From 17d6207e9c13d2a32653361b85355422c8199ad7 Mon Sep 17 00:00:00 2001 From: Pear-231 <61670316+Pear-231@users.noreply.github.com> Date: Sun, 4 Jan 2026 23:00:50 +0000 Subject: [PATCH 1/4] [Chore]: Removed redundant packages that are now part of .NET. --- Shared/SharedCore/Shared.Core.csproj | 3 --- 1 file changed, 3 deletions(-) diff --git a/Shared/SharedCore/Shared.Core.csproj b/Shared/SharedCore/Shared.Core.csproj index 0b062eea..34f19db3 100644 --- a/Shared/SharedCore/Shared.Core.csproj +++ b/Shared/SharedCore/Shared.Core.csproj @@ -34,11 +34,8 @@ - - - From cec9aec6b002d1a19cebe9a6904391b3babbc5f0 Mon Sep 17 00:00:00 2001 From: Pear-231 <61670316+Pear-231@users.noreply.github.com> Date: Mon, 5 Jan 2026 00:53:32 +0000 Subject: [PATCH 2/4] [Feature]: Made pack compression logging more detailed. This helps users find files that shouldn't be compressed, for example wems being compressed due to old RPFM versions. --- .../SharedCore/ErrorHandling/PackFileLog.cs | 148 ++++++++++-------- .../PackFiles/PackFileContainerLoader.cs | 6 +- 2 files changed, 83 insertions(+), 71 deletions(-) diff --git a/Shared/SharedCore/ErrorHandling/PackFileLog.cs b/Shared/SharedCore/ErrorHandling/PackFileLog.cs index d813d869..3a51adf9 100644 --- a/Shared/SharedCore/ErrorHandling/PackFileLog.cs +++ b/Shared/SharedCore/ErrorHandling/PackFileLog.cs @@ -3,21 +3,15 @@ namespace Shared.Core.ErrorHandling { - public class CompressionStats + public class CompressionInformation(long diskSize = 0, long uncompressedSize = 0) { - public long DiskSize { get; set; } - public long UncompressedSize { get; set; } + public long DiskSize { get; set; } = diskSize; + public long UncompressedSize { get; set; } = uncompressedSize; - public CompressionStats(long diskSize = 0, long uncompressedSize = 0) + public void Add(CompressionInformation compressionInformation) { - DiskSize = diskSize; - UncompressedSize = uncompressedSize; - } - - public void Add(CompressionStats stat) - { - DiskSize += stat.DiskSize; - UncompressedSize += stat.UncompressedSize; + DiskSize += compressionInformation.DiskSize; + UncompressedSize += compressionInformation.UncompressedSize; } } @@ -25,22 +19,19 @@ public static class PackFileLog { private static readonly ILogger s_logger = Logging.CreateStatic(typeof(PackFileLog)); - public static Dictionary GetCompressionStats(PackFileContainer container) + public static Dictionary GetCompressionInformation(PackFileContainer container) { - var stats = new Dictionary(); + var compressionInformation = new Dictionary(); foreach (var packFile in container.FileList.Values) { if (packFile.DataSource is PackedFileSource source) { - var format = source.IsCompressed - ? source.CompressionFormat - : CompressionFormat.None; - - if (!stats.TryGetValue(format, out var totals)) + var compressionFormat = source.IsCompressed ? source.CompressionFormat : CompressionFormat.None; + if (!compressionInformation.TryGetValue(compressionFormat, out var totals)) { - totals = new CompressionStats(); - stats[format] = totals; + totals = new CompressionInformation(); + compressionInformation[compressionFormat] = totals; } totals.DiskSize += source.Size; @@ -48,77 +39,98 @@ public static Dictionary GetCompressionStat } } - return stats; + return compressionInformation; } public static void LogPackCompression(PackFileContainer container) { - var stats = GetCompressionStats(container); + var compressionInformation = GetCompressionInformation(container); var totalFiles = container.FileList.Count; - var packSizeFmt = FormatSize(container.OriginalLoadByteSize); + var packSize = FormatSize(container.OriginalLoadByteSize); + + var loadingPart = $"Loading {container.Name}.pack ({totalFiles} files, {packSize})"; - var loadingPart = $"Loading {container.Name}.pack ({totalFiles} files, {packSizeFmt})"; + var fileCountsByCompressionFormat = new Dictionary(); + var fileTypeCountsByCompressionFormat = new Dictionary>(); - var fileCounts = new Dictionary(); - foreach (var pf in container.FileList.Values) + foreach (var packFile in container.FileList.Values) { - if (pf.DataSource is PackedFileSource src) + if (packFile.DataSource is not PackedFileSource packedFileSource) + continue; + + var compressionFormat = packedFileSource.IsCompressed ? packedFileSource.CompressionFormat : CompressionFormat.None; + + if (!fileCountsByCompressionFormat.TryGetValue(compressionFormat, out var fileCount)) + fileCountsByCompressionFormat[compressionFormat] = 1; + else + fileCountsByCompressionFormat[compressionFormat] = fileCount + 1; + + var fileType = string.IsNullOrWhiteSpace(packFile.Extension) ? "no_extension" : packFile.Extension; + + if (!fileTypeCountsByCompressionFormat.TryGetValue(compressionFormat, out var fileTypeCounts)) { - var fmt = src.IsCompressed - ? src.CompressionFormat - : CompressionFormat.None; - - if (!fileCounts.TryGetValue(fmt, out var cnt)) - fileCounts[fmt] = 1; - else - fileCounts[fmt] = cnt + 1; + fileTypeCounts = new Dictionary(StringComparer.OrdinalIgnoreCase); + fileTypeCountsByCompressionFormat[compressionFormat] = fileTypeCounts; } + + if (!fileTypeCounts.TryGetValue(fileType, out var fileTypeCount)) + fileTypeCounts[fileType] = 1; + else + fileTypeCounts[fileType] = fileTypeCount + 1; } - var segments = stats - .OrderBy(kvp => kvp.Key) - .Select(kvp => + var segments = new List(); + + foreach (var compressionEntry in compressionInformation.OrderBy(compressionEntry => compressionEntry.Key)) + { + var compressionFormat = compressionEntry.Key; + var count = fileCountsByCompressionFormat.TryGetValue(compressionFormat, out var fileCount)? fileCount : 0; + var diskSize = FormatSize(compressionEntry.Value.DiskSize); + + var fileSizes = compressionFormat == CompressionFormat.None + ? $"Disk Size: {diskSize}" + : $"Disk Size: {diskSize}, Uncompressed Size: {FormatSize(compressionEntry.Value.UncompressedSize)}"; + + var fileTypes = string.Empty; + if (fileTypeCountsByCompressionFormat.TryGetValue(compressionFormat, out var fileTypeCounts) && fileTypeCounts.Count > 0) { - var fmt = kvp.Key; - var count = fileCounts.TryGetValue(fmt, out var c) ? c : 0; - var disk = FormatSize(kvp.Value.DiskSize); + var fileTypeSegments = new List(); - if (fmt == CompressionFormat.None) - return $"{fmt}: {count} files, {disk} (Disk Size)"; + foreach (var fileTypeEntry in fileTypeCounts.OrderBy(fileTypeEntry => fileTypeEntry.Key, StringComparer.OrdinalIgnoreCase)) + fileTypeSegments.Add($"{fileTypeEntry.Key} ({fileTypeEntry.Value})"); - var unc = FormatSize(kvp.Value.UncompressedSize); - return $"{fmt}: {count} files, {disk} (Disk Size), {unc} (Uncompressed Size)"; - }) - .ToList(); + fileTypes = $": {string.Join(", ", fileTypeSegments)}"; + } + + segments.Add($"{compressionFormat} ({count} files, {fileSizes}){fileTypes}"); + } - var compressionPart = $"File Compression – {string.Join(" | ", segments)}"; - s_logger.Here().Information($"{loadingPart} | {compressionPart}"); + s_logger.Here().Information($"{loadingPart} | {string.Join(" | ", segments)}"); } - public static void LogPacksCompression(IDictionary globalStats) + public static void LogPacksCompression(IDictionary allCompressionInformation) { - var segments = globalStats - .OrderBy(kvp => kvp.Key) - .Select(kvp => - { - var format = kvp.Key; - var diskFormatted = FormatSize(kvp.Value.DiskSize); + var segments = new List(); - if (format == CompressionFormat.None) - return $"{format}: {diskFormatted} (Disk Size)"; + foreach (var compressionEntry in allCompressionInformation.OrderBy(compressionEntry => compressionEntry.Key)) + { + var compressionFormat = compressionEntry.Key; + var diskSize = FormatSize(compressionEntry.Value.DiskSize); - var uncompressedFormatted = FormatSize(kvp.Value.UncompressedSize); - return $"{format}: {diskFormatted} (Disk Size), {uncompressedFormatted} (Uncompressed Size)"; - }) - .ToList(); + if (compressionFormat == CompressionFormat.None) + { + segments.Add($"{compressionFormat}: {diskSize} (Disk Size)"); + continue; + } - var totalDisk = globalStats.Values.Sum(stat => stat.DiskSize); - var totalUncompressed = globalStats.Values.Sum(stat => stat.UncompressedSize); + var uncompressedSize = FormatSize(compressionEntry.Value.UncompressedSize); + segments.Add($"{compressionFormat}: {diskSize} (Disk Size), {uncompressedSize} (Uncompressed Size)"); + } - var totalDiskFormatted = FormatSize(totalDisk); - var totalUncompressedFormatted = FormatSize(totalUncompressed); + var totalDiskSize = FormatSize(allCompressionInformation.Values.Sum(compressionInformation => compressionInformation.DiskSize)); + var totalUncompressedSize = FormatSize(allCompressionInformation.Values.Sum(compressionInformation => compressionInformation.UncompressedSize)); - var totalSegment = $"Total: {totalDiskFormatted} (Disk Size), {totalUncompressedFormatted} (Uncompressed Size)"; + var totalSegment = $"Total: {totalDiskSize} (Disk Size), {totalUncompressedSize} (Uncompressed Size)"; var summary = string.Join(" | ", segments.Append(totalSegment)); s_logger.Here().Information($"Size of compressed files in all packs by format - {summary}"); diff --git a/Shared/SharedCore/PackFiles/PackFileContainerLoader.cs b/Shared/SharedCore/PackFiles/PackFileContainerLoader.cs index 23b95ddd..e47a746e 100644 --- a/Shared/SharedCore/PackFiles/PackFileContainerLoader.cs +++ b/Shared/SharedCore/PackFiles/PackFileContainerLoader.cs @@ -110,7 +110,7 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri } var packList = new List(); - var packsCompressionStats = new ConcurrentDictionary(); + var packsCompressionStats = new ConcurrentDictionary(); Parallel.ForEach(allCaPackFiles, packFilePath => { @@ -124,11 +124,11 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri packList.Add(pack); PackFileLog.LogPackCompression(pack); - var packCompressionStats = PackFileLog.GetCompressionStats(pack); + var packCompressionStats = PackFileLog.GetCompressionInformation(pack); foreach (var kvp in packCompressionStats) { if (!packsCompressionStats.TryGetValue(kvp.Key, out var existingStats)) - packsCompressionStats[kvp.Key] = new CompressionStats(kvp.Value.DiskSize, kvp.Value.UncompressedSize); + packsCompressionStats[kvp.Key] = new CompressionInformation(kvp.Value.DiskSize, kvp.Value.UncompressedSize); else existingStats.Add(kvp.Value); } From 870000d6a644f49161edf16139e3b669095c243e Mon Sep 17 00:00:00 2001 From: Pear-231 <61670316+Pear-231@users.noreply.github.com> Date: Mon, 5 Jan 2026 01:11:51 +0000 Subject: [PATCH 3/4] [Chore]: Added a new broken anim file to the pile! --- GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs b/GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs index 2f8c1f03..9e406e24 100644 --- a/GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs +++ b/GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs @@ -135,7 +135,8 @@ void FileDiscovered(byte[] byteChunk, PackFileContainer container, string fullPa "animations\\battle\\humanoid13b\\golgfag\\docking\\hu13b_golgfag_docking_armed_02.anim", "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_swc_rider1_shoot_back_crossbow_01.anim", "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_swc_rider1_reload_crossbow_01.anim", - "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_sp_rider1_shoot_ready_crossbow_01.anim" + "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_sp_rider1_shoot_ready_crossbow_01.anim", + "animations\\battle\\humanoid01c\\sayl_staff_and_skull\\stand\\props\\hu1c_sayl_staff_and_skull_staff_stand_idle_02.anim" }; if (brokenFiles.Contains(fullPath)) { From 054238b72dd8f995263f512c7922dea73308ed91 Mon Sep 17 00:00:00 2001 From: Pear-231 <61670316+Pear-231@users.noreply.github.com> Date: Mon, 5 Jan 2026 01:12:05 +0000 Subject: [PATCH 4/4] [Bugfix]: Stopped using ReadData(size) as decompression doesn't work with partial reads. --- .../Converters/AnimationBinWh3FileToXmlConverter.cs | 7 ++++--- .../View3D/Services/SkeletonAnimationLookUpHelper.cs | 2 +- Shared/GameFiles/Animation/AnimationFile.cs | 2 +- Shared/SharedCore/PackFiles/Models/DataSource.cs | 9 +++++++-- Shared/SharedCore/PackFiles/Models/PackFileContainer.cs | 1 - 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Editors/SimpleAnimationEditors/AnimationPack/Converters/AnimationBinWh3FileToXmlConverter.cs b/Editors/SimpleAnimationEditors/AnimationPack/Converters/AnimationBinWh3FileToXmlConverter.cs index e29b910a..1d5efb03 100644 --- a/Editors/SimpleAnimationEditors/AnimationPack/Converters/AnimationBinWh3FileToXmlConverter.cs +++ b/Editors/SimpleAnimationEditors/AnimationPack/Converters/AnimationBinWh3FileToXmlConverter.cs @@ -237,7 +237,7 @@ private bool IsAnimFile(string file, IPackFileService pfs, ErrorList errorList, return false; } - var data = theFile.DataSource.ReadData(20); + var data = theFile.DataSource.ReadData(); var headerIsReallyAnimFile = data[0] == 0x06 || data[0] == 0x07 || data[0] == 0x08; //check if version is not 6 7 8 (or just check if it's 2) return endsWithAnim && headerIsReallyAnimFile; } @@ -252,7 +252,8 @@ private bool IsAnimMetaFile(string file, IPackFileService pfs, ErrorList errorLi errorList.Warning(animationSlot, $"Inable to locate {file} for {animationSlot}"); return false; } - var data = theFile.DataSource.ReadData(20); + + var data = theFile.DataSource.ReadData(); var headerIsReallyAnimMetaFile = data[0] == 0x02; //check if version is not 6 7 8 (or just check if it's 2) return endsWithDotMeta && headerIsReallyAnimMetaFile; } @@ -268,7 +269,7 @@ private bool IsSndMetaFile(string file, IPackFileService pfs, ErrorList errorLis return false; } - var data = theFile.DataSource.ReadData(20); + var data = theFile.DataSource.ReadData(); var headerIsReallyAnimMetaFile = data[0] == 0x02; //check if version is not 6 7 8 (or just check if it's 2) return endsWithDotMeta && headerIsReallyAnimMetaFile; } diff --git a/GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs b/GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs index 9e406e24..e27cfdde 100644 --- a/GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs +++ b/GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs @@ -111,7 +111,7 @@ void LoadFromPackFileContainer(PackFileContainer packFileContainer) Parallel.For(0, allAnimsOtherFiles.Count, index => { var animation = allAnimations[index]; - FileDiscovered(animation.Pack.DataSource.ReadData(100), packFileContainer, animation.FileName, ref skeletonFileNameList, ref animationList); + FileDiscovered(animation.Pack.DataSource.ReadData(), packFileContainer, animation.FileName, ref skeletonFileNameList, ref animationList); }); foreach (var skeleton in skeletonFileNameList) diff --git a/Shared/GameFiles/Animation/AnimationFile.cs b/Shared/GameFiles/Animation/AnimationFile.cs index ac3e4d84..eed5fe29 100644 --- a/Shared/GameFiles/Animation/AnimationFile.cs +++ b/Shared/GameFiles/Animation/AnimationFile.cs @@ -128,7 +128,7 @@ public AnimationV8OptimizationData(uint boneCount) public static AnimationHeader GetAnimationHeader(PackFile file) { - var data = file.DataSource.ReadData(100); + var data = file.DataSource.ReadData(); try { return GetAnimationHeader(new ByteChunk(data)); diff --git a/Shared/SharedCore/PackFiles/Models/DataSource.cs b/Shared/SharedCore/PackFiles/Models/DataSource.cs index 03b64838..1674725b 100644 --- a/Shared/SharedCore/PackFiles/Models/DataSource.cs +++ b/Shared/SharedCore/PackFiles/Models/DataSource.cs @@ -22,6 +22,7 @@ public class FileSystemSource : IDataSource public long Size { get; private set; } protected string filepath; + public FileSystemSource(string filepath) : base() { @@ -55,11 +56,13 @@ public class MemorySource : IDataSource public long Size { get; private set; } private byte[] data; + public MemorySource(byte[] data) { Size = data.Length; this.data = data; } + public byte[] ReadData() { return data; @@ -77,6 +80,7 @@ public static MemorySource FromFile(string path) { return new MemorySource(File.ReadAllBytes(path)); } + public ByteChunk ReadDataAsChunk() { return new ByteChunk(ReadData()); @@ -131,6 +135,9 @@ public byte[] ReadData() public byte[] ReadData(int size) { + if (IsCompressed) + throw new NotSupportedException("Partial reads are not supported because decompression requires the full compressed data."); + var data = new byte[size]; using (Stream stream = File.Open(_parent.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { @@ -140,8 +147,6 @@ public byte[] ReadData(int size) if (IsEncrypted) data = PackFileEncryption.Decrypt(data); - if (IsCompressed) - data = PackFileCompression.Decompress(data); return data; } diff --git a/Shared/SharedCore/PackFiles/Models/PackFileContainer.cs b/Shared/SharedCore/PackFiles/Models/PackFileContainer.cs index c722d5a4..04577756 100644 --- a/Shared/SharedCore/PackFiles/Models/PackFileContainer.cs +++ b/Shared/SharedCore/PackFiles/Models/PackFileContainer.cs @@ -62,7 +62,6 @@ public void SaveToByteArray(BinaryWriter writer, GameInformation gameInformation foreach (var file in sortedFiles) { var packFile = file.Value; - var fileSize = (int)packFile.DataSource.Size; // Determine compression info var currentCompressionFormat = CompressionFormat.None;