From 2c27609aa6f553e62e6d3335a5738f4af1edf334 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=91=E9=93=96?= <702287355@qq.com> Date: Fri, 12 Jul 2024 11:39:27 +0800 Subject: [PATCH 1/3] Complete the development of the subcontracting writing function and prepare to adjust the code structure of the writing part in the future --- ValvePak/ValvePak.Test/ValvePak.Test.csproj | 2 +- ValvePak/ValvePak/Package.Save.cs | 199 +++++++++++++++----- 2 files changed, 156 insertions(+), 45 deletions(-) diff --git a/ValvePak/ValvePak.Test/ValvePak.Test.csproj b/ValvePak/ValvePak.Test/ValvePak.Test.csproj index 5f853f4..eda7817 100644 --- a/ValvePak/ValvePak.Test/ValvePak.Test.csproj +++ b/ValvePak/ValvePak.Test/ValvePak.Test.csproj @@ -1,4 +1,4 @@ - + net8.0 diff --git a/ValvePak/ValvePak/Package.Save.cs b/ValvePak/ValvePak/Package.Save.cs index 0567043..d4ce9ae 100644 --- a/ValvePak/ValvePak/Package.Save.cs +++ b/ValvePak/ValvePak/Package.Save.cs @@ -101,31 +101,26 @@ public PackageEntry AddFile(string filePath, byte[] fileData) /// Opens and writes the given filename. /// /// The file to open and write. - public void Write(string filename) + public void Write(string filename, int maxFileBytes = int.MaxValue) { - using var fs = new FileStream(filename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + ArgumentOutOfRangeException.ThrowIfNegative(maxFileBytes); + + using var fs = new FileStream(filename, FileMode.Create, FileAccess.ReadWrite, FileShare.None); fs.SetLength(0); - Write(fs); + Write(fs, maxFileBytes); } /// /// Writes to the given . /// /// The input to write to. - public void Write(Stream stream) + public void Write(FileStream stream, int maxFileBytes) { - if (IsDirVPK) - { - throw new InvalidOperationException("This package was opened from a _dir.vpk, writing back is currently unsupported."); - } - ArgumentNullException.ThrowIfNull(stream); if (!stream.CanSeek || !stream.CanRead) - { throw new InvalidOperationException("Stream must be seekable and readable."); - } using var writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: true); @@ -143,6 +138,9 @@ public void Write(Stream stream) foreach (var entry in typeEntries.Value) { + if (entry.TotalLength >= maxFileBytes) + throw new InvalidOperationException("There are files exceeding max file bytes"); + var directoryName = entry.DirectoryName.Length == 0 ? Space : entry.DirectoryName; if (!typeTree.TryGetValue(directoryName, out var directoryEntries)) @@ -154,21 +152,21 @@ public void Write(Stream stream) directoryEntries.Add(entry); fileDataSectionSize += entry.TotalLength; - - if (fileDataSectionSize > int.MaxValue) - { - throw new InvalidOperationException("Package contents exceed 2GiB, and splitting packages is currently unsupported."); - } } } + ulong packetCount = (fileDataSectionSize / Convert.ToUInt64(maxFileBytes)) + Convert.ToUInt64(fileDataSectionSize % Convert.ToUInt64(maxFileBytes) == 0 ? 0 : 1); + if (packetCount >= 0x7FFF) + throw new InvalidOperationException("The number of packages exceeds 32766"); + + // Header writer.Write(MAGIC); writer.Write(2); // Version writer.Write(0); // TreeSize, to be updated later writer.Write(0); // FileDataSectionSize, to be updated later writer.Write(0); // ArchiveMD5SectionSize - writer.Write(48); // OtherMD5SectionSize + writer.Write(48); //OtherMD5SectionSize writer.Write(0); // SignatureSectionSize var headerSize = (int)(stream.Position - streamOffset); @@ -176,55 +174,167 @@ public void Write(Stream stream) const byte NullByte = 0; - // File tree data - foreach (var typeEntries in tree) + long fileTreeSize; + if (packetCount == 1) { - writer.Write(Encoding.UTF8.GetBytes(typeEntries.Key)); - writer.Write(NullByte); - - foreach (var directoryEntries in typeEntries.Value) + ushort archiveIndex = 0x7FFF; + // File tree data + foreach (var typeEntries in tree) { - writer.Write(Encoding.UTF8.GetBytes(directoryEntries.Key)); + writer.Write(Encoding.UTF8.GetBytes(typeEntries.Key)); writer.Write(NullByte); - foreach (var entry in directoryEntries.Value) + foreach (var directoryEntries in typeEntries.Value) { - var fileLength = entry.TotalLength; + writer.Write(Encoding.UTF8.GetBytes(directoryEntries.Key)); + writer.Write(NullByte); + + foreach (var entry in directoryEntries.Value) + { + var fileLength = entry.TotalLength; + + writer.Write(Encoding.UTF8.GetBytes(entry.FileName)); + writer.Write(NullByte); + writer.Write(entry.CRC32); + writer.Write((short)0); // SmallData, we will put it into data instead + writer.Write(archiveIndex); + writer.Write(fileOffset); + writer.Write(fileLength); + writer.Write(ushort.MaxValue); // terminator, 0xFFFF + + fileOffset += fileLength; + } - writer.Write(Encoding.UTF8.GetBytes(entry.FileName)); writer.Write(NullByte); - writer.Write(entry.CRC32); - writer.Write((short)0); // SmallData, we will put it into data instead - writer.Write(entry.ArchiveIndex); - writer.Write(fileOffset); - writer.Write(fileLength); - writer.Write(ushort.MaxValue); // terminator, 0xFFFF - - fileOffset += fileLength; } writer.Write(NullByte); } writer.Write(NullByte); - } - writer.Write(NullByte); + fileTreeSize = stream.Position - headerSize; - var fileTreeSize = stream.Position - headerSize; + // File data + foreach (var typeEntries in tree) + { + foreach (var directoryEntries in typeEntries.Value) + { + foreach (var entry in directoryEntries.Value) + { + ReadEntry(entry, out var fileData, validateCrc: false); - // File data - foreach (var typeEntries in tree) + writer.Write(fileData); + } + } + } + } + else { - foreach (var directoryEntries in typeEntries.Value) + var trees = new List>>>(); + Dictionary>> activeTree = new Dictionary>>(); + trees.Add(activeTree); + uint totalLength = 0; + + ushort archiveIndex = 0; + foreach (var typeEntries in tree) { - foreach (var entry in directoryEntries.Value) + writer.Write(Encoding.UTF8.GetBytes(typeEntries.Key)); + writer.Write(NullByte); + + Dictionary> activeTree2 = new Dictionary>(); + activeTree[typeEntries.Key] = activeTree2; + foreach (var directoryEntries in typeEntries.Value) { - ReadEntry(entry, out var fileData, validateCrc: false); + writer.Write(Encoding.UTF8.GetBytes(directoryEntries.Key)); + writer.Write(NullByte); + + List entries = new List(); + activeTree2[directoryEntries.Key] = entries; + foreach (var entry in directoryEntries.Value) + { + if (totalLength + entry.TotalLength >= maxFileBytes) + { + if (entries.Count == 0) + { + activeTree2.Remove(directoryEntries.Key); + if (activeTree2.Count == 0) + activeTree.Remove(typeEntries.Key); + + if (activeTree.Count == 0) + trees.Remove(activeTree); + } + + activeTree = new Dictionary>>(); + trees.Add(activeTree); + activeTree2 = new Dictionary>(); + activeTree[typeEntries.Key] = activeTree2; + entries = new List(); + activeTree2[directoryEntries.Key] = entries; + entries.Add(entry); + totalLength = entry.TotalLength; + archiveIndex++; + fileOffset = 0; + } + else + { + totalLength += entry.TotalLength; + entries.Add(entry); + } + + var fileLength = entry.TotalLength; + + writer.Write(Encoding.UTF8.GetBytes(entry.FileName)); + writer.Write(NullByte); + writer.Write(entry.CRC32); + writer.Write((short)0); // SmallData, we will put it into data instead + writer.Write(archiveIndex); + writer.Write(fileOffset); + writer.Write(fileLength); + writer.Write(ushort.MaxValue); // terminator, 0xFFFF + + fileOffset += fileLength; + } + writer.Write(NullByte); + } + writer.Write(NullByte); + } + writer.Write(NullByte); - writer.Write(fileData); + fileTreeSize = stream.Position - headerSize; + + for (ushort i = 0; i < trees.Count; i++) + { + long testtotalSize = 0; + archiveIndex = i; + Dictionary>> _tree = trees[i]; + + string filename = stream.Name; + FileInfo sub_FileInfo = new FileInfo(filename); + + string sub_FileName = Path.GetFileNameWithoutExtension(sub_FileInfo.FullName); + if (sub_FileName.EndsWith("_dir", StringComparison.OrdinalIgnoreCase)) + sub_FileName = $"{sub_FileName[..^4]}"; + + sub_FileName = $"{sub_FileName}_{archiveIndex:D3}"; + string sub_FilePath = $"{sub_FileInfo.Directory}\\{sub_FileName}{sub_FileInfo.Extension}"; + using var fs = new FileStream(sub_FilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None); + using var writer_sub = new BinaryWriter(fs, Encoding.UTF8, leaveOpen: true); + // File data + foreach (var typeEntries in _tree) + { + foreach (var directoryEntries in typeEntries.Value) + { + foreach (var entry in directoryEntries.Value) + { + ReadEntry(entry, out var fileData, validateCrc: false); + testtotalSize += fileData.LongLength; + writer_sub.Write(fileData); + } + } } } + } var afterFileData = stream.Position; @@ -293,5 +403,6 @@ public void Write(Stream stream) ArrayPool.Shared.Return(buffer); } } + } } From e8515e40826ae9ea5e1301bcf0fe8e6a66f08eed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=91=E9=93=96?= <702287355@qq.com> Date: Fri, 12 Jul 2024 23:22:49 +0800 Subject: [PATCH 2/3] Complete the development of package splitting write functionality --- ValvePak/ValvePak/Package.Save.cs | 347 +++++++++++++++++++----------- 1 file changed, 227 insertions(+), 120 deletions(-) diff --git a/ValvePak/ValvePak/Package.Save.cs b/ValvePak/ValvePak/Package.Save.cs index d4ce9ae..06d5e31 100644 --- a/ValvePak/ValvePak/Package.Save.cs +++ b/ValvePak/ValvePak/Package.Save.cs @@ -1,10 +1,15 @@ +using Microsoft.VisualBasic.FileIO; using System; using System.Buffers; +using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.IO.Hashing; +using System.Linq; using System.Security.Cryptography; using System.Text; +using static System.Runtime.InteropServices.JavaScript.JSType; namespace SteamDatabase.ValvePak { @@ -126,37 +131,10 @@ public void Write(FileStream stream, int maxFileBytes) // TODO: input.SetLength() var streamOffset = stream.Position; - ulong fileDataSectionSize = 0; - var tree = new Dictionary>>(); + var _trees = CreatePacketsGroup(Entries, maxFileBytes); - // Precalculate the file tree and set data offsets - foreach (var typeEntries in Entries) - { - var typeTree = new Dictionary>(); - tree[typeEntries.Key] = typeTree; - - foreach (var entry in typeEntries.Value) - { - if (entry.TotalLength >= maxFileBytes) - throw new InvalidOperationException("There are files exceeding max file bytes"); - - var directoryName = entry.DirectoryName.Length == 0 ? Space : entry.DirectoryName; - - if (!typeTree.TryGetValue(directoryName, out var directoryEntries)) - { - directoryEntries = []; - typeTree[directoryName] = directoryEntries; - } - - directoryEntries.Add(entry); - - fileDataSectionSize += entry.TotalLength; - } - } - - ulong packetCount = (fileDataSectionSize / Convert.ToUInt64(maxFileBytes)) + Convert.ToUInt64(fileDataSectionSize % Convert.ToUInt64(maxFileBytes) == 0 ? 0 : 1); - if (packetCount >= 0x7FFF) + if (_trees.Count >= 0x7FFF) throw new InvalidOperationException("The number of packages exceeds 32766"); @@ -168,15 +146,14 @@ public void Write(FileStream stream, int maxFileBytes) writer.Write(0); // ArchiveMD5SectionSize writer.Write(48); //OtherMD5SectionSize writer.Write(0); // SignatureSectionSize - var headerSize = (int)(stream.Position - streamOffset); uint fileOffset = 0; - const byte NullByte = 0; - long fileTreeSize; - if (packetCount == 1) + + if (_trees.Count == 1) { + var tree = _trees[0]; ushort archiveIndex = 0x7FFF; // File tree data foreach (var typeEntries in tree) @@ -213,7 +190,6 @@ public void Write(FileStream stream, int maxFileBytes) writer.Write(NullByte); - fileTreeSize = stream.Position - headerSize; // File data foreach (var typeEntries in tree) @@ -223,7 +199,6 @@ public void Write(FileStream stream, int maxFileBytes) foreach (var entry in directoryEntries.Value) { ReadEntry(entry, out var fileData, validateCrc: false); - writer.Write(fileData); } } @@ -231,111 +206,82 @@ public void Write(FileStream stream, int maxFileBytes) } else { - var trees = new List>>>(); - Dictionary>> activeTree = new Dictionary>>(); - trees.Add(activeTree); - uint totalLength = 0; - - ushort archiveIndex = 0; - foreach (var typeEntries in tree) + for (ushort i = 0; i < 999; i++) { - writer.Write(Encoding.UTF8.GetBytes(typeEntries.Key)); - writer.Write(NullByte); + string sub_FilePath = GetSubFilePath(stream.Name, i); + if (File.Exists(sub_FilePath)) + File.Delete(sub_FilePath); + } + for (ushort i = 0; i < _trees.Count; i++) + { + var tree = _trees[i]; - Dictionary> activeTree2 = new Dictionary>(); - activeTree[typeEntries.Key] = activeTree2; - foreach (var directoryEntries in typeEntries.Value) + string sub_FilePath = GetSubFilePath(stream.Name,i); + + using var fs = new FileStream(sub_FilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None); + using var writer_sub = new BinaryWriter(fs, Encoding.UTF8, leaveOpen: true); + + fileOffset = 0; + foreach (var fileType in tree.Keys) { - writer.Write(Encoding.UTF8.GetBytes(directoryEntries.Key)); - writer.Write(NullByte); + //如果是一种类型的开头 + if (IsTypeHead(fileType, i, _trees)) + { + writer.Write(Encoding.UTF8.GetBytes(fileType)); + writer.Write(NullByte); + } - List entries = new List(); - activeTree2[directoryEntries.Key] = entries; - foreach (var entry in directoryEntries.Value) + var dics = tree[fileType]; + foreach (var dicName in dics.Keys) { - if (totalLength + entry.TotalLength >= maxFileBytes) + //如果是一类地址的开头 + if (IsDirectoryHead(fileType, dicName, i, _trees)) { - if (entries.Count == 0) - { - activeTree2.Remove(directoryEntries.Key); - if (activeTree2.Count == 0) - activeTree.Remove(typeEntries.Key); + writer.Write(Encoding.UTF8.GetBytes(dicName)); + writer.Write(NullByte); + } - if (activeTree.Count == 0) - trees.Remove(activeTree); + var entries = dics[dicName]; + foreach (var entry in entries) + { + if (entry.FileName == "odest_spawn_02") + { + Debug.Write(1); } + var fileLength = entry.TotalLength; + writer.Write(Encoding.UTF8.GetBytes(entry.FileName)); + writer.Write(NullByte); + writer.Write(entry.CRC32); + writer.Write((short)0); // SmallData, we will put it into data instead + writer.Write(i); + writer.Write(fileOffset); + writer.Write(fileLength); + writer.Write(ushort.MaxValue); // terminator, 0xFFFF + + fileOffset += fileLength; - activeTree = new Dictionary>>(); - trees.Add(activeTree); - activeTree2 = new Dictionary>(); - activeTree[typeEntries.Key] = activeTree2; - entries = new List(); - activeTree2[directoryEntries.Key] = entries; - entries.Add(entry); - totalLength = entry.TotalLength; - archiveIndex++; - fileOffset = 0; - } - else - { - totalLength += entry.TotalLength; - entries.Add(entry); + ReadEntry(entry, out var fileData, validateCrc: false); + writer_sub.Write(fileData); } - var fileLength = entry.TotalLength; + //如果是一类地址的结尾 + if (IsDirectoryTail(fileType, dicName, i, _trees)) + writer.Write(NullByte); + } - writer.Write(Encoding.UTF8.GetBytes(entry.FileName)); + //如果是一种类型的结尾 + if (IsTypeTail(fileType, i, _trees)) writer.Write(NullByte); - writer.Write(entry.CRC32); - writer.Write((short)0); // SmallData, we will put it into data instead - writer.Write(archiveIndex); - writer.Write(fileOffset); - writer.Write(fileLength); - writer.Write(ushort.MaxValue); // terminator, 0xFFFF - fileOffset += fileLength; - } - writer.Write(NullByte); } - writer.Write(NullByte); } - writer.Write(NullByte); + writer.Write(NullByte); fileTreeSize = stream.Position - headerSize; + } - for (ushort i = 0; i < trees.Count; i++) - { - long testtotalSize = 0; - archiveIndex = i; - Dictionary>> _tree = trees[i]; - - string filename = stream.Name; - FileInfo sub_FileInfo = new FileInfo(filename); + fileTreeSize = stream.Position - headerSize; - string sub_FileName = Path.GetFileNameWithoutExtension(sub_FileInfo.FullName); - if (sub_FileName.EndsWith("_dir", StringComparison.OrdinalIgnoreCase)) - sub_FileName = $"{sub_FileName[..^4]}"; - - sub_FileName = $"{sub_FileName}_{archiveIndex:D3}"; - string sub_FilePath = $"{sub_FileInfo.Directory}\\{sub_FileName}{sub_FileInfo.Extension}"; - using var fs = new FileStream(sub_FilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None); - using var writer_sub = new BinaryWriter(fs, Encoding.UTF8, leaveOpen: true); - // File data - foreach (var typeEntries in _tree) - { - foreach (var directoryEntries in typeEntries.Value) - { - foreach (var entry in directoryEntries.Value) - { - ReadEntry(entry, out var fileData, validateCrc: false); - testtotalSize += fileData.LongLength; - writer_sub.Write(fileData); - } - } - } - } - - } var afterFileData = stream.Position; var fileDataSize = afterFileData - fileTreeSize - headerSize; @@ -404,5 +350,166 @@ public void Write(FileStream stream, int maxFileBytes) } } + static string GetSubFilePath(string indexFilePath,ushort indexNumber) + { + FileInfo sub_FileInfo = new FileInfo(indexFilePath); + + string sub_FileName = Path.GetFileNameWithoutExtension(sub_FileInfo.FullName); + if (sub_FileName.EndsWith("_dir", StringComparison.OrdinalIgnoreCase)) + sub_FileName = $"{sub_FileName[..^4]}"; + + sub_FileName = $"{sub_FileName}_{indexNumber:D3}"; + return $"{sub_FileInfo.Directory}\\{sub_FileName}{sub_FileInfo.Extension}"; + } + + /// + /// Determine if it is located at the beginning of a set of file types + /// + /// Current file type + /// Current tree index + /// Tree list + /// Is located at the beginning of a set of files + static bool IsTypeHead(string fileType, ushort treeIndex, List>>> trees) + { + var tree = trees[treeIndex]; + if (treeIndex == 0) + return true; + + if (fileType == tree.Keys.First()) + return trees[treeIndex - 1].Keys.Last() != fileType; + else + return true; + + } + /// + /// Determine if it is located at the end of a set of file types + /// + /// Current file type + /// Current tree index + /// Tree list + /// Is located at the end of a set of files + static bool IsTypeTail(string fileType, ushort treeIndex, List>>> trees) + { + if (treeIndex == trees.Count - 1) + return true; + + var tree = trees[treeIndex]; + if (fileType == tree.Keys.Last()) + return trees[treeIndex + 1].Keys.First() != fileType; + else + return true; + + } + + /// + /// Determine if it is located at the beginning of a set of folder addresses + /// + /// Current file type + /// Current directory path + /// Current tree index + /// Tree list + /// Is located at the beginning of a set of folder addresses + static bool IsDirectoryHead(string fileType, string dicName, ushort treeIndex, List>>> trees) + { + if (treeIndex == 0) + return true; + + var tree = trees[treeIndex]; + + if (dicName == tree.Values.First().Keys.First() && fileType == tree.Keys.First()) + return trees[treeIndex - 1].Keys.Last() != fileType && trees[treeIndex - 1].Values.Last().Keys.Last() != dicName; + else + return true; + } + + /// + /// Determine if it is located at the end of a set of folder addresses + /// + /// Current file type + /// Current directory path + /// Current tree index + /// Tree list + /// Is located at the end of a set of folder addresses + static bool IsDirectoryTail(string fileType, string dicName, ushort treeIndex, List>>> trees) + { + if (treeIndex == trees.Count - 1) + return true; + + var tree = trees[treeIndex]; + if (fileType == tree.Keys.Last() && dicName == tree.Values.Last().Keys.Last()) + return trees[treeIndex + 1].Keys.First() != fileType && trees[treeIndex - 1].Values.First().Keys.First() != dicName; + else + return true; + } + + /// + /// Split the current tree into multiple trees based on packet size + /// + /// Tree of data sources + /// Maximum file byte count + /// List of Trees + /// If the size of a single file exceeds the size of the package,throw this exception + static List>>> CreatePacketsGroup(Dictionary> mainTypeTree, int maxFileBytes) + { + + List>>> packets = new List>>>(); + //Create a new tree + Dictionary>> currentTree = new Dictionary>>(); + packets.Add(currentTree); + uint totalLength = 0; + foreach (var typeEntries in mainTypeTree) + { + //Create a new type tree and add the current one to the current tree + Dictionary> currentTypeTree = new Dictionary>(); + currentTree[typeEntries.Key] = currentTypeTree; + + //Group the entries under the current type tree according to their folder location + IEnumerable> dicGroups = typeEntries.Value.GroupBy(s => s.DirectoryName.Length == 0 ? Space : s.DirectoryName); + foreach (var dicGroup in dicGroups) + { + List entries = new List(); + currentTypeTree[dicGroup.Key] = entries; + + foreach (var entry in dicGroup) + { + var fileLength = entry.TotalLength; + //If the size of a single file exceeds the size of the package + if (fileLength >= maxFileBytes) + throw new InvalidOperationException("There are files exceeding max file bytes"); + + // TODO: Search for smaller files to fill in the empty space + if (totalLength + fileLength >= maxFileBytes) + { + if (entries.Count == 0) + { + currentTypeTree.Remove(dicGroup.Key); + if (currentTypeTree.Count == 0) + currentTree.Remove(typeEntries.Key); + + if (currentTree.Count == 0) + packets.Remove(currentTree); + } + + currentTree = new Dictionary>>(); + packets.Add(currentTree); + currentTypeTree = new Dictionary>(); + currentTree[typeEntries.Key] = currentTypeTree; + entries = new List(); + currentTypeTree[dicGroup.Key] = entries; + entries.Add(entry); + totalLength = fileLength; + } + else + { + totalLength += fileLength; + entries.Add(entry); + } + + } + + } + } + return packets; + } } } From 4e9bd4d32ab7f2a938daa2b6c4f05d3cfdf5f7f4 Mon Sep 17 00:00:00 2001 From: FireFrog <702287355@qq.com> Date: Sat, 13 Jul 2024 09:51:14 +0800 Subject: [PATCH 3/3] Redesigned the order of writing sub files, reducing the number of sub files under special conditions --- ValvePak/ValvePak/Package.Save.cs | 369 +++++++++++------------------- 1 file changed, 135 insertions(+), 234 deletions(-) diff --git a/ValvePak/ValvePak/Package.Save.cs b/ValvePak/ValvePak/Package.Save.cs index 06d5e31..6fbf37d 100644 --- a/ValvePak/ValvePak/Package.Save.cs +++ b/ValvePak/ValvePak/Package.Save.cs @@ -9,10 +9,17 @@ using System.Linq; using System.Security.Cryptography; using System.Text; +using System.Text.RegularExpressions; using static System.Runtime.InteropServices.JavaScript.JSType; namespace SteamDatabase.ValvePak { + internal sealed class WriteEntry(ushort archiveIndex, uint fileOffset, PackageEntry entry) + { + internal ushort ArchiveIndex { get; set; } = archiveIndex; + internal uint FileOffset { get; set; } = fileOffset; + internal PackageEntry Entry { get; set; } = entry; + } public partial class Package { /// @@ -122,6 +129,7 @@ public void Write(string filename, int maxFileBytes = int.MaxValue) /// The input to write to. public void Write(FileStream stream, int maxFileBytes) { + ArgumentNullException.ThrowIfNull(stream); if (!stream.CanSeek || !stream.CanRead) @@ -132,11 +140,33 @@ public void Write(FileStream stream, int maxFileBytes) // TODO: input.SetLength() var streamOffset = stream.Position; - var _trees = CreatePacketsGroup(Entries, maxFileBytes); + List entries = Entries.SelectMany(e => e.Value).ToList(); - if (_trees.Count >= 0x7FFF) - throw new InvalidOperationException("The number of packages exceeds 32766"); + if (entries.Any(e => e.TotalLength > maxFileBytes)) + throw new InvalidOperationException("There are files exceeding max file bytes"); + + + var tree = new Dictionary>>(); + + // Precalculate the file tree and set data offsets + foreach (var typeEntries in Entries) + { + var typeTree = new Dictionary>(); + tree[typeEntries.Key] = typeTree; + + foreach (var entry in typeEntries.Value) + { + var directoryName = entry.DirectoryName.Length == 0 ? Space : entry.DirectoryName; + + if (!typeTree.TryGetValue(directoryName, out var directoryEntries)) + { + directoryEntries = []; + typeTree[directoryName] = directoryEntries; + } + directoryEntries.Add(entry); + } + } // Header writer.Write(MAGIC); @@ -147,140 +177,102 @@ public void Write(FileStream stream, int maxFileBytes) writer.Write(48); //OtherMD5SectionSize writer.Write(0); // SignatureSectionSize var headerSize = (int)(stream.Position - streamOffset); - uint fileOffset = 0; const byte NullByte = 0; - long fileTreeSize; - if (_trees.Count == 1) + bool isSingleFile = entries.Sum(s => s.TotalLength) + headerSize + 64 <= maxFileBytes; + + var groups = CreatePacketsGroup(entries, maxFileBytes, isSingleFile); + + if (groups.Count >= 0x7FFF) + throw new InvalidOperationException("The number of packages exceeds 32766"); + + + uint fileOffset = 0; + foreach (var typeEntries in tree) { - var tree = _trees[0]; - ushort archiveIndex = 0x7FFF; - // File tree data - foreach (var typeEntries in tree) + writer.Write(Encoding.UTF8.GetBytes(typeEntries.Key)); + writer.Write(NullByte); + + foreach (var directoryEntries in typeEntries.Value) { - writer.Write(Encoding.UTF8.GetBytes(typeEntries.Key)); + writer.Write(Encoding.UTF8.GetBytes(directoryEntries.Key)); writer.Write(NullByte); - foreach (var directoryEntries in typeEntries.Value) + foreach (var entry in directoryEntries.Value) { - writer.Write(Encoding.UTF8.GetBytes(directoryEntries.Key)); - writer.Write(NullByte); + var fileLength = entry.TotalLength; - foreach (var entry in directoryEntries.Value) + var fullPath = entry.GetFullPath(); + WriteEntry writeEntry = null; + + foreach (var group in groups) { - var fileLength = entry.TotalLength; - - writer.Write(Encoding.UTF8.GetBytes(entry.FileName)); - writer.Write(NullByte); - writer.Write(entry.CRC32); - writer.Write((short)0); // SmallData, we will put it into data instead - writer.Write(archiveIndex); - writer.Write(fileOffset); - writer.Write(fileLength); - writer.Write(ushort.MaxValue); // terminator, 0xFFFF - - fileOffset += fileLength; + if (group.TryGetValue(fullPath, out writeEntry)) + break; } + if (writeEntry is null) + throw new InvalidOperationException("No need write entry found"); + + writer.Write(Encoding.UTF8.GetBytes(entry.FileName)); writer.Write(NullByte); + writer.Write(entry.CRC32); + writer.Write((short)0); // SmallData, we will put it into data instead + writer.Write(writeEntry.ArchiveIndex); + writer.Write(writeEntry.FileOffset); + writer.Write(fileLength); + writer.Write(ushort.MaxValue); // terminator, 0xFFFF + + fileOffset += fileLength; } writer.Write(NullByte); } writer.Write(NullByte); + } + writer.Write(NullByte); - // File data - foreach (var typeEntries in tree) + //clear sub file + for (ushort i = 0; i < 999; i++) + { + string sub_FilePath = GetSubFilePath(stream.Name, i); + if (File.Exists(sub_FilePath)) + File.Delete(sub_FilePath); + } + + if (isSingleFile) + { + //Write file data + foreach (var writeEntry in groups[0].Values) { - foreach (var directoryEntries in typeEntries.Value) - { - foreach (var entry in directoryEntries.Value) - { - ReadEntry(entry, out var fileData, validateCrc: false); - writer.Write(fileData); - } - } + ReadEntry(writeEntry.Entry, out var fileData, validateCrc: false); + writer.Write(fileData); } } else { - for (ushort i = 0; i < 999; i++) + //Create and write sub file data + for (ushort i = 0; i < groups.Count; i++) { string sub_FilePath = GetSubFilePath(stream.Name, i); - if (File.Exists(sub_FilePath)) - File.Delete(sub_FilePath); - } - for (ushort i = 0; i < _trees.Count; i++) - { - var tree = _trees[i]; - string sub_FilePath = GetSubFilePath(stream.Name,i); - using var fs = new FileStream(sub_FilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None); using var writer_sub = new BinaryWriter(fs, Encoding.UTF8, leaveOpen: true); - fileOffset = 0; - foreach (var fileType in tree.Keys) + var group = groups[i]; + foreach (var writeEntry in group.Values) { - //如果是一种类型的开头 - if (IsTypeHead(fileType, i, _trees)) - { - writer.Write(Encoding.UTF8.GetBytes(fileType)); - writer.Write(NullByte); - } - - var dics = tree[fileType]; - foreach (var dicName in dics.Keys) - { - //如果是一类地址的开头 - if (IsDirectoryHead(fileType, dicName, i, _trees)) - { - writer.Write(Encoding.UTF8.GetBytes(dicName)); - writer.Write(NullByte); - } - - var entries = dics[dicName]; - foreach (var entry in entries) - { - if (entry.FileName == "odest_spawn_02") - { - Debug.Write(1); - } - var fileLength = entry.TotalLength; - writer.Write(Encoding.UTF8.GetBytes(entry.FileName)); - writer.Write(NullByte); - writer.Write(entry.CRC32); - writer.Write((short)0); // SmallData, we will put it into data instead - writer.Write(i); - writer.Write(fileOffset); - writer.Write(fileLength); - writer.Write(ushort.MaxValue); // terminator, 0xFFFF - - fileOffset += fileLength; - - ReadEntry(entry, out var fileData, validateCrc: false); - writer_sub.Write(fileData); - } - - //如果是一类地址的结尾 - if (IsDirectoryTail(fileType, dicName, i, _trees)) - writer.Write(NullByte); - } - - //如果是一种类型的结尾 - if (IsTypeTail(fileType, i, _trees)) - writer.Write(NullByte); - + ReadEntry(writeEntry.Entry, out var fileData, validateCrc: false); + writer_sub.Write(fileData); } } - writer.Write(NullByte); - fileTreeSize = stream.Position - headerSize; } - fileTreeSize = stream.Position - headerSize; + + long fileTreeSize = stream.Position - headerSize; var afterFileData = stream.Position; @@ -350,7 +342,13 @@ public void Write(FileStream stream, int maxFileBytes) } } - static string GetSubFilePath(string indexFilePath,ushort indexNumber) + /// + /// Get the sub file name + /// + /// Index file path + /// Index number + /// + static string GetSubFilePath(string indexFilePath, ushort indexNumber) { FileInfo sub_FileInfo = new FileInfo(indexFilePath); @@ -362,154 +360,57 @@ static string GetSubFilePath(string indexFilePath,ushort indexNumber) return $"{sub_FileInfo.Directory}\\{sub_FileName}{sub_FileInfo.Extension}"; } - /// - /// Determine if it is located at the beginning of a set of file types - /// - /// Current file type - /// Current tree index - /// Tree list - /// Is located at the beginning of a set of files - static bool IsTypeHead(string fileType, ushort treeIndex, List>>> trees) - { - var tree = trees[treeIndex]; - if (treeIndex == 0) - return true; - - if (fileType == tree.Keys.First()) - return trees[treeIndex - 1].Keys.Last() != fileType; - else - return true; - - } - /// - /// Determine if it is located at the end of a set of file types - /// - /// Current file type - /// Current tree index - /// Tree list - /// Is located at the end of a set of files - static bool IsTypeTail(string fileType, ushort treeIndex, List>>> trees) - { - if (treeIndex == trees.Count - 1) - return true; - - var tree = trees[treeIndex]; - if (fileType == tree.Keys.Last()) - return trees[treeIndex + 1].Keys.First() != fileType; - else - return true; - - } - - /// - /// Determine if it is located at the beginning of a set of folder addresses - /// - /// Current file type - /// Current directory path - /// Current tree index - /// Tree list - /// Is located at the beginning of a set of folder addresses - static bool IsDirectoryHead(string fileType, string dicName, ushort treeIndex, List>>> trees) - { - if (treeIndex == 0) - return true; - - var tree = trees[treeIndex]; - - if (dicName == tree.Values.First().Keys.First() && fileType == tree.Keys.First()) - return trees[treeIndex - 1].Keys.Last() != fileType && trees[treeIndex - 1].Values.Last().Keys.Last() != dicName; - else - return true; - } - - /// - /// Determine if it is located at the end of a set of folder addresses - /// - /// Current file type - /// Current directory path - /// Current tree index - /// Tree list - /// Is located at the end of a set of folder addresses - static bool IsDirectoryTail(string fileType, string dicName, ushort treeIndex, List>>> trees) - { - if (treeIndex == trees.Count - 1) - return true; - - var tree = trees[treeIndex]; - if (fileType == tree.Keys.Last() && dicName == tree.Values.Last().Keys.Last()) - return trees[treeIndex + 1].Keys.First() != fileType && trees[treeIndex - 1].Values.First().Keys.First() != dicName; - else - return true; - } - /// /// Split the current tree into multiple trees based on packet size /// /// Tree of data sources /// Maximum file byte count /// List of Trees - /// If the size of a single file exceeds the size of the package,throw this exception - static List>>> CreatePacketsGroup(Dictionary> mainTypeTree, int maxFileBytes) - { - List>>> packets = new List>>>(); - //Create a new tree - Dictionary>> currentTree = new Dictionary>>(); - packets.Add(currentTree); + static List> CreatePacketsGroup(List entries, int maxFileBytes, bool isSingleFile) + { + List> groups = new List>(); uint totalLength = 0; - foreach (var typeEntries in mainTypeTree) - { - //Create a new type tree and add the current one to the current tree - Dictionary> currentTypeTree = new Dictionary>(); - currentTree[typeEntries.Key] = currentTypeTree; + ushort archiveIndex = 0; + Dictionary group = new Dictionary(); + groups.Add(group); - //Group the entries under the current type tree according to their folder location - IEnumerable> dicGroups = typeEntries.Value.GroupBy(s => s.DirectoryName.Length == 0 ? Space : s.DirectoryName); - foreach (var dicGroup in dicGroups) + if (isSingleFile) + { + foreach (var entry in entries) { - List entries = new List(); - currentTypeTree[dicGroup.Key] = entries; + group.Add(entry.GetFullPath(), new(0x7FFF, totalLength, entry)); + totalLength += entry.TotalLength; + } + } + else + { + group.Add(entries[0].GetFullPath(), new(archiveIndex, totalLength, entries[0])); + totalLength += entries[0].TotalLength; - foreach (var entry in dicGroup) + entries.RemoveAt(0); + do + { + PackageEntry entry = entries.Find(e => e.TotalLength < (ulong)maxFileBytes - totalLength); + if (entry is not null) { - var fileLength = entry.TotalLength; - //If the size of a single file exceeds the size of the package - if (fileLength >= maxFileBytes) - throw new InvalidOperationException("There are files exceeding max file bytes"); - - // TODO: Search for smaller files to fill in the empty space - if (totalLength + fileLength >= maxFileBytes) - { - if (entries.Count == 0) - { - currentTypeTree.Remove(dicGroup.Key); - if (currentTypeTree.Count == 0) - currentTree.Remove(typeEntries.Key); - - if (currentTree.Count == 0) - packets.Remove(currentTree); - } - - currentTree = new Dictionary>>(); - packets.Add(currentTree); - currentTypeTree = new Dictionary>(); - currentTree[typeEntries.Key] = currentTypeTree; - entries = new List(); - currentTypeTree[dicGroup.Key] = entries; - entries.Add(entry); - totalLength = fileLength; - } - else - { - totalLength += fileLength; - entries.Add(entry); - } - + group.Add(entry.GetFullPath(), new(archiveIndex, totalLength, entry)); + totalLength += entry.TotalLength; + entries.Remove(entry); + } + else + { + group = new Dictionary(); + groups.Add(group); + totalLength = 0; + archiveIndex++; } - } + } while (entries.Count != 0); } - return packets; + + + return groups; } } }