Skip to content

Commit 485a212

Browse files
xPawxPaw
committed
Add support for Blake3 chunk hashes
1 parent fea571a commit 485a212

File tree

9 files changed

+117
-32
lines changed

9 files changed

+117
-32
lines changed

‎ValvePak/ValvePak.Test/Files/fall_2025_rewardfx.vpk‎

14 KB
Binary file not shown.

‎ValvePak/ValvePak.Test/Files/monster_hunter_dashboard_balek3_chunk_hash.vpk‎

99.8 KB
Binary file not shown.

‎ValvePak/ValvePak.Test/PackageTest.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -488,6 +488,30 @@ public void ThrowsDueToMissingPakFile()
488488
Assert.Throws<FileNotFoundException>(() => package.VerifyChunkHashes());
489489
}
490490

491+
[Test]
492+
public void TestVerifyChunkHashes()
493+
{
494+
var path = Path.Combine(TestContext.CurrentContext.TestDirectory, "Files", "fall_2025_rewardfx.vpk");
495+
496+
using var package = new Package();
497+
package.Read(path);
498+
499+
Assert.DoesNotThrow(package.VerifyHashes);
500+
Assert.DoesNotThrow(() => package.VerifyChunkHashes(null));
501+
}
502+
503+
[Test]
504+
public void TestVerifyChunkHashesBlake3()
505+
{
506+
var path = Path.Combine(TestContext.CurrentContext.TestDirectory, "Files", "monster_hunter_dashboard_balek3_chunk_hash.vpk");
507+
508+
using var package = new Package();
509+
package.Read(path);
510+
511+
Assert.DoesNotThrow(package.VerifyHashes);
512+
Assert.DoesNotThrow(() => package.VerifyChunkHashes(null));
513+
}
514+
491515
private static void TestVPKExtraction(string path)
492516
{
493517
using var package = new Package();

ValvePak/ValvePak/ArchiveMD5SectionEntry.cs renamed to ValvePak/ValvePak/ChunkHashFraction.cs

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
namespace SteamDatabase.ValvePak
22
{
33
/// <summary>
4-
/// Represents an entry in the VPK archive MD5 section, containing checksum information for a chunk of archive data.
4+
/// Represents an entry in the VPK archive hashes section, containing checksum information for a chunk of archive data.
55
/// </summary>
6-
public class ArchiveMD5SectionEntry
6+
public class ChunkHashFraction
77
{
88
/// <summary>
99
/// Gets or sets the archive index.
1010
/// </summary>
11-
public required uint ArchiveIndex { get; set; }
11+
public required ushort ArchiveIndex { get; set; }
12+
13+
/// <summary>
14+
/// Gets or sets the hash algorithm type used for this entry.
15+
/// </summary>
16+
public required EHashType HashType { get; set; }
1217

1318
/// <summary>
1419
/// Gets or sets the offset in the package.

‎ValvePak/ValvePak/EHashType.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
namespace SteamDatabase.ValvePak
2+
{
3+
/// <summary>
4+
/// Represents the hash algorithm type used in VPK archive MD5 section entries.
5+
/// </summary>
6+
#pragma warning disable CA1028 // Enum Storage should be Int32
7+
public enum EHashType : ushort
8+
#pragma warning restore CA1028
9+
{
10+
/// <summary>
11+
/// MD5 hash algorithm.
12+
/// </summary>
13+
MD5 = 0,
14+
15+
/// <summary>
16+
/// Blake3 hash algorithm.
17+
/// </summary>
18+
Blake3 = 1,
19+
}
20+
}

‎ValvePak/ValvePak/Package.Read.cs‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -109,10 +109,10 @@ public void ReadEntry(PackageEntry entry, out byte[] output, bool validateCrc =
109109
/// <param name="validateCrc">If true, CRC32 will be calculated and verified for read data.</param>
110110
public void ReadEntry(PackageEntry entry, byte[] output, bool validateCrc = true)
111111
{
112-
ArgumentNullException.ThrowIfNull(entry);
113-
ArgumentNullException.ThrowIfNull(output);
112+
ArgumentNullException.ThrowIfNull(entry);
113+
ArgumentNullException.ThrowIfNull(output);
114114

115-
var totalLength = (int)entry.TotalLength;
115+
var totalLength = (int)entry.TotalLength;
116116

117117
if (output.Length < totalLength)
118118
{
@@ -145,7 +145,7 @@ public void ReadEntry(PackageEntry entry, byte[] output, bool validateCrc = true
145145
}
146146
finally
147147
{
148-
if (entry.ArchiveIndex != 0x7FFF)
148+
if (entry.ArchiveIndex != 0x7FFF)
149149
{
150150
fs.Dispose();
151151
}
@@ -263,23 +263,32 @@ private void ReadArchiveMD5Section()
263263

264264
if (ArchiveMD5SectionSize == 0)
265265
{
266-
ArchiveMD5Entries = [];
266+
AccessPackFileHashes = [];
267267
return;
268268
}
269269

270270
var entries = (int)(ArchiveMD5SectionSize / 28); // 28 is sizeof(VPK_MD5SectionEntry), which is int + int + int + 16 chars
271271

272-
ArchiveMD5Entries = new List<ArchiveMD5SectionEntry>(entries);
272+
AccessPackFileHashes = new List<ChunkHashFraction>(entries);
273273

274274
for (var i = 0; i < entries; i++)
275275
{
276-
ArchiveMD5Entries.Add(new ArchiveMD5SectionEntry
276+
var hashFraction = new ChunkHashFraction
277277
{
278-
ArchiveIndex = Reader.ReadUInt32(),
278+
ArchiveIndex = Reader.ReadUInt16(),
279+
HashType = (EHashType)Reader.ReadUInt16(),
279280
Offset = Reader.ReadUInt32(),
280281
Length = Reader.ReadUInt32(),
281282
Checksum = Reader.ReadBytes(16)
282-
});
283+
};
284+
285+
if (hashFraction.ArchiveIndex == 0 && hashFraction.HashType == (EHashType)0x8000)
286+
{
287+
hashFraction.ArchiveIndex = 0x7FFF;
288+
hashFraction.HashType = EHashType.MD5;
289+
}
290+
291+
AccessPackFileHashes.Add(hashFraction);
283292
}
284293
}
285294

@@ -355,10 +364,10 @@ private Stream GetFileStream(ushort archiveIndex)
355364
/// <param name="entry">Package entry.</param>
356365
/// <returns>Stream for a given package entry contents.</returns>
357366
public Stream GetMemoryMappedStreamIfPossible(PackageEntry entry)
358-
{
359-
ArgumentNullException.ThrowIfNull(entry);
367+
{
368+
ArgumentNullException.ThrowIfNull(entry);
360369

361-
if (entry.Length <= 4096 || entry.SmallData.Length > 0)
370+
if (entry.Length <= 4096 || entry.SmallData.Length > 0)
362371
{
363372
ReadEntry(entry, out var output, false);
364373
return new MemoryStream(output);

‎ValvePak/ValvePak/Package.Verify.cs‎

Lines changed: 39 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public override long Position
5555
private uint FileSizeBeforeWholeFileHash;
5656

5757
/// <summary>
58-
/// Gets the size in bytes of the whole file before <see cref="ArchiveMD5Entries"/>.
58+
/// Gets the size in bytes of the whole file before <see cref="AccessPackFileHashes"/>.
5959
/// </summary>
6060
private uint FileSizeBeforeArchiveMD5Entries;
6161

@@ -105,60 +105,84 @@ public void VerifyHashes()
105105
}
106106

107107
/// <summary>
108-
/// Verify MD5 hashes of individual chunk files provided in the VPK.
108+
/// Verify hashes of individual chunk files provided in the VPK.
109109
/// </summary>
110110
/// <param name="progressReporter">If provided, will report a string with the current verification progress.</param>
111111
public void VerifyChunkHashes(IProgress<string>? progressReporter = null)
112112
{
113113
Stream? stream = null;
114-
var lastArchiveIndex = uint.MaxValue;
114+
var lastArchiveIndex = ushort.MaxValue;
115115

116116
// When created by Valve, entries are sorted, and are 1MB chunks
117-
var allEntries = ArchiveMD5Entries
117+
var allEntries = AccessPackFileHashes
118118
.OrderBy(file => file.Offset)
119119
.GroupBy(file => file.ArchiveIndex)
120120
.OrderBy(x => x.Key)
121121
.SelectMany(x => x);
122122

123+
Span<byte> hash = stackalloc byte[16];
124+
123125
try
124126
{
125127
foreach (var entry in allEntries)
126128
{
127-
if (entry.ArchiveIndex > short.MaxValue)
129+
var hashTypeName = entry.HashType switch
128130
{
129-
throw new InvalidDataException("Unexpected archive index");
130-
}
131+
EHashType.Blake3 => "Blake3",
132+
EHashType.MD5 => "MD5",
133+
_ => string.Empty,
134+
};
131135

132-
progressReporter?.Report($"Verifying MD5 hash at offset {entry.Offset} in archive {entry.ArchiveIndex}.");
136+
progressReporter?.Report($"Verifying {hashTypeName} hash at offset {entry.Offset} in archive {entry.ArchiveIndex}.");
133137

134-
if (lastArchiveIndex != entry.ArchiveIndex)
138+
if (stream == null || lastArchiveIndex != entry.ArchiveIndex)
135139
{
136140
if (lastArchiveIndex != 0x7FFF)
137141
{
138142
stream?.Close();
139143
}
140144

141-
stream = GetFileStream((ushort)entry.ArchiveIndex);
145+
stream = GetFileStream(entry.ArchiveIndex);
142146
lastArchiveIndex = entry.ArchiveIndex;
143147
}
144148
else
145149
{
146-
Debug.Assert(stream != null); // what's actually happening here? did we miss assigning stream to Reader.BaseStream?
147-
148150
var offset = entry.ArchiveIndex == 0x7FFF ? HeaderSize + TreeSize : 0;
149151
stream.Seek(offset, SeekOrigin.Begin);
150152
}
151153

152154
using var subStream = new SubStream(stream, stream.Position + entry.Offset, entry.Length);
153-
var hash = MD5.HashData(subStream);
155+
156+
switch (entry.HashType)
157+
{
158+
case EHashType.Blake3:
159+
using (var hasher = Blake3.Hasher.New())
160+
{
161+
var buffer = new byte[8192]; // TODO: Fix this alloc
162+
int bytesRead;
163+
while ((bytesRead = subStream.Read(buffer, 0, buffer.Length)) > 0)
164+
{
165+
hasher.UpdateWithJoin(buffer.AsSpan(0, bytesRead));
166+
}
167+
hasher.Finalize(hash);
168+
}
169+
break;
170+
171+
case EHashType.MD5:
172+
MD5.HashData(subStream, hash);
173+
break;
174+
175+
default:
176+
throw new InvalidDataException($"Unrecognized hash type: {entry.HashType} ({(ushort)entry.HashType})");
177+
}
154178

155179
if (!hash.SequenceEqual(entry.Checksum))
156180
{
157-
throw new InvalidDataException($"Package checksum mismatch in archive {entry.ArchiveIndex} at {entry.Offset} ({BitConverter.ToString(hash)} != expected {BitConverter.ToString(entry.Checksum)})");
181+
throw new InvalidDataException($"Package checksum mismatch ({hashTypeName}) in archive {entry.ArchiveIndex} at {entry.Offset} ({Convert.ToHexString(hash)} != expected {Convert.ToHexString(entry.Checksum)})");
158182
}
159183
}
160184

161-
progressReporter?.Report("Successfully verified archive MD5 hashes.");
185+
progressReporter?.Report("Successfully verified archive hashes.");
162186
}
163187
finally
164188
{

‎ValvePak/ValvePak/Package.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ public partial class Package : IDisposable
106106
/// <summary>
107107
/// Gets the archive MD5 checksum section entries. Also known as cache line hashes.
108108
/// </summary>
109-
public List<ArchiveMD5SectionEntry> ArchiveMD5Entries { get; private set; } = [];
109+
public List<ChunkHashFraction> AccessPackFileHashes { get; private set; } = [];
110110

111111
private CaseInsensitivePackageEntryComparer? Comparer;
112112

‎ValvePak/ValvePak/ValvePak.csproj‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,10 @@
1818
</PropertyGroup>
1919

2020
<ItemGroup>
21-
<PackageReference Include="System.IO.Hashing" Version="9.0.10" />
21+
<PackageReference Include="Blake3" Version="2.0.0">
22+
<NoWarn>CS8002</NoWarn>
23+
</PackageReference>
24+
<PackageReference Include="System.IO.Hashing" Version="9.0.11" />
2225
</ItemGroup>
2326

2427
<ItemGroup>

0 commit comments

Comments
 (0)