From de9f476ba99ec927be88d8f23d041d1fbb1d9fd8 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 5 Aug 2022 13:02:43 +0300 Subject: [PATCH 01/49] Very rudimentary kv3 text parsing --- .gitattributes | 4 + .../Test Data/TextKV3/basic.kv3 | 4 + .../Test Data/TextKV3/multiline_crlf.kv3 | 7 + .../Test Data/TextKV3/zoo.kv3 | 30 ++ .../ValveKeyValue.Test/TextKV3/Basic.cs | 16 + .../ValveKeyValue.Test/TextKV3/ZooTest.cs | 16 + .../ValveKeyValue.Test.csproj | 2 + .../KeyValues3/KV3TextReader.cs | 323 +++++++++++++++++ .../KeyValues3/KV3TextReaderState.cs | 11 + .../KeyValues3/KV3TextReaderStateMachine.cs | 59 +++ .../KeyValues3/KV3TokenReader.cs | 342 ++++++++++++++++++ .../ValveKeyValue/KVSerializationFormat.cs | 7 +- ValveKeyValue/ValveKeyValue/KVSerializer.cs | 3 + ValveKeyValue/ValveKeyValue/KVTokenType.cs | 15 +- 14 files changed, 837 insertions(+), 2 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/basic.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/multiline_crlf.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/zoo.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs create mode 100644 ValveKeyValue/ValveKeyValue.Test/TextKV3/ZooTest.cs create mode 100644 ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs create mode 100644 ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs create mode 100644 ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs create mode 100644 ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs diff --git a/.gitattributes b/.gitattributes index 2ffde598..b4b74233 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,6 @@ * text=auto *.cs diff=csharp + +# Keep intended line endings to test parser +*_crlf.kv3 eol=crlf +*_lf.kv3 eol=lf diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/basic.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/basic.kv3 new file mode 100644 index 00000000..fcabd606 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/basic.kv3 @@ -0,0 +1,4 @@ + +{ + foo = "bar" +} diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/multiline_crlf.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/multiline_crlf.kv3 new file mode 100644 index 00000000..73ba27d3 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/multiline_crlf.kv3 @@ -0,0 +1,7 @@ + +{ + multiLineStringValue = """ +First line of a multi-line string literal. +Second line of a multi-line string literal. +""" +} diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/zoo.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/zoo.kv3 new file mode 100644 index 00000000..277ae4d6 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/zoo.kv3 @@ -0,0 +1,30 @@ + +{ + boolValue = false + intValue = 128 + doubleValue = 64.000000 + negativeIntValue = -1337 + negativeDoubleValue = -0.1337 + stringValue = "hello world" + stringThatIsAResourceReference = resource:"particles/items3_fx/star_emblem.vpcf" + multiLineStringValue = """ +First line of a multi-line string literal. +Second line of a multi-line string literal. +""" + arrayValue = + [ + 1, + 2, + ] + objectValue = + { + n = 5 + s = "foo" + } + arrayOnSingleLine = [ 16.7551, 20.3763, 19.6448 ] + "quoted.key" = "hello" + // single line comment + /* multi + line + comment */ +} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs new file mode 100644 index 00000000..aafdaaac --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs @@ -0,0 +1,16 @@ +using NUnit.Framework; + +namespace ValveKeyValue.Test.TextKV3 +{ + class BasicTest + { + [Test] + public void DeserializesHeaderAndValue() + { + using var stream = TestDataHelper.OpenResource("TextKV3.basic.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.That((string)data["foo"], Is.EqualTo("bar")); + } + } +} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/ZooTest.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/ZooTest.cs new file mode 100644 index 00000000..c69bab03 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/ZooTest.cs @@ -0,0 +1,16 @@ +using NUnit.Framework; + +namespace ValveKeyValue.Test.TextKV3 +{ + class ZooTest + { + [Test] + public void Test() + { + using var stream = TestDataHelper.OpenResource("TextKV3.zoo.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.Fail(); + } + } +} diff --git a/ValveKeyValue/ValveKeyValue.Test/ValveKeyValue.Test.csproj b/ValveKeyValue/ValveKeyValue.Test/ValveKeyValue.Test.csproj index 60caf9ed..6634dcaa 100644 --- a/ValveKeyValue/ValveKeyValue.Test/ValveKeyValue.Test.csproj +++ b/ValveKeyValue/ValveKeyValue.Test/ValveKeyValue.Test.csproj @@ -11,10 +11,12 @@ + + diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs new file mode 100644 index 00000000..88235a51 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -0,0 +1,323 @@ +using System; +using System.Globalization; +using System.IO; +using ValveKeyValue.Abstraction; + +namespace ValveKeyValue.Deserialization.KeyValues3 +{ + sealed class KV3TextReader : IVisitingReader + { + public KV3TextReader(TextReader textReader, IParsingVisitationListener listener, KVSerializerOptions options) + { + Require.NotNull(textReader, nameof(textReader)); + Require.NotNull(listener, nameof(listener)); + Require.NotNull(options, nameof(options)); + + this.listener = listener; + this.options = options; + + conditionEvaluator = new KVConditionEvaluator(options.Conditions); + tokenReader = new KV3TokenReader(textReader, options); + stateMachine = new KV3TextReaderStateMachine(); + } + + readonly IParsingVisitationListener listener; + readonly KVSerializerOptions options; + + readonly KVConditionEvaluator conditionEvaluator; + readonly KV3TokenReader tokenReader; + readonly KV3TextReaderStateMachine stateMachine; + bool disposed; + + public void ReadObject() + { + Require.NotDisposed(nameof(KV3TextReader), disposed); + + while (stateMachine.IsInObject) + { + KVToken token; + + try + { + token = tokenReader.ReadNextToken(); + } + catch (InvalidDataException ex) + { + throw new KeyValueException(ex.Message, ex); + } + catch (EndOfStreamException ex) + { + throw new KeyValueException("Found end of file while trying to read token.", ex); + } + + switch (token.TokenType) + { + case KVTokenType.Header: + // TODO: Actually parse out the header + stateMachine.SetName("root"); // TODO: Get rid of this + break; + + case KVTokenType.Assignment: + stateMachine.Push(KV3TextReaderState.InObjectBeforeValue); + break; + + case KVTokenType.String: + ReadText(token.Value); + break; + + case KVTokenType.ObjectStart: + BeginNewObject(); + break; + + case KVTokenType.ObjectEnd: + FinalizeCurrentObject(@explicit: true); + break; + + case KVTokenType.Condition: + HandleCondition(token.Value); + break; + + case KVTokenType.EndOfFile: + try + { + FinalizeDocument(); + } + catch (InvalidOperationException ex) + { + throw new KeyValueException("Found end of file when another token type was expected.", ex); + } + + break; + + case KVTokenType.Comment: + break; + + case KVTokenType.IncludeAndMerge: + if (!stateMachine.IsAtStart) + { + throw new KeyValueException("Inclusions are only valid at the beginning of a file."); + } + + stateMachine.AddItemForMerging(token.Value); + break; + + case KVTokenType.IncludeAndAppend: + if (!stateMachine.IsAtStart) + { + throw new KeyValueException("Inclusions are only valid at the beginning of a file."); + } + + stateMachine.AddItemForAppending(token.Value); + break; + + default: + throw new ArgumentOutOfRangeException(nameof(token.TokenType), token.TokenType, "Unhandled token type."); + } + } + } + + public void Dispose() + { + if (!disposed) + { + tokenReader.Dispose(); + disposed = true; + } + } + + void ReadText(string text) + { + switch (stateMachine.Current) + { + // If we're after a value when we find more text, then we must be starting a new key/value pair. + case KV3TextReaderState.InObjectAfterValue: + FinalizeCurrentObject(@explicit: false); + stateMachine.PushObject(); + SetObjectKey(text); + break; + + case KV3TextReaderState.InObjectBeforeKey: + SetObjectKey(text); + break; + + case KV3TextReaderState.InObjectBeforeValue: + var value = ParseValue(text); + var name = stateMachine.CurrentName; + listener.OnKeyValuePair(name, value); + + stateMachine.Push(KV3TextReaderState.InObjectAfterValue); + break; + + default: + throw new InvalidOperationException($"Unhandled text reader state: {stateMachine.Current}."); + } + } + + void SetObjectKey(string name) + { + stateMachine.SetName(name); + stateMachine.Push(KV3TextReaderState.InObjectBetweenKeyAndValue); + } + + void BeginNewObject() + { + if (stateMachine.Current != KV3TextReaderState.Header && stateMachine.Current != KV3TextReaderState.InObjectBetweenKeyAndValue) + { + throw new InvalidOperationException($"Attempted to begin new object while in state {stateMachine.Current}."); + } + + listener.OnObjectStart(stateMachine.CurrentName); + + stateMachine.PushObject(); + stateMachine.Push(KV3TextReaderState.InObjectBeforeKey); + } + + void FinalizeCurrentObject(bool @explicit) + { + if (stateMachine.Current != KV3TextReaderState.InObjectBeforeKey && stateMachine.Current != KV3TextReaderState.InObjectAfterValue) + { + throw new InvalidOperationException($"Attempted to finalize object while in state {stateMachine.Current}."); + } + + stateMachine.PopObject(out var discard); + + if (stateMachine.IsInObject) + { + stateMachine.Push(KV3TextReaderState.InObjectAfterValue); + } + + if (discard) + { + listener.DiscardCurrentObject(); + } + if (@explicit) + { + listener.OnObjectEnd(); + } + } + + void FinalizeDocument() + { + FinalizeCurrentObject(@explicit: true); + + if (stateMachine.IsInObject) + { + throw new InvalidOperationException("Inconsistent state - at end of file whilst inside an object."); + } + + foreach (var includedForMerge in stateMachine.ItemsForMerging) + { + DoIncludeAndMerge(includedForMerge); + } + + foreach (var includedDocument in stateMachine.ItemsForAppending) + { + DoIncludeAndAppend(includedDocument); + } + } + + void HandleCondition(string text) + { + if (stateMachine.Current != KV3TextReaderState.InObjectAfterValue) + { + throw new InvalidDataException($"Found conditional while in state {stateMachine.Current}."); + } + + if (!conditionEvaluator.Evalute(text)) + { + stateMachine.SetDiscardCurrent(); + } + } + + void DoIncludeAndMerge(string filePath) + { + var mergeListener = listener.GetMergeListener(); + + using var stream = OpenFileForInclude(filePath); + using var reader = new KV3TextReader(new StreamReader(stream), mergeListener, options); + reader.ReadObject(); + } + + void DoIncludeAndAppend(string filePath) + { + var appendListener = listener.GetAppendListener(); + + using var stream = OpenFileForInclude(filePath); + using var reader = new KV3TextReader(new StreamReader(stream), appendListener, options); + reader.ReadObject(); + } + + Stream OpenFileForInclude(string filePath) + { + if (options.FileLoader == null) + { + throw new KeyValueException("Inclusions requirer a FileLoader to be provided in KVSerializerOptions."); + } + + var stream = options.FileLoader.OpenFile(filePath); + if (stream == null) + { + throw new KeyValueException("IIncludedFileLoader returned null for included file path."); + } + + return stream; + } + + static KVValue ParseValue(string text) + { + // "0x" + 2 digits per byte. Long is 8 bytes, so s + 16 = 18. + // Expressed this way for readability, rather than using a magic value. + const int HexStringLengthForUnsignedLong = 2 + (sizeof(long) * 2); + + if (text.Length == HexStringLengthForUnsignedLong && text.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + var hexadecimalString = text[2..]; + var data = ParseHexStringAsByteArray(hexadecimalString); + + if (BitConverter.IsLittleEndian) + { + Array.Reverse(data); + } + + var value = BitConverter.ToUInt64(data, 0); + return new KVObjectValue(value, KVValueType.UInt64); + } + + const NumberStyles IntegerNumberStyles = + NumberStyles.AllowLeadingWhite | + NumberStyles.AllowLeadingSign; + + if (int.TryParse(text, IntegerNumberStyles, CultureInfo.InvariantCulture, out var intValue)) + { + return new KVObjectValue(intValue, KVValueType.Int32); + } + + const NumberStyles FloatingPointNumberStyles = + NumberStyles.AllowLeadingWhite | + NumberStyles.AllowDecimalPoint | + NumberStyles.AllowExponent | + NumberStyles.AllowLeadingSign; + + if (float.TryParse(text, FloatingPointNumberStyles, CultureInfo.InvariantCulture, out var floatValue)) + { + return new KVObjectValue(floatValue, KVValueType.FloatingPoint); + } + + return new KVObjectValue(text, KVValueType.String); + } + + static byte[] ParseHexStringAsByteArray(string hexadecimalRepresentation) + { + Require.NotNull(hexadecimalRepresentation, nameof(hexadecimalRepresentation)); + + var data = new byte[hexadecimalRepresentation.Length / 2]; + for (var i = 0; i < data.Length; i++) + { + var currentByteText = hexadecimalRepresentation.Substring(i * 2, 2); + data[i] = byte.Parse(currentByteText, NumberStyles.HexNumber, CultureInfo.InvariantCulture); + } + + return data; + } + } +} diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs new file mode 100644 index 00000000..97fe607e --- /dev/null +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs @@ -0,0 +1,11 @@ +namespace ValveKeyValue.Deserialization.KeyValues3 +{ + enum KV3TextReaderState + { + Header, + InObjectBeforeKey, + InObjectBetweenKeyAndValue, + InObjectBeforeValue, + InObjectAfterValue + } +} diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs new file mode 100644 index 00000000..aa2e9a32 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs @@ -0,0 +1,59 @@ +using System.Collections.Generic; + +namespace ValveKeyValue.Deserialization.KeyValues3 +{ + class KV3TextReaderStateMachine + { + public KV3TextReaderStateMachine() + { + states = new Stack>(); + includedPathsToMerge = new List(); + includedPathsToAppend = new List(); + + PushObject(); + Push(KV3TextReaderState.Header); + } + + readonly Stack> states; + readonly IList includedPathsToMerge; + readonly IList includedPathsToAppend; + + public KV3TextReaderState Current => CurrentObject.States.Peek(); + + public bool IsInObject => states.Count > 0; + + public bool IsAtStart => states.Count == 1 && CurrentObject.States.Count == 1 && Current == KV3TextReaderState.InObjectBeforeKey; + + public void PushObject() => states.Push(new KVPartialState()); + + public void Push(KV3TextReaderState state) => CurrentObject.States.Push(state); + + public void PopObject(out bool discard) + { + var state = states.Pop(); + discard = state.Discard; + } + + public string CurrentName => CurrentObject.Key; + + public void Pop() => CurrentObject.States.Pop(); + + public void SetName(string name) => CurrentObject.Key = name; + + public void SetValue(KVValue value) => CurrentObject.Value = value; + + public void AddItem(KVObject item) => CurrentObject.Items.Add(item); + + public void SetDiscardCurrent() => CurrentObject.Discard = true; + + public IEnumerable ItemsForMerging => includedPathsToMerge; + + public void AddItemForMerging(string item) => includedPathsToMerge.Add(item); + + public IEnumerable ItemsForAppending => includedPathsToAppend; + + public void AddItemForAppending(string item) => includedPathsToAppend.Add(item); + + KVPartialState CurrentObject => states.Peek(); + } +} diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs new file mode 100644 index 00000000..b7141609 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -0,0 +1,342 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace ValveKeyValue.Deserialization.KeyValues3 +{ + class KV3TokenReader : IDisposable + { + const char HeaderStart = '<'; + const char QuotationMark = '"'; + const char ObjectStart = '{'; + const char ObjectEnd = '}'; + const char CommentBegin = '/'; // Although Valve uses the double-slash convention, the KV spec allows for single-slash comments. + const char ConditionBegin = '['; + const char ConditionEnd = ']'; + const char InclusionMark = '#'; + const char Assignment = '='; + + public KV3TokenReader(TextReader textReader, KVSerializerOptions options) + { + Require.NotNull(textReader, nameof(textReader)); + Require.NotNull(options, nameof(options)); + + this.textReader = textReader; + this.options = options; + } + + readonly KVSerializerOptions options; + TextReader textReader; + bool disposed; + int? peekedNext; + + public KVToken ReadNextToken() + { + Require.NotDisposed(nameof(KV3TokenReader), disposed); + SwallowWhitespace(); + + var nextChar = Peek(); + if (IsEndOfFile(nextChar)) + { + return new KVToken(KVTokenType.EndOfFile); + } + + return nextChar switch + { + HeaderStart => ReadHeader(), + ObjectStart => ReadObjectStart(), + ObjectEnd => ReadObjectEnd(), + CommentBegin => ReadComment(), + ConditionBegin => ReadCondition(), + InclusionMark => ReadInclusion(), + Assignment => ReadAssignment(), + _ => ReadString(), + }; + } + + public void Dispose() + { + if (!disposed) + { + textReader.Dispose(); + textReader = null; + + disposed = true; + } + } + + KVToken ReadString() + { + var text = ReadStringRaw(); + return new KVToken(KVTokenType.String, text); + } + + KVToken ReadObjectStart() + { + ReadChar(ObjectStart); + return new KVToken(KVTokenType.ObjectStart); + } + + KVToken ReadAssignment() + { + ReadChar(Assignment); + return new KVToken(KVTokenType.Assignment); + } + + KVToken ReadObjectEnd() + { + ReadChar(ObjectEnd); + return new KVToken(KVTokenType.ObjectEnd); + } + + KVToken ReadHeader() + { + ReadChar('<'); + ReadChar('!'); + ReadChar('-'); + ReadChar('-'); + + var sb = new StringBuilder(); + bool ended; + + while (true) + { + var next = Next(); + + if (next == '\n') + { + throw new InvalidDataException("Found new line while parsing header."); + } + + if (next == '>' && sb.Length >= 2 && sb[^1] == '-' && sb[^2] == '-') + { + ended = true; + break; + } + + sb.Append(next); + } + + if (!ended) + { + throw new InvalidDataException("Did not find header comment ending."); + } + + var text = sb.ToString(); + + return new KVToken(KVTokenType.Header, text); + } + + KVToken ReadComment() + { + ReadChar(CommentBegin); + + var sb = new StringBuilder(); + var next = Next(); + + // Some keyvalues implementations have a bug where only a single slash is needed for a comment + if (next != CommentBegin) + { + sb.Append(next); + } + + while (true) + { + next = Next(); + + if (next == '\n') + { + break; + } + + sb.Append(next); + } + + if (sb.Length > 0 && sb[^1] == '\r') + { + sb.Remove(sb.Length - 1, 1); + } + + var text = sb.ToString(); + + return new KVToken(KVTokenType.Comment, text); + } + + KVToken ReadCondition() + { + ReadChar(ConditionBegin); + var text = ReadUntil(ConditionEnd); + ReadChar(ConditionEnd); + + return new KVToken(KVTokenType.Condition, text); + } + + KVToken ReadInclusion() + { + ReadChar(InclusionMark); + var term = ReadUntil(new[] { ' ', '\t' }); + var value = ReadStringRaw(); + + if (string.Equals(term, "include", StringComparison.Ordinal)) + { + return new KVToken(KVTokenType.IncludeAndAppend, value); + } + else if (string.Equals(term, "base", StringComparison.Ordinal)) + { + return new KVToken(KVTokenType.IncludeAndMerge, value); + } + + throw new InvalidDataException("Unrecognized term after '#' symbol."); + } + + char Next() + { + int next; + + if (peekedNext.HasValue) + { + next = peekedNext.Value; + peekedNext = null; + } + else + { + next = textReader.Read(); + } + + if (next == -1) + { + throw new EndOfStreamException(); + } + + return (char)next; + } + + int Peek() + { + if (peekedNext.HasValue) + { + return peekedNext.Value; + } + + var next = textReader.Read(); + peekedNext = next; + + return next; + } + + void ReadChar(char expectedChar) + { + var next = Next(); + if (next != expectedChar) + { + throw new InvalidDataException($"The syntax is incorrect, expected '{expectedChar}' but got '{next}'."); + } + } + + string ReadUntil(params char[] terminators) + { + var sb = new StringBuilder(); + var escapeNext = false; + + var integerTerminators = new HashSet(terminators.Select(t => (int)t)); + while (!integerTerminators.Contains(Peek()) || escapeNext) + { + var next = Next(); + + if (options.HasEscapeSequences) + { + if (!escapeNext && next == '\\') + { + escapeNext = true; + continue; + } + + if (escapeNext) + { + next = next switch + { + 'r' => '\r', + 'n' => '\n', + 't' => '\t', + '\\' => '\\', + '"' => '"', + _ when options.EnableValveNullByteBugBehavior => '\0', + _ => throw new InvalidDataException($"Unknown escape sequence '\\{next}'."), + }; + + escapeNext = false; + } + } + + sb.Append(next); + } + + var result = sb.ToString(); + + // Valve bug-for-bug compatibility with tier1 KeyValues/CUtlBuffer: an invalid escape sequence is a null byte which + // causes the text to be trimmed to the point of that null byte. + if (options.EnableValveNullByteBugBehavior && result.IndexOf('\0') is var nullByteIndex && nullByteIndex >= 0) + { + result = result[..nullByteIndex]; + } + return result; + } + + string ReadUntilWhitespaceOrQuote() + { + var sb = new StringBuilder(); + + while (true) + { + var next = Peek(); + if (next == -1 || char.IsWhiteSpace((char)next) || next == '"') + { + break; + } + + sb.Append(Next()); + } + + return sb.ToString(); + } + + void SwallowWhitespace() + { + while (PeekWhitespace()) + { + Next(); + } + } + + bool PeekWhitespace() + { + var next = Peek(); + return !IsEndOfFile(next) && char.IsWhiteSpace((char)next); + } + + string ReadStringRaw() + { + SwallowWhitespace(); + if (Peek() == '"') + { + return ReadQuotedStringRaw(); + } + else + { + return ReadUntilWhitespaceOrQuote(); + } + } + + string ReadQuotedStringRaw() + { + ReadChar(QuotationMark); + var text = ReadUntil(QuotationMark); + ReadChar(QuotationMark); + return text; + } + + bool IsEndOfFile(int value) => value == -1; + } +} diff --git a/ValveKeyValue/ValveKeyValue/KVSerializationFormat.cs b/ValveKeyValue/ValveKeyValue/KVSerializationFormat.cs index 6a1f7a6d..20640b9c 100644 --- a/ValveKeyValue/ValveKeyValue/KVSerializationFormat.cs +++ b/ValveKeyValue/ValveKeyValue/KVSerializationFormat.cs @@ -13,6 +13,11 @@ public enum KVSerializationFormat /// /// KeyValues 1 binary format. Used occasionally in Steam. /// - KeyValues1Binary + KeyValues1Binary, + + /// + /// KeyValues 3 textual format. Used in the Source 2 engine. + /// + KeyValues3Text, } } diff --git a/ValveKeyValue/ValveKeyValue/KVSerializer.cs b/ValveKeyValue/ValveKeyValue/KVSerializer.cs index 72db7638..c3e5aae2 100644 --- a/ValveKeyValue/ValveKeyValue/KVSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/KVSerializer.cs @@ -1,6 +1,8 @@ using ValveKeyValue.Abstraction; using ValveKeyValue.Deserialization; using ValveKeyValue.Deserialization.KeyValues1; +using ValveKeyValue.Deserialization.KeyValues3; +using ValveKeyValue.Serialization; using ValveKeyValue.Serialization.KeyValues1; namespace ValveKeyValue @@ -116,6 +118,7 @@ IVisitingReader MakeReader(Stream stream, IParsingVisitationListener listener, K { KVSerializationFormat.KeyValues1Text => new KV1TextReader(new StreamReader(stream, null, true, -1, leaveOpen: true), listener, options), KVSerializationFormat.KeyValues1Binary => new KV1BinaryReader(stream, listener, options.StringTable), + KVSerializationFormat.KeyValues3Text => new KV3TextReader(new StreamReader(stream, null, true, -1, leaveOpen: true), listener, options), _ => throw new ArgumentOutOfRangeException(nameof(format), format, "Invalid serialization format."), }; } diff --git a/ValveKeyValue/ValveKeyValue/KVTokenType.cs b/ValveKeyValue/ValveKeyValue/KVTokenType.cs index f11dceed..4d463b23 100644 --- a/ValveKeyValue/ValveKeyValue/KVTokenType.cs +++ b/ValveKeyValue/ValveKeyValue/KVTokenType.cs @@ -9,6 +9,19 @@ enum KVTokenType Comment, Condition, IncludeAndAppend, - IncludeAndMerge + IncludeAndMerge, + + // KeyValues3 + Header, + Assignment, + CommentBlock, + + SEEK_VALUE, + PROP_NAME, + VALUE_STRUCT, + VALUE_ARRAY, + VALUE_STRING_MULTI, + VALUE_NUMBER, + VALUE_FLAGGED, } } From 88db5816f95063df06f52ea2024e83b7bf2a65c7 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 5 Aug 2022 13:16:48 +0300 Subject: [PATCH 02/49] Basic test for flagged value --- .../Test Data/TextKV3/flagged_value.kv3 | 4 ++++ ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs | 9 +++++++++ 2 files changed, 13 insertions(+) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 new file mode 100644 index 00000000..f8756d44 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 @@ -0,0 +1,4 @@ + +{ + foo = resource:"bar" +} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs index aafdaaac..1e4368f4 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs @@ -12,5 +12,14 @@ public void DeserializesHeaderAndValue() Assert.That((string)data["foo"], Is.EqualTo("bar")); } + + [Test] + public void DeserializesFlaggedValues() + { + using var stream = TestDataHelper.OpenResource("TextKV3.flagged_value.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.That((string)data["foo"], Is.EqualTo("bar")); + } } } From b958d6b09a746ba3b2dd2826ed67d13a645bf37b Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 5 Aug 2022 15:44:19 +0300 Subject: [PATCH 03/49] kv3 does not have includes --- .../KeyValues3/KV3TextReader.cs | 62 ------------------- .../KeyValues3/KV3TextReaderStateMachine.cs | 12 ---- 2 files changed, 74 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 88235a51..174f2e5a 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -92,24 +92,6 @@ public void ReadObject() case KVTokenType.Comment: break; - case KVTokenType.IncludeAndMerge: - if (!stateMachine.IsAtStart) - { - throw new KeyValueException("Inclusions are only valid at the beginning of a file."); - } - - stateMachine.AddItemForMerging(token.Value); - break; - - case KVTokenType.IncludeAndAppend: - if (!stateMachine.IsAtStart) - { - throw new KeyValueException("Inclusions are only valid at the beginning of a file."); - } - - stateMachine.AddItemForAppending(token.Value); - break; - default: throw new ArgumentOutOfRangeException(nameof(token.TokenType), token.TokenType, "Unhandled token type."); } @@ -204,16 +186,6 @@ void FinalizeDocument() { throw new InvalidOperationException("Inconsistent state - at end of file whilst inside an object."); } - - foreach (var includedForMerge in stateMachine.ItemsForMerging) - { - DoIncludeAndMerge(includedForMerge); - } - - foreach (var includedDocument in stateMachine.ItemsForAppending) - { - DoIncludeAndAppend(includedDocument); - } } void HandleCondition(string text) @@ -229,40 +201,6 @@ void HandleCondition(string text) } } - void DoIncludeAndMerge(string filePath) - { - var mergeListener = listener.GetMergeListener(); - - using var stream = OpenFileForInclude(filePath); - using var reader = new KV3TextReader(new StreamReader(stream), mergeListener, options); - reader.ReadObject(); - } - - void DoIncludeAndAppend(string filePath) - { - var appendListener = listener.GetAppendListener(); - - using var stream = OpenFileForInclude(filePath); - using var reader = new KV3TextReader(new StreamReader(stream), appendListener, options); - reader.ReadObject(); - } - - Stream OpenFileForInclude(string filePath) - { - if (options.FileLoader == null) - { - throw new KeyValueException("Inclusions requirer a FileLoader to be provided in KVSerializerOptions."); - } - - var stream = options.FileLoader.OpenFile(filePath); - if (stream == null) - { - throw new KeyValueException("IIncludedFileLoader returned null for included file path."); - } - - return stream; - } - static KVValue ParseValue(string text) { // "0x" + 2 digits per byte. Long is 8 bytes, so s + 16 = 18. diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs index aa2e9a32..7ffa9512 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs @@ -7,16 +7,12 @@ class KV3TextReaderStateMachine public KV3TextReaderStateMachine() { states = new Stack>(); - includedPathsToMerge = new List(); - includedPathsToAppend = new List(); PushObject(); Push(KV3TextReaderState.Header); } readonly Stack> states; - readonly IList includedPathsToMerge; - readonly IList includedPathsToAppend; public KV3TextReaderState Current => CurrentObject.States.Peek(); @@ -46,14 +42,6 @@ public void PopObject(out bool discard) public void SetDiscardCurrent() => CurrentObject.Discard = true; - public IEnumerable ItemsForMerging => includedPathsToMerge; - - public void AddItemForMerging(string item) => includedPathsToMerge.Add(item); - - public IEnumerable ItemsForAppending => includedPathsToAppend; - - public void AddItemForAppending(string item) => includedPathsToAppend.Add(item); - KVPartialState CurrentObject => states.Peek(); } } From 0766277e6238cdcba4f9666890517a485de3d276 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 5 Aug 2022 15:50:33 +0300 Subject: [PATCH 04/49] kv3 does not have conditionals --- .../KeyValues3/KV3TextReader.cs | 25 +------------- .../KeyValues3/KV3TextReaderStateMachine.cs | 7 ++-- .../KeyValues3/KV3TokenReader.cs | 34 +------------------ 3 files changed, 4 insertions(+), 62 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 174f2e5a..dbf8d04e 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -16,7 +16,6 @@ public KV3TextReader(TextReader textReader, IParsingVisitationListener listener, this.listener = listener; this.options = options; - conditionEvaluator = new KVConditionEvaluator(options.Conditions); tokenReader = new KV3TokenReader(textReader, options); stateMachine = new KV3TextReaderStateMachine(); } @@ -24,7 +23,6 @@ public KV3TextReader(TextReader textReader, IParsingVisitationListener listener, readonly IParsingVisitationListener listener; readonly KVSerializerOptions options; - readonly KVConditionEvaluator conditionEvaluator; readonly KV3TokenReader tokenReader; readonly KV3TextReaderStateMachine stateMachine; bool disposed; @@ -73,10 +71,6 @@ public void ReadObject() FinalizeCurrentObject(@explicit: true); break; - case KVTokenType.Condition: - HandleCondition(token.Value); - break; - case KVTokenType.EndOfFile: try { @@ -161,17 +155,13 @@ void FinalizeCurrentObject(bool @explicit) throw new InvalidOperationException($"Attempted to finalize object while in state {stateMachine.Current}."); } - stateMachine.PopObject(out var discard); + stateMachine.PopObject(); if (stateMachine.IsInObject) { stateMachine.Push(KV3TextReaderState.InObjectAfterValue); } - if (discard) - { - listener.DiscardCurrentObject(); - } if (@explicit) { listener.OnObjectEnd(); @@ -188,19 +178,6 @@ void FinalizeDocument() } } - void HandleCondition(string text) - { - if (stateMachine.Current != KV3TextReaderState.InObjectAfterValue) - { - throw new InvalidDataException($"Found conditional while in state {stateMachine.Current}."); - } - - if (!conditionEvaluator.Evalute(text)) - { - stateMachine.SetDiscardCurrent(); - } - } - static KVValue ParseValue(string text) { // "0x" + 2 digits per byte. Long is 8 bytes, so s + 16 = 18. diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs index 7ffa9512..61d1cd54 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs @@ -24,10 +24,9 @@ public KV3TextReaderStateMachine() public void Push(KV3TextReaderState state) => CurrentObject.States.Push(state); - public void PopObject(out bool discard) + public void PopObject() { - var state = states.Pop(); - discard = state.Discard; + states.Pop(); } public string CurrentName => CurrentObject.Key; @@ -40,8 +39,6 @@ public void PopObject(out bool discard) public void AddItem(KVObject item) => CurrentObject.Items.Add(item); - public void SetDiscardCurrent() => CurrentObject.Discard = true; - KVPartialState CurrentObject => states.Peek(); } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index b7141609..8fff516f 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -12,10 +12,7 @@ class KV3TokenReader : IDisposable const char QuotationMark = '"'; const char ObjectStart = '{'; const char ObjectEnd = '}'; - const char CommentBegin = '/'; // Although Valve uses the double-slash convention, the KV spec allows for single-slash comments. - const char ConditionBegin = '['; - const char ConditionEnd = ']'; - const char InclusionMark = '#'; + const char CommentBegin = '/'; const char Assignment = '='; public KV3TokenReader(TextReader textReader, KVSerializerOptions options) @@ -49,8 +46,6 @@ public KVToken ReadNextToken() ObjectStart => ReadObjectStart(), ObjectEnd => ReadObjectEnd(), CommentBegin => ReadComment(), - ConditionBegin => ReadCondition(), - InclusionMark => ReadInclusion(), Assignment => ReadAssignment(), _ => ReadString(), }; @@ -164,33 +159,6 @@ KVToken ReadComment() return new KVToken(KVTokenType.Comment, text); } - KVToken ReadCondition() - { - ReadChar(ConditionBegin); - var text = ReadUntil(ConditionEnd); - ReadChar(ConditionEnd); - - return new KVToken(KVTokenType.Condition, text); - } - - KVToken ReadInclusion() - { - ReadChar(InclusionMark); - var term = ReadUntil(new[] { ' ', '\t' }); - var value = ReadStringRaw(); - - if (string.Equals(term, "include", StringComparison.Ordinal)) - { - return new KVToken(KVTokenType.IncludeAndAppend, value); - } - else if (string.Equals(term, "base", StringComparison.Ordinal)) - { - return new KVToken(KVTokenType.IncludeAndMerge, value); - } - - throw new InvalidDataException("Unrecognized term after '#' symbol."); - } - char Next() { int next; From c8696fe5bf08aa127a4fd293330d4ebd45ab27a5 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 5 Aug 2022 16:21:27 +0300 Subject: [PATCH 05/49] Basic identifier read (flagged values) --- .../KeyValues3/KV3TextReader.cs | 28 +++++++++++++++--- .../KeyValues3/KV3TokenReader.cs | 29 ++++++++----------- ValveKeyValue/ValveKeyValue/KVTokenType.cs | 1 + 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index dbf8d04e..90e7d337 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -59,6 +59,10 @@ public void ReadObject() stateMachine.Push(KV3TextReaderState.InObjectBeforeValue); break; + case KVTokenType.Identifier: + ReadIdentifier(token.Value); + break; + case KVTokenType.String: ReadText(token.Value); break; @@ -101,6 +105,26 @@ public void Dispose() } } + void ReadIdentifier(string text) + { + switch (stateMachine.Current) + { + case KV3TextReaderState.InObjectBeforeKey: + SetObjectKey(text); + break; + + case KV3TextReaderState.InObjectBeforeValue: + if (text.EndsWith(":") || text.EndsWith("+")) + { + // TODO: Parse flag like resource: then read as string + } + break; + + default: + throw new InvalidOperationException($"Unhandled text reader state: {stateMachine.Current}."); + } + } + void ReadText(string text) { switch (stateMachine.Current) @@ -112,10 +136,6 @@ void ReadText(string text) SetObjectKey(text); break; - case KV3TextReaderState.InObjectBeforeKey: - SetObjectKey(text); - break; - case KV3TextReaderState.InObjectBeforeValue: var value = ParseValue(text); var name = stateMachine.CurrentName; diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 8fff516f..e2e44aaf 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -47,7 +47,8 @@ public KVToken ReadNextToken() ObjectEnd => ReadObjectEnd(), CommentBegin => ReadComment(), Assignment => ReadAssignment(), - _ => ReadString(), + _ => ReadStringOrIdentifier(), // TODO: This should read identifiers, strings should only be read as values, keys can't be quoted + // TODO: #[] byte array }; } @@ -62,10 +63,16 @@ public void Dispose() } } - KVToken ReadString() + KVToken ReadStringOrIdentifier() { - var text = ReadStringRaw(); - return new KVToken(KVTokenType.String, text); + SwallowWhitespace(); + + if (Peek() == '"') + { + return new KVToken(KVTokenType.String, ReadQuotedStringRaw()); + } + + return new KVToken(KVTokenType.Identifier, ReadUntilWhitespaceOrQuote()); } KVToken ReadObjectStart() @@ -252,6 +259,7 @@ string ReadUntil(params char[] terminators) return result; } + // TODO: Read until delimeter: "{}[]=, \t\n'\":+;" string ReadUntilWhitespaceOrQuote() { var sb = new StringBuilder(); @@ -284,19 +292,6 @@ bool PeekWhitespace() return !IsEndOfFile(next) && char.IsWhiteSpace((char)next); } - string ReadStringRaw() - { - SwallowWhitespace(); - if (Peek() == '"') - { - return ReadQuotedStringRaw(); - } - else - { - return ReadUntilWhitespaceOrQuote(); - } - } - string ReadQuotedStringRaw() { ReadChar(QuotationMark); diff --git a/ValveKeyValue/ValveKeyValue/KVTokenType.cs b/ValveKeyValue/ValveKeyValue/KVTokenType.cs index 4d463b23..a5e984d7 100644 --- a/ValveKeyValue/ValveKeyValue/KVTokenType.cs +++ b/ValveKeyValue/ValveKeyValue/KVTokenType.cs @@ -13,6 +13,7 @@ enum KVTokenType // KeyValues3 Header, + Identifier, Assignment, CommentBlock, From 84e5f2cf691260121767b8138412d0002fb88a85 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 5 Aug 2022 17:33:26 +0300 Subject: [PATCH 06/49] Support multiline strings --- .../Test Data/TextKV3/multiline.kv3 | 7 ++ .../Test Data/apisurface.txt | 1 + .../ValveKeyValue.Test/TextKV3/Basic.cs | 18 +++ .../KeyValues3/KV3TextReader.cs | 14 +-- .../KeyValues3/KV3TokenReader.cs | 119 ++++++++++-------- 5 files changed, 99 insertions(+), 60 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/multiline.kv3 diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/multiline.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/multiline.kv3 new file mode 100644 index 00000000..73ba27d3 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/multiline.kv3 @@ -0,0 +1,7 @@ + +{ + multiLineStringValue = """ +First line of a multi-line string literal. +Second line of a multi-line string literal. +""" +} diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt b/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt index 69930fbc..d0333740 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt @@ -178,6 +178,7 @@ public sealed enum ValveKeyValue.KVSerializationFormat { KeyValues1Text = 0; KeyValues1Binary = 1; + KeyValues3Text = 2; public int CompareTo(object target); public bool Equals(object obj); diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs index 1e4368f4..53669944 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs @@ -21,5 +21,23 @@ public void DeserializesFlaggedValues() Assert.That((string)data["foo"], Is.EqualTo("bar")); } + + [Test] + public void DeserializesMultilineStrings() + { + using var stream = TestDataHelper.OpenResource("TextKV3.multiline.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.That((string)data["multiLineStringValue"], Is.EqualTo("First line of a multi-line string literal.\nSecond line of a multi-line string literal.")); + } + + [Test] + public void DeserializesMultilineStringsCRLF() + { + using var stream = TestDataHelper.OpenResource("TextKV3.multiline_crlf.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.That((string)data["multiLineStringValue"], Is.EqualTo("First line of a multi-line string literal.\r\nSecond line of a multi-line string literal.")); + } } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 90e7d337..a1627716 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -109,6 +109,13 @@ void ReadIdentifier(string text) { switch (stateMachine.Current) { + // If we're after a value when we find more text, then we must be starting a new key/value pair. + case KV3TextReaderState.InObjectAfterValue: + FinalizeCurrentObject(@explicit: false); + stateMachine.PushObject(); + SetObjectKey(text); + break; + case KV3TextReaderState.InObjectBeforeKey: SetObjectKey(text); break; @@ -129,13 +136,6 @@ void ReadText(string text) { switch (stateMachine.Current) { - // If we're after a value when we find more text, then we must be starting a new key/value pair. - case KV3TextReaderState.InObjectAfterValue: - FinalizeCurrentObject(@explicit: false); - stateMachine.PushObject(); - SetObjectKey(text); - break; - case KV3TextReaderState.InObjectBeforeValue: var value = ParseValue(text); var name = stateMachine.CurrentName; diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index e2e44aaf..32a37d37 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -67,11 +67,11 @@ KVToken ReadStringOrIdentifier() { SwallowWhitespace(); - if (Peek() == '"') + if (Peek() == QuotationMark) { return new KVToken(KVTokenType.String, ReadQuotedStringRaw()); } - + return new KVToken(KVTokenType.Identifier, ReadUntilWhitespaceOrQuote()); } @@ -138,6 +138,7 @@ KVToken ReadComment() var sb = new StringBuilder(); var next = Next(); + // TODO: Read /* */ comments // Some keyvalues implementations have a bug where only a single slash is needed for a comment if (next != CommentBegin) { @@ -210,55 +211,6 @@ void ReadChar(char expectedChar) } } - string ReadUntil(params char[] terminators) - { - var sb = new StringBuilder(); - var escapeNext = false; - - var integerTerminators = new HashSet(terminators.Select(t => (int)t)); - while (!integerTerminators.Contains(Peek()) || escapeNext) - { - var next = Next(); - - if (options.HasEscapeSequences) - { - if (!escapeNext && next == '\\') - { - escapeNext = true; - continue; - } - - if (escapeNext) - { - next = next switch - { - 'r' => '\r', - 'n' => '\n', - 't' => '\t', - '\\' => '\\', - '"' => '"', - _ when options.EnableValveNullByteBugBehavior => '\0', - _ => throw new InvalidDataException($"Unknown escape sequence '\\{next}'."), - }; - - escapeNext = false; - } - } - - sb.Append(next); - } - - var result = sb.ToString(); - - // Valve bug-for-bug compatibility with tier1 KeyValues/CUtlBuffer: an invalid escape sequence is a null byte which - // causes the text to be trimmed to the point of that null byte. - if (options.EnableValveNullByteBugBehavior && result.IndexOf('\0') is var nullByteIndex && nullByteIndex >= 0) - { - result = result[..nullByteIndex]; - } - return result; - } - // TODO: Read until delimeter: "{}[]=, \t\n'\":+;" string ReadUntilWhitespaceOrQuote() { @@ -295,9 +247,70 @@ bool PeekWhitespace() string ReadQuotedStringRaw() { ReadChar(QuotationMark); - var text = ReadUntil(QuotationMark); + + var isMultiline = false; + + var sb = new StringBuilder(); + + // Is there another quote mark? + // TODO: Peek() for more than one character + if (Peek() == QuotationMark) + { + Next(); + + // If the next character is not another quote, it's an empty string + if (Peek() == QuotationMark) + { + isMultiline = true; + + Next(); + + if (Peek() == '\r') + { + Next(); + } + + if (Peek() == '\n') + { + Next(); + } + } + else + { + return string.Empty; + } + } + + // TODO: Single quoted strings may not have new lines + var integerTerminators = new HashSet + { + QuotationMark, + }; + + while (!integerTerminators.Contains(Peek())) + { + sb.Append(Next()); + } + ReadChar(QuotationMark); - return text; + + if (isMultiline) + { + ReadChar(QuotationMark); + ReadChar(QuotationMark); + } + + if (sb.Length > 0 && sb[^1] == '\n') + { + sb.Remove(sb.Length - 1, 1); + } + + if (sb.Length > 0 && sb[^1] == '\r') + { + sb.Remove(sb.Length - 1, 1); + } + + return sb.ToString(); } bool IsEndOfFile(int value) => value == -1; From 17875bacbca1383850062cbcac9a9dd803016577 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 5 Aug 2022 17:46:26 +0300 Subject: [PATCH 07/49] Support multi line comments --- .../Test Data/TextKV3/comments.kv3 | 10 ++++ .../ValveKeyValue.Test/TextKV3/Basic.cs | 14 +++++ .../KeyValues3/KV3TokenReader.cs | 54 ++++++++++++++----- 3 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/comments.kv3 diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/comments.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/comments.kv3 new file mode 100644 index 00000000..3bee395a --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/comments.kv3 @@ -0,0 +1,10 @@ + +{ + // single line comment + /* multi + line + comment */ + foo = "bar" + one = /* comment in between */ "1" + two /* comment in between */ = "2" +} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs index 53669944..56e0697a 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs @@ -39,5 +39,19 @@ public void DeserializesMultilineStringsCRLF() Assert.That((string)data["multiLineStringValue"], Is.EqualTo("First line of a multi-line string literal.\r\nSecond line of a multi-line string literal.")); } + + [Test] + public void DeserializesComments() + { + using var stream = TestDataHelper.OpenResource("TextKV3.comments.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.Multiple(() => + { + Assert.That((string)data["foo"], Is.EqualTo("bar")); + Assert.That((string)data["one"], Is.EqualTo("1")); + Assert.That((string)data["two"], Is.EqualTo("2")); + }); + } } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 32a37d37..5b5c4323 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -137,29 +137,57 @@ KVToken ReadComment() var sb = new StringBuilder(); var next = Next(); + var isMultiline = false; // TODO: Read /* */ comments - // Some keyvalues implementations have a bug where only a single slash is needed for a comment - if (next != CommentBegin) + if (next == '*') { - sb.Append(next); + isMultiline = true; } - - while (true) + else if (next != CommentBegin) { - next = Next(); + // TODO: Return identifier? + throw new InvalidDataException("The syntax is incorrect, or is it?"); + } - if (next == '\n') + if (isMultiline) + { + while (true) { - break; - } + next = Next(); - sb.Append(next); - } + if (next == '*') + { + var nextNext = Peek(); - if (sb.Length > 0 && sb[^1] == '\r') + if (nextNext == '/') + { + Next(); + break; + } + } + + sb.Append(next); + } + } + else { - sb.Remove(sb.Length - 1, 1); + while (true) + { + next = Next(); + + if (next == '\n') + { + break; + } + + sb.Append(next); + } + + if (sb.Length > 0 && sb[^1] == '\r') + { + sb.Remove(sb.Length - 1, 1); + } } var text = sb.ToString(); From dafef2e854bccd70df5a0acc96964e9d671eb91f Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 5 Aug 2022 18:17:58 +0300 Subject: [PATCH 08/49] Partial support for arrays --- .../Test Data/TextKV3/array.kv3 | 8 +++ .../ValveKeyValue.Test/TextKV3/Basic.cs | 9 +++ .../Abstraction/IVisitationListener.cs | 2 + .../Deserialization/KVObjectBuilder.cs | 18 ++++++ .../Deserialization/KVPartialState.cs | 3 + .../KeyValues3/KV3TextReader.cs | 63 ++++++++++++++++++- .../KeyValues3/KV3TextReaderState.cs | 3 +- .../KeyValues3/KV3TokenReader.cs | 43 +++++++++---- ValveKeyValue/ValveKeyValue/KVTokenType.cs | 3 +- .../KeyValues1/KV1BinarySerializer.cs | 2 + .../KeyValues1/KV1TextSerializer.cs | 2 + 11 files changed, 138 insertions(+), 18 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 new file mode 100644 index 00000000..0ede6281 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 @@ -0,0 +1,8 @@ + +{ + arrayValue = + [ + "a", + "b", + ] +} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs index 56e0697a..c6e174f4 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs @@ -53,5 +53,14 @@ public void DeserializesComments() Assert.That((string)data["two"], Is.EqualTo("2")); }); } + + [Test] + public void DeserializesArray() + { + using var stream = TestDataHelper.OpenResource("TextKV3.array.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.True(false); + } } } diff --git a/ValveKeyValue/ValveKeyValue/Abstraction/IVisitationListener.cs b/ValveKeyValue/ValveKeyValue/Abstraction/IVisitationListener.cs index f5109716..b1f7c181 100644 --- a/ValveKeyValue/ValveKeyValue/Abstraction/IVisitationListener.cs +++ b/ValveKeyValue/ValveKeyValue/Abstraction/IVisitationListener.cs @@ -7,5 +7,7 @@ interface IVisitationListener : IDisposable void OnObjectEnd(); void OnKeyValuePair(string name, KVValue value); + + void OnArrayValue(KVValue value); } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs index 36c82a2a..5c86798b 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs @@ -43,6 +43,24 @@ public void OnKeyValuePair(string name, KVValue value) } } + public void OnArrayValue(KVValue value) + { + if (StateStack.Count > 0) + { + var state = StateStack.Peek(); + state.Children.Add(value); + } + else + { + var state = new KVPartialState + { + Value = value + }; + + StateStack.Push(state); + } + } + public void OnObjectEnd() { if (StateStack.Count <= 1) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KVPartialState.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KVPartialState.cs index 685d54a5..12e78e07 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KVPartialState.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KVPartialState.cs @@ -8,6 +8,9 @@ class KVPartialState public IList Items { get; } = new List(); + // TODO: Somehow merge with Items? + public IList Children { get; } = new List(); + public bool Discard { get; set; } } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index a1627716..e2e6b3a4 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -31,6 +31,8 @@ public void ReadObject() { Require.NotDisposed(nameof(KV3TextReader), disposed); + // TODO: Read header here as it's always expected, instead of using the tokenizer. + while (stateMachine.IsInObject) { KVToken token; @@ -56,7 +58,7 @@ public void ReadObject() break; case KVTokenType.Assignment: - stateMachine.Push(KV3TextReaderState.InObjectBeforeValue); + ReadAssignment(); break; case KVTokenType.Identifier: @@ -75,6 +77,14 @@ public void ReadObject() FinalizeCurrentObject(@explicit: true); break; + case KVTokenType.ArrayStart: + BeginNewArray(); + break; + + case KVTokenType.ArrayEnd: + FinalizeCurrentArray(); + break; + case KVTokenType.EndOfFile: try { @@ -105,10 +115,24 @@ public void Dispose() } } + void ReadAssignment() + { + if (stateMachine.Current != KV3TextReaderState.InObjectBetweenKeyAndValue) + { + throw new InvalidOperationException($"Attempted to assign while in state {stateMachine.Current}."); + } + + stateMachine.Push(KV3TextReaderState.InObjectBeforeValue); + } + void ReadIdentifier(string text) { switch (stateMachine.Current) { + case KV3TextReaderState.InArray: + new KVObjectValue(text, KVValueType.String); + break; + // If we're after a value when we find more text, then we must be starting a new key/value pair. case KV3TextReaderState.InObjectAfterValue: FinalizeCurrentObject(@explicit: false); @@ -128,16 +152,23 @@ void ReadIdentifier(string text) break; default: - throw new InvalidOperationException($"Unhandled text reader state: {stateMachine.Current}."); + throw new InvalidOperationException($"Unhandled identifier reader state: {stateMachine.Current}."); } } void ReadText(string text) { + KVValue value; + switch (stateMachine.Current) { + case KV3TextReaderState.InArray: + value = new KVObjectValue(text, KVValueType.String); + listener.OnArrayValue(value); + break; + case KV3TextReaderState.InObjectBeforeValue: - var value = ParseValue(text); + value = new KVObjectValue(text, KVValueType.String); var name = stateMachine.CurrentName; listener.OnKeyValuePair(name, value); @@ -149,6 +180,32 @@ void ReadText(string text) } } + void BeginNewArray() + { + if (stateMachine.Current != KV3TextReaderState.InObjectBeforeValue) + { + throw new InvalidOperationException($"Attempted to begin new array while in state {stateMachine.Current}."); + } + + stateMachine.PushObject(); + stateMachine.Push(KV3TextReaderState.InArray); + } + + void FinalizeCurrentArray() + { + if (stateMachine.Current != KV3TextReaderState.InArray) + { + throw new InvalidOperationException($"Attempted to finalize array while in state {stateMachine.Current}."); + } + + stateMachine.PopObject(); + + if (stateMachine.IsInObject) + { + stateMachine.Push(KV3TextReaderState.InObjectAfterValue); + } + } + void SetObjectKey(string name) { stateMachine.SetName(name); diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs index 97fe607e..46ba1f20 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs @@ -6,6 +6,7 @@ enum KV3TextReaderState InObjectBeforeKey, InObjectBetweenKeyAndValue, InObjectBeforeValue, - InObjectAfterValue + InObjectAfterValue, + InArray, } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 5b5c4323..8ac58fc4 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -12,6 +12,8 @@ class KV3TokenReader : IDisposable const char QuotationMark = '"'; const char ObjectStart = '{'; const char ObjectEnd = '}'; + const char ArrayStart = '['; + const char ArrayEnd = ']'; const char CommentBegin = '/'; const char Assignment = '='; @@ -45,6 +47,8 @@ public KVToken ReadNextToken() HeaderStart => ReadHeader(), ObjectStart => ReadObjectStart(), ObjectEnd => ReadObjectEnd(), + ArrayStart => ReadArrayStart(), + ArrayEnd => ReadArrayEnd(), CommentBegin => ReadComment(), Assignment => ReadAssignment(), _ => ReadStringOrIdentifier(), // TODO: This should read identifiers, strings should only be read as values, keys can't be quoted @@ -75,18 +79,30 @@ KVToken ReadStringOrIdentifier() return new KVToken(KVTokenType.Identifier, ReadUntilWhitespaceOrQuote()); } - KVToken ReadObjectStart() - { - ReadChar(ObjectStart); - return new KVToken(KVTokenType.ObjectStart); - } - KVToken ReadAssignment() { ReadChar(Assignment); return new KVToken(KVTokenType.Assignment); } + KVToken ReadArrayStart() + { + ReadChar(ArrayStart); + return new KVToken(KVTokenType.ArrayStart); + } + + KVToken ReadArrayEnd() + { + ReadChar(ArrayEnd); + return new KVToken(KVTokenType.ArrayEnd); + } + + KVToken ReadObjectStart() + { + ReadChar(ObjectStart); + return new KVToken(KVTokenType.ObjectStart); + } + KVToken ReadObjectEnd() { ReadChar(ObjectEnd); @@ -309,15 +325,16 @@ string ReadQuotedStringRaw() } } - // TODO: Single quoted strings may not have new lines - var integerTerminators = new HashSet + while (Peek() != QuotationMark) { - QuotationMark, - }; + var next = Next(); - while (!integerTerminators.Contains(Peek())) - { - sb.Append(Next()); + if (next == '\n') + { + throw new InvalidDataException("Found new line while parsing literal string."); + } + + sb.Append(next); } ReadChar(QuotationMark); diff --git a/ValveKeyValue/ValveKeyValue/KVTokenType.cs b/ValveKeyValue/ValveKeyValue/KVTokenType.cs index a5e984d7..8a8c0738 100644 --- a/ValveKeyValue/ValveKeyValue/KVTokenType.cs +++ b/ValveKeyValue/ValveKeyValue/KVTokenType.cs @@ -16,11 +16,12 @@ enum KVTokenType Identifier, Assignment, CommentBlock, + ArrayStart, + ArrayEnd, SEEK_VALUE, PROP_NAME, VALUE_STRUCT, - VALUE_ARRAY, VALUE_STRING_MULTI, VALUE_NUMBER, VALUE_FLAGGED, diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs index 4a5705d3..000b2c40 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs @@ -75,6 +75,8 @@ public void OnKeyValuePair(string name, KVValue value) } } + public void OnArrayValue(KVValue value) => throw new NotImplementedException(); + void Write(KV1BinaryNodeType nodeType) { writer.Write((byte)nodeType); diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs index 7473dc97..9155a44a 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs @@ -36,6 +36,8 @@ public void OnObjectEnd() public void OnKeyValuePair(string name, KVValue value) => WriteKeyValuePair(name, value); + public void OnArrayValue(KVValue value) => throw new NotImplementedException(); + public void DiscardCurrentObject() { throw new NotSupportedException("Discard not supported when writing."); From 68dd620fe47a00396ed4f310a6d4521e5059147a Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 5 Aug 2022 20:25:43 +0300 Subject: [PATCH 09/49] Add kv3 guids --- .../Test Data/apisurface.txt | 57 ++++++++----------- .../ValveKeyValue/KeyValues3/Encoding.cs | 12 ++++ .../ValveKeyValue/KeyValues3/Format.cs | 9 +++ 3 files changed, 44 insertions(+), 34 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue/KeyValues3/Encoding.cs create mode 100644 ValveKeyValue/ValveKeyValue/KeyValues3/Format.cs diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt b/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt index d0333740..f2196c6f 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt @@ -5,9 +5,6 @@ public interface ValveKeyValue.IIncludedFileLoader public class ValveKeyValue.KeyValueException { - public .ctor(); - public .ctor(string message); - public .ctor(string message, Exception inner); protected void add_SerializeObjectState(EventHandler`1[[System.Runtime.Serialization.SafeSerializationEventArgs]] value); public bool Equals(object obj); protected void Finalize(); @@ -34,7 +31,6 @@ public class ValveKeyValue.KeyValueException public class ValveKeyValue.KVArrayValue { - public .ctor(); public void Add(ValveKeyValue.KVValue value); public void AddRange(System.Collections.Generic.IEnumerable`1[[ValveKeyValue.KVValue]] values); public void Clear(); @@ -78,8 +74,6 @@ public class ValveKeyValue.KVArrayValue public class ValveKeyValue.KVBinaryBlob { - public .ctor(byte[] value); - public .ctor(Memory`1[[byte]] value); public bool Equals(object obj); protected void Finalize(); public Memory`1[[byte]] get_Bytes(); @@ -110,7 +104,6 @@ public class ValveKeyValue.KVBinaryBlob public class ValveKeyValue.KVDocument { - public .ctor(string name, ValveKeyValue.KVValue value); public void Add(ValveKeyValue.KVObject value); public bool Equals(object obj); protected void Finalize(); @@ -128,7 +121,6 @@ public class ValveKeyValue.KVDocument public sealed class ValveKeyValue.KVIgnoreAttribute { - public .ctor(); public bool Equals(object obj); protected void Finalize(); public object get_TypeId(); @@ -142,8 +134,6 @@ public sealed class ValveKeyValue.KVIgnoreAttribute public class ValveKeyValue.KVObject { - public .ctor(string name, System.Collections.Generic.IEnumerable`1[[ValveKeyValue.KVObject]] items); - public .ctor(string name, ValveKeyValue.KVValue value); public void Add(ValveKeyValue.KVObject value); public bool Equals(object obj); protected void Finalize(); @@ -161,7 +151,6 @@ public class ValveKeyValue.KVObject public sealed class ValveKeyValue.KVPropertyAttribute { - public .ctor(string propertyName); public bool Equals(object obj); protected void Finalize(); public string get_PropertyName(); @@ -212,7 +201,6 @@ public class ValveKeyValue.KVSerializer public sealed class ValveKeyValue.KVSerializerOptions { - public .ctor(); public bool Equals(object obj); protected void Finalize(); public System.Collections.Generic.IList`1[[string]] get_Conditions(); @@ -220,20 +208,17 @@ public sealed class ValveKeyValue.KVSerializerOptions public bool get_EnableValveNullByteBugBehavior(); public ValveKeyValue.IIncludedFileLoader get_FileLoader(); public bool get_HasEscapeSequences(); - public ValveKeyValue.StringTable get_StringTable(); public int GetHashCode(); public Type GetType(); protected object MemberwiseClone(); public void set_EnableValveNullByteBugBehavior(bool value); public void set_FileLoader(ValveKeyValue.IIncludedFileLoader value); public void set_HasEscapeSequences(bool value); - public void set_StringTable(ValveKeyValue.StringTable value); public string ToString(); } public class ValveKeyValue.KVValue { - protected .ctor(); public bool Equals(object obj); protected void Finalize(); public ValveKeyValue.KVValue get_Item(string key); @@ -289,17 +274,13 @@ public sealed enum ValveKeyValue.KVValueType Collection = 1; Array = 2; BinaryBlob = 3; - Boolean = 4; - String = 5; - Int16 = 6; - Int32 = 7; - Int64 = 8; - UInt16 = 9; - UInt32 = 10; - UInt64 = 11; - FloatingPoint = 12; - FloatingPoint64 = 13; - Pointer = 14; + String = 4; + Int32 = 5; + UInt64 = 6; + FloatingPoint = 7; + Pointer = 8; + Int64 = 9; + Boolean = 10; public int CompareTo(object target); public bool Equals(object obj); @@ -315,20 +296,28 @@ public sealed enum ValveKeyValue.KVValueType public string ToString(string format, IFormatProvider provider); } -public sealed class ValveKeyValue.StringTable +public class ValveKeyValue.KeyValues3.Encoding { - public .ctor(); - public .ctor(int capacity); - public .ctor(System.Collections.Generic.IList`1[[string]] values); - public void Add(string value); public bool Equals(object obj); protected void Finalize(); - public string get_Item(int index); + public static Guid get_BinaryBlockCompressed(); + public static Guid get_BinaryBlockLZ4(); + public static Guid get_BinaryBlockUncompressed(); + public static Guid get_Text(); + public int GetHashCode(); + public Type GetType(); + protected object MemberwiseClone(); + public string ToString(); +} + +public class ValveKeyValue.KeyValues3.Format +{ + public bool Equals(object obj); + protected void Finalize(); + public static Guid get_Generic(); public int GetHashCode(); - public int GetOrAdd(string value); public Type GetType(); protected object MemberwiseClone(); - public string[] ToArray(); public string ToString(); } diff --git a/ValveKeyValue/ValveKeyValue/KeyValues3/Encoding.cs b/ValveKeyValue/ValveKeyValue/KeyValues3/Encoding.cs new file mode 100644 index 00000000..3bfac429 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue/KeyValues3/Encoding.cs @@ -0,0 +1,12 @@ +using System; + +namespace ValveKeyValue.KeyValues3 +{ + public class Encoding + { + public static Guid Text { get; } = new(new byte[] { 0x3C, 0x7F, 0x1C, 0xE2, 0x33, 0x8A, 0xC5, 0x41, 0x99, 0x77, 0xA7, 0x6D, 0x3A, 0x32, 0xAA, 0x0D }); + public static Guid BinaryBlockCompressed { get; } = new(new byte[] { 0x46, 0x1A, 0x79, 0x95, 0xBC, 0x95, 0x6C, 0x4F, 0xA7, 0x0B, 0x05, 0xBC, 0xA1, 0xB7, 0xDF, 0xD2 }); + public static Guid BinaryBlockUncompressed { get; } = new(new byte[] { 0x00, 0x05, 0x86, 0x1B, 0xD8, 0xF7, 0xC1, 0x40, 0xAD, 0x82, 0x75, 0xA4, 0x82, 0x67, 0xE7, 0x14 }); + public static Guid BinaryBlockLZ4 { get; } = new(new byte[] { 0x8A, 0x34, 0x47, 0x68, 0xA1, 0x63, 0x5C, 0x4F, 0xA1, 0x97, 0x53, 0x80, 0x6F, 0xD9, 0xB1, 0x19 }); + } +} diff --git a/ValveKeyValue/ValveKeyValue/KeyValues3/Format.cs b/ValveKeyValue/ValveKeyValue/KeyValues3/Format.cs new file mode 100644 index 00000000..017efd84 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue/KeyValues3/Format.cs @@ -0,0 +1,9 @@ +using System; + +namespace ValveKeyValue.KeyValues3 +{ + public class Format + { + public static Guid Generic { get; } = new(new byte[] { 0x7C, 0x16, 0x12, 0x74, 0xE9, 0x06, 0x98, 0x46, 0xAF, 0xF2, 0xE6, 0x3E, 0xB5, 0x90, 0x37, 0xE7 }); + } +} From 5d26bff99ec36f1747b446f52cbb71b5ccbc45c0 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 5 Aug 2022 20:26:00 +0300 Subject: [PATCH 10/49] Parse kv3 header outside of the state machine --- .../KeyValues3/KV3TextReader.cs | 13 +- .../KeyValues3/KV3TextReaderState.cs | 3 +- .../KeyValues3/KV3TextReaderStateMachine.cs | 5 +- .../KeyValues3/KV3TokenReader.cs | 127 +++++++++++++----- 4 files changed, 105 insertions(+), 43 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index e2e6b3a4..69e0f3c3 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -31,7 +31,7 @@ public void ReadObject() { Require.NotDisposed(nameof(KV3TextReader), disposed); - // TODO: Read header here as it's always expected, instead of using the tokenizer. + tokenReader.ReadHeader(); while (stateMachine.IsInObject) { @@ -52,11 +52,6 @@ public void ReadObject() switch (token.TokenType) { - case KVTokenType.Header: - // TODO: Actually parse out the header - stateMachine.SetName("root"); // TODO: Get rid of this - break; - case KVTokenType.Assignment: ReadAssignment(); break; @@ -117,7 +112,7 @@ public void Dispose() void ReadAssignment() { - if (stateMachine.Current != KV3TextReaderState.InObjectBetweenKeyAndValue) + if (stateMachine.Current != KV3TextReaderState.InObjectAfterKey) { throw new InvalidOperationException($"Attempted to assign while in state {stateMachine.Current}."); } @@ -209,12 +204,12 @@ void FinalizeCurrentArray() void SetObjectKey(string name) { stateMachine.SetName(name); - stateMachine.Push(KV3TextReaderState.InObjectBetweenKeyAndValue); + stateMachine.Push(KV3TextReaderState.InObjectAfterKey); } void BeginNewObject() { - if (stateMachine.Current != KV3TextReaderState.Header && stateMachine.Current != KV3TextReaderState.InObjectBetweenKeyAndValue) + if (stateMachine.Current != KV3TextReaderState.InObjectAfterKey) { throw new InvalidOperationException($"Attempted to begin new object while in state {stateMachine.Current}."); } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs index 46ba1f20..8aa742b1 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs @@ -2,9 +2,8 @@ { enum KV3TextReaderState { - Header, InObjectBeforeKey, - InObjectBetweenKeyAndValue, + InObjectAfterKey, InObjectBeforeValue, InObjectAfterValue, InArray, diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs index 61d1cd54..2d401bca 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs @@ -8,8 +8,11 @@ public KV3TextReaderStateMachine() { states = new Stack>(); + // TODO: Get rid of this, kv3 has no root + // Bare values such as 'null' can be root PushObject(); - Push(KV3TextReaderState.Header); + SetName("root"); + Push(KV3TextReaderState.InObjectAfterKey); } readonly Stack> states; diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 8ac58fc4..73db5712 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -1,14 +1,13 @@ using System; -using System.Collections.Generic; using System.IO; -using System.Linq; using System.Text; +using ValveKeyValue.KeyValues3; +using Encoding = ValveKeyValue.KeyValues3.Encoding; namespace ValveKeyValue.Deserialization.KeyValues3 { class KV3TokenReader : IDisposable { - const char HeaderStart = '<'; const char QuotationMark = '"'; const char ObjectStart = '{'; const char ObjectEnd = '}'; @@ -44,7 +43,6 @@ public KVToken ReadNextToken() return nextChar switch { - HeaderStart => ReadHeader(), ObjectStart => ReadObjectStart(), ObjectEnd => ReadObjectEnd(), ArrayStart => ReadArrayStart(), @@ -76,7 +74,7 @@ KVToken ReadStringOrIdentifier() return new KVToken(KVTokenType.String, ReadQuotedStringRaw()); } - return new KVToken(KVTokenType.Identifier, ReadUntilWhitespaceOrQuote()); + return new KVToken(KVTokenType.Identifier, ReadUntilWhitespaceOrDelimeter(QuotationMark)); } KVToken ReadAssignment() @@ -109,42 +107,90 @@ KVToken ReadObjectEnd() return new KVToken(KVTokenType.ObjectEnd); } - KVToken ReadHeader() + public KVToken ReadHeader() { - ReadChar('<'); - ReadChar('!'); - ReadChar('-'); - ReadChar('-'); + var str = ReadUntilWhitespaceOrDelimeter((char)0); - var sb = new StringBuilder(); - bool ended; + if (str != "") + { + throw new InvalidDataException($"The header is incorrect, expected '-->' but got '{str}'."); + } + + if (encodingType.Equals("text", StringComparison.OrdinalIgnoreCase) && encoding != Encoding.Text) + { + throw new InvalidDataException($"Unrecognized format specifier, expected '{Encoding.Text}' but got '{format}'."); + } + + if (encodingType.Equals("generic", StringComparison.OrdinalIgnoreCase) && format != Format.Generic) + { + throw new InvalidDataException($"Unrecognized encoding specifier, expected '{Format.Generic}' but got '{format}'."); + } + + return new KVToken(KVTokenType.Header, string.Empty); } KVToken ReadComment() @@ -255,15 +301,34 @@ void ReadChar(char expectedChar) } } + string ReadUntil(char delimeter) + { + var sb = new StringBuilder(); + + while (true) + { + var next = Peek(); + + if (next == delimeter) + { + break; + } + + sb.Append(Next()); + } + + return sb.ToString(); + } + // TODO: Read until delimeter: "{}[]=, \t\n'\":+;" - string ReadUntilWhitespaceOrQuote() + string ReadUntilWhitespaceOrDelimeter(char delimeter) { var sb = new StringBuilder(); while (true) { var next = Peek(); - if (next == -1 || char.IsWhiteSpace((char)next) || next == '"') + if (next == -1 || char.IsWhiteSpace((char)next) || next == delimeter) { break; } @@ -329,7 +394,7 @@ string ReadQuotedStringRaw() { var next = Next(); - if (next == '\n') + if (!isMultiline && next == '\n') { throw new InvalidDataException("Found new line while parsing literal string."); } From 897103444224b96f5a9e564ade4f1f7a36052952 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 5 Aug 2022 23:10:18 +0300 Subject: [PATCH 11/49] Add tests for header parsing --- .../{Basic.cs => BasicKV3TestCases.cs} | 2 +- .../TextKV3/HeadersTestCase.cs | 82 +++++++++++++++++++ .../KeyValues3/KV3TokenReader.cs | 6 +- 3 files changed, 86 insertions(+), 4 deletions(-) rename ValveKeyValue/ValveKeyValue.Test/TextKV3/{Basic.cs => BasicKV3TestCases.cs} (98%) create mode 100644 ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs similarity index 98% rename from ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs rename to ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index c6e174f4..106f90ed 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Basic.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -2,7 +2,7 @@ namespace ValveKeyValue.Test.TextKV3 { - class BasicTest + class BasicKV3TestCases { [Test] public void DeserializesHeaderAndValue() diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs new file mode 100644 index 00000000..d1fc47e5 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs @@ -0,0 +1,82 @@ +using System; +using System.IO; +using System.Text; +using NUnit.Framework; + +namespace ValveKeyValue.Test.TextKV3 +{ + class HeadersTestCase + { + [TestCase("")] + [TestCase("")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase(""; + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(value)); + var kv = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); + + Assert.That(() => kv.Deserialize(stream), Throws.Exception.TypeOf()); + } + + [Test] + public void IncorrectFormatGenericGuidThrows() + { + var value = ""; + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(value)); + var kv = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); + + Assert.That(() => kv.Deserialize(stream), Throws.Exception.TypeOf()); + } + + [TestCase("")] + [TestCase("")] + [TestCase("")] + [TestCase("}")] + public void InvalidGuidThrows(string value) + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(value)); + var kv = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); + + Assert.That(() => kv.Deserialize(stream), Throws.Exception.TypeOf()); + } + + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + [TestCase("\n{}")] + public void ValidHeadersAreParsed(string value) + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(value)); + var kv = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); + + Assert.Pass(); + } + } +} diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 73db5712..74574067 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -49,7 +49,7 @@ public KVToken ReadNextToken() ArrayEnd => ReadArrayEnd(), CommentBegin => ReadComment(), Assignment => ReadAssignment(), - _ => ReadStringOrIdentifier(), // TODO: This should read identifiers, strings should only be read as values, keys can't be quoted + _ => ReadStringOrIdentifier(), // TODO: This should read identifiers, strings should only be read as values // TODO: #[] byte array }; } @@ -182,10 +182,10 @@ public KVToken ReadHeader() if (encodingType.Equals("text", StringComparison.OrdinalIgnoreCase) && encoding != Encoding.Text) { - throw new InvalidDataException($"Unrecognized format specifier, expected '{Encoding.Text}' but got '{format}'."); + throw new InvalidDataException($"Unrecognized format specifier, expected '{Encoding.Text}' but got '{encoding}'."); } - if (encodingType.Equals("generic", StringComparison.OrdinalIgnoreCase) && format != Format.Generic) + if (formatType.Equals("generic", StringComparison.OrdinalIgnoreCase) && format != Format.Generic) { throw new InvalidDataException($"Unrecognized encoding specifier, expected '{Format.Generic}' but got '{format}'."); } From 3530dae124cee915999aa04a0bd607a93f70dfc0 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Sun, 7 Aug 2022 11:27:09 +0300 Subject: [PATCH 12/49] Add value type to debugger display --- ValveKeyValue/ValveKeyValue/KVObject.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ValveKeyValue/ValveKeyValue/KVObject.cs b/ValveKeyValue/ValveKeyValue/KVObject.cs index 1e907cba..858dc6f8 100644 --- a/ValveKeyValue/ValveKeyValue/KVObject.cs +++ b/ValveKeyValue/ValveKeyValue/KVObject.cs @@ -100,6 +100,17 @@ KVCollectionValue GetCollectionValue() return collection; } - string DebuggerDescription => $"{Name}: {Value}"; + string DebuggerDescription + { + get + { + if (Value.ValueType == KVValueType.String) + { + return $"{Name}: {Value}"; + } + + return $"{Name}: {Value} ({Value.ValueType})"; + } + } } } From 5cf4b603ca5e5533d098376c307e036d4157c80b Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Sun, 7 Aug 2022 11:50:40 +0300 Subject: [PATCH 13/49] Parse basic types --- .../Test Data/TextKV3/types.kv3 | 19 +++++ .../TextKV3/BasicKV3TestCases.cs | 58 ++++++++++++++ .../KeyValues3/KV3TextReader.cs | 77 +++++++++++-------- 3 files changed, 123 insertions(+), 31 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 new file mode 100644 index 00000000..9284bb9b --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 @@ -0,0 +1,19 @@ + +{ + boolFalseValue = false + boolTrueValue = true + nullValue = null + intValue = 128 + doubleValue = 64.000000 + negativeIntValue = -1337 + negativeDoubleValue = -0.1337 + plusIntValue = +1337 + plusDoubleValue = +0.1337 + stringValue = "hello world" + negativeMaxInt = -9223372036854775807 + positiveMaxInt = 18446744073709551615 + doubleMaxValue = 62147483647.1337 + doubleNegativeMaxValue = -62147483647.1337 + doubleExponent = 1.23456E+2 + intWithStringSuffix = 123foobar +} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index 106f90ed..3b42b961 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -62,5 +62,63 @@ public void DeserializesArray() Assert.True(false); } + + [Test] + public void DeserializesBasicTypes() + { + using var stream = TestDataHelper.OpenResource("TextKV3.types.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + Assert.Multiple(() => + { + Assert.That(data["boolFalseValue"].ValueType, Is.EqualTo(KVValueType.Boolean)); + Assert.That((bool)data["boolFalseValue"], Is.EqualTo(false)); + + Assert.That(data["boolTrueValue"].ValueType, Is.EqualTo(KVValueType.Boolean)); + Assert.That((bool)data["boolTrueValue"], Is.EqualTo(true)); + + Assert.That(data["nullValue"].ValueType, Is.EqualTo(KVValueType.Null)); + //Assert.That(data["nullValue"], Is.EqualTo(null)); + + Assert.That(data["intValue"].ValueType, Is.EqualTo(KVValueType.UInt64)); + Assert.That((int)data["intValue"], Is.EqualTo(128)); + + Assert.That(data["doubleValue"].ValueType, Is.EqualTo(KVValueType.FloatingPoint)); + Assert.That((double)data["doubleValue"], Is.EqualTo(64.000000)); + + Assert.That(data["negativeIntValue"].ValueType, Is.EqualTo(KVValueType.Int64)); + Assert.That((long)data["negativeIntValue"], Is.EqualTo(-1337)); + + Assert.That(data["negativeDoubleValue"].ValueType, Is.EqualTo(KVValueType.FloatingPoint)); + Assert.That((double)data["negativeDoubleValue"], Is.EqualTo(-0.1337)); + + Assert.That(data["plusIntValue"].ValueType, Is.EqualTo(KVValueType.UInt64)); + Assert.That((ulong)data["plusIntValue"], Is.EqualTo(+1337)); + + Assert.That(data["plusDoubleValue"].ValueType, Is.EqualTo(KVValueType.FloatingPoint)); + Assert.That((double)data["plusDoubleValue"], Is.EqualTo(+0.1337)); + + Assert.That(data["stringValue"].ValueType, Is.EqualTo(KVValueType.String)); + Assert.That((string)data["stringValue"], Is.EqualTo("hello world")); + + Assert.That(data["negativeMaxInt"].ValueType, Is.EqualTo(KVValueType.Int64)); + Assert.That((long)data["negativeMaxInt"], Is.EqualTo(-9223372036854775807)); + + Assert.That(data["positiveMaxInt"].ValueType, Is.EqualTo(KVValueType.UInt64)); + Assert.That((ulong)data["positiveMaxInt"], Is.EqualTo(18446744073709551615)); + + Assert.That(data["doubleMaxValue"].ValueType, Is.EqualTo(KVValueType.FloatingPoint)); + Assert.That((double)data["doubleMaxValue"], Is.EqualTo(62147483647.1337)); + + Assert.That(data["doubleNegativeMaxValue"].ValueType, Is.EqualTo(KVValueType.FloatingPoint)); + Assert.That((double)data["doubleNegativeMaxValue"], Is.EqualTo(-62147483647.1337)); + + Assert.That(data["doubleExponent"].ValueType, Is.EqualTo(KVValueType.FloatingPoint)); + Assert.That((double)data["doubleExponent"], Is.EqualTo(123.456)); + + // TODO: Should this throw instead because strings need to be quoted? Or should it parse until it hits a non number like 123? + Assert.That(data["intWithStringSuffix"].ValueType, Is.EqualTo(KVValueType.String)); + Assert.That((string)data["intWithStringSuffix"], Is.EqualTo("123foobar")); + }); + } } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 69e0f3c3..45ecbc32 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -140,10 +140,21 @@ void ReadIdentifier(string text) break; case KV3TextReaderState.InObjectBeforeValue: - if (text.EndsWith(":") || text.EndsWith("+")) + if (text.EndsWith(":", StringComparison.Ordinal) || text.EndsWith("+", StringComparison.Ordinal)) { // TODO: Parse flag like resource: then read as string } + + KVValue value = ParseValue(text); + + if (value != null) + { + var name = stateMachine.CurrentName; + listener.OnKeyValuePair(name, value); + + stateMachine.Push(KV3TextReaderState.InObjectAfterValue); + } + break; default: @@ -252,42 +263,46 @@ void FinalizeDocument() static KVValue ParseValue(string text) { - // "0x" + 2 digits per byte. Long is 8 bytes, so s + 16 = 18. - // Expressed this way for readability, rather than using a magic value. - const int HexStringLengthForUnsignedLong = 2 + (sizeof(long) * 2); - - if (text.Length == HexStringLengthForUnsignedLong && text.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + if (text.Equals("false", StringComparison.Ordinal)) { - var hexadecimalString = text[2..]; - var data = ParseHexStringAsByteArray(hexadecimalString); - - if (BitConverter.IsLittleEndian) - { - Array.Reverse(data); - } - - var value = BitConverter.ToUInt64(data, 0); - return new KVObjectValue(value, KVValueType.UInt64); + return new KVObjectValue(false, KVValueType.Boolean); } - - const NumberStyles IntegerNumberStyles = - NumberStyles.AllowLeadingWhite | - NumberStyles.AllowLeadingSign; - - if (int.TryParse(text, IntegerNumberStyles, CultureInfo.InvariantCulture, out var intValue)) + else if (text.Equals("true", StringComparison.Ordinal)) { - return new KVObjectValue(intValue, KVValueType.Int32); + return new KVObjectValue(true, KVValueType.Boolean); } + else if (text.Equals("null", StringComparison.Ordinal)) + { + // TODO: Null is not a string + // TODO: KVObjectValue does not accept null + //value = new KVObjectValue(null, KVValueType.Null); + return new KVObjectValue(string.Empty, KVValueType.Null); + } + else if (char.IsDigit(text[0]) || text[0] == '-' || text[0] == '+') + { + // TODO: Due to Valve's string to int/double conversion functions, it is possible to have 0x hex values (as well as prefixed with minus like -0x) - const NumberStyles FloatingPointNumberStyles = - NumberStyles.AllowLeadingWhite | - NumberStyles.AllowDecimalPoint | - NumberStyles.AllowExponent | - NumberStyles.AllowLeadingSign; + const NumberStyles IntegerNumberStyles = NumberStyles.AllowLeadingSign; - if (float.TryParse(text, FloatingPointNumberStyles, CultureInfo.InvariantCulture, out var floatValue)) - { - return new KVObjectValue(floatValue, KVValueType.FloatingPoint); + if (text[0] == '-' && long.TryParse(text, IntegerNumberStyles, CultureInfo.InvariantCulture, out var intValue)) + { + return new KVObjectValue(intValue, KVValueType.Int64); + } + else if (ulong.TryParse(text, IntegerNumberStyles, CultureInfo.InvariantCulture, out var uintValue)) + { + return new KVObjectValue(uintValue, KVValueType.UInt64); + } + + const NumberStyles FloatingPointNumberStyles = + NumberStyles.AllowDecimalPoint | + NumberStyles.AllowExponent | + NumberStyles.AllowLeadingSign; + + // TODO: + if (double.TryParse(text, FloatingPointNumberStyles, CultureInfo.InvariantCulture, out var floatValue)) + { + return new KVObjectValue(floatValue, KVValueType.FloatingPoint); + } } return new KVObjectValue(text, KVValueType.String); From 934033b5e1d8a61871fea9434656266f1c0ef6cf Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Sun, 7 Aug 2022 12:12:12 +0300 Subject: [PATCH 14/49] Add some text kv3 serialization support --- .../Test Data/TextKV3/types.kv3 | 2 +- .../Test Data/TextKV3/types_serialized.kv3 | 23 +++ .../TextKV3/BasicKV3TestCases.cs | 2 +- .../TextKV3/SerializationTestCase.cs | 32 ++++ .../Abstraction/KVObjectVisitor.cs | 2 + ValveKeyValue/ValveKeyValue/KVSerializer.cs | 2 + .../KeyValues3/KV3TextSerializer.cs | 165 ++++++++++++++++++ 7 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs create mode 100644 ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 index 9284bb9b..758cde3e 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 @@ -4,7 +4,7 @@ boolTrueValue = true nullValue = null intValue = 128 - doubleValue = 64.000000 + doubleValue = 64.123 negativeIntValue = -1337 negativeDoubleValue = -0.1337 plusIntValue = +1337 diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 new file mode 100644 index 00000000..ceb24423 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 @@ -0,0 +1,23 @@ + +{ + boolFalseValue = false + boolTrueValue = true + nullValue = null + intValue = 128 + doubleValue = 64.123 + negativeIntValue = -1337 + negativeDoubleValue = -0.1337 + plusIntValue = 1337 + plusDoubleValue = 0.1337 + stringValue = "hello world" + negativeMaxInt = -9223372036854775807 + positiveMaxInt = 18446744073709551615 + doubleMaxValue = 62147483647.1337 + doubleNegativeMaxValue = -62147483647.1337 + doubleExponent = 123.456 + intWithStringSuffix = "123foobar" + multiLineString = """ +hello +world +""" +} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index 3b42b961..1261928f 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -83,7 +83,7 @@ public void DeserializesBasicTypes() Assert.That((int)data["intValue"], Is.EqualTo(128)); Assert.That(data["doubleValue"].ValueType, Is.EqualTo(KVValueType.FloatingPoint)); - Assert.That((double)data["doubleValue"], Is.EqualTo(64.000000)); + Assert.That((double)data["doubleValue"], Is.EqualTo(64.123)); Assert.That(data["negativeIntValue"].ValueType, Is.EqualTo(KVValueType.Int64)); Assert.That((long)data["negativeIntValue"], Is.EqualTo(-1337)); diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs new file mode 100644 index 00000000..3c705b81 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs @@ -0,0 +1,32 @@ +using System.IO; +using NUnit.Framework; + +namespace ValveKeyValue.Test.TextKV3 +{ + class SerializationTestCase + { + [Test] + public void CreatesTextDocument() + { + using var stream = TestDataHelper.OpenResource("TextKV3.types.kv3"); + var expected = TestDataHelper.ReadTextResource("TextKV3.types_serialized.kv3"); + + var kv = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); + var data = kv.Deserialize(stream); + + data.Add(new KVObject("multiLineString", "hello\nworld")); + + string text; + using (var ms = new MemoryStream()) + { + kv.Serialize(ms, data); + + ms.Seek(0, SeekOrigin.Begin); + using var reader = new StreamReader(ms); + text = reader.ReadToEnd(); + } + + Assert.That(text, Is.EqualTo(expected)); + } + } +} diff --git a/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs b/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs index 431e3a0d..85fbc3c6 100644 --- a/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs +++ b/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs @@ -32,6 +32,8 @@ void VisitObject(string name, KVValue value) case KVValueType.String: case KVValueType.UInt64: case KVValueType.Int64: + case KVValueType.Boolean: + case KVValueType.Null: listener.OnKeyValuePair(name, value); break; diff --git a/ValveKeyValue/ValveKeyValue/KVSerializer.cs b/ValveKeyValue/ValveKeyValue/KVSerializer.cs index c3e5aae2..a8231520 100644 --- a/ValveKeyValue/ValveKeyValue/KVSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/KVSerializer.cs @@ -4,6 +4,7 @@ using ValveKeyValue.Deserialization.KeyValues3; using ValveKeyValue.Serialization; using ValveKeyValue.Serialization.KeyValues1; +using ValveKeyValue.Serialization.KeyValues3; namespace ValveKeyValue { @@ -132,6 +133,7 @@ IVisitationListener MakeSerializer(Stream stream, KVSerializerOptions options) { KVSerializationFormat.KeyValues1Text => new KV1TextSerializer(stream, options), KVSerializationFormat.KeyValues1Binary => new KV1BinarySerializer(stream, options.StringTable), + KVSerializationFormat.KeyValues3Text => new KV3TextSerializer(stream), _ => throw new ArgumentOutOfRangeException(nameof(format), format, "Invalid serialization format."), }; ; diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs new file mode 100644 index 00000000..ab7941a5 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs @@ -0,0 +1,165 @@ +using System; +using System.IO; +using System.Text; +using ValveKeyValue.Abstraction; + +namespace ValveKeyValue.Serialization.KeyValues3 +{ + sealed class KV3TextSerializer : IVisitationListener, IDisposable + { + public KV3TextSerializer(Stream stream) + { + Require.NotNull(stream, nameof(stream)); + + writer = new StreamWriter(stream, new UTF8Encoding(), bufferSize: 1024, leaveOpen: true) + { + NewLine = "\n" + }; + + // TODO: Write correct encoding and format + writer.WriteLine(""); + } + + readonly TextWriter writer; + int indentation = 0; + + public void Dispose() + { + writer.Dispose(); + } + + public void OnObjectStart(string name) + => WriteStartObject(name); + + public void OnObjectEnd() + => WriteEndObject(); + + public void OnKeyValuePair(string name, KVValue value) + => WriteKeyValuePair(name, value); + + public void OnArrayValue(KVValue value) => throw new NotImplementedException(); + + public void DiscardCurrentObject() + { + throw new NotSupportedException("Discard not supported when writing."); + } + + void WriteStartObject(string name) + { + WriteIndentation(); + + // TODO: Dumb hack, we should not have a root name + if (indentation != 0 && name != "root") + { + WriteText(name); + WriteLine(); + } + + WriteIndentation(); + writer.Write('{'); + indentation++; + WriteLine(); + } + + void WriteEndObject() + { + indentation--; + WriteIndentation(); + writer.Write('}'); + writer.WriteLine(); + } + + void WriteKeyValuePair(string name, KVValue value) + { + WriteIndentation(); + + // TODO: We need to quote keys when they contain terminators such as a dot + writer.Write(name); + writer.Write(" = "); + + switch (value.ValueType) + { + case KVValueType.Boolean: + if ((bool)value) + { + writer.Write("true"); + } + else + { + writer.Write("false"); + } + break; + case KVValueType.Null: + writer.Write("null"); + break; + case KVValueType.FloatingPoint: + case KVValueType.Int64: + case KVValueType.UInt64: + writer.Write(value.ToString(null)); + break; + default: + WriteText(value.ToString(null)); + break; + } + + WriteLine(); + } + + void WriteIndentation() + { + if (indentation == 0) + { + return; + } + + var text = new string('\t', indentation); + writer.Write(text); + } + + void WriteText(string text) + { + var isMultiline = text.Contains("\n", StringComparison.Ordinal); + + if (isMultiline) + { + writer.Write("\"\"\"\n"); + } + else + { + writer.Write('"'); + } + + foreach (var @char in text) + { + switch (@char) + { + case '"': + writer.Write("\\\""); + break; + + case '\\': + writer.Write("\\"); + break; + + default: + writer.Write(@char); + break; + } + } + + if (isMultiline) + { + writer.Write("\n\"\"\""); + } + else + { + writer.Write('"'); + } + } + + void WriteLine() + { + writer.WriteLine(); + } + } +} From 0923322c6fd25e1f23879c97b8511435416b4cc2 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Mon, 8 Aug 2022 11:25:56 +0300 Subject: [PATCH 15/49] Prepare kv flag handling --- .../Test Data/TextKV3/flagged_value.kv3 | 2 ++ .../TextKV3/BasicKV3TestCases.cs | 7 ++++++- .../KeyValues3/KV3TextReader.cs | 20 ++++++++++++++++++- .../ValveKeyValue/KeyValues3/KVFlag.cs | 13 ++++++++++++ .../KeyValues3/KVFlaggedObjectValue.cs | 16 +++++++++++++++ 5 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue/KeyValues3/KVFlag.cs create mode 100644 ValveKeyValue/ValveKeyValue/KeyValues3/KVFlaggedObjectValue.cs diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 index f8756d44..5307d0b6 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 @@ -1,4 +1,6 @@ { foo = resource:"bar" + bar = resource+"foo" + flaggedNumber = panorama:-1234 } diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index 1261928f..45e11f3d 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -19,7 +19,12 @@ public void DeserializesFlaggedValues() using var stream = TestDataHelper.OpenResource("TextKV3.flagged_value.kv3"); var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); - Assert.That((string)data["foo"], Is.EqualTo("bar")); + Assert.Multiple(() => + { + Assert.That((string)data["foo"], Is.EqualTo("bar")); + Assert.That((string)data["bar"], Is.EqualTo("foo")); + Assert.That((long)data["flaggedNumber"], Is.EqualTo(-1234)); + }); } [Test] diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 45ecbc32..c771a60d 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -2,6 +2,7 @@ using System.Globalization; using System.IO; using ValveKeyValue.Abstraction; +using ValveKeyValue.KeyValues3; namespace ValveKeyValue.Deserialization.KeyValues3 { @@ -142,7 +143,11 @@ void ReadIdentifier(string text) case KV3TextReaderState.InObjectBeforeValue: if (text.EndsWith(":", StringComparison.Ordinal) || text.EndsWith("+", StringComparison.Ordinal)) { - // TODO: Parse flag like resource: then read as string + var flag = ParseFlag(text[..^1]); + + + // Keep the InObjectBeforeValue state + break; } KVValue value = ParseValue(text); @@ -321,5 +326,18 @@ static byte[] ParseHexStringAsByteArray(string hexadecimalRepresentation) return data; } + + static KVFlag ParseFlag(string flag) + { + return flag switch + { + "resource" => KVFlag.Resource, + "resource_name" => KVFlag.ResourceName, + "panorama" => KVFlag.Panorama, + "soundevent" => KVFlag.SoundEvent, + "subclass" => KVFlag.SubClass, + _ => throw new InvalidDataException($"Unknown flag '{flag}'"), + }; + } } } diff --git a/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlag.cs b/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlag.cs new file mode 100644 index 00000000..bcfc0258 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlag.cs @@ -0,0 +1,13 @@ +namespace ValveKeyValue.KeyValues3 +{ + public enum KVFlag + { + None = 0, + Resource = 1, + ResourceName = 2, + // TODO: We don't know what 4 was + Panorama = 8, + SoundEvent = 16, + SubClass = 32, + } +} diff --git a/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlaggedObjectValue.cs b/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlaggedObjectValue.cs new file mode 100644 index 00000000..72052899 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlaggedObjectValue.cs @@ -0,0 +1,16 @@ +using System; + +namespace ValveKeyValue.KeyValues3 +{ + class KVFlaggedObjectValue : KVObjectValue + where TObject : IConvertible + { + public KVFlag Flag { get; private set; } + + public KVFlaggedObjectValue(TObject value, KVFlag flag, KVValueType valueType) + : base(value, valueType) + { + Flag = flag; + } + } +} From 08b612382c0099dbf0fa364ec638c3b361fdb5a5 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Mon, 8 Aug 2022 11:59:32 +0300 Subject: [PATCH 16/49] Use Valve's delimeters --- .../KeyValues3/KV3TextReader.cs | 3 +- .../KeyValues3/KV3TokenReader.cs | 54 ++++++++----------- ValveKeyValue/ValveKeyValue/KVTokenType.cs | 7 --- 3 files changed, 23 insertions(+), 41 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index c771a60d..194fcdef 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -143,6 +143,7 @@ void ReadIdentifier(string text) case KV3TextReaderState.InObjectBeforeValue: if (text.EndsWith(":", StringComparison.Ordinal) || text.EndsWith("+", StringComparison.Ordinal)) { + // TODO: There can be multiple flags on a single value, panorama+subclass:"thing" or something var flag = ParseFlag(text[..^1]); @@ -283,7 +284,7 @@ static KVValue ParseValue(string text) //value = new KVObjectValue(null, KVValueType.Null); return new KVObjectValue(string.Empty, KVValueType.Null); } - else if (char.IsDigit(text[0]) || text[0] == '-' || text[0] == '+') + else if (text.Length > 0 && (char.IsDigit(text[0]) || text[0] == '-' || text[0] == '+')) { // TODO: Due to Valve's string to int/double conversion functions, it is possible to have 0x hex values (as well as prefixed with minus like -0x) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 74574067..5788368e 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -1,5 +1,7 @@ using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using ValveKeyValue.KeyValues3; using Encoding = ValveKeyValue.KeyValues3.Encoding; @@ -23,9 +25,14 @@ public KV3TokenReader(TextReader textReader, KVSerializerOptions options) this.textReader = textReader; this.options = options; + + // TODO: Valve doesn't terminate on \r + var terminators = "{}[]=, \t\n\r'\":+;".ToCharArray(); + integerTerminators = new HashSet(terminators.Select(t => (int)t)); } readonly KVSerializerOptions options; + readonly HashSet integerTerminators; TextReader textReader; bool disposed; int? peekedNext; @@ -74,7 +81,7 @@ KVToken ReadStringOrIdentifier() return new KVToken(KVTokenType.String, ReadQuotedStringRaw()); } - return new KVToken(KVTokenType.Identifier, ReadUntilWhitespaceOrDelimeter(QuotationMark)); + return new KVToken(KVTokenType.Identifier, ReadToken()); } KVToken ReadAssignment() @@ -109,7 +116,7 @@ KVToken ReadObjectEnd() public KVToken ReadHeader() { - var str = ReadUntilWhitespaceOrDelimeter((char)0); + var str = ReadToken(); if (str != "") { @@ -301,7 +308,7 @@ void ReadChar(char expectedChar) } } - string ReadUntil(char delimeter) + string ReadToken() { var sb = new StringBuilder(); @@ -309,26 +316,7 @@ string ReadUntil(char delimeter) { var next = Peek(); - if (next == delimeter) - { - break; - } - - sb.Append(Next()); - } - - return sb.ToString(); - } - - // TODO: Read until delimeter: "{}[]=, \t\n'\":+;" - string ReadUntilWhitespaceOrDelimeter(char delimeter) - { - var sb = new StringBuilder(); - - while (true) - { - var next = Peek(); - if (next == -1 || char.IsWhiteSpace((char)next) || next == delimeter) + if (next == -1 || integerTerminators.Contains(next)) { break; } diff --git a/ValveKeyValue/ValveKeyValue/KVTokenType.cs b/ValveKeyValue/ValveKeyValue/KVTokenType.cs index 8a8c0738..7d9a4d6c 100644 --- a/ValveKeyValue/ValveKeyValue/KVTokenType.cs +++ b/ValveKeyValue/ValveKeyValue/KVTokenType.cs @@ -18,12 +18,5 @@ enum KVTokenType CommentBlock, ArrayStart, ArrayEnd, - - SEEK_VALUE, - PROP_NAME, - VALUE_STRUCT, - VALUE_STRING_MULTI, - VALUE_NUMBER, - VALUE_FLAGGED, } } From b2f4b4d231f8b06959d7e8d5dd974e82af7e165f Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 9 Aug 2022 12:42:39 +0300 Subject: [PATCH 17/49] flag 4 --- ValveKeyValue/ValveKeyValue/KeyValues3/KVFlag.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlag.cs b/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlag.cs index bcfc0258..52564f70 100644 --- a/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlag.cs +++ b/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlag.cs @@ -5,7 +5,7 @@ public enum KVFlag None = 0, Resource = 1, ResourceName = 2, - // TODO: We don't know what 4 was + MultilineString = 4, Panorama = 8, SoundEvent = 16, SubClass = 32, From 694489dddbd6999ff61e51d0d06af1d75c0884b2 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 9 Aug 2022 16:46:19 +0300 Subject: [PATCH 18/49] case insensitive --- .../ValveKeyValue.Test/TextKV3/HeadersTestCase.cs | 7 +------ .../Deserialization/KeyValues3/KV3TokenReader.cs | 12 ++++++------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs index d1fc47e5..7e2711ac 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs @@ -11,8 +11,6 @@ class HeadersTestCase [TestCase("")] [TestCase("")] [TestCase("\n{}")] - [TestCase("\n{}")] [TestCase("\n{}")] [TestCase("\n{}")] [TestCase("\n{}")] @@ -21,8 +19,6 @@ class HeadersTestCase [TestCase("\n{}")] [TestCase("\n{}")] [TestCase("\n{}")] - [TestCase("\n{}")] - [TestCase("\n{}")] [TestCase("\n{}")] [TestCase("\n{}")] - [TestCase("\n{}")] + [TestCase("\n{}")] [TestCase("\n{}")] [TestCase("\n{}")] [TestCase("\n{}")] diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 5788368e..6090da02 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -10,7 +10,7 @@ namespace ValveKeyValue.Deserialization.KeyValues3 { class KV3TokenReader : IDisposable { - const char QuotationMark = '"'; + const char QuotationMark = '"'; // TODO: Support single quotes 'abc' const char ObjectStart = '{'; const char ObjectEnd = '}'; const char ArrayStart = '['; @@ -126,7 +126,7 @@ public KVToken ReadHeader() SwallowWhitespace(); str = ReadToken(); - if (str != "kv3") + if (!str.Equals("kv3", StringComparison.OrdinalIgnoreCase)) { throw new InvalidDataException($"The header is incorrect, expected 'kv3' but got '{str}'."); } @@ -134,7 +134,7 @@ public KVToken ReadHeader() SwallowWhitespace(); str = ReadToken(); - if (str != "encoding") + if (!str.Equals("encoding", StringComparison.OrdinalIgnoreCase)) { throw new InvalidDataException($"The header is incorrect, expected 'encoding' but got '{str}'."); } @@ -145,7 +145,7 @@ public KVToken ReadHeader() str = ReadToken(); - if (str != "version") + if (!str.Equals("version", StringComparison.OrdinalIgnoreCase)) { throw new InvalidDataException($"The header is incorrect, expected 'version' but got '{str}'."); } @@ -158,7 +158,7 @@ public KVToken ReadHeader() str = ReadToken(); - if (str != "format") + if (!str.Equals("format", StringComparison.OrdinalIgnoreCase)) { throw new InvalidDataException($"The header is incorrect, expected 'format' but got '{str}'."); } @@ -169,7 +169,7 @@ public KVToken ReadHeader() str = ReadToken(); - if (str != "version") + if (!str.Equals("version", StringComparison.OrdinalIgnoreCase)) { throw new InvalidDataException($"The header is incorrect, expected 'version' but got '{str}'."); } From f61912ee3edf1415a936d6d18710c93d2ffc699d Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 9 Aug 2022 18:48:45 +0300 Subject: [PATCH 19/49] Read single quoted strings, read flags, correctly detect identifiers --- .../Test Data/TextKV3/flagged_value.kv3 | 4 +- .../Test Data/TextKV3/types.kv3 | 2 + .../TextKV3/BasicKV3TestCases.cs | 7 ++ .../KeyValues3/KV3TextReader.cs | 44 ++++--- .../KeyValues3/KV3TokenReader.cs | 108 ++++++++++++++---- ValveKeyValue/ValveKeyValue/KVTokenType.cs | 2 + 6 files changed, 127 insertions(+), 40 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 index 5307d0b6..c4e5c5e9 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 @@ -1,6 +1,8 @@ { foo = resource:"bar" - bar = resource+"foo" + bar = resource|"foo" + uppercase = RESOURCE:"foo" flaggedNumber = panorama:-1234 + multipleFlags = resource:resource_name|subclass:"cool value" } diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 index 758cde3e..1f13e6c9 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 @@ -16,4 +16,6 @@ doubleNegativeMaxValue = -62147483647.1337 doubleExponent = 1.23456E+2 intWithStringSuffix = 123foobar + singleQuotes = 'string' + singleQuotesWithQuotesInside = 'string is "pretty" cool' } diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index 45e11f3d..62161306 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -23,6 +23,7 @@ public void DeserializesFlaggedValues() { Assert.That((string)data["foo"], Is.EqualTo("bar")); Assert.That((string)data["bar"], Is.EqualTo("foo")); + Assert.That((string)data["multipleFlags"], Is.EqualTo("cool value")); Assert.That((long)data["flaggedNumber"], Is.EqualTo(-1234)); }); } @@ -123,6 +124,12 @@ public void DeserializesBasicTypes() // TODO: Should this throw instead because strings need to be quoted? Or should it parse until it hits a non number like 123? Assert.That(data["intWithStringSuffix"].ValueType, Is.EqualTo(KVValueType.String)); Assert.That((string)data["intWithStringSuffix"], Is.EqualTo("123foobar")); + + Assert.That(data["singleQuotes"].ValueType, Is.EqualTo(KVValueType.String)); + Assert.That((string)data["singleQuotes"], Is.EqualTo("string")); + + Assert.That(data["singleQuotesWithQuotesInside"].ValueType, Is.EqualTo(KVValueType.String)); + Assert.That((string)data["singleQuotesWithQuotesInside"], Is.EqualTo("string is \"pretty\" cool")); }); } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 194fcdef..e8281065 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -57,6 +57,14 @@ public void ReadObject() ReadAssignment(); break; + case KVTokenType.Comma: + ReadComma(); + break; + + case KVTokenType.Flag: + ReadFlag(token.Value); + break; + case KVTokenType.Identifier: ReadIdentifier(token.Value); break; @@ -121,6 +129,24 @@ void ReadAssignment() stateMachine.Push(KV3TextReaderState.InObjectBeforeValue); } + void ReadComma() + { + if (stateMachine.Current != KV3TextReaderState.InArray) + { + throw new InvalidOperationException($"Attempted to have a comma character while in state {stateMachine.Current}."); + } + } + + void ReadFlag(string text) + { + if (stateMachine.Current != KV3TextReaderState.InArray && stateMachine.Current != KV3TextReaderState.InObjectBeforeValue) + { + throw new InvalidOperationException($"Attempted to read flag while in state {stateMachine.Current}."); + } + + var flag = ParseFlag(text); + } + void ReadIdentifier(string text) { switch (stateMachine.Current) @@ -141,16 +167,6 @@ void ReadIdentifier(string text) break; case KV3TextReaderState.InObjectBeforeValue: - if (text.EndsWith(":", StringComparison.Ordinal) || text.EndsWith("+", StringComparison.Ordinal)) - { - // TODO: There can be multiple flags on a single value, panorama+subclass:"thing" or something - var flag = ParseFlag(text[..^1]); - - - // Keep the InObjectBeforeValue state - break; - } - KVValue value = ParseValue(text); if (value != null) @@ -170,17 +186,15 @@ void ReadIdentifier(string text) void ReadText(string text) { - KVValue value; + var value = ParseValue(text); switch (stateMachine.Current) { case KV3TextReaderState.InArray: - value = new KVObjectValue(text, KVValueType.String); listener.OnArrayValue(value); break; case KV3TextReaderState.InObjectBeforeValue: - value = new KVObjectValue(text, KVValueType.String); var name = stateMachine.CurrentName; listener.OnKeyValuePair(name, value); @@ -284,7 +298,7 @@ static KVValue ParseValue(string text) //value = new KVObjectValue(null, KVValueType.Null); return new KVObjectValue(string.Empty, KVValueType.Null); } - else if (text.Length > 0 && (char.IsDigit(text[0]) || text[0] == '-' || text[0] == '+')) + else if (text.Length > 0 && ((text[0] >= '0' && text[0] <= '9') || text[0] == '-' || text[0] == '+')) { // TODO: Due to Valve's string to int/double conversion functions, it is possible to have 0x hex values (as well as prefixed with minus like -0x) @@ -330,7 +344,7 @@ static byte[] ParseHexStringAsByteArray(string hexadecimalRepresentation) static KVFlag ParseFlag(string flag) { - return flag switch + return flag.ToLowerInvariant() switch { "resource" => KVFlag.Resource, "resource_name" => KVFlag.ResourceName, diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 6090da02..38a7e9fb 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -10,13 +10,13 @@ namespace ValveKeyValue.Deserialization.KeyValues3 { class KV3TokenReader : IDisposable { - const char QuotationMark = '"'; // TODO: Support single quotes 'abc' const char ObjectStart = '{'; const char ObjectEnd = '}'; const char ArrayStart = '['; const char ArrayEnd = ']'; const char CommentBegin = '/'; const char Assignment = '='; + const char Comma = ','; public KV3TokenReader(TextReader textReader, KVSerializerOptions options) { @@ -27,7 +27,8 @@ public KV3TokenReader(TextReader textReader, KVSerializerOptions options) this.options = options; // TODO: Valve doesn't terminate on \r - var terminators = "{}[]=, \t\n\r'\":+;".ToCharArray(); + // Dota 2 binary from 2017 used "+" as a terminate (for flagged values), but then they changed it to "|" + var terminators = "{}[]=, \t\n\r'\":|;".ToCharArray(); integerTerminators = new HashSet(terminators.Select(t => (int)t)); } @@ -56,7 +57,8 @@ public KVToken ReadNextToken() ArrayEnd => ReadArrayEnd(), CommentBegin => ReadComment(), Assignment => ReadAssignment(), - _ => ReadStringOrIdentifier(), // TODO: This should read identifiers, strings should only be read as values + Comma => ReadComma(), + _ => ReadStringOrIdentifier(), // TODO: #[] byte array }; } @@ -76,12 +78,23 @@ KVToken ReadStringOrIdentifier() { SwallowWhitespace(); - if (Peek() == QuotationMark) + var token = ReadToken(); + var type = KVTokenType.String; + + if (IsIdentifier(token)) { - return new KVToken(KVTokenType.String, ReadQuotedStringRaw()); + type = KVTokenType.Identifier; + + var next = Peek(); + + if (next == ':' || next == '|') + { + Next(); + type = KVTokenType.Flag; + } } - return new KVToken(KVTokenType.Identifier, ReadToken()); + return new KVToken(type, token); } KVToken ReadAssignment() @@ -90,6 +103,12 @@ KVToken ReadAssignment() return new KVToken(KVTokenType.Assignment); } + KVToken ReadComma() + { + ReadChar(Comma); + return new KVToken(KVTokenType.Comma); + } + KVToken ReadArrayStart() { ReadChar(ArrayStart); @@ -308,15 +327,54 @@ void ReadChar(char expectedChar) } } + bool IsIdentifier(string text) + { + for (var i = 0; i < text.Length; i++) + { + var c = text[i]; + + if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')) + { + continue; + } + + if (c >= '0' && c <= '9') + { + continue; + } + + if (c == '_' || c == ':' || c == '.') + { + continue; + } + + return false; + } + + return true; + } + string ReadToken() { + // TODO: while true + // swallow whitespace + // swallow comments + // swallow whitespace + + var next = Peek(); + + if (next == '"' || next == '\'') + { + return ReadQuotedStringRaw((char)next); + } + var sb = new StringBuilder(); while (true) { - var next = Peek(); + next = Peek(); - if (next == -1 || integerTerminators.Contains(next)) + if (next <= ' ' || integerTerminators.Contains(next)) { break; } @@ -341,9 +399,9 @@ bool PeekWhitespace() return !IsEndOfFile(next) && char.IsWhiteSpace((char)next); } - string ReadQuotedStringRaw() + string ReadQuotedStringRaw(char quotationMark) { - ReadChar(QuotationMark); + ReadChar(quotationMark); var isMultiline = false; @@ -351,12 +409,12 @@ string ReadQuotedStringRaw() // Is there another quote mark? // TODO: Peek() for more than one character - if (Peek() == QuotationMark) + if (quotationMark == '"' && Peek() == '"') { Next(); // If the next character is not another quote, it's an empty string - if (Peek() == QuotationMark) + if (Peek() == '"') { isMultiline = true; @@ -378,7 +436,9 @@ string ReadQuotedStringRaw() } } - while (Peek() != QuotationMark) + // TODO: Figure out '\' character + + while (Peek() != quotationMark) { var next = Next(); @@ -390,22 +450,22 @@ string ReadQuotedStringRaw() sb.Append(next); } - ReadChar(QuotationMark); + ReadChar(quotationMark); if (isMultiline) { - ReadChar(QuotationMark); - ReadChar(QuotationMark); - } + ReadChar('"'); + ReadChar('"'); - if (sb.Length > 0 && sb[^1] == '\n') - { - sb.Remove(sb.Length - 1, 1); - } + if (sb.Length > 0 && sb[^1] == '\n') + { + sb.Remove(sb.Length - 1, 1); + } - if (sb.Length > 0 && sb[^1] == '\r') - { - sb.Remove(sb.Length - 1, 1); + if (sb.Length > 0 && sb[^1] == '\r') + { + sb.Remove(sb.Length - 1, 1); + } } return sb.ToString(); diff --git a/ValveKeyValue/ValveKeyValue/KVTokenType.cs b/ValveKeyValue/ValveKeyValue/KVTokenType.cs index 7d9a4d6c..9ad10db3 100644 --- a/ValveKeyValue/ValveKeyValue/KVTokenType.cs +++ b/ValveKeyValue/ValveKeyValue/KVTokenType.cs @@ -14,7 +14,9 @@ enum KVTokenType // KeyValues3 Header, Identifier, + Flag, Assignment, + Comma, CommentBlock, ArrayStart, ArrayEnd, From 585b8a2ec25a3e66e18cceef97722caabcfd504a Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 9 Aug 2022 19:14:03 +0300 Subject: [PATCH 20/49] Support quoted keys --- .../ValveKeyValue.Test/Test Data/TextKV3/types.kv3 | 7 +++++++ .../ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs | 9 +++++++++ .../Deserialization/KeyValues3/KV3TextReader.cs | 10 ++++++++++ .../Deserialization/KeyValues3/KV3TokenReader.cs | 2 +- 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 index 1f13e6c9..cd42ff0a 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 @@ -18,4 +18,11 @@ intWithStringSuffix = 123foobar singleQuotes = 'string' singleQuotesWithQuotesInside = 'string is "pretty" cool' + key_with._various.separators = "test" + "quoted key with : {} terminators" = "test quoted key" + """ +this is a multi +line +key +""" = "multi line key parsed" } diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index 62161306..22fc432a 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -130,6 +130,15 @@ public void DeserializesBasicTypes() Assert.That(data["singleQuotesWithQuotesInside"].ValueType, Is.EqualTo(KVValueType.String)); Assert.That((string)data["singleQuotesWithQuotesInside"], Is.EqualTo("string is \"pretty\" cool")); + + Assert.That(data["key_with._various.separators"].ValueType, Is.EqualTo(KVValueType.String)); + Assert.That((string)data["key_with._various.separators"], Is.EqualTo("test")); + + Assert.That(data["quoted key with : {} terminators"].ValueType, Is.EqualTo(KVValueType.String)); + Assert.That((string)data["quoted key with : {} terminators"], Is.EqualTo("test quoted key")); + + Assert.That(data["this is a multi\nline\nkey"].ValueType, Is.EqualTo(KVValueType.String)); + Assert.That((string)data["this is a multi\nline\nkey"], Is.EqualTo("multi line key parsed")); }); } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index e8281065..5c3d4776 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -194,6 +194,16 @@ void ReadText(string text) listener.OnArrayValue(value); break; + case KV3TextReaderState.InObjectAfterValue: + FinalizeCurrentObject(@explicit: false); + stateMachine.PushObject(); + SetObjectKey(text); + break; + + case KV3TextReaderState.InObjectBeforeKey: + SetObjectKey(text); + break; + case KV3TextReaderState.InObjectBeforeValue: var name = stateMachine.CurrentName; listener.OnKeyValuePair(name, value); diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 38a7e9fb..21c9124d 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -343,7 +343,7 @@ bool IsIdentifier(string text) continue; } - if (c == '_' || c == ':' || c == '.') + if (c == '_' || c == '.') { continue; } From 7a819d2d7f6a3a85f1cc2904dc7ce974c3a80988 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 9 Aug 2022 19:16:04 +0300 Subject: [PATCH 21/49] still check for : --- .../KeyValues3/KV3TextReader.cs | 20 +++++++++++-------- .../KeyValues3/KV3TokenReader.cs | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 5c3d4776..83e9138a 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -186,13 +186,14 @@ void ReadIdentifier(string text) void ReadText(string text) { - var value = ParseValue(text); - switch (stateMachine.Current) { case KV3TextReaderState.InArray: - listener.OnArrayValue(value); - break; + { + var value = ParseValue(text); + listener.OnArrayValue(value); + break; + } case KV3TextReaderState.InObjectAfterValue: FinalizeCurrentObject(@explicit: false); @@ -205,11 +206,14 @@ void ReadText(string text) break; case KV3TextReaderState.InObjectBeforeValue: - var name = stateMachine.CurrentName; - listener.OnKeyValuePair(name, value); + { + var name = stateMachine.CurrentName; + var value = ParseValue(text); + listener.OnKeyValuePair(name, value); - stateMachine.Push(KV3TextReaderState.InObjectAfterValue); - break; + stateMachine.Push(KV3TextReaderState.InObjectAfterValue); + break; + } default: throw new InvalidOperationException($"Unhandled text reader state: {stateMachine.Current}."); diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 21c9124d..38a7e9fb 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -343,7 +343,7 @@ bool IsIdentifier(string text) continue; } - if (c == '_' || c == '.') + if (c == '_' || c == ':' || c == '.') { continue; } From 85ce390cbc811e81de09243326548243d2018968 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 9 Aug 2022 19:28:57 +0300 Subject: [PATCH 22/49] Remove InObjectBeforeValue state --- .../Deserialization/KeyValues3/KV3TextReader.cs | 10 ++++------ .../Deserialization/KeyValues3/KV3TextReaderState.cs | 1 - 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 83e9138a..5db894ed 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -125,8 +125,6 @@ void ReadAssignment() { throw new InvalidOperationException($"Attempted to assign while in state {stateMachine.Current}."); } - - stateMachine.Push(KV3TextReaderState.InObjectBeforeValue); } void ReadComma() @@ -139,7 +137,7 @@ void ReadComma() void ReadFlag(string text) { - if (stateMachine.Current != KV3TextReaderState.InArray && stateMachine.Current != KV3TextReaderState.InObjectBeforeValue) + if (stateMachine.Current != KV3TextReaderState.InArray && stateMachine.Current != KV3TextReaderState.InObjectAfterKey) { throw new InvalidOperationException($"Attempted to read flag while in state {stateMachine.Current}."); } @@ -166,7 +164,7 @@ void ReadIdentifier(string text) SetObjectKey(text); break; - case KV3TextReaderState.InObjectBeforeValue: + case KV3TextReaderState.InObjectAfterKey: KVValue value = ParseValue(text); if (value != null) @@ -205,7 +203,7 @@ void ReadText(string text) SetObjectKey(text); break; - case KV3TextReaderState.InObjectBeforeValue: + case KV3TextReaderState.InObjectAfterKey: { var name = stateMachine.CurrentName; var value = ParseValue(text); @@ -222,7 +220,7 @@ void ReadText(string text) void BeginNewArray() { - if (stateMachine.Current != KV3TextReaderState.InObjectBeforeValue) + if (stateMachine.Current != KV3TextReaderState.InObjectAfterKey) { throw new InvalidOperationException($"Attempted to begin new array while in state {stateMachine.Current}."); } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs index 8aa742b1..dae406f7 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs @@ -4,7 +4,6 @@ enum KV3TextReaderState { InObjectBeforeKey, InObjectAfterKey, - InObjectBeforeValue, InObjectAfterValue, InArray, } From bed2bb8ce4d64ec76ab8e465ece01276d4ac5dfc Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 9 Aug 2022 19:52:32 +0300 Subject: [PATCH 23/49] Write quoted keys when necessary; test objects --- .../Test Data/TextKV3/array.kv3 | 15 +++++ .../Test Data/TextKV3/comments.kv3 | 4 +- .../Test Data/TextKV3/object.kv3 | 9 +++ .../Test Data/TextKV3/types_serialized.kv3 | 5 ++ .../Test Data/TextKV3/zoo.kv3 | 30 ---------- .../TextKV3/BasicKV3TestCases.cs | 9 +++ .../ValveKeyValue.Test/TextKV3/ZooTest.cs | 16 ----- .../KeyValues3/KV3TextReader.cs | 2 +- .../KeyValues3/KV3TokenReader.cs | 11 +++- ValveKeyValue/ValveKeyValue/KVTokenType.cs | 1 + .../KeyValues3/KV3TextSerializer.cs | 59 ++++++++++++++++++- 11 files changed, 108 insertions(+), 53 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/object.kv3 delete mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/zoo.kv3 delete mode 100644 ValveKeyValue/ValveKeyValue.Test/TextKV3/ZooTest.cs diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 index 0ede6281..5ab87e27 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 @@ -5,4 +5,19 @@ "a", "b", ] + arrayOnSingleLine = [ 16.7551, 20.3763, 19.6448 ] + arrayNoSpace=[1.3763,19.6448] + arrayMixedTypes = + [ + "a", + 1, + true, + false, + { + foo = "bar" + }, + [ + 1, 3, 3, 7 + ] + ] } diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/comments.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/comments.kv3 index 3bee395a..1df43096 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/comments.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/comments.kv3 @@ -5,6 +5,6 @@ line comment */ foo = "bar" - one = /* comment in between */ "1" - two /* comment in between */ = "2" + one = /* comment in between */ "1" /* comment after */ + two /* comment in between */ = "2" /* comment after */ } diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/object.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/object.kv3 new file mode 100644 index 00000000..06d3a2e5 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/object.kv3 @@ -0,0 +1,9 @@ + +{ + a = { + foo = "bar" + b = { + c = "d" + } + } +} diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 index ceb24423..7bc790b5 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 @@ -16,6 +16,11 @@ doubleNegativeMaxValue = -62147483647.1337 doubleExponent = 123.456 intWithStringSuffix = "123foobar" + singleQuotes = "string" + singleQuotesWithQuotesInside = "string is \"pretty\" cool" + key_with._various.separators = "test" + "quoted key with : {} terminators" = "test quoted key" + "this is a multi\nline\nkey" = "multi line key parsed" multiLineString = """ hello world diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/zoo.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/zoo.kv3 deleted file mode 100644 index 277ae4d6..00000000 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/zoo.kv3 +++ /dev/null @@ -1,30 +0,0 @@ - -{ - boolValue = false - intValue = 128 - doubleValue = 64.000000 - negativeIntValue = -1337 - negativeDoubleValue = -0.1337 - stringValue = "hello world" - stringThatIsAResourceReference = resource:"particles/items3_fx/star_emblem.vpcf" - multiLineStringValue = """ -First line of a multi-line string literal. -Second line of a multi-line string literal. -""" - arrayValue = - [ - 1, - 2, - ] - objectValue = - { - n = 5 - s = "foo" - } - arrayOnSingleLine = [ 16.7551, 20.3763, 19.6448 ] - "quoted.key" = "hello" - // single line comment - /* multi - line - comment */ -} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index 22fc432a..3c4826cc 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -69,6 +69,15 @@ public void DeserializesArray() Assert.True(false); } + [Test] + public void DeserializesNestedObject() + { + using var stream = TestDataHelper.OpenResource("TextKV3.object.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.That((string)data["a"]["b"]["c"], Is.EqualTo("d")); + } + [Test] public void DeserializesBasicTypes() { diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/ZooTest.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/ZooTest.cs deleted file mode 100644 index c69bab03..00000000 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/ZooTest.cs +++ /dev/null @@ -1,16 +0,0 @@ -using NUnit.Framework; - -namespace ValveKeyValue.Test.TextKV3 -{ - class ZooTest - { - [Test] - public void Test() - { - using var stream = TestDataHelper.OpenResource("TextKV3.zoo.kv3"); - var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); - - Assert.Fail(); - } - } -} diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 5db894ed..e43a612d 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -252,7 +252,7 @@ void SetObjectKey(string name) void BeginNewObject() { - if (stateMachine.Current != KV3TextReaderState.InObjectAfterKey) + if (stateMachine.Current != KV3TextReaderState.InArray && stateMachine.Current != KV3TextReaderState.InObjectAfterKey) { throw new InvalidOperationException($"Attempted to begin new object while in state {stateMachine.Current}."); } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 38a7e9fb..5f23eb90 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -12,6 +12,7 @@ class KV3TokenReader : IDisposable { const char ObjectStart = '{'; const char ObjectEnd = '}'; + const char BinaryArrayMarker = '#'; const char ArrayStart = '['; const char ArrayEnd = ']'; const char CommentBegin = '/'; @@ -26,7 +27,6 @@ public KV3TokenReader(TextReader textReader, KVSerializerOptions options) this.textReader = textReader; this.options = options; - // TODO: Valve doesn't terminate on \r // Dota 2 binary from 2017 used "+" as a terminate (for flagged values), but then they changed it to "|" var terminators = "{}[]=, \t\n\r'\":|;".ToCharArray(); integerTerminators = new HashSet(terminators.Select(t => (int)t)); @@ -53,13 +53,13 @@ public KVToken ReadNextToken() { ObjectStart => ReadObjectStart(), ObjectEnd => ReadObjectEnd(), + BinaryArrayMarker when Peek() == ArrayStart => ReadBinaryArrayStart(), ArrayStart => ReadArrayStart(), ArrayEnd => ReadArrayEnd(), CommentBegin => ReadComment(), Assignment => ReadAssignment(), Comma => ReadComma(), _ => ReadStringOrIdentifier(), - // TODO: #[] byte array }; } @@ -109,6 +109,13 @@ KVToken ReadComma() return new KVToken(KVTokenType.Comma); } + KVToken ReadBinaryArrayStart() + { + ReadChar(BinaryArrayMarker); + ReadChar(ArrayStart); + return new KVToken(KVTokenType.BinaryArrayStart); + } + KVToken ReadArrayStart() { ReadChar(ArrayStart); diff --git a/ValveKeyValue/ValveKeyValue/KVTokenType.cs b/ValveKeyValue/ValveKeyValue/KVTokenType.cs index 9ad10db3..3b34e2af 100644 --- a/ValveKeyValue/ValveKeyValue/KVTokenType.cs +++ b/ValveKeyValue/ValveKeyValue/KVTokenType.cs @@ -20,5 +20,6 @@ enum KVTokenType CommentBlock, ArrayStart, ArrayEnd, + BinaryArrayStart, } } diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs index ab7941a5..ec96b67f 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs @@ -73,8 +73,7 @@ void WriteKeyValuePair(string name, KVValue value) { WriteIndentation(); - // TODO: We need to quote keys when they contain terminators such as a dot - writer.Write(name); + WriteKey(name); writer.Write(" = "); switch (value.ValueType) @@ -157,6 +156,62 @@ void WriteText(string text) } } + void WriteKey(string key) + { + var escaped = false; + var sb = new StringBuilder(key.Length + 2); + sb.Append('"'); + + foreach (var @char in key) + { + switch (@char) + { + case '\t': + escaped = true; + sb.Append('\\'); + sb.Append('t'); + break; + + case '\n': + escaped = true; + sb.Append('\\'); + sb.Append('n'); + break; + + case ' ': + escaped = true; + sb.Append(' '); + break; + + case '"': + escaped = true; + sb.Append('\\'); + sb.Append('"'); + break; + + case '\'': + escaped = true; + sb.Append('\\'); + sb.Append('\''); + break; + + default: + sb.Append(@char); + break; + } + } + + if (escaped) + { + sb.Append('"'); + writer.Write(sb.ToString()); + } + else + { + writer.Write(key); + } + } + void WriteLine() { writer.WriteLine(); From c7bbcf3e9a1ed277b63ca93aa79ef0c19cc149eb Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 9 Aug 2022 20:29:35 +0300 Subject: [PATCH 24/49] Support escaping """ --- .../Test Data/TextKV3/comments.kv3 | 4 +- .../Test Data/TextKV3/multiline.kv3 | 6 ++ .../TextKV3/BasicKV3TestCases.cs | 7 +- .../KeyValues3/KV3TokenReader.cs | 86 +++++++++++++------ .../KeyValues3/KV3TextSerializer.cs | 38 ++++---- 5 files changed, 94 insertions(+), 47 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/comments.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/comments.kv3 index 1df43096..9d773e33 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/comments.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/comments.kv3 @@ -4,7 +4,7 @@ /* multi line comment */ - foo = "bar" - one = /* comment in between */ "1" /* comment after */ + foo = "bar" // comment after + one = /* comment in between */ /* another comment in between */ "1" /* comment after */ two /* comment in between */ = "2" /* comment after */ } diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/multiline.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/multiline.kv3 index 73ba27d3..e6cd87b3 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/multiline.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/multiline.kv3 @@ -4,4 +4,10 @@ First line of a multi-line string literal. Second line of a multi-line string literal. """ + multiLineWithQuotesInside = """ +hmm this """is awkward +\""" yes +""" + singleQuotesButWithNewLineAnyway = "hello +valve" } diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index 3c4826cc..2beae1fa 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -34,7 +34,12 @@ public void DeserializesMultilineStrings() using var stream = TestDataHelper.OpenResource("TextKV3.multiline.kv3"); var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); - Assert.That((string)data["multiLineStringValue"], Is.EqualTo("First line of a multi-line string literal.\nSecond line of a multi-line string literal.")); + Assert.Multiple(() => + { + Assert.That((string)data["multiLineStringValue"], Is.EqualTo("First line of a multi-line string literal.\nSecond line of a multi-line string literal.")); + Assert.That((string)data["multiLineWithQuotesInside"], Is.EqualTo("hmm this \"\"\"is awkward\n\\\"\"\" yes")); + Assert.That((string)data["singleQuotesButWithNewLineAnyway"], Is.EqualTo("hello\nvalve")); + }); } [Test] diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 5f23eb90..39bc7e4a 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -363,11 +363,6 @@ bool IsIdentifier(string text) string ReadToken() { - // TODO: while true - // swallow whitespace - // swallow comments - // swallow whitespace - var next = Peek(); if (next == '"' || next == '\'') @@ -432,37 +427,69 @@ string ReadQuotedStringRaw(char quotationMark) Next(); } - if (Peek() == '\n') - { - Next(); - } + ReadChar('\n'); } else { return string.Empty; } } - - // TODO: Figure out '\' character - - while (Peek() != quotationMark) + if (isMultiline) { - var next = Next(); + var escapeNext = false; - if (!isMultiline && next == '\n') + // Scan until \n""" + while (true) { - throw new InvalidDataException("Found new line while parsing literal string."); - } + var next = Next(); - sb.Append(next); - } + if (next == '\\') + { + // TODO: Is valve keeping the \ character in the string? + escapeNext = true; + } - ReadChar(quotationMark); + if (!escapeNext && next == '\n') + { + sb.Append(next); - if (isMultiline) - { - ReadChar('"'); - ReadChar('"'); + next = Next(); + + // TODO: This is absolutely terrible + if (next == '"') + { + next = Next(); + + if (next == '"') + { + next = Next(); + + if (next == '"') + { + break; + } + else + { + sb.Append(next); + } + } + else + { + sb.Append(next); + } + } + else + { + sb.Append(next); + } + } + else + { + escapeNext = false; + + sb.Append(next); + } + } if (sb.Length > 0 && sb[^1] == '\n') { @@ -474,6 +501,17 @@ string ReadQuotedStringRaw(char quotationMark) sb.Remove(sb.Length - 1, 1); } } + else + { + // TODO: Figure out '\' character escapes, does Valve actually unescape anything? + while (Peek() != quotationMark) + { + var next = Next(); + sb.Append(next); + } + + ReadChar(quotationMark); + } return sb.ToString(); } diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs index ec96b67f..4e6ac108 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs @@ -122,36 +122,34 @@ void WriteText(string text) if (isMultiline) { writer.Write("\"\"\"\n"); + + text = text.Replace("\"\"\"", "\\\"\"\""); + + writer.Write(text); + writer.Write("\n\"\"\""); } else { writer.Write('"'); - } - foreach (var @char in text) - { - switch (@char) + foreach (var @char in text) { - case '"': - writer.Write("\\\""); - break; + switch (@char) + { + case '"': + writer.Write("\\\""); + break; - case '\\': - writer.Write("\\"); - break; + case '\\': + writer.Write("\\"); + break; - default: - writer.Write(@char); - break; + default: + writer.Write(@char); + break; + } } - } - if (isMultiline) - { - writer.Write("\n\"\"\""); - } - else - { writer.Write('"'); } } From b29f10c06fd27d763a9c462a1cf04ff5937c58df Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 9 Aug 2022 20:43:16 +0300 Subject: [PATCH 25/49] Parse binary blobs --- .../Test Data/TextKV3/array.kv3 | 6 +- .../Test Data/TextKV3/binary_blob.kv3 | 8 +++ .../TextKV3/BasicKV3TestCases.cs | 9 +++ .../KeyValues3/KV3TextReader.cs | 55 +++++++++++++------ .../KeyValues3/KV3TextReaderState.cs | 1 + .../KeyValues3/KV3TokenReader.cs | 10 ++-- ValveKeyValue/ValveKeyValue/KVTokenType.cs | 2 +- 7 files changed, 66 insertions(+), 25 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/binary_blob.kv3 diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 index 5ab87e27..4eb445de 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 @@ -18,6 +18,10 @@ }, [ 1, 3, 3, 7 - ] + ], + #[ + 11 FF + ], + resource:"hello.world" ] } diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/binary_blob.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/binary_blob.kv3 new file mode 100644 index 00000000..4323149a --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/binary_blob.kv3 @@ -0,0 +1,8 @@ + +{ + array = + #[ + 00 11 22 33 44 55 66 77 88 99 + AA BB CC DD FF + ] +} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index 2beae1fa..adf7d727 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -74,6 +74,15 @@ public void DeserializesArray() Assert.True(false); } + [Test] + public void DeserializesBinaryBlob() + { + using var stream = TestDataHelper.OpenResource("TextKV3.binary_blob.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.True(false); + } + [Test] public void DeserializesNestedObject() { diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index e43a612d..41bf03e8 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -73,6 +73,10 @@ public void ReadObject() ReadText(token.Value); break; + case KVTokenType.BinaryBlobStart: + BeginBinaryBlob(); + break; + case KVTokenType.ObjectStart: BeginNewObject(); break; @@ -149,6 +153,12 @@ void ReadIdentifier(string text) { switch (stateMachine.Current) { + case KV3TextReaderState.InBinaryBlob: + { + var value = ParseHexCharacter(text); + break; + } + case KV3TextReaderState.InArray: new KVObjectValue(text, KVValueType.String); break; @@ -165,17 +175,19 @@ void ReadIdentifier(string text) break; case KV3TextReaderState.InObjectAfterKey: - KVValue value = ParseValue(text); - - if (value != null) { - var name = stateMachine.CurrentName; - listener.OnKeyValuePair(name, value); + KVValue value = ParseValue(text); - stateMachine.Push(KV3TextReaderState.InObjectAfterValue); - } + if (value != null) + { + var name = stateMachine.CurrentName; + listener.OnKeyValuePair(name, value); - break; + stateMachine.Push(KV3TextReaderState.InObjectAfterValue); + } + + break; + } default: throw new InvalidOperationException($"Unhandled identifier reader state: {stateMachine.Current}."); @@ -218,9 +230,20 @@ void ReadText(string text) } } + void BeginBinaryBlob() + { + if (stateMachine.Current != KV3TextReaderState.InArray && stateMachine.Current != KV3TextReaderState.InObjectAfterKey) + { + throw new InvalidOperationException($"Attempted to begin new binary blob while in state {stateMachine.Current}."); + } + + stateMachine.PushObject(); + stateMachine.Push(KV3TextReaderState.InBinaryBlob); + } + void BeginNewArray() { - if (stateMachine.Current != KV3TextReaderState.InObjectAfterKey) + if (stateMachine.Current != KV3TextReaderState.InArray && stateMachine.Current != KV3TextReaderState.InObjectAfterKey) { throw new InvalidOperationException($"Attempted to begin new array while in state {stateMachine.Current}."); } @@ -231,7 +254,7 @@ void BeginNewArray() void FinalizeCurrentArray() { - if (stateMachine.Current != KV3TextReaderState.InArray) + if (stateMachine.Current != KV3TextReaderState.InArray && stateMachine.Current != KV3TextReaderState.InBinaryBlob) { throw new InvalidOperationException($"Attempted to finalize array while in state {stateMachine.Current}."); } @@ -340,18 +363,14 @@ static KVValue ParseValue(string text) return new KVObjectValue(text, KVValueType.String); } - static byte[] ParseHexStringAsByteArray(string hexadecimalRepresentation) + static byte ParseHexCharacter(string hexadecimalRepresentation) { - Require.NotNull(hexadecimalRepresentation, nameof(hexadecimalRepresentation)); - - var data = new byte[hexadecimalRepresentation.Length / 2]; - for (var i = 0; i < data.Length; i++) + if (hexadecimalRepresentation.Length != 2) { - var currentByteText = hexadecimalRepresentation.Substring(i * 2, 2); - data[i] = byte.Parse(currentByteText, NumberStyles.HexNumber, CultureInfo.InvariantCulture); + throw new InvalidDataException("Expected hex byte (eg. 00-FF)"); } - return data; + return byte.Parse(hexadecimalRepresentation, NumberStyles.HexNumber, CultureInfo.InvariantCulture); } static KVFlag ParseFlag(string flag) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs index dae406f7..0119b37f 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs @@ -6,5 +6,6 @@ enum KV3TextReaderState InObjectAfterKey, InObjectAfterValue, InArray, + InBinaryBlob, } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 39bc7e4a..0e82bf9f 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -12,7 +12,7 @@ class KV3TokenReader : IDisposable { const char ObjectStart = '{'; const char ObjectEnd = '}'; - const char BinaryArrayMarker = '#'; + const char BinaryBlobMarker = '#'; const char ArrayStart = '['; const char ArrayEnd = ']'; const char CommentBegin = '/'; @@ -53,7 +53,7 @@ public KVToken ReadNextToken() { ObjectStart => ReadObjectStart(), ObjectEnd => ReadObjectEnd(), - BinaryArrayMarker when Peek() == ArrayStart => ReadBinaryArrayStart(), + BinaryBlobMarker => ReadBinaryArrayStart(), ArrayStart => ReadArrayStart(), ArrayEnd => ReadArrayEnd(), CommentBegin => ReadComment(), @@ -111,9 +111,9 @@ KVToken ReadComma() KVToken ReadBinaryArrayStart() { - ReadChar(BinaryArrayMarker); - ReadChar(ArrayStart); - return new KVToken(KVTokenType.BinaryArrayStart); + ReadChar(BinaryBlobMarker); + ReadChar(ArrayStart); // TODO: Strictly speaking Valve allows bare # without [ to be read as literal value (but what would that be?) + return new KVToken(KVTokenType.BinaryBlobStart); } KVToken ReadArrayStart() diff --git a/ValveKeyValue/ValveKeyValue/KVTokenType.cs b/ValveKeyValue/ValveKeyValue/KVTokenType.cs index 3b34e2af..d28f6815 100644 --- a/ValveKeyValue/ValveKeyValue/KVTokenType.cs +++ b/ValveKeyValue/ValveKeyValue/KVTokenType.cs @@ -20,6 +20,6 @@ enum KVTokenType CommentBlock, ArrayStart, ArrayEnd, - BinaryArrayStart, + BinaryBlobStart, } } From d700bbd7449b17daf856a3342f7b0201bb01dee4 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 9 Aug 2022 21:54:47 +0300 Subject: [PATCH 26/49] Changes for array --- .../Deserialization/KVPartialState.cs | 2 + .../KeyValues3/KV3TextReader.cs | 81 ++++++------------- .../KeyValues3/KV3TextReaderState.cs | 1 - .../KeyValues3/KV3TextReaderStateMachine.cs | 8 +- .../KeyValues3/KV3TokenReader.cs | 26 +++++- ValveKeyValue/ValveKeyValue/KVTokenType.cs | 2 +- ValveKeyValue/ValveKeyValue/Utils.cs | 27 +++++++ 7 files changed, 79 insertions(+), 68 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue/Utils.cs diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KVPartialState.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KVPartialState.cs index 12e78e07..7dcb18f7 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KVPartialState.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KVPartialState.cs @@ -12,5 +12,7 @@ class KVPartialState public IList Children { get; } = new List(); public bool Discard { get; set; } + + public bool IsArray { get; set; } } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 41bf03e8..a70f7bb7 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -66,15 +66,15 @@ public void ReadObject() break; case KVTokenType.Identifier: - ReadIdentifier(token.Value); + ReadText(token.Value); break; case KVTokenType.String: ReadText(token.Value); break; - case KVTokenType.BinaryBlobStart: - BeginBinaryBlob(); + case KVTokenType.BinaryBlob: + ReadBinaryBlob(token.Value); break; case KVTokenType.ObjectStart: @@ -149,79 +149,56 @@ void ReadFlag(string text) var flag = ParseFlag(text); } - void ReadIdentifier(string text) + void ReadText(string text) { switch (stateMachine.Current) { - case KV3TextReaderState.InBinaryBlob: + case KV3TextReaderState.InArray: { - var value = ParseHexCharacter(text); + var value = ParseValue(text); + listener.OnArrayValue(value); break; } - case KV3TextReaderState.InArray: - new KVObjectValue(text, KVValueType.String); - break; - - // If we're after a value when we find more text, then we must be starting a new key/value pair. - case KV3TextReaderState.InObjectAfterValue: - FinalizeCurrentObject(@explicit: false); - stateMachine.PushObject(); - SetObjectKey(text); - break; - case KV3TextReaderState.InObjectBeforeKey: SetObjectKey(text); break; case KV3TextReaderState.InObjectAfterKey: { - KVValue value = ParseValue(text); - - if (value != null) - { - var name = stateMachine.CurrentName; - listener.OnKeyValuePair(name, value); - - stateMachine.Push(KV3TextReaderState.InObjectAfterValue); - } + var name = stateMachine.CurrentName; + var value = ParseValue(text); + listener.OnKeyValuePair(name, value); + stateMachine.Push(KV3TextReaderState.InObjectBeforeKey); break; } default: - throw new InvalidOperationException($"Unhandled identifier reader state: {stateMachine.Current}."); + throw new InvalidOperationException($"Unhandled text reader state: {stateMachine.Current}."); } } - void ReadText(string text) + void ReadBinaryBlob(string text) { + var bytes = Utils.ParseHexStringAsByteArray(text); + //var value = new KVObjectValue(bytes, KVValueType.BinaryBlob); + var value = new KVObjectValue(0x00, KVValueType.BinaryBlob); // TODO: wrong + switch (stateMachine.Current) { case KV3TextReaderState.InArray: { - var value = ParseValue(text); listener.OnArrayValue(value); break; } - case KV3TextReaderState.InObjectAfterValue: - FinalizeCurrentObject(@explicit: false); - stateMachine.PushObject(); - SetObjectKey(text); - break; - - case KV3TextReaderState.InObjectBeforeKey: - SetObjectKey(text); - break; - case KV3TextReaderState.InObjectAfterKey: { var name = stateMachine.CurrentName; - var value = ParseValue(text); listener.OnKeyValuePair(name, value); - stateMachine.Push(KV3TextReaderState.InObjectAfterValue); + stateMachine.Push(KV3TextReaderState.InObjectBeforeKey); break; } @@ -230,17 +207,6 @@ void ReadText(string text) } } - void BeginBinaryBlob() - { - if (stateMachine.Current != KV3TextReaderState.InArray && stateMachine.Current != KV3TextReaderState.InObjectAfterKey) - { - throw new InvalidOperationException($"Attempted to begin new binary blob while in state {stateMachine.Current}."); - } - - stateMachine.PushObject(); - stateMachine.Push(KV3TextReaderState.InBinaryBlob); - } - void BeginNewArray() { if (stateMachine.Current != KV3TextReaderState.InArray && stateMachine.Current != KV3TextReaderState.InObjectAfterKey) @@ -249,21 +215,22 @@ void BeginNewArray() } stateMachine.PushObject(); + stateMachine.SetArrayCurrent(); stateMachine.Push(KV3TextReaderState.InArray); } void FinalizeCurrentArray() { - if (stateMachine.Current != KV3TextReaderState.InArray && stateMachine.Current != KV3TextReaderState.InBinaryBlob) + if (stateMachine.Current != KV3TextReaderState.InArray) { throw new InvalidOperationException($"Attempted to finalize array while in state {stateMachine.Current}."); } stateMachine.PopObject(); - if (stateMachine.IsInObject) + if (stateMachine.IsInObject && !stateMachine.IsInArray) { - stateMachine.Push(KV3TextReaderState.InObjectAfterValue); + stateMachine.Push(KV3TextReaderState.InObjectBeforeKey); } } @@ -288,7 +255,7 @@ void BeginNewObject() void FinalizeCurrentObject(bool @explicit) { - if (stateMachine.Current != KV3TextReaderState.InObjectBeforeKey && stateMachine.Current != KV3TextReaderState.InObjectAfterValue) + if (stateMachine.Current != KV3TextReaderState.InObjectBeforeKey) { throw new InvalidOperationException($"Attempted to finalize object while in state {stateMachine.Current}."); } @@ -297,7 +264,7 @@ void FinalizeCurrentObject(bool @explicit) if (stateMachine.IsInObject) { - stateMachine.Push(KV3TextReaderState.InObjectAfterValue); + stateMachine.Push(KV3TextReaderState.InObjectBeforeKey); } if (@explicit) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs index 0119b37f..dae406f7 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs @@ -6,6 +6,5 @@ enum KV3TextReaderState InObjectAfterKey, InObjectAfterValue, InArray, - InBinaryBlob, } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs index 2d401bca..d27205ab 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs @@ -21,7 +21,7 @@ public KV3TextReaderStateMachine() public bool IsInObject => states.Count > 0; - public bool IsAtStart => states.Count == 1 && CurrentObject.States.Count == 1 && Current == KV3TextReaderState.InObjectBeforeKey; + public bool IsInArray => states.Count > 0 && CurrentObject.IsArray; public void PushObject() => states.Push(new KVPartialState()); @@ -34,13 +34,9 @@ public void PopObject() public string CurrentName => CurrentObject.Key; - public void Pop() => CurrentObject.States.Pop(); - public void SetName(string name) => CurrentObject.Key = name; - public void SetValue(KVValue value) => CurrentObject.Value = value; - - public void AddItem(KVObject item) => CurrentObject.Items.Add(item); + public void SetArrayCurrent() => CurrentObject.IsArray = true; KVPartialState CurrentObject => states.Peek(); } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 0e82bf9f..764aa032 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -53,7 +53,7 @@ public KVToken ReadNextToken() { ObjectStart => ReadObjectStart(), ObjectEnd => ReadObjectEnd(), - BinaryBlobMarker => ReadBinaryArrayStart(), + BinaryBlobMarker => ReadBinaryBlob(), ArrayStart => ReadArrayStart(), ArrayEnd => ReadArrayEnd(), CommentBegin => ReadComment(), @@ -109,11 +109,31 @@ KVToken ReadComma() return new KVToken(KVTokenType.Comma); } - KVToken ReadBinaryArrayStart() + KVToken ReadBinaryBlob() { ReadChar(BinaryBlobMarker); ReadChar(ArrayStart); // TODO: Strictly speaking Valve allows bare # without [ to be read as literal value (but what would that be?) - return new KVToken(KVTokenType.BinaryBlobStart); + + var sb = new StringBuilder(); + + while (true) + { + var next = Next(); + + if (char.IsWhiteSpace(next)) + { + continue; + } + + if (next == ArrayEnd) + { + break; + } + + sb.Append(next); + } + + return new KVToken(KVTokenType.BinaryBlob, sb.ToString()); } KVToken ReadArrayStart() diff --git a/ValveKeyValue/ValveKeyValue/KVTokenType.cs b/ValveKeyValue/ValveKeyValue/KVTokenType.cs index d28f6815..83e68f74 100644 --- a/ValveKeyValue/ValveKeyValue/KVTokenType.cs +++ b/ValveKeyValue/ValveKeyValue/KVTokenType.cs @@ -20,6 +20,6 @@ enum KVTokenType CommentBlock, ArrayStart, ArrayEnd, - BinaryBlobStart, + BinaryBlob, } } diff --git a/ValveKeyValue/ValveKeyValue/Utils.cs b/ValveKeyValue/ValveKeyValue/Utils.cs new file mode 100644 index 00000000..e5494e33 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue/Utils.cs @@ -0,0 +1,27 @@ +using System; +using System.Globalization; + +namespace ValveKeyValue +{ + internal class Utils + { + public static byte[] ParseHexStringAsByteArray(string hexadecimalRepresentation) + { + Require.NotNull(hexadecimalRepresentation, nameof(hexadecimalRepresentation)); + + var data = new byte[hexadecimalRepresentation.Length / 2]; + for (var i = 0; i < data.Length; i++) + { + var currentByteText = hexadecimalRepresentation.Substring(i * 2, 2); + data[i] = byte.Parse(currentByteText, NumberStyles.HexNumber, CultureInfo.InvariantCulture); + } + + if (BitConverter.IsLittleEndian) + { + Array.Reverse(data); + } + + return data; + } + } +} From df76133473ab65c8c8e33e2105f51c9ebd86b181 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Wed, 10 Aug 2022 11:37:19 +0300 Subject: [PATCH 27/49] Add OnArrayStart/OnArrayEnd --- .../Abstraction/IVisitationListener.cs | 4 ++ .../Deserialization/KVObjectBuilder.cs | 56 +++++++++++++++++++ .../KeyValues3/KV3TextReader.cs | 4 ++ ValveKeyValue/ValveKeyValue/KVObject.cs | 22 ++++++++ .../KeyValues1/KV1BinarySerializer.cs | 2 + .../KeyValues1/KV1TextSerializer.cs | 2 + .../KeyValues3/KV3TextSerializer.cs | 2 + 7 files changed, 92 insertions(+) diff --git a/ValveKeyValue/ValveKeyValue/Abstraction/IVisitationListener.cs b/ValveKeyValue/ValveKeyValue/Abstraction/IVisitationListener.cs index b1f7c181..f97a40f9 100644 --- a/ValveKeyValue/ValveKeyValue/Abstraction/IVisitationListener.cs +++ b/ValveKeyValue/ValveKeyValue/Abstraction/IVisitationListener.cs @@ -8,6 +8,10 @@ interface IVisitationListener : IDisposable void OnKeyValuePair(string name, KVValue value); + void OnArrayStart(string name); + void OnArrayValue(KVValue value); + + void OnArrayEnd(); } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs index 5c86798b..c4f98ba2 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs @@ -76,6 +76,21 @@ public void OnObjectEnd() parentState.Items.Add(completedObject); } + public void OnArrayEnd() + { + if (StateStack.Count <= 1) + { + return; + } + + var state = StateStack.Pop(); + + var completedObject = MakeArray(state); + + var parentState = StateStack.Peek(); + parentState.Items.Add(completedObject); + } + public void DiscardCurrentObject() { var state = StateStack.Peek(); @@ -98,6 +113,16 @@ public void OnObjectStart(string name) StateStack.Push(state); } + public void OnArrayStart(string name) + { + var state = new KVPartialState + { + Key = name, + IsArray = true, + }; + StateStack.Push(state); + } + public IParsingVisitationListener GetMergeListener() { var builder = new KVMergingObjectBuilder(this); @@ -133,6 +158,11 @@ KVObject MakeObject(KVPartialState state) return null; } + if (state.IsArray) + { + throw new InvalidCastException("Tried to make an object ouf of an array."); + } + KVObject @object; if (state.Value != null) @@ -146,5 +176,31 @@ KVObject MakeObject(KVPartialState state) return @object; } + + KVObject MakeArray(KVPartialState state) + { + if (state.Discard) + { + return null; + } + + if (!state.IsArray) + { + throw new InvalidCastException("Tried to make an array out of an object."); + } + + KVObject @object; + + if (state.Value != null) + { + @object = new KVObject(state.Key, state.Value); + } + else + { + @object = new KVObject(state.Key, state.Children); + } + + return @object; + } } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index a70f7bb7..7aa3625b 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -214,6 +214,8 @@ void BeginNewArray() throw new InvalidOperationException($"Attempted to begin new array while in state {stateMachine.Current}."); } + listener.OnArrayStart(stateMachine.CurrentName); + stateMachine.PushObject(); stateMachine.SetArrayCurrent(); stateMachine.Push(KV3TextReaderState.InArray); @@ -232,6 +234,8 @@ void FinalizeCurrentArray() { stateMachine.Push(KV3TextReaderState.InObjectBeforeKey); } + + listener.OnArrayEnd(); } void SetObjectKey(string name) diff --git a/ValveKeyValue/ValveKeyValue/KVObject.cs b/ValveKeyValue/ValveKeyValue/KVObject.cs index 858dc6f8..da034219 100644 --- a/ValveKeyValue/ValveKeyValue/KVObject.cs +++ b/ValveKeyValue/ValveKeyValue/KVObject.cs @@ -41,6 +41,23 @@ public KVObject(string name, IEnumerable items) Value = value; } + /// + /// Initializes a new instance of the class. + /// + /// Name of this object. + /// Child items of this object. + public KVObject(string name, IEnumerable items) + { + Require.NotNull(name, nameof(name)); + Require.NotNull(items, nameof(items)); + + Name = name; + var value = new KVArrayValue(); + value.AddRange(items); + + Value = value; + } + /// /// Gets the name of this object. /// @@ -90,6 +107,11 @@ public void Add(KVObject value) /// public IEnumerable Children => (Value as KVCollectionValue) ?? Enumerable.Empty(); + /// + /// Gets the children of this . + /// + public IEnumerable ChildrenValues => (Value as KVArrayValue) ?? Enumerable.Empty(); + KVCollectionValue GetCollectionValue() { if (Value is not KVCollectionValue collection) diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs index 000b2c40..715f3ecf 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs @@ -75,7 +75,9 @@ public void OnKeyValuePair(string name, KVValue value) } } + public void OnArrayStart(string name) => throw new NotImplementedException(); public void OnArrayValue(KVValue value) => throw new NotImplementedException(); + public void OnArrayEnd() => throw new NotImplementedException(); void Write(KV1BinaryNodeType nodeType) { diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs index 9155a44a..c867f95d 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs @@ -36,7 +36,9 @@ public void OnObjectEnd() public void OnKeyValuePair(string name, KVValue value) => WriteKeyValuePair(name, value); + public void OnArrayStart(string name) => throw new NotImplementedException(); public void OnArrayValue(KVValue value) => throw new NotImplementedException(); + public void OnArrayEnd() => throw new NotImplementedException(); public void DiscardCurrentObject() { diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs index 4e6ac108..97895d4f 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs @@ -37,7 +37,9 @@ public void OnObjectEnd() public void OnKeyValuePair(string name, KVValue value) => WriteKeyValuePair(name, value); + public void OnArrayStart(string name) => throw new NotImplementedException(); public void OnArrayValue(KVValue value) => throw new NotImplementedException(); + public void OnArrayEnd() => throw new NotImplementedException(); public void DiscardCurrentObject() { From 97c5cbdc3b44bf00c5c04cd350273505bf152dc4 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Wed, 10 Aug 2022 11:51:24 +0300 Subject: [PATCH 28/49] Fix nested objects/arrays in arrays --- .../Deserialization/KVObjectBuilder.cs | 20 +++++++++++++++++-- .../KeyValues3/KV3TextReader.cs | 2 +- ValveKeyValue/ValveKeyValue/KVObject.cs | 6 +++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs index c4f98ba2..1eb659ea 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs @@ -73,7 +73,15 @@ public void OnObjectEnd() var completedObject = MakeObject(state); var parentState = StateStack.Peek(); - parentState.Items.Add(completedObject); + + if (parentState.IsArray) + { + parentState.Children.Add(completedObject.Value); // TODO: Avoid wrapping it into KVObject in the first place? + } + else + { + parentState.Items.Add(completedObject); + } } public void OnArrayEnd() @@ -88,7 +96,15 @@ public void OnArrayEnd() var completedObject = MakeArray(state); var parentState = StateStack.Peek(); - parentState.Items.Add(completedObject); + + if (parentState.IsArray) + { + parentState.Children.Add(completedObject.Value); // TODO: Avoid wrapping it into KVObject in the first place? + } + else + { + parentState.Items.Add(completedObject); + } } public void DiscardCurrentObject() diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 7aa3625b..0873cb77 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -266,7 +266,7 @@ void FinalizeCurrentObject(bool @explicit) stateMachine.PopObject(); - if (stateMachine.IsInObject) + if (stateMachine.IsInObject && !stateMachine.IsInArray) { stateMachine.Push(KV3TextReaderState.InObjectBeforeKey); } diff --git a/ValveKeyValue/ValveKeyValue/KVObject.cs b/ValveKeyValue/ValveKeyValue/KVObject.cs index da034219..7dfbb684 100644 --- a/ValveKeyValue/ValveKeyValue/KVObject.cs +++ b/ValveKeyValue/ValveKeyValue/KVObject.cs @@ -17,7 +17,7 @@ public partial class KVObject /// Value of this object. public KVObject(string name, KVValue value) { - Require.NotNull(name, nameof(name)); + //Require.NotNull(name, nameof(name)); // Objects in an array will not have a name Require.NotNull(value, nameof(value)); Name = name; @@ -31,7 +31,7 @@ public KVObject(string name, KVValue value) /// Child items of this object. public KVObject(string name, IEnumerable items) { - Require.NotNull(name, nameof(name)); + //Require.NotNull(name, nameof(name)); // Objects in an array will not have a name Require.NotNull(items, nameof(items)); Name = name; @@ -48,7 +48,7 @@ public KVObject(string name, IEnumerable items) /// Child items of this object. public KVObject(string name, IEnumerable items) { - Require.NotNull(name, nameof(name)); + //Require.NotNull(name, nameof(name)); // Objects in an array will not have a name Require.NotNull(items, nameof(items)); Name = name; From ae3e24c0861fc5342a75cc2977ba01309777a5a6 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Wed, 10 Aug 2022 21:19:09 +0300 Subject: [PATCH 29/49] Some preparation for kv flags --- .../Test Data/TextKV3/flagged_value.kv3 | 11 ++++++++++ .../TextKV3/BasicKV3TestCases.cs | 21 +++++++++++++++++++ .../Deserialization/KVPartialState.cs | 2 ++ .../KeyValues3/KV3TextReader.cs | 6 +++++- .../KeyValues3/KV3TextReaderStateMachine.cs | 11 ++++++++++ .../ValveKeyValue/{KeyValues3 => }/KVFlag.cs | 5 ++++- ValveKeyValue/ValveKeyValue/KVValue.cs | 5 +++++ .../KeyValues3/KVFlaggedObjectValue.cs | 16 -------------- 8 files changed, 59 insertions(+), 18 deletions(-) rename ValveKeyValue/ValveKeyValue/{KeyValues3 => }/KVFlag.cs (79%) delete mode 100644 ValveKeyValue/ValveKeyValue/KeyValues3/KVFlaggedObjectValue.cs diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 index c4e5c5e9..7c625688 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value.kv3 @@ -5,4 +5,15 @@ uppercase = RESOURCE:"foo" flaggedNumber = panorama:-1234 multipleFlags = resource:resource_name|subclass:"cool value" + soundEvent = soundEvent:"event sound" + noFlags = 5 + + flaggedObject = panorama:{ + 1 = soundEvent:"test1" + 2 = "test2" + 3 = subclass:[ + "test3" + ] + 4 = resource_name:"test4" + } } diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index adf7d727..4beba826 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -21,10 +21,31 @@ public void DeserializesFlaggedValues() Assert.Multiple(() => { + Assert.That(data["foo"].Flag, Is.EqualTo(KVFlag.Resource)); Assert.That((string)data["foo"], Is.EqualTo("bar")); + + Assert.That(data["bar"].Flag, Is.EqualTo(KVFlag.Resource)); Assert.That((string)data["bar"], Is.EqualTo("foo")); + + Assert.That(data["multipleFlags"].Flag, Is.EqualTo(KVFlag.Resource | KVFlag.ResourceName | KVFlag.SubClass)); Assert.That((string)data["multipleFlags"], Is.EqualTo("cool value")); + + Assert.That(data["flaggedNumber"].Flag, Is.EqualTo(KVFlag.Panorama)); Assert.That((long)data["flaggedNumber"], Is.EqualTo(-1234)); + + Assert.That(data["soundEvent"].Flag, Is.EqualTo(KVFlag.SoundEvent)); + Assert.That((string)data["soundEvent"], Is.EqualTo("event sound")); + + Assert.That(data["noFlags"].Flag, Is.EqualTo(KVFlag.None)); + Assert.That((long)data["noFlags"], Is.EqualTo(5)); + + /* TODO + Assert.That(data["flaggedObject"].Flag, Is.EqualTo(KVFlag.Panorama)); + Assert.That(data["flaggedObject"]["1"].Flag, Is.EqualTo(KVFlag.SoundEvent)); + Assert.That(data["flaggedObject"]["2"].Flag, Is.EqualTo(KVFlag.None)); + Assert.That(data["flaggedObject"]["3"].Flag, Is.EqualTo(KVFlag.SubClass)); + Assert.That(data["flaggedObject"]["4"].Flag, Is.EqualTo(KVFlag.ResourceName)); + */ }); } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KVPartialState.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KVPartialState.cs index 7dcb18f7..130ab05a 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KVPartialState.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KVPartialState.cs @@ -4,6 +4,8 @@ class KVPartialState { public string Key { get; set; } + public KVFlag Flag { get; set; } + public KVValue Value { get; set; } public IList Items { get; } = new List(); diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 0873cb77..c0468b9f 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -2,7 +2,6 @@ using System.Globalization; using System.IO; using ValveKeyValue.Abstraction; -using ValveKeyValue.KeyValues3; namespace ValveKeyValue.Deserialization.KeyValues3 { @@ -147,6 +146,8 @@ void ReadFlag(string text) } var flag = ParseFlag(text); + + stateMachine.SetFlag(flag); } void ReadText(string text) @@ -156,6 +157,7 @@ void ReadText(string text) case KV3TextReaderState.InArray: { var value = ParseValue(text); + value.Flag = stateMachine.GetAndResetFlag(); listener.OnArrayValue(value); break; } @@ -168,6 +170,7 @@ void ReadText(string text) { var name = stateMachine.CurrentName; var value = ParseValue(text); + value.Flag = stateMachine.GetAndResetFlag(); listener.OnKeyValuePair(name, value); stateMachine.Push(KV3TextReaderState.InObjectBeforeKey); @@ -184,6 +187,7 @@ void ReadBinaryBlob(string text) var bytes = Utils.ParseHexStringAsByteArray(text); //var value = new KVObjectValue(bytes, KVValueType.BinaryBlob); var value = new KVObjectValue(0x00, KVValueType.BinaryBlob); // TODO: wrong + value.Flag = stateMachine.GetAndResetFlag(); switch (stateMachine.Current) { diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs index d27205ab..66bb1a6e 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs @@ -36,6 +36,17 @@ public void PopObject() public void SetName(string name) => CurrentObject.Key = name; + public void SetFlag(KVFlag flag) => CurrentObject.Flag |= flag; + + public KVFlag GetAndResetFlag() + { + var flag = CurrentObject.Flag; + + CurrentObject.Flag = KVFlag.None; + + return flag; + } + public void SetArrayCurrent() => CurrentObject.IsArray = true; KVPartialState CurrentObject => states.Peek(); diff --git a/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlag.cs b/ValveKeyValue/ValveKeyValue/KVFlag.cs similarity index 79% rename from ValveKeyValue/ValveKeyValue/KeyValues3/KVFlag.cs rename to ValveKeyValue/ValveKeyValue/KVFlag.cs index 52564f70..22e04573 100644 --- a/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlag.cs +++ b/ValveKeyValue/ValveKeyValue/KVFlag.cs @@ -1,5 +1,8 @@ -namespace ValveKeyValue.KeyValues3 +using System; + +namespace ValveKeyValue { + [Flags] public enum KVFlag { None = 0, diff --git a/ValveKeyValue/ValveKeyValue/KVValue.cs b/ValveKeyValue/ValveKeyValue/KVValue.cs index ba0a5dc0..18dc7fa9 100644 --- a/ValveKeyValue/ValveKeyValue/KVValue.cs +++ b/ValveKeyValue/ValveKeyValue/KVValue.cs @@ -10,6 +10,11 @@ public abstract partial class KVValue : IConvertible /// public abstract KVValueType ValueType { get; } + /// + /// Gets or sets the current flags of this . + /// + public KVFlag Flag { get; set; } + /// /// Gets the child with the given key. /// diff --git a/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlaggedObjectValue.cs b/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlaggedObjectValue.cs deleted file mode 100644 index 72052899..00000000 --- a/ValveKeyValue/ValveKeyValue/KeyValues3/KVFlaggedObjectValue.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System; - -namespace ValveKeyValue.KeyValues3 -{ - class KVFlaggedObjectValue : KVObjectValue - where TObject : IConvertible - { - public KVFlag Flag { get; private set; } - - public KVFlaggedObjectValue(TObject value, KVFlag flag, KVValueType valueType) - : base(value, valueType) - { - Flag = flag; - } - } -} From c0381cc01d9f8e5f88ddaedb7f3ccfd86dd12c50 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Thu, 11 Aug 2022 11:14:10 +0300 Subject: [PATCH 30/49] Implement flags on objects/arrays --- .../ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs | 2 -- .../ValveKeyValue/Abstraction/IVisitationListener.cs | 4 ++-- .../ValveKeyValue/Abstraction/KVObjectVisitor.cs | 2 +- .../ValveKeyValue/Deserialization/KVObjectBuilder.cs | 12 +++++++++--- .../Deserialization/KeyValues1/KV1BinaryReader.cs | 2 +- .../Deserialization/KeyValues1/KV1TextReader.cs | 2 +- .../Deserialization/KeyValues3/KV3TextReader.cs | 5 +++-- .../Serialization/KeyValues1/KV1BinarySerializer.cs | 4 ++-- .../Serialization/KeyValues1/KV1TextSerializer.cs | 4 ++-- .../Serialization/KeyValues3/KV3TextSerializer.cs | 4 ++-- 10 files changed, 23 insertions(+), 18 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index 4beba826..2c097517 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -39,13 +39,11 @@ public void DeserializesFlaggedValues() Assert.That(data["noFlags"].Flag, Is.EqualTo(KVFlag.None)); Assert.That((long)data["noFlags"], Is.EqualTo(5)); - /* TODO Assert.That(data["flaggedObject"].Flag, Is.EqualTo(KVFlag.Panorama)); Assert.That(data["flaggedObject"]["1"].Flag, Is.EqualTo(KVFlag.SoundEvent)); Assert.That(data["flaggedObject"]["2"].Flag, Is.EqualTo(KVFlag.None)); Assert.That(data["flaggedObject"]["3"].Flag, Is.EqualTo(KVFlag.SubClass)); Assert.That(data["flaggedObject"]["4"].Flag, Is.EqualTo(KVFlag.ResourceName)); - */ }); } diff --git a/ValveKeyValue/ValveKeyValue/Abstraction/IVisitationListener.cs b/ValveKeyValue/ValveKeyValue/Abstraction/IVisitationListener.cs index f97a40f9..8625d1bb 100644 --- a/ValveKeyValue/ValveKeyValue/Abstraction/IVisitationListener.cs +++ b/ValveKeyValue/ValveKeyValue/Abstraction/IVisitationListener.cs @@ -2,13 +2,13 @@ namespace ValveKeyValue.Abstraction { interface IVisitationListener : IDisposable { - void OnObjectStart(string name); + void OnObjectStart(string name, KVFlag flag); void OnObjectEnd(); void OnKeyValuePair(string name, KVValue value); - void OnArrayStart(string name); + void OnArrayStart(string name, KVFlag flag); void OnArrayValue(KVValue value); diff --git a/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs b/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs index 85fbc3c6..0749beea 100644 --- a/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs +++ b/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs @@ -21,7 +21,7 @@ void VisitObject(string name, KVValue value) switch (value.ValueType) { case KVValueType.Collection: - listener.OnObjectStart(name); + listener.OnObjectStart(name, value.Flag); VisitValue((IEnumerable)value); listener.OnObjectEnd(); break; diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs index 1eb659ea..49b15553 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KVObjectBuilder.cs @@ -120,20 +120,22 @@ public void DiscardCurrentObject() } } - public void OnObjectStart(string name) + public void OnObjectStart(string name, KVFlag flag) { var state = new KVPartialState { - Key = name + Key = name, + Flag = flag, }; StateStack.Push(state); } - public void OnArrayStart(string name) + public void OnArrayStart(string name, KVFlag flag) { var state = new KVPartialState { Key = name, + Flag = flag, IsArray = true, }; StateStack.Push(state); @@ -190,6 +192,8 @@ KVObject MakeObject(KVPartialState state) @object = new KVObject(state.Key, state.Items); } + @object.Value.Flag = state.Flag; + return @object; } @@ -216,6 +220,8 @@ KVObject MakeArray(KVPartialState state) @object = new KVObject(state.Key, state.Children); } + @object.Value.Flag = state.Flag; + return @object; } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs index 7f367afb..6eb0b148 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs @@ -96,7 +96,7 @@ void ReadValue(KV1BinaryNodeType type) switch (type) { case KV1BinaryNodeType.ChildObject: - listener.OnObjectStart(name); + listener.OnObjectStart(name, KVFlag.None); ReadObjectCore(); listener.OnObjectEnd(); return; diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1TextReader.cs index 14399372..b6bdc807 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1TextReader.cs @@ -155,7 +155,7 @@ void BeginNewObject() throw new InvalidOperationException($"Attempted to begin new object while in state {stateMachine.Current} at {tokenReader.PreviousTokenPosition}."); } - listener.OnObjectStart(stateMachine.CurrentName); + listener.OnObjectStart(stateMachine.CurrentName, KVFlag.None); stateMachine.PushObject(); stateMachine.Push(KV1TextReaderState.InObjectBeforeKey); diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index c0468b9f..71c340f4 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -218,7 +218,7 @@ void BeginNewArray() throw new InvalidOperationException($"Attempted to begin new array while in state {stateMachine.Current}."); } - listener.OnArrayStart(stateMachine.CurrentName); + listener.OnArrayStart(stateMachine.CurrentName, stateMachine.GetAndResetFlag()); stateMachine.PushObject(); stateMachine.SetArrayCurrent(); @@ -244,6 +244,7 @@ void FinalizeCurrentArray() void SetObjectKey(string name) { + stateMachine.GetAndResetFlag(); stateMachine.SetName(name); stateMachine.Push(KV3TextReaderState.InObjectAfterKey); } @@ -255,7 +256,7 @@ void BeginNewObject() throw new InvalidOperationException($"Attempted to begin new object while in state {stateMachine.Current}."); } - listener.OnObjectStart(stateMachine.CurrentName); + listener.OnObjectStart(stateMachine.CurrentName, stateMachine.GetAndResetFlag()); stateMachine.PushObject(); stateMachine.Push(KV3TextReaderState.InObjectBeforeKey); diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs index 715f3ecf..46868dde 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs @@ -24,7 +24,7 @@ public void Dispose() writer.Dispose(); } - public void OnObjectStart(string name) + public void OnObjectStart(string name, KVFlag flag) { objectDepth++; Write(KV1BinaryNodeType.ChildObject); @@ -75,7 +75,7 @@ public void OnKeyValuePair(string name, KVValue value) } } - public void OnArrayStart(string name) => throw new NotImplementedException(); + public void OnArrayStart(string name, KVFlag flag) => throw new NotImplementedException(); public void OnArrayValue(KVValue value) => throw new NotImplementedException(); public void OnArrayEnd() => throw new NotImplementedException(); diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs index c867f95d..cb804fc0 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs @@ -27,7 +27,7 @@ public void Dispose() writer.Dispose(); } - public void OnObjectStart(string name) + public void OnObjectStart(string name, KVFlag flag) => WriteStartObject(name); public void OnObjectEnd() @@ -36,7 +36,7 @@ public void OnObjectEnd() public void OnKeyValuePair(string name, KVValue value) => WriteKeyValuePair(name, value); - public void OnArrayStart(string name) => throw new NotImplementedException(); + public void OnArrayStart(string name, KVFlag flag) => throw new NotImplementedException(); public void OnArrayValue(KVValue value) => throw new NotImplementedException(); public void OnArrayEnd() => throw new NotImplementedException(); diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs index 97895d4f..e9da2068 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs @@ -28,7 +28,7 @@ public void Dispose() writer.Dispose(); } - public void OnObjectStart(string name) + public void OnObjectStart(string name, KVFlag flag) => WriteStartObject(name); public void OnObjectEnd() @@ -37,7 +37,7 @@ public void OnObjectEnd() public void OnKeyValuePair(string name, KVValue value) => WriteKeyValuePair(name, value); - public void OnArrayStart(string name) => throw new NotImplementedException(); + public void OnArrayStart(string name, KVFlag flag) => throw new NotImplementedException(); public void OnArrayValue(KVValue value) => throw new NotImplementedException(); public void OnArrayEnd() => throw new NotImplementedException(); From a18b987fdbb1270a57e440bc4a6043b24da2ebe9 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Thu, 11 Aug 2022 11:41:43 +0300 Subject: [PATCH 31/49] Serialize arrays and flags --- .../Test Data/TextKV3/array.kv3 | 1 + .../TextKV3/flagged_value_serialized.kv3 | 19 +++ .../TextKV3/SerializationTestCase.cs | 48 +++++++ .../Abstraction/KVObjectVisitor.cs | 29 ++++- .../KeyValues3/KV3TextSerializer.cs | 117 ++++++++++++++++-- 5 files changed, 200 insertions(+), 14 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_serialized.kv3 diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 index 4eb445de..a89d5c82 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 @@ -13,6 +13,7 @@ 1, true, false, + null, { foo = "bar" }, diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_serialized.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_serialized.kv3 new file mode 100644 index 00000000..d8d99e46 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_serialized.kv3 @@ -0,0 +1,19 @@ + +{ + foo = resource:"bar" + bar = resource:"foo" + uppercase = resource:"foo" + flaggedNumber = panorama:-1234 + multipleFlags = resource|resource_name|subclass:"cool value" + soundEvent = soundevent:"event sound" + noFlags = 5 + flaggedObject = panorama:{ + 1 = soundevent:"test1" + 2 = "test2" + 3 = subclass:[ + "test3", + ] + 4 = resource_name:"test4" + } + test = "success" +} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs index 3c705b81..029e8c23 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs @@ -28,5 +28,53 @@ public void CreatesTextDocument() Assert.That(text, Is.EqualTo(expected)); } + + [Test] + public void SerializesArray() + { + using var stream = TestDataHelper.OpenResource("TextKV3.array.kv3"); + var expected = TestDataHelper.ReadTextResource("TextKV3.array.kv3"); + + var kv = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); + var data = kv.Deserialize(stream); + + data.Add(new KVObject("test", "success")); + + string text; + using (var ms = new MemoryStream()) + { + kv.Serialize(ms, data); + + ms.Seek(0, SeekOrigin.Begin); + using var reader = new StreamReader(ms); + text = reader.ReadToEnd(); + } + + Assert.That(text, Is.EqualTo(expected)); + } + + [Test] + public void SerializesFlags() + { + using var stream = TestDataHelper.OpenResource("TextKV3.flagged_value.kv3"); + var expected = TestDataHelper.ReadTextResource("TextKV3.flagged_value_serialized.kv3"); + + var kv = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); + var data = kv.Deserialize(stream); + + data.Add(new KVObject("test", "success")); + + string text; + using (var ms = new MemoryStream()) + { + kv.Serialize(ms, data); + + ms.Seek(0, SeekOrigin.Begin); + using var reader = new StreamReader(ms); + text = reader.ReadToEnd(); + } + + Assert.That(text, Is.EqualTo(expected)); + } } } diff --git a/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs b/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs index 0749beea..12d04a51 100644 --- a/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs +++ b/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs @@ -13,10 +13,10 @@ public KVObjectVisitor(IVisitationListener listener) public void Visit(KVObject @object) { - VisitObject(@object.Name, @object.Value); + VisitObject(@object.Name, @object.Value, false); } - void VisitObject(string name, KVValue value) + void VisitObject(string name, KVValue value, bool isArray) { switch (value.ValueType) { @@ -26,6 +26,16 @@ void VisitObject(string name, KVValue value) listener.OnObjectEnd(); break; + case KVValueType.BinaryBlob: + // TODO: write binary blobs + break; + + case KVValueType.Array: + listener.OnArrayStart(name, value.Flag); + VisitArray((IEnumerable)value); + listener.OnArrayEnd(); + break; + case KVValueType.FloatingPoint: case KVValueType.Int32: case KVValueType.Pointer: @@ -34,6 +44,11 @@ void VisitObject(string name, KVValue value) case KVValueType.Int64: case KVValueType.Boolean: case KVValueType.Null: + if (isArray) + { + listener.OnArrayValue(value); + break; + } listener.OnKeyValuePair(name, value); break; @@ -46,7 +61,15 @@ void VisitValue(IEnumerable collection) { foreach (var item in collection) { - VisitObject(item.Name, item.Value); + VisitObject(item.Name, item.Value, false); + } + } + + void VisitArray(IEnumerable collection) + { + foreach (var item in collection) + { + VisitObject(null, item, true); } } } diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs index e9da2068..3836e4b8 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs @@ -29,7 +29,7 @@ public void Dispose() } public void OnObjectStart(string name, KVFlag flag) - => WriteStartObject(name); + => WriteStartObject(name, flag); public void OnObjectEnd() => WriteEndObject(); @@ -37,27 +37,53 @@ public void OnObjectEnd() public void OnKeyValuePair(string name, KVValue value) => WriteKeyValuePair(name, value); - public void OnArrayStart(string name, KVFlag flag) => throw new NotImplementedException(); - public void OnArrayValue(KVValue value) => throw new NotImplementedException(); - public void OnArrayEnd() => throw new NotImplementedException(); + public void OnArrayStart(string name, KVFlag flag) + { + WriteIndentation(); + + WriteKey(name); + WriteFlag(flag); + + writer.Write('['); + indentation++; + WriteLine(); + } + + public void OnArrayValue(KVValue value) + { + WriteIndentation(); + + WriteValue(value); + + writer.Write(','); + writer.WriteLine(); // TODO: If short, no line? + } + + public void OnArrayEnd() + { + indentation--; + WriteIndentation(); + writer.Write(']'); + writer.WriteLine(); + } public void DiscardCurrentObject() { throw new NotSupportedException("Discard not supported when writing."); } - void WriteStartObject(string name) + void WriteStartObject(string name, KVFlag flag) { WriteIndentation(); // TODO: Dumb hack, we should not have a root name if (indentation != 0 && name != "root") { - WriteText(name); - WriteLine(); + WriteKey(name); } - WriteIndentation(); + WriteFlag(flag); + writer.Write('{'); indentation++; WriteLine(); @@ -76,7 +102,15 @@ void WriteKeyValuePair(string name, KVValue value) WriteIndentation(); WriteKey(name); - writer.Write(" = "); + + WriteValue(value); + + WriteLine(); + } + + void WriteValue(KVValue value) + { + WriteFlag(value.Flag); switch (value.ValueType) { @@ -102,8 +136,6 @@ void WriteKeyValuePair(string name, KVValue value) WriteText(value.ToString(null)); break; } - - WriteLine(); } void WriteIndentation() @@ -158,6 +190,11 @@ void WriteText(string text) void WriteKey(string key) { + if (key == null) + { + return; + } + var escaped = false; var sb = new StringBuilder(key.Length + 2); sb.Append('"'); @@ -210,11 +247,69 @@ void WriteKey(string key) { writer.Write(key); } + + writer.Write(" = "); + } + + void WriteFlag(KVFlag kvFlag) + { + if (kvFlag == KVFlag.None) + { + return; + } + + var flags = (int)kvFlag; + var i = 0; + var currentFlag = -1; + var more = false; + + while (i < flags) + { + var flag = (1 << ++currentFlag); + + i += flag; + + if ((flag & flags) == 0) + { + continue; + } + + var serialized = SerializeFlagName((KVFlag)flag); + + if (serialized == null) + { + continue; + } + + if (more) + { + writer.Write('|'); + } + + writer.Write(serialized); + + more = true; + } + + writer.Write(':'); } void WriteLine() { writer.WriteLine(); } + + string SerializeFlagName(KVFlag flag) + { + return flag switch + { + KVFlag.Resource => "resource", + KVFlag.ResourceName => "resource_name", + KVFlag.Panorama => "panorama", + KVFlag.SoundEvent => "soundevent", + KVFlag.SubClass => "subclass", + _ => null, + }; + } } } From bd3d4477e502efeb437dfae7f77fb58a32d5f01b Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Thu, 11 Aug 2022 11:59:54 +0300 Subject: [PATCH 32/49] Test that we serialize to kv1 without flags and arrays --- .../Test Data/TextKV3/array.kv3 | 7 ++- .../Test Data/TextKV3/array_kv1.vdf | 43 +++++++++++++ .../Test Data/TextKV3/flagged_value_kv1.vdf | 21 +++++++ .../TextKV3/Kv3ToKv1TestCase.cs | 60 +++++++++++++++++++ .../KeyValues1/KV1TextSerializer.cs | 36 ++++++++++- 5 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_kv1.vdf create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_kv1.vdf create mode 100644 ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv3ToKv1TestCase.cs diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 index a89d5c82..cefef2aa 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array.kv3 @@ -23,6 +23,11 @@ #[ 11 FF ], - resource:"hello.world" + resource:"hello.world", + """ +multiline +string +""", + -69.420 ] } diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_kv1.vdf b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_kv1.vdf new file mode 100644 index 00000000..924c7fd0 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_kv1.vdf @@ -0,0 +1,43 @@ +"root" +{ + "arrayValue" + { + "0" "a" + "1" "b" + } + "arrayOnSingleLine" + { + "0" "16.7551" + "1" "20.3763" + "2" "19.6448" + } + "arrayNoSpace" + { + "0" "1.3763" + "1" "19.6448" + } + "arrayMixedTypes" + { + "0" "a" + "1" "1" + "2" "True" + "3" "False" + "4" "" + "5" + { + "foo" "bar" + } + "6" + { + "0" "1" + "1" "3" + "2" "3" + "3" "7" + } + "7" "hello.world" + "8" "multiline +string" + "9" "-69.42" + } + "test" "success" +} diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_kv1.vdf b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_kv1.vdf new file mode 100644 index 00000000..2b3b84a0 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_kv1.vdf @@ -0,0 +1,21 @@ +"root" +{ + "foo" "bar" + "bar" "foo" + "uppercase" "foo" + "flaggedNumber" "-1234" + "multipleFlags" "cool value" + "soundEvent" "event sound" + "noFlags" "5" + "flaggedObject" + { + "1" "test1" + "2" "test2" + "3" + { + "0" "test3" + } + "4" "test4" + } + "test" "success" +} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv3ToKv1TestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv3ToKv1TestCase.cs new file mode 100644 index 00000000..433d4c0c --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv3ToKv1TestCase.cs @@ -0,0 +1,60 @@ +using System.IO; +using NUnit.Framework; + +namespace ValveKeyValue.Test.TextKV3 +{ + class Kv3ToKv1TestCase + { + [Test] + public void SerializesAndDropsFlags() + { + using var stream = TestDataHelper.OpenResource("TextKV3.flagged_value.kv3"); + var expected = TestDataHelper.ReadTextResource("TextKV3.flagged_value_kv1.vdf"); + + var kv1 = KVSerializer.Create(KVSerializationFormat.KeyValues1Text); + var kv3 = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); + var data = kv3.Deserialize(stream); + + data.Add(new KVObject("test", "success")); + + string text; + using (var ms = new MemoryStream()) + { + kv1.Serialize(ms, data); + + ms.Seek(0, SeekOrigin.Begin); + using var reader = new StreamReader(ms); + text = reader.ReadToEnd(); + } + + Assert.That(text, Is.EqualTo(expected)); + } + + [Test] + public void SerializesArraysToObjects() + { + using var stream = TestDataHelper.OpenResource("TextKV3.array.kv3"); + var expected = TestDataHelper.ReadTextResource("TextKV3.array_kv1.vdf"); + + var kv1 = KVSerializer.Create(KVSerializationFormat.KeyValues1Text); + var kv3 = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); + var data = kv3.Deserialize(stream); + + data.Add(new KVObject("test", "success")); + + string text; + using (var ms = new MemoryStream()) + { + kv1.Serialize(ms, data); + + ms.Seek(0, SeekOrigin.Begin); + using var reader = new StreamReader(ms); + text = reader.ReadToEnd(); + } + + System.Console.WriteLine(text); + + Assert.That(text, Is.EqualTo(expected)); + } + } +} diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs index cb804fc0..122880e6 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs @@ -1,5 +1,6 @@ using System.Globalization; using System.Text; +using System.Xml.Linq; using ValveKeyValue.Abstraction; namespace ValveKeyValue.Serialization.KeyValues1 @@ -21,6 +22,7 @@ public KV1TextSerializer(Stream stream, KVSerializerOptions options) readonly KVSerializerOptions options; readonly TextWriter writer; int indentation = 0; + Stack arrayCount = new(); public void Dispose() { @@ -36,9 +38,26 @@ public void OnObjectEnd() public void OnKeyValuePair(string name, KVValue value) => WriteKeyValuePair(name, value); - public void OnArrayStart(string name, KVFlag flag) => throw new NotImplementedException(); - public void OnArrayValue(KVValue value) => throw new NotImplementedException(); - public void OnArrayEnd() => throw new NotImplementedException(); + public void OnArrayStart(string name, KVFlag flag) + { + WriteStartObject(name); + arrayCount.Push(0); + } + + public void OnArrayValue(KVValue value) + { + var count = arrayCount.Pop(); + + WriteKeyValuePair(count.ToString(), value); + + arrayCount.Push(count + 1); + } + + public void OnArrayEnd() + { + WriteEndObject(); + arrayCount.Pop(); + } public void DiscardCurrentObject() { @@ -47,6 +66,15 @@ public void DiscardCurrentObject() void WriteStartObject(string name) { + if (name == null) + { + var count = arrayCount.Pop(); + + name = count.ToString(); + + arrayCount.Push(count + 1); + } + WriteIndentation(); WriteText(name); WriteLine(); @@ -66,6 +94,8 @@ void WriteEndObject() void WriteKeyValuePair(string name, IConvertible value) { + // TODO: Handle true, false, null value types + WriteIndentation(); WriteText(name); writer.Write('\t'); From b1fa5d2474cdda5b0faf6524fdd523ee7da11158 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Thu, 11 Aug 2022 12:03:26 +0300 Subject: [PATCH 33/49] Add expected array serialization --- .../Test Data/TextKV3/array_serialized.kv3 | 42 +++++++++++++++++++ .../TextKV3/Kv3ToKv1TestCase.cs | 2 - .../TextKV3/SerializationTestCase.cs | 2 +- .../KeyValues3/KV3TextSerializer.cs | 2 + 4 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_serialized.kv3 diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_serialized.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_serialized.kv3 new file mode 100644 index 00000000..e69142e3 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_serialized.kv3 @@ -0,0 +1,42 @@ + +{ + arrayValue = [ + "a", + "b", + ] + arrayOnSingleLine = [ + 16.7551, + 20.3763, + 19.6448, + ] + arrayNoSpace = [ + 1.3763, + 19.6448, + ] + arrayMixedTypes = [ + "a", + 1, + true, + false, + null, + { + foo = "bar" + }, + [ + 1, + 3, + 3, + 7, + ], + #[ + 11 FF + ], + resource:"hello.world", + """ +multiline +string +""", + -69.42, + ] + test = "success" +} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv3ToKv1TestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv3ToKv1TestCase.cs index 433d4c0c..7be5493d 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv3ToKv1TestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv3ToKv1TestCase.cs @@ -52,8 +52,6 @@ public void SerializesArraysToObjects() text = reader.ReadToEnd(); } - System.Console.WriteLine(text); - Assert.That(text, Is.EqualTo(expected)); } } diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs index 029e8c23..44d03ec9 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs @@ -33,7 +33,7 @@ public void CreatesTextDocument() public void SerializesArray() { using var stream = TestDataHelper.OpenResource("TextKV3.array.kv3"); - var expected = TestDataHelper.ReadTextResource("TextKV3.array.kv3"); + var expected = TestDataHelper.ReadTextResource("TextKV3.array_serialized.kv3"); var kv = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); var data = kv.Deserialize(stream); diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs index 3836e4b8..3eaf4c17 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs @@ -64,6 +64,7 @@ public void OnArrayEnd() indentation--; WriteIndentation(); writer.Write(']'); + // TODO: Write comma if we're in an array writer.WriteLine(); } @@ -94,6 +95,7 @@ void WriteEndObject() indentation--; WriteIndentation(); writer.Write('}'); + // TODO: Write comma if we're in an array writer.WriteLine(); } From f709b88ef1c484e7281c9aa2954f2b24304a4acd Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Thu, 11 Aug 2022 20:09:59 +0300 Subject: [PATCH 34/49] Simplify --- .../Test Data/TextKV3/types.kv3 | 1 + .../Test Data/TextKV3/types_serialized.kv3 | 1 + .../TextKV3/BasicKV3TestCases.cs | 3 + .../KeyValues3/KV3TokenReader.cs | 119 +++++++----------- 4 files changed, 53 insertions(+), 71 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 index cd42ff0a..79839af1 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 @@ -25,4 +25,5 @@ this is a multi line key """ = "multi line key parsed" + empty.string = "" } diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 index 7bc790b5..807c81ae 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 @@ -21,6 +21,7 @@ key_with._various.separators = "test" "quoted key with : {} terminators" = "test quoted key" "this is a multi\nline\nkey" = "multi line key parsed" + empty.string = "" multiLineString = """ hello world diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index 2c097517..b9fbba51 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -181,6 +181,9 @@ public void DeserializesBasicTypes() Assert.That(data["this is a multi\nline\nkey"].ValueType, Is.EqualTo(KVValueType.String)); Assert.That((string)data["this is a multi\nline\nkey"], Is.EqualTo("multi line key parsed")); + + Assert.That(data["empty.string"].ValueType, Is.EqualTo(KVValueType.String)); + Assert.That((string)data["empty.string"], Is.EqualTo(string.Empty)); }); } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 764aa032..2e5c22e9 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -74,6 +74,42 @@ public void Dispose() } } + KVToken ReadAssignment() + { + ReadChar(Assignment); + return new KVToken(KVTokenType.Assignment); + } + + KVToken ReadComma() + { + ReadChar(Comma); + return new KVToken(KVTokenType.Comma); + } + + KVToken ReadArrayStart() + { + ReadChar(ArrayStart); + return new KVToken(KVTokenType.ArrayStart); + } + + KVToken ReadArrayEnd() + { + ReadChar(ArrayEnd); + return new KVToken(KVTokenType.ArrayEnd); + } + + KVToken ReadObjectStart() + { + ReadChar(ObjectStart); + return new KVToken(KVTokenType.ObjectStart); + } + + KVToken ReadObjectEnd() + { + ReadChar(ObjectEnd); + return new KVToken(KVTokenType.ObjectEnd); + } + KVToken ReadStringOrIdentifier() { SwallowWhitespace(); @@ -97,18 +133,6 @@ KVToken ReadStringOrIdentifier() return new KVToken(type, token); } - KVToken ReadAssignment() - { - ReadChar(Assignment); - return new KVToken(KVTokenType.Assignment); - } - - KVToken ReadComma() - { - ReadChar(Comma); - return new KVToken(KVTokenType.Comma); - } - KVToken ReadBinaryBlob() { ReadChar(BinaryBlobMarker); @@ -136,30 +160,6 @@ KVToken ReadBinaryBlob() return new KVToken(KVTokenType.BinaryBlob, sb.ToString()); } - KVToken ReadArrayStart() - { - ReadChar(ArrayStart); - return new KVToken(KVTokenType.ArrayStart); - } - - KVToken ReadArrayEnd() - { - ReadChar(ArrayEnd); - return new KVToken(KVTokenType.ArrayEnd); - } - - KVToken ReadObjectStart() - { - ReadChar(ObjectStart); - return new KVToken(KVTokenType.ObjectStart); - } - - KVToken ReadObjectEnd() - { - ReadChar(ObjectEnd); - return new KVToken(KVTokenType.ObjectEnd); - } - public KVToken ReadHeader() { var str = ReadToken(); @@ -254,7 +254,6 @@ KVToken ReadComment() var next = Next(); var isMultiline = false; - // TODO: Read /* */ comments if (next == '*') { isMultiline = true; @@ -324,7 +323,7 @@ char Next() next = textReader.Read(); } - if (next == -1) + if (IsEndOfFile(next)) { throw new EndOfStreamException(); } @@ -454,6 +453,7 @@ string ReadQuotedStringRaw(char quotationMark) return string.Empty; } } + if (isMultiline) { var escapeNext = false; @@ -471,37 +471,19 @@ string ReadQuotedStringRaw(char quotationMark) if (!escapeNext && next == '\n') { - sb.Append(next); + var a = Next(); + var b = Next(); + var c = Next(); - next = Next(); - - // TODO: This is absolutely terrible - if (next == '"') - { - next = Next(); - - if (next == '"') - { - next = Next(); - - if (next == '"') - { - break; - } - else - { - sb.Append(next); - } - } - else - { - sb.Append(next); - } - } - else + if (a == '"' && b == '"' && c == '"') { - sb.Append(next); + break; } + + sb.Append(next); + sb.Append(a); + sb.Append(b); + sb.Append(c); } else { @@ -511,11 +493,6 @@ string ReadQuotedStringRaw(char quotationMark) } } - if (sb.Length > 0 && sb[^1] == '\n') - { - sb.Remove(sb.Length - 1, 1); - } - if (sb.Length > 0 && sb[^1] == '\r') { sb.Remove(sb.Length - 1, 1); From d517ae9754255dd771d75e9bba5a295cb5b52cad Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Thu, 11 Aug 2022 20:17:07 +0300 Subject: [PATCH 35/49] Put common token reader methods in one class --- .../KeyValues3/KV3TextReader.cs | 7 +- .../KeyValues3/KV3TokenReader.cs | 85 +------------------ 2 files changed, 4 insertions(+), 88 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 71c340f4..0c94706f 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -7,21 +7,18 @@ namespace ValveKeyValue.Deserialization.KeyValues3 { sealed class KV3TextReader : IVisitingReader { - public KV3TextReader(TextReader textReader, IParsingVisitationListener listener, KVSerializerOptions options) + public KV3TextReader(TextReader textReader, IParsingVisitationListener listener) { Require.NotNull(textReader, nameof(textReader)); Require.NotNull(listener, nameof(listener)); - Require.NotNull(options, nameof(options)); this.listener = listener; - this.options = options; - tokenReader = new KV3TokenReader(textReader, options); + tokenReader = new KV3TokenReader(textReader); stateMachine = new KV3TextReaderStateMachine(); } readonly IParsingVisitationListener listener; - readonly KVSerializerOptions options; readonly KV3TokenReader tokenReader; readonly KV3TextReaderStateMachine stateMachine; diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 2e5c22e9..96a44a9f 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -8,7 +8,7 @@ namespace ValveKeyValue.Deserialization.KeyValues3 { - class KV3TokenReader : IDisposable + class KV3TokenReader : KVTokenReader { const char ObjectStart = '{'; const char ObjectEnd = '}'; @@ -19,24 +19,14 @@ class KV3TokenReader : IDisposable const char Assignment = '='; const char Comma = ','; - public KV3TokenReader(TextReader textReader, KVSerializerOptions options) + public KV3TokenReader(TextReader textReader) : base(textReader) { - Require.NotNull(textReader, nameof(textReader)); - Require.NotNull(options, nameof(options)); - - this.textReader = textReader; - this.options = options; - // Dota 2 binary from 2017 used "+" as a terminate (for flagged values), but then they changed it to "|" var terminators = "{}[]=, \t\n\r'\":|;".ToCharArray(); integerTerminators = new HashSet(terminators.Select(t => (int)t)); } - readonly KVSerializerOptions options; readonly HashSet integerTerminators; - TextReader textReader; - bool disposed; - int? peekedNext; public KVToken ReadNextToken() { @@ -63,17 +53,6 @@ public KVToken ReadNextToken() }; } - public void Dispose() - { - if (!disposed) - { - textReader.Dispose(); - textReader = null; - - disposed = true; - } - } - KVToken ReadAssignment() { ReadChar(Assignment); @@ -309,50 +288,6 @@ KVToken ReadComment() return new KVToken(KVTokenType.Comment, text); } - char Next() - { - int next; - - if (peekedNext.HasValue) - { - next = peekedNext.Value; - peekedNext = null; - } - else - { - next = textReader.Read(); - } - - if (IsEndOfFile(next)) - { - throw new EndOfStreamException(); - } - - return (char)next; - } - - int Peek() - { - if (peekedNext.HasValue) - { - return peekedNext.Value; - } - - var next = textReader.Read(); - peekedNext = next; - - return next; - } - - void ReadChar(char expectedChar) - { - var next = Next(); - if (next != expectedChar) - { - throw new InvalidDataException($"The syntax is incorrect, expected '{expectedChar}' but got '{next}'."); - } - } - bool IsIdentifier(string text) { for (var i = 0; i < text.Length; i++) @@ -406,20 +341,6 @@ string ReadToken() return sb.ToString(); } - void SwallowWhitespace() - { - while (PeekWhitespace()) - { - Next(); - } - } - - bool PeekWhitespace() - { - var next = Peek(); - return !IsEndOfFile(next) && char.IsWhiteSpace((char)next); - } - string ReadQuotedStringRaw(char quotationMark) { ReadChar(quotationMark); @@ -512,7 +433,5 @@ string ReadQuotedStringRaw(char quotationMark) return sb.ToString(); } - - bool IsEndOfFile(int value) => value == -1; } } From ecd2dde52d69e74743173aaa21c873c214e49024 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 2 Dec 2022 20:47:53 +0200 Subject: [PATCH 36/49] Put commas after objects and arrays in arrays --- .../KeyValues1/KV1TextSerializer.cs | 1 - .../KeyValues3/KV3TextSerializer.cs | 34 ++++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs index 122880e6..31f736a8 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs @@ -1,6 +1,5 @@ using System.Globalization; using System.Text; -using System.Xml.Linq; using ValveKeyValue.Abstraction; namespace ValveKeyValue.Serialization.KeyValues1 diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs index 3eaf4c17..1040945e 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Text; using ValveKeyValue.Abstraction; @@ -22,6 +23,9 @@ public KV3TextSerializer(Stream stream) readonly TextWriter writer; int indentation = 0; + Stack inArray = new(); + + bool IsInArray => inArray.Count > 0 && inArray.Peek(); public void Dispose() { @@ -29,16 +33,26 @@ public void Dispose() } public void OnObjectStart(string name, KVFlag flag) - => WriteStartObject(name, flag); + { + inArray.Push(false); + + WriteStartObject(name, flag); + } public void OnObjectEnd() - => WriteEndObject(); + { + inArray.Pop(); + + WriteEndObject(); + } public void OnKeyValuePair(string name, KVValue value) => WriteKeyValuePair(name, value); public void OnArrayStart(string name, KVFlag flag) { + inArray.Push(true); + WriteIndentation(); WriteKey(name); @@ -61,10 +75,17 @@ public void OnArrayValue(KVValue value) public void OnArrayEnd() { + inArray.Pop(); + indentation--; WriteIndentation(); writer.Write(']'); - // TODO: Write comma if we're in an array + + if (IsInArray) + { + writer.Write(','); + } + writer.WriteLine(); } @@ -95,7 +116,12 @@ void WriteEndObject() indentation--; WriteIndentation(); writer.Write('}'); - // TODO: Write comma if we're in an array + + if (IsInArray) + { + writer.Write(','); + } + writer.WriteLine(); } From 52af7c28318ea189f92423a02cee8fd428d6218d Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 2 Dec 2022 21:42:11 +0200 Subject: [PATCH 37/49] Implement binary blobs --- .../Test Data/TextKV3/array_kv1.vdf | 7 +-- .../TextKV3/BasicKV3TestCases.cs | 14 +++++- .../Abstraction/KVObjectVisitor.cs | 5 +- .../KeyValues3/KV3TextReader.cs | 9 ++-- .../KeyValues3/KV3TextSerializer.cs | 48 +++++++++++++++++++ ValveKeyValue/ValveKeyValue/Utils.cs | 15 ++++++ 6 files changed, 85 insertions(+), 13 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_kv1.vdf b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_kv1.vdf index 924c7fd0..942164b6 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_kv1.vdf +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_kv1.vdf @@ -34,10 +34,11 @@ "2" "3" "3" "7" } - "7" "hello.world" - "8" "multiline + "7" "11 FF" + "8" "hello.world" + "9" "multiline string" - "9" "-69.42" + "10" "-69.42" } "test" "success" } diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index b9fbba51..cb1badb3 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -90,7 +90,12 @@ public void DeserializesArray() using var stream = TestDataHelper.OpenResource("TextKV3.array.kv3"); var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); - Assert.True(false); + Assert.That(data["arrayValue"].ValueType, Is.EqualTo(KVValueType.Array)); + Assert.That(data["arrayOnSingleLine"].ValueType, Is.EqualTo(KVValueType.Array)); + Assert.That(data["arrayNoSpace"].ValueType, Is.EqualTo(KVValueType.Array)); + Assert.That(data["arrayMixedTypes"].ValueType, Is.EqualTo(KVValueType.Array)); + + // TODO: Test all the children values } [Test] @@ -99,7 +104,12 @@ public void DeserializesBinaryBlob() using var stream = TestDataHelper.OpenResource("TextKV3.binary_blob.kv3"); var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); - Assert.True(false); + Assert.That(data["array"].ValueType, Is.EqualTo(KVValueType.BinaryBlob)); + Assert.That(((KVBinaryBlob)data["array"]).Bytes, Is.EqualTo(new byte[] + { + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xFF + })); } [Test] diff --git a/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs b/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs index 12d04a51..25858b06 100644 --- a/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs +++ b/ValveKeyValue/ValveKeyValue/Abstraction/KVObjectVisitor.cs @@ -26,16 +26,13 @@ void VisitObject(string name, KVValue value, bool isArray) listener.OnObjectEnd(); break; - case KVValueType.BinaryBlob: - // TODO: write binary blobs - break; - case KVValueType.Array: listener.OnArrayStart(name, value.Flag); VisitArray((IEnumerable)value); listener.OnArrayEnd(); break; + case KVValueType.BinaryBlob: // TODO: Should binary blobs have their own method? case KVValueType.FloatingPoint: case KVValueType.Int32: case KVValueType.Pointer: diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 0c94706f..95236973 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -181,10 +181,11 @@ void ReadText(string text) void ReadBinaryBlob(string text) { - var bytes = Utils.ParseHexStringAsByteArray(text); - //var value = new KVObjectValue(bytes, KVValueType.BinaryBlob); - var value = new KVObjectValue(0x00, KVValueType.BinaryBlob); // TODO: wrong - value.Flag = stateMachine.GetAndResetFlag(); + var bytes = Utils.ParseHexStringAsByteArrayNoReverse(text); + var value = new KVBinaryBlob(bytes) + { + Flag = stateMachine.GetAndResetFlag() + }; switch (stateMachine.Current) { diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs index 1040945e..a27d192f 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs @@ -1,5 +1,7 @@ using System; +using System.Collections; using System.Collections.Generic; +using System.Drawing; using System.IO; using System.Text; using ValveKeyValue.Abstraction; @@ -142,6 +144,9 @@ void WriteValue(KVValue value) switch (value.ValueType) { + case KVValueType.BinaryBlob: + WriteBinaryBlob((KVBinaryBlob)value); + break; case KVValueType.Boolean: if ((bool)value) { @@ -166,6 +171,49 @@ void WriteValue(KVValue value) } } + void WriteBinaryBlob(KVBinaryBlob value) + { + // TODO: Verify this against Valve + if (value.Bytes.Length > 32) + { + writer.WriteLine(); + WriteIndentation(); + } + + writer.Write('#'); + writer.Write('['); + writer.WriteLine(); + indentation++; + WriteIndentation(); + + var count = 0; + + foreach (var oneByte in value.Bytes) + { + writer.Write(oneByte.ToString("X2")); + + if (++count % 32 == 0) + { + writer.WriteLine(); + WriteIndentation(); + } + else if (count != value.Bytes.Length) + { + writer.Write(' '); + } + } + + indentation--; + + if (count % 32 != 0) + { + writer.WriteLine(); + WriteIndentation(); + } + + writer.Write(']'); + } + void WriteIndentation() { if (indentation == 0) diff --git a/ValveKeyValue/ValveKeyValue/Utils.cs b/ValveKeyValue/ValveKeyValue/Utils.cs index e5494e33..5d4f8282 100644 --- a/ValveKeyValue/ValveKeyValue/Utils.cs +++ b/ValveKeyValue/ValveKeyValue/Utils.cs @@ -5,6 +5,21 @@ namespace ValveKeyValue { internal class Utils { + // TODO: Need to figure out whether Array.Reverse is actually correct for kv1 + public static byte[] ParseHexStringAsByteArrayNoReverse(string hexadecimalRepresentation) + { + Require.NotNull(hexadecimalRepresentation, nameof(hexadecimalRepresentation)); + + var data = new byte[hexadecimalRepresentation.Length / 2]; + for (var i = 0; i < data.Length; i++) + { + var currentByteText = hexadecimalRepresentation.Substring(i * 2, 2); + data[i] = byte.Parse(currentByteText, NumberStyles.HexNumber, CultureInfo.InvariantCulture); + } + + return data; + } + public static byte[] ParseHexStringAsByteArray(string hexadecimalRepresentation) { Require.NotNull(hexadecimalRepresentation, nameof(hexadecimalRepresentation)); From d800e2e9be75895cb5cc0d5d1a46682948d4c89a Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Sat, 3 Dec 2022 22:36:03 +0200 Subject: [PATCH 38/49] Remove unused usings --- ValveKeyValue/ValveKeyValue/KVSerializer.cs | 1 - .../ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs | 2 -- 2 files changed, 3 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue/KVSerializer.cs b/ValveKeyValue/ValveKeyValue/KVSerializer.cs index a8231520..62d5e375 100644 --- a/ValveKeyValue/ValveKeyValue/KVSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/KVSerializer.cs @@ -2,7 +2,6 @@ using ValveKeyValue.Deserialization; using ValveKeyValue.Deserialization.KeyValues1; using ValveKeyValue.Deserialization.KeyValues3; -using ValveKeyValue.Serialization; using ValveKeyValue.Serialization.KeyValues1; using ValveKeyValue.Serialization.KeyValues3; diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs index a27d192f..5bf64c28 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs @@ -1,7 +1,5 @@ using System; -using System.Collections; using System.Collections.Generic; -using System.Drawing; using System.IO; using System.Text; using ValveKeyValue.Abstraction; From df7d4373b773878a3961df6a5f8f566348280acb Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Sun, 4 Dec 2022 14:40:26 +0200 Subject: [PATCH 39/49] Add test for nested arrays and objects --- .../Test Data/TextKV3/array_nested.kv3 | 40 +++++++++++++++++++ .../TextKV3/SerializationTestCase.cs | 21 ++++++++++ 2 files changed, 61 insertions(+) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_nested.kv3 diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_nested.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_nested.kv3 new file mode 100644 index 00000000..643a1517 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_nested.kv3 @@ -0,0 +1,40 @@ + +{ + array = [ + 1, + 2, + 3, + { + array2 = [ + 4, + 5, + 6, + { + something = "something" + array3 = [ + 7, + 8, + 9, + ] + test = "abc" + }, + 10, + ] + test2 = "def" + }, + "string", + 11, + 12, + [ + 13, + 14, + 15, + [ + 16, + 17, + 18, + ], + ], + 19, + ] +} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs index 44d03ec9..69dff8e5 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs @@ -53,6 +53,27 @@ public void SerializesArray() Assert.That(text, Is.EqualTo(expected)); } + [Test] + public void SerializesNestedArray() + { + var expected = TestDataHelper.ReadTextResource("TextKV3.array_nested.kv3"); + + var kv = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); + var data = kv.Deserialize(expected); + + string text; + using (var ms = new MemoryStream()) + { + kv.Serialize(ms, data); + + ms.Seek(0, SeekOrigin.Begin); + using var reader = new StreamReader(ms); + text = reader.ReadToEnd(); + } + + Assert.That(text, Is.EqualTo(expected)); + } + [Test] public void SerializesFlags() { From cc6b98dca8f91fa5d36e33d646c5cc1ee9d68016 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Sun, 4 Dec 2022 15:02:30 +0200 Subject: [PATCH 40/49] Implement ICollection/IList on KVArrayValue --- .../ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index cb1badb3..aaf3cebd 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -95,6 +95,12 @@ public void DeserializesArray() Assert.That(data["arrayNoSpace"].ValueType, Is.EqualTo(KVValueType.Array)); Assert.That(data["arrayMixedTypes"].ValueType, Is.EqualTo(KVValueType.Array)); + var arrayValue = (KVArrayValue)data["arrayValue"]; + + Assert.That(arrayValue.Count, Is.EqualTo(2)); + Assert.That(arrayValue[0].ToString(), Is.EqualTo("a")); + Assert.That(arrayValue[1].ToString(), Is.EqualTo("b")); + // TODO: Test all the children values } From 080a360cb77918dd3359d834839de3a679875006 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 3 Mar 2023 11:52:39 +0200 Subject: [PATCH 41/49] Return KVFile from read header --- .../Deserialization/KeyValues3/KV3TextReader.cs | 2 +- .../Deserialization/KeyValues3/KV3TokenReader.cs | 8 ++++++-- ValveKeyValue/ValveKeyValue/KVFile.cs | 11 +++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue/KVFile.cs diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 95236973..9e1e8523 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -28,7 +28,7 @@ public void ReadObject() { Require.NotDisposed(nameof(KV3TextReader), disposed); - tokenReader.ReadHeader(); + var file = tokenReader.ReadHeader(); while (stateMachine.IsInObject) { diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 96a44a9f..a3a52342 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -139,7 +139,7 @@ KVToken ReadBinaryBlob() return new KVToken(KVTokenType.BinaryBlob, sb.ToString()); } - public KVToken ReadHeader() + public KVFile ReadHeader() { var str = ReadToken(); @@ -222,7 +222,11 @@ public KVToken ReadHeader() throw new InvalidDataException($"Unrecognized encoding specifier, expected '{Format.Generic}' but got '{format}'."); } - return new KVToken(KVTokenType.Header, string.Empty); + return new KVFile + { + Encoding = encoding, + Format = format, + }; } KVToken ReadComment() diff --git a/ValveKeyValue/ValveKeyValue/KVFile.cs b/ValveKeyValue/ValveKeyValue/KVFile.cs new file mode 100644 index 00000000..f45bd816 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue/KVFile.cs @@ -0,0 +1,11 @@ +using System; + +namespace ValveKeyValue +{ + public class KVFile + { + public Guid Encoding { get; set; } + public Guid Format { get; set; } + public KVObject Root { get; set; } + } +} From 2057ebaa1c2fcfe24888fe08d55237e0d32a898e Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Mon, 6 Mar 2023 12:05:47 +0200 Subject: [PATCH 42/49] Asset.Pass breaks analyzer --- ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs index 7e2711ac..3e398dc2 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs @@ -71,7 +71,7 @@ public void ValidHeadersAreParsed(string value) using var stream = new MemoryStream(Encoding.UTF8.GetBytes(value)); var kv = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); - Assert.Pass(); + Assert.That(() => true); } } } From 4882f3dae0d0054b12fcda1b4eb6aef2a5c389af Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Mon, 6 Mar 2023 12:23:25 +0200 Subject: [PATCH 43/49] Add test cases for different types of roots --- .../Test Data/TextKV3/root_array.kv3 | 23 +++ .../Test Data/TextKV3/root_binary_blob.kv3 | 5 + .../Test Data/TextKV3/root_flagged_object.kv3 | 4 + .../Test Data/TextKV3/root_flagged_string.kv3 | 2 + .../Test Data/TextKV3/root_float.kv3 | 2 + .../Test Data/TextKV3/root_multiline.kv3 | 5 + .../Test Data/TextKV3/root_null.kv3 | 2 + .../Test Data/TextKV3/root_number.kv3 | 2 + .../TextKV3/root_number_negative.kv3 | 2 + .../Test Data/TextKV3/root_string.kv3 | 2 + .../TextKV3/RootTypesTestCase.cs | 148 ++++++++++++++++++ 11 files changed, 197 insertions(+) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_array.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_binary_blob.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_flagged_object.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_flagged_string.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_float.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_multiline.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_null.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_number.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_number_negative.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_string.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/TextKV3/RootTypesTestCase.cs diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_array.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_array.kv3 new file mode 100644 index 00000000..23180206 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_array.kv3 @@ -0,0 +1,23 @@ + +[ + "a", + 1, + true, + false, + null, + { + foo = "bar" + }, + [ + 1, 3, 3, 7 + ], + #[ + 11 FF + ], + resource:"hello.world", + """ +multiline +string +""", + -69.420 +] diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_binary_blob.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_binary_blob.kv3 new file mode 100644 index 00000000..576d37da --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_binary_blob.kv3 @@ -0,0 +1,5 @@ + +#[ + 00 11 22 33 44 55 66 77 88 99 + AA BB CC DD FF +] diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_flagged_object.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_flagged_object.kv3 new file mode 100644 index 00000000..b225c276 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_flagged_object.kv3 @@ -0,0 +1,4 @@ + +panorama:{ + foo = resource:"bar" +} diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_flagged_string.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_flagged_string.kv3 new file mode 100644 index 00000000..3c25bf65 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_flagged_string.kv3 @@ -0,0 +1,2 @@ + +resource:"cool_resource.txt" diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_float.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_float.kv3 new file mode 100644 index 00000000..e980b0c1 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_float.kv3 @@ -0,0 +1,2 @@ + +-1337.401 diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_multiline.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_multiline.kv3 new file mode 100644 index 00000000..a79c6405 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_multiline.kv3 @@ -0,0 +1,5 @@ + +""" +First line of a multi-line string literal. +Second line of a multi-line string literal. +""" diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_null.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_null.kv3 new file mode 100644 index 00000000..e00a7e2f --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_null.kv3 @@ -0,0 +1,2 @@ + +null \ No newline at end of file diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_number.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_number.kv3 new file mode 100644 index 00000000..97480416 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_number.kv3 @@ -0,0 +1,2 @@ + +1234567890 \ No newline at end of file diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_number_negative.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_number_negative.kv3 new file mode 100644 index 00000000..b9b5ff00 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_number_negative.kv3 @@ -0,0 +1,2 @@ + +-1234567890 \ No newline at end of file diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_string.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_string.kv3 new file mode 100644 index 00000000..e627908c --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/root_string.kv3 @@ -0,0 +1,2 @@ + +"cool 123 string" diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/RootTypesTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/RootTypesTestCase.cs new file mode 100644 index 00000000..e685b105 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/RootTypesTestCase.cs @@ -0,0 +1,148 @@ +using NUnit.Framework; + +namespace ValveKeyValue.Test.TextKV3 +{ + class RootTypesTestCase + { + [Test] + public void DeserializesRootNull() + { + using var stream = TestDataHelper.OpenResource("TextKV3.root_null.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.Multiple(() => + { + Assert.That(data.Name, Is.EqualTo("root")); + Assert.That(data.Value.ValueType, Is.EqualTo(KVValueType.Null)); + Assert.That((string)data.Value, Is.EqualTo("")); // TODO: This should be a null value + }); + } + + [Test] + public void DeserializesRootString() + { + using var stream = TestDataHelper.OpenResource("TextKV3.root_string.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.Multiple(() => + { + Assert.That(data.Name, Is.EqualTo("root")); + Assert.That(data.Value.ValueType, Is.EqualTo(KVValueType.String)); + Assert.That((string)data.Value, Is.EqualTo("cool 123 string")); + }); + } + + [Test] + public void DeserializesRootMultilineString() + { + using var stream = TestDataHelper.OpenResource("TextKV3.root_multiline.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.Multiple(() => + { + Assert.That(data.Name, Is.EqualTo("root")); + Assert.That((string)data.Value, Is.EqualTo("First line of a multi-line string literal.\nSecond line of a multi-line string literal.")); + }); + } + + [Test] + public void DeserializesRootFlaggedString() + { + using var stream = TestDataHelper.OpenResource("TextKV3.root_flagged_string.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.Multiple(() => + { + Assert.That(data.Name, Is.EqualTo("root")); + Assert.That(data.Value.ValueType, Is.EqualTo(KVValueType.String)); + Assert.That(data.Value.Flag, Is.EqualTo(KVFlag.Resource)); + Assert.That((string)data.Value, Is.EqualTo("cool_resource.txt")); + }); + } + + [Test] + public void DeserializesRootFlaggedObject() + { + using var stream = TestDataHelper.OpenResource("TextKV3.root_flagged_object.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.Multiple(() => + { + Assert.That(data.Name, Is.EqualTo("root")); + Assert.That(data.Value.ValueType, Is.EqualTo(KVValueType.Collection)); + Assert.That(data.Value.Flag, Is.EqualTo(KVFlag.Panorama)); + Assert.That(data["foo"].Flag, Is.EqualTo(KVFlag.Resource)); + Assert.That((string)data["foo"], Is.EqualTo("bar")); + }); + } + + [Test] + public void DeserializesRootBinaryBlob() + { + using var stream = TestDataHelper.OpenResource("TextKV3.root_binary_blob.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.Multiple(() => + { + Assert.That(data.Name, Is.EqualTo("root")); + Assert.That(data.Value.ValueType, Is.EqualTo(KVValueType.BinaryBlob)); + Assert.That(((KVBinaryBlob)data.Value).Bytes, Is.EqualTo(new byte[] + { + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xFF + })); + }); + } + + [Test] + public void DeserializesRootNumber() + { + using var stream = TestDataHelper.OpenResource("TextKV3.root_number.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.Multiple(() => + { + Assert.That(data.Name, Is.EqualTo("root")); + Assert.That(data.Value.ValueType, Is.EqualTo(KVValueType.UInt64)); + Assert.That((int)data.Value, Is.EqualTo(1234567890)); + }); + } + + [Test] + public void DeserializesRootNumberNegative() + { + using var stream = TestDataHelper.OpenResource("TextKV3.root_number_negative.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.Multiple(() => + { + Assert.That(data.Name, Is.EqualTo("root")); + Assert.That(data.Value.ValueType, Is.EqualTo(KVValueType.Int64)); + Assert.That((int)data.Value, Is.EqualTo(-1234567890)); + }); + } + + [Test] + public void DeserializesRootFloat() + { + using var stream = TestDataHelper.OpenResource("TextKV3.root_float.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.Multiple(() => + { + Assert.That(data.Name, Is.EqualTo("root")); + Assert.That(data.Value.ValueType, Is.EqualTo(KVValueType.FloatingPoint)); + Assert.That((float)data.Value, Is.EqualTo(-1337.401f)); + }); + } + + [Test] + public void DeserializesRootArray() + { + using var stream = TestDataHelper.OpenResource("TextKV3.root_array.kv3"); + var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); + + Assert.That(data.Value.ValueType, Is.EqualTo(KVValueType.Array)); + } + } +} From 8bbccd7049a72e7b888f65fd28b744d4230a34db Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Sat, 8 Apr 2023 11:06:28 +0300 Subject: [PATCH 44/49] Add kv1 -> kv3 test --- .../Test Data/TextKV3/array_from_kv1.kv3 | 41 +++++++++++++ .../TextKV3/flagged_value_from_kv1.kv3 | 20 +++++++ .../TextKV3/Kv1ToKv3TestCase.cs | 59 +++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_from_kv1.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_from_kv1.kv3 create mode 100644 ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv1ToKv3TestCase.cs diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_from_kv1.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_from_kv1.kv3 new file mode 100644 index 00000000..0e046a7d --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_from_kv1.kv3 @@ -0,0 +1,41 @@ + +{ + arrayValue = { + 0 = "a" + 1 = "b" + } + arrayOnSingleLine = { + 0 = 16.7551 + 1 = 20.3763 + 2 = 19.6448 + } + arrayNoSpace = { + 0 = 1.3763 + 1 = 19.6448 + } + arrayMixedTypes = { + 0 = "a" + 1 = "1" + 2 = "True" + 3 = "False" + 4 = "" + 5 = { + foo = "bar" + } + 6 = { + 0 = "1" + 1 = "3" + 2 = "3" + 3 = "7" + } + 7 = "11 FF" + 8 = "hello.world" + 9 = """ +multiline +string +""" + 10 = -69.42 + } + test = "success" + test = "success" +} diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_from_kv1.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_from_kv1.kv3 new file mode 100644 index 00000000..1a955fb6 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_from_kv1.kv3 @@ -0,0 +1,20 @@ + +{ + foo = "bar" + bar = "foo" + uppercase = "foo" + flaggedNumber = "-1234" + multipleFlags = "cool value" + soundEvent = "event sound" + noFlags = "5" + flaggedObject = { + 1 = "test1" + 2 = "test2" + 3 = { + 0 = "test3" + } + 4 = "test4" + } + test = "success" + test = "success" +} diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv1ToKv3TestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv1ToKv3TestCase.cs new file mode 100644 index 00000000..e6037eb6 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv1ToKv3TestCase.cs @@ -0,0 +1,59 @@ +using System; +using System.IO; +using NUnit.Framework; + +namespace ValveKeyValue.Test.TextKV3 +{ + class Kv1ToKv3TestCase + { + [Test] + public void SerializesBasicObjects() + { + using var stream = TestDataHelper.OpenResource("TextKV3.flagged_value_kv1.vdf"); + var expected = TestDataHelper.ReadTextResource("TextKV3.flagged_value_from_kv1.kv3"); + + var kv1 = KVSerializer.Create(KVSerializationFormat.KeyValues1Text); + var kv3 = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); + var data = kv1.Deserialize(stream); + + data.Add(new KVObject("test", "success")); + + string text; + using (var ms = new MemoryStream()) + { + kv3.Serialize(ms, data); + + ms.Seek(0, SeekOrigin.Begin); + using var reader = new StreamReader(ms); + text = reader.ReadToEnd(); + } + + Assert.That(text, Is.EqualTo(expected)); + } + + [Test] + public void SerializesAndKeepsLinearObjects() // TODO: Perhaps in the future KV1 arrays can use the KVArray type so it can be emitted as an array + { + using var stream = TestDataHelper.OpenResource("TextKV3.array_kv1.vdf"); + var expected = TestDataHelper.ReadTextResource("TextKV3.array_from_kv1.kv3"); + + var kv1 = KVSerializer.Create(KVSerializationFormat.KeyValues1Text); + var kv3 = KVSerializer.Create(KVSerializationFormat.KeyValues3Text); + var data = kv1.Deserialize(stream); + + data.Add(new KVObject("test", "success")); + + string text; + using (var ms = new MemoryStream()) + { + kv3.Serialize(ms, data); + + ms.Seek(0, SeekOrigin.Begin); + using var reader = new StreamReader(ms); + text = reader.ReadToEnd(); + } + + Assert.That(text, Is.EqualTo(expected)); + } + } +} From 5911cc6ebcd5614faeacd120422185ffc58b7f41 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 4 Jul 2023 12:12:49 +0300 Subject: [PATCH 45/49] Improve kv3 key escaping --- .../Test Data/TextKV3/array_from_kv1.kv3 | 44 +++++++++---------- .../TextKV3/flagged_value_from_kv1.kv3 | 10 ++--- .../TextKV3/flagged_value_serialized.kv3 | 8 ++-- .../Test Data/TextKV3/types.kv3 | 6 +++ .../Test Data/TextKV3/types_serialized.kv3 | 6 +++ .../KeyValues3/KV3TokenReader.cs | 1 + .../KeyValues3/KV3TextSerializer.cs | 19 +++++--- 7 files changed, 57 insertions(+), 37 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_from_kv1.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_from_kv1.kv3 index 0e046a7d..4e33631b 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_from_kv1.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/array_from_kv1.kv3 @@ -1,40 +1,40 @@ { arrayValue = { - 0 = "a" - 1 = "b" + "0" = "a" + "1" = "b" } arrayOnSingleLine = { - 0 = 16.7551 - 1 = 20.3763 - 2 = 19.6448 + "0" = 16.7551 + "1" = 20.3763 + "2" = 19.6448 } arrayNoSpace = { - 0 = 1.3763 - 1 = 19.6448 + "0" = 1.3763 + "1" = 19.6448 } arrayMixedTypes = { - 0 = "a" - 1 = "1" - 2 = "True" - 3 = "False" - 4 = "" - 5 = { + "0" = "a" + "1" = "1" + "2" = "True" + "3" = "False" + "4" = "" + "5" = { foo = "bar" } - 6 = { - 0 = "1" - 1 = "3" - 2 = "3" - 3 = "7" + "6" = { + "0" = "1" + "1" = "3" + "2" = "3" + "3" = "7" } - 7 = "11 FF" - 8 = "hello.world" - 9 = """ + "7" = "11 FF" + "8" = "hello.world" + "9" = """ multiline string """ - 10 = -69.42 + "10" = -69.42 } test = "success" test = "success" diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_from_kv1.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_from_kv1.kv3 index 1a955fb6..a23ab7c5 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_from_kv1.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_from_kv1.kv3 @@ -8,12 +8,12 @@ soundEvent = "event sound" noFlags = "5" flaggedObject = { - 1 = "test1" - 2 = "test2" - 3 = { - 0 = "test3" + "1" = "test1" + "2" = "test2" + "3" = { + "0" = "test3" } - 4 = "test4" + "4" = "test4" } test = "success" test = "success" diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_serialized.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_serialized.kv3 index d8d99e46..859d794e 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_serialized.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/flagged_value_serialized.kv3 @@ -8,12 +8,12 @@ soundEvent = soundevent:"event sound" noFlags = 5 flaggedObject = panorama:{ - 1 = soundevent:"test1" - 2 = "test2" - 3 = subclass:[ + "1" = soundevent:"test1" + "2" = "test2" + "3" = subclass:[ "test3", ] - 4 = resource_name:"test4" + "4" = resource_name:"test4" } test = "success" } diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 index 79839af1..22b2563d 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types.kv3 @@ -26,4 +26,10 @@ line key """ = "multi line key parsed" empty.string = "" + 1 = "one" + a = "alpha" + 22 = "two" + a3 = "three" + bb = "bravo" + "" = "empty key" } diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 index 807c81ae..b089f37d 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/types_serialized.kv3 @@ -22,6 +22,12 @@ "quoted key with : {} terminators" = "test quoted key" "this is a multi\nline\nkey" = "multi line key parsed" empty.string = "" + "1" = "one" + a = "alpha" + "22" = "two" + a3 = "three" + bb = "bravo" + "" = "empty key" multiLineString = """ hello world diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index a3a52342..48e5d948 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -308,6 +308,7 @@ bool IsIdentifier(string text) continue; } + // TODO: Disallow : because it's a token terminator? if (c == '_' || c == ':' || c == '.') { continue; diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs index 5bf64c28..715f8bea 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs @@ -269,10 +269,16 @@ void WriteKey(string key) return; } - var escaped = false; + var escaped = key.Length == 0; // Quote empty strings var sb = new StringBuilder(key.Length + 2); sb.Append('"'); + if (key.Length > 0 && key[0] >= '0' && key[0] <= '9') + { + // Quote when first character is a digit + escaped = true; + } + foreach (var @char in key) { switch (@char) @@ -289,11 +295,6 @@ void WriteKey(string key) sb.Append('n'); break; - case ' ': - escaped = true; - sb.Append(' '); - break; - case '"': escaped = true; sb.Append('\\'); @@ -307,6 +308,12 @@ void WriteKey(string key) break; default: + // TODO: Use char.IsAscii* functions from newer .NET + if (@char != '.' && @char != '_' && !((@char >= 'A' && @char <= 'Z') || (@char >= 'a' && @char <= 'z') || (@char >= '0' && @char <= '9'))) + { + escaped = true; + } + sb.Append(@char); break; } From 0a61d2dbe0a374bb256be4983d67da7bdd2e613d Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Mon, 24 Jul 2023 13:02:32 +0300 Subject: [PATCH 46/49] Make KVFile extend KVObject --- .../ValveKeyValue/Deserialization/IVisitingReader.cs | 2 +- .../Deserialization/KeyValues1/KV1BinaryReader.cs | 4 +++- .../Deserialization/KeyValues1/KV1TextReader.cs | 8 +++++--- .../Deserialization/KeyValues3/KV3TextReader.cs | 6 ++++-- .../Deserialization/KeyValues3/KV3TokenReader.cs | 4 ++-- ValveKeyValue/ValveKeyValue/KVFile.cs | 11 +++++++---- ValveKeyValue/ValveKeyValue/KVHeader.cs | 10 ++++++++++ ValveKeyValue/ValveKeyValue/KVSerializer.cs | 9 ++++----- 8 files changed, 36 insertions(+), 18 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue/KVHeader.cs diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/IVisitingReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/IVisitingReader.cs index 7b6eaa4e..0f2246ee 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/IVisitingReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/IVisitingReader.cs @@ -2,6 +2,6 @@ namespace ValveKeyValue.Deserialization { interface IVisitingReader : IDisposable { - void ReadObject(); + KVHeader ReadHeader(); } } diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs index 6eb0b148..c022e8c4 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs @@ -32,7 +32,7 @@ public KV1BinaryReader(Stream stream, IVisitationListener listener, StringTable bool disposed; KV1BinaryNodeType endMarker = KV1BinaryNodeType.End; - public void ReadObject() + public KVHeader ReadHeader() { Require.NotDisposed(nameof(KV1TextReader), disposed); @@ -54,6 +54,8 @@ public void ReadObject() { throw new KeyValueException("Error while parsing binary KeyValues.", ex); } + + return new KVHeader(); } public void Dispose() diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1TextReader.cs index b6bdc807..aef2f90f 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1TextReader.cs @@ -27,7 +27,7 @@ public KV1TextReader(TextReader textReader, IParsingVisitationListener listener, readonly KV1TextReaderStateMachine stateMachine; bool disposed; - public void ReadObject() + public KVHeader ReadHeader() { Require.NotDisposed(nameof(KV1TextReader), disposed); @@ -103,6 +103,8 @@ public void ReadObject() throw new ArgumentOutOfRangeException(nameof(token.TokenType), token.TokenType, "Unhandled token type."); } } + + return new KVHeader(); } public void Dispose() @@ -224,7 +226,7 @@ void DoIncludeAndMerge(string filePath) using var stream = OpenFileForInclude(filePath); using var reader = new KV1TextReader(new StreamReader(stream), mergeListener, options); - reader.ReadObject(); + reader.ReadHeader(); } void DoIncludeAndAppend(string filePath) @@ -233,7 +235,7 @@ void DoIncludeAndAppend(string filePath) using var stream = OpenFileForInclude(filePath); using var reader = new KV1TextReader(new StreamReader(stream), appendListener, options); - reader.ReadObject(); + reader.ReadHeader(); } Stream OpenFileForInclude(string filePath) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index 9e1e8523..ea4f34fa 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -24,11 +24,11 @@ public KV3TextReader(TextReader textReader, IParsingVisitationListener listener) readonly KV3TextReaderStateMachine stateMachine; bool disposed; - public void ReadObject() + public KVHeader ReadHeader() { Require.NotDisposed(nameof(KV3TextReader), disposed); - var file = tokenReader.ReadHeader(); + var header = tokenReader.ReadHeader(); while (stateMachine.IsInObject) { @@ -108,6 +108,8 @@ public void ReadObject() throw new ArgumentOutOfRangeException(nameof(token.TokenType), token.TokenType, "Unhandled token type."); } } + + return header; } public void Dispose() diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 48e5d948..7a6124c0 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -139,7 +139,7 @@ KVToken ReadBinaryBlob() return new KVToken(KVTokenType.BinaryBlob, sb.ToString()); } - public KVFile ReadHeader() + public KVHeader ReadHeader() { var str = ReadToken(); @@ -222,7 +222,7 @@ public KVFile ReadHeader() throw new InvalidDataException($"Unrecognized encoding specifier, expected '{Format.Generic}' but got '{format}'."); } - return new KVFile + return new KVHeader { Encoding = encoding, Format = format, diff --git a/ValveKeyValue/ValveKeyValue/KVFile.cs b/ValveKeyValue/ValveKeyValue/KVFile.cs index f45bd816..bae07fc6 100644 --- a/ValveKeyValue/ValveKeyValue/KVFile.cs +++ b/ValveKeyValue/ValveKeyValue/KVFile.cs @@ -2,10 +2,13 @@ namespace ValveKeyValue { - public class KVFile + public class KVFile : KVObject { - public Guid Encoding { get; set; } - public Guid Format { get; set; } - public KVObject Root { get; set; } + public KVHeader Header { get; } + + public KVFile(KVHeader header, string name, KVValue value) : base(name, value) + { + Header = header; + } } } diff --git a/ValveKeyValue/ValveKeyValue/KVHeader.cs b/ValveKeyValue/ValveKeyValue/KVHeader.cs new file mode 100644 index 00000000..6f02091a --- /dev/null +++ b/ValveKeyValue/ValveKeyValue/KVHeader.cs @@ -0,0 +1,10 @@ +using System; + +namespace ValveKeyValue +{ + public class KVHeader + { + public Guid Encoding { get; set; } + public Guid Format { get; set; } + } +} diff --git a/ValveKeyValue/ValveKeyValue/KVSerializer.cs b/ValveKeyValue/ValveKeyValue/KVSerializer.cs index 62d5e375..a8fe6d24 100644 --- a/ValveKeyValue/ValveKeyValue/KVSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/KVSerializer.cs @@ -38,13 +38,12 @@ public KVDocument Deserialize(Stream stream, KVSerializerOptions options = null) Require.NotNull(stream, nameof(stream)); var builder = new KVObjectBuilder(); - using (var reader = MakeReader(stream, builder, options ?? KVSerializerOptions.DefaultOptions)) - { - reader.ReadObject(); - } + using var reader = MakeReader(stream, builder, options ?? KVSerializerOptions.DefaultOptions); + var header = reader.ReadHeader(); var root = builder.GetObject(); - return new KVDocument(root.Name, root.Value); + + return new KVDocument(header, root.Name, root.Value); // TODO } /// From 4101c389130f124ddea71b970e2930826e0cc993 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Mon, 24 Jul 2023 13:17:05 +0300 Subject: [PATCH 47/49] Make KVFlag singular --- .../KeyValues3/KV3TextReaderStateMachine.cs | 2 +- ValveKeyValue/ValveKeyValue/KVFlag.cs | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs index 66bb1a6e..127ec45d 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs @@ -36,7 +36,7 @@ public void PopObject() public void SetName(string name) => CurrentObject.Key = name; - public void SetFlag(KVFlag flag) => CurrentObject.Flag |= flag; + public void SetFlag(KVFlag flag) => CurrentObject.Flag = flag; public KVFlag GetAndResetFlag() { diff --git a/ValveKeyValue/ValveKeyValue/KVFlag.cs b/ValveKeyValue/ValveKeyValue/KVFlag.cs index 22e04573..25e20c31 100644 --- a/ValveKeyValue/ValveKeyValue/KVFlag.cs +++ b/ValveKeyValue/ValveKeyValue/KVFlag.cs @@ -2,15 +2,13 @@ namespace ValveKeyValue { - [Flags] public enum KVFlag { None = 0, Resource = 1, ResourceName = 2, - MultilineString = 4, - Panorama = 8, - SoundEvent = 16, - SubClass = 32, + Panorama = 3, + SoundEvent = 4, + SubClass = 5, } } From 5d45a35c9eb0b4a29564022f51c0737ed68f725d Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 9 Feb 2024 21:42:39 +0200 Subject: [PATCH 48/49] Rebase --- .gitattributes | 4 -- .../Test Data/TextKV3/.gitattributes | 3 ++ .../Test Data/apisurface.txt | 46 +++++++++++++++++++ .../TextKV3/BasicKV3TestCases.cs | 8 ++-- .../TextKV3/HeadersTestCase.cs | 3 -- .../TextKV3/Kv1ToKv3TestCase.cs | 4 -- .../TextKV3/Kv3ToKv1TestCase.cs | 3 -- .../TextKV3/RootTypesTestCase.cs | 4 +- .../TextKV3/SerializationTestCase.cs | 3 -- .../KeyValues3/KV3TextReader.cs | 4 +- .../KeyValues3/KV3TextReaderState.cs | 2 +- .../KeyValues3/KV3TextReaderStateMachine.cs | 2 - .../KeyValues3/KV3TokenReader.cs | 3 -- ValveKeyValue/ValveKeyValue/KVDocument.cs | 6 ++- ValveKeyValue/ValveKeyValue/KVFile.cs | 14 ------ ValveKeyValue/ValveKeyValue/KVFlag.cs | 2 - ValveKeyValue/ValveKeyValue/KVHeader.cs | 2 - .../ValveKeyValue/KeyValues3/Encoding.cs | 2 - .../ValveKeyValue/KeyValues3/Format.cs | 2 - .../KeyValues1/KV1TextSerializer.cs | 2 +- .../KeyValues3/KV3TextSerializer.cs | 23 +++++----- ValveKeyValue/ValveKeyValue/Utils.cs | 42 ----------------- 22 files changed, 72 insertions(+), 112 deletions(-) create mode 100644 ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/.gitattributes delete mode 100644 ValveKeyValue/ValveKeyValue/KVFile.cs delete mode 100644 ValveKeyValue/ValveKeyValue/Utils.cs diff --git a/.gitattributes b/.gitattributes index b4b74233..2ffde598 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,2 @@ * text=auto *.cs diff=csharp - -# Keep intended line endings to test parser -*_crlf.kv3 eol=crlf -*_lf.kv3 eol=lf diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/.gitattributes b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/.gitattributes new file mode 100644 index 00000000..317b9fcd --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/TextKV3/.gitattributes @@ -0,0 +1,3 @@ +# Keep intended line endings to test parser +*.kv3 eol=lf +*_crlf.kv3 eol=crlf diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt b/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt index f2196c6f..87654957 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt @@ -39,6 +39,7 @@ public class ValveKeyValue.KVArrayValue public bool Equals(object obj); protected void Finalize(); public int get_Count(); + public ValveKeyValue.KVFlag get_Flag(); public bool get_IsReadOnly(); public ValveKeyValue.KVValue get_Item(int key); public ValveKeyValue.KVValue get_Item(string key); @@ -52,6 +53,7 @@ public class ValveKeyValue.KVArrayValue protected object MemberwiseClone(); public bool Remove(ValveKeyValue.KVValue item); public void RemoveAt(int index); + public void set_Flag(ValveKeyValue.KVFlag value); public void set_Item(int key, ValveKeyValue.KVValue value); public bool ToBoolean(IFormatProvider provider); public byte ToByte(IFormatProvider provider); @@ -77,12 +79,14 @@ public class ValveKeyValue.KVBinaryBlob public bool Equals(object obj); protected void Finalize(); public Memory`1[[byte]] get_Bytes(); + public ValveKeyValue.KVFlag get_Flag(); public ValveKeyValue.KVValue get_Item(string key); public ValveKeyValue.KVValueType get_ValueType(); public int GetHashCode(); public Type GetType(); public TypeCode GetTypeCode(); protected object MemberwiseClone(); + public void set_Flag(ValveKeyValue.KVFlag value); public bool ToBoolean(IFormatProvider provider); public byte ToByte(IFormatProvider provider); public char ToChar(IFormatProvider provider); @@ -108,6 +112,8 @@ public class ValveKeyValue.KVDocument public bool Equals(object obj); protected void Finalize(); public System.Collections.Generic.IEnumerable`1[[ValveKeyValue.KVObject]] get_Children(); + public System.Collections.Generic.IEnumerable`1[[ValveKeyValue.KVValue]] get_ChildrenValues(); + public ValveKeyValue.KVHeader get_Header(); public ValveKeyValue.KVValue get_Item(string key); public string get_Name(); public ValveKeyValue.KVValue get_Value(); @@ -119,6 +125,43 @@ public class ValveKeyValue.KVDocument public string ToString(); } +public sealed enum ValveKeyValue.KVFlag +{ + None = 0; + Resource = 1; + ResourceName = 2; + Panorama = 3; + SoundEvent = 4; + SubClass = 5; + + public int CompareTo(object target); + public bool Equals(object obj); + protected void Finalize(); + public int GetHashCode(); + public Type GetType(); + public TypeCode GetTypeCode(); + public bool HasFlag(Enum flag); + protected object MemberwiseClone(); + public string ToString(); + public string ToString(IFormatProvider provider); + public string ToString(string format); + public string ToString(string format, IFormatProvider provider); +} + +public class ValveKeyValue.KVHeader +{ + public bool Equals(object obj); + protected void Finalize(); + public Guid get_Encoding(); + public Guid get_Format(); + public int GetHashCode(); + public Type GetType(); + protected object MemberwiseClone(); + public void set_Encoding(Guid value); + public void set_Format(Guid value); + public string ToString(); +} + public sealed class ValveKeyValue.KVIgnoreAttribute { public bool Equals(object obj); @@ -138,6 +181,7 @@ public class ValveKeyValue.KVObject public bool Equals(object obj); protected void Finalize(); public System.Collections.Generic.IEnumerable`1[[ValveKeyValue.KVObject]] get_Children(); + public System.Collections.Generic.IEnumerable`1[[ValveKeyValue.KVValue]] get_ChildrenValues(); public ValveKeyValue.KVValue get_Item(string key); public string get_Name(); public ValveKeyValue.KVValue get_Value(); @@ -221,6 +265,7 @@ public class ValveKeyValue.KVValue { public bool Equals(object obj); protected void Finalize(); + public ValveKeyValue.KVFlag get_Flag(); public ValveKeyValue.KVValue get_Item(string key); public ValveKeyValue.KVValueType get_ValueType(); public int GetHashCode(); @@ -249,6 +294,7 @@ public class ValveKeyValue.KVValue public static ValveKeyValue.KVValue op_Implicit(long value); public static ValveKeyValue.KVValue op_Implicit(string value); public static ValveKeyValue.KVValue op_Implicit(ulong value); + public void set_Flag(ValveKeyValue.KVFlag value); public bool ToBoolean(IFormatProvider provider); public byte ToByte(IFormatProvider provider); public char ToChar(IFormatProvider provider); diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs index aaf3cebd..8f51a7cd 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/BasicKV3TestCases.cs @@ -1,5 +1,3 @@ -using NUnit.Framework; - namespace ValveKeyValue.Test.TextKV3 { class BasicKV3TestCases @@ -27,7 +25,7 @@ public void DeserializesFlaggedValues() Assert.That(data["bar"].Flag, Is.EqualTo(KVFlag.Resource)); Assert.That((string)data["bar"], Is.EqualTo("foo")); - Assert.That(data["multipleFlags"].Flag, Is.EqualTo(KVFlag.Resource | KVFlag.ResourceName | KVFlag.SubClass)); + Assert.That(data["multipleFlags"].Flag, Is.EqualTo(KVFlag.SubClass)); Assert.That((string)data["multipleFlags"], Is.EqualTo("cool value")); Assert.That(data["flaggedNumber"].Flag, Is.EqualTo(KVFlag.Panorama)); @@ -97,7 +95,7 @@ public void DeserializesArray() var arrayValue = (KVArrayValue)data["arrayValue"]; - Assert.That(arrayValue.Count, Is.EqualTo(2)); + Assert.That(arrayValue, Has.Count.EqualTo(2)); Assert.That(arrayValue[0].ToString(), Is.EqualTo("a")); Assert.That(arrayValue[1].ToString(), Is.EqualTo("b")); @@ -111,7 +109,7 @@ public void DeserializesBinaryBlob() var data = KVSerializer.Create(KVSerializationFormat.KeyValues3Text).Deserialize(stream); Assert.That(data["array"].ValueType, Is.EqualTo(KVValueType.BinaryBlob)); - Assert.That(((KVBinaryBlob)data["array"]).Bytes, Is.EqualTo(new byte[] + Assert.That(((KVBinaryBlob)data["array"]).Bytes.ToArray(), Is.EqualTo(new byte[] { 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xFF diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs index 3e398dc2..4c715620 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/HeadersTestCase.cs @@ -1,7 +1,4 @@ -using System; -using System.IO; using System.Text; -using NUnit.Framework; namespace ValveKeyValue.Test.TextKV3 { diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv1ToKv3TestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv1ToKv3TestCase.cs index e6037eb6..5eba97b6 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv1ToKv3TestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv1ToKv3TestCase.cs @@ -1,7 +1,3 @@ -using System; -using System.IO; -using NUnit.Framework; - namespace ValveKeyValue.Test.TextKV3 { class Kv1ToKv3TestCase diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv3ToKv1TestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv3ToKv1TestCase.cs index 7be5493d..3ede1317 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv3ToKv1TestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/Kv3ToKv1TestCase.cs @@ -1,6 +1,3 @@ -using System.IO; -using NUnit.Framework; - namespace ValveKeyValue.Test.TextKV3 { class Kv3ToKv1TestCase diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/RootTypesTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/RootTypesTestCase.cs index e685b105..b27bdcbb 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/RootTypesTestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/RootTypesTestCase.cs @@ -1,5 +1,3 @@ -using NUnit.Framework; - namespace ValveKeyValue.Test.TextKV3 { class RootTypesTestCase @@ -86,7 +84,7 @@ public void DeserializesRootBinaryBlob() { Assert.That(data.Name, Is.EqualTo("root")); Assert.That(data.Value.ValueType, Is.EqualTo(KVValueType.BinaryBlob)); - Assert.That(((KVBinaryBlob)data.Value).Bytes, Is.EqualTo(new byte[] + Assert.That(((KVBinaryBlob)data.Value).Bytes.ToArray(), Is.EqualTo(new byte[] { 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xFF diff --git a/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs index 69dff8e5..71ac6899 100644 --- a/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/TextKV3/SerializationTestCase.cs @@ -1,6 +1,3 @@ -using System.IO; -using NUnit.Framework; - namespace ValveKeyValue.Test.TextKV3 { class SerializationTestCase diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs index ea4f34fa..a59a0f23 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReader.cs @@ -1,6 +1,4 @@ -using System; using System.Globalization; -using System.IO; using ValveKeyValue.Abstraction; namespace ValveKeyValue.Deserialization.KeyValues3 @@ -183,7 +181,7 @@ void ReadText(string text) void ReadBinaryBlob(string text) { - var bytes = Utils.ParseHexStringAsByteArrayNoReverse(text); + var bytes = HexStringHelper.ParseHexStringAsByteArray(text); var value = new KVBinaryBlob(bytes) { Flag = stateMachine.GetAndResetFlag() diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs index dae406f7..b4a0a685 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderState.cs @@ -1,4 +1,4 @@ -namespace ValveKeyValue.Deserialization.KeyValues3 +namespace ValveKeyValue.Deserialization.KeyValues3 { enum KV3TextReaderState { diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs index 127ec45d..b233dc88 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TextReaderStateMachine.cs @@ -1,5 +1,3 @@ -using System.Collections.Generic; - namespace ValveKeyValue.Deserialization.KeyValues3 { class KV3TextReaderStateMachine diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs index 7a6124c0..91625ced 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues3/KV3TokenReader.cs @@ -1,6 +1,3 @@ -using System; -using System.Collections.Generic; -using System.IO; using System.Linq; using System.Text; using ValveKeyValue.KeyValues3; diff --git a/ValveKeyValue/ValveKeyValue/KVDocument.cs b/ValveKeyValue/ValveKeyValue/KVDocument.cs index aebd995d..2417ddfa 100644 --- a/ValveKeyValue/ValveKeyValue/KVDocument.cs +++ b/ValveKeyValue/ValveKeyValue/KVDocument.cs @@ -2,9 +2,11 @@ namespace ValveKeyValue { public class KVDocument : KVObject { - public KVDocument(string name, KVValue value) : base(name, value) + public KVHeader Header { get; } + + public KVDocument(KVHeader header, string name, KVValue value) : base(name, value) { - // KV3 will require a header field that contains format/encoding here. + Header = header; } } } diff --git a/ValveKeyValue/ValveKeyValue/KVFile.cs b/ValveKeyValue/ValveKeyValue/KVFile.cs deleted file mode 100644 index bae07fc6..00000000 --- a/ValveKeyValue/ValveKeyValue/KVFile.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; - -namespace ValveKeyValue -{ - public class KVFile : KVObject - { - public KVHeader Header { get; } - - public KVFile(KVHeader header, string name, KVValue value) : base(name, value) - { - Header = header; - } - } -} diff --git a/ValveKeyValue/ValveKeyValue/KVFlag.cs b/ValveKeyValue/ValveKeyValue/KVFlag.cs index 25e20c31..eea986e0 100644 --- a/ValveKeyValue/ValveKeyValue/KVFlag.cs +++ b/ValveKeyValue/ValveKeyValue/KVFlag.cs @@ -1,5 +1,3 @@ -using System; - namespace ValveKeyValue { public enum KVFlag diff --git a/ValveKeyValue/ValveKeyValue/KVHeader.cs b/ValveKeyValue/ValveKeyValue/KVHeader.cs index 6f02091a..17ab54aa 100644 --- a/ValveKeyValue/ValveKeyValue/KVHeader.cs +++ b/ValveKeyValue/ValveKeyValue/KVHeader.cs @@ -1,5 +1,3 @@ -using System; - namespace ValveKeyValue { public class KVHeader diff --git a/ValveKeyValue/ValveKeyValue/KeyValues3/Encoding.cs b/ValveKeyValue/ValveKeyValue/KeyValues3/Encoding.cs index 3bfac429..d2f6b21b 100644 --- a/ValveKeyValue/ValveKeyValue/KeyValues3/Encoding.cs +++ b/ValveKeyValue/ValveKeyValue/KeyValues3/Encoding.cs @@ -1,5 +1,3 @@ -using System; - namespace ValveKeyValue.KeyValues3 { public class Encoding diff --git a/ValveKeyValue/ValveKeyValue/KeyValues3/Format.cs b/ValveKeyValue/ValveKeyValue/KeyValues3/Format.cs index 017efd84..b2c6ebb2 100644 --- a/ValveKeyValue/ValveKeyValue/KeyValues3/Format.cs +++ b/ValveKeyValue/ValveKeyValue/KeyValues3/Format.cs @@ -1,5 +1,3 @@ -using System; - namespace ValveKeyValue.KeyValues3 { public class Format diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs index 31f736a8..57a1bfe3 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1TextSerializer.cs @@ -21,7 +21,7 @@ public KV1TextSerializer(Stream stream, KVSerializerOptions options) readonly KVSerializerOptions options; readonly TextWriter writer; int indentation = 0; - Stack arrayCount = new(); + readonly Stack arrayCount = new(); public void Dispose() { diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs index 715f8bea..db6359bc 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues3/KV3TextSerializer.cs @@ -1,6 +1,3 @@ -using System; -using System.Collections.Generic; -using System.IO; using System.Text; using ValveKeyValue.Abstraction; @@ -23,7 +20,7 @@ public KV3TextSerializer(Stream stream) readonly TextWriter writer; int indentation = 0; - Stack inArray = new(); + readonly Stack inArray = new(); bool IsInArray => inArray.Count > 0 && inArray.Peek(); @@ -171,8 +168,10 @@ void WriteValue(KVValue value) void WriteBinaryBlob(KVBinaryBlob value) { + var bytes = value.Bytes.Span; + // TODO: Verify this against Valve - if (value.Bytes.Length > 32) + if (bytes.Length > 32) { writer.WriteLine(); WriteIndentation(); @@ -184,18 +183,20 @@ void WriteBinaryBlob(KVBinaryBlob value) indentation++; WriteIndentation(); - var count = 0; + var i = 0; - foreach (var oneByte in value.Bytes) + for (; i < bytes.Length; i++) { - writer.Write(oneByte.ToString("X2")); + var b = bytes[i]; + writer.Write(HexStringHelper.HexToCharUpper(b >> 4)); + writer.Write(HexStringHelper.HexToCharUpper(b)); - if (++count % 32 == 0) + if (i > 0 && i % 32 == 0) { writer.WriteLine(); WriteIndentation(); } - else if (count != value.Bytes.Length) + else if (i != bytes.Length - 1) { writer.Write(' '); } @@ -203,7 +204,7 @@ void WriteBinaryBlob(KVBinaryBlob value) indentation--; - if (count % 32 != 0) + if (i % 32 != 0) { writer.WriteLine(); WriteIndentation(); diff --git a/ValveKeyValue/ValveKeyValue/Utils.cs b/ValveKeyValue/ValveKeyValue/Utils.cs deleted file mode 100644 index 5d4f8282..00000000 --- a/ValveKeyValue/ValveKeyValue/Utils.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.Globalization; - -namespace ValveKeyValue -{ - internal class Utils - { - // TODO: Need to figure out whether Array.Reverse is actually correct for kv1 - public static byte[] ParseHexStringAsByteArrayNoReverse(string hexadecimalRepresentation) - { - Require.NotNull(hexadecimalRepresentation, nameof(hexadecimalRepresentation)); - - var data = new byte[hexadecimalRepresentation.Length / 2]; - for (var i = 0; i < data.Length; i++) - { - var currentByteText = hexadecimalRepresentation.Substring(i * 2, 2); - data[i] = byte.Parse(currentByteText, NumberStyles.HexNumber, CultureInfo.InvariantCulture); - } - - return data; - } - - public static byte[] ParseHexStringAsByteArray(string hexadecimalRepresentation) - { - Require.NotNull(hexadecimalRepresentation, nameof(hexadecimalRepresentation)); - - var data = new byte[hexadecimalRepresentation.Length / 2]; - for (var i = 0; i < data.Length; i++) - { - var currentByteText = hexadecimalRepresentation.Substring(i * 2, 2); - data[i] = byte.Parse(currentByteText, NumberStyles.HexNumber, CultureInfo.InvariantCulture); - } - - if (BitConverter.IsLittleEndian) - { - Array.Reverse(data); - } - - return data; - } - } -} From 196429d12e58050d518daf109cdd6a52b86e84fb Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Sat, 5 Apr 2025 23:01:20 +0300 Subject: [PATCH 49/49] Fix --- ValveKeyValue/ValveKeyValue.Test/KVValueToStringTestCase.cs | 2 +- ValveKeyValue/ValveKeyValue/KVSerializer.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ValveKeyValue/ValveKeyValue.Test/KVValueToStringTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/KVValueToStringTestCase.cs index dbb0f1f4..d10398ba 100644 --- a/ValveKeyValue/ValveKeyValue.Test/KVValueToStringTestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/KVValueToStringTestCase.cs @@ -14,7 +14,7 @@ public static IEnumerable ToStringTestCases { yield return new TestCaseData(new KVObject("a", "blah").Value).Returns("blah"); yield return new TestCaseData(new KVObject("a", "yay").Value).Returns("yay"); - yield return new TestCaseData(new KVObject("a", []).Value).Returns("[Collection]").SetName("{m} - Empty Collection"); + yield return new TestCaseData(new KVObject("a", Enumerable.Empty()).Value).Returns("[Collection]").SetName("{m} - Empty Collection"); yield return new TestCaseData(new KVObject("a", [new KVObject("boo", "aah")]).Value).Returns("[Collection]").SetName("{m} - Collection With Value"); } } diff --git a/ValveKeyValue/ValveKeyValue/KVSerializer.cs b/ValveKeyValue/ValveKeyValue/KVSerializer.cs index a8fe6d24..77fb4569 100644 --- a/ValveKeyValue/ValveKeyValue/KVSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/KVSerializer.cs @@ -117,7 +117,7 @@ IVisitingReader MakeReader(Stream stream, IParsingVisitationListener listener, K { KVSerializationFormat.KeyValues1Text => new KV1TextReader(new StreamReader(stream, null, true, -1, leaveOpen: true), listener, options), KVSerializationFormat.KeyValues1Binary => new KV1BinaryReader(stream, listener, options.StringTable), - KVSerializationFormat.KeyValues3Text => new KV3TextReader(new StreamReader(stream, null, true, -1, leaveOpen: true), listener, options), + KVSerializationFormat.KeyValues3Text => new KV3TextReader(new StreamReader(stream, null, true, -1, leaveOpen: true), listener), _ => throw new ArgumentOutOfRangeException(nameof(format), format, "Invalid serialization format."), }; }