Skip to content

Commit 4b6fc58

Browse files
committed
Fix filtering by Timestamp
We need to fixup datetime filters by prefixing them with the conversion operator `datetime'[VALUE]'` as expected by OData. This seems to be a bug in the DataServiceContext we use to create the query and URL. The fix covers both document and table repositories. Fixes #328
1 parent ed8c7fd commit 4b6fc58

13 files changed

Lines changed: 392 additions & 11 deletions

readme.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ The `DocumentRepository.Create` and `DocumentPartition.Create` factory methods p
147147
to document-based storage, exposing the a similar API as column-based storage.
148148

149149
Document repositories cause entities to be persisted as a single document column, alongside type and version
150-
information to handle versioning a the app level as needed.
150+
information to handle versioning at the app level as needed.
151151

152152
The API is mostly the same as for column-based repositories (document repositories implement
153153
the same underlying `ITableStorage` interface):
@@ -201,6 +201,9 @@ The `Type` column persisted in the documents table is the `Type.FullName` of the
201201
The major and minor version components are also provided as individual columns for easier querying
202202
by various version ranges, using `IDocumentRepository.EnumerateAsync(predicate)`.
203203

204+
If the serialized documents need to access the `Timestamp` managed by Azure Table
205+
Storage, you can implement `IDocumentTimestamp` in your entity type.
206+
204207
<!-- #documents -->
205208

206209
In addition to the default built-in JSON plain-text based serializer (which uses the
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
using System;
2+
using System.Runtime.Serialization;
3+
4+
namespace ProtobufNetTest
5+
{
6+
[DataContract(Name = nameof(DateTimeOffset))]
7+
public class DateTimeOffsetSurrogate
8+
{
9+
[DataMember(Order = 1)]
10+
public long? Value { get; set; }
11+
12+
public static implicit operator DateTimeOffset(DateTimeOffsetSurrogate surrogate)
13+
{
14+
return DateTimeOffset.FromUnixTimeMilliseconds(surrogate.Value.GetValueOrDefault());
15+
}
16+
17+
public static implicit operator DateTimeOffset?(DateTimeOffsetSurrogate surrogate)
18+
{
19+
return surrogate != null ? DateTimeOffset.FromUnixTimeMilliseconds(surrogate.Value.GetValueOrDefault()) : null;
20+
}
21+
22+
public static implicit operator DateTimeOffsetSurrogate(DateTimeOffset source)
23+
{
24+
return new DateTimeOffsetSurrogate
25+
{
26+
Value = source.ToUnixTimeMilliseconds()
27+
};
28+
}
29+
30+
public static implicit operator DateTimeOffsetSurrogate(DateTimeOffset? source)
31+
{
32+
return new DateTimeOffsetSurrogate
33+
{
34+
Value = source?.ToUnixTimeMilliseconds()
35+
};
36+
}
37+
}
38+
}

src/TableStorage.Protobuf/ProtobufDocumentSerializer.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System.IO;
55
using ProtoBuf;
66
using ProtoBuf.Meta;
7+
using ProtobufNetTest;
78

89
namespace Devlooped
910
{
@@ -18,6 +19,12 @@ partial class ProtobufDocumentSerializer : IBinaryDocumentSerializer
1819
/// </summary>
1920
public static IDocumentSerializer Default { get; } = new ProtobufDocumentSerializer();
2021

22+
static ProtobufDocumentSerializer()
23+
{
24+
RuntimeTypeModel.Default.Add(typeof(DateTimeOffset), false).SetSurrogate(typeof(DateTimeOffsetSurrogate));
25+
RuntimeTypeModel.Default.Add(typeof(DateTimeOffset?), false).SetSurrogate(typeof(DateTimeOffsetSurrogate));
26+
}
27+
2128
/// <inheritdoc />
2229
public T? Deserialize<T>(byte[] data)
2330
{

src/TableStorage.Source/TableStorage.Source.csproj

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,7 @@
1414
<PackageReference Include="Devlooped.CloudStorageAccount" Version="1.3.0" />
1515
<PackageReference Include="Azure.Data.Tables" Version="12.10.0" />
1616
<PackageReference Include="Microsoft.OData.Client" Version="7.17.0" />
17-
<PackageReference Include="PolySharp" Version="1.15.0">
18-
<PrivateAssets>all</PrivateAssets>
19-
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
20-
</PackageReference>
17+
<PackageReference Include="PolySharp" Version="1.15.0" PrivateAssets="all" />
2118
<PackageReference Include="System.Text.Json" Version="6.0.10" />
2219
</ItemGroup>
2320

src/TableStorage/DocumentRepository`1.cs

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,12 @@ async IAsyncEnumerable<T> EnumerateBinaryAsync(Func<IQueryable<IDocumentEntity>,
152152
{
153153
var value = binarySerializer!.Deserialize<T>(document);
154154
if (value != null)
155+
{
156+
if (value is IDocumentTimestamp ts)
157+
ts.Timestamp = entity.Timestamp;
158+
155159
yield return value;
160+
}
156161
}
157162
}
158163
}
@@ -169,8 +174,11 @@ async IAsyncEnumerable<T> EnumerateBinaryAsync(Func<IQueryable<IDocumentEntity>,
169174
if (document == null)
170175
return default;
171176

172-
return binarySerializer!.Deserialize<T>(document);
177+
var value = binarySerializer!.Deserialize<T>(document);
178+
if (value is IDocumentTimestamp ts)
179+
ts.Timestamp = result.Value.Timestamp;
173180

181+
return value;
174182
}
175183
catch (RequestFailedException ex) when (ex.Status == 404)
176184
{
@@ -239,7 +247,11 @@ async IAsyncEnumerable<T> EnumerateStringAsync(Func<IQueryable<IDocumentEntity>,
239247
{
240248
var value = stringSerializer!.Deserialize<T>(data);
241249
if (value != null)
250+
{
251+
if (value is IDocumentTimestamp ts)
252+
ts.Timestamp = document.Timestamp;
242253
yield return value;
254+
}
243255
}
244256
}
245257
}
@@ -256,7 +268,11 @@ async IAsyncEnumerable<T> EnumerateStringAsync(Func<IQueryable<IDocumentEntity>,
256268
if (document == null)
257269
return default;
258270

259-
return stringSerializer!.Deserialize<T>(document);
271+
var value = stringSerializer!.Deserialize<T>(document);
272+
if (value is IDocumentTimestamp ts)
273+
ts.Timestamp = result.Value.Timestamp;
274+
275+
return value;
260276
}
261277
catch (RequestFailedException ex) when (ex.Status == 404)
262278
{
Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
//<auto-generated/>
2+
#nullable enable
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Globalization;
6+
using System.Linq;
7+
using System.Text.RegularExpressions;
8+
using Microsoft.OData.Edm;
9+
using Microsoft.OData.UriParser;
10+
using Microsoft.OData.UriParser.Aggregation;
11+
12+
namespace Devlooped;
13+
14+
/// <summary>
15+
/// Helper class that fixes filter expressions containing DateTimeOffset comparisons.
16+
/// </summary>
17+
internal static class FilterExpressionFixer
18+
{
19+
/// <summary>
20+
/// Fixes filter expressions involving DateTimeOffset by converting them to the format expected by Azure Table Storage.
21+
/// </summary>
22+
/// <param name="filter">The filter expression to fix.</param>
23+
/// <returns>A fixed filter expression where DateTimeOffset comparisons are properly formatted.</returns>
24+
public static string? Fix(string? filter)
25+
{
26+
if (string.IsNullOrEmpty(filter))
27+
return filter;
28+
29+
var parser = new UriQueryExpressionParser(int.MaxValue);
30+
var node = parser.ParseFilter(filter);
31+
32+
if (node == null)
33+
return filter;
34+
35+
try
36+
{
37+
return node.Accept(new QueryTokenToFilterStringVisitor());
38+
}
39+
catch (NotSupportedException)
40+
{
41+
return filter;
42+
}
43+
}
44+
45+
class QueryTokenToFilterStringVisitor : ISyntacticTreeVisitor<string>
46+
{
47+
public string Visit(BinaryOperatorToken token)
48+
{
49+
string left = token.Left.Accept(this);
50+
string right = token.Right.Accept(this);
51+
string op = ToOperatorString(token.OperatorKind);
52+
return $"{left} {op} {right}";
53+
}
54+
55+
public string Visit(UnaryOperatorToken token)
56+
{
57+
string operand = token.Operand.Accept(this);
58+
string op = ToOperatorString(token.OperatorKind);
59+
return $"{op} {operand}";
60+
}
61+
62+
public string Visit(LiteralToken token)
63+
{
64+
// You may want to handle different literal types more carefully
65+
if (token.Value is string s)
66+
return $"'{s.Replace("'", "''")}'";
67+
if (token.Value is bool b)
68+
return b ? "true" : "false";
69+
if (token.Value is DateTimeOffset dto)
70+
return $"datetime'{dto.UtcDateTime.ToString("o", CultureInfo.InvariantCulture)}'";
71+
if (token.Value is DateTime dt)
72+
return $"datetime'{dt.ToUniversalTime().ToString("o", CultureInfo.InvariantCulture)}'";
73+
if (token.Value is Guid g)
74+
return $"guid'{g.ToString()}'";
75+
if (token.Value == null)
76+
return "null";
77+
78+
return Convert.ToString(token.Value, CultureInfo.InvariantCulture)!;
79+
}
80+
81+
public string Visit(EndPathToken token)
82+
{
83+
// Handles property/field access
84+
if (token.NextToken != null)
85+
return $"{token.Identifier}/{token.NextToken.Accept(this)}";
86+
return token.Identifier;
87+
}
88+
89+
public string Visit(DottedIdentifierToken token)
90+
{
91+
if (token.NextToken != null)
92+
return $"{token.Identifier}/{token.NextToken.Accept(this)}";
93+
return token.Identifier;
94+
}
95+
96+
public string Visit(FunctionCallToken token)
97+
{
98+
var args = token.Arguments?.Select(a => a.Accept(this)).ToArray() ?? Array.Empty<string>();
99+
string source = token.Source != null ? token.Source.Accept(this) + "/" : "";
100+
return $"{source}{token.Name}({string.Join(",", args)})";
101+
}
102+
103+
public string Visit(FunctionParameterToken token) => token.ValueToken.Accept(this);
104+
105+
public string Visit(AnyToken token)
106+
{
107+
var parent = token.Parent?.Accept(this);
108+
var expr = token.Expression?.Accept(this);
109+
return $"{parent}/any({token.Parameter}:{expr})";
110+
}
111+
112+
public string Visit(AllToken token)
113+
{
114+
var parent = token.Parent?.Accept(this);
115+
var expr = token.Expression?.Accept(this);
116+
return $"{parent}/all({token.Parameter}:{expr})";
117+
}
118+
119+
public string Visit(InToken token)
120+
{
121+
string left = token.Left.Accept(this);
122+
string right = token.Right.Accept(this);
123+
return $"{left} in {right}";
124+
}
125+
126+
// Add more Visit methods for other token types as needed...
127+
128+
// Helper to convert operator enums to OData string
129+
static string ToOperatorString(BinaryOperatorKind op)
130+
{
131+
return op switch
132+
{
133+
BinaryOperatorKind.And => "and",
134+
BinaryOperatorKind.Or => "or",
135+
BinaryOperatorKind.Equal => "eq",
136+
BinaryOperatorKind.NotEqual => "ne",
137+
BinaryOperatorKind.GreaterThan => "gt",
138+
BinaryOperatorKind.GreaterThanOrEqual => "ge",
139+
BinaryOperatorKind.LessThan => "lt",
140+
BinaryOperatorKind.LessThanOrEqual => "le",
141+
_ => throw new NotSupportedException($"Operator {op} not supported")
142+
};
143+
}
144+
145+
static string ToOperatorString(UnaryOperatorKind op)
146+
{
147+
return op switch
148+
{
149+
UnaryOperatorKind.Not => "not",
150+
_ => throw new NotSupportedException($"Operator {op} not supported")
151+
};
152+
}
153+
154+
// Implement stubs for other ISyntacticTreeVisitor<string> methods
155+
public string Visit(CustomQueryOptionToken token) => throw new NotSupportedException();
156+
public string Visit(AggregateExpressionToken token) => throw new NotSupportedException();
157+
public string Visit(CountSegmentToken tokenIn) => throw new NotImplementedException();
158+
public string Visit(ExpandToken tokenIn) => throw new NotImplementedException();
159+
public string Visit(ExpandTermToken tokenIn) => throw new NotImplementedException();
160+
public string Visit(LambdaToken tokenIn) => throw new NotImplementedException();
161+
public string Visit(InnerPathToken tokenIn) => throw new NotImplementedException();
162+
public string Visit(OrderByToken tokenIn) => throw new NotImplementedException();
163+
public string Visit(RangeVariableToken tokenIn) => throw new NotImplementedException();
164+
public string Visit(SelectToken tokenIn) => throw new NotImplementedException();
165+
public string Visit(SelectTermToken tokenIn) => throw new NotImplementedException();
166+
public string Visit(StarToken tokenIn) => throw new NotImplementedException();
167+
public string Visit(AggregateToken tokenIn) => throw new NotImplementedException();
168+
public string Visit(EntitySetAggregateToken tokenIn) => throw new NotImplementedException();
169+
public string Visit(GroupByToken tokenIn) => throw new NotImplementedException();
170+
// ...and so on for all required token types
171+
}
172+
}
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
using System;
2+
3+
namespace Devlooped;
4+
5+
/// <summary>
6+
/// Opts-in to receiving the document timestamp from the <see cref="IDocumentRepository{T}"/>
7+
/// when retrieving the document from the table storage.
8+
/// </summary>
9+
public interface IDocumentTimestamp
10+
{
11+
/// <summary>
12+
/// The timestamp of the document.
13+
/// </summary>
14+
DateTimeOffset? Timestamp { get; set; }
15+
}

src/TableStorage/TableRepositoryQuery`1.cs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ public TableRepositoryQuery(CloudStorageAccount account, IStringDocumentSerializ
4646

4747
public async IAsyncEnumerator<T> GetAsyncEnumerator(CancellationToken cancellation = default)
4848
{
49-
var query = (DataServiceQuery)new DataServiceContext(account.TableEndpoint).CreateQuery<T>(tableName)
49+
var context = new DataServiceContext(account.TableEndpoint);
50+
var query = (DataServiceQuery)context.CreateQuery<T>(tableName)
5051
.Provider.CreateQuery(expression);
5152

5253
// OData will translate the enum value in a filter to TYPENAME.'ENUMVALUE'.
@@ -115,6 +116,9 @@ public async IAsyncEnumerator<T> GetAsyncEnumerator(CancellationToken cancellati
115116
qs["$filter"] = filter;
116117
}
117118

119+
// Fix DateTimeOffset filters to ensure proper handling
120+
qs["$filter"] = FilterExpressionFixer.Fix(qs["$filter"]);
121+
118122
var builder = new UriBuilder(query.RequestUri)
119123
{
120124
Query = string.Join("&", qs.AllKeys.Select(x => $"{x}={qs[x]}"))

src/TableStorage/TableStorage.csproj

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
44
<AssemblyName>Devlooped.TableStorage</AssemblyName>
@@ -16,6 +16,11 @@
1616
<PackageReference Include="System.Text.Json" Version="6.0.10" />
1717
</ItemGroup>
1818

19+
<ItemGroup Condition="'$(TargetFramework)' == 'net8.0'">
20+
<PackageReference Update="Azure.Data.Tables" Version="12.11.0" />
21+
<PackageReference Update="Microsoft.OData.Client" Version="8.2.3" />
22+
</ItemGroup>
23+
1924
<ItemGroup>
2025
<InternalsVisibleTo Include="Tests" />
2126
<Compile Remove="System\*.cs" Condition="'$(TargetFramework)' == 'net6.0' or '$(TargetFramework)' == 'net8.0'" />

0 commit comments

Comments
 (0)