Skip to content

Implement JSON Path parser and related routines #9062

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Noremos wants to merge 7 commits into
base: master
Choose a base branch
from
Open

Implement JSON Path parser and related routines #9062

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c913906
Implement JSON Path parser and related routines
Jun 15, 2026
71b0339
Do not run locale-depended JSON unit tests
Jun 15, 2026
554244c
Correct JSON files in engine vcxproj
Jun 15, 2026
5a1c3d6
Copy jpath-parse-conflicts.txt from temp to src
Jun 15, 2026
cc27f99
Allow fail when coping jpath-parse-conflicts.txt to src/jrd/json/path…
Jun 15, 2026
d740346
Use accurate int types in JSON decimal convert tests
Jun 15, 2026
d3fc83e
Fix incorrect syntax in jpath-parse-conflicts.txt copy
Jun 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions builds/posix/Makefile.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -973,6 +973,7 @@ clean_gpre_gen:

clean_yacc_gen:
$(RM) $(ROOT)/src/dsql/parse.cpp $(ROOT)/src/dsql/dsql.tab.h $(TMP_ROOT)/include/gen/parse.h types.y y.* $(OBJ)/.parse-gen-sentinel
$(RM) $(ROOT)/jrd/json/path/JsonPathParser.cpp $(GEN_ROOT)/jtypes.y $(GEN_ROOT)/json_y.y $(TMP_ROOT)/include/gen/jparse.h jtypes.y y.* $(OBJ)/.jparse-gen-sentinel

#___________________________________________________________________________
# Extra platform specific targets
Expand Down
24 changes: 24 additions & 0 deletions builds/posix/make.shared.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,30 @@ $(OBJ)/.parse-gen-sentinel: $(SRC_ROOT)/dsql/parse.y $(SRC_ROOT)/dsql/btyacc_fb.
touch $@


# This rule creates JsonPathParser.cpp from JsonPathParser.y

$(OBJ)/jrd/json/path/JsonPathParser.cpp $(SRC_ROOT)/include/gen/jparse.h: $(OBJ)/.jparse-gen-sentinel ;

$(OBJ)/.jparse-gen-sentinel: $(SRC_ROOT)/jrd/json/path/JsonPathParser.y $(SRC_ROOT)/jrd/json/path/btyacc_json.ske
sed -n '/%type .*/p' < $< > $(GEN_ROOT)/jtypes.y
sed 's/%type .*//' < $< > $(GEN_ROOT)/json_y.y
($(BTYACC) -l -d -b jpath -S $(SRC_ROOT)/jrd/json/path/btyacc_json.ske $(GEN_ROOT)/json_y.y; echo $$? > $(GEN_ROOT)/json_y.status) 2>&1 | tee $(GEN_ROOT)/json_y.txt
(exit `cat $(GEN_ROOT)/json_y.status`)
sed -n -e "s/.*btyacc: \(.*conflicts.*\)/\1/p" $(GEN_ROOT)/json_y.txt > $(TMP_ROOT)/jpath-parse-conflicts.txt
cp $(TMP_ROOT)/jpath-parse-conflicts.txt $(SRC_ROOT)/jrd/json/path/jpath-parse-conflicts.txt
if [ ! -f $(SRC_ROOT)/jrd/json/path/jpath-parse-conflicts.txt ] || ! cmp -s $(TMP_ROOT)/jpath-parse-conflicts.txt $(SRC_ROOT)/jrd/json/path/jpath-parse-conflicts.txt; then \
cp $(TMP_ROOT)/jpath-parse-conflicts.txt $(SRC_ROOT)/jrd/json/path/jpath-parse-conflicts.txt || true; \
fi
sed -i -e 's/#define \([A-Z].*\)/#define TOK_\1/' $(GEN_ROOT)/jpath_tab.h
sed -i -e 's/#define TOK_YY\(.*\)/#define YY\1/' $(GEN_ROOT)/jpath_tab.h
$(MV) $(GEN_ROOT)/jpath_tab.h $(SRC_ROOT)/include/gen/jparse.h
$(MV) $(GEN_ROOT)/jpath_tab.c $(OBJ)/jrd/json/path/JsonPathParser.cpp
touch $(OBJ)/jrd/json/path/JsonPathParser.cpp

# Explicit dependence on generated header (jparser)
$(OBJ)/jrd/json/path/JsonPath.o $(OBJ)/jrd/json/path/JsonPathParser.o $(OBJ)/jrd/RecordSourceNodes.o $(OBJ)/dsql/ExprNodes.o: $(SRC_ROOT)/include/gen/jparse.h


# gpre_meta needs a special boot build since there is no database.

$(SRC_ROOT)/gpre/gpre_meta.cpp: $(SRC_ROOT)/gpre/gpre_meta.epp
Expand Down
9 changes: 7 additions & 2 deletions builds/posix/make.shared.variables
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,13 +86,18 @@ AllObjects += $(Chacha_Objects)
Profiler_Objects:= $(call dirObjects,plugins/profiler)
AllObjects += $(Profiler_Objects)

JSON_Objects:= $(call dirObjects,jrd/json) \
$(call dirObjects,jrd/json/path) \
$(call dirObjects,jrd/json/classes)

# Engine
Engine_Objects:= $(call dirObjects,jrd) $(call dirObjects,dsql) $(call dirObjects,jrd/extds) \
$(call dirObjects,jrd/optimizer) $(call dirObjects,jrd/recsrc) $(call dirObjects,jrd/replication) \
$(call dirObjects,jrd/sys-packages) $(call dirObjects,jrd/trace) \
$(call makeObjects,lock,lock.cpp)
$(call makeObjects,lock,lock.cpp) $(JSON_Objects)

Engine_Test_Objects:= $(call dirObjects,jrd/tests) $(call dirObjects,lock/tests)
Engine_Test_Objects:= $(call dirObjects,jrd/tests) $(call dirObjects,lock/tests) \
$(call dirObjects,jrd/tests/json) $(call dirObjects,jrd/tests/json/classes)

AllObjects += $(Engine_Objects) $(Engine_Test_Objects)

Expand Down
17 changes: 17 additions & 0 deletions builds/win32/msvc15/engine_static.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,14 @@
<ClCompile Include="..\..\..\src\utilities\gsec\gsec.cpp" />
<ClCompile Include="..\..\..\src\utilities\gstat\ppg.cpp" />
<ClCompile Include="..\..\..\src\utilities\nbackup\nbackup.cpp" />
<ClCompile Include="..\..\..\src\jrd\json\JsonRuntime.cpp" />
<ClCompile Include="..\..\..\src\jrd\json\JsonParser.cpp" />
<ClCompile Include="..\..\..\src\jrd\json\path\JsonPath.cpp" />
<ClCompile Include="..\..\..\src\jrd\json\path\JsonPathParser.cpp" />
<ClCompile Include="..\..\..\src\jrd\json\path\JPathParser.cpp" />
<ClCompile Include="..\..\..\src\jrd\json\classes\JsonTypes.cpp" />
<ClCompile Include="..\..\..\src\jrd\json\classes\JsonScalar.cpp" />
<ClCompile Include="..\..\..\src\jrd\json\classes\JsonDatetime.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\src\dsql\AggNodes.h" />
Expand Down Expand Up @@ -402,6 +410,12 @@
<ClInclude Include="..\..\..\src\jrd\vio_proto.h" />
<ClInclude Include="..\..\..\src\jrd\VirtualTable.h" />
<ClInclude Include="..\..\..\src\jrd\WorkerAttachment.h" />
<ClCompile Include="..\..\..\src\jrd\json\JsonRuntime.h" />
<ClCompile Include="..\..\..\src\jrd\json\path\JsonPath.h" />
<ClCompile Include="..\..\..\src\jrd\json\path\JPathParser.h" />
<ClCompile Include="..\..\..\src\jrd\json\classes\JsonTypes.h" />
<ClCompile Include="..\..\..\src\jrd\json\classes\JsonScalar.h" />
<ClCompile Include="..\..\..\src\jrd\json\classes\JsonDatetime.h" />
</ItemGroup>
<ItemGroup>
<None Include="..\..\..\src\dsql\DdlNodes.epp" />
Expand All @@ -422,6 +436,9 @@
<None Include="..\..\..\src\jrd\SystemTriggers.epp" />
<None Include="..\..\..\src\jrd\Package.epp" />
<None Include="..\..\..\src\utilities\gstat\dba.epp" />
<None Include="..\..\..\src\jrd\json\path\JsonPathParser.y">
<FileType>Document</FileType>
</None>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{B32D1B09-8161-451E-8D20-D30F26094EC0}</ProjectGuid>
Expand Down
21 changes: 21 additions & 0 deletions builds/win32/msvc15/engine_static.vcxproj.filters
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@
<Filter Include="Replication\Headers">
<UniqueIdentifier>{4bcf3dd7-c9c1-4148-bfeb-d5960bf35faa}</UniqueIdentifier>
</Filter>
<Filter Include="JSON">
<UniqueIdentifier>{49f32cf0-a84a-4f19-8efd-58524e855ce2}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\src\jrd\recsrc\WindowedStream.cpp">
Expand Down Expand Up @@ -1214,5 +1217,23 @@
<None Include="..\..\..\src\dsql\parse.y">
<Filter>DSQL</Filter>
</None>
<ClCompile Include="..\..\..\src\jrd\json\JsonPath.cpp">
<Filter>JSON</Filter>
</ClCompile>
<None Include="..\..\..\src\jrd\json\path\JsonPathParser.y">
<Filter>JSON</Filter>
</None>
<ClCompile Include="..\..\..\src\jrd\json\path\JsonPathParser.cpp">
<Filter>JSON</Filter>
</ClCompile>
<ClCompile Include="..\..\..\src\jrd\json\path\JPathParser.cpp">
<Filter>JSON</Filter>
</ClCompile>
<ClCompile Include="..\..\..\src\jrd\json\classes\JsonTypes.cpp">
<Filter>JSON</Filter>
</ClCompile>
<ClCompile Include="..\..\..\src\jrd\json\classes\JsonDatetime.cpp">
<Filter>JSON</Filter>
</ClCompile>
</ItemGroup>
</Project>
23 changes: 23 additions & 0 deletions builds/win32/parse.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,27 @@
@del y_tab.c
@del sed*


@echo Generating JsonPathParser.cpp

@sed -n "/%%type .*/p" < %FB_ROOT_PATH%\src\jrd\json\path\JsonPathParser.y > jtypes.y
@sed "s/%%type .*//" < %FB_ROOT_PATH%\src\jrd\json\path\JsonPathParser.y > json_y.y

%FB_ROOT_PATH%\temp\%FB_OBJ_DIR%\btyacc\btyacc -l -d -b jpath -S %FB_ROOT_PATH%\src\jrd\json\path\btyacc_json.ske json_y.y 2>json_y.txt
@if errorlevel 1 (type json_y.txt && exit /B 1)
@type json_y.txt

@sed -i "s/#define \([A-Z].*\)/#define TOK_\1/" jpath_tab.h
@sed -i "s/#define TOK_YY\(.*\)/#define YY\1/" jpath_tab.h
@sed -n -e "s/.*btyacc: \(.*conflicts.*\)/\1/p" json_y.txt > %FB_ROOT_PATH%\src\dsql\jpath-parse-conflicts.txt

@copy jpath_tab.h %FB_ROOT_PATH%\src\include\gen\jparse.h > nul
@copy jpath_tab.c %FB_ROOT_PATH%\src\jrd\json\path\JsonPathParser.cpp > nul
@del json_y.y
@del json_y.txt
@del jtypes.y
@del jpath_tab.h
@del jpath_tab.c
@del sed*

:END
199 changes: 199 additions & 0 deletions doc/sql.extensions/README.json.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
# JSON

Functional to parse, generate, query and store JSON

## JSON Path

The JSON Path is uses to query data from JSON.
For example, the path `$[*].name` with JSON `[{"name":"John", "score":10}, {"name":"Sam", "score":50}]` will produce the sequence `["John","name"]`.
To filter elements, a filer can be used: `$[*] ? (@.score > 15)`. Child filter is also supported: `$[*] ? (@.score > 15).name`.

The JSON Path has 2 modes: lax (default) and strict. The first one allows to flex path and JSON content matching.
Missing fields and incorrect array ranges are ignored. The strict one produces an error. The full rules:
1) Invalid array range: lax - allowed, strict - error
2) Index of array range: lax - allowed, strict - error
3) Missing field: lax - allowed, strict - error
4) Error inside a JSON Path Filter: lax - hides (the), strict - throw error
5) Unwrapping. lax: for `$.name` with `[{"name":"John"},{"name":"Sam"}]`, the array will be unwrapped. A path will be consider as follow: `$[*].name`. Strict - error
6) Path unwrapping before methods (except `size()` and `type()`) in lax mode
7) Path unwrapping before filters in lax mode
8) Path unwrapping before unary `-` and `+`
9) Path unwrapping for arithmetic operands`
10) Wrapping. lax: for `$[*].name` with `{"name":"John"}`, the object will be wrapped. A path will be consider as follow: `$.name`. Strict - error

The JSON PATH is used in JSON_VALUE, JSON_QUERY, JSON_EXISTS and JSON_TABLE
The full syntax:

```sql
<JSON path expression> ::=
<JSON path mode> <JSON path expr>

<JSON path mode> ::= lax | strict

<JSON path> ::=
'$' [<JSON path accessors>]
| '@' [<JSON path accessors>]
| <scalar>
| <Passing variable> [<JSON path accessors>]
| '(' <JSON path expr> ')'

<Passing variable> ::= '$' <identifier>

<JSON path expr> ::=
[<JSON unary>] <JSON path>
| <JSON path expr> <arithmetic expression> <JSON path expr>
| <JSON path expr> <JSON item method>
| <JSON path expr> <JSON filter expression>

<JSON path accessors> ::= <JSON accessor op> [<JSON accessor op> ...]

<JSON accessor op> ::=
<JSON member accessor>
| <JSON wildcard member accessor>
| <JSON array accessor>
| <JSON filter expression>
| <JSON item method>

<JSON member accessor> ::=
'.' <quoted string literal>
| '.' <string literal>

<JSON wildcard member accessor> ::=
'.' '*'

<JSON array accessor> ::=
'[' <JSON subscript list> ']'

<JSON subscript list> ::=
<JSON subscript> [ { <comma> <JSON subscript> }... ]

<JSON subscript> ::=
<integer>
| 'last'
| 'last' - <integer>
| 'last' + <integer>
| <integer> to <integer>
| '*'
| <JSON path expr>


<JSON unary> ::=
'+'
| '-'

<arithmetic expression> ::=
'+'
| '-'
| '*'
| '/'
| '%'


<JSON filter expression> ::=
'?' '(' <JSON comparison predicate> ')'

<JSON item method> ::=
type '(' ')'
| size '(' ')'
| double '(' ')'
| ceiling '(' ')'
| floor '(' ')'
| abs '(' ')'
| datetime '(' [ <JSON datetime template> ] ')'
| keyvalue '(' ')' [<JSON path accessors>]
| bigint '(' ')'
| boolean '(' ')'
| date '(' ')'
| integer '(' ')'
| number '(' ')'
| string '(' ')'
| time '(' ')'
| time_tz '('')'
| timestamp '('')'
| timestamp_tz '('')'

<JSON datetime template> ::=
<string literal>


<JSON comparison predicate> ::=
<JSON delimited predicate>
| <JSON non-delimited predicate>

<JSON delimited predicate> ::=
<JSON exists path predicate>
| '(' <JSON boolean disjunction> ')'

<JSON exists path predicate> ::=
exists '(' <JSON path expr> ')'


<JSON non-delimited predicate> ::=
<JSON comparison predicate>
| <JSON like_regex predicate>
| <JSON starts with predicate>
| <JSON unknown predicate>
| <JSON path expr>

<JSON comparison predicate> ::=
<JSON boolean negation> <JSON comp op> <JSON boolean negation>

<JSON comp op> ::=
'=='
| '!='
| '<>'
| '<'
| '>'
| '<='
| '>='

<JSON like_regex predicate> ::=
<JSON path expr> like_regex <JSON like_regex pattern> [ flag <JSON like_regex flags> ]

<JSON like_regex pattern> ::=
<string literal> [. <JSON item method> ]
| <JSON path>

<JSON like_regex flags> ::= <string literal>

<JSON unary expression> ::=
<JSON accessor expression>

<JSON starts with predicate> ::=
<JSON path expr> starts with <JSON starts with initial>

<JSON starts with initial> ::=
<string literal> [. <JSON item method> ]
| <JSON path>

<JSON unknown predicate> ::=
'(' <JSON boolean disjunction> ')' is unknown

<JSON boolean negation> ::=
<JSON comparison predicate>
| '!' <JSON delimited predicate>


<JSON boolean disjunction> ::=
<JSON boolean conjunction>
| <JSON boolean disjunction> '||' <JSON boolean conjunction>

<JSON boolean conjunction> ::=
<JSON boolean negation>
| <JSON boolean conjunction> '&&' <JSON boolean negation>
```

### Nodes
JSON Path should be a compile time constant

### Examples:
```sql
SELECT JSON_VALUE('[1,2,3]', '$[2]') FROM RDB$DATABASE;
SELECT JSON_QUERY('[1,2,3]', '$ ? (@ > 1)' WITH WRAPPER) FROM RDB$DATABASE;
SELECT JSON_QUERY('[1,2,3]', 'strict $[*] ? (@ > 1)' WITH WRAPPER) FROM RDB$DATABASE;
SELECT JSON_QUERY('[1,-2,3]', 'strict $[*].abs()' WITH WRAPPER) FROM RDB$DATABASE;

SELECT JSON_QUERY('[{"name":"John", "score":10}, {"name":"Sam", "score":50}]', '$ ? (@.score > 15).name' WITH WRAPPER) FROM RDB$DATABASE;
SELECT JSON_QUERY('{"items":[{"name":"John"}, {"name":"Sam", "score":50}]}', '$.items ? (exists(@.score) && @.score < 15).name' WITH WRAPPER) FROM RDB$DATABASE;

```
28 changes: 28 additions & 0 deletions src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,34 @@ project_group(parse "Boot/Custom build steps")
set_source_files_properties(dsql/parse.cpp include/gen/parse.h PROPERTIES GENERATED TRUE)


set(jpath_parse_src
${CMAKE_CURRENT_SOURCE_DIR}/jrd/json/path/JsonPathParser.y
${CMAKE_CURRENT_SOURCE_DIR}/jrd/json/path/btyacc_json.ske
)
add_custom_command(
OUTPUT jpath_tab.h jpath_tab.c
DEPENDS
btyacc
${jpath_parse_src}
COMMAND sed -n "/%type .*/p" ${CMAKE_CURRENT_SOURCE_DIR}/jrd/json/path/JsonPathParser.y > jtypes.y
COMMAND sed "s/%type .*//" ${CMAKE_CURRENT_SOURCE_DIR}/jrd/json/path/JsonPathParser.y > json_y.y
COMMAND btyacc -l -d -b jpath -S ${CMAKE_CURRENT_SOURCE_DIR}/jrd/json/path/btyacc_json.ske json_y.y

COMMAND sed -i "s/#define \\([A-Z].*\\)/#define TOK_\\1/g" jpath_tab.h
COMMAND sed -i "s/#define TOK_YY\\(.*\\)/#define YY\\1/g" jpath_tab.h

COMMAND ${CMAKE_COMMAND} -E copy_if_different jpath_tab.h include/gen/jparse.h
COMMAND ${CMAKE_COMMAND} -E copy_if_different jpath_tab.c dsql/JsonPathParser.cpp
COMMENT "Generating JsonPathParser.cpp, JsonPathParser.h"
VERBATIM
)
add_custom_target(parse_jpath
DEPENDS jpath_tab.h jpath_tab.c
SOURCES ${parse_src} jpath_tab.h jpath_tab.c
)
project_group(parse_jpath "Boot/Custom build steps")
set_source_files_properties(dsql/JsonPathParser.cpp include/gen/jparse.h PROPERTIES GENERATED TRUE)

########################################
# BUILD_STEP UpdateCloopInterfaces
########################################
Expand Down
Loading
Loading