Skip to content

Commit 9b03bce

Browse files
KamiKami
authored
Merge pull request #3766 from nmaludy/feature/json-jinja-filters
Implement new JSON Jinja filters
2 parents ac17343 + 8c8018f commit 9b03bce

File tree

10 files changed

+263
-2
lines changed

10 files changed

+263
-2
lines changed

‎CHANGELOG.rst‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,9 @@ Added
3535
StackStorm role mappings. This means that the same role can now be granted via multiple RBAC
3636
mapping files.
3737
#3763
38-
38+
* Add new Jinja filters ``from_json_string``, ``from_yaml_string``, and ``jsonpath_query``.
39+
#3763
40+
3941
Fixed
4042
~~~~~
4143

‎contrib/examples/actions/workflows/tests/mistral-test-func-from-json-string.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
version: '2.0'
2+
3+
examples.mistral-test-func-from-json-string:
4+
description: A workflow for testing from_json_string custom filter in mistral
5+
type: direct
6+
input:
7+
- input_str
8+
output:
9+
result_jinja: <% $.result_jinja %>
10+
result_yaql: <% $.result_yaql %>
11+
tasks:
12+
task1:
13+
action: std.noop
14+
publish:
15+
result_jinja: "{{ from_json_string(_.input_str) }}"
16+
result_yaql: '<% from_json_string($.input_str) %>'

‎contrib/examples/actions/workflows/tests/mistral-test-func-from-yaml-string.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
version: '2.0'
2+
3+
examples.mistral-test-func-from-yaml-string:
4+
description: A workflow for testing from_yaml_string custom filter in mistral
5+
type: direct
6+
input:
7+
- input_str
8+
output:
9+
result_jinja: <% $.result_jinja %>
10+
result_yaql: <% $.result_yaql %>
11+
tasks:
12+
task1:
13+
action: std.noop
14+
publish:
15+
result_jinja: "{{ from_yaml_string(_.input_str) }}"
16+
result_yaql: '<% from_yaml_string($.input_str) %>'

‎contrib/examples/actions/workflows/tests/mistral-test-func-jsonpath-query.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
version: '2.0'
2+
3+
examples.mistral-test-func-jsonpath-query:
4+
description: A workflow for testing jsonpath_query custom filter in mistral
5+
type: direct
6+
input:
7+
- input_obj
8+
- input_query
9+
output:
10+
result_jinja: <% $.result_jinja %>
11+
result_yaql: <% $.result_yaql %>
12+
tasks:
13+
14+
task2:
15+
action: std.noop
16+
publish:
17+
result_jinja: '{{ jsonpath_query(_.input_obj, _.input_query) }}'
18+
result_yaql: '<% jsonpath_query($.input_obj, $.input_query) %>'

‎st2common/st2common/jinja/filters/data.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,21 @@
1717
import yaml
1818

1919
__all__ = [
20+
'from_json_string',
21+
'from_yaml_string',
2022
'to_json_string',
2123
'to_yaml_string',
2224
]
2325

2426

27+
def from_json_string(value):
28+
return json.loads(value)
29+
30+
31+
def from_yaml_string(value):
32+
return yaml.safe_load(value)
33+
34+
2535
def to_json_string(value, indent=4, sort_keys=False, separators=(',', ':')):
2636
return json.dumps(value, indent=indent, separators=separators,
2737
sort_keys=sort_keys)

‎st2common/st2common/jinja/filters/jsonpath_query.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
import jsonpath_rw
17+
18+
__all__ = [
19+
'jsonpath_query',
20+
]
21+
22+
23+
def jsonpath_query(value, query):
24+
"""Extracts data from an object `value` using a JSONPath `query`.
25+
:link: https://github.com/kennknowles/python-jsonpath-rw
26+
:param value: a object (dict, array, etc) to query
27+
:param query: a JSONPath query expression (string)
28+
:returns: the result of the query executed on the value
29+
:rtype: dict, array, int, string, bool
30+
"""
31+
expr = jsonpath_rw.parse(query)
32+
matches = [match.value for match in expr.find(value)]
33+
if not matches:
34+
return None
35+
return matches

‎st2common/st2common/util/jinja.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,15 @@ def get_filters():
6060
from st2common.jinja.filters import time
6161
from st2common.jinja.filters import version
6262
from st2common.jinja.filters import json_escape
63+
from st2common.jinja.filters import jsonpath_query
6364

6465
# IMPORTANT NOTE - these filters were recently duplicated in st2mistral so that
6566
# they are also available in Mistral workflows. Please ensure any additions you
6667
# make here are also made there so that feature parity is maintained.
6768
return {
6869
'decrypt_kv': crypto.decrypt_kv,
70+
'from_json_string': data.from_json_string,
71+
'from_yaml_string': data.from_yaml_string,
6972
'to_json_string': data.to_json_string,
7073
'to_yaml_string': data.to_yaml_string,
7174

@@ -89,7 +92,8 @@ def get_filters():
8992
'version_strip_patch': version.version_strip_patch,
9093
'use_none': use_none,
9194

92-
'json_escape': json_escape.json_escape
95+
'json_escape': json_escape.json_escape,
96+
'jsonpath_query': jsonpath_query.jsonpath_query
9397
}
9498

9599

‎st2common/tests/unit/test_jinja_render_data_filters.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,32 @@
2222

2323
class JinjaUtilsDataFilterTestCase(unittest2.TestCase):
2424

25+
def test_filter_from_json_string(self):
26+
env = jinja_utils.get_jinja_environment()
27+
expected_obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}
28+
obj_json_str = '{"a": "b", "c": {"d": "e", "f": 1, "g": true}}'
29+
30+
template = '{{k1 | from_json_string}}'
31+
32+
obj_str = env.from_string(template).render({'k1': obj_json_str})
33+
obj = eval(obj_str)
34+
self.assertDictEqual(obj, expected_obj)
35+
36+
def test_filter_from_yaml_string(self):
37+
env = jinja_utils.get_jinja_environment()
38+
expected_obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}
39+
obj_yaml_str = ("---\n"
40+
"a: b\n"
41+
"c:\n"
42+
" d: e\n"
43+
" f: 1\n"
44+
" g: true\n")
45+
46+
template = '{{k1 | from_yaml_string}}'
47+
obj_str = env.from_string(template).render({'k1': obj_yaml_str})
48+
obj = eval(obj_str)
49+
self.assertDictEqual(obj, expected_obj)
50+
2551
def test_filter_to_json_string(self):
2652
env = jinja_utils.get_jinja_environment()
2753
obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}

‎st2common/tests/unit/test_jinja_render_jsonpath_query_filters.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
17+
import unittest2
18+
19+
from st2common.util import jinja as jinja_utils
20+
21+
22+
class JinjaUtilsJsonpathQueryTestCase(unittest2.TestCase):
23+
24+
def test_jsonpath_query_static(self):
25+
env = jinja_utils.get_jinja_environment()
26+
obj = {'people': [{'first': 'James', 'last': 'd'},
27+
{'first': 'Jacob', 'last': 'e'},
28+
{'first': 'Jayden', 'last': 'f'},
29+
{'missing': 'different'}],
30+
'foo': {'bar': 'baz'}}
31+
32+
template = '{{ obj | jsonpath_query("people[*].first") }}'
33+
actual_str = env.from_string(template).render({'obj': obj})
34+
actual = eval(actual_str)
35+
expected = ['James', 'Jacob', 'Jayden']
36+
self.assertEqual(actual, expected)
37+
38+
def test_jsonpath_query_dynamic(self):
39+
env = jinja_utils.get_jinja_environment()
40+
obj = {'people': [{'first': 'James', 'last': 'd'},
41+
{'first': 'Jacob', 'last': 'e'},
42+
{'first': 'Jayden', 'last': 'f'},
43+
{'missing': 'different'}],
44+
'foo': {'bar': 'baz'}}
45+
query = "people[*].last"
46+
47+
template = '{{ obj | jsonpath_query(query) }}'
48+
actual_str = env.from_string(template).render({'obj': obj,
49+
'query': query})
50+
actual = eval(actual_str)
51+
expected = ['d', 'e', 'f']
52+
self.assertEqual(actual, expected)
53+
54+
def test_jsonpath_query_no_results(self):
55+
env = jinja_utils.get_jinja_environment()
56+
obj = {'people': [{'first': 'James', 'last': 'd'},
57+
{'first': 'Jacob', 'last': 'e'},
58+
{'first': 'Jayden', 'last': 'f'},
59+
{'missing': 'different'}],
60+
'foo': {'bar': 'baz'}}
61+
query = "query_returns_no_results"
62+
63+
template = '{{ obj | jsonpath_query(query) }}'
64+
actual_str = env.from_string(template).render({'obj': obj,
65+
'query': query})
66+
actual = eval(actual_str)
67+
expected = None
68+
self.assertEqual(actual, expected)

‎st2tests/integration/mistral/test_filters.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,46 @@
2525
]
2626

2727

28+
class FromJsonStringFiltersTest(base.TestWorkflowExecution):
29+
30+
def test_from_json_string(self):
31+
32+
execution = self._execute_workflow(
33+
'examples.mistral-test-func-from-json-string',
34+
parameters={
35+
"input_str": '{"a": "b"}'
36+
}
37+
)
38+
execution = self._wait_for_completion(execution)
39+
self._assert_success(execution, num_tasks=1)
40+
jinja_dict = execution.result['result_jinja']
41+
yaql_dict = execution.result['result_yaql']
42+
self.assertTrue(isinstance(jinja_dict, dict))
43+
self.assertEqual(jinja_dict["a"], "b")
44+
self.assertTrue(isinstance(yaql_dict, dict))
45+
self.assertEqual(yaql_dict["a"], "b")
46+
47+
48+
class FromYamlStringFiltersTest(base.TestWorkflowExecution):
49+
50+
def test_from_yaml_string(self):
51+
52+
execution = self._execute_workflow(
53+
'examples.mistral-test-func-from-yaml-string',
54+
parameters={
55+
"input_str": 'a: b'
56+
}
57+
)
58+
execution = self._wait_for_completion(execution)
59+
self._assert_success(execution, num_tasks=1)
60+
jinja_dict = execution.result['result_jinja']
61+
yaql_dict = execution.result['result_yaql']
62+
self.assertTrue(isinstance(jinja_dict, dict))
63+
self.assertEqual(jinja_dict["a"], "b")
64+
self.assertTrue(isinstance(yaql_dict, dict))
65+
self.assertEqual(yaql_dict["a"], "b")
66+
67+
2868
class JsonEscapeFiltersTest(base.TestWorkflowExecution):
2969

3070
def test_json_escape(self):
@@ -44,6 +84,32 @@ def test_json_escape(self):
4484
self.assertEqual(yaql_dict["title"], breaking_str)
4585

4686

87+
class JsonpathQueryFiltersTest(base.TestWorkflowExecution):
88+
89+
def test_jsonpath_query(self):
90+
91+
execution = self._execute_workflow(
92+
'examples.mistral-test-func-jsonpath-query',
93+
parameters={
94+
"input_obj": {'people': [{'first': 'James', 'last': 'Smith'},
95+
{'first': 'Jacob', 'last': 'Alberts'},
96+
{'first': 'Jayden', 'last': 'Davis'},
97+
{'missing': 'different'}]},
98+
"input_query": "people[*].last"
99+
}
100+
)
101+
expected_result = ['Smith', 'Alberts', 'Davis']
102+
103+
execution = self._wait_for_completion(execution)
104+
self._assert_success(execution, num_tasks=1)
105+
jinja_result = execution.result['result_jinja']
106+
yaql_result = execution.result['result_yaql']
107+
self.assertTrue(isinstance(jinja_result, list))
108+
self.assertEqual(jinja_result, expected_result)
109+
self.assertTrue(isinstance(yaql_result, list))
110+
self.assertEqual(yaql_result, expected_result)
111+
112+
47113
class RegexMatchFiltersTest(base.TestWorkflowExecution):
48114

49115
def test_regex_match(self):

0 commit comments

Comments
 (0)