Skip to content

Commit 7bf8d1a

Browse files
SwiddisSwiddis
committed
Implement string escaping for paths instead of ident escaping
Signed-off-by: Simeon Widdis <[email protected]>
1 parent b5d6996 commit 7bf8d1a

File tree

9 files changed

+61
-9
lines changed

9 files changed

+61
-9
lines changed

‎core/src/main/java/org/opensearch/sql/ast/tree/SPath.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
package org.opensearch.sql.ast.tree;
77

8-
import static org.opensearch.sql.common.utils.StringUtils.unquoteIdentifier;
8+
import static org.opensearch.sql.common.utils.StringUtils.unquoteText;
99

1010
import com.google.common.collect.ImmutableList;
1111
import java.util.List;
@@ -50,7 +50,7 @@ public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
5050

5151
public Eval rewriteAsEval() {
5252
String outField = this.outField;
53-
String unquotedPath = unquoteIdentifier(this.path);
53+
String unquotedPath = unquoteText(this.path);
5454
if (outField == null) {
5555
outField = unquotedPath;
5656
}

‎docs/category.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
"user/ppl/cmd/search.rst",
4747
"user/ppl/cmd/showdatasources.rst",
4848
"user/ppl/cmd/sort.rst",
49+
"user/ppl/cmd/spath.rst",
4950
"user/ppl/cmd/stats.rst",
5051
"user/ppl/cmd/streamstats.rst",
5152
"user/ppl/cmd/subquery.rst",

‎docs/user/dql/metadata.rst‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ Example 1: Show All Indices Information
3535
SQL query::
3636

3737
os> SHOW TABLES LIKE '%'
38-
fetched rows / total rows = 22/22
38+
fetched rows / total rows = 23/23
3939
+----------------+-------------+-------------------+------------+---------+----------+------------+-----------+---------------------------+----------------+
4040
| TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION |
4141
|----------------+-------------+-------------------+------------+---------+----------+------------+-----------+---------------------------+----------------|
@@ -54,6 +54,7 @@ SQL query::
5454
| docTestCluster | null | otellogs | BASE TABLE | null | null | null | null | null | null |
5555
| docTestCluster | null | people | BASE TABLE | null | null | null | null | null | null |
5656
| docTestCluster | null | state_country | BASE TABLE | null | null | null | null | null | null |
57+
| docTestCluster | null | structured | BASE TABLE | null | null | null | null | null | null |
5758
| docTestCluster | null | time_data | BASE TABLE | null | null | null | null | null | null |
5859
| docTestCluster | null | time_data2 | BASE TABLE | null | null | null | null | null | null |
5960
| docTestCluster | null | time_test | BASE TABLE | null | null | null | null | null | null |

‎docs/user/ppl/cmd/spath.rst‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,10 @@ The simplest spath is to extract a single field. This extracts `n` from the `doc
3737

3838
PPL query::
3939

40-
PPL> source=test_spath | spath input=doc n;
40+
os> source=structured | spath input=doc_n n | fields doc_n n;
4141
fetched rows / total rows = 3/3
4242
+----------+---+
43-
| doc | n |
43+
| doc_n | n |
4444
|----------+---|
4545
| {"n": 1} | 1 |
4646
| {"n": 2} | 2 |
@@ -54,10 +54,10 @@ These queries demonstrate more JSON path uses, like traversing nested fields and
5454

5555
PPL query::
5656

57-
PPL> source=test_spath | spath input=doc output=first_element list{0} | spath input=doc output=all_elements list{} | spath input=doc output=nested nest_out.nest_in;
57+
os> source=structured | spath input=doc_list output=first_element list{0} | spath input=doc_list output=all_elements list{} | spath input=doc_list output=nested nest_out.nest_in | fields doc_list first_element all_elements nested;
5858
fetched rows / total rows = 3/3
5959
+------------------------------------------------------+---------------+--------------+--------+
60-
| doc | first_element | all_elements | nested |
60+
| doc_list | first_element | all_elements | nested |
6161
|------------------------------------------------------+---------------+--------------+--------|
6262
| {"list": [1, 2, 3, 4], "nest_out": {"nest_in": "a"}} | 1 | [1,2,3,4] | a |
6363
| {"list": [], "nest_out": {"nest_in": "a"}} | null | [] | a |
@@ -71,10 +71,27 @@ The example shows extracting an inner field and doing statistics on it, using th
7171

7272
PPL query::
7373

74-
PPL> source=test_spath | spath input=doc n | eval n=cast(n as int) | stats sum(n);
74+
os> source=structured | spath input=doc_n n | eval n=cast(n as int) | stats sum(n) | fields `sum(n)`;
7575
fetched rows / total rows = 1/1
7676
+--------+
7777
| sum(n) |
7878
|--------|
7979
| 6 |
8080
+--------+
81+
82+
Example 4: Escaped paths
83+
============================
84+
85+
`spath` can escape paths with strings to accept any path that `json_extract` does. This includes escaping complex field names as array components.
86+
87+
PPL query::
88+
89+
os> source=structured | spath output=a input=doc_escape "['a fancy field name']" | spath output=b input=doc_escape "['a.b.c']" | fields a b;
90+
fetched rows / total rows = 3/3
91+
+-------+---+
92+
| a | b |
93+
|-------+---|
94+
| true | 0 |
95+
| true | 1 |
96+
| false | 2 |
97+
+-------+---+

‎doctest/test_data/structured.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{"doc_n":"{\"n\": 1}","doc_escape":"{\"a fancy field name\": true,\"a.b.c\": 0}","doc_list":"{\"list\": [1, 2, 3, 4], \"nest_out\": {\"nest_in\": \"a\"}}","obj_field":{"field": "a"}}
2+
{"doc_n":"{\"n\": 2}","doc_escape":"{\"a fancy field name\": true,\"a.b.c\": 1}","doc_list":"{\"list\": [], \"nest_out\": {\"nest_in\": \"a\"}}","obj_field":{"field": "b"}}
3+
{"doc_n":"{\"n\": 3}","doc_escape":"{\"a fancy field name\": false,\"a.b.c\": 2}","doc_list":"{\"list\": [5, 6], \"nest_out\": {\"nest_in\": \"a\"}}","obj_field":{"field": "c"}}

‎doctest/test_docs.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
'weblogs': 'weblogs.json',
3838
'json_test': 'json_test.json',
3939
'state_country': 'state_country.json',
40+
'structured': 'structured.json',
4041
'occupation': 'occupation.json',
4142
'worker': 'worker.json',
4243
'work_information': 'work_information.json',

‎doctest/test_mapping/structured.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"mappings": {
3+
"properties": {
4+
"doc_n": {
5+
"type": "text"
6+
},
7+
"doc_list": {
8+
"type": "text"
9+
},
10+
"doc_escape": {
11+
"type": "text"
12+
},
13+
"obj_field": {
14+
"properties": {
15+
"field": { "type": "text" }
16+
}
17+
}
18+
}
19+
}
20+
}

‎ppl/src/main/antlr/OpenSearchPPLParser.g4‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,7 @@ spathParameter
404404

405405
indexablePath
406406
: pathElement (DOT pathElement)*
407+
| stringLiteral
407408
;
408409

409410
pathElement

‎ppl/src/test/java/org/opensearch/sql/ppl/utils/SPathRewriteTest.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,17 @@ public void testSpathArgumentDeshuffle() {
6868
@Test
6969
public void testSpathEscapedParse() {
7070
SPath sp =
71-
(SPath) plan("source = t | spath input=f output=o path=`attributes.['cluster.name']`");
71+
(SPath) plan("source = t | spath input=f output=o path=\"attributes.['cluster.name']\"");
7272
Eval ev = (Eval) plan("source = t | eval o=json_extract(f, \"attributes.['cluster.name']\")");
7373

7474
assertEquals(ev, sp.rewriteAsEval());
7575
}
76+
77+
@Test
78+
public void testSpathEscapedSpaces() {
79+
SPath sp = (SPath) plan("source = t | spath input=f output=o path=\"['abc def ghi']\"");
80+
Eval ev = (Eval) plan("source = t | eval o=json_extract(f, \"['abc def ghi']\")");
81+
82+
assertEquals(ev, sp.rewriteAsEval());
83+
}
7684
}

0 commit comments

Comments
 (0)