From 1c651a8a10943a716f3a9885c092433c6e362845 Mon Sep 17 00:00:00 2001 From: brigadier Date: Thu, 11 Mar 2021 21:31:28 +0300 Subject: [PATCH 1/8] Allow tuples in input terms with the [tuples_to_lists] option --- README.md | 9 ++++-- src/jsx_config.erl | 17 +++++++++--- src/jsx_config.hrl | 3 +- src/jsx_encoder.erl | 67 +++++++++++++++++++++++---------------------- src/jsx_to_json.erl | 5 +++- 5 files changed, 59 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index f4e27ceb..2d7841c6 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ see below | `datetime()` * arrays json arrays are represented with erlang lists of json values as described - in this section + in this section, or by tuples with the `tuples_to_lists`option * objects @@ -229,6 +229,7 @@ see below | `datetime()` strings and are assumed to be UTC time. no conversion is attempted of json [iso8601][iso8601] strings in decoded json + ### incomplete input ### **jsx** can handle incomplete json texts. if the option `stream` is passed to the decoder @@ -272,7 +273,8 @@ json_term() = [json_term()] | float() | binary() | atom() - | datetime() + | datetime() + | tuple() ``` the erlang representation of json. binaries should be `utf8` encoded, or close @@ -318,6 +320,7 @@ option() = dirty_strings | return_tail | uescape | unescaped_jsonp + | tuples-to_lists strict_option() = comments | trailing_commas @@ -495,7 +498,7 @@ encode(Term, Opts) -> JSON Term = json_term() JSON = json_text() - Opts = [option() | space | {space, N} | indent | {indent, N}] + Opts = [option() | space | {space, N} | indent | {indent, N} | tuples-to_lists] N = pos_integer() ``` diff --git a/src/jsx_config.erl b/src/jsx_config.erl index ba1d872a..1749f13b 100644 --- a/src/jsx_config.erl +++ b/src/jsx_config.erl @@ -60,6 +60,7 @@ | {indent, non_neg_integer()} | {depth, non_neg_integer()} | {newline, binary()} + | {tuples_to_lists, boolean()} | legacy_option() | {legacy_option(), boolean()}. -type legacy_option() :: strict_comments @@ -114,6 +115,8 @@ parse_config([multi_term|Rest], Config) -> parse_config(Rest, Config#config{multi_term=true}); parse_config([return_tail|Rest], Config) -> parse_config(Rest, Config#config{return_tail=true}); +parse_config([tuples_to_lists|Rest], Config) -> + parse_config(Rest, Config#config{tuples_to_lists = true}); %% retained for backwards compat, now does nothing however parse_config([repeat_keys|Rest], Config) -> parse_config(Rest, Config); @@ -215,7 +218,8 @@ valid_flags() -> stream, uescape, error_handler, - incomplete_handler + incomplete_handler, + tuples_to_lists ]. @@ -259,7 +263,8 @@ config_test_() -> strict_escapes = true, strict_control_codes = true, stream = true, - uescape = true + uescape = true, + tuples_to_lists = true }, parse_config([dirty_strings, escaped_forward_slashes, @@ -270,7 +275,8 @@ config_test_() -> repeat_keys, strict, stream, - uescape + uescape, + tuples_to_lists ]) ) }, @@ -342,7 +348,9 @@ config_to_list_test_() -> stream, uescape, unescaped_jsonp, + tuples_to_lists, strict + ], config_to_list( #config{escaped_forward_slashes = true, @@ -356,7 +364,8 @@ config_to_list_test_() -> strict_escapes = true, strict_control_codes = true, stream = true, - uescape = true + uescape = true, + tuples_to_lists = true } ) )}, diff --git a/src/jsx_config.hrl b/src/jsx_config.hrl index c89963c5..2f2b19a0 100644 --- a/src/jsx_config.hrl +++ b/src/jsx_config.hrl @@ -14,5 +14,6 @@ uescape = false :: boolean(), unescaped_jsonp = false :: boolean(), error_handler = false :: false | jsx_config:handler(), - incomplete_handler = false :: false | jsx_config:handler() + incomplete_handler = false :: false | jsx_config:handler(), + tuples_to_lists = false :: boolean() }). diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index a1242a79..8ffc4fa5 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -22,58 +22,59 @@ -module(jsx_encoder). - --export([encoder/3, encode/1, encode/2]). +-include("jsx_config.hrl"). +-export([encoder/3, encode/2, encode/3]). -spec encoder(Handler::module(), State::any(), Config::jsx_config:options()) -> jsx:encoder(). encoder(Handler, State, Config) -> Parser = jsx:parser(Handler, State, Config), - fun(Term) -> Parser(encode(Term) ++ [end_json]) end. + fun(Term) -> Parser(encode(Term, jsx_config:parse_config(Config)) ++ [end_json]) end. --spec encode(Term::any()) -> [any(), ...]. +-spec encode(Term::any(), Config::jsx_config:options()) -> [any(), ...]. -encode(Term) -> encode(Term, ?MODULE). +encode(Term, Config) -> encode(Term, ?MODULE, Config). --spec encode(Term::any(), EntryPoint::module()) -> [any(), ...]. +-spec encode(Term::any(), EntryPoint::module(), Config::jsx_config:options()) -> [any(), ...]. -encode(Map, _EntryPoint) when is_map(Map), map_size(Map) < 1 -> +encode(Map, _EntryPoint, _Config) when is_map(Map), map_size(Map) < 1 -> [start_object, end_object]; -encode(Term, EntryPoint) when is_map(Term) -> - [start_object] ++ unpack(Term, EntryPoint); -encode(Term, EntryPoint) -> encode_(Term, EntryPoint). +encode(Term, EntryPoint, Config) when is_map(Term) -> + [start_object] ++ unpack(Term, EntryPoint, Config); +encode(Term, EntryPoint, Config) -> encode_(Term, EntryPoint, Config). -encode_([], _EntryPoint) -> [start_array, end_array]; -encode_([{}], _EntryPoint) -> [start_object, end_object]; +encode_([], _EntryPoint, _Config) -> [start_array, end_array]; +encode_([{}], _EntryPoint,#config{tuples_to_lists = false}) -> [start_object, end_object]; %% datetime special case -encode_([{{_,_,_},{_,_,_}} = DateTime|Rest], EntryPoint) -> - [start_array] ++ [DateTime] ++ unhitch(Rest, EntryPoint); -encode_([{_, _}|_] = Term, EntryPoint) -> - [start_object] ++ unzip(Term, EntryPoint); -encode_(Term, EntryPoint) when is_list(Term) -> - [start_array] ++ unhitch(Term, EntryPoint); - -encode_(Else, _EntryPoint) -> [Else]. +encode_([{{_,_,_},{_,_,_}} = DateTime|Rest], EntryPoint, #config{tuples_to_lists = false} = Config) -> + [start_array] ++ [DateTime] ++ unhitch(Rest, EntryPoint, Config); +encode_([{_, _}|_] = Term, EntryPoint, #config{tuples_to_lists = false} = Config) -> + [start_object] ++ unzip(Term, EntryPoint, Config); +encode_(Term, EntryPoint, Config) when is_list(Term) -> + [start_array] ++ unhitch(Term, EntryPoint, Config); +encode_(T, EntryPoint, #config{tuples_to_lists = true} = Config) when is_tuple(T)-> + encode_(tuple_to_list(T), EntryPoint, Config); +encode_(Else, _EntryPoint, _Config) -> [Else]. -unzip([{K, V}|Rest], EntryPoint) when is_integer(K); is_binary(K); is_atom(K) -> - [K] ++ EntryPoint:encode(V, EntryPoint) ++ unzip(Rest, EntryPoint); -unzip([], _) -> [end_object]; -unzip(_, _) -> erlang:error(badarg). +unzip([{K, V}|Rest], EntryPoint, Config) when is_integer(K); is_binary(K); is_atom(K) -> + [K] ++ EntryPoint:encode(V, EntryPoint, Config) ++ unzip(Rest, EntryPoint, Config); +unzip([], _, _) -> [end_object]; +unzip(_, _, _) -> erlang:error(badarg). -unhitch([V|Rest], EntryPoint) -> - EntryPoint:encode(V, EntryPoint) ++ unhitch(Rest, EntryPoint); -unhitch([], _) -> [end_array]. +unhitch([V|Rest], EntryPoint, Config) -> + EntryPoint:encode(V, EntryPoint, Config) ++ unhitch(Rest, EntryPoint, Config); +unhitch([], _, _) -> [end_array]. -unpack(Map, EntryPoint) -> unpack(Map, maps:keys(Map), EntryPoint). +unpack(Map, EntryPoint, Config) -> unpack(Map, maps:keys(Map), EntryPoint, Config). -unpack(Map, [K|Rest], EntryPoint) when is_integer(K); is_binary(K); is_atom(K) -> - [K] ++ EntryPoint:encode(maps:get(K, Map), EntryPoint) ++ unpack(Map, Rest, EntryPoint); -unpack(_, [], _) -> [end_object]. +unpack(Map, [K|Rest], EntryPoint, Config) when is_integer(K); is_binary(K); is_atom(K) -> + [K] ++ EntryPoint:encode(maps:get(K, Map), EntryPoint, Config) ++ unpack(Map, Rest, EntryPoint, Config); +unpack(_, [], _, _) -> [end_object]. -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -105,11 +106,11 @@ improper_lists_test_() -> [ {"improper proplist", ?_assertError( badarg, - encode([{<<"key">>, <<"value">>}, false]) + encode([{<<"key">>, <<"value">>}, false], #config{}) )}, {"improper list", ?_assertError( badarg, - encode([{literal, true}, false, null]) + encode([{literal, true}, false, null], #config{}) )} ]. diff --git a/src/jsx_to_json.erl b/src/jsx_to_json.erl index d20add61..15543a00 100644 --- a/src/jsx_to_json.erl +++ b/src/jsx_to_json.erl @@ -33,7 +33,8 @@ space = 0, indent = 0, depth = 0, - newline = <<$\n>> + newline = <<$\n>>, + tuples_to_lists = false }). -type config() :: proplists:proplist(). @@ -62,6 +63,8 @@ parse_config([{indent, Val}|Rest], Config) when is_integer(Val), Val > 0 -> parse_config(Rest, Config#config{indent = Val}); parse_config([indent|Rest], Config) -> parse_config(Rest, Config#config{indent = 1}); +parse_config([tuples_to_lists|Rest], Config) -> + parse_config(Rest, Config#config{tuples_to_lists = true}); parse_config([{newline, Val}|Rest], Config) when is_binary(Val) -> parse_config(Rest, Config#config{newline = Val}); parse_config([{K, _}|Rest] = Options, Config) -> From 9e055df40e97937f1e74a4d2f1a4c27baff03620 Mon Sep 17 00:00:00 2001 From: Evgeny M Date: Thu, 11 Mar 2021 22:55:31 +0300 Subject: [PATCH 2/8] Update README.md Co-authored-by: Paulo F. Oliveira --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2d7841c6..b3cad5e3 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ see below | `datetime()` * arrays json arrays are represented with erlang lists of json values as described - in this section, or by tuples with the `tuples_to_lists`option + in this section, or by tuples with the `tuples_to_lists` option * objects From 0f340e040eec18b624484ca538d35d88fd2397bf Mon Sep 17 00:00:00 2001 From: Evgeny M Date: Thu, 11 Mar 2021 22:55:36 +0300 Subject: [PATCH 3/8] Update README.md Co-authored-by: Paulo F. Oliveira --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b3cad5e3..29aaea77 100644 --- a/README.md +++ b/README.md @@ -320,7 +320,7 @@ option() = dirty_strings | return_tail | uescape | unescaped_jsonp - | tuples-to_lists + | tuples_to_lists strict_option() = comments | trailing_commas From 10a7fffaae251d4c97c1403eeec9e59a2f3bd41b Mon Sep 17 00:00:00 2001 From: Evgeny M Date: Thu, 11 Mar 2021 22:55:42 +0300 Subject: [PATCH 4/8] Update README.md Co-authored-by: Paulo F. Oliveira --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 29aaea77..98d60bd2 100644 --- a/README.md +++ b/README.md @@ -498,7 +498,7 @@ encode(Term, Opts) -> JSON Term = json_term() JSON = json_text() - Opts = [option() | space | {space, N} | indent | {indent, N} | tuples-to_lists] + Opts = [option() | space | {space, N} | indent | {indent, N} | tuples_to_lists] N = pos_integer() ``` From 28f7079889501c3ec59a1bcb1f802eddbef99757 Mon Sep 17 00:00:00 2001 From: Evgeny M Date: Fri, 12 Mar 2021 09:29:36 +0300 Subject: [PATCH 5/8] Update README.md Co-authored-by: Paulo F. Oliveira --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 98d60bd2..3883274d 100644 --- a/README.md +++ b/README.md @@ -229,7 +229,6 @@ see below | `datetime()` strings and are assumed to be UTC time. no conversion is attempted of json [iso8601][iso8601] strings in decoded json - ### incomplete input ### **jsx** can handle incomplete json texts. if the option `stream` is passed to the decoder From 0d300321171018ef77474d0bd707eb6268dc62d6 Mon Sep 17 00:00:00 2001 From: Evgeny M Date: Fri, 12 Mar 2021 09:29:43 +0300 Subject: [PATCH 6/8] Update src/jsx_config.erl Co-authored-by: Paulo F. Oliveira --- src/jsx_config.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/jsx_config.erl b/src/jsx_config.erl index 1749f13b..5627c94d 100644 --- a/src/jsx_config.erl +++ b/src/jsx_config.erl @@ -350,7 +350,6 @@ config_to_list_test_() -> unescaped_jsonp, tuples_to_lists, strict - ], config_to_list( #config{escaped_forward_slashes = true, From a87e43e2ea6c34ebea3fcdaa7642b50490d6e1c3 Mon Sep 17 00:00:00 2001 From: brigadier Date: Sat, 13 Mar 2021 17:15:14 +0300 Subject: [PATCH 7/8] disable_timestamp_euristics config flag disables conversion of {A, B, C} and {{A, B, C}, {D, E, F}} to datetime. Converts to lists instead. --- src/jsx_config.erl | 16 ++++++++++++---- src/jsx_config.hrl | 3 ++- src/jsx_encoder.erl | 7 +++++++ src/jsx_to_json.erl | 3 ++- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/jsx_config.erl b/src/jsx_config.erl index 5627c94d..8babc718 100644 --- a/src/jsx_config.erl +++ b/src/jsx_config.erl @@ -61,6 +61,7 @@ | {depth, non_neg_integer()} | {newline, binary()} | {tuples_to_lists, boolean()} + | {disable_timestamp_euristics, boolean()} | legacy_option() | {legacy_option(), boolean()}. -type legacy_option() :: strict_comments @@ -117,6 +118,8 @@ parse_config([return_tail|Rest], Config) -> parse_config(Rest, Config#config{return_tail=true}); parse_config([tuples_to_lists|Rest], Config) -> parse_config(Rest, Config#config{tuples_to_lists = true}); +parse_config([disable_timestamp_euristics|Rest], Config) -> + parse_config(Rest, Config#config{disable_timestamp_euristics = true}); %% retained for backwards compat, now does nothing however parse_config([repeat_keys|Rest], Config) -> parse_config(Rest, Config); @@ -219,7 +222,8 @@ valid_flags() -> uescape, error_handler, incomplete_handler, - tuples_to_lists + tuples_to_lists, + disable_timestamp_euristics ]. @@ -264,7 +268,8 @@ config_test_() -> strict_control_codes = true, stream = true, uescape = true, - tuples_to_lists = true + tuples_to_lists = true, + disable_timestamp_euristics = true }, parse_config([dirty_strings, escaped_forward_slashes, @@ -276,7 +281,8 @@ config_test_() -> strict, stream, uescape, - tuples_to_lists + tuples_to_lists, + disable_timestamp_euristics ]) ) }, @@ -349,6 +355,7 @@ config_to_list_test_() -> uescape, unescaped_jsonp, tuples_to_lists, + disable_timestamp_euristics, strict ], config_to_list( @@ -364,7 +371,8 @@ config_to_list_test_() -> strict_control_codes = true, stream = true, uescape = true, - tuples_to_lists = true + tuples_to_lists = true, + disable_timestamp_euristics = true } ) )}, diff --git a/src/jsx_config.hrl b/src/jsx_config.hrl index 2f2b19a0..11c4f5ad 100644 --- a/src/jsx_config.hrl +++ b/src/jsx_config.hrl @@ -15,5 +15,6 @@ unescaped_jsonp = false :: boolean(), error_handler = false :: false | jsx_config:handler(), incomplete_handler = false :: false | jsx_config:handler(), - tuples_to_lists = false :: boolean() + tuples_to_lists = false :: boolean(), + disable_timestamp_euristics = false :: boolean() }). diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index 8ffc4fa5..cd817fd0 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -51,6 +51,13 @@ encode_([{}], _EntryPoint,#config{tuples_to_lists = false}) -> [start_object, en %% datetime special case encode_([{{_,_,_},{_,_,_}} = DateTime|Rest], EntryPoint, #config{tuples_to_lists = false} = Config) -> [start_array] ++ [DateTime] ++ unhitch(Rest, EntryPoint, Config); +encode_({{A,B,C},{D,E,F}} = DateTime, _EntryPoint, #config{tuples_to_lists = true, disable_timestamp_euristics = false} = _Config) + when is_integer(A), is_integer(B), is_integer(C), is_integer(D), is_integer(E), is_integer(F) -> + [DateTime]; +encode_({A,B,C} = Timestamp, _EntryPoint, #config{tuples_to_lists = true, disable_timestamp_euristics = false} = _Config) + when is_integer(A), is_integer(B), is_integer(C) -> + [Timestamp]; + encode_([{_, _}|_] = Term, EntryPoint, #config{tuples_to_lists = false} = Config) -> [start_object] ++ unzip(Term, EntryPoint, Config); encode_(Term, EntryPoint, Config) when is_list(Term) -> diff --git a/src/jsx_to_json.erl b/src/jsx_to_json.erl index 15543a00..58bc722a 100644 --- a/src/jsx_to_json.erl +++ b/src/jsx_to_json.erl @@ -34,7 +34,8 @@ indent = 0, depth = 0, newline = <<$\n>>, - tuples_to_lists = false + tuples_to_lists = false, + disable_timestamp_euristics = false }). -type config() :: proplists:proplist(). From a4abe37fa463f83b2bdc44b81ae6a8ee75dfcf4a Mon Sep 17 00:00:00 2001 From: brigadier Date: Sun, 14 Mar 2021 17:44:20 +0300 Subject: [PATCH 8/8] euristics -> heuristics --- src/jsx_config.erl | 16 ++++++++-------- src/jsx_config.hrl | 2 +- src/jsx_encoder.erl | 4 ++-- src/jsx_to_json.erl | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/jsx_config.erl b/src/jsx_config.erl index 8babc718..7acc11f8 100644 --- a/src/jsx_config.erl +++ b/src/jsx_config.erl @@ -61,7 +61,7 @@ | {depth, non_neg_integer()} | {newline, binary()} | {tuples_to_lists, boolean()} - | {disable_timestamp_euristics, boolean()} + | {disable_timestamp_heuristics, boolean()} | legacy_option() | {legacy_option(), boolean()}. -type legacy_option() :: strict_comments @@ -118,8 +118,8 @@ parse_config([return_tail|Rest], Config) -> parse_config(Rest, Config#config{return_tail=true}); parse_config([tuples_to_lists|Rest], Config) -> parse_config(Rest, Config#config{tuples_to_lists = true}); -parse_config([disable_timestamp_euristics|Rest], Config) -> - parse_config(Rest, Config#config{disable_timestamp_euristics = true}); +parse_config([disable_timestamp_heuristics|Rest], Config) -> + parse_config(Rest, Config#config{disable_timestamp_heuristics = true}); %% retained for backwards compat, now does nothing however parse_config([repeat_keys|Rest], Config) -> parse_config(Rest, Config); @@ -223,7 +223,7 @@ valid_flags() -> error_handler, incomplete_handler, tuples_to_lists, - disable_timestamp_euristics + disable_timestamp_heuristics ]. @@ -269,7 +269,7 @@ config_test_() -> stream = true, uescape = true, tuples_to_lists = true, - disable_timestamp_euristics = true + disable_timestamp_heuristics = true }, parse_config([dirty_strings, escaped_forward_slashes, @@ -282,7 +282,7 @@ config_test_() -> stream, uescape, tuples_to_lists, - disable_timestamp_euristics + disable_timestamp_heuristics ]) ) }, @@ -355,7 +355,7 @@ config_to_list_test_() -> uescape, unescaped_jsonp, tuples_to_lists, - disable_timestamp_euristics, + disable_timestamp_heuristics, strict ], config_to_list( @@ -372,7 +372,7 @@ config_to_list_test_() -> stream = true, uescape = true, tuples_to_lists = true, - disable_timestamp_euristics = true + disable_timestamp_heuristics = true } ) )}, diff --git a/src/jsx_config.hrl b/src/jsx_config.hrl index 11c4f5ad..a03fd756 100644 --- a/src/jsx_config.hrl +++ b/src/jsx_config.hrl @@ -16,5 +16,5 @@ error_handler = false :: false | jsx_config:handler(), incomplete_handler = false :: false | jsx_config:handler(), tuples_to_lists = false :: boolean(), - disable_timestamp_euristics = false :: boolean() + disable_timestamp_heuristics = false :: boolean() }). diff --git a/src/jsx_encoder.erl b/src/jsx_encoder.erl index cd817fd0..6961d4aa 100644 --- a/src/jsx_encoder.erl +++ b/src/jsx_encoder.erl @@ -51,10 +51,10 @@ encode_([{}], _EntryPoint,#config{tuples_to_lists = false}) -> [start_object, en %% datetime special case encode_([{{_,_,_},{_,_,_}} = DateTime|Rest], EntryPoint, #config{tuples_to_lists = false} = Config) -> [start_array] ++ [DateTime] ++ unhitch(Rest, EntryPoint, Config); -encode_({{A,B,C},{D,E,F}} = DateTime, _EntryPoint, #config{tuples_to_lists = true, disable_timestamp_euristics = false} = _Config) +encode_({{A,B,C},{D,E,F}} = DateTime, _EntryPoint, #config{tuples_to_lists = true, disable_timestamp_heuristics = false} = _Config) when is_integer(A), is_integer(B), is_integer(C), is_integer(D), is_integer(E), is_integer(F) -> [DateTime]; -encode_({A,B,C} = Timestamp, _EntryPoint, #config{tuples_to_lists = true, disable_timestamp_euristics = false} = _Config) +encode_({A,B,C} = Timestamp, _EntryPoint, #config{tuples_to_lists = true, disable_timestamp_heuristics = false} = _Config) when is_integer(A), is_integer(B), is_integer(C) -> [Timestamp]; diff --git a/src/jsx_to_json.erl b/src/jsx_to_json.erl index 58bc722a..1e585f27 100644 --- a/src/jsx_to_json.erl +++ b/src/jsx_to_json.erl @@ -35,7 +35,7 @@ depth = 0, newline = <<$\n>>, tuples_to_lists = false, - disable_timestamp_euristics = false + disable_timestamp_heuristics = false }). -type config() :: proplists:proplist().