diff --git a/.github/ubuntu/clickhouse.sh b/.github/ubuntu/clickhouse.sh index 1b4adbd..632c33a 100755 --- a/.github/ubuntu/clickhouse.sh +++ b/.github/ubuntu/clickhouse.sh @@ -38,5 +38,17 @@ for pkg in clickhouse-common-static clickhouse-server; do rm "${pkg}.deb" done +if [ "${CH_VERSION%%.*}" -lt 25 ]; then + # Enable the JSON type in release prior to 2025. + printf "~~~~ Starting ClickHouse %s ~~~~\n" "$CH_VERSION" + setting=allow_experimental_object_type + if [ "${CH_VERSION:0:4}" == "24.8" ]; then + # The setting was renamed. + setting=allow_experimental_json_type + fi + + perl -i -pe "s{^((\s+))}{\$1\n\$2 <$setting>1}" /etc/clickhouse-server/users.xml +fi + printf "~~~~ Starting ClickHouse %s ~~~~\n" "$CH_VERSION" /etc/init.d/clickhouse-server start diff --git a/src/pglink.c b/src/pglink.c index f7cbdd6..550e543 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -760,6 +760,7 @@ static char *str_types_map[][2] = { {"UUID", "UUID"}, {"IPv4", "inet"}, {"IPv6", "inet"}, + {"JSON", "JSON"}, {NULL, NULL}, }; diff --git a/test/expected/json.out b/test/expected/json.out new file mode 100644 index 0000000..6a076f6 --- /dev/null +++ b/test/expected/json.out @@ -0,0 +1,86 @@ +SET datestyle = 'ISO'; +CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary'); +CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http'); +CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback; +CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback; +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test'); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query('CREATE DATABASE json_test'); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query($$ + CREATE TABLE json_test.things ( + id Int32 NOT NULL, + data JSON NOT NULL + ) ENGINE = MergeTree PARTITION BY id ORDER BY (id); +$$); + clickhouse_raw_query +---------------------- + +(1 row) + +CREATE SCHEMA json_bin; +CREATE SCHEMA json_http; +IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO json_bin; +\d json_bin.things + Foreign table "json_bin.things" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + id | integer | | not null | | + data | json | | not null | | +Server: binary_json_loopback +FDW options: (database 'json_test', table_name 'things', engine 'MergeTree') + +IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO json_http; +\d json_http.things + Foreign table "json_http.things" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + id | integer | | not null | | + data | json | | not null | | +Server: http_json_loopback +FDW options: (database 'json_test', table_name 'things', engine 'MergeTree') + +-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422 +INSERT INTO json_bin.things VALUES + (1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'), + (2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}') +; +ERROR: pg_clickhouse: could not prepare insert - unsupported column type: JSON +INSERT INTO json_http.things VALUES + (1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'), + (2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'), + (3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'), + (4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}') +; +SELECT * FROM json_bin.things ORDER BY id; +ERROR: pg_clickhouse: unsupported column type: JSON +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC +SELECT * FROM json_http.things ORDER BY id; + id | data +----+---------------------------------------------------------- + 1 | {"id":1,"name":"widget","size":"large","stocked":true} + 2 | {"id":2,"name":"sprocket","size":"small","stocked":true} + 3 | {"id":3,"name":"gizmo","size":"medium","stocked":true} + 4 | {"id":4,"name":"doodad","size":"large","stocked":false} +(4 rows) + +SELECT clickhouse_raw_query('DROP DATABASE json_test'); + clickhouse_raw_query +---------------------- + +(1 row) + +DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback; +DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback; +DROP SERVER binary_json_loopback CASCADE; +NOTICE: drop cascades to foreign table json_bin.things +DROP SERVER http_json_loopback CASCADE; +NOTICE: drop cascades to foreign table json_http.things diff --git a/test/expected/json_1.out b/test/expected/json_1.out new file mode 100644 index 0000000..e0d9f71 --- /dev/null +++ b/test/expected/json_1.out @@ -0,0 +1,86 @@ +SET datestyle = 'ISO'; +CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary'); +CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http'); +CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback; +CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback; +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test'); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query('CREATE DATABASE json_test'); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query($$ + CREATE TABLE json_test.things ( + id Int32 NOT NULL, + data JSON NOT NULL + ) ENGINE = MergeTree PARTITION BY id ORDER BY (id); +$$); + clickhouse_raw_query +---------------------- + +(1 row) + +CREATE SCHEMA json_bin; +CREATE SCHEMA json_http; +IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO json_bin; +\d json_bin.things + Foreign table "json_bin.things" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + id | integer | | not null | | + data | json | | not null | | +Server: binary_json_loopback +FDW options: (database 'json_test', table_name 'things', engine 'MergeTree') + +IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO json_http; +\d json_http.things + Foreign table "json_http.things" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + id | integer | | not null | | + data | json | | not null | | +Server: http_json_loopback +FDW options: (database 'json_test', table_name 'things', engine 'MergeTree') + +-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422 +INSERT INTO json_bin.things VALUES + (1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'), + (2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}') +; +ERROR: pg_clickhouse: could not prepare insert - unsupported column type: JSON +INSERT INTO json_http.things VALUES + (1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'), + (2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'), + (3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'), + (4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}') +; +SELECT * FROM json_bin.things ORDER BY id; +ERROR: pg_clickhouse: unsupported column type: JSON +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC +SELECT * FROM json_http.things ORDER BY id; + id | data +----+------------------------------------------------------------ + 1 | {"id":"1","name":"widget","size":"large","stocked":true} + 2 | {"id":"2","name":"sprocket","size":"small","stocked":true} + 3 | {"id":"3","name":"gizmo","size":"medium","stocked":true} + 4 | {"id":"4","name":"doodad","size":"large","stocked":false} +(4 rows) + +SELECT clickhouse_raw_query('DROP DATABASE json_test'); + clickhouse_raw_query +---------------------- + +(1 row) + +DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback; +DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback; +DROP SERVER binary_json_loopback CASCADE; +NOTICE: drop cascades to foreign table json_bin.things +DROP SERVER http_json_loopback CASCADE; +NOTICE: drop cascades to foreign table json_http.things diff --git a/test/expected/json_2.out b/test/expected/json_2.out new file mode 100644 index 0000000..7de0c31 --- /dev/null +++ b/test/expected/json_2.out @@ -0,0 +1,71 @@ +SET datestyle = 'ISO'; +CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary'); +CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http'); +CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback; +CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback; +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test'); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query('CREATE DATABASE json_test'); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query($$ + CREATE TABLE json_test.things ( + id Int32 NOT NULL, + data JSON NOT NULL + ) ENGINE = MergeTree PARTITION BY id ORDER BY (id); +$$); + clickhouse_raw_query +---------------------- + +(1 row) + +CREATE SCHEMA json_bin; +CREATE SCHEMA json_http; +IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO json_bin; +ERROR: pg_clickhouse: could not map things.data type <'json')> +\d json_bin.things +IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO json_http; +ERROR: pg_clickhouse: could not map things.data type <'json')> +\d json_http.things +-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422 +INSERT INTO json_bin.things VALUES + (1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'), + (2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}') +; +ERROR: relation "json_bin.things" does not exist +LINE 1: INSERT INTO json_bin.things VALUES + ^ +INSERT INTO json_http.things VALUES + (1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'), + (2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'), + (3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'), + (4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}') +; +ERROR: relation "json_http.things" does not exist +LINE 1: INSERT INTO json_http.things VALUES + ^ +SELECT * FROM json_bin.things ORDER BY id; +ERROR: relation "json_bin.things" does not exist +LINE 1: SELECT * FROM json_bin.things ORDER BY id; + ^ +SELECT * FROM json_http.things ORDER BY id; +ERROR: relation "json_http.things" does not exist +LINE 1: SELECT * FROM json_http.things ORDER BY id; + ^ +SELECT clickhouse_raw_query('DROP DATABASE json_test'); + clickhouse_raw_query +---------------------- + +(1 row) + +DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback; +DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback; +DROP SERVER binary_json_loopback CASCADE; +DROP SERVER http_json_loopback CASCADE; diff --git a/test/expected/result_map.txt b/test/expected/result_map.txt index 16736c3..459247d 100644 --- a/test/expected/result_map.txt +++ b/test/expected/result_map.txt @@ -100,3 +100,16 @@ import_schema.sql ClickHouse | File ------------|------------------- 22-25 | import_schema.out + +json.sql +-------- + + Postgres | File +----------|---------- + 13-18 | json.out + + ClickHouse | File +------------|------------ + 25.8+ | json.out + 24.8-25.3 | json_1.out + 22-24.3 | json_2.out diff --git a/test/sql/json.sql b/test/sql/json.sql new file mode 100644 index 0000000..532a862 --- /dev/null +++ b/test/sql/json.sql @@ -0,0 +1,43 @@ +SET datestyle = 'ISO'; +CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary'); +CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http'); +CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback; +CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback; + +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test'); +SELECT clickhouse_raw_query('CREATE DATABASE json_test'); +SELECT clickhouse_raw_query($$ + CREATE TABLE json_test.things ( + id Int32 NOT NULL, + data JSON NOT NULL + ) ENGINE = MergeTree PARTITION BY id ORDER BY (id); +$$); + +CREATE SCHEMA json_bin; +CREATE SCHEMA json_http; +IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO json_bin; +\d json_bin.things +IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO json_http; +\d json_http.things + +-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422 +INSERT INTO json_bin.things VALUES + (1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'), + (2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}') +; + +INSERT INTO json_http.things VALUES + (1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'), + (2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'), + (3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'), + (4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}') +; + +SELECT * FROM json_bin.things ORDER BY id; +SELECT * FROM json_http.things ORDER BY id; + +SELECT clickhouse_raw_query('DROP DATABASE json_test'); +DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback; +DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback; +DROP SERVER binary_json_loopback CASCADE; +DROP SERVER http_json_loopback CASCADE;