Skip to content

Commit d4d9e2f

Browse files
AmandeepSinghCSAmandeepSinghCS
authored
removed reference columns from custom sql statement rule (#190)
1 parent f578f05 commit d4d9e2f

13 files changed

Lines changed: 60 additions & 157 deletions

‎clouddq/templates/dbt/macros/create_rule_binding_view.sql‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ validation_results AS (
8585

8686
{% for rule_id, rule_configs in rule_configs_dict.items() %}
8787
{%- if rule_configs.get('rule_type') == "CUSTOM_SQL_STATEMENT" -%}
88-
{{ validate_complex_rule(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name, include_reference_columns, configs) }}
88+
{{ validate_complex_rule(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name, configs) }}
8989
{%- else -%}
9090
{{ validate_simple_rule(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name, include_reference_columns, configs) }}
9191
{%- endif -%}
@@ -107,12 +107,6 @@ all_validation_results AS (
107107
r.simple_rule_row_is_valid AS simple_rule_row_is_valid,
108108
r.complex_rule_validation_errors_count AS complex_rule_validation_errors_count,
109109
r.complex_rule_validation_success_flag AS complex_rule_validation_success_flag,
110-
{% for ref_column_name in include_reference_columns %}
111-
r.{{ ref_column_name }} AS {{ ref_column_name }},
112-
{% if loop.last %}
113-
{{ '\n' }}
114-
{% endif %}
115-
{%- endfor -%}
116110
(SELECT COUNT(*) FROM data) AS rows_validated,
117111
last_mod.last_modified,
118112
'{{ metadata|tojson }}' AS metadata_json_string,

‎clouddq/templates/dbt/macros/failed_records_query.sql‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ last_mod AS (
7272
validation_results AS (
7373

7474
{%- if rule_type == "CUSTOM_SQL_STATEMENT" -%}
75-
{{ validate_complex_rule_failed_records_query(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name, include_reference_columns) }}
75+
{{ validate_complex_rule_failed_records_query(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name) }}
7676
{%- else -%}
7777
{{ validate_simple_rule_failed_records_query(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name, include_reference_columns) }}
7878
{%- endif -%}
@@ -88,9 +88,15 @@ all_validation_results AS (
8888
r.simple_rule_row_is_valid AS _dq_validation_simple_rule_row_is_valid,
8989
r.complex_rule_validation_errors_count AS _dq_validation_complex_rule_validation_errors_count,
9090
r.complex_rule_validation_success_flag AS _dq_validation_complex_rule_validation_success_flag,
91-
{% for ref_column_name in include_reference_columns %}
92-
r.{{ ref_column_name }} AS {{ ref_column_name }},
93-
{%- endfor -%}
91+
{{ '\n' }}
92+
{%- if rule_type != "CUSTOM_SQL_STATEMENT" -%}
93+
{% for ref_column_name in include_reference_columns %}
94+
r.{{ ref_column_name }} AS {{ ref_column_name }},
95+
{%- endfor -%}
96+
{%- else -%}
97+
r.custom_sql_statement_validation_errors,
98+
{%- endif -%}
99+
{{ '\n' }}
94100
FROM
95101
validation_results r
96102
)

‎clouddq/templates/dbt/macros/macros.sql‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,6 @@
2020
'{{ fully_qualified_table_name }}' AS table_id,
2121
'{{ column_name }}' AS column_id,
2222
data.{{ column_name }} AS column_value,
23-
{% for ref_column_name in include_reference_columns %}
24-
data.{{ ref_column_name }} AS {{ ref_column_name }},
25-
{%- endfor -%}
2623
{% if rule_configs.get("dimension") %}
2724
'{{ rule_configs.get("dimension") }}' AS dimension,
2825
{% else %}
@@ -55,17 +52,14 @@
5552
zero_record.rule_binding_id = data.rule_binding_id
5653
{% endmacro -%}
5754

58-
{% macro validate_complex_rule(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name, include_reference_columns, configs) -%}
55+
{% macro validate_complex_rule(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name, configs) -%}
5956
SELECT
6057
CURRENT_TIMESTAMP() AS execution_ts,
6158
'{{ rule_binding_id }}' AS rule_binding_id,
6259
'{{ rule_id }}' AS rule_id,
6360
'{{ fully_qualified_table_name }}' AS table_id,
6461
CAST(NULL AS STRING) AS column_id,
6562
NULL AS column_value,
66-
{% for ref_column_name in include_reference_columns %}
67-
custom_sql_statement_validation_errors.{{ ref_column_name }} AS {{ ref_column_name }},
68-
{%- endfor -%}
6963
{% if rule_configs.get("dimension") %}
7064
'{{ rule_configs.get("dimension") }}' AS dimension,
7165
{% else %}
@@ -138,16 +132,14 @@
138132
zero_record.rule_binding_id = data.rule_binding_id
139133
{% endmacro -%}
140134

141-
{% macro validate_complex_rule_failed_records_query(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name, include_reference_columns) -%}
135+
{% macro validate_complex_rule_failed_records_query(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name) -%}
142136
SELECT
143137
'{{ rule_binding_id }}' AS rule_binding_id,
144138
'{{ rule_id }}' AS rule_id,
145139
'{{ fully_qualified_table_name }}' AS table_id,
146140
CAST(NULL AS STRING) AS column_id,
147141
NULL AS column_value,
148-
{% for ref_column_name in include_reference_columns %}
149-
custom_sql_statement_validation_errors.{{ ref_column_name }} AS {{ ref_column_name }},
150-
{%- endfor -%}
142+
custom_sql_statement_validation_errors,
151143
{% if rule_configs.get("dimension") %}
152144
'{{ rule_configs.get("dimension") }}' AS dimension,
153145
{% else %}

‎tests/resources/bq_native_default_partitioned_sql_expected.sql‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,6 @@ CURRENT_TIMESTAMP() AS execution_ts,
4040
'<your-gcp-project-id>.austin_311.contact_details_ingestion_time_partitioned' AS table_id,
4141
CAST(NULL AS STRING) AS column_id,
4242
NULL AS column_value,
43-
custom_sql_statement_validation_errors.row_id AS row_id,
44-
custom_sql_statement_validation_errors.contact_type AS contact_type,
45-
custom_sql_statement_validation_errors.value AS value,
4643
CAST(NULL AS STRING) AS dimension,
4744
CAST(NULL AS BOOLEAN) AS simple_rule_row_is_valid,
4845
TRUE AS skip_null_count,
@@ -79,9 +76,7 @@ validation_results AS (SELECT
7976
'<your-gcp-project-id>.austin_311.contact_details_ingestion_time_partitioned' AS table_id,
8077
CAST(NULL AS STRING) AS column_id,
8178
NULL AS column_value,
82-
custom_sql_statement_validation_errors.row_id AS row_id,
83-
custom_sql_statement_validation_errors.contact_type AS contact_type,
84-
custom_sql_statement_validation_errors.value AS value,
79+
custom_sql_statement_validation_errors,
8580
CAST(NULL AS STRING) AS dimension,
8681
CAST(NULL AS BOOLEAN) AS simple_rule_row_is_valid,
8782
TRUE AS skip_null_count,
@@ -126,9 +121,8 @@ CAST(r.dimension AS STRING) AS _dq_validation_dimension,
126121
r.simple_rule_row_is_valid AS _dq_validation_simple_rule_row_is_valid,
127122
r.complex_rule_validation_errors_count AS _dq_validation_complex_rule_validation_errors_count,
128123
r.complex_rule_validation_success_flag AS _dq_validation_complex_rule_validation_success_flag,
129-
r.row_id AS row_id,
130-
r.contact_type AS contact_type,
131-
r.value AS value,FROM
124+
r.custom_sql_statement_validation_errors,
125+
FROM
132126
validation_results r
133127
)
134128
SELECT
@@ -172,9 +166,6 @@ CURRENT_TIMESTAMP() AS execution_ts,
172166
'<your-gcp-project-id>.austin_311.contact_details_ingestion_time_partitioned' AS table_id,
173167
'value' AS column_id,
174168
data.value AS column_value,
175-
data.row_id AS row_id,
176-
data.contact_type AS contact_type,
177-
data.value AS value,
178169
CAST(NULL AS STRING) AS dimension,
179170
CASE
180171
WHEN value IS NOT NULL THEN TRUE
@@ -243,7 +234,8 @@ r.complex_rule_validation_errors_count AS _dq_validation_complex_rule_validation
243234
r.complex_rule_validation_success_flag AS _dq_validation_complex_rule_validation_success_flag,
244235
r.row_id AS row_id,
245236
r.contact_type AS contact_type,
246-
r.value AS value,FROM
237+
r.value AS value,
238+
FROM
247239
validation_results r
248240
)
249241
SELECT
@@ -276,9 +268,6 @@ r.skip_null_count AS skip_null_count,
276268
r.simple_rule_row_is_valid AS simple_rule_row_is_valid,
277269
r.complex_rule_validation_errors_count AS complex_rule_validation_errors_count,
278270
r.complex_rule_validation_success_flag AS complex_rule_validation_success_flag,
279-
r.row_id AS row_id,
280-
r.contact_type AS contact_type,
281-
r.value AS value,
282271
(SELECT COUNT(*) FROM data) AS rows_validated,
283272
last_mod.last_modified,
284273
'{"brand": "one"}' AS metadata_json_string,

‎tests/resources/bq_native_partitioned_sql_expected.sql‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,6 @@ CURRENT_TIMESTAMP() AS execution_ts,
4040
'<your-gcp-project-id>.austin_311.contact_details_partitioned' AS table_id,
4141
CAST(NULL AS STRING) AS column_id,
4242
NULL AS column_value,
43-
custom_sql_statement_validation_errors.row_id AS row_id,
44-
custom_sql_statement_validation_errors.contact_type AS contact_type,
45-
custom_sql_statement_validation_errors.value AS value,
4643
CAST(NULL AS STRING) AS dimension,
4744
CAST(NULL AS BOOLEAN) AS simple_rule_row_is_valid,
4845
TRUE AS skip_null_count,
@@ -79,9 +76,7 @@ validation_results AS (SELECT
7976
'<your-gcp-project-id>.austin_311.contact_details_partitioned' AS table_id,
8077
CAST(NULL AS STRING) AS column_id,
8178
NULL AS column_value,
82-
custom_sql_statement_validation_errors.row_id AS row_id,
83-
custom_sql_statement_validation_errors.contact_type AS contact_type,
84-
custom_sql_statement_validation_errors.value AS value,
79+
custom_sql_statement_validation_errors,
8580
CAST(NULL AS STRING) AS dimension,
8681
CAST(NULL AS BOOLEAN) AS simple_rule_row_is_valid,
8782
TRUE AS skip_null_count,
@@ -126,9 +121,8 @@ CAST(r.dimension AS STRING) AS _dq_validation_dimension,
126121
r.simple_rule_row_is_valid AS _dq_validation_simple_rule_row_is_valid,
127122
r.complex_rule_validation_errors_count AS _dq_validation_complex_rule_validation_errors_count,
128123
r.complex_rule_validation_success_flag AS _dq_validation_complex_rule_validation_success_flag,
129-
r.row_id AS row_id,
130-
r.contact_type AS contact_type,
131-
r.value AS value,FROM
124+
r.custom_sql_statement_validation_errors,
125+
FROM
132126
validation_results r
133127
)
134128
SELECT
@@ -172,9 +166,6 @@ CURRENT_TIMESTAMP() AS execution_ts,
172166
'<your-gcp-project-id>.austin_311.contact_details_partitioned' AS table_id,
173167
'value' AS column_id,
174168
data.value AS column_value,
175-
data.row_id AS row_id,
176-
data.contact_type AS contact_type,
177-
data.value AS value,
178169
CAST(NULL AS STRING) AS dimension,
179170
CASE
180171
WHEN value IS NOT NULL THEN TRUE
@@ -243,7 +234,8 @@ r.complex_rule_validation_errors_count AS _dq_validation_complex_rule_validation
243234
r.complex_rule_validation_success_flag AS _dq_validation_complex_rule_validation_success_flag,
244235
r.row_id AS row_id,
245236
r.contact_type AS contact_type,
246-
r.value AS value,FROM
237+
r.value AS value,
238+
FROM
247239
validation_results r
248240
)
249241
SELECT
@@ -276,9 +268,6 @@ r.skip_null_count AS skip_null_count,
276268
r.simple_rule_row_is_valid AS simple_rule_row_is_valid,
277269
r.complex_rule_validation_errors_count AS complex_rule_validation_errors_count,
278270
r.complex_rule_validation_success_flag AS complex_rule_validation_success_flag,
279-
r.row_id AS row_id,
280-
r.contact_type AS contact_type,
281-
r.value AS value,
282271
(SELECT COUNT(*) FROM data) AS rows_validated,
283272
last_mod.last_modified,
284273
'{"brand": "one"}' AS metadata_json_string,

‎tests/resources/bq_native_sql_expected.sql‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,6 @@ CURRENT_TIMESTAMP() AS execution_ts,
4040
'<your-gcp-project-id>.austin_311.311_service_requests' AS table_id,
4141
'unique_key' AS column_id,
4242
data.unique_key AS column_value,
43-
data.unique_key AS unique_key,
44-
data.complaint_description AS complaint_description,
45-
data.source AS source,
46-
data.status AS status,
4743
CAST(NULL AS STRING) AS dimension,
4844
CASE
4945
WHEN unique_key IS NOT NULL THEN TRUE
@@ -114,7 +110,8 @@ CAST(NULL AS BOOLEAN) AS complex_rule_validation_success_flag,
114110
r.unique_key AS unique_key,
115111
r.complaint_description AS complaint_description,
116112
r.source AS source,
117-
r.status AS status,FROM
113+
r.status AS status,
114+
FROM
118115
validation_results r
119116
)
120117
SELECT
@@ -147,10 +144,6 @@ r.skip_null_count AS skip_null_count,
147144
r.simple_rule_row_is_valid AS simple_rule_row_is_valid,
148145
r.complex_rule_validation_errors_count AS complex_rule_validation_errors_count,
149146
r.complex_rule_validation_success_flag AS complex_rule_validation_success_flag,
150-
r.unique_key AS unique_key,
151-
r.complaint_description AS complaint_description,
152-
r.source AS source,
153-
r.status AS status,
154147
(SELECT COUNT(*) FROM data) AS rows_validated,
155148
last_mod.last_modified,
156149
'{"brand": "one"}' AS metadata_json_string,

‎tests/resources/dataplex_gcs_metadata_sql_expected.sql‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,6 @@ CURRENT_TIMESTAMP() AS execution_ts,
4040
'<your-gcp-project-id>.<your_dataplex_zone_name>.asset_bucket' AS table_id,
4141
CAST(NULL AS STRING) AS column_id,
4242
NULL AS column_value,
43-
custom_sql_statement_validation_errors.row_id AS row_id,
44-
custom_sql_statement_validation_errors.contact_type AS contact_type,
45-
custom_sql_statement_validation_errors.value AS value,
4643
CAST(NULL AS STRING) AS dimension,
4744
CAST(NULL AS BOOLEAN) AS simple_rule_row_is_valid,
4845
TRUE AS skip_null_count,
@@ -79,9 +76,7 @@ validation_results AS (SELECT
7976
'<your-gcp-project-id>.<your_dataplex_zone_name>.asset_bucket' AS table_id,
8077
CAST(NULL AS STRING) AS column_id,
8178
NULL AS column_value,
82-
custom_sql_statement_validation_errors.row_id AS row_id,
83-
custom_sql_statement_validation_errors.contact_type AS contact_type,
84-
custom_sql_statement_validation_errors.value AS value,
79+
custom_sql_statement_validation_errors,
8580
CAST(NULL AS STRING) AS dimension,
8681
CAST(NULL AS BOOLEAN) AS simple_rule_row_is_valid,
8782
TRUE AS skip_null_count,
@@ -126,9 +121,8 @@ CAST(r.dimension AS STRING) AS _dq_validation_dimension,
126121
r.simple_rule_row_is_valid AS _dq_validation_simple_rule_row_is_valid,
127122
r.complex_rule_validation_errors_count AS _dq_validation_complex_rule_validation_errors_count,
128123
r.complex_rule_validation_success_flag AS _dq_validation_complex_rule_validation_success_flag,
129-
r.row_id AS row_id,
130-
r.contact_type AS contact_type,
131-
r.value AS value,FROM
124+
r.custom_sql_statement_validation_errors,
125+
FROM
132126
validation_results r
133127
)
134128
SELECT
@@ -172,9 +166,6 @@ CURRENT_TIMESTAMP() AS execution_ts,
172166
'<your-gcp-project-id>.<your_dataplex_zone_name>.asset_bucket' AS table_id,
173167
'value' AS column_id,
174168
data.value AS column_value,
175-
data.row_id AS row_id,
176-
data.contact_type AS contact_type,
177-
data.value AS value,
178169
CAST(NULL AS STRING) AS dimension,
179170
CASE
180171
WHEN value IS NOT NULL THEN TRUE
@@ -243,7 +234,8 @@ r.complex_rule_validation_errors_count AS _dq_validation_complex_rule_validation
243234
r.complex_rule_validation_success_flag AS _dq_validation_complex_rule_validation_success_flag,
244235
r.row_id AS row_id,
245236
r.contact_type AS contact_type,
246-
r.value AS value,FROM
237+
r.value AS value,
238+
FROM
247239
validation_results r
248240
)
249241
SELECT
@@ -276,9 +268,6 @@ r.skip_null_count AS skip_null_count,
276268
r.simple_rule_row_is_valid AS simple_rule_row_is_valid,
277269
r.complex_rule_validation_errors_count AS complex_rule_validation_errors_count,
278270
r.complex_rule_validation_success_flag AS complex_rule_validation_success_flag,
279-
r.row_id AS row_id,
280-
r.contact_type AS contact_type,
281-
r.value AS value,
282271
(SELECT COUNT(*) FROM data) AS rows_validated,
283272
last_mod.last_modified,
284273
'{"brand": "one"}' AS metadata_json_string,

‎tests/resources/dataplex_gcs_partitioned_metadata_sql_expected.sql‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,6 @@ CURRENT_TIMESTAMP() AS execution_ts,
4040
'<your-gcp-project-id>.<your_dataplex_zone_name>.partitioned_gcs_asset' AS table_id,
4141
CAST(NULL AS STRING) AS column_id,
4242
NULL AS column_value,
43-
custom_sql_statement_validation_errors.row_id AS row_id,
44-
custom_sql_statement_validation_errors.contact_type AS contact_type,
45-
custom_sql_statement_validation_errors.value AS value,
4643
CAST(NULL AS STRING) AS dimension,
4744
CAST(NULL AS BOOLEAN) AS simple_rule_row_is_valid,
4845
TRUE AS skip_null_count,
@@ -79,9 +76,7 @@ validation_results AS (SELECT
7976
'<your-gcp-project-id>.<your_dataplex_zone_name>.partitioned_gcs_asset' AS table_id,
8077
CAST(NULL AS STRING) AS column_id,
8178
NULL AS column_value,
82-
custom_sql_statement_validation_errors.row_id AS row_id,
83-
custom_sql_statement_validation_errors.contact_type AS contact_type,
84-
custom_sql_statement_validation_errors.value AS value,
79+
custom_sql_statement_validation_errors,
8580
CAST(NULL AS STRING) AS dimension,
8681
CAST(NULL AS BOOLEAN) AS simple_rule_row_is_valid,
8782
TRUE AS skip_null_count,
@@ -126,9 +121,8 @@ CAST(r.dimension AS STRING) AS _dq_validation_dimension,
126121
r.simple_rule_row_is_valid AS _dq_validation_simple_rule_row_is_valid,
127122
r.complex_rule_validation_errors_count AS _dq_validation_complex_rule_validation_errors_count,
128123
r.complex_rule_validation_success_flag AS _dq_validation_complex_rule_validation_success_flag,
129-
r.row_id AS row_id,
130-
r.contact_type AS contact_type,
131-
r.value AS value,FROM
124+
r.custom_sql_statement_validation_errors,
125+
FROM
132126
validation_results r
133127
)
134128
SELECT
@@ -172,9 +166,6 @@ CURRENT_TIMESTAMP() AS execution_ts,
172166
'<your-gcp-project-id>.<your_dataplex_zone_name>.partitioned_gcs_asset' AS table_id,
173167
'value' AS column_id,
174168
data.value AS column_value,
175-
data.row_id AS row_id,
176-
data.contact_type AS contact_type,
177-
data.value AS value,
178169
CAST(NULL AS STRING) AS dimension,
179170
CASE
180171
WHEN value IS NOT NULL THEN TRUE
@@ -243,7 +234,8 @@ r.complex_rule_validation_errors_count AS _dq_validation_complex_rule_validation
243234
r.complex_rule_validation_success_flag AS _dq_validation_complex_rule_validation_success_flag,
244235
r.row_id AS row_id,
245236
r.contact_type AS contact_type,
246-
r.value AS value,FROM
237+
r.value AS value,
238+
FROM
247239
validation_results r
248240
)
249241
SELECT
@@ -276,9 +268,6 @@ r.skip_null_count AS skip_null_count,
276268
r.simple_rule_row_is_valid AS simple_rule_row_is_valid,
277269
r.complex_rule_validation_errors_count AS complex_rule_validation_errors_count,
278270
r.complex_rule_validation_success_flag AS complex_rule_validation_success_flag,
279-
r.row_id AS row_id,
280-
r.contact_type AS contact_type,
281-
r.value AS value,
282271
(SELECT COUNT(*) FROM data) AS rows_validated,
283272
last_mod.last_modified,
284273
'{"brand": "one"}' AS metadata_json_string,

‎tests/resources/dataplex_metadata_configs_file_sql_expected.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ validation_results AS (SELECT
7676
'<your-gcp-project-id>.<your_bigquery_dataset_id>.contact_details' AS table_id,
7777
CAST(NULL AS STRING) AS column_id,
7878
NULL AS column_value,
79+
custom_sql_statement_validation_errors,
7980
CAST(NULL AS STRING) AS dimension,
8081
CAST(NULL AS BOOLEAN) AS simple_rule_row_is_valid,
8182
TRUE AS skip_null_count,
@@ -120,6 +121,7 @@ CAST(r.dimension AS STRING) AS _dq_validation_dimension,
120121
r.simple_rule_row_is_valid AS _dq_validation_simple_rule_row_is_valid,
121122
r.complex_rule_validation_errors_count AS _dq_validation_complex_rule_validation_errors_count,
122123
r.complex_rule_validation_success_flag AS _dq_validation_complex_rule_validation_success_flag,
124+
r.custom_sql_statement_validation_errors,
123125
FROM
124126
validation_results r
125127
)

0 commit comments

Comments
 (0)