diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b94b14c..a6abd9a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +### Release [1.9.6] + +#### Bugs +* Alter destination table along with mv using `on_schema_changes` ([#534](https://github.com/ClickHouse/dbt-clickhouse/pull/534)) + ### Release [1.9.5], 2025-10-20 #### Bugs @@ -18,7 +23,6 @@ * All documentation is now hosted on [clickhouse.com/docs](https://clickhouse.com/docs/integrations/dbt). The README in this repository has been updated to reflect this change and now includes a quick start guide and links to the full documentation. ([#526](https://github.com/ClickHouse/dbt-clickhouse/pull/526)). * List of supported/tested ClickHouse versions has been updated to include only [actively supported versions](https://github.com/ClickHouse/ClickHouse/blob/master/SECURITY.md) ([#517](https://github.com/ClickHouse/dbt-clickhouse/pull/517)). - ### Release [1.9.3], 2025-09-08 #### Bugs diff --git a/dbt/include/clickhouse/macros/materializations/materialized_view.sql b/dbt/include/clickhouse/macros/materializations/materialized_view.sql index 3b3f48ad..2c86ad38 100644 --- a/dbt/include/clickhouse/macros/materializations/materialized_view.sql +++ b/dbt/include/clickhouse/macros/materializations/materialized_view.sql @@ -86,6 +86,15 @@ select 1 {%- endcall %} + {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%} + {{ log('on_schema_change strategy for destination table of MV: ' + on_schema_change, info=True) }} + {%- if on_schema_change != 'ignore' -%} + {%- set column_changes = adapter.check_incremental_schema_changes(on_schema_change, existing_relation, sql) -%} + {% if column_changes %} + {% do clickhouse__apply_column_changes(column_changes, existing_relation) %} + {% set existing_relation = load_cached_relation(this) %} + {% endif %} + {%- endif %} -- try to alter view first to replace sql, else drop and create {{ clickhouse__update_mvs(target_relation, cluster_clause, refreshable_clause, views) }} diff --git a/tests/integration/adapter/materialized_view/test_materialized_view.py b/tests/integration/adapter/materialized_view/test_materialized_view.py index 73e3ca57..10616164 100644 --- a/tests/integration/adapter/materialized_view/test_materialized_view.py +++ b/tests/integration/adapter/materialized_view/test_materialized_view.py @@ -27,6 +27,7 @@ materialized='materialized_view' if var('run_type', '') != 'view_conversion' else 'view', engine='MergeTree()', order_by='(id)', + on_schema_change='sync_all_columns', schema='catchup' if var('run_type', '') == 'catchup' else 'custom_schema', **({'catchup': False} if var('run_type', '') == 'catchup' else {}) ) }} @@ -53,7 +54,8 @@ when name like 'Dade' and age != 11 then 'crash override' when name like 'Kate' then 'acid burn' else 'N/A' - end as hacker_alias + end as hacker_alias, + id as id2 from {{ source('raw', 'people') }} where department = 'engineering' @@ -245,6 +247,19 @@ def test_update_incremental(self, project): ) assert len(result) == 2 + # assert that the destination table is updated with the new column + table_description_after_update = project.run_sql( + f"DESCRIBE TABLE {schema}.hackers", fetch="all" + ) + assert any(col[0] == "id2" and col[1] == "Int32" for col in table_description_after_update) + + # run again without extended schema, to make sure table is updated back without the id2 column + run_dbt() + table_description_after_revert_update = project.run_sql( + f"DESCRIBE TABLE {schema}.hackers", fetch="all" + ) + assert not any(col[0] == "id2" for col in table_description_after_revert_update) + def test_update_full_refresh(self, project): schema = quote_identifier(project.test_schema + "_custom_schema") # create our initial materialized view diff --git a/tests/integration/adapter/materialized_view/test_multiple_materialized_views.py b/tests/integration/adapter/materialized_view/test_multiple_materialized_views.py index 4678c9ee..377c0a99 100644 --- a/tests/integration/adapter/materialized_view/test_multiple_materialized_views.py +++ b/tests/integration/adapter/materialized_view/test_multiple_materialized_views.py @@ -28,6 +28,7 @@ materialized='materialized_view', engine='MergeTree()', order_by='(id)', + on_schema_change='sync_all_columns', schema='custom_schema_for_multiple_mv', ) }} @@ -70,7 +71,8 @@ when name like 'Dade' and age != 11 then 'crash override' when name like 'Kate' then 'acid burn' else 'N/A' - end as hacker_alias + end as hacker_alias, + id as id2 from {{ source('raw', 'people') }} where department = 'engineering' --mv1:end @@ -82,7 +84,8 @@ id, name, -- sales people are not cool enough to have a hacker alias - 'N/A' as hacker_alias + 'N/A' as hacker_alias, + id as id2 from {{ source('raw', 'people') }} where department = 'sales' --mv2:end @@ -219,6 +222,17 @@ def test_update_incremental(self, project): assert result[0][0] == "crash_override" assert result[1][0] == "zero cool" + # assert that the destination table is updated with the new column + table_description_after_update = project.run_sql(f"DESCRIBE {schema}.hackers", fetch="all") + assert any(col[0] == "id2" and col[1] == "Int32" for col in table_description_after_update) + + # run again without extended schema, to make sure table is updated back without the id2 column + run_dbt() + table_description_after_revert_update = project.run_sql( + f"DESCRIBE TABLE {schema}.hackers", fetch="all" + ) + assert not any(col[0] == "id2" for col in table_description_after_revert_update) + def test_update_full_refresh(self, project): schema = quote_identifier(project.test_schema + "_custom_schema_for_multiple_mv") # create our initial materialized view