diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index dca93f6..0a02845 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -49,6 +49,9 @@ For `operations`, the SQL is often set via `.queries(["SQL"])`. This method can ### 4. Assertions Assertions in Dataform are strict. They expect a single `SELECT` statement. Prepending a `SET` statement will cause a syntax error in BigQuery because assertions are often wrapped in subqueries or views by Dataform. We explicitly skip assertions in this package. +### 5. Outer DECLARE Detection +Operations where `DECLARE` is the first statement at the outer level are automatically skipped. BigQuery requires `DECLARE` before any other statements in a script, so prepending `SET @@reservation` would fail. The package strips leading whitespace and SQL comments (`--`, `#`, `/* */`) to reliably detect this case. `DECLARE` inside `BEGIN...END` or `EXECUTE IMMEDIATE` is not flagged — reservation is applied normally in those cases. + ## Release Process See [CONTRIBUTING.md](../CONTRIBUTING.md#release-process) for the full release workflow steps. diff --git a/README.md b/README.md index a7c8341..f6d3503 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,13 @@ autoAssignActions(RESERVATION_CONFIG); With automated assignement, you don't need to edit your individual action files — the package handles everything globally. +#### Limitations of Automated Assignment + +* `DECLARE` at the top level of the SQL (the first real statement after whitespace/comments). + The automation skips operations where `DECLARE` is the first statement at the outer level. BigQuery requires `DECLARE` to appear before any other statements in a script, so prepending `SET @@reservation` would cause a syntax error. This detection works automatically without any configuration needed. + + Use manual assignment for any actions that require top-level `DECLARE` statements. + ### Manual Assignment (Optional) For more granular control, you can manually apply reservations per file. Create a setter function in your global scope under `/includes` directory: diff --git a/index.js b/index.js index 68ed98e..7c7bb1f 100644 --- a/index.js +++ b/index.js @@ -107,6 +107,42 @@ function createReservationSetter(config) { } } +/** + * Checks if SQL has a DECLARE statement at the outer (top) level. + * DECLARE inside BEGIN...END blocks or EXECUTE IMMEDIATE strings is not flagged. + * @param {string|Array} sql - SQL query or array of queries + * @returns {boolean} True if outer DECLARE detected + */ +function hasOuterDeclare(sql) { + if (Array.isArray(sql)) { + return sql.some(q => hasOuterDeclare(q)) + } + + // Strip leading whitespace and SQL comments to find the first real statement + let s = (sql || '').trimStart() + let changed = true + while (changed) { + changed = false + if (s.startsWith('--')) { + const idx = s.indexOf('\n') + s = idx === -1 ? '' : s.slice(idx + 1).trimStart() + changed = true + } + if (s.startsWith('#')) { + const idx = s.indexOf('\n') + s = idx === -1 ? '' : s.slice(idx + 1).trimStart() + changed = true + } + if (s.startsWith('/*')) { + const idx = s.indexOf('*/') + s = idx === -1 ? '' : s.slice(idx + 2).trimStart() + changed = true + } + } + + return /^DECLARE\b/i.test(s) +} + /** * Helper to apply reservation to a single action * @param {Object} action - Dataform action object @@ -150,6 +186,12 @@ function applyReservationToAction(action, configSets) { const originalQueriesFn = action.queries action.queries = function (queries) { let queriesArray = queries + + // Check for outer DECLARE before wrapping + if (hasOuterDeclare(queries)) { + return originalQueriesFn.apply(this, [queries]) + } + if (typeof queries === 'function') { queriesArray = (ctx) => { const result = queries(ctx) @@ -190,13 +232,16 @@ function applyReservationToAction(action, configSets) { } // 2. Try contextableQueries (Operations Builders before resolution) else if (action.contextableQueries) { - if (Array.isArray(action.contextableQueries)) { - if (!action.contextableQueries.includes(statement)) { - action.contextableQueries.unshift(statement) - } - } else if (typeof action.contextableQueries === 'string') { - if (!action.contextableQueries.includes(statement)) { - action.contextableQueries = [statement, action.contextableQueries] + // Skip if there is an outer DECLARE + if (!hasOuterDeclare(action.contextableQueries)) { + if (Array.isArray(action.contextableQueries)) { + if (!action.contextableQueries.includes(statement)) { + action.contextableQueries.unshift(statement) + } + } else if (typeof action.contextableQueries === 'string') { + if (!action.contextableQueries.includes(statement)) { + action.contextableQueries = [statement, action.contextableQueries] + } } } } @@ -219,13 +264,16 @@ function applyReservationToAction(action, configSets) { } // 4. Try proto.queries (Compiled Operations or Resolved Builders) else if (proto.queries) { - if (Array.isArray(proto.queries)) { - if (!proto.queries.includes(statement)) { - proto.queries.unshift(statement) - } - } else if (typeof proto.queries === 'string') { - if (!proto.queries.includes(statement)) { - proto.queries = [statement, proto.queries] + // Skip if there is an outer DECLARE + if (!hasOuterDeclare(proto.queries)) { + if (Array.isArray(proto.queries)) { + if (!proto.queries.includes(statement)) { + proto.queries.unshift(statement) + } + } else if (typeof proto.queries === 'string') { + if (!proto.queries.includes(statement)) { + proto.queries = [statement, proto.queries] + } } } } diff --git a/test/index.test.js b/test/index.test.js index 1b97024..36c2ff9 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -488,5 +488,113 @@ describe('Dataform package', () => { ).length expect(reservationCount).toBe(1) }) + + test('should handle mixed case DECLARE at outer level', () => { + const config = [ + { + tag: 'test', + reservation: 'projects/test/locations/US/reservations/prod', + actions: ['test-project.test-schema.mixed_case'] + } + ] + + autoAssignActions(config) + global.operate('mixed_case').queries(` + declare x INT64 DEFAULT 1; + SELECT x; + `) + + const action = global.dataform.actions[0] + expect(action.proto.queries).not.toContain('SET @@reservation=\'projects/test/locations/US/reservations/prod\';') + }) + + test('should not skip DECLARE inside BEGIN...END block', () => { + const config = [ + { + tag: 'test', + reservation: 'projects/test/locations/US/reservations/prod', + actions: ['test-project.test-schema.begin_declare'] + } + ] + + autoAssignActions(config) + global.operate('begin_declare').queries(` + --DECLARE x INT64 DEFAULT 1; + # comment + BEGIN + DECLARE x INT64 DEFAULT 1; + SELECT x; + END; + `) + + const action = global.dataform.actions[0] + expect(action.proto.queries[0]).toBe('SET @@reservation=\'projects/test/locations/US/reservations/prod\';') + }) + + test('should not skip DECLARE inside EXECUTE IMMEDIATE', () => { + const config = [ + { + tag: 'test', + reservation: 'projects/test/locations/US/reservations/prod', + actions: ['test-project.test-schema.exec_declare'] + } + ] + + autoAssignActions(config) + global.operate('exec_declare').queries(` + /* + block comment + DECLARE x INT64; + */ + EXECUTE IMMEDIATE "DECLARE x INT64; SET x = 1; SELECT x;" + `) + + const action = global.dataform.actions[0] + expect(action.proto.queries[0]).toBe('SET @@reservation=\'projects/test/locations/US/reservations/prod\';') + }) + + test('should skip DECLARE after SQL comments', () => { + const config = [ + { + tag: 'test', + reservation: 'projects/test/locations/US/reservations/prod', + actions: ['test-project.test-schema.comment_declare'] + } + ] + + autoAssignActions(config) + global.operate('comment_declare').queries(` + -- set up variables + # comment + /* block comment */ + /* + multi-line block comment + */ + DECLARE x INT64 DEFAULT 1; + SELECT x; + `) + + const action = global.dataform.actions[0] + expect(action.proto.queries).not.toContain('SET @@reservation=\'projects/test/locations/US/reservations/prod\';') + }) + + test('should handle array of queries with outer DECLARE', () => { + const config = [ + { + tag: 'test', + reservation: 'projects/test/locations/US/reservations/prod', + actions: ['test-project.test-schema.array_queries'] + } + ] + + autoAssignActions(config) + global.operate('array_queries').queries([ + 'DECLARE x INT64 DEFAULT 1;', + 'SELECT x;' + ]) + + const action = global.dataform.actions[0] + expect(action.proto.queries).not.toContain('SET @@reservation=\'projects/test/locations/US/reservations/prod\';') + }) }) })