Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/copilot-instructions.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ For `operations`, the SQL is often set via `.queries(["SQL"])`. This method can
### 4. Assertions
Assertions in Dataform are strict. They expect a single `SELECT` statement. Prepending a `SET` statement will cause a syntax error in BigQuery because assertions are often wrapped in subqueries or views by Dataform. We explicitly skip assertions in this package.

### 5. Outer DECLARE Detection
Operations where `DECLARE` is the first statement at the outer level are automatically skipped. BigQuery requires `DECLARE` before any other statements in a script, so prepending `SET @@reservation` would fail. The package strips leading whitespace and SQL comments (`--`, `#`, `/* */`) to reliably detect this case. `DECLARE` inside `BEGIN...END` or `EXECUTE IMMEDIATE` is not flagged — reservation is applied normally in those cases.

## Release Process

See [CONTRIBUTING.md](../CONTRIBUTING.md#release-process) for the full release workflow steps.
Expand Down
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,13 @@ autoAssignActions(RESERVATION_CONFIG);

With automated assignement, you don't need to edit your individual action files — the package handles everything globally.

#### Limitations of Automated Assignment

* `DECLARE` at the top level of the SQL (the first real statement after whitespace/comments).
The automation skips operations where `DECLARE` is the first statement at the outer level. BigQuery requires `DECLARE` to appear before any other statements in a script, so prepending `SET @@reservation` would cause a syntax error. This detection works automatically without any configuration needed.

Use manual assignment for any actions that require top-level `DECLARE` statements.

### Manual Assignment (Optional)

For more granular control, you can manually apply reservations per file. Create a setter function in your global scope under `/includes` directory:
Expand Down
76 changes: 62 additions & 14 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,42 @@ function createReservationSetter(config) {
}
}

/**
* Checks if SQL has a DECLARE statement at the outer (top) level.
* DECLARE inside BEGIN...END blocks or EXECUTE IMMEDIATE strings is not flagged.
* @param {string|Array} sql - SQL query or array of queries
* @returns {boolean} True if outer DECLARE detected
*/
function hasOuterDeclare(sql) {
if (Array.isArray(sql)) {
return sql.some(q => hasOuterDeclare(q))
}

// Strip leading whitespace and SQL comments to find the first real statement
let s = (sql || '').trimStart()
let changed = true
while (changed) {
changed = false
if (s.startsWith('--')) {
const idx = s.indexOf('\n')
s = idx === -1 ? '' : s.slice(idx + 1).trimStart()
changed = true
}
if (s.startsWith('#')) {
const idx = s.indexOf('\n')
s = idx === -1 ? '' : s.slice(idx + 1).trimStart()
changed = true
}
if (s.startsWith('/*')) {
const idx = s.indexOf('*/')
s = idx === -1 ? '' : s.slice(idx + 2).trimStart()
changed = true
}
}

return /^DECLARE\b/i.test(s)
}

/**
* Helper to apply reservation to a single action
* @param {Object} action - Dataform action object
Expand Down Expand Up @@ -150,6 +186,12 @@ function applyReservationToAction(action, configSets) {
const originalQueriesFn = action.queries
action.queries = function (queries) {
let queriesArray = queries

// Check for outer DECLARE before wrapping
if (hasOuterDeclare(queries)) {
return originalQueriesFn.apply(this, [queries])
}

if (typeof queries === 'function') {
queriesArray = (ctx) => {
const result = queries(ctx)
Expand Down Expand Up @@ -190,13 +232,16 @@ function applyReservationToAction(action, configSets) {
}
// 2. Try contextableQueries (Operations Builders before resolution)
else if (action.contextableQueries) {
if (Array.isArray(action.contextableQueries)) {
if (!action.contextableQueries.includes(statement)) {
action.contextableQueries.unshift(statement)
}
} else if (typeof action.contextableQueries === 'string') {
if (!action.contextableQueries.includes(statement)) {
action.contextableQueries = [statement, action.contextableQueries]
// Skip if there is an outer DECLARE
if (!hasOuterDeclare(action.contextableQueries)) {
if (Array.isArray(action.contextableQueries)) {
if (!action.contextableQueries.includes(statement)) {
action.contextableQueries.unshift(statement)
}
} else if (typeof action.contextableQueries === 'string') {
if (!action.contextableQueries.includes(statement)) {
action.contextableQueries = [statement, action.contextableQueries]
}
}
}
}
Expand All @@ -219,13 +264,16 @@ function applyReservationToAction(action, configSets) {
}
// 4. Try proto.queries (Compiled Operations or Resolved Builders)
else if (proto.queries) {
if (Array.isArray(proto.queries)) {
if (!proto.queries.includes(statement)) {
proto.queries.unshift(statement)
}
} else if (typeof proto.queries === 'string') {
if (!proto.queries.includes(statement)) {
proto.queries = [statement, proto.queries]
// Skip if there is an outer DECLARE
if (!hasOuterDeclare(proto.queries)) {
if (Array.isArray(proto.queries)) {
if (!proto.queries.includes(statement)) {
proto.queries.unshift(statement)
}
} else if (typeof proto.queries === 'string') {
if (!proto.queries.includes(statement)) {
proto.queries = [statement, proto.queries]
}
}
}
}
Expand Down
108 changes: 108 additions & 0 deletions test/index.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -488,5 +488,113 @@ describe('Dataform package', () => {
).length
expect(reservationCount).toBe(1)
})

test('should handle mixed case DECLARE at outer level', () => {
const config = [
{
tag: 'test',
reservation: 'projects/test/locations/US/reservations/prod',
actions: ['test-project.test-schema.mixed_case']
}
]

autoAssignActions(config)
global.operate('mixed_case').queries(`
declare x INT64 DEFAULT 1;
SELECT x;
`)

const action = global.dataform.actions[0]
expect(action.proto.queries).not.toContain('SET @@reservation=\'projects/test/locations/US/reservations/prod\';')
})

test('should not skip DECLARE inside BEGIN...END block', () => {
const config = [
{
tag: 'test',
reservation: 'projects/test/locations/US/reservations/prod',
actions: ['test-project.test-schema.begin_declare']
}
]

autoAssignActions(config)
global.operate('begin_declare').queries(`
--DECLARE x INT64 DEFAULT 1;
# comment
BEGIN
DECLARE x INT64 DEFAULT 1;
SELECT x;
END;
`)

const action = global.dataform.actions[0]
expect(action.proto.queries[0]).toBe('SET @@reservation=\'projects/test/locations/US/reservations/prod\';')
})

test('should not skip DECLARE inside EXECUTE IMMEDIATE', () => {
const config = [
{
tag: 'test',
reservation: 'projects/test/locations/US/reservations/prod',
actions: ['test-project.test-schema.exec_declare']
}
]

autoAssignActions(config)
global.operate('exec_declare').queries(`
/*
block comment
DECLARE x INT64;
*/
EXECUTE IMMEDIATE "DECLARE x INT64; SET x = 1; SELECT x;"
`)

const action = global.dataform.actions[0]
expect(action.proto.queries[0]).toBe('SET @@reservation=\'projects/test/locations/US/reservations/prod\';')
})

test('should skip DECLARE after SQL comments', () => {
const config = [
{
tag: 'test',
reservation: 'projects/test/locations/US/reservations/prod',
actions: ['test-project.test-schema.comment_declare']
}
]

autoAssignActions(config)
global.operate('comment_declare').queries(`
-- set up variables
# comment
/* block comment */
/*
multi-line block comment
*/
DECLARE x INT64 DEFAULT 1;
SELECT x;
`)

const action = global.dataform.actions[0]
expect(action.proto.queries).not.toContain('SET @@reservation=\'projects/test/locations/US/reservations/prod\';')
})

test('should handle array of queries with outer DECLARE', () => {
const config = [
{
tag: 'test',
reservation: 'projects/test/locations/US/reservations/prod',
actions: ['test-project.test-schema.array_queries']
}
]

autoAssignActions(config)
global.operate('array_queries').queries([
'DECLARE x INT64 DEFAULT 1;',
'SELECT x;'
])

const action = global.dataform.actions[0]
expect(action.proto.queries).not.toContain('SET @@reservation=\'projects/test/locations/US/reservations/prod\';')
})
})
})