diff --git a/docker/.env.example b/docker/.env.example index 3f3c6f35f1..b905426791 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -51,3 +51,10 @@ EXPERIMENTAL_ENGINE_RUST_VERSION=false # Wren Engine # OPTIONAL: set if you want to use local storage for the Wren Engine LOCAL_STORAGE=. + +# MySQL Database Configuration +MYSQL_HOST=mysql-database +MYSQL_PORT=3306 +MYSQL_USER=wrenai_user +MYSQL_PASSWORD=wrenai_pwd +MYSQL_DB=wrenai_db \ No newline at end of file diff --git a/wren-ui/README.md b/wren-ui/README.md index 97febdc8a6..0f33a19053 100644 --- a/wren-ui/README.md +++ b/wren-ui/README.md @@ -28,6 +28,31 @@ export PG_URL=postgres://user:password@localhost:5432/dbname ``` - `PG_URL` is the connection string of your postgres database. +To use MySQL or MariaDB as the database, set `DB_TYPE=mysql` and provide the connection settings via environment variables below. (The `mysql2` driver used by Wren UI supports both MySQL and MariaDB.) + +```bash +# windows +SET DB_TYPE=mysql +SET MYSQL_HOST=localhost +SET MYSQL_PORT=3306 +SET MYSQL_USER=root +SET MYSQL_PASSWORD=your_password +SET MYSQL_DB=your_database + +# linux or mac +export DB_TYPE=mysql +export MYSQL_HOST=localhost +export MYSQL_PORT=3306 +export MYSQL_USER=root +export MYSQL_PASSWORD=your_password +export MYSQL_DB=your_database +``` + +Notes: +- Ensure the user has sufficient privileges to create tables and run migrations. +- For MariaDB, the same variables apply; just point them to your MariaDB instance. +- If you use a non-default port, adjust `MYSQL_PORT` accordingly. + To switch back to using SQLite, you can reassign the `DB_TYPE` to `sqlite`. ``` # windows diff --git a/wren-ui/knexfile.js b/wren-ui/knexfile.js index 72c26263a4..77d66f1fd1 100644 --- a/wren-ui/knexfile.js +++ b/wren-ui/knexfile.js @@ -9,6 +9,19 @@ if (process.env.DB_TYPE === 'pg') { client: 'pg', connection: process.env.PG_URL, }; +} else if (process.env.DB_TYPE === 'mysql') { + console.log('Using MySQL at '+process.env.MYSQL_HOST+':'+(process.env.MYSQL_PORT || 3306)); + module.exports = { + client: 'mysql2', + connection: { + host: process.env.MYSQL_HOST, + port: +(process.env.MYSQL_PORT || 3306), + user: process.env.MYSQL_USER, + password: process.env.MYSQL_PASSWORD, + database: process.env.MYSQL_DB || 'wren_ui', + }, + pool: { min: 2, max: 10 }, + }; } else { console.log('Using SQLite'); module.exports = { diff --git a/wren-ui/migrations/20240125070643_create_project_table.js b/wren-ui/migrations/20240125070643_create_project_table.js index 1821bfd9e3..b8fd7d7376 100644 --- a/wren-ui/migrations/20240125070643_create_project_table.js +++ b/wren-ui/migrations/20240125070643_create_project_table.js @@ -24,23 +24,48 @@ exports.up = function (knex) { table.string('dataset_id').nullable().comment('big query datasetId'); // duckdb - table - .jsonb('init_sql') - .nullable() - .comment('init sql for establishing duckdb environment'); + if (knex.client.config.client === 'mysql2') { + table + .json('init_sql') + .nullable() + .comment('init sql for establishing duckdb environment'); + } else { + table + .jsonb('init_sql') + .nullable() + .comment('init sql for establishing duckdb environment'); + } // knex jsonb ref: https://knexjs.org/guide/schema-builder.html#json - table - .jsonb('extensions') - .nullable() - .comment( - 'duckdb extensions, will be a array-like string like, eg: ["extension1", "extension2"]', - ); - table - .jsonb('configurations') - .nullable() - .comment( - 'duckdb configurations that can be set in session, eg: { "key1": "value1", "key2": "value2" }', - ); + if (knex.client.config.client === 'mysql2') { + table + .json('extensions') + .nullable() + .comment( + 'duckdb extensions, will be a array-like string like, eg: ["extension1", "extension2"]', + ); + } else { + table + .jsonb('extensions') + .nullable() + .comment( + 'duckdb extensions, will be a array-like string like, eg: ["extension1", "extension2"]', + ); + } + if (knex.client.config.client === 'mysql2') { + table + .json('configurations') + .nullable() + .comment( + 'duckdb configurations that can be set in session, eg: { "key1": "value1", "key2": "value2" }', + ); + } else { + table + .jsonb('configurations') + .nullable() + .comment( + 'duckdb configurations that can be set in session, eg: { "key1": "value1", "key2": "value2" }', + ); + } // not sure to store or not, the catalog & schema in the manifest table.string('catalog').comment('catalog name'); diff --git a/wren-ui/migrations/20240319083758_create_deploy_table.js b/wren-ui/migrations/20240319083758_create_deploy_table.js index 05ca3f11ae..5d1600e1c0 100644 --- a/wren-ui/migrations/20240319083758_create_deploy_table.js +++ b/wren-ui/migrations/20240319083758_create_deploy_table.js @@ -8,7 +8,11 @@ exports.up = function (knex) { table.integer('project_id').comment('Reference to project.id'); // basic info - table.jsonb('manifest').comment('the deployed manifest'); + if (knex.client.config.client === 'mysql2') { + table.json('manifest').comment('the deployed manifest'); + } else { + table.jsonb('manifest').comment('the deployed manifest'); + } table.string('hash').comment('the hash of the manifest'); // status diff --git a/wren-ui/migrations/20240327030000_create_ask_table.js b/wren-ui/migrations/20240327030000_create_ask_table.js index f48374d876..05e6c641d0 100644 --- a/wren-ui/migrations/20240327030000_create_ask_table.js +++ b/wren-ui/migrations/20240327030000_create_ask_table.js @@ -15,7 +15,11 @@ exports.up = function (knex) { }) .createTable('thread_response', (table) => { table.increments('id').comment('ID'); - table.integer('thread_id').comment('Reference to thread.id'); + if (knex.client.config.client === 'mysql2') { + table.integer('thread_id').unsigned().comment('Reference to thread.id'); + } else { + table.integer('thread_id').comment('Reference to thread.id'); + } table.foreign('thread_id').references('thread.id').onDelete('CASCADE'); // query id from AI service @@ -24,8 +28,13 @@ exports.up = function (knex) { // response from AI service table.text('question').comment('the question of the response'); table.string('status').comment('the status of the response'); - table.jsonb('detail').nullable().comment('the detail of the response'); - table.jsonb('error').nullable().comment('the error message if any'); + if (knex.client.config.client === 'mysql2') { + table.json('detail').nullable().comment('the detail of the response'); + table.json('error').nullable().comment('the error message if any'); + } else { + table.jsonb('detail').nullable().comment('the detail of the response'); + table.jsonb('error').nullable().comment('the error message if any'); + } // timestamps table.timestamps(true, true); diff --git a/wren-ui/migrations/20240419090558_add_foreign_key_to_model_column_and_metric_measure.js b/wren-ui/migrations/20240419090558_add_foreign_key_to_model_column_and_metric_measure.js index 82e8231a75..d7972088bd 100644 --- a/wren-ui/migrations/20240419090558_add_foreign_key_to_model_column_and_metric_measure.js +++ b/wren-ui/migrations/20240419090558_add_foreign_key_to_model_column_and_metric_measure.js @@ -5,9 +5,15 @@ exports.up = function (knex) { return knex.schema .alterTable('model_column', (table) => { + if (knex.client.config.client === 'mysql2') { + table.integer('model_id').unsigned().alter(); + } table.foreign('model_id').references('model.id').onDelete('CASCADE'); }) .alterTable('metric_measure', (table) => { + if (knex.client.config.client === 'mysql2') { + table.integer('metric_id').unsigned().alter(); + } table.foreign('metric_id').references('metric.id').onDelete('CASCADE'); }); }; diff --git a/wren-ui/migrations/20240446090560_update_relationship_table.js b/wren-ui/migrations/20240446090560_update_relationship_table.js index b35b7bd807..3e4a3422d2 100644 --- a/wren-ui/migrations/20240446090560_update_relationship_table.js +++ b/wren-ui/migrations/20240446090560_update_relationship_table.js @@ -5,12 +5,18 @@ exports.up = function (knex) { return knex.schema .alterTable('relation', (table) => { + if (knex.client.config.client === 'mysql2') { + table.integer('from_column_id').unsigned().alter(); + } table .foreign('from_column_id') .references('model_column.id') .onDelete('CASCADE'); }) .alterTable('relation', (table) => { + if (knex.client.config.client === 'mysql2') { + table.integer('to_column_id').unsigned().alter(); + } table .foreign('to_column_id') .references('model_column.id') diff --git a/wren-ui/migrations/20240530062133_update_project_table.js b/wren-ui/migrations/20240530062133_update_project_table.js index 8c5168114a..64a9c1ba2e 100644 --- a/wren-ui/migrations/20240530062133_update_project_table.js +++ b/wren-ui/migrations/20240530062133_update_project_table.js @@ -6,10 +6,17 @@ // create connectionInfo column in project table exports.up = function (knex) { return knex.schema.table('project', (table) => { - table - .jsonb('connection_info') - .nullable() - .comment('Connection information for the project'); + if (knex.client.config.client === 'mysql2') { + table + .json('connection_info') + .nullable() + .comment('Connection information for the project'); + } else { + table + .jsonb('connection_info') + .nullable() + .comment('Connection information for the project'); + } }); }; diff --git a/wren-ui/migrations/20240530105955_drop_project_table_columns.js b/wren-ui/migrations/20240530105955_drop_project_table_columns.js index b7be89adb9..d6abcfd50a 100644 --- a/wren-ui/migrations/20240530105955_drop_project_table_columns.js +++ b/wren-ui/migrations/20240530105955_drop_project_table_columns.js @@ -23,12 +23,21 @@ exports.up = function (knex) { */ exports.down = function (knex) { return knex.schema.table('project', (table) => { - table - .jsonb('configurations') - .nullable() - .comment( - 'duckdb configurations that can be set in session, eg: { "key1": "value1", "key2": "value2" }', - ); + if (knex.client.config.client === 'mysql2') { + table + .json('configurations') + .nullable() + .comment( + 'duckdb configurations that can be set in session, eg: { "key1": "value1", "key2": "value2" }', + ); + } else { + table + .jsonb('configurations') + .nullable() + .comment( + 'duckdb configurations that can be set in session, eg: { "key1": "value1", "key2": "value2" }', + ); + } table .text('credentials') .nullable() @@ -39,12 +48,21 @@ exports.down = function (knex) { .comment('gcp project id, big query specific'); table.string('dataset_id').nullable().comment('big query datasetId'); table.text('init_sql'); - table - .jsonb('extensions') - .nullable() - .comment( - 'duckdb extensions, will be a array-like string like, eg: ["extension1", "extension2"]', - ); + if (knex.client.config.client === 'mysql2') { + table + .json('extensions') + .nullable() + .comment( + 'duckdb extensions, will be a array-like string like, eg: ["extension1", "extension2"]', + ); + } else { + table + .jsonb('extensions') + .nullable() + .comment( + 'duckdb extensions, will be a array-like string like, eg: ["extension1", "extension2"]', + ); + } table .string('host') .nullable() diff --git a/wren-ui/migrations/20240610070534_create_schema_change_table.js b/wren-ui/migrations/20240610070534_create_schema_change_table.js index 0d605e50f8..2fae39b1d4 100644 --- a/wren-ui/migrations/20240610070534_create_schema_change_table.js +++ b/wren-ui/migrations/20240610070534_create_schema_change_table.js @@ -8,8 +8,13 @@ exports.up = function (knex) { table.integer('project_id').comment('Reference to project.id'); // schema change info - table.jsonb('change').nullable(); - table.jsonb('resolve').nullable(); + if (knex.client.config.client === 'mysql2') { + table.json('change').nullable(); + table.json('resolve').nullable(); + } else { + table.jsonb('change').nullable(); + table.jsonb('resolve').nullable(); + } // timestamps table.timestamps(true, true); diff --git a/wren-ui/migrations/20240928165009_create_model_nested_column.js b/wren-ui/migrations/20240928165009_create_model_nested_column.js index 310d8c9756..64ece802c6 100644 --- a/wren-ui/migrations/20240928165009_create_model_nested_column.js +++ b/wren-ui/migrations/20240928165009_create_model_nested_column.js @@ -6,7 +6,11 @@ exports.up = function (knex) { return knex.schema.createTable('model_nested_column', (table) => { table.increments('id').comment('ID'); table.integer('model_id').comment('Reference to model ID'); - table.integer('column_id').comment('Reference to column ID'); + if (knex.client.config.client === 'mysql2') { + table.integer('column_id').unsigned().comment('Reference to column ID'); + } else { + table.integer('column_id').comment('Reference to column ID'); + } table .string('column_path') .comment( diff --git a/wren-ui/migrations/20241106232204_update_project_table.js b/wren-ui/migrations/20241106232204_update_project_table.js index 801ceaf5f7..6319a42d9b 100644 --- a/wren-ui/migrations/20241106232204_update_project_table.js +++ b/wren-ui/migrations/20241106232204_update_project_table.js @@ -4,10 +4,17 @@ */ exports.up = function (knex) { return knex.schema.alterTable('project', (table) => { - table - .jsonb('questions') - .nullable() - .comment('The recommended questions generated by AI'); + if (knex.client.config.client === 'mysql2') { + table + .json('questions') + .nullable() + .comment('The recommended questions generated by AI'); + } else { + table + .jsonb('questions') + .nullable() + .comment('The recommended questions generated by AI'); + } table .string('query_id') .nullable() @@ -16,10 +23,17 @@ exports.up = function (knex) { .string('questions_status') .nullable() .comment('The status of the recommended question pipeline'); - table - .jsonb('questions_error') - .nullable() - .comment('The error of the recommended question pipeline'); + if (knex.client.config.client === 'mysql2') { + table + .json('questions_error') + .nullable() + .comment('The error of the recommended question pipeline'); + } else { + table + .jsonb('questions_error') + .nullable() + .comment('The error of the recommended question pipeline'); + } }); }; diff --git a/wren-ui/migrations/20241107171828_update_thread_table.js b/wren-ui/migrations/20241107171828_update_thread_table.js index 4d5fbad755..81dd44f5c5 100644 --- a/wren-ui/migrations/20241107171828_update_thread_table.js +++ b/wren-ui/migrations/20241107171828_update_thread_table.js @@ -4,10 +4,17 @@ */ exports.up = function (knex) { return knex.schema.alterTable('thread', (table) => { - table - .jsonb('questions') - .nullable() - .comment('The recommended questions generated by AI'); + if (knex.client.config.client === 'mysql2') { + table + .json('questions') + .nullable() + .comment('The recommended questions generated by AI'); + } else { + table + .jsonb('questions') + .nullable() + .comment('The recommended questions generated by AI'); + } table .string('query_id') .nullable() @@ -16,10 +23,17 @@ exports.up = function (knex) { .string('questions_status') .nullable() .comment('The status of the recommended question pipeline'); - table - .jsonb('questions_error') - .nullable() - .comment('The error of the recommended question pipeline'); + if (knex.client.config.client === 'mysql2') { + table + .json('questions_error') + .nullable() + .comment('The error of the recommended question pipeline'); + } else { + table + .jsonb('questions_error') + .nullable() + .comment('The error of the recommended question pipeline'); + } }); }; diff --git a/wren-ui/migrations/20241207000000_update_thread_response_for_answer.js b/wren-ui/migrations/20241207000000_update_thread_response_for_answer.js index a73324c037..a122fb2d2f 100644 --- a/wren-ui/migrations/20241207000000_update_thread_response_for_answer.js +++ b/wren-ui/migrations/20241207000000_update_thread_response_for_answer.js @@ -41,10 +41,17 @@ exports.up = async function (knex) { .text('sql') .nullable() .comment('the SQL query generated by AI service'); - table - .jsonb('answer_detail') - .defaultTo('{}') - .comment('AI generated text-based answer detail'); + if (knex.client.config.client === 'mysql2') { + table + .json('answer_detail') + .defaultTo('{}') + .comment('AI generated text-based answer detail'); + } else { + table + .jsonb('answer_detail') + .defaultTo('{}') + .comment('AI generated text-based answer detail'); + } table .integer('view_id') .nullable() @@ -106,7 +113,11 @@ exports.down = async function (knex) { await knex.schema.alterTable('thread_response', (table) => { table.string('query_id').comment('the query id generated by AI service'); table.string('status').comment('the status of the response'); - table.jsonb('error').nullable().comment('the error message if any'); + if (knex.client.config.client === 'mysql2') { + table.json('error').nullable().comment('the error message if any'); + } else { + table.jsonb('error').nullable().comment('the error message if any'); + } table.dropColumn('sql'); table.dropColumn('answer_detail'); }); diff --git a/wren-ui/migrations/20241210072534_update_thread_response_table.js b/wren-ui/migrations/20241210072534_update_thread_response_table.js index 0b216d6649..84d8b7b683 100644 --- a/wren-ui/migrations/20241210072534_update_thread_response_table.js +++ b/wren-ui/migrations/20241210072534_update_thread_response_table.js @@ -4,7 +4,11 @@ */ exports.up = function (knex) { return knex.schema.alterTable('thread_response', (table) => { - table.jsonb('chart_detail').nullable(); + if (knex.client.config.client === 'mysql2') { + table.json('chart_detail').nullable(); + } else { + table.jsonb('chart_detail').nullable(); + } }); }; diff --git a/wren-ui/migrations/20250102074255_create_dashboard_table.js b/wren-ui/migrations/20250102074255_create_dashboard_table.js index 93447d641b..06ba635941 100644 --- a/wren-ui/migrations/20250102074255_create_dashboard_table.js +++ b/wren-ui/migrations/20250102074255_create_dashboard_table.js @@ -5,10 +5,18 @@ exports.up = async function (knex) { await knex.schema.createTable('dashboard', (table) => { table.increments('id').primary(); - table - .integer('project_id') - .notNullable() - .comment('Reference to project.id'); + if (knex.client.config.client === 'mysql2') { + table + .integer('project_id') + .unsigned() + .notNullable() + .comment('Reference to project.id'); + } else { + table + .integer('project_id') + .notNullable() + .comment('Reference to project.id'); + } table.string('name').notNullable().comment('The dashboard name'); table.foreign('project_id').references('project.id').onDelete('CASCADE'); diff --git a/wren-ui/migrations/20250102074256_create_dashboard_item_table.js b/wren-ui/migrations/20250102074256_create_dashboard_item_table.js index 8531216a23..9f605af91c 100644 --- a/wren-ui/migrations/20250102074256_create_dashboard_item_table.js +++ b/wren-ui/migrations/20250102074256_create_dashboard_item_table.js @@ -5,29 +5,51 @@ exports.up = function (knex) { return knex.schema.createTable('dashboard_item', (table) => { table.increments('id').primary(); - table - .integer('dashboard_id') - .notNullable() - .comment('Reference to dashboard.id'); + if (knex.client.config.client === 'mysql2') { + table + .integer('dashboard_id') + .unsigned() + .notNullable() + .comment('Reference to dashboard.id'); + } else { + table + .integer('dashboard_id') + .notNullable() + .comment('Reference to dashboard.id'); + } table .string('type') .notNullable() .comment( 'The chart type of the dashboard item, such as: bar, table, number, etc', ); - table - .jsonb('layout') - .notNullable() - .comment( - 'The layout of the dashboard item, according to which library it is, such as: { x: 0, y: 0, w: 6, h: 6 }', - ); - table - .jsonb('detail') - .notNullable() - .comment( - 'The detail of the dashboard item, such as: { chartSchema: {...}, sql: "..." } ', - ); - + if (knex.client.config.client === 'mysql2') { + table + .json('layout') + .notNullable() + .comment( + 'The layout of the dashboard item, according to which library it is, such as: { x: 0, y: 0, w: 6, h: 6 }', + ); + table + .json('detail') + .notNullable() + .comment( + 'The detail of the dashboard item, such as: { chartSchema: {...}, sql: "..." } ', + ); + } else { + table + .jsonb('layout') + .notNullable() + .comment( + 'The layout of the dashboard item, according to which library it is, such as: { x: 0, y: 0, w: 6, h: 6 }', + ); + table + .jsonb('detail') + .notNullable() + .comment( + 'The detail of the dashboard item, such as: { chartSchema: {...}, sql: "..." } ', + ); + } table .foreign('dashboard_id') .references('dashboard.id') diff --git a/wren-ui/migrations/20250102074256_create_sql_pair_table.js b/wren-ui/migrations/20250102074256_create_sql_pair_table.js index d22b952dd0..452593e7ee 100644 --- a/wren-ui/migrations/20250102074256_create_sql_pair_table.js +++ b/wren-ui/migrations/20250102074256_create_sql_pair_table.js @@ -5,10 +5,18 @@ exports.up = function (knex) { return knex.schema.createTable('sql_pair', (table) => { table.increments('id').primary(); - table - .integer('project_id') - .notNullable() - .comment('Reference to project.id'); + if (knex.client.config.client === 'mysql2') { + table + .integer('project_id') + .unsigned() + .notNullable() + .comment('Reference to project.id'); + } else { + table + .integer('project_id') + .notNullable() + .comment('Reference to project.id'); + } table.text('sql').notNullable(); table.string('question', 1000).notNullable(); table.timestamps(true, true); diff --git a/wren-ui/migrations/20250311046282_create_instruction_table.js b/wren-ui/migrations/20250311046282_create_instruction_table.js index fcb84483b6..23de6b4301 100644 --- a/wren-ui/migrations/20250311046282_create_instruction_table.js +++ b/wren-ui/migrations/20250311046282_create_instruction_table.js @@ -5,12 +5,24 @@ exports.up = function (knex) { return knex.schema.createTable('instruction', (table) => { table.increments('id').primary(); + if (knex.client.config.client === 'mysql2') { table - .integer('project_id') - .notNullable() - .comment('Reference to project.id'); + .integer('project_id') + .unsigned() + .notNullable() + .comment('Reference to project.id'); + } else { + table + .integer('project_id') + .notNullable() + .comment('Reference to project.id'); + } table.text('instruction').notNullable().comment('The instruction text'); - table.jsonb('questions').notNullable().comment('The questions array'); + if (knex.client.config.client === 'mysql2') { + table.json('questions').notNullable().comment('The questions array'); + } else { + table.jsonb('questions').notNullable().comment('The questions array'); + } table .boolean('is_default') .notNullable() diff --git a/wren-ui/migrations/20250423000000_create_dashboard_cache_refresh_table.js b/wren-ui/migrations/20250423000000_create_dashboard_cache_refresh_table.js index 69a76ac390..73f6e2782f 100644 --- a/wren-ui/migrations/20250423000000_create_dashboard_cache_refresh_table.js +++ b/wren-ui/migrations/20250423000000_create_dashboard_cache_refresh_table.js @@ -2,8 +2,13 @@ exports.up = function (knex) { return knex.schema.createTable('dashboard_item_refresh_job', (table) => { table.increments('id').primary(); table.string('hash').notNullable().comment('uuid for the refresh job'); - table.integer('dashboard_id').notNullable(); - table.integer('dashboard_item_id').notNullable(); + if (knex.client.config.client === 'mysql2') { + table.integer('dashboard_id').unsigned().notNullable(); + table.integer('dashboard_item_id').unsigned().notNullable(); + }else{ + table.integer('dashboard_id').notNullable(); + table.integer('dashboard_item_id').notNullable(); + } table.timestamp('started_at').notNullable(); table.timestamp('finished_at'); table.string('status').notNullable(); // 'success', 'failed', 'in_progress' diff --git a/wren-ui/migrations/20250509000000_create_asking_task.js b/wren-ui/migrations/20250509000000_create_asking_task.js index bd0fcd5e0d..9dd2fc1e31 100644 --- a/wren-ui/migrations/20250509000000_create_asking_task.js +++ b/wren-ui/migrations/20250509000000_create_asking_task.js @@ -7,19 +7,39 @@ exports.up = function (knex) { table.increments('id').primary(); table.string('query_id').notNullable().unique(); table.text('question'); - table.jsonb('detail').defaultTo('{}'); + if (knex.client.config.client === 'mysql2') { + table.json('detail').defaultTo('{}'); + } else { + table.jsonb('detail').defaultTo('{}'); + } - table - .integer('thread_id') - .references('id') - .inTable('thread') - .onDelete('CASCADE'); + if (knex.client.config.client === 'mysql2') { + table + .integer('thread_id') + .unsigned() + .references('id') + .inTable('thread') + .onDelete('CASCADE'); - table - .integer('thread_response_id') - .references('id') - .inTable('thread_response') - .onDelete('CASCADE'); + table + .integer('thread_response_id') + .unsigned() + .references('id') + .inTable('thread_response') + .onDelete('CASCADE'); + } else { + table + .integer('thread_id') + .references('id') + .inTable('thread') + .onDelete('CASCADE'); + + table + .integer('thread_response_id') + .references('id') + .inTable('thread_response') + .onDelete('CASCADE'); + } table.timestamps(true, true); }); diff --git a/wren-ui/migrations/20250509000001_add_task_id_to_thread.js b/wren-ui/migrations/20250509000001_add_task_id_to_thread.js index d483cc2bca..d384c8b39d 100644 --- a/wren-ui/migrations/20250509000001_add_task_id_to_thread.js +++ b/wren-ui/migrations/20250509000001_add_task_id_to_thread.js @@ -4,12 +4,22 @@ */ exports.up = function (knex) { return knex.schema.alterTable('thread_response', (table) => { - table - .integer('asking_task_id') - .nullable() - .references('id') - .inTable('asking_task') - .onDelete('SET NULL'); + if (knex.client.config.client === 'mysql2') { + table + .integer('asking_task_id') + .unsigned() + .nullable() + .references('id') + .inTable('asking_task') + .onDelete('SET NULL'); + }else{ + table + .integer('asking_task_id') + .nullable() + .references('id') + .inTable('asking_task') + .onDelete('SET NULL'); + } }); }; diff --git a/wren-ui/migrations/20250510000000_add_adjustment_to_thread_response.js b/wren-ui/migrations/20250510000000_add_adjustment_to_thread_response.js index 5c06d6ddf5..799cc90a21 100644 --- a/wren-ui/migrations/20250510000000_add_adjustment_to_thread_response.js +++ b/wren-ui/migrations/20250510000000_add_adjustment_to_thread_response.js @@ -4,12 +4,21 @@ */ exports.up = function (knex) { return knex.schema.alterTable('thread_response', (table) => { - table - .jsonb('adjustment') - .nullable() - .comment( - 'Adjustment data for thread response, including type and payload', - ); + if (knex.client.config.client === 'mysql2') { + table + .json('adjustment') + .nullable() + .comment( + 'Adjustment data for thread response, including type and payload', + ); + } else { + table + .jsonb('adjustment') + .nullable() + .comment( + 'Adjustment data for thread response, including type and payload', + ); + } }); }; diff --git a/wren-ui/migrations/20250511000000-create-api-history.js b/wren-ui/migrations/20250511000000-create-api-history.js index aa88cf449a..dfd445509f 100644 --- a/wren-ui/migrations/20250511000000-create-api-history.js +++ b/wren-ui/migrations/20250511000000-create-api-history.js @@ -7,7 +7,11 @@ exports.up = function (knex) { table.string('id').primary(); // Project - table.integer('project_id').notNullable(); + if (knex.client.config.client === 'mysql2') { + table.integer('project_id').unsigned().notNullable(); + } else { + table.integer('project_id').notNullable(); + } table .foreign('project_id') .references('id') @@ -20,12 +24,21 @@ exports.up = function (knex) { // API Type table.string('api_type').notNullable(); - // Request - table.jsonb('headers'); - table.jsonb('request_payload'); + if (knex.client.config.client === 'mysql2') { + // Request + table.json('headers'); + table.json('request_payload'); - // Response - table.jsonb('response_payload'); + // Response + table.json('response_payload'); + } else { + // Request + table.jsonb('headers'); + table.jsonb('request_payload'); + + // Response + table.jsonb('response_payload'); + } // Result table.integer('status_code').notNullable(); diff --git a/wren-ui/package.json b/wren-ui/package.json index 925c1ce544..fb2eb18074 100644 --- a/wren-ui/package.json +++ b/wren-ui/package.json @@ -29,6 +29,7 @@ "log4js": "^6.9.1", "micro": "^9.4.1", "micro-cors": "^0.1.1", + "mysql2": "^3.15.3", "next": "14.2.32", "pg": "^8.8.0", "pg-cursor": "^2.7.4", diff --git a/wren-ui/src/apollo/server/config.ts b/wren-ui/src/apollo/server/config.ts index ad43e2da84..08676688fb 100644 --- a/wren-ui/src/apollo/server/config.ts +++ b/wren-ui/src/apollo/server/config.ts @@ -11,6 +11,12 @@ export interface IConfig { debug?: boolean; // sqlite sqliteFile?: string; + // mysql / mariadb + mysqlHost?: string; + mysqlPort?: number; + mysqlUser?: string; + mysqlPassword?: string; + mysqlDb?: string; persistCredentialDir?: string; @@ -61,6 +67,12 @@ const defaultConfig = { // sqlite sqliteFile: './db.sqlite3', + // mysql / mariadb + mysqlHost: 'localhost', + mysqlPort: 3306, + mysqlUser: '', + mysqlPassword: '', + mysqlDb: 'wren_ui', persistCredentialDir: `${process.cwd()}/.tmp`, @@ -90,6 +102,14 @@ const config = { debug: process.env.DEBUG === 'true', // sqlite sqliteFile: process.env.SQLITE_FILE, + // mysql / mariadb + mysqlHost: process.env.MYSQL_HOST, + mysqlPort: process.env.MYSQL_PORT + ? parseInt(process.env.MYSQL_PORT) + : undefined, + mysqlUser: process.env.MYSQL_USER, + mysqlPassword: process.env.MYSQL_PASSWORD, + mysqlDb: process.env.MYSQL_DB, persistCredentialDir: (() => { if ( diff --git a/wren-ui/src/apollo/server/repositories/baseRepository.ts b/wren-ui/src/apollo/server/repositories/baseRepository.ts index f76fe27654..5b58fa7b61 100644 --- a/wren-ui/src/apollo/server/repositories/baseRepository.ts +++ b/wren-ui/src/apollo/server/repositories/baseRepository.ts @@ -105,10 +105,16 @@ export class BaseRepository implements IBasicRepository { public async createOne(data: Partial, queryOptions?: IQueryOptions) { const executer = queryOptions?.tx ? queryOptions.tx : this.knex; - const [result] = await executer(this.tableName) - .insert(this.transformToDBData(data)) - .returning('*'); - return this.transformFromDBData(result); + const query = executer(this.tableName).insert(this.transformToDBData(data)); + + if (this.knex.client.config.client != 'mysql2') { + const [result] = await query.returning('*'); + return this.transformFromDBData(result); + } + // MySQL does not support this, so retrieve the newly created record. + const [id] = await query; + const inserted = await executer(this.tableName).where({ id }).first(); + return this.transformFromDBData(inserted); } public async createMany(data: Partial[], queryOptions?: IQueryOptions) { @@ -119,14 +125,42 @@ export class BaseRepository implements IBasicRepository { for (let i = 0; i < batchCount; i++) { const start = i * batchSize; const end = Math.min((i + 1) * batchSize, data.length); - const batchValues = data.slice(start, end); - const chunk = await executer(this.tableName) - .insert(batchValues.map(this.transformToDBData)) - .returning('*'); - result.push(...chunk); + const batchValuesOriginal = data.slice(start, end); + // IMPORTANT: we need to convert each object to snake_case before inserting. + // createOne already did this; createMany didn't, which caused camelCase columns (e.g., displayName) + // to break in databases where the actual column is display_name. + const batchValues = batchValuesOriginal.map((v) => + this.transformToDBData(v), + ); + const query = executer(this.tableName).insert(batchValues); + + if (this.knex.client.config.client != 'mysql2') { + // PostgreSQL and similar + const chunk = await query.returning('*'); + result.push(...chunk.map(this.transformFromDBData)); + } else { + // MySQL / MariaDB: manually fetch the inserted records + const insertedIds = await query; // Returns the first ID in the sequence + const firstId = Array.isArray(insertedIds) + ? insertedIds[0] + : insertedIds; + + // Search for the range of inserted IDs (if the table uses autoincrement) + // ⚠️ This only works well if the 'id' field is auto increment. + if (typeof firstId === 'number') { + const lastId = firstId + batchValues.length - 1; + const rows = await executer(this.tableName) + .whereBetween('id', [firstId, lastId]) + .orderBy('id', 'asc'); + result.push(...rows.map(this.transformFromDBData)); + } else { + // Fallback without numeric ID — returns the raw inserted data + // Here batchValues are already in snake_case format; transformFromDBData converts to camelCase. + result.push(...batchValues.map(this.transformFromDBData)); + } + } } - - return result.map((data) => this.transformFromDBData(data)); + return result; } public async updateOne( @@ -135,11 +169,17 @@ export class BaseRepository implements IBasicRepository { queryOptions?: IQueryOptions, ) { const executer = queryOptions?.tx ? queryOptions.tx : this.knex; - const [result] = await executer(this.tableName) + const query = executer(this.tableName) .where({ id }) - .update(this.transformToDBData(data)) - .returning('*'); - return this.transformFromDBData(result); + .update(this.transformToDBData(data)); + if (this.knex.client.config.client != 'mysql2') { + const [result] = await query.returning('*'); + return this.transformFromDBData(result); + } + // MySQL: manually fetch the updated record + await query; + const updated = await executer(this.tableName).where({ id }).first(); + return this.transformFromDBData(updated); } public async deleteOne(id: string, queryOptions?: IQueryOptions) { diff --git a/wren-ui/src/apollo/server/repositories/threadResponseRepository.ts b/wren-ui/src/apollo/server/repositories/threadResponseRepository.ts index f51d1eda91..2490adcb5a 100644 --- a/wren-ui/src/apollo/server/repositories/threadResponseRepository.ts +++ b/wren-ui/src/apollo/server/repositories/threadResponseRepository.ts @@ -170,11 +170,18 @@ export class ThreadResponseRepository adjustment: data.adjustment ? JSON.stringify(data.adjustment) : undefined, }; const executer = queryOptions?.tx ? queryOptions.tx : this.knex; - const [result] = await executer(this.tableName) + const query = executer(this.tableName) .where({ id }) - .update(this.transformToDBData(transformedData as any)) - .returning('*'); - return this.transformFromDBData(result); + .update(this.transformToDBData(transformedData as any)); + + if (this.knex.client.config.client != 'mysql2') { + const [result] = await query.returning('*'); + return this.transformFromDBData(result); + } + // MySQL: manually fetch the updated record + await query; + const result = await executer(this.tableName).where({ id }).first(); + return result ? this.transformFromDBData(result) : undefined; } protected override transformFromDBData = (data: any): ThreadResponse => { diff --git a/wren-ui/src/apollo/server/utils/knex.ts b/wren-ui/src/apollo/server/utils/knex.ts index b7c74bba53..332ad9df66 100644 --- a/wren-ui/src/apollo/server/utils/knex.ts +++ b/wren-ui/src/apollo/server/utils/knex.ts @@ -16,6 +16,18 @@ export const bootstrapKnex = (options: KnexOptions) => { debug, pool: { min: 2, max: 10 }, }); + } else if (options.dbType === 'mysql') { + return require('knex')({ + client: 'mysql2', + connection: { + host: process.env.MYSQL_HOST, + port: +(process.env.MYSQL_PORT || 3306), + user: process.env.MYSQL_USER, + password: process.env.MYSQL_PASSWORD, + database: process.env.MYSQL_DB, + }, + pool: { min: 2, max: 10 }, + }); } else { console.log('using sqlite'); /* eslint-disable @typescript-eslint/no-var-requires */ diff --git a/wren-ui/yarn.lock b/wren-ui/yarn.lock index 01ea74558e..57cdd10e21 100644 --- a/wren-ui/yarn.lock +++ b/wren-ui/yarn.lock @@ -4965,6 +4965,13 @@ __metadata: languageName: node linkType: hard +"aws-ssl-profiles@npm:^1.1.1": + version: 1.1.2 + resolution: "aws-ssl-profiles@npm:1.1.2" + checksum: 10c0/e5f59a4146fe3b88ad2a84f814886c788557b80b744c8cbcb1cbf8cf5ba19cc006a7a12e88819adc614ecda9233993f8f1d1f3b612cbc2f297196df9e8f4f66e + languageName: node + linkType: hard + "axe-core@npm:=4.7.0": version: 4.7.0 resolution: "axe-core@npm:4.7.0" @@ -6522,6 +6529,13 @@ __metadata: languageName: node linkType: hard +"denque@npm:^2.1.0": + version: 2.1.0 + resolution: "denque@npm:2.1.0" + checksum: 10c0/f9ef81aa0af9c6c614a727cb3bd13c5d7db2af1abf9e6352045b86e85873e629690f6222f4edd49d10e4ccf8f078bbeec0794fafaf61b659c0589d0c511ec363 + languageName: node + linkType: hard + "depd@npm:~1.1.2": version: 1.1.2 resolution: "depd@npm:1.1.2" @@ -7915,6 +7929,15 @@ __metadata: languageName: node linkType: hard +"generate-function@npm:^2.3.1": + version: 2.3.1 + resolution: "generate-function@npm:2.3.1" + dependencies: + is-property: "npm:^1.0.2" + checksum: 10c0/4645cf1da90375e46a6f1dc51abc9933e5eafa4cd1a44c2f7e3909a30a4e9a1a08c14cd7d5b32da039da2dba2a085e1ed4597b580c196c3245b2d35d8bc0de5d + languageName: node + linkType: hard + "gensync@npm:^1.0.0-beta.2": version: 1.0.0-beta.2 resolution: "gensync@npm:1.0.0-beta.2" @@ -8547,6 +8570,15 @@ __metadata: languageName: node linkType: hard +"iconv-lite@npm:^0.7.0": + version: 0.7.0 + resolution: "iconv-lite@npm:0.7.0" + dependencies: + safer-buffer: "npm:>= 2.1.2 < 3.0.0" + checksum: 10c0/2382400469071c55b6746c531eed5fa4d033e5db6690b7331fb2a5f59a30d7a9782932e92253db26df33c1cf46fa200a3fbe524a2a7c62037c762283f188ec2f + languageName: node + linkType: hard + "ieee754@npm:^1.1.13": version: 1.2.1 resolution: "ieee754@npm:1.2.1" @@ -8986,6 +9018,13 @@ __metadata: languageName: node linkType: hard +"is-property@npm:^1.0.2": + version: 1.0.2 + resolution: "is-property@npm:1.0.2" + checksum: 10c0/33ab65a136e4ba3f74d4f7d9d2a013f1bd207082e11cedb160698e8d5394644e873c39668d112a402175ccbc58a087cef87198ed46829dbddb479115a0257283 + languageName: node + linkType: hard + "is-regex@npm:^1.1.4": version: 1.1.4 resolution: "is-regex@npm:1.1.4" @@ -10215,6 +10254,13 @@ __metadata: languageName: node linkType: hard +"long@npm:^5.2.1": + version: 5.3.2 + resolution: "long@npm:5.3.2" + checksum: 10c0/7130fe1cbce2dca06734b35b70d380ca3f70271c7f8852c922a7c62c86c4e35f0c39290565eca7133c625908d40e126ac57c02b1b1a4636b9457d77e1e60b981 + languageName: node + linkType: hard + "longest-streak@npm:^3.0.0": version: 3.1.0 resolution: "longest-streak@npm:3.1.0" @@ -10283,13 +10329,20 @@ __metadata: languageName: node linkType: hard -"lru-cache@npm:^7.7.1": +"lru-cache@npm:^7.14.1, lru-cache@npm:^7.7.1": version: 7.18.3 resolution: "lru-cache@npm:7.18.3" checksum: 10c0/b3a452b491433db885beed95041eb104c157ef7794b9c9b4d647be503be91769d11206bb573849a16b4cc0d03cbd15ffd22df7960997788b74c1d399ac7a4fed languageName: node linkType: hard +"lru.min@npm:^1.0.0": + version: 1.1.3 + resolution: "lru.min@npm:1.1.3" + checksum: 10c0/62567c9d9e6e3b1b3793853ac509007082d93dc838819998a9911a3058b7ca53045ba6a477ab8ccb983788591dd7ff90e05f3b073ba0d30d8b8245c9ef17a06d + languageName: node + linkType: hard + "luxon@npm:^3.6.1": version: 3.6.1 resolution: "luxon@npm:3.6.1" @@ -11308,6 +11361,32 @@ __metadata: languageName: node linkType: hard +"mysql2@npm:^3.15.3": + version: 3.15.3 + resolution: "mysql2@npm:3.15.3" + dependencies: + aws-ssl-profiles: "npm:^1.1.1" + denque: "npm:^2.1.0" + generate-function: "npm:^2.3.1" + iconv-lite: "npm:^0.7.0" + long: "npm:^5.2.1" + lru.min: "npm:^1.0.0" + named-placeholders: "npm:^1.1.3" + seq-queue: "npm:^0.0.5" + sqlstring: "npm:^2.3.2" + checksum: 10c0/e10c51eebb2b2783837b732f1f4edc9e0ea15d9c5d80167e739b1dc97a323c786f5b3261e229f586b2903c44abcc71422c473113dfb261fa6215efcbbb5fe6ef + languageName: node + linkType: hard + +"named-placeholders@npm:^1.1.3": + version: 1.1.3 + resolution: "named-placeholders@npm:1.1.3" + dependencies: + lru-cache: "npm:^7.14.1" + checksum: 10c0/cd83b4bbdf358b2285e3c51260fac2039c9d0546632b8a856b3eeabd3bfb3d5b597507ab319b97c281a4a70d748f38bc66fa218a61cb44f55ad997ad5d9c9935 + languageName: node + linkType: hard + "nanoid@npm:^3.3.6": version: 3.3.8 resolution: "nanoid@npm:3.3.8" @@ -13789,6 +13868,13 @@ __metadata: languageName: node linkType: hard +"seq-queue@npm:^0.0.5": + version: 0.0.5 + resolution: "seq-queue@npm:0.0.5" + checksum: 10c0/ec870fc392f0e6e99ec0e551c3041c1a66144d1580efabae7358e572de127b0ad2f844c95a4861d2e6203f836adea4c8196345b37bed55331ead8f22d99ac84c + languageName: node + linkType: hard + "set-blocking@npm:^2.0.0": version: 2.0.0 resolution: "set-blocking@npm:2.0.0" @@ -14112,6 +14198,13 @@ __metadata: languageName: node linkType: hard +"sqlstring@npm:^2.3.2": + version: 2.3.3 + resolution: "sqlstring@npm:2.3.3" + checksum: 10c0/3b5dd7badb3d6312f494cfa6c9a381ee630fbe3dbd571c4c9eb8ecdb99a7bf5a1f7a5043191d768797f6b3c04eed5958ac6a5f948b998f0a138294c6d3125fbd + languageName: node + linkType: hard + "ssri@npm:^12.0.0": version: 12.0.0 resolution: "ssri@npm:12.0.0" @@ -16061,6 +16154,7 @@ __metadata: log4js: "npm:^6.9.1" micro: "npm:^9.4.1" micro-cors: "npm:^0.1.1" + mysql2: "npm:^3.15.3" next: "npm:14.2.32" next-with-less: "npm:^3.0.1" pg: "npm:^8.8.0"