diff --git a/.github/workflows/db-migrations.yaml b/.github/workflows/db-migrations.yaml
new file mode 100644
index 000000000..b36e9a721
--- /dev/null
+++ b/.github/workflows/db-migrations.yaml
@@ -0,0 +1,62 @@
+name: DB Migrations
+
+on:
+ push:
+ branches:
+ - develop
+ pull_request:
+ workflow_dispatch:
+
+jobs:
+ alembic-migrations:
+ strategy:
+ matrix:
+ python-version: ["3.10"]
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install dependencies
+ run: |
+ pip install --upgrade --upgrade-strategy eager -r requirements.txt
+ pip install --upgrade --upgrade-strategy eager -r requirements-dev.txt
+
+ - name: Set DB env vars
+ run: |
+ echo "DB_FILE=$RUNNER_TEMP/dashai-ci.sqlite3" >> "$GITHUB_ENV"
+ echo "DATABASE_URL=sqlite:///$RUNNER_TEMP/dashai-ci.sqlite3" >> "$GITHUB_ENV"
+ echo "Temp dir: $RUNNER_TEMP"
+
+ - name: Show Alembic info
+ run: |
+ alembic --version
+ echo "DB will be at: $DATABASE_URL"
+
+ - name: Prepare temp dir
+ run: |
+ mkdir -p "$(dirname "$DB_FILE")"
+ rm -f "$DB_FILE"
+
+ # upgrade to head
+ - name: Upgrade to head
+ run: |
+ alembic -x url="$DATABASE_URL" upgrade head
+
+ # Checks downgrade and upgrade again (reversibility)
+ - name: Downgrade to base and upgrade again (reversibility check)
+ run: |
+ alembic -x url="$DATABASE_URL" downgrade base
+ alembic -x url="$DATABASE_URL" upgrade head
+
+ - name: Check for pending autogenerate (python-based)
+ env:
+ PYTHONPATH: "${PYTHONPATH}:."
+ run: python -m scripts.ci_alembic_check
diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml
index 0872ac1c4..4356ba6eb 100644
--- a/.github/workflows/pre-commit.yaml
+++ b/.github/workflows/pre-commit.yaml
@@ -1,26 +1,27 @@
name: Pre-commit checks
-on:
- push
+on: push
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- - name: 'Check out repository code'
+ - name: "Check out repository code"
uses: actions/checkout@v3
- name: Use Node.js
uses: actions/setup-node@v3
with:
- node-version: '18.x'
- cache: 'yarn'
+ node-version: "18.x"
+ cache: "yarn"
cache-dependency-path: DashAI/front/yarn.lock
- name: Install frontend deps
working-directory: DashAI/front
run: yarn install --frozen-lockfile
- - name: Setup latest Python 3.x
- uses: actions/setup-python@v3
+ - name: Setup Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
- name: Install pre-commit
run: |
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 79c3bedcb..0fdb94455 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -15,13 +15,13 @@ repos:
rev: v0.12.10
hooks:
- id: ruff
- args: [--fix]
+ args: [--fix, "--show-fixes"]
- id: ruff-format
- repo: https://github.com/pre-commit/mirrors-prettier
rev: "v4.0.0-alpha.8"
hooks:
- id: prettier
- files: \.[jt]sx?$ # *.js, *.jsx, *.ts, *.tsx
+ files: \.[jt]sx?$ # *.js, *.jsx, *.ts, *.tsx
types: [file]
args:
- "--config=DashAI/front/.prettierrc"
diff --git a/DashAI/alembic/env.py b/DashAI/alembic/env.py
new file mode 100644
index 000000000..0f3c44a1b
--- /dev/null
+++ b/DashAI/alembic/env.py
@@ -0,0 +1,91 @@
+import os
+from pathlib import Path
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config, pool
+from alembic import context
+
+from DashAI.back.dependencies.database import Base
+
+config = context.config
+
+
+def _resolve_url() -> str:
+ """
+ Prioridad url sql:
+ 1) alembic -x url=...
+ 2) env var DATABASE_URL
+ 3) value already present in alembic.ini (config.get_main_option)
+ 4) fallback using the same logic as the application (~/.DashAI/db.sqlite)
+ """
+ # 1) cli -x url
+ xargs = context.get_x_argument(as_dictionary=True)
+ if xargs and xargs.get("url"):
+ return xargs["url"]
+
+ # 2) environment
+ env_url = os.getenv("DATABASE_URL")
+ if env_url:
+ return env_url
+
+ # 3) value from alembic.ini (if any)
+ cfg_url = config.get_main_option("sqlalchemy.url")
+ if cfg_url and cfg_url != "sqlite:///":
+ return cfg_url
+
+ # 4) fallback - same as application config
+ # Use the same default path as the application: ~/.DashAI/db.sqlite
+ from DashAI.back.config import DefaultSettings
+ settings = DefaultSettings()
+ local_path = Path(settings.LOCAL_PATH).expanduser().absolute()
+ db_path = local_path / settings.SQLITE_DB_PATH
+ return f"sqlite:///{db_path}"
+
+
+config.set_main_option("sqlalchemy.url", _resolve_url())
+
+if config.config_file_name is not None:
+ fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+
+
+def run_migrations_offline():
+ url = config.get_main_option("sqlalchemy.url")
+ context.configure(
+ url=url,
+ target_metadata=target_metadata,
+ literal_binds=True,
+ compare_type=True,
+ compare_server_default=True,
+ )
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+def run_migrations_online():
+ ini = dict(config.get_section(config.config_ini_section) or {})
+
+ connectable = engine_from_config(
+ ini,
+ prefix="sqlalchemy.",
+ poolclass=pool.NullPool,
+ future=True,
+ )
+
+ with connectable.connect() as connection:
+ context.configure(
+ connection=connection,
+ target_metadata=target_metadata,
+ compare_type=True,
+ compare_server_default=True,
+ render_as_batch=True,
+ )
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()
diff --git a/alembic/script.py.mako b/DashAI/alembic/script.py.mako
similarity index 57%
rename from alembic/script.py.mako
rename to DashAI/alembic/script.py.mako
index 55df2863d..fbc4b07dc 100644
--- a/alembic/script.py.mako
+++ b/DashAI/alembic/script.py.mako
@@ -5,15 +5,17 @@ Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
+from typing import Sequence, Union
+
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
-revision = ${repr(up_revision)}
-down_revision = ${repr(down_revision)}
-branch_labels = ${repr(branch_labels)}
-depends_on = ${repr(depends_on)}
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
diff --git a/DashAI/alembic/versions/57ff87150ae2_experiments_to_model_session.py b/DashAI/alembic/versions/57ff87150ae2_experiments_to_model_session.py
new file mode 100644
index 000000000..bd2815f5f
--- /dev/null
+++ b/DashAI/alembic/versions/57ff87150ae2_experiments_to_model_session.py
@@ -0,0 +1,76 @@
+"""Experiments to model session
+
+Revision ID: 57ff87150ae2
+Revises: 7662169fa0e0
+Create Date: 2026-01-19 16:14:03.739306
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import sqlite
+
+# revision identifiers, used by Alembic.
+revision: str = '57ff87150ae2'
+down_revision: Union[str, None] = '7662169fa0e0'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.create_table('model_session',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('dataset_id', sa.Integer(), nullable=False),
+ sa.Column('name', sa.String(), nullable=False),
+ sa.Column('task_name', sa.String(), nullable=False),
+ sa.Column('input_columns', sa.JSON(), nullable=False),
+ sa.Column('output_columns', sa.JSON(), nullable=False),
+ sa.Column('train_metrics', sa.JSON(), nullable=True),
+ sa.Column('validation_metrics', sa.JSON(), nullable=True),
+ sa.Column('test_metrics', sa.JSON(), nullable=True),
+ sa.Column('splits', sa.JSON(), nullable=False),
+ sa.Column('created', sa.DateTime(), nullable=False),
+ sa.Column('last_modified', sa.DateTime(), nullable=False),
+ sa.ForeignKeyConstraint(['dataset_id'], ['dataset.id'], name=op.f('fk_model_session_dataset_id_dataset')),
+ sa.PrimaryKeyConstraint('id', name=op.f('pk_model_session')),
+ sa.UniqueConstraint('name', name=op.f('uq_model_session_name'))
+ )
+ with op.batch_alter_table('run', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('model_session_id', sa.Integer(), nullable=False))
+ batch_op.drop_constraint(batch_op.f('fk_run_experiment_id_experiment'), type_='foreignkey')
+ batch_op.create_foreign_key(batch_op.f('fk_run_model_session_id_model_session'), 'model_session', ['model_session_id'], ['id'], ondelete='CASCADE')
+ batch_op.drop_column('experiment_id')
+ op.drop_table('experiment')
+
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table('run', schema=None) as batch_op:
+ batch_op.add_column(sa.Column('experiment_id', sa.INTEGER(), nullable=False))
+ batch_op.drop_constraint(batch_op.f('fk_run_model_session_id_model_session'), type_='foreignkey')
+ batch_op.create_foreign_key(batch_op.f('fk_run_experiment_id_experiment'), 'experiment', ['experiment_id'], ['id'], ondelete='CASCADE')
+ batch_op.drop_column('model_session_id')
+
+ op.create_table('experiment',
+ sa.Column('id', sa.INTEGER(), nullable=False),
+ sa.Column('dataset_id', sa.INTEGER(), nullable=False),
+ sa.Column('name', sa.VARCHAR(), nullable=False),
+ sa.Column('task_name', sa.VARCHAR(), nullable=False),
+ sa.Column('input_columns', sqlite.JSON(), nullable=False),
+ sa.Column('output_columns', sqlite.JSON(), nullable=False),
+ sa.Column('train_metrics', sqlite.JSON(), nullable=True),
+ sa.Column('validation_metrics', sqlite.JSON(), nullable=True),
+ sa.Column('test_metrics', sqlite.JSON(), nullable=True),
+ sa.Column('splits', sqlite.JSON(), nullable=False),
+ sa.Column('created', sa.DATETIME(), nullable=False),
+ sa.Column('last_modified', sa.DATETIME(), nullable=False),
+ sa.ForeignKeyConstraint(['dataset_id'], ['dataset.id'], name=op.f('fk_experiment_dataset_id_dataset')),
+ sa.PrimaryKeyConstraint('id', name=op.f('pk_experiment')),
+ sa.UniqueConstraint('name', name=op.f('uq_experiment_name'))
+ )
+ op.drop_table('model_session')
+ # ### end Alembic commands ###
diff --git a/DashAI/alembic/versions/7662169fa0e0_initial_migrations.py b/DashAI/alembic/versions/7662169fa0e0_initial_migrations.py
new file mode 100644
index 000000000..82b1a4dc7
--- /dev/null
+++ b/DashAI/alembic/versions/7662169fa0e0_initial_migrations.py
@@ -0,0 +1,283 @@
+"""Initial migrations
+
+Revision ID: 7662169fa0e0
+Revises:
+Create Date: 2026-01-07 10:58:08.748292
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '7662169fa0e0'
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.create_table('dataset',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('name', sa.String(), nullable=False),
+ sa.Column('huey_id', sa.String(), nullable=True),
+ sa.Column('created', sa.DateTime(), nullable=False),
+ sa.Column('last_modified', sa.DateTime(), nullable=False),
+ sa.Column('file_path', sa.String(), nullable=False),
+ sa.Column('status', sa.Enum('NOT_STARTED', 'DELIVERED', 'STARTED', 'FINISHED', 'ERROR', name='datasetstatus'), nullable=False),
+ sa.PrimaryKeyConstraint('id'),
+ sa.UniqueConstraint('name')
+ )
+ op.create_table('generative_session',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('created', sa.DateTime(), nullable=False),
+ sa.Column('last_modified', sa.DateTime(), nullable=False),
+ sa.Column('task_name', sa.String(), nullable=False),
+ sa.Column('model_name', sa.String(), nullable=False),
+ sa.Column('parameters', sa.JSON(), nullable=False),
+ sa.Column('name', sa.String(), nullable=False),
+ sa.Column('description', sa.String(), nullable=True),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.create_table('global_explainer',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('name', sa.String(), nullable=False),
+ sa.Column('run_id', sa.Integer(), nullable=False),
+ sa.Column('huey_id', sa.String(), nullable=True),
+ sa.Column('explainer_name', sa.String(), nullable=False),
+ sa.Column('explanation_path', sa.String(), nullable=True),
+ sa.Column('plot_path', sa.String(), nullable=True),
+ sa.Column('parameters', sa.JSON(), nullable=False),
+ sa.Column('created', sa.DateTime(), nullable=False),
+ sa.Column('status', sa.Enum('NOT_STARTED', 'DELIVERED', 'STARTED', 'FINISHED', 'ERROR', name='explainerstatus'), nullable=False),
+ sa.PrimaryKeyConstraint('id'),
+ sa.UniqueConstraint('name')
+ )
+ op.create_table('local_explainer',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('name', sa.String(), nullable=False),
+ sa.Column('run_id', sa.Integer(), nullable=False),
+ sa.Column('huey_id', sa.String(), nullable=True),
+ sa.Column('explainer_name', sa.String(), nullable=False),
+ sa.Column('dataset_id', sa.Integer(), nullable=False),
+ sa.Column('explanation_path', sa.String(), nullable=True),
+ sa.Column('plots_path', sa.String(), nullable=True),
+ sa.Column('parameters', sa.JSON(), nullable=False),
+ sa.Column('fit_parameters', sa.JSON(), nullable=False),
+ sa.Column('scope', sa.JSON(), nullable=False),
+ sa.Column('created', sa.DateTime(), nullable=False),
+ sa.Column('status', sa.Enum('NOT_STARTED', 'DELIVERED', 'STARTED', 'FINISHED', 'ERROR', name='explainerstatus'), nullable=False),
+ sa.PrimaryKeyConstraint('id'),
+ sa.UniqueConstraint('name')
+ )
+ op.create_table('pipeline',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('name', sa.String(), nullable=False),
+ sa.Column('steps', sa.JSON(), nullable=True),
+ sa.Column('edges', sa.JSON(), nullable=True),
+ sa.Column('exploration', sa.JSON(), nullable=True),
+ sa.Column('train', sa.JSON(), nullable=True),
+ sa.Column('prediction', sa.JSON(), nullable=True),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.create_table('plugin',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('name', sa.String(), nullable=False),
+ sa.Column('author', sa.String(), nullable=False),
+ sa.Column('installed_version', sa.String(), nullable=False),
+ sa.Column('lastest_version', sa.String(), nullable=False),
+ sa.Column('status', sa.Enum('REGISTERED', 'INSTALLED', 'ERROR', name='pluginstatus'), nullable=False),
+ sa.Column('summary', sa.String(), nullable=False),
+ sa.Column('description', sa.String(), nullable=False),
+ sa.Column('description_content_type', sa.String(), nullable=False),
+ sa.Column('created', sa.DateTime(), nullable=False),
+ sa.Column('last_modified', sa.DateTime(), nullable=False),
+ sa.PrimaryKeyConstraint('id'),
+ sa.UniqueConstraint('name')
+ )
+ op.create_table('experiment',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('dataset_id', sa.Integer(), nullable=False),
+ sa.Column('name', sa.String(), nullable=False),
+ sa.Column('task_name', sa.String(), nullable=False),
+ sa.Column('input_columns', sa.JSON(), nullable=False),
+ sa.Column('output_columns', sa.JSON(), nullable=False),
+ sa.Column('train_metrics', sa.JSON(), nullable=True),
+ sa.Column('validation_metrics', sa.JSON(), nullable=True),
+ sa.Column('test_metrics', sa.JSON(), nullable=True),
+ sa.Column('splits', sa.JSON(), nullable=False),
+ sa.Column('created', sa.DateTime(), nullable=False),
+ sa.Column('last_modified', sa.DateTime(), nullable=False),
+ sa.ForeignKeyConstraint(['dataset_id'], ['dataset.id'], ),
+ sa.PrimaryKeyConstraint('id'),
+ sa.UniqueConstraint('name')
+ )
+ op.create_table('generative_process',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('created', sa.DateTime(), nullable=False),
+ sa.Column('last_modified', sa.DateTime(), nullable=False),
+ sa.Column('session_id', sa.Integer(), nullable=False),
+ sa.Column('status', sa.Enum('NOT_STARTED', 'DELIVERED', 'STARTED', 'FINISHED', 'ERROR', name='runstatus'), nullable=False),
+ sa.Column('delivery_time', sa.DateTime(), nullable=True),
+ sa.Column('start_time', sa.DateTime(), nullable=True),
+ sa.Column('end_time', sa.DateTime(), nullable=True),
+ sa.ForeignKeyConstraint(['session_id'], ['generative_session.id'], ondelete='CASCADE'),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.create_table('notebook',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('dataset_id', sa.Integer(), nullable=False),
+ sa.Column('created', sa.DateTime(), nullable=False),
+ sa.Column('last_modified', sa.DateTime(), nullable=False),
+ sa.Column('file_path', sa.String(), nullable=False),
+ sa.Column('name', sa.String(), nullable=True),
+ sa.Column('description', sa.String(), nullable=True),
+ sa.ForeignKeyConstraint(['dataset_id'], ['dataset.id'], ondelete='CASCADE'),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.create_table('parameter_history',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('session_id', sa.Integer(), nullable=False),
+ sa.Column('parameters', sa.JSON(), nullable=False),
+ sa.Column('modified_at', sa.DateTime(), nullable=False),
+ sa.ForeignKeyConstraint(['session_id'], ['generative_session.id'], ondelete='CASCADE'),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.create_table('tag',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('plugin_id', sa.Integer(), nullable=False),
+ sa.Column('name', sa.Enum('DashAI', 'Package', 'Task', 'Model', 'Metric', 'Dataloader', 'Converter', 'Explainer', name='plugintag'), nullable=False),
+ sa.ForeignKeyConstraint(['plugin_id'], ['plugin.id'], ),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.create_table('converter_list',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('notebook_id', sa.Integer(), nullable=False),
+ sa.Column('huey_id', sa.String(), nullable=True),
+ sa.Column('converter', sa.String(), nullable=False),
+ sa.Column('parameters', sa.JSON(), nullable=False),
+ sa.Column('created', sa.DateTime(), nullable=False),
+ sa.Column('last_modified', sa.DateTime(), nullable=False),
+ sa.Column('status', sa.Enum('NOT_STARTED', 'DELIVERED', 'STARTED', 'FINISHED', 'ERROR', name='converterliststatus'), nullable=False),
+ sa.Column('delivery_time', sa.DateTime(), nullable=True),
+ sa.Column('start_time', sa.DateTime(), nullable=True),
+ sa.Column('end_time', sa.DateTime(), nullable=True),
+ sa.ForeignKeyConstraint(['notebook_id'], ['notebook.id'], ondelete='CASCADE'),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.create_table('explorer',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('notebook_id', sa.Integer(), nullable=False),
+ sa.Column('huey_id', sa.String(), nullable=True),
+ sa.Column('created', sa.DateTime(), nullable=False),
+ sa.Column('last_modified', sa.DateTime(), nullable=False),
+ sa.Column('columns', sa.JSON(), nullable=False),
+ sa.Column('exploration_type', sa.String(), nullable=False),
+ sa.Column('parameters', sa.JSON(), nullable=False),
+ sa.Column('exploration_path', sa.String(), nullable=True),
+ sa.Column('name', sa.String(), nullable=True),
+ sa.Column('delivery_time', sa.DateTime(), nullable=True),
+ sa.Column('start_time', sa.DateTime(), nullable=True),
+ sa.Column('end_time', sa.DateTime(), nullable=True),
+ sa.Column('status', sa.Enum('NOT_STARTED', 'DELIVERED', 'STARTED', 'FINISHED', 'ERROR', name='explorerstatus'), nullable=False),
+ sa.ForeignKeyConstraint(['notebook_id'], ['notebook.id'], ondelete='CASCADE'),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.create_table('process_data',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('data', sa.String(), nullable=False),
+ sa.Column('data_type', sa.String(), nullable=False),
+ sa.Column('process_id', sa.Integer(), nullable=False),
+ sa.Column('is_input', sa.Boolean(), nullable=False),
+ sa.ForeignKeyConstraint(['process_id'], ['generative_process.id'], ondelete='CASCADE'),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.create_table('run',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('experiment_id', sa.Integer(), nullable=False),
+ sa.Column('huey_id', sa.String(), nullable=True),
+ sa.Column('created', sa.DateTime(), nullable=False),
+ sa.Column('last_modified', sa.DateTime(), nullable=False),
+ sa.Column('model_name', sa.String(), nullable=False),
+ sa.Column('parameters', sa.JSON(), nullable=False),
+ sa.Column('split_indexes', sa.JSON(), nullable=True),
+ sa.Column('optimizer_name', sa.String(), nullable=False),
+ sa.Column('optimizer_parameters', sa.JSON(), nullable=False),
+ sa.Column('plot_history_path', sa.String(), nullable=True),
+ sa.Column('plot_slice_path', sa.String(), nullable=True),
+ sa.Column('plot_contour_path', sa.String(), nullable=True),
+ sa.Column('plot_importance_path', sa.String(), nullable=True),
+ sa.Column('goal_metric', sa.String(), nullable=False),
+ sa.Column('artifacts', sa.JSON(), nullable=True),
+ sa.Column('name', sa.String(), nullable=False),
+ sa.Column('description', sa.String(), nullable=True),
+ sa.Column('run_path', sa.String(), nullable=True),
+ sa.Column('status', sa.Enum('NOT_STARTED', 'DELIVERED', 'STARTED', 'FINISHED', 'ERROR', name='runstatus'), nullable=False),
+ sa.Column('delivery_time', sa.DateTime(), nullable=True),
+ sa.Column('start_time', sa.DateTime(), nullable=True),
+ sa.Column('end_time', sa.DateTime(), nullable=True),
+ sa.ForeignKeyConstraint(['experiment_id'], ['experiment.id'], ondelete='CASCADE'),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.create_table('metric',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('run_id', sa.Integer(), nullable=False),
+ sa.Column('split', sa.Enum('TRAIN', 'VALIDATION', 'TEST', name='splitenum'), nullable=False),
+ sa.Column('level', sa.Enum('LAST', 'TRIAL', 'STEP', 'EPOCH', name='levelenum'), nullable=False),
+ sa.Column('name', sa.String(), nullable=False),
+ sa.Column('value', sa.Float(), nullable=False),
+ sa.Column('step', sa.Integer(), nullable=False),
+ sa.Column('timestamp', sa.DateTime(), nullable=False),
+ sa.ForeignKeyConstraint(['run_id'], ['run.id'], ondelete='CASCADE'),
+ sa.PrimaryKeyConstraint('id')
+ )
+ with op.batch_alter_table('metric', schema=None) as batch_op:
+ batch_op.create_index(batch_op.f('ix_metric_run_id'), ['run_id'], unique=False)
+ batch_op.create_index(batch_op.f('ix_metric_timestamp'), ['timestamp'], unique=False)
+
+ op.create_table('prediction',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('run_id', sa.Integer(), nullable=False),
+ sa.Column('dataset_id', sa.Integer(), nullable=True),
+ sa.Column('huey_id', sa.String(), nullable=True),
+ sa.Column('created', sa.DateTime(), nullable=False),
+ sa.Column('last_modified', sa.DateTime(), nullable=False),
+ sa.Column('results_path', sa.String(), nullable=True),
+ sa.Column('status', sa.Enum('NOT_STARTED', 'DELIVERED', 'STARTED', 'FINISHED', 'ERROR', name='predictionstatus'), nullable=False),
+ sa.Column('delivery_time', sa.DateTime(), nullable=True),
+ sa.Column('start_time', sa.DateTime(), nullable=True),
+ sa.Column('end_time', sa.DateTime(), nullable=True),
+ sa.ForeignKeyConstraint(['dataset_id'], ['dataset.id'], ),
+ sa.ForeignKeyConstraint(['run_id'], ['run.id'], ondelete='CASCADE'),
+ sa.PrimaryKeyConstraint('id')
+ )
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_table('prediction')
+ with op.batch_alter_table('metric', schema=None) as batch_op:
+ batch_op.drop_index(batch_op.f('ix_metric_timestamp'))
+ batch_op.drop_index(batch_op.f('ix_metric_run_id'))
+
+ op.drop_table('metric')
+ op.drop_table('run')
+ op.drop_table('process_data')
+ op.drop_table('explorer')
+ op.drop_table('converter_list')
+ op.drop_table('tag')
+ op.drop_table('parameter_history')
+ op.drop_table('notebook')
+ op.drop_table('generative_process')
+ op.drop_table('experiment')
+ op.drop_table('plugin')
+ op.drop_table('pipeline')
+ op.drop_table('local_explainer')
+ op.drop_table('global_explainer')
+ op.drop_table('generative_session')
+ op.drop_table('dataset')
+ # ### end Alembic commands ###
diff --git a/DashAI/back/app.py b/DashAI/back/app.py
index 3164801bb..71cc377dd 100644
--- a/DashAI/back/app.py
+++ b/DashAI/back/app.py
@@ -13,7 +13,7 @@
from DashAI.back.api.front_api import router as app_router
from DashAI.back.container import build_container
from DashAI.back.dependencies.config_builder import build_config_dict
-from DashAI.back.dependencies.database.models import Base
+from DashAI.back.dependencies.database.migrate import migrate_on_startup
logger = logging.getLogger(__name__)
@@ -81,15 +81,16 @@ def create_app(
logger.debug("3. Creating app container and setting up dependency injection.")
container = build_container(config=config)
- logger.debug("5. Creating database.")
- Base.metadata.create_all(bind=container["engine"])
-
- logger.debug("6. Initializing FastAPI application.")
+ logger.debug("4. Applying database migrations.")
+ migrate_on_startup(
+ sqlite_file_path=pathlib.Path(config["SQLITE_DB_PATH"]),
+ )
+ logger.debug("5. Initializing FastAPI application.")
app = FastAPI(title="DashAI")
api_v0 = FastAPI(title="DashAI API v0")
api_v1 = FastAPI(title="DashAI API v1")
- logger.debug("7. Mounting API routers.")
+ logger.debug("6. Mounting API routers.")
api_v0.include_router(api_router_v0)
api_v1.include_router(api_router_v1)
diff --git a/DashAI/back/dependencies/database/migrate.py b/DashAI/back/dependencies/database/migrate.py
new file mode 100644
index 000000000..ffcbda073
--- /dev/null
+++ b/DashAI/back/dependencies/database/migrate.py
@@ -0,0 +1,66 @@
+import logging
+import os
+import shutil
+import time
+from pathlib import Path
+
+from alembic import command
+from alembic.config import Config
+
+from DashAI.back.dependencies.database.utils import resolve_db_url
+
+logger = logging.getLogger(__name__)
+
+
+def alembic_config(db_url: str) -> Config:
+ package_root = Path(__file__).absolute().parents[3]
+ script_location = package_root / "alembic"
+
+ cfg = Config()
+
+ cfg.set_main_option("script_location", str(script_location))
+ cfg.set_main_option("sqlalchemy.url", db_url)
+ cfg.set_main_option("prepend_sys_path", str(package_root))
+ cfg.set_main_option("version_path_separator", "os")
+ return cfg
+
+
+def run_migrations(db_url: str) -> None:
+ cfg = alembic_config(db_url=db_url)
+ command.upgrade(cfg, "head")
+
+
+def backup_and_recreate_db(db_url: str, sqlite_file_path: Path) -> None:
+ # Only try to backup if it's a local SQLite file and DATABASE_URL env var not set
+ env_url = os.getenv("DATABASE_URL")
+ if not env_url and sqlite_file_path.exists():
+ ts = time.strftime("%Y%m%d-%H%M%S")
+ backup = sqlite_file_path.with_suffix(f".bak-{ts}.sqlite3")
+ shutil.copy2(sqlite_file_path, backup)
+ sqlite_file_path.unlink()
+
+ run_migrations(db_url=db_url)
+
+
+def migrate_on_startup(sqlite_file_path: Path) -> None:
+ db_url = resolve_db_url(sqlite_file_path)
+
+ try:
+ logger.info(f"Running migrations on database at {db_url}")
+ run_migrations(db_url=db_url)
+ except Exception as exc:
+ logger.error(
+ (
+ f"Error during migration: {exc}. "
+ "Attempting to backup and recreate the database."
+ )
+ )
+ try:
+ backup_and_recreate_db(db_url=db_url, sqlite_file_path=sqlite_file_path)
+ except Exception as backup_exc:
+ logger.error(
+ f"Error during backup and recreate: {backup_exc}. "
+ f"Original migration error: {exc}."
+ )
+ raise backup_exc from exc
+ raise exc
diff --git a/DashAI/back/dependencies/database/models.py b/DashAI/back/dependencies/database/models.py
index 2f564f079..84433a742 100644
--- a/DashAI/back/dependencies/database/models.py
+++ b/DashAI/back/dependencies/database/models.py
@@ -11,6 +11,7 @@
Float,
ForeignKey,
Integer,
+ MetaData,
String,
)
from sqlalchemy.ext.declarative import declarative_base
@@ -31,7 +32,16 @@
logger = logging.getLogger(__name__)
-Base = declarative_base()
+naming_convention = {
+ "ix": "ix_%(table_name)s_%(column_0_name)s",
+ "uq": "uq_%(table_name)s_%(column_0_name)s",
+ "ck": "ck_%(constraint_name)s",
+ "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+ "pk": "pk_%(table_name)s",
+}
+
+metadata = MetaData(naming_convention=naming_convention)
+Base = declarative_base(metadata=metadata)
class Dataset(Base):
diff --git a/DashAI/back/dependencies/database/sqlite_database.py b/DashAI/back/dependencies/database/sqlite_database.py
index 6601f72de..e450a184f 100644
--- a/DashAI/back/dependencies/database/sqlite_database.py
+++ b/DashAI/back/dependencies/database/sqlite_database.py
@@ -7,6 +7,8 @@
from sqlalchemy.engine import Engine
from sqlalchemy.orm import sessionmaker
+from DashAI.back.dependencies.database.utils import resolve_db_url
+
logger = logging.getLogger(__name__)
@@ -32,8 +34,7 @@ def setup_sqlite_db(config: Dict[str, str]) -> Tuple[Engine, sessionmaker]:
for creating database sessions.
"""
- if not str(config["SQLITE_DB_PATH"]).startswith("sqlite:///"):
- db_url = "sqlite:///" + str(config["SQLITE_DB_PATH"])
+ db_url = resolve_db_url(config["SQLITE_DB_PATH"])
logger.info("Using %s as SQLite path.", db_url)
diff --git a/DashAI/back/dependencies/database/utils.py b/DashAI/back/dependencies/database/utils.py
index e9bbbd991..5fe4b3d44 100644
--- a/DashAI/back/dependencies/database/utils.py
+++ b/DashAI/back/dependencies/database/utils.py
@@ -1,4 +1,6 @@
import logging
+import os
+from pathlib import Path
from fastapi import Depends
from kink import di, inject
@@ -276,3 +278,23 @@ def find_entity_by_huey_id(huey_id: str) -> dict:
}
return None
+
+
+def resolve_db_url(sqlite_file_path: Path) -> str:
+ """
+ Resolve database URL with the same priority as alembic:
+ 1) env var DATABASE_URL
+ 2) fallback to provided sqlite file path
+
+ This is mainly to use in tests, where we set the env var to a temp path.
+ """
+ # 1) environment variable
+ env_url = os.getenv("DATABASE_URL")
+ if env_url:
+ return env_url
+
+ if not str(sqlite_file_path).startswith("sqlite:///"):
+ return f"sqlite:///{sqlite_file_path}"
+
+ # 2) fallback to provided sqlite file path
+ return sqlite_file_path
diff --git a/MANIFEST.in b/MANIFEST.in
index 51353ff30..a649ed52f 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,8 +1,11 @@
include requirements.txt
include requirements-dev.txt
+include DashAI/alembic.ini
+recursive-include DashAI/alembic *
recursive-include DashAI/front/build *
recursive-include DashAI/back/models/parameters/models_schemas *
recursive-include DashAI/back/dataloaders/dataloaders_schemas *
recursive-include DashAI/back/dataloaders/description_schemas *
recursive-include DashAI/back/dataloaders/params_schemas *
+recursive-include DashAI/back/types *
prune tests/
diff --git a/README.rst b/README.rst
index be6b32dae..1e2169a1a 100644
--- a/README.rst
+++ b/README.rst
@@ -199,6 +199,97 @@ You can check all available options through the command:
$ python -m DashAI --help
+Database Migrations
+==================
+
+Migrations are managed through `Alembic `_.
+
+They are automatically executed when starting DashAI. However, if you want to
+run them manually, you can do so using the following command (inside the
+`DashAI/` folder):
+
+.. code-block:: bash
+
+ $ alembic upgrade head
+
+This command applies all pending migrations up to the latest revision.
+
+---
+
+Creating a New Migration
+------------------------
+
+After modifying the database models, a new migration can be generated using:
+
+.. code-block:: bash
+
+ $ alembic revision --autogenerate -m "<>"
+
+Where ``<>`` is a brief description of the changes introduced
+(e.g., *add model metadata table*, *update dataset schema*).
+
+Generated migrations are located in the ``alembic/versions`` directory and
+**must be committed to the repository**.
+
+It is strongly recommended to review the autogenerated migration file before
+applying it, as Alembic may not always detect complex changes correctly.
+
+---
+
+Applying Migrations
+-------------------
+
+To apply all pending migrations:
+
+.. code-block:: bash
+
+ $ alembic upgrade head
+
+To upgrade to a specific revision:
+
+.. code-block:: bash
+
+ $ alembic upgrade
+
+---
+
+Downgrading Migrations
+----------------------
+
+If you need to revert database changes, migrations can be downgraded using:
+
+.. code-block:: bash
+
+ $ alembic downgrade -1
+
+This command reverts the last applied migration.
+
+To downgrade to a specific revision:
+
+.. code-block:: bash
+
+ $ alembic downgrade
+
+---
+
+Checking Migration Status
+-------------------------
+
+To view the current migration applied to the database:
+
+.. code-block:: bash
+
+ $ alembic current
+
+To list the full migration history:
+
+.. code-block:: bash
+
+ $ alembic history
+
+---
+
+
Testing
=======
diff --git a/alembic.ini b/alembic.ini
index 8ee69ae36..e71144cf7 100644
--- a/alembic.ini
+++ b/alembic.ini
@@ -2,7 +2,7 @@
[alembic]
# path to migration scripts
-script_location = alembic
+script_location = DashAI/alembic
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
# Uncomment the line below if you want the files to be prepended with date and time
@@ -60,7 +60,9 @@ version_path_separator = os # Use os.pathsep. Default configuration used for ne
# are written from script.py.mako
# output_encoding = utf-8
-sqlalchemy.url = sqlite:///DashAI/back/database/DashAI.sqlite
+# SQLite URL points to the same database used by the application
+# Note: This will be overridden by environment variables or CLI arguments if provided
+sqlalchemy.url = sqlite:///
[post_write_hooks]
diff --git a/alembic/env.py b/alembic/env.py
deleted file mode 100644
index 7907b3085..000000000
--- a/alembic/env.py
+++ /dev/null
@@ -1,67 +0,0 @@
-from logging.config import fileConfig
-
-from sqlalchemy import engine_from_config, pool
-
-from alembic import context
-from DashAI.back.database.models import Base
-
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
-config = context.config
-
-# Interpret the config file for Python logging.
-# This line sets up loggers basically.
-if config.config_file_name is not None:
- fileConfig(config.config_file_name)
-
-target_metadata = Base.metadata
-
-
-def run_migrations_offline() -> None:
- """Run migrations in 'offline' mode.
-
- This configures the context with just a URL
- and not an Engine, though an Engine is acceptable
- here as well. By skipping the Engine creation
- we don't even need a DBAPI to be available.
-
- Calls to context.execute() here emit the given string to the
- script output.
-
- """
- url = config.get_main_option("sqlalchemy.url")
- context.configure(
- url=url,
- target_metadata=target_metadata,
- literal_binds=True,
- dialect_opts={"paramstyle": "named"},
- )
-
- with context.begin_transaction():
- context.run_migrations()
-
-
-def run_migrations_online() -> None:
- """Run migrations in 'online' mode.
-
- In this scenario we need to create an Engine
- and associate a connection with the context.
-
- """
- connectable = engine_from_config(
- config.get_section(config.config_ini_section, {}),
- prefix="sqlalchemy.",
- poolclass=pool.NullPool,
- )
-
- with connectable.connect() as connection:
- context.configure(connection=connection, target_metadata=target_metadata)
-
- with context.begin_transaction():
- context.run_migrations()
-
-
-if context.is_offline_mode():
- run_migrations_offline()
-else:
- run_migrations_online()
diff --git a/alembic/versions/2f997776b9a1_add_splits_metadata_to_experiment_model_.py b/alembic/versions/2f997776b9a1_add_splits_metadata_to_experiment_model_.py
deleted file mode 100644
index cb96a15fe..000000000
--- a/alembic/versions/2f997776b9a1_add_splits_metadata_to_experiment_model_.py
+++ /dev/null
@@ -1,25 +0,0 @@
-"""Add splits metadata to Experiment model on DB
-
-Revision ID: 2f997776b9a1
-Revises: 7ba93a3d42a8
-Create Date: 2023-10-09 12:34:12.786244
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = "2f997776b9a1"
-down_revision = "7ba93a3d42a8"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- pass
- # ### end Alembic commands ###
-
-
-def downgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- pass
- # ### end Alembic commands ###
diff --git a/alembic/versions/4bce47006c88_add_splits_metadata_to_experiment_model_.py b/alembic/versions/4bce47006c88_add_splits_metadata_to_experiment_model_.py
deleted file mode 100644
index c8c7a000f..000000000
--- a/alembic/versions/4bce47006c88_add_splits_metadata_to_experiment_model_.py
+++ /dev/null
@@ -1,28 +0,0 @@
-"""Add splits metadata to Experiment model on DB
-
-Revision ID: 4bce47006c88
-Revises: 2f997776b9a1
-Create Date: 2023-10-09 12:57:13.075246
-
-"""
-import sqlalchemy as sa
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "4bce47006c88"
-down_revision = "2f997776b9a1"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.add_column("experiment", sa.Column("splits", sa.JSON(), nullable=False))
- # ### end Alembic commands ###
-
-
-def downgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.drop_column("experiment", "splits")
- # ### end Alembic commands ###
diff --git a/alembic/versions/649c500185b8_delete_task_name_and_adds_dataloader_.py b/alembic/versions/649c500185b8_delete_task_name_and_adds_dataloader_.py
deleted file mode 100644
index dd781e15c..000000000
--- a/alembic/versions/649c500185b8_delete_task_name_and_adds_dataloader_.py
+++ /dev/null
@@ -1,32 +0,0 @@
-"""Delete task_name and adds dataloader name to Dataset model
-
-Revision ID: 649c500185b8
-Revises: 80faf11605a2
-Create Date: 2023-08-09 08:43:29.112965
-
-"""
-import sqlalchemy as sa
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "649c500185b8"
-down_revision = "80faf11605a2"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.add_column("dataset", sa.Column("feature_names", sa.JSON(), nullable=False))
- op.add_column("dataset", sa.Column("dataset_name", sa.String(), nullable=False))
- op.drop_column("dataset", "task_name")
- # ### end Alembic commands ###
-
-
-def downgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.add_column("dataset", sa.Column("task_name", sa.VARCHAR(), nullable=False))
- op.drop_column("dataset", "dataset_name")
- op.drop_column("dataset", "feature_names")
- # ### end Alembic commands ###
diff --git a/alembic/versions/7ba93a3d42a8_delete_feature_names_to_experiment_.py b/alembic/versions/7ba93a3d42a8_delete_feature_names_to_experiment_.py
deleted file mode 100644
index 7ddac412d..000000000
--- a/alembic/versions/7ba93a3d42a8_delete_feature_names_to_experiment_.py
+++ /dev/null
@@ -1,31 +0,0 @@
-"""Delete feature_names to Experiment model on DB
-
-Revision ID: 7ba93a3d42a8
-Revises: 7e990d4a75ec
-Create Date: 2023-09-25 10:03:29.526284
-
-"""
-import sqlalchemy as sa
-from sqlalchemy.dialects import sqlite
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "7ba93a3d42a8"
-down_revision = "7e990d4a75ec"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.drop_column("experiment", "feature_names")
- # ### end Alembic commands ###
-
-
-def downgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.add_column(
- "experiment", sa.Column("feature_names", sqlite.JSON(), nullable=False)
- )
- # ### end Alembic commands ###
diff --git a/alembic/versions/7e990d4a75ec_add_metadata_to_experiment_model_on_db.py b/alembic/versions/7e990d4a75ec_add_metadata_to_experiment_model_on_db.py
deleted file mode 100644
index 216084ee6..000000000
--- a/alembic/versions/7e990d4a75ec_add_metadata_to_experiment_model_on_db.py
+++ /dev/null
@@ -1,30 +0,0 @@
-"""Add metadata to Experiment model on DB
-
-Revision ID: 7e990d4a75ec
-Revises: 8307d4c14824
-Create Date: 2023-09-22 22:45:34.448464
-
-"""
-import sqlalchemy as sa
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "7e990d4a75ec"
-down_revision = "8307d4c14824"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.add_column("experiment", sa.Column("input_columns", sa.JSON(), nullable=False))
- op.add_column("experiment", sa.Column("output_columns", sa.JSON(), nullable=False))
- # ### end Alembic commands ###
-
-
-def downgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.drop_column("experiment", "output_columns")
- op.drop_column("experiment", "input_columns")
- # ### end Alembic commands ###
diff --git a/alembic/versions/80faf11605a2_initial_migration.py b/alembic/versions/80faf11605a2_initial_migration.py
deleted file mode 100644
index d3da13249..000000000
--- a/alembic/versions/80faf11605a2_initial_migration.py
+++ /dev/null
@@ -1,30 +0,0 @@
-"""Initial migration
-
-Revision ID: 80faf11605a2
-Revises:
-Create Date: 2023-08-08 09:13:58.794747
-
-"""
-import sqlalchemy as sa
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "80faf11605a2"
-down_revision = None
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.drop_column("experiment", "step")
- # ### end Alembic commands ###
-
-
-def downgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.add_column(
- "experiment", sa.Column("step", sa.VARCHAR(length=19), nullable=False)
- )
- # ### end Alembic commands ###
diff --git a/alembic/versions/8307d4c14824_add_metadata_to_experiment_model_on_db.py b/alembic/versions/8307d4c14824_add_metadata_to_experiment_model_on_db.py
deleted file mode 100644
index 39c1b41d7..000000000
--- a/alembic/versions/8307d4c14824_add_metadata_to_experiment_model_on_db.py
+++ /dev/null
@@ -1,28 +0,0 @@
-"""Add metadata to Experiment model on DB
-
-Revision ID: 8307d4c14824
-Revises: d267eb740e0f
-Create Date: 2023-09-22 21:46:37.521272
-
-"""
-import sqlalchemy as sa
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "8307d4c14824"
-down_revision = "d267eb740e0f"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.add_column("experiment", sa.Column("feature_names", sa.JSON(), nullable=False))
- # ### end Alembic commands ###
-
-
-def downgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.drop_column("experiment", "feature_names")
- # ### end Alembic commands ###
diff --git a/alembic/versions/d267eb740e0f_delete_dataloader_name_in_dataset_model.py b/alembic/versions/d267eb740e0f_delete_dataloader_name_in_dataset_model.py
deleted file mode 100644
index 13ef6cc77..000000000
--- a/alembic/versions/d267eb740e0f_delete_dataloader_name_in_dataset_model.py
+++ /dev/null
@@ -1,28 +0,0 @@
-"""Delete dataloader name in Dataset model
-
-Revision ID: d267eb740e0f
-Revises: 649c500185b8
-Create Date: 2023-08-09 20:45:43.229983
-
-"""
-import sqlalchemy as sa
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "d267eb740e0f"
-down_revision = "649c500185b8"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.drop_column("dataset", "dataset_name")
- # ### end Alembic commands ###
-
-
-def downgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.add_column("dataset", sa.Column("dataset_name", sa.VARCHAR(), nullable=False))
- # ### end Alembic commands ###
diff --git a/alembic/versions/e40a1eeee0ef_delete_feature_names_from_datasets.py b/alembic/versions/e40a1eeee0ef_delete_feature_names_from_datasets.py
deleted file mode 100644
index ac89ee989..000000000
--- a/alembic/versions/e40a1eeee0ef_delete_feature_names_from_datasets.py
+++ /dev/null
@@ -1,29 +0,0 @@
-"""Delete feature_names from datasets
-
-Revision ID: e40a1eeee0ef
-Revises: 4bce47006c88
-Create Date: 2023-11-28 15:58:18.913263
-
-"""
-import sqlalchemy as sa
-from sqlalchemy.dialects import sqlite
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "e40a1eeee0ef"
-down_revision = "4bce47006c88"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.drop_column("dataset", "feature_names")
- # ### end Alembic commands ###
-
-
-def downgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
- op.add_column("dataset", sa.Column("feature_names", sqlite.JSON(), nullable=False))
- # ### end Alembic commands ###
diff --git a/scripts/ci_alembic_check.py b/scripts/ci_alembic_check.py
new file mode 100644
index 000000000..84dbdd7bc
--- /dev/null
+++ b/scripts/ci_alembic_check.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+import os
+import sys
+
+from alembic.autogenerate import compare_metadata
+from alembic.migration import MigrationContext
+from sqlalchemy import create_engine
+
+try:
+ from DashAI.back.dependencies.database.models import Base
+
+ metadata = Base.metadata
+except Exception as e:
+ print("ERROR importing project metadata:", e, file=sys.stderr)
+ sys.exit(2)
+
+database_url = os.environ.get("DATABASE_URL")
+if not database_url:
+ print("DATABASE_URL is not defined", file=sys.stderr)
+ sys.exit(2)
+
+engine = create_engine(database_url)
+with engine.connect() as conn:
+ ctx = MigrationContext.configure(conn)
+ diffs = compare_metadata(ctx, metadata)
+
+if diffs:
+ print("¡Detected schema drift! Autogenerate produced changes:")
+ for d in diffs:
+ print(d)
+ sys.exit(1)
+
+print("No pending schema changes detected.")
+sys.exit(0)