diff --git a/api.py b/api.py index dc080478d..2288185fb 100644 --- a/api.py +++ b/api.py @@ -71,6 +71,7 @@ migrate_datasets_table_to_filesystem, migrate_models_table_to_filesystem, migrate_tasks_table_to_filesystem, + migrate_config_table_to_filesystem, ) from transformerlab.shared.request_context import set_current_org_id from lab.dirs import set_organization_id as lab_set_org_id @@ -107,6 +108,7 @@ async def lifespan(app: FastAPI): asyncio.create_task(migrate_models_table_to_filesystem()) asyncio.create_task(migrate_datasets_table_to_filesystem()) asyncio.create_task(migrate_tasks_table_to_filesystem()) + asyncio.create_task(migrate_config_table_to_filesystem()) asyncio.create_task(run_over_and_over()) print("FastAPI LIFESPAN: 🏁 🏁 🏁 Begin API Server 🏁 🏁 🏁", flush=True) yield diff --git a/requirements-no-gpu-uv.txt b/requirements-no-gpu-uv.txt index 8a73db1c4..968bfbe28 100644 --- a/requirements-no-gpu-uv.txt +++ b/requirements-no-gpu-uv.txt @@ -548,7 +548,7 @@ tqdm==4.66.5 # peft # sentence-transformers # transformers -transformerlab==0.0.17 +transformerlab==0.0.20 # via -r requirements.in transformerlab-inference==0.2.50 # via -r requirements.in diff --git a/requirements-rocm-uv.txt b/requirements-rocm-uv.txt index 17d5e7176..09cf13dd3 100644 --- a/requirements-rocm-uv.txt +++ b/requirements-rocm-uv.txt @@ -551,7 +551,7 @@ tqdm==4.66.5 # peft # sentence-transformers # transformers -transformerlab==0.0.17 +transformerlab==0.0.20 # via -r requirements-rocm.in transformerlab-inference==0.2.50 # via -r requirements-rocm.in diff --git a/requirements-rocm.in b/requirements-rocm.in index be3f56680..f09630a74 100644 --- a/requirements-rocm.in +++ b/requirements-rocm.in @@ -32,7 +32,7 @@ hf_xet macmon-python mcp[cli] transformerlab-inference==0.2.50 -transformerlab==0.0.17 +transformerlab==0.0.20 diffusers==0.33.1 pyrsmi controlnet_aux==0.0.10 diff --git a/requirements-uv.txt b/requirements-uv.txt index f77340260..0ff57dfaf 100644 --- a/requirements-uv.txt +++ b/requirements-uv.txt @@ -586,7 +586,7 @@ tqdm==4.66.5 # peft # sentence-transformers # transformers -transformerlab==0.0.17 +transformerlab==0.0.20 # via -r requirements.in transformerlab-inference==0.2.50 # via -r requirements.in diff --git a/requirements.in b/requirements.in index 84f17711d..9eceb1463 100644 --- a/requirements.in +++ b/requirements.in @@ -31,7 +31,7 @@ markitdown[all] hf_xet macmon-python transformerlab-inference==0.2.50 -transformerlab==0.0.17 +transformerlab==0.0.20 diffusers==0.33.1 nvidia-ml-py mcp[cli] diff --git a/test/db/test_db.py b/test/db/test_db.py index 63349cc12..d02a2e64d 100644 --- a/test/db/test_db.py +++ b/test/db/test_db.py @@ -24,8 +24,6 @@ get_plugins, get_plugin, save_plugin, - config_get, - config_set, get_training_template, get_training_templates, create_training_template, @@ -33,6 +31,7 @@ delete_training_template, export_job_create, ) +from lab.config import Config as fs_config # noqa: E402 from transformerlab.db.sync import ( # noqa: E402 job_create_sync, @@ -177,7 +176,7 @@ async def test_get_plugins_of_type_returns_list(): @pytest.mark.asyncio async def test_config_get_returns_none_for_missing(): - value = await config_get("missing_config_key") + value = fs_config.get_value_by_key("missing_config_key") assert value is None @@ -419,23 +418,23 @@ async def test_get_plugins(self): class TestConfig: @pytest.mark.asyncio async def test_config_set_and_get(self): - await config_set("test_key", "test_value") - value = await config_get("test_key") + fs_config.set_value_by_key("test_key", "test_value") + value = fs_config.get_value_by_key("test_key") assert value == "test_value" # now try to set the same key with a different value - await config_set("test_key", "test_value2") - value = await config_get("test_key") + fs_config.set_value_by_key("test_key", "test_value2") + value = fs_config.get_value_by_key("test_key") assert value == "test_value2" # now try to get a key that does not exist - value = await config_get("test_key2_SHOULD_NOT_EXIST") + value = fs_config.get_value_by_key("test_key2_SHOULD_NOT_EXIST") assert value is None # now try to set a key with None value - await config_set("test_key3", None) - value = await config_get("test_key3") + fs_config.set_value_by_key("test_key3", None) + value = fs_config.get_value_by_key("test_key3") assert value is None # now try to set a key with empty string value - await config_set("test_key4", "") - value = await config_get("test_key4") + fs_config.set_value_by_key("test_key4", "") + value = fs_config.get_value_by_key("test_key4") assert value == "" diff --git a/transformerlab/db/db.py b/transformerlab/db/db.py index 68727f909..f5fbf6720 100644 --- a/transformerlab/db/db.py +++ b/transformerlab/db/db.py @@ -1,7 +1,6 @@ import json from sqlalchemy import select, delete, text, update -from sqlalchemy.dialects.sqlite import insert # Correct import for SQLite upsert # from sqlalchemy import create_engine from sqlalchemy.ext.asyncio import AsyncSession @@ -21,7 +20,7 @@ workflow_run_delete, ) from transformerlab.shared.models import models -from transformerlab.shared.models.models import Config, Plugin +from transformerlab.shared.models.models import Plugin from transformerlab.db.utils import sqlalchemy_to_dict, sqlalchemy_list_to_dict from transformerlab.db.session import async_session @@ -462,22 +461,3 @@ async def delete_plugin(name: str): return False -############### -# Config MODEL -############### - - -async def config_get(key: str): - async with async_session() as session: - result = await session.execute(select(Config.value).where(Config.key == key)) - row = result.scalar_one_or_none() - return row - - -async def config_set(key: str, value: str): - stmt = insert(Config).values(key=key, value=value) - stmt = stmt.on_conflict_do_update(index_elements=["key"], set_={"value": value}) - async with async_session() as session: - await session.execute(stmt) - await session.commit() - return diff --git a/transformerlab/db/filesystem_migrations.py b/transformerlab/db/filesystem_migrations.py index 6b925f236..d364df266 100644 --- a/transformerlab/db/filesystem_migrations.py +++ b/transformerlab/db/filesystem_migrations.py @@ -331,3 +331,60 @@ async def migrate_tasks_table_to_filesystem(): except Exception as e: # Do not block startup on migration issues print(f"Tasks migration skipped due to error: {e}") + + +async def migrate_config_table_to_filesystem(): + """ + One-time migration: copy rows from the legacy config DB table into the filesystem + registry via transformerlab-sdk, then drop/rename the table. + Safe to run multiple times; it will no-op if table is missing or empty. + """ + try: + from transformerlab.db.session import async_session + from sqlalchemy import text as sqlalchemy_text + from lab.config import Config as fs_config + + # Check table exists + async with async_session() as session: + result = await session.execute( + sqlalchemy_text("SELECT name FROM sqlite_master WHERE type='table' AND name='config'") + ) + exists = result.fetchone() is not None + if not exists: + return + + # Read rows + rows = [] + try: + async with async_session() as session: + result = await session.execute(sqlalchemy_text("SELECT key, value FROM config")) + rows = [dict(r) for r in result.mappings().all()] + except Exception as e: + print(f"Failed to read config for migration: {e}") + rows = [] + + migrated = 0 + for row in rows: + key = row.get("key") + if not key: + continue + value = row.get("value", None) + try: + fs_config.set_value_by_key(key, value) + migrated += 1 + except Exception as e: + print(f"Error migrating config key {key}: {e}") + continue + + # Rename legacy table + try: + async with async_session() as session: + await session.execute(sqlalchemy_text("ALTER TABLE config RENAME TO zzz_archived_config")) + await session.commit() + except Exception: + pass + + if migrated: + print(f"Config migration completed: {migrated} entries migrated to filesystem store.") + except Exception as e: + print(f"Config migration skipped due to error: {e}") diff --git a/transformerlab/routers/config.py b/transformerlab/routers/config.py index bb56313b7..85475a62c 100644 --- a/transformerlab/routers/config.py +++ b/transformerlab/routers/config.py @@ -1,5 +1,5 @@ from fastapi import APIRouter -import transformerlab.db.db as db +from lab.config import Config as fs_config router = APIRouter(prefix="/config", tags=["config"]) @@ -7,11 +7,11 @@ @router.get("/get/{key}", summary="") async def config_get(key: str): - value = await db.config_get(key=key) + value = fs_config.get_value_by_key(key) return value @router.get("/set", summary="") async def config_set(k: str, v: str): - await db.config_set(key=k, value=v) + fs_config.set_value_by_key(k, v) return {"key": k, "value": v} diff --git a/transformerlab/routers/model.py b/transformerlab/routers/model.py index 4d5e1e8d1..b76e83930 100644 --- a/transformerlab/routers/model.py +++ b/transformerlab/routers/model.py @@ -4,7 +4,6 @@ import datetime import dateutil.relativedelta from typing import Annotated -import transformerlab.db.db as db import transformerlab.db.jobs as db_jobs from fastapi import APIRouter, Body from fastchat.model.model_adapter import get_conversation_template @@ -311,7 +310,8 @@ async def model_details_from_filesystem(model_id: str): async def login_to_huggingface(): from huggingface_hub import get_token, login - token = await db.config_get("HuggingfaceUserAccessToken") + from lab.config import Config as fs_config + token = fs_config.get_value_by_key("HuggingfaceUserAccessToken") saved_token_in_hf_cache = get_token() # print(f"Saved token in HF cache: {saved_token_in_hf_cache}") @@ -342,7 +342,8 @@ async def login_to_wandb(): # TODO: Move all of these logins and their tests to another router outside 'model' to maintain clarity import wandb - token = await db.config_get("WANDB_API_KEY") + from lab.config import Config as fs_config + token = fs_config.get_value_by_key("WANDB_API_KEY") if token is None: return {"message": "WANDB_API not set"} @@ -372,7 +373,8 @@ def test_wandb_login(): @router.get(path="/model/set_openai_api_key") async def set_openai_api_key(): - token = await db.config_get("OPENAI_API_KEY") + from lab.config import Config as fs_config + token = fs_config.get_value_by_key("OPENAI_API_KEY") if not token or token == "": return {"message": "OPENAI_API_KEY not configured in database"} @@ -386,7 +388,8 @@ async def set_openai_api_key(): @router.get(path="/model/set_anthropic_api_key") async def set_anthropic_api_key(): - token = await db.config_get("ANTHROPIC_API_KEY") + from lab.config import Config as fs_config + token = fs_config.get_value_by_key("ANTHROPIC_API_KEY") if not token or token == "": return {"message": "ANTHROPIC_API_KEY not configured in database"} @@ -400,7 +403,8 @@ async def set_anthropic_api_key(): @router.get(path="/model/set_custom_api_key") async def set_custom_api_key(): - token_str = await db.config_get("CUSTOM_MODEL_API_KEY") + from lab.config import Config as fs_config + token_str = fs_config.get_value_by_key("CUSTOM_MODEL_API_KEY") if not token_str or token_str == "": return {"message": "CUSTOM_MODEL_API_KEY not configured in database"} diff --git a/transformerlab/routers/tools.py b/transformerlab/routers/tools.py index abb6aaf69..78628f6a2 100644 --- a/transformerlab/routers/tools.py +++ b/transformerlab/routers/tools.py @@ -7,7 +7,6 @@ from fastapi.responses import JSONResponse from pydantic import BaseModel -import transformerlab.db.db as db # MCP client imports try: @@ -64,7 +63,8 @@ async def get_all_tools(): # Get MCP server config directly from database mcp_config = None try: - config_text = await db.config_get(key="MCP_SERVER") + from lab.config import Config as fs_config + config_text = fs_config.get_value_by_key("MCP_SERVER") if config_text: mcp_config = json.loads(config_text) except Exception: diff --git a/transformerlab/shared/models/models.py b/transformerlab/shared/models/models.py index 108d83179..a1d2ba312 100644 --- a/transformerlab/shared/models/models.py +++ b/transformerlab/shared/models/models.py @@ -7,16 +7,6 @@ class Base(DeclarativeBase): pass -class Config(Base): - """Configuration key-value store model.""" - - __tablename__ = "config" - - id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True) - key: Mapped[str] = mapped_column(String, unique=True, index=True, nullable=False) - value: Mapped[Optional[str]] = mapped_column(String, nullable=True) - - # I believe we are not using the following table anymore as the filesystem # is being used to track plugins class Plugin(Base):