feat(seeder): Add configurable variables for data seed

packages/dsw-config/dsw/config/keys.py

@@ -51,6 +51,10 @@ def cast_optional_dict(value: typing.Any) -> dict | None:
     return value
 
 
+def cast_dict(value: typing.Any) -> dict:
+    return cast_optional_dict(value) or {}
+
+
 class ConfigKey[T]:
 
     def __init__(self, *, yaml_path: list[str],

packages/dsw-data-seeder/config.example.yml

@@ -28,3 +28,8 @@ extraS3s:
     username: minio
     password: minioPassword
     bucket: engine-wizard
+
+seed:
+  jobTimeout: 300000
+  variables:
+    someVariable: someValue

packages/dsw-data-seeder/dsw/data_seeder/config.py

@@ -5,6 +5,7 @@
     ConfigKey,
     ConfigKeys,
     ConfigKeysContainer,
+    cast_dict,
     cast_int,
     cast_optional_int,
     cast_str,
@@ -20,22 +21,29 @@
 )
 
 
-class _ExperimentalKeys(ConfigKeysContainer):
+class _SeedKeys(ConfigKeysContainer):
     job_timeout = ConfigKey(
-        yaml_path=['experimental', 'jobTimeout'],
-        var_names=['EXPERIMENTAL_JOB_TIMEOUT'],
+        yaml_path=['seed', 'jobTimeout'],
+        var_names=['SEED_JOB_TIMEOUT'],
         default=None,
         cast=cast_optional_int,
     )
+    variables = ConfigKey(
+        yaml_path=['seed', 'variables'],
+        var_names=[],
+        default=None,
+        cast=cast_dict,
+    )
 
 
 class DataSeederConfigKeys(ConfigKeys):
-    experimental = _ExperimentalKeys
+    seed = _SeedKeys
 
 
 @dataclasses.dataclass
-class ExperimentalConfig(ConfigModel):
+class SeedConfig(ConfigModel):
     job_timeout: int | None
+    variables: dict
 
 
 @dataclasses.dataclass
@@ -48,7 +56,7 @@ class SeederConfig:
     general: GeneralConfig
     extra_dbs: dict[str, DatabaseConfig]
     extra_s3s: dict[str, S3Config]
-    experimental: ExperimentalConfig
+    seed: SeedConfig
 
     def __str__(self):
         return f'SeederConfig\n' \
@@ -59,7 +67,7 @@ def __str__(self):
                f'{self.log}' \
                f'{self.sentry}' \
                f'{self.cloud}' \
-               f'{self.experimental}' \
+               f'{self.seed}' \
                f'====================\n'
 
 
@@ -140,9 +148,10 @@ def extra_s3s(self) -> dict[str, S3Config]:
         return result
 
     @property
-    def experimental(self) -> ExperimentalConfig:
-        return ExperimentalConfig(
-            job_timeout=self.get(self.keys.experimental.job_timeout),
+    def seed(self) -> SeedConfig:
+        return SeedConfig(
+            job_timeout=self.get(self.keys.seed.job_timeout),
+            variables=self.get(self.keys.seed.variables),
         )
 
     @property
@@ -156,5 +165,5 @@ def config(self) -> SeederConfig:
             general=self.general,
             extra_dbs=self.extra_dbs,
             extra_s3s=self.extra_s3s,
-            experimental=self.experimental,
+            seed=self.seed,
         )

packages/dsw-data-seeder/dsw/data_seeder/seeder.py

@@ -1,4 +1,5 @@
 import collections
+import contextlib
 import dataclasses
 import json
 import logging
@@ -19,7 +20,7 @@
 from . import consts
 from .build_info import BUILD_INFO
 from .config import SeederConfig
-from .context import Context
+from .context import Context, ContextNotInitializedError
 
 
 LOG = logging.getLogger(__name__)
@@ -150,27 +151,38 @@ def load_s3_object_names(self):
 
 
 class SeedRecipe:
+    UUID_PLACEHOLDER = '{{-UUID[n]-}}'
+    VAR_PLACEHOLDER = '{{-VAR[name]-}}'
 
     def __init__(self, *, name: str, description: str, root: pathlib.Path,
                  db: SeedRecipeDB, s3: SeedRecipeS3,
-                 uuids_count: int, uuids_placeholder: str | None,
-                 init_wait: float):
+                 uuids_count: int, uuids_placeholder: str, uuids_names: list[str],
+                 vars_placeholder: str, variables: dict, init_wait: float):
         self.name = name
         self.description = description
         self.root = root
         self.db = db
         self.s3 = s3
         self.prepared = False
         self.uuids_count = uuids_count
+        self.uuids_names = uuids_names
         self.uuids_placeholder = uuids_placeholder
+        self.vars_placeholder = vars_placeholder
         self.uuids_replacement: dict[str, str] = {}
+        self.vars_replacement: dict[str, str] = {
+            self.vars_placeholder.replace('[name]', f'[{key}]'): str(value)
+            for key, value in variables.items()
+        }
         self.init_wait = init_wait
 
     def _prepare_uuids(self):
         if self.uuids_placeholder is not None:
             for i in range(self.uuids_count):
                 key = self.uuids_placeholder.replace('[n]', f'[{i}]')
                 self.uuids_replacement[key] = str(uuid.uuid4())
+            for name in self.uuids_names:
+                key = self.uuids_placeholder.replace('[n]', f'[{name}]')
+                self.uuids_replacement[key] = str(uuid.uuid4())
 
     def _prepare_s3_objects(self):
         for s3_object in self.s3.objects:
@@ -189,6 +201,8 @@ def _replace_db_script(self, script: str, tenant_uuid: str) -> str:
         result = script.replace(self.db.tenant_placeholder, tenant_uuid)
         for uuid_key, uuid_value in self.uuids_replacement.items():
             result = result.replace(uuid_key, uuid_value)
+        for var_key, var_value in self.vars_replacement.items():
+            result = result.replace(var_key, var_value)
         return result
 
     def iterate_db_scripts(self, tenant_uuid: str):
@@ -217,22 +231,29 @@ def __str__(self):
                f'S3 Filename Replace:\n' \
                f'{replaces}'
 
-    @staticmethod
-    def load_from_json(recipe_file: pathlib.Path) -> 'SeedRecipe':
+    @classmethod
+    def load_from_json(cls, recipe_file: pathlib.Path) -> 'SeedRecipe':
         data = json.loads(recipe_file.read_text(
             encoding=consts.DEFAULT_ENCODING,
         ))
         db: dict[str, typing.Any] = data.get('db', {})
         s3: dict[str, typing.Any] = data.get('s3', {})
         root_dir = recipe_file.parent
+        variables = {}
+        with contextlib.suppress(ContextNotInitializedError):
+            variables.update(Context.get().app.cfg.seed.variables.copy())
+        variables.update(data.get('variables', {}).get('values', {}))
         return SeedRecipe(
             name=data['name'],
             description=data.get('description', ''),
             root=root_dir,
             db=SeedRecipeDB.from_dict(db, root_dir),
             s3=SeedRecipeS3.from_dict(s3, root_dir),
             uuids_count=data.get('uuids', {}).get('count', 0),
-            uuids_placeholder=data.get('uuids', {}).get('placeholder', None),
+            uuids_names=data.get('uuids', {}).get('names', []),
+            uuids_placeholder=data.get('uuids', {}).get('placeholder', cls.UUID_PLACEHOLDER),
+            vars_placeholder=data.get('variables', {}).get('placeholder', cls.VAR_PLACEHOLDER),
+            variables=variables,
             init_wait=data.get('initWait', 0),
         )
 
@@ -242,8 +263,8 @@ def load_from_dir(recipes_dir: pathlib.Path) -> dict[str, 'SeedRecipe']:
         recipes = (SeedRecipe.load_from_json(f) for f in recipe_files)
         return {r.name: r for r in recipes}
 
-    @staticmethod
-    def create_default():
+    @classmethod
+    def create_default(cls):
         return SeedRecipe(
             name='default',
             description='Default dummy recipe',
@@ -256,8 +277,11 @@ def create_default():
                 copy={},
                 filename_replace={},
             ),
+            uuids_names=[],
             uuids_count=0,
-            uuids_placeholder=None,
+            uuids_placeholder=cls.UUID_PLACEHOLDER,
+            vars_placeholder=cls.VAR_PLACEHOLDER,
+            variables={},
             init_wait=20,
         )
 
@@ -330,7 +354,7 @@ def _run_preparation(self) -> CommandQueue:
             channel=consts.CMD_CHANNEL,
             component=consts.CMD_COMPONENT,
             wait_timeout=Context.get().app.cfg.db.queue_timeout,
-            work_timeout=Context.get().app.cfg.experimental.job_timeout,
+            work_timeout=Context.get().app.cfg.seed.job_timeout,
         )
 
     def run(self):

packages/dsw-data-seeder/example/example.seed.json

@@ -33,7 +33,18 @@
   },
   "uuids": {
     "count": 3,
-    "placeholder": "{{-UUID[n]-}}"
+    "placeholder": "{{-UUID[n]-}}",
+    "names": [
+      "tenantId",
+      "userId",
+      "configId"
+    ]
+  },
+  "variables": {
+    "placeholder": "{{-VAR[name]-}}",
+    "values": {
+      "name": "value"
+    }
   },
   "initWait": 1.0
-}
+}