[4.0.1] Pipeline architecture, multi-series isolation & search CLI

.gitattributes

@@ -0,0 +1,6 @@
+* text=auto
+
+*.sh text eol=lf
+*.py text eol=lf
+Dockerfile text eol=lf
+*.dockerignore text eol=lf

.gitignore

@@ -42,3 +42,4 @@ cookies.txt
 test_episodes.json
 /models
 /tmp
+/preprocessor/scripts/scribe_compare

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 fail_fast: false
-exclude: '^(bot/RANCZO-WIDEO/|bot/RANCZO-TRANSKRYPCJE/)'
+exclude: '^(bot/RANCZO-WIDEO/|bot/RANCZO-TRANSKRYPCJE/|scripts/)'
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v6.0.0
@@ -37,6 +37,7 @@ repos:
       - id: chmod
         args: ["755"]
         files: (.*scripts\/.*.py$|\.sh$)
+        exclude: (^preprocessor/entrypoint\.sh$|^preprocessor/scripts/)
       - id: remove-tabs
         args: [--whitespaces-count, '4']
   - repo: https://github.com/PyCQA/isort

VERSION

@@ -0,0 +1 @@
+4.0.1

bot/services/reindex/reindex_service.py

@@ -22,7 +22,7 @@
 from bot.services.reindex.video_path_transformer import VideoPathTransformer
 from bot.services.reindex.zip_extractor import ZipExtractor
 from bot.settings import settings
-from preprocessor.search.elastic_manager import ElasticSearchManager
+from preprocessor.search.elastic_manager import ElasticSearchManager  # pylint: disable=no-name-in-module
 
 
 @dataclass

preprocessor/Dockerfile

@@ -38,12 +38,18 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     pip install --no-cache-dir --upgrade pip setuptools wheel \
     && pip install --no-cache-dir \
         -r /app/requirements.txt \
-        vllm==0.13.0 \
         --extra-index-url https://pypi.nvidia.com \
+    && pip install --no-cache-dir --pre vllm \
+        --extra-index-url https://wheels.vllm.ai/nightly \
+    && pip install --no-cache-dir \
+        git+https://github.com/huggingface/transformers.git@main \
+    && pip uninstall -y flashinfer \
     && pip uninstall -y onnxruntime \
     && pip install --no-cache-dir \
         onnxruntime-gpu==1.21.0 \
         --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/ \
+    && pip uninstall -y nvidia-cudnn-cu11 || true \
+    && pip install --no-cache-dir --force-reinstall --no-deps nvidia-cudnn-cu12 \
     && pip uninstall -y nvidia-nccl-cu11 || true \
     && pip install --no-cache-dir --force-reinstall --no-deps nvidia-nccl-cu12
 
@@ -58,10 +64,7 @@ RUN mkdir -p \
     /models/whisper \
     /models/insightface \
     /models/ultralytics \
-    /models/emotion_model \
-    /app/output_data/characters \
-    /app/output_data/scraped_pages \
-    /app/output_data/processing_metadata
+    /models/emotion_model
 
 COPY bot /app/bot
 COPY preprocessor /app/preprocessor