fixing k8s wrapper to work, and dequeue.py to choose which mode of execution it wants docker/kubernetes and use corresponding wrapper

Author: ranita-pal

docker-app/qfieldcloud/core/management/commands/dequeue.py

@@ -1,3 +1,4 @@
+import os
 import logging
 import signal
 from time import sleep
@@ -9,15 +10,30 @@
 from django.db import connection, transaction
 from django.db.models import Q
 from qfieldcloud.core.models import Job
-from worker_wrapper.wrapper import (
-    ApplyDeltaJobRun,
-    CreateProjectJobRun,
-    JobRun,
-    PackageJobRun,
-    ProcessProjectfileJobRun,
-    cancel_orphaned_workers,
+
+execution_mode = os.environ.get(
+    "QFC_WORKER_EXECUTION_MODE",
+    "docker",
 )
 
+if execution_mode == "kubernetes":
+    from worker_wrapper.wrapper_k8s import (
+        ApplyDeltaJobRun,
+        CreateProjectJobRun,
+        JobRun,
+        PackageJobRun,
+        ProcessProjectfileJobRun,
+        cancel_orphaned_workers,
+    )
+else:
+    from worker_wrapper.wrapper import (
+        ApplyDeltaJobRun,
+        CreateProjectJobRun,
+        JobRun,
+        PackageJobRun,
+        ProcessProjectfileJobRun,
+        cancel_orphaned_workers,
+    )
 
 class GracefulKiller:
     alive = True

docker-app/requirements/requirements_worker_wrapper.txt

@@ -13,16 +13,8 @@ requests==2.34.2
 tenacity==9.1.4
     # via -r requirements/requirements_worker_wrapper.in
 urllib3==2.7.0
-    # via
-    #   kubernetes
-    #   requests
-charset-normalizer==3.4.7
-    # via requests
-docker==7.1.0
     # via -r /requirements/requirements_worker_wrapper.in
 durationpy==0.10
-    # via kubernetes
-idna==3.15
     # via requests
 kubernetes==35.0.0
     # via -r /requirements/requirements_worker_wrapper.in
@@ -31,8 +23,6 @@ oauthlib==3.3.1
 python-dateutil==2.9.0.post0
     # via kubernetes
 pyyaml==6.0.3
-    # via kubernetes
-requests==2.32.4
     # via
     #   docker
     #   kubernetes
@@ -44,8 +34,6 @@ six==1.17.0
     #   kubernetes
     #   python-dateutil
 tenacity==9.1.4
-    # via -r /requirements/requirements_worker_wrapper.in
-urllib3==2.2.3
     # via
     #   docker
     #   kubernetes

docker-app/worker_wrapper/wrapper_k8s.py

@@ -90,7 +90,7 @@ class JobRun:
 
     def __init__(self, job_id: str) -> None:
         self.job = None
-        self.container_timeout_secs = config.WORKER_TIMEOUT_S
+        self.container_timeout_secs = settings.QFIELDCLOUD_WORKER_TIMEOUT_S
 
         try:
             self.job_id = job_id
@@ -505,7 +505,10 @@ def _run_kubernetes_job(
         )
 
         logger.info(
-            f"Kubernetes Job {job_name} finished: {status}"
+            "Kubernetes Job %s finished: succeeded=%s failed=%s",
+            job_name,
+            status.succeeded,
+            status.failed,
         )
 
         if not success:
@@ -851,6 +854,8 @@ def after_worker_run(self) -> None:
             "outputs"
         ]["project_details"]["project_details"]
 
+        update_fields = ["project_details"]
+
         # Since the `Project.qgis_version` field is newly added, we want to backfill it for old projects that didn't have it set,
         # but the `process_projectfile` job can detect the QGIS version from the project file and return it in the feedback, we can set it here.
         # NOTE `Project.qgis_version` is used by `wrapper.JobRun.get_qgis_image()` to detect
@@ -866,10 +871,12 @@ def after_worker_run(self) -> None:
         project.save(update_fields=update_fields)
 
         QgisProject.objects.update_from_details(
-            project, project.the_qgis_file.latest_version, project.project_details
+            project,
+            project.the_qgis_file.latest_version,
+            project.project_details,
         )
 
-    def after_docker_exception(self) -> None:
+    def after_worker_exception(self) -> None:
         project = self.job.project
 
         if hasattr(project, "qgis_project"):
@@ -889,4 +896,6 @@ class CreateProjectJobRun(JobRun):
     ]
 
 def cancel_orphaned_workers() -> None:
-    logger.info("cancel_orphaned_workers disabled for Kubernetes backend")
+    logger.info(
+        "cancel_orphaned_workers disabled for Kubernetes backend"
+    )