110325_work

worker_health/create_task-talos-2404.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+# set -x
+
+# check that count argument is provided
+if [ -z "$1" ]; then
+    echo "Usage: $0 <count> [additional arguments]"
+    exit 1
+fi
+
+COUNT="$1"
+shift
+
+./create_tc_task.py -q releng-hardware/gecko-t-linux-talos-2404 -c "$COUNT" "$@"

worker_health/lw_2404.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+set -x
+
+pipenv run ./list_workers.py releng-hardware gecko-t-linux-talos-2404 "$@"

worker_health/quar_lt_a55_alpha.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+PROV="proj-autophone"
+WTYPE="gecko-t-lambda-alpha-a55"
+
+pipenv run ./quarantine_tool.py "$PROV" "$WTYPE" "$@"

worker_health/quar_talos_2404.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+PROV="releng-hardware"
+WTYPE="gecko-t-linux-talos-2404"
+
+pipenv run ./quarantine_tool.py "$PROV" "$WTYPE" "$@"

worker_health/wait_for_tc_idle.py

@@ -9,6 +9,7 @@
 #     t-linux64-ms-239,t-linux64-ms-240 -v
 
 import argparse
+import logging
 import time
 from worker_health import status
 
@@ -28,28 +29,42 @@ def main():
     parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose output")
     args = parser.parse_args()
 
+    # Configure logging
+    logging.basicConfig(
+        level=logging.CRITICAL,
+        format="%(asctime)s - %(levelname)s - %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    logger = logging.getLogger(__name__)
+    log_level = logging.DEBUG if args.verbose else logging.INFO
+    logger.setLevel(log_level)
+
+    SLEEP_INTERVAL = 10  # seconds
+
     si = status.Status(args.provisioner, args.worker_type)
     hosts_with_non_completed_or_failed_jobs = si.get_hosts_running_jobs(args.hosts)
 
+    logger.info(f"Waiting for hosts to be idle: {args.hosts}...")
     while True:
         # pprint.pprint(hosts_with_non_completed_or_failed_jobs)
-        time.sleep(10)
         if args.single:
             # check if any hosts are idle
             input_set = set(args.hosts)
             result_set = set(hosts_with_non_completed_or_failed_jobs)
             difference = input_set - result_set
             if difference:
-                if args.verbose:
-                    print(f"Hosts no longer busy: {difference}")
+                logger.debug(f"Hosts no longer busy: {difference}")
                 break
             pass
         else:
             # check if all hosts are idle
             if not hosts_with_non_completed_or_failed_jobs:
-                if args.verbose:
-                    print(f"All hosts are idle: {args.hosts}")
+                logger.debug(f"All hosts are idle: {args.hosts}")
                 break
+        logger.debug(
+            f"Hosts with non-completed or failed jobs: {hosts_with_non_completed_or_failed_jobs}. Sleeping {SLEEP_INTERVAL} seconds before rechecking...",
+        )
+        time.sleep(SLEEP_INTERVAL)
 
 
 if __name__ == "__main__":

worker_health/wfi_moon.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+PROV="releng-hardware"
+WTYPE="gecko-t-linux-talos-1804"
+
+pipenv run ./wait_for_tc_idle.py -p $PROV -w $WTYPE "$@"

worker_health/worker_health/status.py

@@ -177,4 +177,4 @@ def list_workers_py(self):
     # TODO: pull out into binary `list_workers`
     # si.list_workers_human()
     # si.list_workers_csv()
-    # si.list_workers_py()
+    si.list_workers_py()