diff --git a/worker_health/create_task-talos-2404.sh b/worker_health/create_task-talos-2404.sh new file mode 100755 index 00000000..e0eac886 --- /dev/null +++ b/worker_health/create_task-talos-2404.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -e +# set -x + +# check that count argument is provided +if [ -z "$1" ]; then + echo "Usage: $0 [additional arguments]" + exit 1 +fi + +COUNT="$1" +shift + +./create_tc_task.py -q releng-hardware/gecko-t-linux-talos-2404 -c "$COUNT" "$@" diff --git a/worker_health/fitness-awsmetal-all.sh b/worker_health/fitness-awsmetal-all.sh deleted file mode 100755 index 415cd7cb..00000000 --- a/worker_health/fitness-awsmetal-all.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e -# set -x - -./fitness_check.py -p gecko-t -hh -s t-linux-metal "$@" diff --git a/worker_health/fitness-awsmetal.sh b/worker_health/fitness-awsmetal.sh deleted file mode 100755 index eab70c61..00000000 --- a/worker_health/fitness-awsmetal.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e -# set -x - -./fitness_check.py -p gecko-t -hh -s -o t-linux-metal "$@" diff --git a/worker_health/get_quarantined_android_p2_unit.sh b/worker_health/get_quarantined_android_p2_unit.sh deleted file mode 100755 index 79de78e9..00000000 --- a/worker_health/get_quarantined_android_p2_unit.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -./quarantine_tool.py proj-autophone gecko-t-bitbar-gw-unit-p2 show "$@" diff --git a/worker_health/get_quarantined_talos-1804.sh b/worker_health/get_quarantined_talos-1804.sh deleted file mode 100755 index 69ef35b1..00000000 --- a/worker_health/get_quarantined_talos-1804.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -./quarantine_tool.py releng-hardware gecko-t-linux-talos-1804 show "$@" diff --git a/worker_health/get_quarantined_talos.sh b/worker_health/get_quarantined_talos.sh deleted file mode 100755 index cf00d332..00000000 --- a/worker_health/get_quarantined_talos.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -./quarantine_tool.py releng-hardware gecko-t-linux-talos show "$@" diff --git a/worker_health/lw_2404.sh b/worker_health/lw_2404.sh new file mode 100755 index 00000000..d5b5709c --- /dev/null +++ b/worker_health/lw_2404.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail +set -x + +pipenv run ./list_workers.py releng-hardware gecko-t-linux-talos-2404 "$@" diff --git a/worker_health/missing-workers-m2.sh b/worker_health/missing_workers-m2.sh similarity index 100% rename from worker_health/missing-workers-m2.sh rename to worker_health/missing_workers-m2.sh diff --git a/worker_health/missing-workers-r8.py b/worker_health/missing_workers-r8.py similarity index 100% rename from worker_health/missing-workers-r8.py rename to worker_health/missing_workers-r8.py diff --git a/worker_health/quar_lt_a55_alpha.sh b/worker_health/quar_lt_a55_alpha.sh new file mode 100755 index 00000000..5e136bbb --- /dev/null +++ b/worker_health/quar_lt_a55_alpha.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e +set -x + +PROV="proj-autophone" +WTYPE="gecko-t-lambda-alpha-a55" + +pipenv run ./quarantine_tool.py "$PROV" "$WTYPE" "$@" diff --git a/worker_health/quar_moon.sh b/worker_health/quar_talos_1804.sh similarity index 100% rename from worker_health/quar_moon.sh rename to worker_health/quar_talos_1804.sh diff --git a/worker_health/quar_talos_2404.sh b/worker_health/quar_talos_2404.sh new file mode 100755 index 00000000..0b0d9580 --- /dev/null +++ b/worker_health/quar_talos_2404.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e +set -x + +PROV="releng-hardware" +WTYPE="gecko-t-linux-talos-2404" + +pipenv run ./quarantine_tool.py "$PROV" "$WTYPE" "$@" diff --git a/worker_health/wait_for_tc_idle.py b/worker_health/wait_for_tc_idle.py index d23bc512..88fb084a 100755 --- a/worker_health/wait_for_tc_idle.py +++ b/worker_health/wait_for_tc_idle.py @@ -9,6 +9,7 @@ # t-linux64-ms-239,t-linux64-ms-240 -v import argparse +import logging import time from worker_health import status @@ -28,28 +29,42 @@ def main(): parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose output") args = parser.parse_args() + # Configure logging + logging.basicConfig( + level=logging.CRITICAL, + format="%(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + logger = logging.getLogger(__name__) + log_level = logging.DEBUG if args.verbose else logging.INFO + logger.setLevel(log_level) + + SLEEP_INTERVAL = 10 # seconds + si = status.Status(args.provisioner, args.worker_type) hosts_with_non_completed_or_failed_jobs = si.get_hosts_running_jobs(args.hosts) + logger.info(f"Waiting for hosts to be idle: {args.hosts}...") while True: # pprint.pprint(hosts_with_non_completed_or_failed_jobs) - time.sleep(10) if args.single: # check if any hosts are idle input_set = set(args.hosts) result_set = set(hosts_with_non_completed_or_failed_jobs) difference = input_set - result_set if difference: - if args.verbose: - print(f"Hosts no longer busy: {difference}") + logger.debug(f"Hosts no longer busy: {difference}") break pass else: # check if all hosts are idle if not hosts_with_non_completed_or_failed_jobs: - if args.verbose: - print(f"All hosts are idle: {args.hosts}") + logger.debug(f"All hosts are idle: {args.hosts}") break + logger.debug( + f"Hosts with non-completed or failed jobs: {hosts_with_non_completed_or_failed_jobs}. Sleeping {SLEEP_INTERVAL} seconds before rechecking...", + ) + time.sleep(SLEEP_INTERVAL) if __name__ == "__main__": diff --git a/worker_health/wfi_moon.sh b/worker_health/wfi_moon.sh new file mode 100755 index 00000000..abb04f01 --- /dev/null +++ b/worker_health/wfi_moon.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e +set -x + +PROV="releng-hardware" +WTYPE="gecko-t-linux-talos-1804" + +pipenv run ./wait_for_tc_idle.py -p $PROV -w $WTYPE "$@" diff --git a/worker_health/worker_health/status.py b/worker_health/worker_health/status.py index 755981a9..5b9e655e 100644 --- a/worker_health/worker_health/status.py +++ b/worker_health/worker_health/status.py @@ -177,4 +177,4 @@ def list_workers_py(self): # TODO: pull out into binary `list_workers` # si.list_workers_human() # si.list_workers_csv() - # si.list_workers_py() + si.list_workers_py()