Notification of stuck stages

.simplecov

@@ -14,6 +14,7 @@ SimpleCov.start do
   add_filter %r{^/(spec|config)/}
   add_filter 'database_loader.rb'
   add_filter 'workers/slack_username2_id.rb'
+  add_filter 'workers/github_notify_watch_dog.rb'
   add_group 'Models', 'lib/models'
   add_group 'GitHub Functions', 'lib/github'
   add_group 'Bamboo CI Functions', 'lib/bamboo_ci'

config/delayed_job.rb

@@ -10,6 +10,7 @@
 
 require_relative '../lib/helpers/github_logger'
 require_relative '../database_loader'
+require_relative '../workers/github_notify_watch_dog'
 
 require 'delayed_job'
 require 'active_support'

lib/github_ci_app.rb

@@ -43,6 +43,7 @@
 require_relative '../workers/timeout_execution'
 require_relative '../workers/ci_job_fetch_topotest_failures'
 require_relative '../workers/slack_username2_id'
+require_relative '../workers/github_notify_watch_dog'
 
 # Slack libs
 require_relative 'slack/slack'

lib/slack_bot/slack_bot.rb

@@ -136,6 +136,17 @@ def stage_in_progress_notification(stage)
     end
   end
 
+  def notify_watch_dog(slack_user_id, message)
+    url = "#{GitHubApp::Configuration.instance.config['slack_bot_url']}/github/user"
+
+    post_request(URI(url),
+                 machine: 'slack_bot.netdef.org',
+                 body: {
+                   message: message,
+                   slack_user_id: slack_user_id
+                 }.to_json)
+  end
+
   private
 
   def current_execution?(check_suite)

workers/github_notify_watch_dog.rb

@@ -0,0 +1,32 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  github_notify_watch_dog.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2024 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+class GithubNotifyWatchDog
+  class << self
+    def run
+      check_stages
+
+      GithubNotifyWatchDog
+        .delay(run_at: 1.hour.from_now.utc, queue: 'github_notify_watch_dog')
+        .run
+    end
+
+    def check_stages
+      Stage.where(status: :in_progress).where(updated_at: ..3.hour.ago).each do |stage|
+        GitHubApp::Configuration.instance.config['notify_users_when_stage_stuck']&.each do |slack_id|
+          SlackBot
+            .instance
+            .notify_watch_dog(slack_id,
+                              "Stage #{stage.name} is stuck in progress - #{stage.check_suite.bamboo_ci_ref}")
+        end
+      end
+    end
+  end
+end