From 7b78f2313ec1279def6012736b462347941f6252 Mon Sep 17 00:00:00 2001
From: Rodrigo Nardi <rnardi@netdef.org>
Date: Thu, 21 Nov 2024 13:16:44 -0300
Subject: [PATCH 1/4] Notification of stuck stages

Added background script to notify when a stage is stuck

Signed-off-by: Rodrigo Nardi <rnardi@netdef.org>
---
 config/delayed_job.rb              |  4 ++++
 lib/github_ci_app.rb               |  1 +
 lib/slack_bot/slack_bot.rb         | 11 ++++++++++
 workers/github_notify_watch_dog.rb | 32 ++++++++++++++++++++++++++++++
 4 files changed, 48 insertions(+)
 create mode 100644 workers/github_notify_watch_dog.rb

diff --git a/config/delayed_job.rb b/config/delayed_job.rb
index 09f7aa4..2f3387d 100644
--- a/config/delayed_job.rb
+++ b/config/delayed_job.rb
@@ -40,3 +40,7 @@ class << self
 
 config = YAML.load_file('config/database.yml')[ENV.fetch('RACK_ENV', 'development')]
 ActiveRecord::Base.establish_connection(config)
+
+GithubNotifyWatchDog
+  .delay(run_at: 1.hour.from_now.utc, queue: 'github_notify_watch_dog')
+  .run
diff --git a/lib/github_ci_app.rb b/lib/github_ci_app.rb
index e6479d2..986452f 100644
--- a/lib/github_ci_app.rb
+++ b/lib/github_ci_app.rb
@@ -43,6 +43,7 @@
 require_relative '../workers/timeout_execution'
 require_relative '../workers/ci_job_fetch_topotest_failures'
 require_relative '../workers/slack_username2_id'
+require_relative '../workers/github_notify_watch_dog'
 
 # Slack libs
 require_relative 'slack/slack'
diff --git a/lib/slack_bot/slack_bot.rb b/lib/slack_bot/slack_bot.rb
index 48ad5fc..8bc8b1c 100644
--- a/lib/slack_bot/slack_bot.rb
+++ b/lib/slack_bot/slack_bot.rb
@@ -136,6 +136,17 @@ def stage_in_progress_notification(stage)
     end
   end
 
+  def notify_watch_dog(slack_user_id, message)
+    url = "#{GitHubApp::Configuration.instance.config['slack_bot_url']}/github/user"
+
+    post_request(URI(url),
+                 machine: 'slack_bot.netdef.org',
+                 body: {
+                   message: message,
+                   slack_user_id: slack_user_id
+                 }.to_json)
+  end
+
   private
 
   def current_execution?(check_suite)
diff --git a/workers/github_notify_watch_dog.rb b/workers/github_notify_watch_dog.rb
new file mode 100644
index 0000000..2314e86
--- /dev/null
+++ b/workers/github_notify_watch_dog.rb
@@ -0,0 +1,32 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  github_notify_watch_dog.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2024 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+class GithubNotifyWatchDog
+  class << self
+    def run
+      check_stages
+
+      GithubNotifyWatchDog
+        .delay(run_at: 1.hour.from_now.utc, queue: 'github_notify_watch_dog')
+        .run
+    end
+
+    def check_stages
+      Stage.where(status: :in_progress).where(updated_at: ..3.hour.ago).each do |stage|
+        GitHubApp::Configuration.instance.config['notify_users_when_stage_stuck']&.each do |slack_id|
+          SlackBot
+            .instance
+            .notify_watch_dog(slack_id,
+                              "Stage #{stage.name} is stuck in progress - #{stage.check_suite.bamboo_ci_ref}")
+        end
+      end
+    end
+  end
+end

From e262795dfd2e00cd2b6c9149288d7ff56c59c664 Mon Sep 17 00:00:00 2001
From: Rodrigo Nardi <rnardi@netdef.org>
Date: Tue, 3 Dec 2024 10:49:33 -0300
Subject: [PATCH 2/4] SimpleCov

Ignoring github_notify_watch_dog.rb

Signed-off-by: Rodrigo Nardi <rnardi@netdef.org>
---
 .simplecov | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.simplecov b/.simplecov
index b382a47..6de03bb 100644
--- a/.simplecov
+++ b/.simplecov
@@ -14,6 +14,7 @@ SimpleCov.start do
   add_filter %r{^/(spec|config)/}
   add_filter 'database_loader.rb'
   add_filter 'workers/slack_username2_id.rb'
+  add_filter 'workers/github_notify_watch_dog.rb'
   add_group 'Models', 'lib/models'
   add_group 'GitHub Functions', 'lib/github'
   add_group 'Bamboo CI Functions', 'lib/bamboo_ci'

From 6f1c9b8c0605fe475cffb742981ba41c55370980 Mon Sep 17 00:00:00 2001
From: Rodrigo Nardi <rnardi@netdef.org>
Date: Tue, 3 Dec 2024 10:57:58 -0300
Subject: [PATCH 3/4] DelayedJob

Loading GithubNotifyWatchDog

Signed-off-by: Rodrigo Nardi <rnardi@netdef.org>
---
 config/delayed_job.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config/delayed_job.rb b/config/delayed_job.rb
index 2f3387d..35d8ffa 100644
--- a/config/delayed_job.rb
+++ b/config/delayed_job.rb
@@ -10,6 +10,7 @@
 
 require_relative '../lib/helpers/github_logger'
 require_relative '../database_loader'
+require_relative '../workers/github_notify_watch_dog'
 
 require 'delayed_job'
 require 'active_support'

From 2099a3f7712717fe961b22bfad5b1908d20b3b66 Mon Sep 17 00:00:00 2001
From: Rodrigo Nardi <rnardi@netdef.org>
Date: Wed, 4 Dec 2024 11:16:12 -0300
Subject: [PATCH 4/4] DelayedJob

Removing GithubNotifyWatchDog

Signed-off-by: Rodrigo Nardi <rnardi@netdef.org>
---
 config/delayed_job.rb | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/config/delayed_job.rb b/config/delayed_job.rb
index 35d8ffa..9d26a0f 100644
--- a/config/delayed_job.rb
+++ b/config/delayed_job.rb
@@ -41,7 +41,3 @@ class << self
 
 config = YAML.load_file('config/database.yml')[ENV.fetch('RACK_ENV', 'development')]
 ActiveRecord::Base.establish_connection(config)
-
-GithubNotifyWatchDog
-  .delay(run_at: 1.hour.from_now.utc, queue: 'github_notify_watch_dog')
-  .run