Make background scripts more reliable

- Move starting & stopping to `Control`, and call those methods
  - Keeping autorun separate to avoid spamming the log files
- Check whether a script is running before attempting to start it
- Check actual process IDs, rather than hoping the database is accurate
- Kill all instances of a script, rather than just the one in the DB
- More logging to make debugging easier

Author: JasonTarka

src/models/Control.php

@@ -52,7 +52,37 @@ class Control extends Model {
     if ($result->numRows() > 0) {
       $pid = intval(must_have_idx($result->mapRows()[0], 'pid'));
     }
-    return $pid;
+
+    // Check that the process is actually running
+    $actual_pid = await self::genRunningScriptPid($name);
+    if ($pid == $actual_pid) {
+      return $pid;
+    }
+    if ($actual_pid == 0 && $pid != 0) {
+      Utils::logMessage("Warning: Script ($name) started as PID $pid but is no longer running");
+      await self::genStopScriptLog($pid);
+    } else if ($actual_pid != 0 && $pid == 0) {
+      Utils::logMessage("Warning: Script ($name) was not started, but is running as PID $actual_pid");
+    } else if ($actual_pid != $pid) {
+      Utils::logMessage("Warning: Script ($name) started as PID $pid but is actually running as $actual_pid");
+      await self::genStopScriptLog($pid);
+    }
+
+    return $actual_pid;
+  }
+
+  public static async function genRunningScriptPid(string $name): Awaitable<int> {
+    $lines = array();
+    exec( "ps -C hhvm --format 'pid args' | grep '$name.php'", $lines );
+    if (count($lines) === 0) {
+      return 0;
+    } else if (count($lines) > 1) {
+      Utils::logMessage("Error: multiple instances of ($name.php) running!");
+    }
+
+    $matches = array();
+    $results = preg_match('/\s*([0-9]+)\s/', $lines[0], $matches);
+    return $results > 0 ? intval($matches[1]) : 0;
   }
 
   public static async function genClearScriptLog(): Awaitable<void> {
@@ -166,6 +196,7 @@ class Control extends Model {
   }
 
   public static async function genPause(): Awaitable<void> {
+    Utils::logMessage('Pausing game');
     await \HH\Asio\va(
       Announcement::genCreateAuto('Game has been paused!'), // Announce game paused
       ActivityLog::genCreateGenericLog('Game has been paused!'), // Log game paused
@@ -183,6 +214,7 @@ class Control extends Model {
   }
 
   public static async function genUnpause(): Awaitable<void> {
+    Utils::logMessage('Unpausing game');
     await Configuration::genUpdate('scoring', '1'); // Enable scoring
     list($config_pause_ts, $config_start_ts, $config_end_ts) =
       await \HH\Asio\va(
@@ -279,30 +311,43 @@ class Control extends Model {
     }
   }
 
+  public static async function genRunScript($name): Awaitable<void> {
+    Utils::logMessage("Starting background script: $name");
+    $running = await self::checkScriptRunning($name);
+    if ($running) {
+      Utils::logMessage("Warning: $name is already running");
+    }
+
+    $document_root = Utils::get_src_root();
+    $cmd =
+      "hhvm -vRepo.Central.Path=/var/run/hhvm/.hhvm.hhbc_$name ".
+      $document_root.
+      "/scripts/$name.php >> /var/log/fbctf/$name.log 2>&1 & echo $!";
+    Utils::logMessage("Using command: [$cmd]");
+    $pid = shell_exec($cmd);
+    await Control::genStartScriptLog(intval($pid), $name, $cmd);
+  }
+
+  public static async function genStopScript($name): Awaitable<void> {
+    Utils::logMessage("Stopping background script: $name");
+
+    do {
+      // Kill running process
+      $pid = await Control::genScriptPid($name);
+      if ($pid > 0) {
+        Utils::logMessage("Killing $name with PID $pid");
+        exec('kill -9 '.escapeshellarg(strval($pid)));
+      }
+      // Mark process as stopped
+      await Control::genStopScriptLog($pid);
+    } while ($pid > 0); // In case there are multiple instances
+  }
+
   public static async function checkScriptRunning(
     string $name,
   ): Awaitable<bool> {
-    $db = await self::genDb();
-    $host = await Control::genServerAddr();
-    $result = await $db->queryf(
-      'SELECT pid FROM scripts WHERE name = %s AND host = %s AND status = 1',
-      $name,
-      $host,
-    );
-    $status = false;
-    if ($result->numRows() >= 1) {
-      foreach ($result->mapRows() as $row) {
-        $pid = intval(must_have_idx($row, 'pid'));
-        $status = file_exists("/proc/$pid");
-        if ($status === false) {
-          await Control::genStopScriptLog($pid);
-          await Control::genClearScriptLog();
-        }
-      }
-      return $status;
-    } else {
-      return false;
-    }
+    $pid = await self::genScriptPid($name);
+    return $pid !== 0;
   }
 
   public static async function importGame(): Awaitable<bool> {

src/models/Level.php

@@ -1283,26 +1283,12 @@ public static function getBasesResponses(
 
   // Bases processing and scoring.
   public static async function genBaseScoring(): Awaitable<void> {
-    Utils::logMessage('Starting background script: bases');
-    $document_root = Utils::get_src_root();
-    $cmd =
-      'hhvm -vRepo.Central.Path=/var/run/hhvm/.hhvm.hhbc_bases '.
-      $document_root.
-      '/scripts/bases.php >> /var/log/fbctf/bases.log 2>&1 & echo $!';
-    Utils::logMessage("Using command: [$cmd]");
-    $pid = shell_exec($cmd);
-    await Control::genStartScriptLog(intval($pid), 'bases', $cmd);
+    await Control::genRunScript('bases');
   }
 
   // Stop bases processing and scoring process.
   public static async function genStopBaseScoring(): Awaitable<void> {
-    // Kill running process
-    $pid = await Control::genScriptPid('bases');
-    if ($pid > 0) {
-      exec('kill -9 '.escapeshellarg(strval($pid)));
-    }
-    // Mark process as stopped
-    await Control::genStopScriptLog($pid);
+    await Control::genStopScript('bases');
   }
 
   // Check if a level already exists by type, title and entity.

src/models/Progressive.php

@@ -146,25 +146,11 @@ private static function progressiveFromRow(
 
   // Kick off the progressive scoreboard in the background.
   public static async function genRun(): Awaitable<void> {
-    Utils::logMessage('Starting background script: progressive scoreboard');
-    $document_root = Utils::get_src_root();
-    $cmd =
-      'hhvm -vRepo.Central.Path=/var/run/hhvm/.hhvm.hhbc_progressive '.
-      $document_root.
-      "/scripts/progressive.php >> /var/log/fbctf/progressive.log 2>&1 & echo $!";
-    Utils::logMessage("Using command: [$cmd]");
-    $pid = shell_exec($cmd);
-    await Control::genStartScriptLog(intval($pid), 'progressive', $cmd);
+    await Control::genRunScript('progressive');
   }
 
   // Stop the progressive scoreboard process in the background
   public static async function genStop(): Awaitable<void> {
-    // Kill running process
-    $pid = await Control::genScriptPid('progressive');
-    if ($pid > 0) {
-      exec('kill -9 '.escapeshellarg(strval($pid)));
-    }
-    // Mark process as stopped
-    await Control::genStopScriptLog($pid);
+    await Control::genStopScript('progressive');
   }
 }