fix: backup failure overlap issue (#1514)

Author: NishaSharma14

app/Jobs/DataBackupJob.php

@@ -17,6 +17,19 @@ class DataBackupJob implements ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
+    /**
+     * Number of times the job may be attempted.
+     */
+    public int $tries = 3;
+
+    /**
+     * Delay (in seconds) between retry attempts, to ride out transient
+     * pg_dump failures caused by concurrent schema changes (e.g. deployments).
+     *
+     * @var array<int, int>
+     */
+    public array $backoff = [60, 300];
+
     /**
      * Execute the job.
      */

config/database.php

@@ -19,7 +19,8 @@
             'dump' => [
                 'use_single_transaction',
                 'timeout' => 620 * 100, // 51 minute timeout
-                'exclude_tables' => ['versions'],
+                // mols/fps are ephemeral RDKit search caches dropped & rebuilt daily by nmrxiv:index-molecules, excluded to avoid pg_dump racing that DDL
+                'exclude_tables' => ['versions', 'mols', 'fps'],
             ],
             'search_path' => 'public',
         ],

routes/console.php

@@ -20,13 +20,14 @@
 
 Schedule::command('nmrxiv:publish-embargo-projects')->daily();
 Schedule::command('nmrxiv:delete-projects')->daily();
-Schedule::command('nmrxiv:index-molecules')->daily();
+// Staggered away from the backup dump (both defaulted to ->daily(), i.e. midnight) since this rebuilds the mols/fps RDKit tables via DROP/CREATE DDL
+Schedule::command('nmrxiv:index-molecules')->dailyAt('02:00');
 Schedule::command('nmrxiv:index-spectra-metadata-stats')->daily();
 Schedule::command('nmrxiv:delete-citations')->weekly();
 Schedule::command('nmrxiv:delete-authors')->weekly();
 if (App::environment('production')) {
     Schedule::command('nmrxiv:backup-postgres-dump')
-        ->daily()
+        ->dailyAt('04:00')
         ->onOneServer()
         ->withoutOverlapping();
 }