diff --git a/Gemfile.lock b/Gemfile.lock index 5e5259f..cc2cee3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -139,6 +139,8 @@ GEM nio4r (2.7.4) nokogiri (1.18.8-arm64-darwin) racc (~> 1.4) + nokogiri (1.18.8-x86_64-linux-gnu) + racc (~> 1.4) parallel (1.27.0) parser (3.3.8.0) ast (~> 2.4.1) @@ -238,6 +240,7 @@ GEM ruby-progressbar (1.13.0) securerandom (0.4.1) sqlite3 (2.7.2-arm64-darwin) + sqlite3 (2.7.2-x86_64-linux-gnu) stimulus-rails (1.3.4) railties (>= 6.0.0) stringio (3.1.7) @@ -261,6 +264,7 @@ GEM PLATFORMS arm64-darwin + x86_64-linux DEPENDENCIES bootsnap diff --git a/app/jobs/data_drip/dripper.rb b/app/jobs/data_drip/dripper.rb index e79b598..609ceee 100644 --- a/app/jobs/data_drip/dripper.rb +++ b/app/jobs/data_drip/dripper.rb @@ -1,6 +1,6 @@ module DataDrip class Dripper < ActiveJob::Base - queue_as :data_drip + queue_as :within_24_hours def perform(backfill_run) backfill_run.running! @@ -13,27 +13,29 @@ def perform(backfill_run) ) scope = new_backfill.scope - scope = - scope.limit( - backfill_run.amount_of_elements - ) if backfill_run.amount_of_elements.present? && - backfill_run.amount_of_elements > 0 + if backfill_run.amount_of_elements.present? && + backfill_run.amount_of_elements > 0 + scope = + scope.limit( + backfill_run.amount_of_elements + ) + end batch_ids = scope - .find_in_batches(batch_size: backfill_run.batch_size) - .map do |batch| - { - finish_id: batch.last.id, - start_id: batch.first.id, - actual_size: batch.size - } - end + .find_in_batches(batch_size: backfill_run.batch_size) + .map do |batch| + { + finish_id: batch.last.id, + start_id: batch.first.id, + actual_size: batch.size + } + end backfill_run.update(total_count: scope.count) if backfill_run.amount_of_elements.present? && - backfill_run.amount_of_elements < backfill_run.batch_size + backfill_run.amount_of_elements < backfill_run.batch_size backfill_run.batch_size = backfill_run.amount_of_elements backfill_run.save end diff --git a/app/jobs/data_drip/dripper_child.rb b/app/jobs/data_drip/dripper_child.rb index 57a2a29..c919ec5 100644 --- a/app/jobs/data_drip/dripper_child.rb +++ b/app/jobs/data_drip/dripper_child.rb @@ -1,6 +1,6 @@ module DataDrip class DripperChild < ActiveJob::Base - queue_as :data_drip_child + queue_as :within_24_hours def perform(backfill_run_batch) parent = backfill_run_batch.backfill_run