Merge pull request #2103 from alphagov/reduce-batch-size

Reduce batch size
alphagov · Jun 5, 2024 · c1aa187 · c1aa187
2 parents 796d9bc + 5def533
commit c1aa187
Show file tree

Hide file tree

Showing 6 changed files with 6 additions and 6 deletions.
diff --git a/app/domain/etl/feedex/processor.rb b/app/domain/etl/feedex/processor.rb
@@ -18,7 +18,7 @@ def process
 
 private
 
-  BATCH_SIZE = 10_000
+  BATCH_SIZE = 5_000
 
   def extract_events
     batch = 1

diff --git a/app/domain/etl/ga/internal_search_processor.rb b/app/domain/etl/ga/internal_search_processor.rb
@@ -25,7 +25,7 @@ def extract_events
     batch = 1
     Etl::GA::InternalSearchService.find_in_batches(date:) do |events|
       log process: :ga, message: "Processing #{events.length} events in batch #{batch}"
-      Events::GA.import(events, batch_size: 10_000)
+      Events::GA.import(events, batch_size: 5_000)
       batch += 1
     end
   end

diff --git a/app/domain/etl/ga/user_feedback_processor.rb b/app/domain/etl/ga/user_feedback_processor.rb
@@ -25,7 +25,7 @@ def extract_events
     batch = 1
     Etl::GA::UserFeedbackService.find_in_batches(date:) do |events|
       log process: :ga, message: "Processing #{events.length} events in batch #{batch}"
-      Events::GA.import(events, batch_size: 10_000)
+      Events::GA.import(events, batch_size: 5_000)
       batch += 1
     end
   end

diff --git a/app/domain/etl/ga/views_and_navigation_processor.rb b/app/domain/etl/ga/views_and_navigation_processor.rb
@@ -25,7 +25,7 @@ def extract_events
     batch = 1
     Etl::GA::ViewsAndNavigationService.find_in_batches(date:) do |events|
       log process: :ga, message: "Processing #{events.length} events in batch #{batch}"
-      Events::GA.import(events, batch_size: 10_000)
+      Events::GA.import(events, batch_size: 5_000)
       batch += 1
     end
   end

diff --git a/app/domain/etl/main/metrics_processor.rb b/app/domain/etl/main/metrics_processor.rb
@@ -27,7 +27,7 @@ def create_metrics
         log process: :metrics, message: "about to get the Dimensions::Date"
         dimensions_date = Dimensions::Date.find_existing_or_create(date)
         log process: :metrics, message: "got the Dimensions::Date"
-        Dimensions::Edition.live.find_in_batches(batch_size: 10_000)
+        Dimensions::Edition.live.find_in_batches(batch_size: 5_000)
           .with_index do |batch, index|
           log process: :metrics, message: "processing #{batch.length} items in batch #{index}"
           values = batch.pluck(:id).map do |value|

diff --git a/spec/integration/master/daily_metrics_spec.rb b/spec/integration/master/daily_metrics_spec.rb
@@ -170,7 +170,7 @@ def stub_feedex_response
       'page_size': 3,
     }.to_json
 
-    stub_request(:get, "http://support-api.dev.gov.uk/feedback-by-day/#{yesterday}?page=1&per_page=10000")
+    stub_request(:get, "http://support-api.dev.gov.uk/feedback-by-day/#{yesterday}?page=1&per_page=5000")
       .to_return(status: 200, body: response, headers: {})
   end
 end