504 HARVEST TIMEOUTS (#106)

DigitalNZ · Apr 3, 2019 · 0bfa66e · 0bfa66e
1 parent 119eb3d
commit 0bfa66e
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 3 deletions.
diff --git a/app/workers/enrichment_worker.rb b/app/workers/enrichment_worker.rb
@@ -63,14 +63,18 @@ def fetch_records(page = 0)
     if job.record_id.nil?
       if job.harvest_job.present?
         SupplejackApi::Record.find({ 'fragments.job_id' => job.harvest_job.id.to_s }, page: page)
-
       else
         SupplejackApi::Record.find({ 'fragments.source_id' => job.parser.source.source_id }, page: page)
       end
     else
       klass = job.preview? ? SupplejackApi::PreviewRecord : SupplejackApi::Record
       klass.find({ record_id: job.record_id }, page: page)
     end
+  rescue ActiveResource::ServerError => e
+    # Temporary logging to help surface more information around DB timeouts
+    search_query = job.harvest_job.present? ? "fragments.job_id: #{job.harvest_job.id}" : "fragments.source_id: #{job.parser.source.source_id}"
+    Airbrake.notify(e, error_message: "Record Timeout: The record lookup failed with query #{search_query}.  #{e&.message}")
+    raise
   end
 
   # rubocop:disable Metrics/MethodLength

diff --git a/app/workers/source_check_worker.rb b/app/workers/source_check_worker.rb
@@ -20,7 +20,11 @@ def perform(id)
   private
 
   def source_records
-    JSON.parse(RestClient.get("#{ENV['API_HOST']}/harvester/sources/#{source.id}/link_check_records", params: { api_key: ENV['HARVESTER_API_KEY'] }))
+    api_url = "#{ENV['API_HOST']}/harvester/sources/#{source.id}/link_check_records"
+    JSON.parse(RestClient::Request.execute(method: :get, url: api_url, timeout: 90, headers: { params: { api_key: ENV['HARVESTER_API_KEY'] } }))
+  rescue RestClient::GatewayTimeout => e
+    # Temporary logging to help surface more information around API timeouts
+    Airbrake.notify(e, error_message: "Api Timeout: Request timed out on #{url}. #{e&.message}")
   end
 
   def source_active?

diff --git a/config/mongoid.yml b/config/mongoid.yml
@@ -72,6 +72,7 @@ production:
         max_pool_size: 500
         user: <%= ENV['MONGOID_USER'] %>
         password: <%= ENV['MONGOID_PASSWORD'] %>
+        timeout: 90
     api:
       database: dnz_api_production
       hosts:
@@ -81,3 +82,5 @@ production:
         read:
           mode: :primary_preferred
         max_pool_size: 500
+        timeout: 90
+
diff --git a/spec/workers/source_check_worker_spec.rb b/spec/workers/source_check_worker_spec.rb
@@ -79,7 +79,7 @@
     end
 
     it 'retrieves landing urls from the API to check' do
-      expect(RestClient).to receive(:get).with("#{ENV['API_HOST']}/harvester/sources/#{source.id}/link_check_records", params: { api_key: ENV['HARVESTER_API_KEY'] }).and_return response
+      expect(RestClient::Request).to receive(:execute).with(method: :get, url: "#{ENV['API_HOST']}/harvester/sources/#{source.id}/link_check_records", timeout: 90, headers: { params: { api_key: ENV['HARVESTER_API_KEY'] }}).and_return response
       worker.send(:source_records)
     end
   end