Csv background job (#1281)

* adjust service to handle blob

* broadcast completion

* add loading spinnder

* i18n and clean up

* wording

* turbo frame not needed

* fix broadcast / minor tweaks

* change change to current user

get current org in service

* set org with actsastenant

* update service tests

* test index and create

* clean up

* assert errors instance variable

* proper order

* standard

app/controllers/organizations/staff/external_form_upload_controller.rb

@@ -11,8 +11,15 @@ def index
 
       def create
         authorize! :external_form_upload, context: {organization: Current.organization}
-        import = Organizations::Importers::CsvImportService.new(params[:files]).call
-        render turbo_stream: turbo_stream.replace("results", partial: "organizations/staff/external_form_upload/upload_results", locals: {import: import})
+        file = params[:files]
+        @blob = ActiveStorage::Blob.create_and_upload!(io: file, filename: file.original_filename)
+
+        CsvImportJob.perform_later(@blob.signed_id, current_user.id)
+
+        flash.now[:notice] = t(".processing_file")
+        respond_to do |format|
+          format.turbo_stream
+        end
       end
     end
   end

app/jobs/csv_import_job.rb

@@ -0,0 +1,11 @@
+class CsvImportJob < ApplicationJob
+  queue_as :default
+
+  def perform(blob_signed_id, current_user_id)
+    blob = ActiveStorage::Blob.find_signed(blob_signed_id)
+
+    Organizations::Importers::CsvImportService.new(blob, current_user_id).call
+  ensure
+    blob.purge_later
+  end
+end

app/services/organizations/importers/csv_import_service.rb

@@ -4,61 +4,66 @@ module Organizations
   module Importers
     class CsvImportService
       Status = Data.define(:success?, :count, :no_match, :errors)
-      def initialize(file)
+      def initialize(file, current_user_id)
         @file = file
-        @organization = Current.organization
+        @organization = ActsAsTenant.current_tenant
+        @current_user = User.find(current_user_id)
         @count = 0
         @no_match = []
         @errors = []
       end
 
       def call
         catch(:halt_import) do
-          validate_file
-
-          CSV.foreach(@file.to_path, headers: true, skip_blanks: true).with_index(2) do |row, index|
-            # Header may be different depending on which form applicaiton was used(e.g. google forms) or how it was created(User creates form with "Email Address")
-            email = row[@email_header].downcase
-            # Google forms uses "Timestamp", other services may use a different header
-            csv_timestamp = Time.parse(row["Timestamp"]) if row["Timestamp"].present?
-
-            person = Person.find_by(email:, organization: @organization)
-            previously_matched_form_submission = FormSubmission.where(person:, csv_timestamp:)
-
-            if person.nil?
-              @no_match << [index, email]
-            elsif previously_matched_form_submission.present?
-              next
-            else
-              ActiveRecord::Base.transaction do
-                create_form_answers(FormSubmission.create!(person:, csv_timestamp:), row)
-                @count += 1
+          @file.download do |data|
+            validate_file(data)
+            CSV.parse(data, headers: true, skip_blanks: true).each_with_index do |row, index|
+              # Header may be different depending on which form applicaiton was used(e.g. google forms) or how it was created(User creates form with "Email Address")
+              email = row[@email_header].downcase
+              # Google forms uses "Timestamp", other services may use a different header
+              csv_timestamp = Time.parse(row["Timestamp"]) if row["Timestamp"].present?
+
+              person = Person.find_by(email:, organization: @organization)
+              previously_matched_form_submission = FormSubmission.where(person:, csv_timestamp:)
+
+              if person.nil?
+                @no_match << [index + 2, email]
+              elsif previously_matched_form_submission.present?
+                next
+              else
+                ActiveRecord::Base.transaction do
+                  create_form_answers(FormSubmission.create!(person:, csv_timestamp:), row)
+                  @count += 1
+                end
               end
+            rescue => e
+              @errors << [index + 2, e]
             end
-          rescue => e
-            @errors << [index, e]
           end
         end
-        Status.new(@errors.empty?, @count, @no_match, @errors)
+        Turbo::StreamsChannel.broadcast_replace_to ["csv_import", @current_user],
+          target: "results",
+          partial: "organizations/staff/external_form_upload/upload_results",
+          locals: {
+            import: Status.new(@errors.empty?, @count, @no_match, @errors)
+          }
       end
 
       private
 
-      def validate_file
+      def validate_file(data)
         raise FileTypeError unless @file.content_type == "text/csv"
-
-        first_row = CSV.foreach(@file.to_path).first
-        raise FileEmptyError if first_row.nil?
+        first_row = CSV.new(data).shift
 
         raise TimestampColumnError unless first_row.include?("Timestamp")
 
         email_headers = ["Email", "email", "Email Address", "email address"]
-        email_headers.each do |e|
-          @email_header = e if first_row.include?(e)
+        email_headers.each do |header|
+          @email_header = header if first_row.include?(header)
         end
         raise EmailColumnError unless @email_header
-      rescue FileTypeError, FileEmptyError, TimestampColumnError, EmailColumnError => e
-        @errors << e
+      rescue FileTypeError, TimestampColumnError, EmailColumnError => e
+        @errors << [1, e]
         throw :halt_import
       end
 

app/views/organizations/staff/external_form_upload/_upload_in_progress.html.erb

@@ -0,0 +1,5 @@
+<div id="results" class="d-flex justify-content-center">
+  <div class="spinner-border text-primary" role="status">
+    <span class="visually-hidden">Loading...</span>
+  </div>
+</div>

app/views/organizations/staff/external_form_upload/_upload_results.html.erb

@@ -1,21 +1,21 @@
-<turbo-frame id="results">
-    <% if import.success? %>
-        <div class="alert alert-success">
-            <h5 class="alert-heading" role="alert"><%= t('.success_heading') %></h5>
-            <p><%= t('.submissions_added', count: import.count) %></p>
-        </div>
-    <% else %>
-        <div class="alert alert-danger">
-            <h5 class="alert-heading" role="alert"><%= t('.error_heading', count: import.errors.count) %></h5>
-            <p><%= t('.submissions_added', count: import.count) %></p>
-            <%= t('.review_errors', count: import.count) %>
-            <ul>
-                <% import.errors.each do |error| %>
-                    <li>
-                        <%= t('.row', number: error[0], message: error[1].message) %>
-                    </li>
-                <% end %>
-            </ul>
-         </div>
-    <% end %>
-</turbo-frame>
+<%= tag.div id: "results" do %>
+  <% if import.success? %>
+    <div class="alert alert-success">
+      <h5 class="alert-heading" role="alert"><%= t(".success_heading") %></h5>
+      <p><%= t(".submissions_added", count: import.count) %></p>
+    </div>
+  <% else %>
+    <div class="alert alert-danger">
+      <h5 class="alert-heading" role="alert"><%= t(".error_heading", count: import.errors.count) %></h5>
+      <p><%= t(".submissions_added", count: import.count) %></p>
+      <%= t(".review_errors", count: import.count) %>
+      <ul>
+        <% import.errors.each do |error| %>
+          <li>
+            <%= t(".row", number: error[0], message: error[1].message) %>
+          </li>
+        <% end %>
+      </ul>
+    </div>
+  <% end %>
+<% end %>

app/views/organizations/staff/external_form_upload/create.turbo_stream.erb

@@ -0,0 +1,3 @@
+<%= turbo_stream.replace("flash", partial: "layouts/shared/flash_messages") %>
+
+<%= turbo_stream.replace("results", partial: "upload_in_progress") %>

app/views/organizations/staff/external_form_upload/index.html.erb

@@ -1,14 +1,21 @@
 <%= render DashboardPageComponent.new(crumb: :external_form) do |c| %>
-  <% c.with_header_title { t('.header') } %>
+  <% c.with_header_title { t(".header") } %>
   <% c.with_body do %>
     <section>
       <div>
         <div>
-          <p><%= t('.description') %></p>
-          <p class="fw-bold"><%= t('.csv_header_requirements') %></p>
-          <%= render "organizations/staff/shared/attachment_form", instance: nil, title: 'Files', url: staff_external_form_upload_index_path, multiple: false, attachment_type: 'files' %>
+          <p><%= t(".description") %></p>
+          <p class="fw-bold"><%= t(".csv_header_requirements") %></p>
+          <%= render "organizations/staff/shared/attachment_form",
+          instance: nil,
+          title: "Files",
+          url: staff_external_form_upload_index_path,
+          multiple: false,
+          attachment_type: "files" %>
         </div>
       </div>
     </section>
+    <%= turbo_stream_from ["csv_import", current_user] %>
+    <%= tag.div id: "results" %>
   <% end %>
-<% end %>
+<% end %>

app/views/organizations/staff/shared/_attachment_form.html.erb

@@ -16,19 +16,18 @@
                           direct_upload: true,
                           class: "custom-attachments",
                           hide_label: true %>
-          <small class="form-text text-muted"> 
+          <small class="form-text text-muted">
             <% if attachment_type == 'files' %>
               Files must be .pdf, .png, or .jpeg under 2MB.
             <% elsif attachment_type == 'images' %>
-              Images must be .jpg or .png under 1MB. 
-            <%end%>
+              Images must be .jpg or .png under 1MB.
+            <% end %>
           </small>
         </div>
         <div class="ps-2">
           <%= form.submit t("general.attach"), class: "btn btn-outline-success" %>
         </div>
       </div>
     <% end %>
-    <%= turbo_frame_tag "results" %>
   </div>
 </div>

config/locales/en.yml

@@ -677,6 +677,8 @@ en:
           description: "If you collect information from adopters from a third party service, like Google Forms, you can export a CSV file from Google Forms and upload it here. We will import the questions and answers for any adopters who have an account on your Homeward Tails website, providing they use the same email address. Then you will be able to see the information for each adopter when reviewing applications."
           header: "Upload CSV File"
           csv_header_requirements: "Your CSV file must have column headers of 'Email' and 'Timestamp' for the import to work."
+        create:
+          processing_file: "File uploaded for processing"
         upload_results:
           success_heading: "File successfully scanned"
           error_heading: "File scanned: %{count} error(s) present"

test/controllers/organizations/staff/external_form_upload_controller_test.rb

@@ -3,55 +3,25 @@
 module Organizations
   module Staff
     class ExternalFormUploadControllerTest < ActionDispatch::IntegrationTest
-      context "success" do
-        setup do
-          file = fixture_file_upload("google_form_sample.csv", "text/csv")
-          @params = {files: file}
-          admin = create(:admin)
-          @adopter = create(:adopter, email: "[email protected]")
-          @adopter2 = create(:adopter, email: "[email protected]")
-          sign_in admin
-        end
-
-        should "Creates new form submission" do
-          assert_difference "@adopter.person.form_submissions.count" do
-            post staff_external_form_upload_index_path, params: @params
-          end
-        end
-
-        should "It does not create form answers for adopter2" do
-          assert_no_difference "@adopter2.person.form_submissions.count" do
-            post staff_external_form_upload_index_path, params: @params
-          end
-        end
-
-        should "shows success feedback" do
-          post staff_external_form_upload_index_path, params: @params, as: :turbo_stream
-
-          assert_response :success
-          assert_turbo_stream(action: "replace", count: 1) do
-            assert_select "h5", text: "File successfully scanned"
-          end
-        end
+      setup do
+        file = fixture_file_upload("google_form_sample.csv", "text/csv")
+        @params = {files: file}
+        admin = create(:admin)
+        sign_in admin
       end
 
-      context "error" do
-        setup do
-          file = fixture_file_upload("google_form_error_sample.csv", "text/csv")
-          @params = {files: file}
-          admin = create(:admin)
-          create(:adopter, email: "[email protected]")
-          sign_in admin
-        end
+      should "get index" do
+        get staff_external_form_upload_index_path
+        assert_response :success
+      end
 
-        should "shows error feedback" do
+      should "Creates new form submission" do
+        assert_enqueued_with(job: CsvImportJob) do
           post staff_external_form_upload_index_path, params: @params, as: :turbo_stream
-
-          assert_response :success
-          assert_turbo_stream(action: "replace", count: 1) do
-            assert_select "h5", text: "File scanned: 1 error(s) present"
-          end
         end
+
+        assert_response :success
+        assert_equal "File uploaded for processing", flash[:notice]
       end
     end
   end