[BUG] EXTRACTION FORMAT / DOCUMENT FORMAT MISMATCH: As Tim, I want to be able to safely change the format of an extraction, without it crashing the transformation preview, so that I don't get stuck trying to develop my pipeline.

Gemfile.lock

@@ -239,11 +239,11 @@ GEM
       net-protocol
     netrc (0.11.0)
     nio4r (2.7.0)
-    nokogiri (1.15.4-aarch64-linux)
+    nokogiri (1.16.2-aarch64-linux)
       racc (~> 1.4)
-    nokogiri (1.15.4-arm64-darwin)
+    nokogiri (1.16.2-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.15.4-x86_64-linux)
+    nokogiri (1.16.2-x86_64-linux)
       racc (~> 1.4)
     optparse (0.3.1)
     orm_adapter (0.5.0)
@@ -260,7 +260,7 @@ GEM
     puma (5.6.8)
       nio4r (~> 2.0)
     raabro (1.4.0)
-    racc (1.7.1)
+    racc (1.7.3)
     rack (2.2.8)
     rack-mini-profiler (3.3.0)
       rack (>= 1.2.0)

app/controllers/transformation_definitions_controller.rb

@@ -77,7 +77,12 @@ def clone
   def assign_show_variables
     @fields = @transformation_definition.fields.order(created_at: :desc).map(&:to_h)
     @props = transformation_app_state
-    @extraction_jobs = @harvest_definition.extraction_definition.extraction_jobs.order(created_at: :desc)
+
+    @extraction_jobs = if @harvest_definition.extraction_definition.present?
+                         @harvest_definition.extraction_definition.extraction_jobs.order(created_at: :desc)
+                       else
+                         []
+                       end
   end
 
   def find_pipeline

app/frontend/js/components/CodeEditor.jsx

@@ -30,22 +30,34 @@ const CodeEditor = ({ initContent, onChange, format = "ruby", ...props }) => {
     }
   };
 
+  const errorDocument = () => {
+    return JSON.stringify({
+      error:
+        "Something went wrong fetching the response from the content source. This can also happen when the format specified in the extraction definition is not the same as the format of the extracted document.",
+    });
+  };
+
   const doc = () => {
+    let content;
     if (format == "JSON") {
-      let content;
-
       try {
         content = JSON.stringify(JSON.parse(initContent), null, 2);
       } catch (err) {
-        content = JSON.stringify({
-          error:
-            "Something went wrong fetching the response from the content source",
-        });
+        content = errorDocument();
       }
 
       return content;
     } else if (format == "XML") {
-      return xmlFormat(initContent, { indentation: "  ", lineSeparator: "\n" });
+      try {
+        content = xmlFormat(initContent, {
+          indentation: "  ",
+          lineSeparator: "\n",
+        });
+      } catch (err) {
+        content = errorDocument();
+      }
+
+      return content;
     } else {
       return initContent;
     }

app/frontend/js/modals/transformationDefinitionSettingsModal.js

@@ -53,6 +53,17 @@ function extractTransformationPreviewData(form) {
 }
 
 function displayInitialPreview(data) {
+  let { id, result, tooltip } = data;
+
+  if (result == "") {
+    displayError(id);
+    tooltip.disable();
+  } else {
+    displayPreview(data);
+  }
+}
+
+function displayPreview(data) {
   let {
     id,
     recordSelector,
@@ -63,10 +74,7 @@ function displayInitialPreview(data) {
     completed,
   } = data;
 
-  if (result == "") {
-    displayError(id);
-    tooltip.disable();
-  } else {
+  try {
     if (format == "JSON") {
       result = JSON.stringify(JSON.parse(result), null, 2);
     } else if (format == "XML") {
@@ -75,34 +83,60 @@ function displayInitialPreview(data) {
         lineSeparator: "\n",
       });
     }
+  } catch (error) {
+    displayError(id);
+    tooltip.disable();
+    return;
+  }
 
-    editor(`#js-record-selector-result-${id}`, format, true, result);
+  editor(`#js-record-selector-result-${id}`, format, true, result);
 
-    tooltip.disable();
+  setupTooltip(tooltip, recordSelector, transformationDefinitionUpdateButton);
+  setupModal(recordSelector, completed);
+  bindRecordSelectorInput(
+    recordSelector,
+    transformationDefinitionUpdateButton,
+    tooltip
+  );
+}
 
-    if (recordSelector.value == "") {
-      transformationDefinitionUpdateButton.disabled = true;
-      tooltip.enable();
-    }
+function setupTooltip(
+  tooltip,
+  recordSelector,
+  transformationDefinitionUpdateButton
+) {
+  tooltip.disable();
 
-    if (recordSelector.value == "" && completed == "true") {
-      new Modal(
-        document.getElementById("update-transformation-definition-modal")
-      ).show();
-    }
+  if (recordSelector.value == "") {
+    transformationDefinitionUpdateButton.disabled = true;
+    tooltip.enable();
+  }
+}
 
-    recordSelector.addEventListener("input", (event) => {
-      if (event.target.value == "") {
-        transformationDefinitionUpdateButton.disabled = true;
-        tooltip.enable();
-      } else {
-        transformationDefinitionUpdateButton.disabled = false;
-        tooltip.disable();
-      }
-    });
+function setupModal(recordSelector, completed) {
+  if (recordSelector.value == "" && completed == "true") {
+    new Modal(
+      document.getElementById("update-transformation-definition-modal")
+    ).show();
   }
 }
 
+function bindRecordSelectorInput(
+  recordSelector,
+  transformationDefinitionUpdateButton,
+  tooltip
+) {
+  recordSelector.addEventListener("input", (event) => {
+    if (event.target.value == "") {
+      transformationDefinitionUpdateButton.disabled = true;
+      tooltip.enable();
+    } else {
+      transformationDefinitionUpdateButton.disabled = false;
+      tooltip.disable();
+    }
+  });
+}
+
 function bindRecordSelectorTestEventListeners(id) {
   bindTestForm(
     "test",

app/models/transformation_definition.rb

@@ -23,6 +23,8 @@ class TransformationDefinition < ApplicationRecord
   #
   # @return Array
   def records(page = 1)
+    return [] if extraction_job.blank?
+
     Transformation::RawRecordsExtractor.new(self, extraction_job).records(page)
   end
 
@@ -34,7 +36,7 @@ def to_h
   end
 
   def shared?
-    @shared = harvest_definitions.count > 1 if @shared.nil?
+    @shared = harvest_definitions.count > 1 if @shared.blank?
     @shared
   end
 

app/slices/raw_record_slice.rb

@@ -13,10 +13,24 @@ def call
     {
       page: @page,
       record: @record,
-      totalPages: @job.documents.total_pages,
+      totalPages: total_pages,
       totalRecords: @records.length,
-      format: @job.format,
+      format:,
       body: @records[@record - 1]
     }
   end
+
+  private
+
+  def total_pages
+    return 0 if @job.blank?
+
+    @job.documents.total_pages
+  end
+
+  def format
+    return 'JSON' if @job.blank?
+
+    @job.format
+  end
 end

app/views/transformation_definitions/_create_edit_modal.html.erb

@@ -53,11 +53,15 @@
             </div>
 
             <div class='col-8'>
-              <%= form.select(
-                    :extraction_job_id,
-                    options_from_collection_for_select(extraction_jobs, 'id', 'name', model.extraction_job_id),
-                    {}, class: 'form-select'
-                  ) %>
+              <% if extraction_jobs.any? %>
+                <%= form.select(
+                      :extraction_job_id,
+                      options_from_collection_for_select(extraction_jobs, 'id', 'name', model.extraction_job_id),
+                      {}, class: 'form-select'
+                    ) %>
+              <% else %>
+                <p class='text-danger'>There is no available extracted data.</p>
+              <% end %>
             </div>
 
             <div class="col-4">
@@ -108,7 +112,7 @@
   </div>
 <% end %>
 
-<% if model.persisted? %>
+<% if model.persisted? && model.extraction_job.present? %>
   <span
     id='js-transformation-definition-preview-data-<%= harvest_definition.id %>'
     data-format="<%= model.extraction_job.format %>"

app/views/transformation_definitions/show.html.erb

@@ -22,9 +22,9 @@
   <span id='react-nav-tabs'></span>
 <% end %>
 
-<% if @transformation_definition.extraction_job.completed? || @transformation_definition.extraction_job.cancelled? %>
+<% if @transformation_definition.extraction_job.present? && (@transformation_definition.extraction_job.completed? || @transformation_definition.extraction_job.cancelled?) %>
   <div id="js-transformation-app" data-props="<%= @props %>"></div>
-<% else %>
+<% elsif @transformation_definition.extraction_job.present? %>
   <div class="container">
     <div class="row">
       <div class="col text-center">
@@ -35,6 +35,14 @@
       </div>
     </div>
   </div>
+<% else %>
+  <div class="container">
+    <div class="row">
+      <div class="col text-center">
+        <p class='text-center'>There is no available extracted data.</p>
+      </div>
+    </div>
+  </div>
 <% end %>
 
 <%= render 'transformation_definitions/create_edit_modal',