Full LENZ ingest

.gitignore

@@ -36,6 +36,7 @@
 
 # Ignore master key for decrypting credentials and more.
 /config/master.key
+/config/credentials.yml.enc
 
 /app/assets/builds/*
 !/app/assets/builds/.keep

Dockerfile

@@ -3,7 +3,7 @@ FROM ruby:3.2.2-alpine3.18
 WORKDIR /app
 
 ARG BUILD_PACKAGES="build-base curl-dev git"
-ARG DEV_PACKAGES="bash mysql-client mariadb-dev yaml-dev zlib-dev nodejs yarn libxml2 libxml2-dev libxslt libxslt-dev gmp-dev"
+ARG DEV_PACKAGES="bash mysql-client mariadb-dev yaml-dev zlib-dev nodejs yarn libxml2 libxml2-dev libxslt libxslt-dev gmp-dev openjdk8-jre"
 ARG RUBY_PACKAGES="tzdata"
 
 WORKDIR /app
@@ -34,6 +34,8 @@ ARG RAILS_ENV="production"
 ENV RAILS_ENV=$RAILS_ENV
 ARG RAILS_MASTER_KEY
 ENV RAILS_MASTER_KEY=$RAILS_MASTER_KEY
+ARG RAILS_SECRET_KEY_BASE
+ENV SECRET_KEY_BASE=$RAILS_SECRET_KEY_BASE
 
 RUN bundle exec rails assets:precompile
 

Gemfile

@@ -42,6 +42,7 @@ gem 'faraday-follow_redirects'
 gem 'jsonpath'
 gem 'nokogiri'
 gem 'sidekiq'
+gem 'yomu'
 
 # transformation related
 gem 'webmock'

Gemfile.lock

@@ -13,77 +13,77 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (7.0.7.2)
-      actionpack (= 7.0.7.2)
-      activesupport (= 7.0.7.2)
+    actioncable (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (7.0.7.2)
-      actionpack (= 7.0.7.2)
-      activejob (= 7.0.7.2)
-      activerecord (= 7.0.7.2)
-      activestorage (= 7.0.7.2)
-      activesupport (= 7.0.7.2)
+    actionmailbox (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activejob (= 7.0.8.1)
+      activerecord (= 7.0.8.1)
+      activestorage (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       mail (>= 2.7.1)
       net-imap
       net-pop
       net-smtp
-    actionmailer (7.0.7.2)
-      actionpack (= 7.0.7.2)
-      actionview (= 7.0.7.2)
-      activejob (= 7.0.7.2)
-      activesupport (= 7.0.7.2)
+    actionmailer (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      actionview (= 7.0.8.1)
+      activejob (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       mail (~> 2.5, >= 2.5.4)
       net-imap
       net-pop
       net-smtp
       rails-dom-testing (~> 2.0)
-    actionpack (7.0.7.2)
-      actionview (= 7.0.7.2)
-      activesupport (= 7.0.7.2)
+    actionpack (7.0.8.1)
+      actionview (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       rack (~> 2.0, >= 2.2.4)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (7.0.7.2)
-      actionpack (= 7.0.7.2)
-      activerecord (= 7.0.7.2)
-      activestorage (= 7.0.7.2)
-      activesupport (= 7.0.7.2)
+    actiontext (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activerecord (= 7.0.8.1)
+      activestorage (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (7.0.7.2)
-      activesupport (= 7.0.7.2)
+    actionview (7.0.8.1)
+      activesupport (= 7.0.8.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (7.0.7.2)
-      activesupport (= 7.0.7.2)
+    activejob (7.0.8.1)
+      activesupport (= 7.0.8.1)
       globalid (>= 0.3.6)
-    activemodel (7.0.7.2)
-      activesupport (= 7.0.7.2)
-    activerecord (7.0.7.2)
-      activemodel (= 7.0.7.2)
-      activesupport (= 7.0.7.2)
+    activemodel (7.0.8.1)
+      activesupport (= 7.0.8.1)
+    activerecord (7.0.8.1)
+      activemodel (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
     activerecord-nulldb-adapter (0.9.0)
       activerecord (>= 5.2.0, < 7.1)
-    activestorage (7.0.7.2)
-      actionpack (= 7.0.7.2)
-      activejob (= 7.0.7.2)
-      activerecord (= 7.0.7.2)
-      activesupport (= 7.0.7.2)
+    activestorage (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activejob (= 7.0.8.1)
+      activerecord (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (7.0.7.2)
+    activesupport (7.0.8.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
     addressable (2.8.4)
       public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
-    bcrypt (3.1.18)
+    bcrypt (3.1.20)
     better_html (2.0.2)
       actionview (>= 6.0)
       activesupport (>= 6.0)
@@ -108,16 +108,16 @@ GEM
     chunky_png (1.4.0)
     coderay (1.1.3)
     colorize (1.1.0)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
     crack (0.4.5)
       rexml
     crass (1.0.6)
-    date (3.3.3)
+    date (3.3.4)
     debug (1.7.2)
       irb (>= 1.5.0)
       reline (>= 0.3.1)
-    devise (4.9.2)
+    devise (4.9.3)
       bcrypt (~> 3.0)
       orm_adapter (~> 0.1)
       railties (>= 4.1.0)
@@ -128,7 +128,7 @@ GEM
       devise (~> 4.0)
       railties (~> 7.0)
       rotp (~> 6.0)
-    devise_invitable (2.0.8)
+    devise_invitable (2.0.9)
       actionmailer (>= 5.0)
       devise (>= 4.6)
     diff-lcs (1.5.0)
@@ -171,7 +171,7 @@ GEM
     fugit (1.8.1)
       et-orbi (~> 1, >= 1.2.7)
       raabro (~> 1.4)
-    globalid (1.2.0)
+    globalid (1.2.1)
       activesupport (>= 6.1)
     hashdiff (1.0.1)
     http (5.1.1)
@@ -183,7 +183,7 @@ GEM
     http-cookie (1.0.5)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
-    i18n (1.14.1)
+    i18n (1.14.4)
       concurrent-ruby (~> 1.0)
     io-console (0.6.0)
     irb (1.6.4)
@@ -210,32 +210,32 @@ GEM
     llhttp-ffi (0.4.0)
       ffi-compiler (~> 1.0)
       rake (~> 13.0)
-    loofah (2.21.3)
+    loofah (2.22.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     mail (2.8.1)
       mini_mime (>= 0.1.1)
       net-imap
       net-pop
       net-smtp
-    marcel (1.0.2)
+    marcel (1.0.4)
     matrix (0.4.2)
     method_source (1.0.0)
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2023.0218.1)
     mini_mime (1.1.5)
-    minitest (5.19.0)
+    minitest (5.22.2)
     multi_json (1.15.0)
-    mysql2 (0.5.5)
-    net-imap (0.3.7)
+    mysql2 (0.5.6)
+    net-imap (0.4.10)
       date
       net-protocol
     net-pop (0.1.2)
       net-protocol
-    net-protocol (0.2.1)
+    net-protocol (0.2.2)
       timeout
-    net-smtp (0.3.3)
+    net-smtp (0.4.0.1)
       net-protocol
     netrc (0.11.0)
     nio4r (2.7.0)
@@ -261,49 +261,49 @@ GEM
       nio4r (~> 2.0)
     raabro (1.4.0)
     racc (1.7.3)
-    rack (2.2.8)
+    rack (2.2.8.1)
     rack-mini-profiler (3.3.0)
       rack (>= 1.2.0)
     rack-proxy (0.7.6)
       rack
     rack-test (2.1.0)
       rack (>= 1.3)
-    rails (7.0.7.2)
-      actioncable (= 7.0.7.2)
-      actionmailbox (= 7.0.7.2)
-      actionmailer (= 7.0.7.2)
-      actionpack (= 7.0.7.2)
-      actiontext (= 7.0.7.2)
-      actionview (= 7.0.7.2)
-      activejob (= 7.0.7.2)
-      activemodel (= 7.0.7.2)
-      activerecord (= 7.0.7.2)
-      activestorage (= 7.0.7.2)
-      activesupport (= 7.0.7.2)
+    rails (7.0.8.1)
+      actioncable (= 7.0.8.1)
+      actionmailbox (= 7.0.8.1)
+      actionmailer (= 7.0.8.1)
+      actionpack (= 7.0.8.1)
+      actiontext (= 7.0.8.1)
+      actionview (= 7.0.8.1)
+      activejob (= 7.0.8.1)
+      activemodel (= 7.0.8.1)
+      activerecord (= 7.0.8.1)
+      activestorage (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       bundler (>= 1.15.0)
-      railties (= 7.0.7.2)
+      railties (= 7.0.8.1)
     rails-dom-testing (2.2.0)
       activesupport (>= 5.0.0)
       minitest
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.6.0)
       loofah (~> 2.21)
       nokogiri (~> 1.14)
-    railties (7.0.7.2)
-      actionpack (= 7.0.7.2)
-      activesupport (= 7.0.7.2)
+    railties (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
       zeitwerk (~> 2.5)
     rainbow (3.1.1)
-    rake (13.0.6)
+    rake (13.1.0)
     redis-client (0.17.0)
       connection_pool
     regexp_parser (2.8.0)
     reline (0.3.3)
       io-console (~> 0.5)
-    responders (3.1.0)
+    responders (3.1.1)
       actionpack (>= 5.2)
       railties (>= 5.2)
     rest-client (2.1.0)
@@ -394,8 +394,8 @@ GEM
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       sprockets (>= 3.0.0)
-    thor (1.2.2)
-    timeout (0.4.0)
+    thor (1.3.1)
+    timeout (0.4.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unf (0.1.4)
@@ -425,8 +425,9 @@ GEM
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    yard (0.9.34)
-    zeitwerk (2.6.11)
+    yard (0.9.36)
+    yomu (0.1.5)
+    zeitwerk (2.6.13)
 
 PLATFORMS
   aarch64-linux-musl
@@ -475,6 +476,7 @@ DEPENDENCIES
   webdrivers
   webmock
   yard
+  yomu
 
 RUBY VERSION
    ruby 3.2.2p53

app/controllers/extraction_definitions_controller.rb

@@ -98,7 +98,8 @@ def find_destinations
   def extraction_definition_params
     safe_params = params.require(:extraction_definition).permit(
       :pipeline_id, :name, :format, :base_url, :throttle, :page, :per_page,
-      :total_selector, :kind, :destination_id, :source_id, :enrichment_url, :paginated, :split, :split_selector
+      :total_selector, :kind, :destination_id, :source_id, :enrichment_url, :paginated, :split, :split_selector,
+      :extract_text_from_file
     )
     merge_last_edited_by(safe_params)
   end

app/frontend/js/apps/ExtractionApp/components/HeaderActions.jsx

@@ -60,11 +60,12 @@ const HeaderActions = () => {
 
   return createPortal(
     <>
-      {!appDetails.extractionDefinition.split && (
-        <button className="btn btn-success me-2" onClick={handlePreviewClick}>
-          <i className="bi bi-play" aria-hidden="true"></i> Preview
-        </button>
-      )}
+      {!appDetails.extractionDefinition.split &&
+        !appDetails.extractionDefinition.extract_text_from_file && (
+          <button className="btn btn-success me-2" onClick={handlePreviewClick}>
+            <i className="bi bi-play" aria-hidden="true"></i> Preview
+          </button>
+        )}
 
       {appDetails.extractionDefinition.split && <RunSample />}
 

app/models/extraction_definition.rb

@@ -36,6 +36,7 @@ class ExtractionDefinition < ApplicationRecord
 
   validates :name, uniqueness: true
   validates :split_selector, presence: true, if: :split?
+  validates :s3_bucket, presence: true, if: :s3?
 
   validates :throttle, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 60_000 }
 

app/models/harvest_report.rb

@@ -115,6 +115,7 @@ def statuses
   def idle_offset
     return 0 if extraction_end_time.blank?
     return @idle_offset if @idle_offset.present?
+    return 0 if transformation_start_time.blank? || extraction_end_time.blank?
 
     @idle_offset = transformation_start_time - extraction_end_time
     @idle_offset = 0 if @idle_offset.negative?