Merge branch 'feature/stream-ontology-process' into test

Gemfile

@@ -14,6 +14,7 @@ gem 'sinatra', '~> 1.0'
 gem 'sinatra-advanced-routes'
 gem 'sinatra-contrib', '~> 1.0'
 gem 'request_store'
+gem "parallel"
 gem 'addressable', '~> 2.8'
 gem 'json-ld', '~> 3.0.2'
 
@@ -40,6 +41,10 @@ gem 'newrelic_rpm'
 # HTTP server
 gem 'unicorn'
 gem 'unicorn-worker-killer'
+gem "puma", "~> 6.4"
+gem "faye-websocket", "~> 0.11.3"
+
+
 
 # Templating
 gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107
@@ -63,7 +68,7 @@ group :development do
   gem 'capistrano-rbenv', require: false
   gem 'ed25519', '>= 1.2', '< 2.0', require: false
   gem 'pry'
-  gem 'shotgun', github: 'palexander/shotgun', branch: 'ncbo'
+  gem "listen", "~> 3.9"
 end
 
 

Gemfile.lock

@@ -11,7 +11,7 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/goo.git
-  revision: 10b90c17af12c71bfc95bfb4fc0bba5e47ff77af
+  revision: 84aa3db7677e6cc8c56d6fb44d2263304d28e55c
   branch: feature/add-model-based-search
   specs:
     goo (0.0.2)
@@ -57,7 +57,7 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git
-  revision: f703399b348c7bbaf7c5e2e12cc46ce2919cc989
+  revision: 27f7e42dfabde091ce1ed2b34c88b8a4808603d5
   branch: test-uri-deref-and-index
   specs:
     ontologies_linked_data (0.0.1)
@@ -93,14 +93,6 @@ GIT
     rack-post-body-to-params (0.1.8)
       activesupport (>= 2.3)
 
-GIT
-  remote: https://github.com/palexander/shotgun.git
-  revision: db198224aaab2e4cb9b049adccb30e387d88bc3b
-  branch: ncbo
-  specs:
-    shotgun (0.9)
-      rack (>= 1.0)
-
 GEM
   remote: https://rubygems.org/
   specs:
@@ -111,7 +103,7 @@ GEM
       public_suffix (>= 2.0.2, < 6.0)
     airbrussh (1.5.1)
       sshkit (>= 1.6.1, != 1.7.0)
-    backports (3.24.1)
+    backports (3.25.0)
     base64 (0.2.0)
     bcrypt (3.1.20)
     bcrypt_pbkdf (1.1.0)
@@ -141,6 +133,7 @@ GEM
     docile (1.4.0)
     domain_name (0.6.20240107)
     ed25519 (1.3.0)
+    eventmachine (1.2.7)
     faraday (1.10.3)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
@@ -164,6 +157,9 @@ GEM
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
     faraday-retry (1.0.3)
+    faye-websocket (0.11.3)
+      eventmachine (>= 0.12.0)
+      websocket-driver (>= 0.5.1)
     ffi (1.16.3)
     gapic-common (0.21.1)
       faraday (>= 1.9, < 3.a)
@@ -191,12 +187,12 @@ GEM
       representable (~> 3.0)
       retriable (>= 2.0, < 4.a)
       rexml
-    google-cloud-core (1.6.1)
+    google-cloud-core (1.7.0)
       google-cloud-env (>= 1.0, < 3.a)
       google-cloud-errors (~> 1.0)
     google-cloud-env (2.1.1)
       faraday (>= 1.0, < 3.a)
-    google-cloud-errors (1.3.1)
+    google-cloud-errors (1.4.0)
     google-protobuf (3.25.3-x86_64-darwin)
     google-protobuf (3.25.3-x86_64-linux)
     googleapis-common-protos (1.5.0)
@@ -241,6 +237,9 @@ GEM
     kgio (2.11.4)
     libxml-ruby (5.0.2)
     link_header (0.0.8)
+    listen (3.9.0)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
     logger (1.6.0)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
@@ -252,7 +251,7 @@ GEM
     method_source (1.0.0)
     mime-types (3.5.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2024.0206)
+    mime-types-data (3.2024.0305)
     mini_mime (1.1.5)
     minitest (4.7.5)
     minitest-stub_any_instance (1.0.3)
@@ -278,17 +277,21 @@ GEM
     net-ssh (7.2.1)
     netrc (0.11.0)
     newrelic_rpm (9.7.1)
+    nio4r (2.7.0)
     oj (2.18.5)
     omni_logger (0.1.4)
       logger
     os (1.1.4)
+    parallel (1.24.0)
     parseconfig (1.1.2)
     pony (1.13.1)
       mail (>= 2.0)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (5.0.4)
+    puma (6.4.2)
+      nio4r (~> 2.0)
     rack (1.6.13)
     rack-accept (0.4.5)
       rack (>= 0.4)
@@ -307,6 +310,9 @@ GEM
     rack-timeout (0.6.3)
     raindrops (0.20.1)
     rake (10.5.0)
+    rb-fsevent (0.11.2)
+    rb-inotify (0.10.1)
+      ffi (~> 1.0)
     rdf (3.2.11)
       link_header (~> 0.0, >= 0.0.8)
     rdf-raptor (3.2.0)
@@ -405,6 +411,9 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
+    websocket-driver (0.7.6)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
 
 PLATFORMS
   x86_64-darwin-23
@@ -422,11 +431,13 @@ DEPENDENCIES
   cube-ruby
   ed25519 (>= 1.2, < 2.0)
   faraday (~> 1.9)
+  faye-websocket (~> 0.11.3)
   ffi
   goo!
   haml (~> 5.2.2)
   json-ld (~> 3.0.2)
   json-schema (~> 2.0)
+  listen (~> 3.9)
   minitest (~> 4.0)
   minitest-stub_any_instance
   multi_json (~> 1.0)
@@ -436,8 +447,10 @@ DEPENDENCIES
   newrelic_rpm
   oj (~> 2.0)
   ontologies_linked_data!
+  parallel
   parseconfig
   pry
+  puma (~> 6.4)
   rack
   rack-accept (~> 0.4)
   rack-attack (~> 6.6.1)
@@ -454,7 +467,6 @@ DEPENDENCIES
   redis-rack-cache (~> 2.0)
   redis-store (= 1.9.1)
   request_store
-  shotgun!
   simplecov
   simplecov-cobertura
   sinatra (~> 1.0)

app.log

@@ -0,0 +1,59 @@
+# Logfile created on 2024-03-06 04:41:57 +0000 by logger.rb/v1.6.0
+Starting to process INRAETHES-0/submissions/1
+Java call [java -DentityExpansionLimit=2500000 -Xmx10240M -jar /srv/ontoportal/bundle/ruby/2.7.0/bundler/gems/ontologies_linked_data-d37aeafbd7be/bin/owlapi-wrapper.jar -m /srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/CSOPRA\ Summary.owl -o /srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1 -r true]
+2024-03-06T04:41:58 [main] INFO  o.s.n.o.w.OntologyParserCommand - Parsing invocation with values: ParserInvocation [inputRepositoryFolder=null, outputRepositoryFolder=/srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1, masterFileName=/srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/CSOPRA Summary.owl, invocationId=0, parserLog=, userReasoner= true]
+
+2024-03-06T04:41:58 [main] INFO  o.s.n.owlapi.wrapper.OntologyParser - executor ...
+
+2024-03-06T04:41:59 [main] INFO  o.s.n.owlapi.wrapper.OntologyParser - Input repository folder is null. Unique file being parsed.
+
+2024-03-06T04:42:04 [main] INFO  o.s.n.o.w.metrics.OntologyMetrics - Calculating metrics for /srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/CSOPRA Summary.owl
+
+2024-03-06T04:42:05 [main] INFO  o.s.n.o.w.metrics.OntologyMetrics - Finished metrics calculation for /srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/CSOPRA Summary.owl in 21 milliseconds
+
+2024-03-06T04:42:05 [main] INFO  o.s.n.o.w.metrics.OntologyMetrics - Generated metrics CSV file for /srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/CSOPRA Summary.owl
+
+2024-03-06T04:42:05 [main] INFO  o.s.n.owlapi.wrapper.OntologyParser - Ontology document format: org.semanticweb.owlapi.formats.RDFXMLDocumentFormat
+
+2024-03-06T04:42:05 [main] INFO  o.s.n.owlapi.wrapper.OntologyParser - useImports: http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl
+
+2024-03-06T04:42:05 [main] INFO  o.s.n.owlapi.wrapper.OntologyParser - useImports: http://purl.obolibrary.org/obo/agro-edit.owl
+
+2024-03-06T04:42:05 [main] INFO  o.s.n.owlapi.wrapper.OntologyParser - isPrefixOWLOntologyFormat: true
+
+2024-03-06T04:42:05 [main] INFO  o.s.n.owlapi.wrapper.OntologyParser - isPrefixOWLOntologyFormat: true
+
+2024-03-06T04:42:05 [main] INFO  o.s.n.owlapi.wrapper.OntologyParser - isPrefixOWLOntologyFormat: true
+
+2024-03-06T04:42:05 [main] INFO  o.s.n.owlapi.wrapper.OntologyParser - isOBO: false
+
+2024-03-06T04:42:05 [main] INFO  o.s.n.owlapi.wrapper.OntologyParser - Serializing ontology in RDF ...
+
+2024-03-06T04:42:07 [main] INFO  o.s.n.owlapi.wrapper.OntologyParser - Serialization done!
+
+2024-03-06T04:42:07 [main] INFO  o.s.n.o.w.OntologyParserCommand - Parse result: true
+
+2024-03-06T04:42:07 [main] INFO  o.s.n.o.w.OntologyParserCommand - Output triples in: {}/srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/owlapi.xrdf
+
+2024-03-06T04:42:07 [main] INFO  o.s.n.o.w.OntologyParserCommand - Finished parsing!
+
+OWLAPI Java command: parsing finished successfully.
+Output size 7726913 in `/srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/owlapi.xrdf`
+OWL_IMPORT_MISSING: file:///opt/owl/anaee-onto-onto-fr.owl
+Triples /srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/owlapi.xrdf appended in <http://data.bioontology.org/ontologies/INRAETHES-0/submissions/1>
+Empty page encountered. Retrying 1 times...
+Empty page encountered. Retrying 2 times...
+Empty page encountered. Retrying 3 times...
+Empty page encountered. Retrying 4 times...
+Empty page encountered. Retrying 5 times...
+Empty page encountered. Retrying 6 times...
+Empty page encountered. Retrying 7 times...
+Empty page encountered. Retrying 8 times...
+Empty page encountered. Retrying 9 times...
+Empty page encountered. Retrying 10 times...
+Empty page 1 of 2 persisted after retrying 10 times. Missing Labels Generation of 1 aborted...
+Starting to process INRAETHES-0/submissions/1
+Extraction metadata from ontology http://csopralibs.org#
+Additional metadata extracted.
+Submission processing of http://data.bioontology.org/ontologies/INRAETHES-0/submissions/1 completed successfully
+Starting to process INRAETHES-0/submissions/1

app.rb

@@ -187,3 +187,16 @@
   Pry.start binding, :quiet => true
   exit
 end
+
+
+require_relative 'lib/websockets/websockets_server'
+
+configure do
+  set :websocket_server, WebSocketServer.new(lambda { |env|
+    [200, { 'Content-Type' => 'text/plain' }, ['Hello World']]
+  })
+end
+
+Faye::WebSocket.load_adapter('puma')
+
+

bin/ontoportal

@@ -177,7 +177,8 @@ run_command() {
 dev() {
   echo "Starting OntoPortal API development server..."
 
-  local custom_command="bundle exec shotgun --host 0.0.0.0  --env=development --port 9393"
+  local custom_command="ENVIRONMENT=development bundle exec puma -C config/puma.rb"
+  #local custom_command="ENVIRONMENT=development bundle exec rackup -p 9393 --host 0.0.0.0"
   run_command "$custom_command" "$@"
 }
 

config/puma.rb

@@ -0,0 +1,21 @@
+workers Integer(ENV['WEB_CONCURRENCY'] || 0)
+threads_count = Integer(ENV['MAX_THREADS'] || 5)
+threads threads_count, threads_count
+
+pid = Process.pid
+env = ENV['RACK_ENV'] || ENV['ENVIRONMENT'] || 'production'
+
+environment env
+port 9393
+
+if env == 'development'
+  puts "Reload enabled"
+  # Use the `listen` gem to reload the application when files change
+  require 'listen'
+
+  listener = Listen.to('.', only: /\.rb$/) do |modified, added, removed|
+    # Reload the application when Ruby files change
+    Process.kill('SIGUSR1', pid)
+  end
+  listener.start
+end