diff --git a/Gemfile b/Gemfile index d698f63b..22968fdf 100644 --- a/Gemfile +++ b/Gemfile @@ -14,6 +14,7 @@ gem 'sinatra', '~> 1.0' gem 'sinatra-advanced-routes' gem 'sinatra-contrib', '~> 1.0' gem 'request_store' +gem "parallel" # Rack middleware gem 'ffi' @@ -38,6 +39,10 @@ gem 'newrelic_rpm' # HTTP server gem 'unicorn' gem 'unicorn-worker-killer' +gem "puma", "~> 6.4" +gem "faye-websocket", "~> 0.11.3" + + # Templating gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107 @@ -60,7 +65,7 @@ group :development do gem 'capistrano-rbenv', require: false gem 'ed25519', '>= 1.2', '< 2.0', require: false gem 'pry' - gem 'shotgun', github: 'palexander/shotgun', branch: 'ncbo' + gem "listen", "~> 3.9" end diff --git a/Gemfile.lock b/Gemfile.lock index 3948fd1d..eeae2011 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 1be1c83158177935cfc77230d42054de984e2d09 + revision: 6c51346b6f150a69391794b7909b30592acbbe0e branch: development specs: goo (0.0.2) @@ -57,7 +57,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 18a45c2b8a4f8d6c9f56181b4f25b7e0b116a44c + revision: d37aeafbd7bef120917fb4d601f8287a9a859f69 branch: development specs: ontologies_linked_data (0.0.1) @@ -93,14 +93,6 @@ GIT rack-post-body-to-params (0.1.8) activesupport (>= 2.3) -GIT - remote: https://github.com/palexander/shotgun.git - revision: db198224aaab2e4cb9b049adccb30e387d88bc3b - branch: ncbo - specs: - shotgun (0.9) - rack (>= 1.0) - GEM remote: https://rubygems.org/ specs: @@ -141,6 +133,7 @@ GEM docile (1.4.0) domain_name (0.6.20240107) ed25519 (1.3.0) + eventmachine (1.2.7) faraday (1.10.3) faraday-em_http (~> 1.0) faraday-em_synchrony (~> 1.0) @@ -164,6 +157,9 @@ GEM faraday-patron (1.0.0) faraday-rack (1.0.0) faraday-retry (1.0.3) + faye-websocket (0.11.3) + eventmachine (>= 0.12.0) + websocket-driver (>= 0.5.1) ffi (1.16.3) gapic-common (0.21.1) faraday (>= 1.9, < 3.a) @@ -233,11 +229,14 @@ GEM json-schema (2.8.1) addressable (>= 2.4) json_pure (2.7.1) - jwt (2.8.0) + jwt (2.8.1) base64 kgio (2.11.4) libxml-ruby (5.0.2) link_header (0.0.8) + listen (3.9.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) logger (1.6.0) macaddr (1.7.2) systemu (~> 2.6.5) @@ -275,10 +274,12 @@ GEM net-ssh (7.2.1) netrc (0.11.0) newrelic_rpm (9.7.1) + nio4r (2.7.0) oj (2.18.5) omni_logger (0.1.4) logger os (1.1.4) + parallel (1.24.0) parseconfig (1.1.2) pony (1.13.1) mail (>= 2.0) @@ -286,6 +287,8 @@ GEM coderay (~> 1.1) method_source (~> 1.0) public_suffix (5.0.4) + puma (6.4.2) + nio4r (~> 2.0) rack (1.6.13) rack-accept (0.4.5) rack (>= 0.4) @@ -304,6 +307,9 @@ GEM rack-timeout (0.6.3) raindrops (0.20.1) rake (10.5.0) + rb-fsevent (0.11.2) + rb-inotify (0.10.1) + ffi (~> 1.0) rdf (3.2.11) link_header (~> 0.0, >= 0.0.8) rdf-raptor (3.2.0) @@ -402,6 +408,9 @@ GEM addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) + websocket-driver (0.7.6) + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.5) PLATFORMS x86_64-darwin-23 @@ -418,10 +427,12 @@ DEPENDENCIES cube-ruby ed25519 (>= 1.2, < 2.0) faraday (~> 1.9) + faye-websocket (~> 0.11.3) ffi goo! haml (~> 5.2.2) json-schema (~> 2.0) + listen (~> 3.9) minitest (~> 4.0) minitest-stub_any_instance multi_json (~> 1.0) @@ -431,8 +442,10 @@ DEPENDENCIES newrelic_rpm oj (~> 2.0) ontologies_linked_data! + parallel parseconfig pry + puma (~> 6.4) rack rack-accept (~> 0.4) rack-attack (~> 6.6.1) @@ -449,7 +462,6 @@ DEPENDENCIES redis-rack-cache (~> 2.0) redis-store (= 1.9.1) request_store - shotgun! simplecov simplecov-cobertura sinatra (~> 1.0) @@ -461,4 +473,4 @@ DEPENDENCIES webmock BUNDLED WITH - 2.4.21 + 2.4.22 diff --git a/app.log b/app.log new file mode 100644 index 00000000..d9ab1338 --- /dev/null +++ b/app.log @@ -0,0 +1,59 @@ +# Logfile created on 2024-03-06 04:41:57 +0000 by logger.rb/v1.6.0 +Starting to process INRAETHES-0/submissions/1 +Java call [java -DentityExpansionLimit=2500000 -Xmx10240M -jar /srv/ontoportal/bundle/ruby/2.7.0/bundler/gems/ontologies_linked_data-d37aeafbd7be/bin/owlapi-wrapper.jar -m /srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/CSOPRA\ Summary.owl -o /srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1 -r true] +2024-03-06T04:41:58 [main] INFO o.s.n.o.w.OntologyParserCommand - Parsing invocation with values: ParserInvocation [inputRepositoryFolder=null, outputRepositoryFolder=/srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1, masterFileName=/srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/CSOPRA Summary.owl, invocationId=0, parserLog=, userReasoner= true] + +2024-03-06T04:41:58 [main] INFO o.s.n.owlapi.wrapper.OntologyParser - executor ... + +2024-03-06T04:41:59 [main] INFO o.s.n.owlapi.wrapper.OntologyParser - Input repository folder is null. Unique file being parsed. + +2024-03-06T04:42:04 [main] INFO o.s.n.o.w.metrics.OntologyMetrics - Calculating metrics for /srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/CSOPRA Summary.owl + +2024-03-06T04:42:05 [main] INFO o.s.n.o.w.metrics.OntologyMetrics - Finished metrics calculation for /srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/CSOPRA Summary.owl in 21 milliseconds + +2024-03-06T04:42:05 [main] INFO o.s.n.o.w.metrics.OntologyMetrics - Generated metrics CSV file for /srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/CSOPRA Summary.owl + +2024-03-06T04:42:05 [main] INFO o.s.n.owlapi.wrapper.OntologyParser - Ontology document format: org.semanticweb.owlapi.formats.RDFXMLDocumentFormat + +2024-03-06T04:42:05 [main] INFO o.s.n.owlapi.wrapper.OntologyParser - useImports: http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl + +2024-03-06T04:42:05 [main] INFO o.s.n.owlapi.wrapper.OntologyParser - useImports: http://purl.obolibrary.org/obo/agro-edit.owl + +2024-03-06T04:42:05 [main] INFO o.s.n.owlapi.wrapper.OntologyParser - isPrefixOWLOntologyFormat: true + +2024-03-06T04:42:05 [main] INFO o.s.n.owlapi.wrapper.OntologyParser - isPrefixOWLOntologyFormat: true + +2024-03-06T04:42:05 [main] INFO o.s.n.owlapi.wrapper.OntologyParser - isPrefixOWLOntologyFormat: true + +2024-03-06T04:42:05 [main] INFO o.s.n.owlapi.wrapper.OntologyParser - isOBO: false + +2024-03-06T04:42:05 [main] INFO o.s.n.owlapi.wrapper.OntologyParser - Serializing ontology in RDF ... + +2024-03-06T04:42:07 [main] INFO o.s.n.owlapi.wrapper.OntologyParser - Serialization done! + +2024-03-06T04:42:07 [main] INFO o.s.n.o.w.OntologyParserCommand - Parse result: true + +2024-03-06T04:42:07 [main] INFO o.s.n.o.w.OntologyParserCommand - Output triples in: {}/srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/owlapi.xrdf + +2024-03-06T04:42:07 [main] INFO o.s.n.o.w.OntologyParserCommand - Finished parsing! + +OWLAPI Java command: parsing finished successfully. +Output size 7726913 in `/srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/owlapi.xrdf` +OWL_IMPORT_MISSING: file:///opt/owl/anaee-onto-onto-fr.owl +Triples /srv/ontoportal/ontologies_api/test/data/ontology_files/repo/INRAETHES-0/1/owlapi.xrdf appended in +Empty page encountered. Retrying 1 times... +Empty page encountered. Retrying 2 times... +Empty page encountered. Retrying 3 times... +Empty page encountered. Retrying 4 times... +Empty page encountered. Retrying 5 times... +Empty page encountered. Retrying 6 times... +Empty page encountered. Retrying 7 times... +Empty page encountered. Retrying 8 times... +Empty page encountered. Retrying 9 times... +Empty page encountered. Retrying 10 times... +Empty page 1 of 2 persisted after retrying 10 times. Missing Labels Generation of 1 aborted... +Starting to process INRAETHES-0/submissions/1 +Extraction metadata from ontology http://csopralibs.org# +Additional metadata extracted. +Submission processing of http://data.bioontology.org/ontologies/INRAETHES-0/submissions/1 completed successfully +Starting to process INRAETHES-0/submissions/1 diff --git a/app.rb b/app.rb index 46457c86..9d477e93 100644 --- a/app.rb +++ b/app.rb @@ -187,3 +187,16 @@ Pry.start binding, :quiet => true exit end + + +require_relative 'lib/websockets/websockets_server' + +configure do + set :websocket_server, WebSocketServer.new(lambda { |env| + [200, { 'Content-Type' => 'text/plain' }, ['Hello World']] + }) +end + +Faye::WebSocket.load_adapter('puma') + + diff --git a/bin/ontoportal b/bin/ontoportal index 4840dad3..34b0d950 100755 --- a/bin/ontoportal +++ b/bin/ontoportal @@ -177,7 +177,8 @@ run_command() { dev() { echo "Starting OntoPortal API development server..." - local custom_command="bundle exec shotgun --host 0.0.0.0 --env=development" + local custom_command="ENVIRONMENT=development bundle exec puma -C config/puma.rb" + #local custom_command="ENVIRONMENT=development bundle exec rackup -p 9393 --host 0.0.0.0" run_command "$custom_command" "$@" } diff --git a/config/puma.rb b/config/puma.rb new file mode 100644 index 00000000..e44b324a --- /dev/null +++ b/config/puma.rb @@ -0,0 +1,21 @@ +workers Integer(ENV['WEB_CONCURRENCY'] || 0) +threads_count = Integer(ENV['MAX_THREADS'] || 5) +threads threads_count, threads_count + +pid = Process.pid +env = ENV['RACK_ENV'] || ENV['ENVIRONMENT'] || 'production' + +environment env +port 9393 + +if env == 'development' + puts "Reload enabled" + # Use the `listen` gem to reload the application when files change + require 'listen' + + listener = Listen.to('.', only: /\.rb$/) do |modified, added, removed| + # Reload the application when Ruby files change + Process.kill('SIGUSR1', pid) + end + listener.start +end \ No newline at end of file diff --git a/controllers/upload_ontology_controller.rb b/controllers/upload_ontology_controller.rb new file mode 100644 index 00000000..b7122ab4 --- /dev/null +++ b/controllers/upload_ontology_controller.rb @@ -0,0 +1,130 @@ +# app.rb +require 'sinatra/base' +require 'faye/websocket' +require 'thread' + +class UploadOntologyController < ApplicationController + require 'logger' + require 'observer' + + class ObservableLogger + include Observable + + def initialize(logfile) + @logger = Logger.new(logfile) + end + + def log(severity, message) + @logger.send(severity, message) + changed + notify_observers(severity, message) + end + + def flush + @logger.flush + end + + def info(message) + log(:info, message) + end + + def warn(message) + log(:warn, message) + end + + def error(message) + log(:error, message) + end + + def fatal(message) + log(:fatal, message) + end + end + + get '/ontologies/upload/ws' do + upload_ontology_ws_subscribe + end + + post '/ontologies/upload' do + ont, sub = perform_step(:ontology_creation, + "Start creating an ontology", + "Ontology created", + "Ontology creation failed" + ) do + create_ontology_from_file('SKOS') + end + + Thread.new do + # actions = { + # :process_rdf => true, + # :generate_labels => true, + # :extract_metadata => true, + # :index_search => true, + # :index_properties => true, + # :run_metrics => true, + # :process_annotator => true, + # :diff => true, + # :remote_pull => false + # } + break if ont.nil? + + begin + + steps = { + :process_rdf => { + label: 'Parse and save in RDF Store', + options: { process_rdf: true, extract_metadata: false, generate_labels: true } + }, + :extract_metadata => { + label: "Extract metadata", + options: { extract_metadata: true } + }, + # :index_search => { + # label: "Indexing terms", + # options: { index_search: true }, + # }, + :index_properties => { + label: "Indexing properties", + options: { index_properties: true }, + }, + :run_metrics => { + label: "Computing metrics", + options: { run_metrics: true } + } + } + + steps.each do |step, options| + perform_step(step, + "Start #{options[:label]}", + "#{options[:label]} ended successfully", + "#{options[:label]} failed") do + logger = ObservableLogger.new('app.log') + + observer = lambda do |severity, message| + broadcast_update(step, 1, message) + end + + logger.add_observer(observer, :call) + sub.process_submission(logger, options[:options]) + end + sub = LinkedData::Models::OntologySubmission.find(sub.id).first + sub.bring_remaining + end + + if sub.ready? + broadcast_update(:result, 2, "#{ont.acronym} parsed successfully") + broadcast_update(:result, 2, sub.metrics.to_hash.to_json) + end + rescue StandardError => e + broadcast_update(:result, 0, "#{ont.acronym} processing error: #{e.message}") + ensure + ont.delete + upload_ontology_ws_unsubscribe + end + end + status 200 + end + +end + + diff --git a/docker-compose.yml b/docker-compose.yml index 9b12fa66..e186e80d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,38 +1,48 @@ x-app: &app - build: - context: . - args: - RUBY_VERSION: '2.7' - # Increase the version number in the image tag every time Dockerfile or its arguments is changed - image: ontologies_ld-dev:0.0.2 + image: agroportal/ontologies_api:development environment: &env - # default bundle config resolves to /usr/local/bundle/config inside of the container - # we are setting it to local app directory if we need to use 'bundle config local' BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle BUNDLE_PATH: /srv/ontoportal/bundle COVERAGE: 'true' # enable simplecov code coverage REDIS_HOST: redis-ut REDIS_PORT: 6379 - SOLR_TERM_SEARCH_URL: http://solr-term-ut:8983/solr/term_search_core1 - SOLR_PROP_SEARCH_URL: http://solr-prop-ut:8984/solr/prop_search_core1 + SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr + SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr stdin_open: true tty: true command: /bin/bash - volumes: - # bundle volume for hosting gems installed by bundle; it speeds up gem install in local development + volumes: # <-- This seems to be incorrectly indented in your original YAML - bundle:/srv/ontoportal/bundle - - .:/srv/ontoportal/ontologies_linked_data - # mount directory containing development version of the gems if you need to use 'bundle config local' - #- /Users/alexskr/ontoportal:/Users/alexskr/ontoportal + - .:/srv/ontoportal/ontologies_api depends_on: &depends_on - solr-prop-ut: - condition: service_healthy - solr-term-ut: + solr-ut: condition: service_healthy redis-ut: condition: service_healthy services: + api: + <<: *app + env_file: + .env + environment: + <<: *env + GOO_BACKEND_NAME: 4store + GOO_PORT: 9000 + GOO_HOST: 4store-ut + GOO_PATH_QUERY: /sparql/ + GOO_PATH_DATA: /data/ + GOO_PATH_UPDATE: /update/ + + ports: + - "9393:9393" + profiles: + - fs + depends_on: + - solr-ut + - redis-ut + - mgrep-ut + - 4store-ut mgrep-ut: image: ontoportal/mgrep-ncbo:0.1 @@ -61,13 +71,11 @@ services: profiles: - fs - solr-term-ut: + solr-ut: image: solr:8 - volumes: - - ./test/solr/configsets:/configsets:ro ports: - "8983:8983" - command: [ "solr-precreate", "term_search_core1", "/configsets/term_search" ] + command: bin/solr start -cloud -f healthcheck: test: [ "CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1" ] start_period: 5s @@ -75,20 +83,6 @@ services: timeout: 5s retries: 5 - solr-prop-ut: - image: solr:8 - volumes: - - ./test/solr/configsets:/configsets:ro - ports: - - "8984:8983" - command: [ "solr-precreate", "prop_search_core1", "/configsets/property_search" ] - healthcheck: - test: [ "CMD-SHELL", "curl -sf http://localhost:8983/solr/prop_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1" ] - start_period: 5s - interval: 10s - timeout: 5s - retries: 5 - agraph-ut: image: franzinc/agraph:v8.1.0 platform: linux/amd64 @@ -99,9 +93,6 @@ services: ports: # - 10035:10035 - 10000-10035:10000-10035 - volumes: - - agdata:/agraph/data - # - ./agraph/etc:/agraph/etc command: > bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start ; agtool repos create ontoportal_test --supersede @@ -155,5 +146,4 @@ services: - gb volumes: - bundle: - agdata: \ No newline at end of file + bundle: \ No newline at end of file diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 24893eef..1df6cd56 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -439,6 +439,10 @@ def file_from_request end return nil, nil end + + def websockets + settings.websocket_server + end private def naive_expiring_cache_write(key, object, timeout = 60) diff --git a/helpers/upload_ontology_helper.rb b/helpers/upload_ontology_helper.rb new file mode 100644 index 00000000..fd725b3f --- /dev/null +++ b/helpers/upload_ontology_helper.rb @@ -0,0 +1,68 @@ +require 'sinatra/base' + +module Sinatra + module Helpers + module UploadOntologyHelper + + def create_ontology_from_file(format = nil ) + LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ + process_submission: false, + acronym: 'INRAETHES', + name: 'INRAETHES', + ont_count: 1, + submission_count: 1, + ontology_format: format || 'OWL' + }) + ont = Ontology.find('INRAETHES-0').include(:acronym).first + sub = ont.latest_submission(status: :any) + add_file_to_submission(ont, sub) + [ont, sub] + end + + def perform_step(step, start_msg, end_msg, error_msg, &block) + broadcast_update(step, 1, start_msg) + outputs = nil + + time = Benchmark.realtime do + begin + outputs = block.call + rescue StandardError => e + outputs = nil + end + end + + if outputs + broadcast_update(step, 2, end_msg, time) + else + broadcast_update(step, 0, error_msg, time) + end + outputs + end + + def broadcast_update(id, status, message, time = nil) + msg = { + id: id, + status: status, + time: time, + message: message + } + upload_ontology_ws_broadcast(msg.to_json) + end + + def upload_ontology_ws_subscribe(channel_id = "",env = request.env) + websockets.subscribe("UploadOntologyController:#{channel_id}", env) + end + + def upload_ontology_ws_unsubscribe(channel_id = "") + websockets.unsubscribe("UploadOntologyController:#{channel_id}") + end + + def upload_ontology_ws_broadcast(channel_id = "", message) + websockets.broadcast("UploadOntologyController:#{channel_id}", message) + end + + end + end +end + +helpers Sinatra::Helpers::UploadOntologyHelper \ No newline at end of file diff --git a/index.html b/index.html new file mode 100644 index 00000000..0ebcec82 --- /dev/null +++ b/index.html @@ -0,0 +1,135 @@ + + + + + + Real-time File Processing + + + + + + + +
+

Real-time File Processing

+
+
+ + +
+ +
+
+
+ + + + + + + diff --git a/lib/websockets/websockets_server.rb b/lib/websockets/websockets_server.rb new file mode 100644 index 00000000..9c2dfad7 --- /dev/null +++ b/lib/websockets/websockets_server.rb @@ -0,0 +1,44 @@ +require 'faye/websocket' +require 'json' + +class WebSocketServer + + def initialize(app) + @app = app + @clients = Hash.new { |h, k| h[k] = [] } + end + + def subscribe(channel, env) + if Faye::WebSocket.websocket?(env) + ws = Faye::WebSocket.new(env) + + ws.on :open do |event| + puts 'WebSocket connection opened' + @clients[channel] ||= [] + @clients[channel] << ws + end + + ws.on :message do |event| + end + + ws.on :close do |event| + puts 'WebSocket connection closed' + @clients.delete(channel) + ws = nil + end + + ws.rack_response + else + @app.call(env) + end + end + + def unsubscribe(channel) + @clients[channel].each { |socket| socket.close} + end + + def broadcast(channel, message) + @clients[channel].each { |socket| socket.send(message)} + end + +end