diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c636a15 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +Dockerfile +docker-compose.yml + +deployment/ + +.venv +pyproject.toml +README.md +requirements.txt +**/__pycache__ +**/*.py[cod] + +vendor +tmp diff --git a/.gitignore b/.gitignore index 54ec10f..69b1302 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,6 @@ spec/fixtures/.DS_Store !.elasticbeanstalk/*.global.yml # Elasticsearch data -es_data \ No newline at end of file +es_data + +vendor/ diff --git a/Gemfile b/Gemfile index f996d8c..994e09d 100644 --- a/Gemfile +++ b/Gemfile @@ -9,7 +9,7 @@ gem "active_elastic_job" gem "aws-sdk", "~> 2" # https://github.com/tawan/active-elastic-job/pull/95 gem "elasticsearch" gem "elasticsearch-model" -gem "elasticsearch-persistence" +gem "elasticsearch-persistence", "~> 7.0" gem "jbuilder", "~> 2.11" gem "puma", "~> 5.3" gem "rails" @@ -17,6 +17,7 @@ gem "sanitize" gem "htmlentities" gem "charlock_holmes" gem "iso_country_codes" +gem "public_suffix" # Reduces boot times through caching; required in config/boot.rb gem "bootsnap", ">= 1.4.2", require: false diff --git a/Gemfile.lock b/Gemfile.lock index 890513a..61d075c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,38 +1,40 @@ GEM remote: https://rubygems.org/ specs: - actioncable (6.0.3.2) - actionpack (= 6.0.3.2) + actioncable (6.1.7.6) + actionpack (= 6.1.7.6) + activesupport (= 6.1.7.6) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.0.3.2) - actionpack (= 6.0.3.2) - activejob (= 6.0.3.2) - activerecord (= 6.0.3.2) - activestorage (= 6.0.3.2) - activesupport (= 6.0.3.2) + actionmailbox (6.1.7.6) + actionpack (= 6.1.7.6) + activejob (= 6.1.7.6) + activerecord (= 6.1.7.6) + activestorage (= 6.1.7.6) + activesupport (= 6.1.7.6) mail (>= 2.7.1) - actionmailer (6.0.3.2) - actionpack (= 6.0.3.2) - actionview (= 6.0.3.2) - activejob (= 6.0.3.2) + actionmailer (6.1.7.6) + actionpack (= 6.1.7.6) + actionview (= 6.1.7.6) + activejob (= 6.1.7.6) + activesupport (= 6.1.7.6) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.0.3.2) - actionview (= 6.0.3.2) - activesupport (= 6.0.3.2) - rack (~> 2.0, >= 2.0.8) + actionpack (6.1.7.6) + actionview (= 6.1.7.6) + activesupport (= 6.1.7.6) + rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.0.3.2) - actionpack (= 6.0.3.2) - activerecord (= 6.0.3.2) - activestorage (= 6.0.3.2) - activesupport (= 6.0.3.2) + actiontext (6.1.7.6) + actionpack (= 6.1.7.6) + activerecord (= 6.1.7.6) + activestorage (= 6.1.7.6) + activesupport (= 6.1.7.6) nokogiri (>= 1.8.5) - actionview (6.0.3.2) - activesupport (= 6.0.3.2) + actionview (6.1.7.6) + activesupport (= 6.1.7.6) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) @@ -40,160 +42,176 @@ GEM active_elastic_job (2.0.1) aws-sdk (~> 2) rails (>= 4.2) - activejob (6.0.3.2) - activesupport (= 6.0.3.2) + activejob (6.1.7.6) + activesupport (= 6.1.7.6) globalid (>= 0.3.6) - activemodel (6.0.3.2) - activesupport (= 6.0.3.2) - activerecord (6.0.3.2) - activemodel (= 6.0.3.2) - activesupport (= 6.0.3.2) - activestorage (6.0.3.2) - actionpack (= 6.0.3.2) - activejob (= 6.0.3.2) - activerecord (= 6.0.3.2) - marcel (~> 0.3.1) - activesupport (6.0.3.2) + activemodel (6.1.7.6) + activesupport (= 6.1.7.6) + activerecord (6.1.7.6) + activemodel (= 6.1.7.6) + activesupport (= 6.1.7.6) + activestorage (6.1.7.6) + actionpack (= 6.1.7.6) + activejob (= 6.1.7.6) + activerecord (= 6.1.7.6) + activesupport (= 6.1.7.6) + marcel (~> 1.0) + mini_mime (>= 1.1.0) + activesupport (6.1.7.6) concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) - zeitwerk (~> 2.2, >= 2.2.2) - addressable (2.7.0) - public_suffix (>= 2.0.2, < 5.0) - ast (2.4.1) - aws-eventstream (1.1.0) - aws-sdk (2.11.505) - aws-sdk-resources (= 2.11.505) - aws-sdk-core (2.11.505) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + zeitwerk (~> 2.3) + addressable (2.8.6) + public_suffix (>= 2.0.2, < 6.0) + ast (2.4.2) + aws-eventstream (1.3.0) + aws-sdk (2.11.632) + aws-sdk-resources (= 2.11.632) + aws-sdk-core (2.11.632) aws-sigv4 (~> 1.0) jmespath (~> 1.0) - aws-sdk-resources (2.11.505) - aws-sdk-core (= 2.11.505) - aws-sigv4 (1.1.3) - aws-eventstream (~> 1.0, >= 1.0.2) - bootsnap (1.7.5) - msgpack (~> 1.0) + aws-sdk-resources (2.11.632) + aws-sdk-core (= 2.11.632) + aws-sigv4 (1.8.0) + aws-eventstream (~> 1, >= 1.0.2) + base64 (0.2.0) + bootsnap (1.17.0) + msgpack (~> 1.2) builder (3.2.4) byebug (11.1.3) charlock_holmes (0.7.7) - concurrent-ruby (1.1.8) + concurrent-ruby (1.2.2) crack (0.4.5) rexml crass (1.0.6) - diff-lcs (1.4.4) - docile (1.3.4) - elasticsearch (7.7.0) - elasticsearch-api (= 7.7.0) - elasticsearch-transport (= 7.7.0) - elasticsearch-api (7.7.0) + date (3.3.4) + diff-lcs (1.5.0) + docile (1.4.0) + elasticsearch (7.17.10) + elasticsearch-api (= 7.17.10) + elasticsearch-transport (= 7.17.10) + elasticsearch-api (7.17.10) multi_json - elasticsearch-model (7.1.0) + elasticsearch-model (7.2.1) activesupport (> 3) - elasticsearch (> 1) + elasticsearch (~> 7) hashie - elasticsearch-persistence (7.1.0) + elasticsearch-persistence (7.2.1) activemodel (> 4) activesupport (> 4) elasticsearch (~> 7) - elasticsearch-model (= 7.1.0) + elasticsearch-model (= 7.2.1) hashie - elasticsearch-transport (7.7.0) - faraday (~> 1) + elasticsearch-transport (7.17.10) + faraday (>= 1, < 3) multi_json - erubi (1.10.0) - faraday (1.0.1) - multipart-post (>= 1.2, < 3) - ffi (1.15.1) - globalid (0.4.2) - activesupport (>= 4.2.0) - hashdiff (1.0.1) - hashie (4.1.0) + erubi (1.12.0) + faraday (2.8.1) + base64 + faraday-net_http (>= 2.0, < 3.1) + ruby2_keywords (>= 0.0.4) + faraday-net_http (3.0.2) + ffi (1.16.3) + globalid (1.2.1) + activesupport (>= 6.1) + hashdiff (1.1.0) + hashie (5.0.0) htmlentities (4.3.4) - i18n (1.8.7) + i18n (1.14.1) concurrent-ruby (~> 1.0) iso_country_codes (0.7.8) - jbuilder (2.11.2) + jbuilder (2.11.5) + actionview (>= 5.0.0) activesupport (>= 5.0.0) - jmespath (1.4.0) + jmespath (1.6.2) + json (2.7.1) listen (3.5.1) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) - loofah (2.8.0) + loofah (2.22.0) crass (~> 1.0.2) - nokogiri (>= 1.5.9) - mail (2.7.1) + nokogiri (>= 1.12.0) + mail (2.8.1) mini_mime (>= 0.1.1) - marcel (0.3.3) - mimemagic (~> 0.3.2) + net-imap + net-pop + net-smtp + marcel (1.0.2) method_source (1.0.0) - mimemagic (0.3.10) - nokogiri (~> 1) - rake - mini_mime (1.0.2) - mini_portile2 (2.5.3) - minitest (5.14.3) - msgpack (1.4.2) - multi_json (1.14.1) - multipart-post (2.1.1) - nio4r (2.5.7) - nokogiri (1.11.7) - mini_portile2 (~> 2.5.0) + mini_mime (1.1.5) + mini_portile2 (2.8.5) + minitest (5.20.0) + msgpack (1.7.2) + multi_json (1.15.0) + net-imap (0.3.7) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.2) + timeout + net-smtp (0.4.0) + net-protocol + nio4r (2.7.0) + nokogiri (1.13.10) + mini_portile2 (~> 2.8.0) racc (~> 1.4) - nokogumbo (2.0.4) - nokogiri (~> 1.8, >= 1.8.4) - parallel (1.20.1) - parser (3.0.0.0) + parallel (1.24.0) + parser (3.2.2.4) ast (~> 2.4.1) - public_suffix (4.0.6) - puma (5.3.2) + racc + public_suffix (5.0.4) + puma (5.6.7) nio4r (~> 2.0) - racc (1.5.2) - rack (2.2.3) - rack-test (1.1.0) - rack (>= 1.0, < 3) - rails (6.0.3.2) - actioncable (= 6.0.3.2) - actionmailbox (= 6.0.3.2) - actionmailer (= 6.0.3.2) - actionpack (= 6.0.3.2) - actiontext (= 6.0.3.2) - actionview (= 6.0.3.2) - activejob (= 6.0.3.2) - activemodel (= 6.0.3.2) - activerecord (= 6.0.3.2) - activestorage (= 6.0.3.2) - activesupport (= 6.0.3.2) - bundler (>= 1.3.0) - railties (= 6.0.3.2) + racc (1.7.3) + rack (2.2.8) + rack-test (2.1.0) + rack (>= 1.3) + rails (6.1.7.6) + actioncable (= 6.1.7.6) + actionmailbox (= 6.1.7.6) + actionmailer (= 6.1.7.6) + actionpack (= 6.1.7.6) + actiontext (= 6.1.7.6) + actionview (= 6.1.7.6) + activejob (= 6.1.7.6) + activemodel (= 6.1.7.6) + activerecord (= 6.1.7.6) + activestorage (= 6.1.7.6) + activesupport (= 6.1.7.6) + bundler (>= 1.15.0) + railties (= 6.1.7.6) sprockets-rails (>= 2.0.0) - rails-dom-testing (2.0.3) - activesupport (>= 4.2.0) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.3.0) - loofah (~> 2.3) - railties (6.0.3.2) - actionpack (= 6.0.3.2) - activesupport (= 6.0.3.2) + rails-html-sanitizer (1.5.0) + loofah (~> 2.19, >= 2.19.1) + railties (6.1.7.6) + actionpack (= 6.1.7.6) + activesupport (= 6.1.7.6) method_source - rake (>= 0.8.7) - thor (>= 0.20.3, < 2.0) - rainbow (3.0.0) - rake (13.0.3) - rb-fsevent (0.11.0) + rake (>= 12.2) + thor (~> 1.0) + rainbow (3.1.1) + rake (13.1.0) + rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) - regexp_parser (2.0.3) - rexml (3.2.5) - rspec-core (3.10.1) - rspec-support (~> 3.10.0) - rspec-expectations (3.10.1) + regexp_parser (2.8.3) + rexml (3.2.6) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-mocks (3.10.1) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-rails (4.0.2) + rspec-support (~> 3.12.0) + rspec-rails (4.1.2) actionpack (>= 4.2) activesupport (>= 4.2) railties (>= 4.2) @@ -201,71 +219,78 @@ GEM rspec-expectations (~> 3.10) rspec-mocks (~> 3.10) rspec-support (~> 3.10) - rspec-support (3.10.1) - rubocop (1.8.1) + rspec-support (3.12.1) + rubocop (1.50.2) + json (~> 2.3) parallel (~> 1.10) - parser (>= 3.0.0.0) + parser (>= 3.2.0.0) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) - rexml - rubocop-ast (>= 1.2.0, < 2.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.4.0) - parser (>= 2.7.1.5) - rubocop-packaging (0.5.1) - rubocop (>= 0.89, < 2.0) - rubocop-performance (1.9.2) - rubocop (>= 0.90.0, < 2.0) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + rubocop-md (1.2.2) + rubocop (>= 1.0) + rubocop-minitest (0.30.0) + rubocop (>= 1.39, < 2.0) + rubocop-packaging (0.5.2) + rubocop (>= 1.33, < 2.0) + rubocop-performance (1.17.1) + rubocop (>= 1.7.0, < 2.0) rubocop-ast (>= 0.4.0) - rubocop-rails (2.9.1) + rubocop-rails (2.19.1) activesupport (>= 4.2.0) rack (>= 1.1) - rubocop (>= 0.90.0, < 2.0) - rubocop-rails_config (1.2.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-rails_config (1.14.0) railties (>= 5.0) - rubocop (>= 1.0) - rubocop-ast (>= 1.0.1) + rubocop (>= 1.48.0) + rubocop-ast (>= 1.26.0) + rubocop-md + rubocop-minitest (~> 0.22) rubocop-packaging (~> 0.5) - rubocop-performance (~> 1.3) + rubocop-performance (~> 1.11) rubocop-rails (~> 2.0) - ruby-progressbar (1.11.0) - sanitize (5.2.3) + ruby-progressbar (1.13.0) + ruby2_keywords (0.0.5) + sanitize (6.1.0) crass (~> 1.0.2) - nokogiri (>= 1.8.0) - nokogumbo (~> 2.0) - shoulda-matchers (4.5.1) - activesupport (>= 4.2.0) - simplecov (0.21.2) + nokogiri (>= 1.12.0) + shoulda-matchers (5.3.0) + activesupport (>= 5.2.0) + simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) - simplecov_json_formatter (0.1.2) + simplecov_json_formatter (0.1.4) spring (2.1.1) spring-watcher-listen (2.0.1) listen (>= 2.7, < 4.0) spring (>= 1.2, < 3.0) - sprockets (4.0.2) + sprockets (4.2.1) concurrent-ruby (~> 1.0) - rack (> 1, < 3) - sprockets-rails (3.2.1) - actionpack (>= 4.0) - activesupport (>= 4.0) + rack (>= 2.2.4, < 4) + sprockets-rails (3.4.2) + actionpack (>= 5.2) + activesupport (>= 5.2) sprockets (>= 3.0.0) - thor (1.0.1) - thread_safe (0.3.6) - tzinfo (1.2.9) - thread_safe (~> 0.1) - unicode-display_width (2.0.0) - webmock (3.13.0) - addressable (>= 2.3.6) + thor (1.3.0) + timeout (0.4.1) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.5.0) + webmock (3.19.1) + addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - websocket-driver (0.7.3) + websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) - zeitwerk (2.4.2) + zeitwerk (2.6.12) PLATFORMS ruby @@ -278,11 +303,12 @@ DEPENDENCIES charlock_holmes elasticsearch elasticsearch-model - elasticsearch-persistence + elasticsearch-persistence (~> 7.0) htmlentities iso_country_codes jbuilder (~> 2.11) listen (>= 3.0.5, < 3.6) + public_suffix puma (~> 5.3) rails rspec-rails (~> 4.0) @@ -295,7 +321,7 @@ DEPENDENCIES webmock RUBY VERSION - ruby 2.6.4p104 + ruby 2.6.10p210 BUNDLED WITH - 1.17.3 + 1.17.2 diff --git a/app/controllers/healthcheck_controller.rb b/app/controllers/healthcheck_controller.rb new file mode 100644 index 0000000..202f3e4 --- /dev/null +++ b/app/controllers/healthcheck_controller.rb @@ -0,0 +1,5 @@ +class HealthcheckController < ApplicationController + def index + render status: 200, json: { status: 'ok' } + end +end diff --git a/config/routes.rb b/config/routes.rb index 6f619e8..24357c7 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,4 +1,5 @@ Rails.application.routes.draw do get 'consolidated_screening_list/search', to: 'consolidated#search', defaults: {format: :json} get 'v2/consolidated_screening_list/search', to: 'consolidated#search', defaults: {format: :json} + get '/health', to: 'healthcheck#index' end diff --git a/config/secrets.yml b/config/secrets.yml new file mode 100644 index 0000000..81b55d5 --- /dev/null +++ b/config/secrets.yml @@ -0,0 +1,2 @@ +production: + secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> diff --git a/docker/csl-python/Dockerfile b/docker/csl-python/Dockerfile index 1e61c93..886b1ea 100644 --- a/docker/csl-python/Dockerfile +++ b/docker/csl-python/Dockerfile @@ -1,16 +1,69 @@ -FROM python:3.6-slim-buster +######################################## +# Backend Builder stage +######################################## +FROM python:3.11-bookworm AS builder + # python +ENV PYTHONUNBUFFERED=1 \ + # prevents python creating .pyc files + PYTHONDONTWRITEBYTECODE=1 \ + # pip + PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_DEFAULT_TIMEOUT=100 \ + # do not ask any interactive question + POETRY_NO_INTERACTION=1 -WORKDIR /csl-python -COPY . /csl-python/ +WORKDIR /repo -RUN pip install --no-cache-dir -r requirements.txt +RUN set -ex \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* -ENV ELASTICSEARCH_HOST="elastic" -ENV ELASTICSEARCH_PORT=9200 +COPY pyproject.toml poetry.lock ./ -COPY ./entrypoint.sh /usr/bin/ -RUN chmod +x /usr/bin/entrypoint.sh -COPY ./wait-for-it.sh /usr/bin/ -RUN chmod +x /usr/bin/wait-for-it.sh + # Upgrade pip and install poetry +RUN pip install --no-cache-dir pip==23.3.1 poetry==1.7.1 && \ + # export poetry configuration file to requirements.txt + poetry export --without-hashes -o /repo/requirements.txt && \ + # Build python packages with wheel format + pip wheel -r /repo/requirements.txt --wheel-dir=/repo/wheels && \ + # Build python packages with sdist format for final stage + find /repo/wheels/ -name '*.whl' > /repo/requirements-wheel.txt -CMD ["/usr/bin/entrypoint.sh"] +######################################## +# Backend Final stage +######################################## +FROM python:3.11-slim-bookworm + +ENV WORKDIR=/app + +WORKDIR ${WORKDIR} + +COPY --from=builder /repo/ /repo/ +COPY . ${WORKDIR}/ +COPY ./entrypoint.sh /usr/local/bin/ +COPY ./wait-for-it.sh /usr/local/bin/ + +RUN chmod +x /usr/local/bin/*.sh + + +RUN set -ex \ + && apt-get update \ + # Security updates + && apt-get full-upgrade -y \ + && apt-get install -y --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + \ + # create a non-root user + && useradd --system --no-create-home -u 1000 appuser \ + \ + # Install only required dependencies + && pip install --no-cache-dir --no-index \ + --find-links=/repo/wheels \ + -r /repo/requirements-wheel.txt + +# Run as non-root user to avoid security concern +USER appuser + +CMD ["/usr/local/bin/entrypoint.sh"] diff --git a/docker/csl-python/import_source.py b/docker/csl-python/import_source.py index 12424f7..ac9afcb 100644 --- a/docker/csl-python/import_source.py +++ b/docker/csl-python/import_source.py @@ -3,14 +3,17 @@ import re import time import traceback -from logging.handlers import WatchedFileHandler +from datetime import datetime import requests from elasticsearch import Elasticsearch from elasticsearch.exceptions import ConnectionError +import warnings +from elasticsearch.exceptions import ElasticsearchWarning +warnings.simplefilter('ignore', ElasticsearchWarning) -STOPWORDS = {'and', 'the', 'los'} +STOP_WORDS = {'and', 'the', 'los'} COMMON_WORDS = { 'co', 'company', 'corp', 'corporation', 'inc', 'incorporated', 'limited', 'ltd', 'mr', 'mrs', 'ms', 'organization', 'sa', 'sas', 'llc', 'university', 'univ' @@ -19,7 +22,7 @@ def make_names(doc): doc['name_idx'] = filter_alnum_and_space(doc['name']) - doc['name_idx'] = remove_words(doc['name_idx'], STOPWORDS) + doc['name_idx'] = remove_words(doc['name_idx'], STOP_WORDS) if has_any_common_words(doc['name_idx']): make_names_with_common(doc, 'name') @@ -36,7 +39,7 @@ def make_names(doc): def make_alt_names(doc): doc['alt_idx'] = [filter_alnum_and_space(n) for n in doc['alt_names']] - doc['alt_idx'] = [remove_words(n, STOPWORDS) for n in doc['alt_idx']] + doc['alt_idx'] = [remove_words(n, STOP_WORDS) for n in doc['alt_idx']] if has_any_common_words(' '.join(doc['alt_idx'])): make_alt_names_with_common(doc) @@ -112,11 +115,6 @@ def get_json_data(url): steam_handler.setLevel(logging.INFO) logger.addHandler(steam_handler) -file_handler = WatchedFileHandler('/var/log/csl.log') -file_handler.setLevel(logging.INFO) -logger.addHandler(file_handler) - - class BaseImporter: ES_INDEX_NAME = None SOURCE_NAME = None @@ -137,7 +135,7 @@ def do_import(self, data): doc = make_full_addresses(doc) doc = make_source_object(self.SOURCE_NAME, doc) - self._es.index(index=self.ES_INDEX_NAME, body=doc, id=doc_id) + self._es.index(index=self.ES_INDEX_NAME, document=doc, id=doc_id) _count += 1 es.indices.refresh(index=self.ES_INDEX_NAME) @@ -175,27 +173,25 @@ def is_isn_source(doc) -> bool: if __name__ == '__main__': - hosts = os.getenv('ELASTICSEARCH_HOST', 'localhost') - port = os.environ.get('ELASTICSEARCH_PORT', 9200) - es = Elasticsearch(hosts=hosts, port=port) + + es_url = os.environ.get('ELASTICSEARCH_URL', 'http://127.0.0.1:9200') + + es = Elasticsearch(hosts=es_url) source_importers = [importer_cls(es) for importer_cls in SOURCE_IMPORTER_CLASSES] - while True: - logger.info('Start import CSL source') - try: - if not es.ping(): - raise ConnectionError + logger.info('Start import CSL source: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + try: + if not es.ping(): + raise ConnectionError - json_data = get_json_data('https://api.trade.gov/static/consolidated_screening_list/consolidated.json') + json_data = get_json_data('https://api.trade.gov/static/consolidated_screening_list/consolidated.json') - for importer in source_importers: - importer.do_import(json_data) + for importer in source_importers: + importer.do_import(json_data) - logger.info('Finish import CSL source') - except ConnectionError: - logger.error('Connect ES server failed') - except Exception as e: + logger.info('Finish import CSL source: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + except ConnectionError: + logger.error('Connect ES server failed') + except Exception as e: logger.error(f'Import CSL source failed: {e}, {traceback.format_exc()}') - - time.sleep(1800) diff --git a/docker/csl-python/poetry.lock b/docker/csl-python/poetry.lock new file mode 100644 index 0000000..1b1ef02 --- /dev/null +++ b/docker/csl-python/poetry.lock @@ -0,0 +1,196 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "certifi" +version = "2023.11.17" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, +] + +[[package]] +name = "chardet" +version = "4.0.0" +description = "Universal encoding detector for Python 2 and 3" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, + {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "elasticsearch" +version = "7.13.1" +description = "Python client for Elasticsearch" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4" +files = [ + {file = "elasticsearch-7.13.1-py2.py3-none-any.whl", hash = "sha256:a09ae1de8869efa6ef2d9a0a9b9f6d9260b0c2506e83dd32bc1119a23fff49a5"}, + {file = "elasticsearch-7.13.1.tar.gz", hash = "sha256:d6bcca0b2e5665d08e6fe6fadc2d4d321affd76ce483603078fc9d3ccd2bc0f9"}, +] + +[package.dependencies] +certifi = "*" +urllib3 = ">=1.21.1,<2" + +[package.extras] +async = ["aiohttp (>=3,<4)"] +develop = ["black", "coverage", "jinja2", "mock", "pytest", "pytest-cov", "pyyaml", "requests (>=2.0.0,<3.0.0)", "sphinx (<1.7)", "sphinx-rtd-theme"] +docs = ["sphinx (<1.7)", "sphinx-rtd-theme"] +requests = ["requests (>=2.4.0,<3.0.0)"] + +[[package]] +name = "idna" +version = "2.10" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, + {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "urllib3" +version = "1.26.18" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.11" +content-hash = "0bbae87f8f5a2d420fa905f4721dea53d06df6287761a1edf981c0707b73e948" diff --git a/docker/csl-python/pyproject.toml b/docker/csl-python/pyproject.toml new file mode 100644 index 0000000..2618a1b --- /dev/null +++ b/docker/csl-python/pyproject.toml @@ -0,0 +1,21 @@ +[tool.poetry] +name = "csl-python" +version = "0.1.0" +description = "" +authors = ["GoFreight"] +readme = "README.md" +packages = [{include = "csl_python"}] + +[tool.poetry.dependencies] +python = "^3.11" +certifi = "*" +chardet = "^4" +elasticsearch = "7.13.1" +idna = "^2" +requests = "^2" +urllib3 = "^1" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/docker/csl-python/requirements.txt b/docker/csl-python/requirements.txt deleted file mode 100644 index 39c9871..0000000 --- a/docker/csl-python/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -certifi==2021.5.30 -chardet==4.0.0 -elasticsearch==7.13.1 -idna==2.10 -requests==2.25.1 -urllib3==1.26.5 diff --git a/docker/csl/Dockerfile b/docker/csl/Dockerfile index 137b2cd..bb94028 100644 --- a/docker/csl/Dockerfile +++ b/docker/csl/Dockerfile @@ -1,21 +1,46 @@ -FROM ruby:2.6 +# Stage 1: Build the Rails app +FROM ruby:2.6.10-bullseye as builder -RUN gem install bundler +# Set environment variables +ENV LANG=C.UTF-8 \ + RAILS_ENV=production \ + BUNDLE_PATH=/bundle \ + BUNDLE_JOBS=4 -WORKDIR /csl -COPY . /csl/ +# set the working directory +WORKDIR /repo -RUN bundle install +# copy the Gemfile and Gemfile.lock +COPY Gemfile Gemfile.lock ./ -ENV ELASTICSEARCH_URL="elastic:9200" +# Install gems +RUN gem install bundler -v 2.4.22 && \ + bundle install --without development test + +# Stage 2: Create a smaller image with Debian Bullseye +FROM ruby:2.6.10-slim-bullseye AS final + +# Set environment variables +ENV LANG=C.UTF-8 \ + RAILS_ENV=production \ + BUNDLE_PATH=/bundle + +# Install security updates +RUN apt-get update \ + && apt-get -y full-upgrade --no-install-recommends \ + && apt-get install -y libicu67 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY . . +COPY --from=builder /bundle /bundle +COPY ./docker/csl/* /usr/bin/ + +RUN bundle install --without development test && \ + chmod +x /usr/bin/*.sh -COPY ./docker/csl/entrypoint.sh /usr/bin/ -RUN chmod +x /usr/bin/entrypoint.sh -COPY ./docker/csl/import.sh /usr/bin/ -RUN chmod +x /usr/bin/import.sh -COPY ./docker/csl/wait-for-it.sh /usr/bin/ -RUN chmod +x /usr/bin/wait-for-it.sh -COPY ./docker/csl/development.rb /csl/config/environments/ EXPOSE 3000 CMD ["/usr/bin/entrypoint.sh"] diff --git a/docker/csl/entrypoint.sh b/docker/csl/entrypoint.sh index d1cfe06..eafed2d 100755 --- a/docker/csl/entrypoint.sh +++ b/docker/csl/entrypoint.sh @@ -2,6 +2,7 @@ /usr/bin/wait-for-it.sh elastic:9200 -t 30 -- echo "Elasticsearch server is ready" +# DB Migration if [ "$RECREATE_DB" = "true" ] ; then bundle exec rake db:create /usr/bin/import.sh diff --git a/docker/csl/import.sh b/docker/csl/import.sh index 87aa98a..c1a97ac 100755 --- a/docker/csl/import.sh +++ b/docker/csl/import.sh @@ -6,17 +6,17 @@ echo "********** CSL server - Start import source list **********" souces=( "CapData" "DplData" - "DtcData" + # "DtcData" "ElData" "FseData" - "IsnData" + # "IsnData" "MeuData" "PlcData" "SdnData" "SsiData" "UvlData" ) - + ## get item count using ${arrayname[@]} ## for source in "${souces[@]}" do diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7461a2c..374e2f4 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,39 +1,59 @@ -version: "3.3" +version: "3.8" services: elastic: - container_name: elasticsearch - # user: $USER - image: docker.elastic.co/elasticsearch/elasticsearch:7.4.2 - volumes: - - ./es_data:/usr/share/elasticsearch/data - environment: + container_name: csl-es + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.16 + restart: on-failure + environment: - discovery.type=single-node + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ports: + - 9200:9200 + - 9300:9300 + volumes: + - elastic:/usr/share/elasticsearch/data + kibana: + container_name: csl-kibana + image: docker.elastic.co/kibana/kibana:7.17.16 + restart: on-failure + links: + - elastic:elasticsearch + ports: + - 5601:5601 + depends_on: + - elastic + volumes: + - kibana:/usr/share/kibana/data + rails: build: context: ./../ dockerfile: $PWD/csl/Dockerfile - container_name: csl + container_name: csl-api environment: - RECREATE_DB=true + - SECRET_KEY_BASE=fake-key + - ELASTICSEARCH_URL=elastic:9200 + command: ["tail", "-f", "/dev/null"] + volumes: + - $PWD/../:/app + ports: + - 3000:3000 depends_on: - elastic + python: - build: ./csl-python/ + build: + context: ./csl-python/ + dockerfile: $PWD/csl-python/Dockerfile container_name: csl-python + environment: + - RECREATE_DB=true + - ELASTICSEARCH_URL=http://elastic:9200 depends_on: + - rails - elastic - csl-proxy: - build: ./nginx/ - container_name: csl-proxy - volumes: - - "./nginx/ssl/:/ssl/:ro" - - "./nginx/conf.d/:/etc/nginx/conf.d/:ro" - ports: - - "${HTTPS_PORT:-11999}:11999" - logging: - driver: json-file - options: - max-size: "30m" - max-file: "10" - depends_on: - - rails \ No newline at end of file + +volumes: + elastic: + kibana: diff --git a/lib/csl_repository.rb b/lib/csl_repository.rb index 0e6daa0..88a8019 100644 --- a/lib/csl_repository.rb +++ b/lib/csl_repository.rb @@ -137,7 +137,10 @@ class CslRepository < BaseRepository expiration_date: { type: "date", format: "yyyy-MM-dd" - } + }, + number: { + type: "keyword" + }, } indexes :nationalities, type: "keyword" @@ -157,7 +160,7 @@ class CslRepository < BaseRepository type: "integer" indexes :_updated_at, type: "date", - format: "strictDateOptionalTime" + format: "strict_date_optional_time" end end end