diff --git a/.ebextensions/01_create_indices.config b/.ebextensions/01_create_indices.config index 1b5a02a..203484c 100644 --- a/.ebextensions/01_create_indices.config +++ b/.ebextensions/01_create_indices.config @@ -1,4 +1,4 @@ container_commands: 01create: command: rake db:create - leader_only: true \ No newline at end of file + leader_only: true diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..439b703 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,10 @@ +## Reason + +## Changes + +## Test Scope + +## Checks + +- [ ] Unit tests are included, or please explain why it's not applicable. +- [ ] Keep pull requests small so they can be easily reviewed. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..42f73bb --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,126 @@ +# Reusable workflow to run lint, test, build, push docker image. +name: Reusable CI Workflow + +on: + workflow_call: + # GitHub Actions doesn't directly support passing secrets between workflows. + # Workflows that call reusable workflows in the same organization or + # enterprise can use the inherit keyword to implicitly pass the secrets. + # Therefore, we should pass the secret to the workflow_call workflow. + # Ref: https://github.com/orgs/community/discussions/23107 + secrets: + HC_GITHUB_SSH_KEY: + required: true + SLACK_WEBHOOK_URL: + required: true + inputs: + should_push_image: + description: 'Whether to push image to ECR' + required: true + type: boolean + outputs: + app_version: + value: ${{ jobs.test.outputs.app_version }} + +jobs: + lint: + runs-on: [self-hosted, general, small] + steps: + - name: checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 1024 + + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: pre-commit (PR) + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha}} + HEAD_SHA: ${{ github.event.pull_request.head.sha}} + run: | + python -m pip install pre-commit + # the bug for virualenv/setuptools/pip + # ref: https://github.com/pypa/setuptools/issues/2353 + SETUPTOOLS_USE_DISTUTILS=stdlib pre-commit run --from-ref $BASE_SHA --to-ref $HEAD_SHA --all-files || (git --no-pager diff && false) + + - name: pre-commit (push) + if: github.event_name == 'push' + env: + BASE_SHA: ${{ github.event.before}} + HEAD_SHA: ${{ github.event.after}} + run: | + python -m pip install pre-commit + SETUPTOOLS_USE_DISTUTILS=stdlib pre-commit run --from-ref $BASE_SHA --to-ref $HEAD_SHA --all-files || (git --no-pager diff && false) + + test: + runs-on: [self-hosted, general, small] + needs: + - lint + timeout-minutes: 10 + outputs: + app_version: ${{ steps.app-version.outputs.APP_VERSION }} + env: + PRIVATE_GH_REPO_SSH_KEY_PATH: /tmp/ssh_key + steps: + - name: checkout repo + uses: actions/checkout@v4 + + - id: app-version + name: Set APP_VERSION + run: | + APP_VERSION=$(make gen-version) + echo "APP_VERSION=$APP_VERSION" >> $GITHUB_ENV + echo "APP_VERSION=$APP_VERSION" >> $GITHUB_OUTPUT + + - name: prepare ssh key for private github repo + run: | + echo "${{ secrets.HC_GITHUB_SSH_KEY }}" > $PRIVATE_GH_REPO_SSH_KEY_PATH + mkdir -p ~/.ssh && cp $PRIVATE_GH_REPO_SSH_KEY_PATH ~/.ssh/id_rsa + + - name: build docker images + run: | + make build-docker-image IMAGE_TAG=$APP_VERSION + + - name: run tests + run: | + make test + + # Scan the Docker images and packages + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ env.IMAGE_NAME }}:${{ env.APP_VERSION }}" + format: 'table' + exit-code: '1' + ignore-unfixed: true + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH' + env: + IMAGE_NAME: gofreight/csl + + - name: push docker images + if: ${{ inputs.should_push_image }} + run: | + aws ecr describe-repositories --repository-names $IMAGE_NAME > /dev/null 2>&1 || aws ecr create-repository --repository-name $IMAGE_NAME + aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com + make push-docker-image IMAGE_TAG=$APP_VERSION + env: + AWS_ACCOUNT_ID: 478041131377 + AWS_REGION: us-west-2 + IMAGE_NAME: gofreight/csl + + - name: tear down + run: | + make tear-down + + - name: slack notification + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_COLOR: '#ff0000' + SLACK_MESSAGE: | + Repository: ${{ github.repository }} diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml new file mode 100644 index 0000000..a76fafc --- /dev/null +++ b/.github/workflows/pr-check.yml @@ -0,0 +1,12 @@ +name: PR Validation + +on: [pull_request] + +jobs: + ci: + uses: ./.github/workflows/ci.yml + secrets: + HC_GITHUB_SSH_KEY: ${{ secrets.HC_GITHUB_SSH_KEY }} # required for install private package from github private repo with SSH key + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PR_CI_WEBHOOK_URL }} + with: + should_push_image: false diff --git a/.github/workflows/release-production.yml b/.github/workflows/release-production.yml new file mode 100644 index 0000000..f90f72c --- /dev/null +++ b/.github/workflows/release-production.yml @@ -0,0 +1,36 @@ +name: Release for Production + +on: + release: + types: + - created + workflow_dispatch: + +jobs: + deploy-production: + runs-on: [self-hosted, general, small] + steps: + - uses: actions/checkout@v4 + with: + ref: release + fetch-tags: true + + - name: Validate if GITHUB_RES belongs to Git Tag + run: | + echo $GITHUB_REF + git describe --tags $GITHUB_REF + + - name: Release the specified git tag to production + run: | + git fetch --unshallow --tags + git reset --hard $GITHUB_REF + git push -f origin HEAD:release + + - name: slack notification + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_CI_CRITICAL_WEBHOOK_URL }} + SLACK_COLOR: '#ff0000' + SLACK_MESSAGE: | + Repository: ${{ github.repository }} diff --git a/.github/workflows/release-staging.yml b/.github/workflows/release-staging.yml new file mode 100644 index 0000000..dfc87c3 --- /dev/null +++ b/.github/workflows/release-staging.yml @@ -0,0 +1,24 @@ +name: Release for Staging + +on: + push: + branches: + - "main" + +jobs: + ci: + uses: ./.github/workflows/ci.yml + secrets: + HC_GITHUB_SSH_KEY: ${{ secrets.HC_GITHUB_SSH_KEY }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GENERAL_CI_WEBHOOK_URL }} + with: + should_push_image: true + + deploy-staging: + needs: ci + uses: ./.github/workflows/update-image-tag.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GENERAL_CI_WEBHOOK_URL }} + with: + commit_sha: ${{ github.sha }} + image_tag: ${{ needs.ci.outputs.app_version }} diff --git a/.github/workflows/update-image-tag.yml b/.github/workflows/update-image-tag.yml new file mode 100644 index 0000000..e5b177e --- /dev/null +++ b/.github/workflows/update-image-tag.yml @@ -0,0 +1,54 @@ +# Reuseable workflow to update image tag in values-version.yaml +name: Update Image Tag Workflow + +on: + workflow_call: + # GitHub Actions doesn't directly support passing secrets between workflows. + # Workflows that call reusable workflows in the same organization or + # enterprise can use the inherit keyword to implicitly pass the secrets. + # Therefore, we should pass the secret to the workflow_call workflow. + # Ref: https://github.com/orgs/community/discussions/23107 + secrets: + SLACK_WEBHOOK_URL: + required: true + inputs: + commit_sha: + description: 'base commit_sha to update' + required: true + type: string + image_tag: + description: 'Image tag to update' + required: true + type: string + +jobs: + update_image_tag: + runs-on: [self-hosted, general, small] + steps: + - name: Checkout commit + uses: actions/checkout@v4 + with: + ref: ${{ inputs.commit_sha }} + + - name: Update image tag in values-version.yaml + run: | + make update-image-tag IMAGE_TAG=${{ inputs.image_tag }} + + - name: Commit and push changes + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add deployment/kubernetes/charts/csl/values-version.yaml + git commit -m "Update image.tag via GitHub Action" + git tag ${{ inputs.image_tag }} + git push origin ${{ inputs.image_tag }} + git push -f origin HEAD:release-stage + + - name: slack notification + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_COLOR: '#ff0000' + SLACK_MESSAGE: | + Repository: ${{ github.repository }} diff --git a/.gitignore b/.gitignore index 69b1302..ad403de 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,8 @@ spec/fixtures/.DS_Store # Elasticsearch data es_data +# Rails vendor/ + +# Python virtualenv +.venv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..14e3976 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,79 @@ +# default_stages: [commit, push] +fail_fast: false +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + # check and format the syntax + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-json + # - id: check-yaml + exclude: ^deployment/kubernetes/charts + - id: check-toml + - id: check-xml + - id: double-quote-string-fixer + # environment checker + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + # check the git-related commit or object + - id: check-merge-conflict + - id: check-added-large-files + # check sensitive information + - id: debug-statements + - id: detect-private-key + + - repo: https://github.com/myint/autoflake + rev: v2.2.1 + hooks: + - id: autoflake + args: + - --in-place + - --remove-unused-variables + - --remove-all-unused-imports + - --ignore-init-module-imports # TODO: remove this + + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + args: + - --profile=black + - --filter-files + + # - repo: https://github.com/psf/black + # rev: 23.11.0 + # hooks: + # - id: black + # language_version: python3.11 + # args: + # - --target-version=py311 + # - --line-length=120 + # - --skip-string-normalization + + # - repo: https://github.com/zricethezav/gitleaks + # rev: v8.18.1 + # hooks: + # - id: gitleaks + # # run the detect mode and show all the leak credentials + # entry: gitleaks detect --verbose --redact + + - repo: https://github.com/hadolint/hadolint + rev: v2.12.1-beta + hooks: + - id: hadolint-docker + + - repo: https://github.com/asottile/pyupgrade + rev: v3.15.0 + hooks: + - id: pyupgrade + + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + + - repo: https://github.com/PyCQA/bandit + rev: 1.7.6 + hooks: + - id: bandit diff --git a/.rubocop.yml b/.rubocop.yml index fa519e1..70b3b2e 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -6,4 +6,4 @@ AllCops: TargetRubyVersion: 2.6 Layout/EmptyLinesAroundAccessModifier: - Enabled: false \ No newline at end of file + Enabled: false diff --git a/.ruby-gemset b/.ruby-gemset index 7749993..e57fe3a 100644 --- a/.ruby-gemset +++ b/.ruby-gemset @@ -1,2 +1 @@ csl - diff --git a/Gemfile b/Gemfile index 994e09d..86f32e9 100644 --- a/Gemfile +++ b/Gemfile @@ -6,18 +6,17 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" } ruby "~> 2.6" gem "active_elastic_job" -gem "aws-sdk", "~> 2" # https://github.com/tawan/active-elastic-job/pull/95 +gem 'aws-sdk', '~> 2' gem "elasticsearch" gem "elasticsearch-model" -gem "elasticsearch-persistence", "~> 7.0" -gem "jbuilder", "~> 2.11" -gem "puma", "~> 5.3" +gem "elasticsearch-persistence" +gem "jbuilder" +gem "puma" gem "rails" gem "sanitize" gem "htmlentities" gem "charlock_holmes" gem "iso_country_codes" -gem "public_suffix" # Reduces boot times through caching; required in config/boot.rb gem "bootsnap", ">= 1.4.2", require: false diff --git a/Gemfile.lock b/Gemfile.lock index 61d075c..754dd8c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -141,7 +141,6 @@ GEM marcel (1.0.2) method_source (1.0.0) mini_mime (1.1.5) - mini_portile2 (2.8.5) minitest (5.20.0) msgpack (1.7.2) multi_json (1.15.0) @@ -155,15 +154,14 @@ GEM net-smtp (0.4.0) net-protocol nio4r (2.7.0) - nokogiri (1.13.10) - mini_portile2 (~> 2.8.0) + nokogiri (1.13.10-x86_64-linux) racc (~> 1.4) parallel (1.24.0) parser (3.2.2.4) ast (~> 2.4.1) racc public_suffix (5.0.4) - puma (5.6.7) + puma (6.4.1) nio4r (~> 2.0) racc (1.7.3) rack (2.2.8) @@ -293,7 +291,7 @@ GEM zeitwerk (2.6.12) PLATFORMS - ruby + x86_64-linux DEPENDENCIES active_elastic_job @@ -303,13 +301,12 @@ DEPENDENCIES charlock_holmes elasticsearch elasticsearch-model - elasticsearch-persistence (~> 7.0) + elasticsearch-persistence htmlentities iso_country_codes - jbuilder (~> 2.11) + jbuilder listen (>= 3.0.5, < 3.6) - public_suffix - puma (~> 5.3) + puma rails rspec-rails (~> 4.0) rubocop-rails_config @@ -324,4 +321,4 @@ RUBY VERSION ruby 2.6.10p210 BUNDLED WITH - 1.17.2 + 2.4.22 diff --git a/LICENSE.md b/LICENSE.md index fd7042a..ad9dfdd 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f48964f --- /dev/null +++ b/Makefile @@ -0,0 +1,44 @@ +ECR_WEST_DOMAIN = 478041131377.dkr.ecr.us-west-2.amazonaws.com +IMAGE_NAME = gofreight/csl +NAMESPACE_POSTFIX = csl +RELEASE_NAME_POSTFIX = csl +CHART_PATH = deployment/kubernetes/charts/csl +GITHUB_REPO_OWNER = "hardcoretech" +GITHUB_REPO_NAME = csl + +.PHONY: test + +help: # Show the help menu + @printf "Usage: make [OPTION]\n" + @printf "\n" + @perl -nle 'print $$& if m{^[\w-]+:.*?#.*$$}' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?#"} {printf " %-18s %s\n", $$1, $$2}' + +build-docker-image: # Build the docker image + docker build -t $(IMAGE_NAME)-api:$${IMAGE_TAG:-latest} --platform linux/amd64 . + docker build -t $(IMAGE_NAME)-data-import:$${IMAGE_TAG:-latest} --platform linux/amd64 . + +push-docker-image: # Push the docker image to ECR + docker tag "$(IMAGE_NAME)-api:$${IMAGE_TAG}" "$(ECR_WEST_DOMAIN)/$(IMAGE_NAME):$${IMAGE_TAG}" + docker push "$(ECR_WEST_DOMAIN)/$(IMAGE_NAME)-api:$${IMAGE_TAG}" + docker tag "$(IMAGE_NAME)-data-import:$${IMAGE_TAG}" "$(ECR_WEST_DOMAIN)/$(IMAGE_NAME):$${IMAGE_TAG}" + docker push "$(ECR_WEST_DOMAIN)/$(IMAGE_NAME)-data-import:$${IMAGE_TAG}" + +test: # Run any type of tests you want + @echo "No tests yet" + +tear-down: # Prune the docker resources and deprecated images + docker system prune -f --volumes + docker image prune -a --force --filter "until=48h" + +update-image-tag: # Update the image tag in the values-version.yaml file + @echo "image:\n tag: $(IMAGE_TAG)\n" > $(CHART_PATH)/values-version.yaml + +gen-version: # Generate the version string + @echo v`date -u +"%y%m%d-%H%M"` + +run-dev: # Run the app at local environment + @docker compose -f docker/docker-compose.yml up -d --remove-orphans --no-color + +rm-dev: # Remove the app at local environment + @docker compose -f docker/docker-compose.yml down -v --remove-orphans diff --git a/README.md b/README.md index dc9839b..f30902b 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,37 @@ Consolidated Screening List (CSL) The Consolidated Screening List (CSL) is a list of parties for which the United States Government maintains restrictions on certain exports, reexports or transfers of items. This project creates a searchable API endpoint - around those restrictions as well as automated jobs to keep the data current. + around those restrictions as well as automated jobs to keep the data current. See [this site](https://www.export.gov/article?id=Consolidated-Screening-List) for more information about the CSL. # Installation +## Getting Started + +Use docker compose to provision development environment. + +### Set up development environment +``` +make run-dev +``` + +The development environment includes +- Elasticsearch +- Kibana +- Rails API: Search API to query data from elasticsearch +- Data import tool: Python script. Run one-time only + + +### Tear down development environment +``` +make rm-dev +``` + +--- + + +# Installation (Deprecated) + ### Ruby This repository has been tested against [Ruby 2.6](http://www.ruby-lang.org/en/downloads/). @@ -23,12 +49,12 @@ Install bundler and other required gems: gem install bundler bundle install - + The `charlock_holmes` gem requires the UCI libraries to be installed. If you are using Homebrew, it's probably as simple as this: - + brew install icu4c -More information about the gem can be found [here](https://github.com/brianmario/charlock_holmes) +More information about the gem can be found [here](https://github.com/brianmario/charlock_holmes) ### ElasticSearch @@ -49,17 +75,17 @@ You can also run it via Docker: Create the indexes: bundle exec rake db:create - + Fire up a web server to host the JSON API: bundle exec rails s - + Import data for a few sources: bundle exec rake ita:import_synchronously[ScreeningList::DplData] bundle exec rake ita:import_synchronously[ScreeningList::DtcData] bundle exec rake ita:import_synchronously[ScreeningList::SdnData] - + See `app/importers/screening_list/*_data.rb` for a complete list. Run some searches: @@ -69,7 +95,7 @@ Run some searches: ### Test Suite -Make sure Elasticsearch 7 is running locally on port 9200 before running the test suite: +Make sure Elasticsearch 7 is running locally on port 9200 before running the test suite: rake spec @@ -82,20 +108,20 @@ After running your tests, view the code coverage report by opening `coverage/ind CSL is essentially a Rails application and a set of recurring jobs that interact with an Elasticsearch cluster. There are many ways to orchestrate such a setup, including Elastic Beanstalk, OpsWorks/Chef, CloudFormation, Kubernetes , Capistrano, and of course just - installing and configuring resources manually. The recurring import jobs in this project are based - on [ActiveJob](https://edgeguides.rubyonrails.org/active_job_basics.html), which allows for different queueing backends. + installing and configuring resources manually. The recurring import jobs in this project are based + on [ActiveJob](https://edgeguides.rubyonrails.org/active_job_basics.html), which allows for different queueing backends. However, in this section we will focus on deploying to Elastic Beanstalk. #### Why Beanstalk - + Deploying via Beanstalk offers several benefits out of the box: * Capacity: Specify different instance types for both Worker tier and Web Server tier, and let Beanstalk automatically scale up/down based on time of day or some metric like CPU load. Beanstalk handles spreading workers across multiple availability zones. * Availability: Beanstalk monitors the health of the containers running the Web tier and Worker tier and - automatically reprovisions them in case of failure. -* Visibilty: Beanstalk provides a URL to access the Web Tier or you can provide your own custom domain. + automatically reprovisions them in case of failure. +* Visibilty: Beanstalk provides a URL to access the Web Tier or you can provide your own custom domain. * Crons: Asynchronous/recurring jobs are specified via using traditional cron syntax in a `cron.yaml` file that is part of the code repository versus some separate orchestration configuration. Unlike single-machine cron, the Worker tier is auto-scalable to accommodate any workload volume. @@ -105,35 +131,35 @@ Deploying via Beanstalk offers several benefits out of the box: with health checks. #### Architecture - + With Elastic Beanstalk, we have three components in a CSL deployment: the Web Server tier, the Worker tier, and a standard SQS queue: - + ![Rails app in Beanstalk](https://raw.githubusercontent.com/tawan/active-elastic-job/master/docs/architecture.png) - + * The Web Server tier hosts the JSON API for CSL search requests. * The Worker tier dequeues import jobs from the SQS queue and processes the tasks. * The SQS queue provides the queueing backend to asynchronously process the various import jobs for different screening lists as well as to upload CSV/TSV versions of the entire data set to an S3 bucket. Beanstalk automatically handles polling the queue for a task, dispatching it to a worker, and deleting the task from the - queue on successful completion of the work. The workflow is as follows: - + queue on successful completion of the work. The workflow is as follows: + ![Beanstalk SQS worker interaction](https://3.bp.blogspot.com/-SKsLwkbwetM/WCvMPSk4vAI/AAAAAAACiPk/XtcGAJgtUvAqZzgOED0MhFAQuF69lNUnACLcB/s1600/SQSD.gif) - + Outside of Elastic Beanstalk, CSL interacts with an Elasticsearch 7 cluster and AWS S3. Just as there are many ways - to deploy a web app, there are also many ways to run an Elasticsearch cluster, including hosted AWS + to deploy a web app, there are also many ways to run an Elasticsearch cluster, including hosted AWS Elasticsearch, hosted Elasticsearch from Elastic, orchestrated via OpsWorks/Chef, CloudFormation, Kubernetes, and manually provisioning a cluster. #### Deployment Even inside of Elastic Beanstalk, there are a number of ways to set up a Rails application and configure a deployment -. Please refer to the [documentation](https://docs.aws.amazon.com/en_pv/elasticbeanstalk/latest/dg/GettingStarted.html) to see which way is best for you. +. Please refer to the [documentation](https://docs.aws.amazon.com/en_pv/elasticbeanstalk/latest/dg/GettingStarted.html) to see which way is best for you. * Elasticsearch: Once you have your Elasticsearch 7 cluster provisioned, make a note of the cluster URL, usually available on port 9200. -* Specify the following environment variables in both your Web tier and Worker tier configurations under Software: +* Specify the following environment variables in both your Web tier and Worker tier configurations under Software: * `AWS_ACCESS_KEY_ID`, `AWS_REGION`, and `AWS_SECRET_ACCESS_KEY`: For accessing the S3 bucket where the CSV/TSV exports will be uploaded. * `ELASTICSEARCH_URL`: The access point for your Elasticsearch 7 cluster. @@ -143,7 +169,7 @@ Even inside of Elastic Beanstalk, there are a number of ways to set up a Rails a Once you have deployed CSL to Beanstalk (via Zip file, S3, etc), the various indices will automatically be created in Elasticsearch. The screening list imports and the CSV/TSV exports will begin based on the schedule defined in the - `cron.yaml` file. + `cron.yaml` file. ### Provenance diff --git a/app/models/settings.json b/app/models/settings.json index fefb572..f4eb956 100644 --- a/app/models/settings.json +++ b/app/models/settings.json @@ -49,4 +49,4 @@ } } } -} \ No newline at end of file +} diff --git a/config/credentials.yml.enc b/config/credentials.yml.enc index e654e8b..eb80e25 100644 --- a/config/credentials.yml.enc +++ b/config/credentials.yml.enc @@ -1 +1 @@ -qNvlkq67AbkyIx4OToaeg98OYlqkJ9zf59Pitll2wg33IRTkXt0Dg1GTQBSF10sRL9SIsTqj0ju3XiYV9F+vEo0S7n2bErCUJHREhu4Y8EBkI6NtZEyaouquMFs0/3PxrTQXTJd/F1X/0FAcq/cP0X50ccVlDDRPI5IqV/tJLfFLfD5AH8qccIBbEpiEBrMliX2dIyH/JShS8HFnxCbxOyogPSVSiHa1K2NG3inyx6PUU9DI6gyfz4zFtZGOfzykt32MiiFd52eTL1uHlBv0ENkvZ5/ZIdvlBRH428TfEWp9CYT5PId5OL8dmaC2kaBbV9Dqa4oRFIhUG1LzT+S4ueUuFJyQa0KaXw+eXr9c5KxQBsA/4k+BBMKQx57Et/ZfP2Dyk756/XwVSfDsSqpYIc/sCd4wvdqDKiNy--4gB+DLeqKxj3VQGh--0yDSFTBqXH28tuqFhc9qnw== \ No newline at end of file +qNvlkq67AbkyIx4OToaeg98OYlqkJ9zf59Pitll2wg33IRTkXt0Dg1GTQBSF10sRL9SIsTqj0ju3XiYV9F+vEo0S7n2bErCUJHREhu4Y8EBkI6NtZEyaouquMFs0/3PxrTQXTJd/F1X/0FAcq/cP0X50ccVlDDRPI5IqV/tJLfFLfD5AH8qccIBbEpiEBrMliX2dIyH/JShS8HFnxCbxOyogPSVSiHa1K2NG3inyx6PUU9DI6gyfz4zFtZGOfzykt32MiiFd52eTL1uHlBv0ENkvZ5/ZIdvlBRH428TfEWp9CYT5PId5OL8dmaC2kaBbV9Dqa4oRFIhUG1LzT+S4ueUuFJyQa0KaXw+eXr9c5KxQBsA/4k+BBMKQx57Et/ZfP2Dyk756/XwVSfDsSqpYIc/sCd4wvdqDKiNy--4gB+DLeqKxj3VQGh--0yDSFTBqXH28tuqFhc9qnw== diff --git a/config/environments/test.rb b/config/environments/test.rb index d446287..7436c58 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -5,7 +5,7 @@ Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. - + config.cache_classes = false # Do not eager load code on boot. This avoids loading your whole application diff --git a/config/routes.rb b/config/routes.rb index 24357c7..669996b 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,5 @@ Rails.application.routes.draw do get 'consolidated_screening_list/search', to: 'consolidated#search', defaults: {format: :json} get 'v2/consolidated_screening_list/search', to: 'consolidated#search', defaults: {format: :json} - get '/health', to: 'healthcheck#index' + get 'health', to: 'healthcheck#index' end diff --git a/config/secrets.yml b/config/secrets.yml index 81b55d5..08ca13e 100644 --- a/config/secrets.yml +++ b/config/secrets.yml @@ -1,2 +1,2 @@ production: - secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> + secret_key_base: <%= ENV["ENC_SECRET_KEY_BASE"] %> diff --git a/config/spring.rb b/config/spring.rb index e46c08a..dc6385b 100644 --- a/config/spring.rb +++ b/config/spring.rb @@ -11,4 +11,4 @@ load path + '/ruby-debug-ide/multiprocess/starter.rb' end end -end \ No newline at end of file +end diff --git a/deployment/kubernetes/charts/csl/Chart.yaml b/deployment/kubernetes/charts/csl/Chart.yaml new file mode 100644 index 0000000..22e5c6c --- /dev/null +++ b/deployment/kubernetes/charts/csl/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: csl +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.1.0" diff --git a/deployment/kubernetes/charts/csl/templates/_helpers.tpl b/deployment/kubernetes/charts/csl/templates/_helpers.tpl new file mode 100644 index 0000000..98da09e --- /dev/null +++ b/deployment/kubernetes/charts/csl/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "project.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "project.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "project.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "project.labels" -}} +helm.sh/chart: {{ include "project.chart" . }} +{{ include "project.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "project.selectorLabels" -}} +app.kubernetes.io/name: {{ include "project.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/deployment/kubernetes/charts/csl/templates/configmap-env.yaml b/deployment/kubernetes/charts/csl/templates/configmap-env.yaml new file mode 100644 index 0000000..7a215fe --- /dev/null +++ b/deployment/kubernetes/charts/csl/templates/configmap-env.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-env + labels: + {{- include "project.labels" . | nindent 4 }} +data: + {{- range $key, $val := .Values.infraEnv }} + {{ $key }}: {{ $val | quote }} + {{- end }} + + {{- range $key, $val := .Values.env }} + {{ $key }}: {{ $val | quote }} + {{- end }} diff --git a/deployment/kubernetes/charts/csl/templates/configmap-migrate-env.yaml b/deployment/kubernetes/charts/csl/templates/configmap-migrate-env.yaml new file mode 100644 index 0000000..c9e3942 --- /dev/null +++ b/deployment/kubernetes/charts/csl/templates/configmap-migrate-env.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-migrate-env + labels: + {{- include "project.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" +data: + {{- range $key, $val := .Values.infraEnv }} + {{ $key }}: {{ $val | quote }} + {{- end }} + + {{- range $key, $val := .Values.env }} + {{ $key }}: {{ $val | quote }} + {{- end }} diff --git a/deployment/kubernetes/charts/csl/templates/cronjob.yaml b/deployment/kubernetes/charts/csl/templates/cronjob.yaml new file mode 100644 index 0000000..86e84e2 --- /dev/null +++ b/deployment/kubernetes/charts/csl/templates/cronjob.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: "{{ .Release.Name }}-cron-{{ .name }}" + labels: + {{- include "project.labels" . | nindent 4 }} +spec: + revisionHistoryLimit: 1 + schedule: "*/30 * * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + restartPolicy: OnFailure + containers: + - name: data-import + image: "{{ .Values.cronjob.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.cronjob.image.pullPolicy }} + resources: + requests: + cpu: {{ .Values.cronjob.resources.requests.cpu }} + memory: {{ .Values.cronjob.resources.requests.memory }} + limits: + cpu: {{ .Values.cronjob.resources.limits.cpu }} + memory: {{ .Values.cronjob.resources.limits.memory }} + envFrom: + - configMapRef: + name: {{ .Release.Name }}-env diff --git a/deployment/kubernetes/charts/csl/templates/deployment-app.yaml b/deployment/kubernetes/charts/csl/templates/deployment-app.yaml new file mode 100644 index 0000000..d647d21 --- /dev/null +++ b/deployment/kubernetes/charts/csl/templates/deployment-app.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }} + labels: + {{- include "project.labels" . | nindent 4 }} +spec: + revisionHistoryLimit: 2 + {{- if not .Values.app.scaledObject.enabled }} + replicas: {{ .Values.app.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "project.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.app.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "project.selectorLabels" . | nindent 8 }} + spec: + securityContext: + {{- toYaml .Values.app.podSecurityContext | nindent 8 }} + nodeSelector: + {{- toYaml .Values.app.nodeSelector | nindent 8 }} + containers: + - name: app + securityContext: + {{- toYaml .Values.app.containerSecurityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["bundle", "exec", "rails", "s", "-b", "0.0.0.0"] + envFrom: + - configMapRef: + name: {{ .Release.Name }}-env + ports: + - name: http + containerPort: {{ .Values.app.containerPort }} + protocol: TCP + resources: + {{- toYaml .Values.app.resources | nindent 12 }} + + {{- if .Values.app.startupProbe }} + startupProbe: + {{- toYaml .Values.app.startupProbe | nindent 12 }} + {{- end }} + {{- if .Values.app.readinessProbe }} + readinessProbe: + {{- toYaml .Values.app.readinessProbe | nindent 12 }} + {{- end }} + {{- if .Values.app.livenessProbe }} + livenessProbe: + {{- toYaml .Values.app.livenessProbe | nindent 12 }} + {{- end }} diff --git a/deployment/kubernetes/charts/csl/templates/deployment-celery-beat.yaml b/deployment/kubernetes/charts/csl/templates/deployment-celery-beat.yaml new file mode 100644 index 0000000..e9835cc --- /dev/null +++ b/deployment/kubernetes/charts/csl/templates/deployment-celery-beat.yaml @@ -0,0 +1,34 @@ +{{- if .Values.celeryBeat.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-celery-beat + labels: + {{- include "project.labels" . | nindent 4 }} +spec: + revisionHistoryLimit: 2 + replicas: 1 + selector: + matchLabels: + {{- include "project.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: celery-beat + template: + metadata: + labels: + {{- include "project.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: celery-beat + spec: + restartPolicy: Always + terminationGracePeriodSeconds: 30 + nodeSelector: + {{- toYaml .Values.celeryBeat.nodeSelector | nindent 8 }} + containers: + - name: celery-beat + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ .Release.Name }}-env + resources: + {{- toYaml .Values.celeryBeat.resources | nindent 12 }} +{{- end }} diff --git a/deployment/kubernetes/charts/csl/templates/deployment-celery.yaml b/deployment/kubernetes/charts/csl/templates/deployment-celery.yaml new file mode 100644 index 0000000..0d8334b --- /dev/null +++ b/deployment/kubernetes/charts/csl/templates/deployment-celery.yaml @@ -0,0 +1,60 @@ +{{- range $celery := .Values.celeries }} + +{{- if $celery.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $.Release.Name }}-{{ $celery.name }} + labels: + {{- include "project.labels" $ | nindent 4 }} +spec: + revisionHistoryLimit: 2 + {{- if not $celery.scaledObject.enabled }} + replicas: {{ $celery.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "project.selectorLabels" $ | nindent 6 }} + app.kubernetes.io/component: {{ $celery.name }} + template: + metadata: + {{- with $celery.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "project.selectorLabels" $ | nindent 8 }} + app.kubernetes.io/component: {{ $celery.name }} + spec: + securityContext: + {{- toYaml $celery.podSecurityContext | nindent 8 }} + nodeSelector: + {{- toYaml $celery.nodeSelector | nindent 8 }} + containers: + - name: {{ $celery.name }} + securityContext: + {{- toYaml $celery.containerSecurityContext | nindent 12 }} + image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag }}" + imagePullPolicy: {{ $.Values.image.pullPolicy }} + command: + {{- toYaml $celery.command | nindent 12 }} + envFrom: + - configMapRef: + name: {{ $.Release.Name }}-env + resources: + {{- toYaml $celery.resources | nindent 12 }} + + {{- if $celery.startupProbe }} + startupProbe: + {{- toYaml $celery.startupProbe | nindent 12 }} + {{- end }} + {{- if $celery.readinessProbe }} + readinessProbe: + {{- toYaml $celery.readinessProbe | nindent 12 }} + {{- end }} + {{- if $celery.livenessProbe }} + livenessProbe: + {{- toYaml $celery.livenessProbe | nindent 12 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/deployment/kubernetes/charts/csl/templates/ingress-internal.yaml b/deployment/kubernetes/charts/csl/templates/ingress-internal.yaml new file mode 100644 index 0000000..96dda4c --- /dev/null +++ b/deployment/kubernetes/charts/csl/templates/ingress-internal.yaml @@ -0,0 +1,26 @@ +{{- if .Values.internalIngress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Release.Name }}-internal + namespace: {{ .Release.Namespace }} + labels: + {{- include "project.labels" . | nindent 4 }} + {{- with .Values.internalIngress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: {{ .Values.internalIngress.className }} + rules: + - host: {{ .Values.internalIngress.host | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ .Release.Name }} + port: + number: {{ .Values.service.port }} +{{- end }} diff --git a/deployment/kubernetes/charts/csl/templates/ingress.yaml b/deployment/kubernetes/charts/csl/templates/ingress.yaml new file mode 100644 index 0000000..c97785d --- /dev/null +++ b/deployment/kubernetes/charts/csl/templates/ingress.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "project.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.className }} + rules: + - host: {{ .Values.ingress.host | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ .Release.Name }} + port: + number: {{ .Values.service.port }} diff --git a/deployment/kubernetes/charts/csl/templates/job-migrate-db.yaml b/deployment/kubernetes/charts/csl/templates/job-migrate-db.yaml new file mode 100644 index 0000000..66c3b38 --- /dev/null +++ b/deployment/kubernetes/charts/csl/templates/job-migrate-db.yaml @@ -0,0 +1,28 @@ +{{- if .Values.app.migratedJob.enabled -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-migrate + labels: + {{- include "project.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-4" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + template: + spec: + nodeSelector: + {{- toYaml .Values.app.nodeSelector | nindent 8 }} + containers: + - name: migrate-db + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + {{- toYaml .Values.app.migratedJob.command | nindent 12 }} + envFrom: + - configMapRef: + name: {{ .Release.Name }}-migrate-env + restartPolicy: Never + backoffLimit: 1 +{{- end }} diff --git a/deployment/kubernetes/charts/csl/templates/scaledObject-app.yaml b/deployment/kubernetes/charts/csl/templates/scaledObject-app.yaml new file mode 100644 index 0000000..e52e66e --- /dev/null +++ b/deployment/kubernetes/charts/csl/templates/scaledObject-app.yaml @@ -0,0 +1,17 @@ +{{- if .Values.app.scaledObject.enabled }} +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ .Release.Name }} + labels: + {{- include "project.labels" . | nindent 4 }} +spec: + scaleTargetRef: + name: {{ .Release.Name }} + minReplicaCount: {{ .Values.app.scaledObject.minReplicaCount }} + maxReplicaCount: {{ .Values.app.scaledObject.maxReplicaCount }} + triggers: +{{- with .Values.app.scaledObject.triggers }} +{{ toYaml . | indent 2 }} +{{ end }} +{{- end }} diff --git a/deployment/kubernetes/charts/csl/templates/scaledObject-celery.yaml b/deployment/kubernetes/charts/csl/templates/scaledObject-celery.yaml new file mode 100644 index 0000000..94e3c96 --- /dev/null +++ b/deployment/kubernetes/charts/csl/templates/scaledObject-celery.yaml @@ -0,0 +1,20 @@ +{{- range $celery := .Values.celeries }} + +{{- if $celery.scaledObject.enabled }} +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ $.Release.Name }}-celery + labels: + {{- include "project.labels" $ | nindent 4 }} +spec: + scaleTargetRef: + name: {{ $.Release.Name }}-{{ $celery.name }} + minReplicaCount: {{ $celery.scaledObject.minReplicaCount }} + maxReplicaCount: {{ $celery.scaledObject.maxReplicaCount }} + triggers: +{{- with $celery.scaledObject.triggers }} +{{ toYaml . | indent 2 }} +{{ end }} +{{- end }} +{{- end }} diff --git a/deployment/kubernetes/charts/csl/templates/service-app.yaml b/deployment/kubernetes/charts/csl/templates/service-app.yaml new file mode 100644 index 0000000..ee14227 --- /dev/null +++ b/deployment/kubernetes/charts/csl/templates/service-app.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "project.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "project.selectorLabels" . | nindent 4 }} diff --git a/deployment/kubernetes/charts/csl/values-infra-cred-prod.yaml b/deployment/kubernetes/charts/csl/values-infra-cred-prod.yaml new file mode 100644 index 0000000..fd5d6cd --- /dev/null +++ b/deployment/kubernetes/charts/csl/values-infra-cred-prod.yaml @@ -0,0 +1,17 @@ +infraEnv: + ELASTICSEARCH_URL: ENC[AES256_GCM,data:IfvHqva3aP/26H96CTy9T8o3MGoQy5bvMl5v5qEobN9OqyH16C4XP/KEHsg421gJ3sD4D+XOUh+qsqOJz9ZoMH0lyxa5ztMaVr0bVMfYSA2Xp7mfitTGcl0ivXEyVabXbP4EUA==,iv:nI1fZDHUrXkX0lirsXvoRiGHPvcS8SfplpbImnku1gE=,tag:SdoD/2O3urTgsZ+v5rt1zg==,type:str] +sops: + kms: + - arn: arn:aws:kms:us-west-2:478041131377:key/3531ff74-9fc2-4088-982f-e24170377116 + created_at: "2024-01-05T08:30:26Z" + enc: AQICAHg0Er1cllAzsJBfqLJDdcEGCgPovBlIgMnXYfrsx9VtQwHEugOamNErCel7vtrDcvtGAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM4BHz/dHGzucTjxKsAgEQgDskNaZlHeNmaMtZpi3WV7I/CcfNObj8O2WIII5TknDXDL9+HI4TvodgXLeNNwTtZauweS5SXvT0b4yyDQ== + aws_profile: "" + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2024-01-05T08:30:26Z" + mac: ENC[AES256_GCM,data:D+R5MQ+FwNy4/LjNZjco9Dru995TApViqUaqpiUxHOysifKGPdJDf/RaCHmsE3hs5CnuNmv7VEa+4qStjuREOXZfl7OUwZAGAx00TmIqlMiqOPPt4tNeuwqKJa+KqIU2e7AYyQ3B6QPB8CpKGf9xqbGrjiPv8ibyF4jmIxsrN5s=,iv:Fqbwd5V/L3UQUjJNGn7y2uBC7tDkpSVT4hqPYsNM/XU=,tag:E511SaluH5XmoTC0LsM7Fw==,type:str] + pgp: [] + encrypted_regex: ^infraEnv + version: 3.8.1 diff --git a/deployment/kubernetes/charts/csl/values-infra-cred-stage.yaml b/deployment/kubernetes/charts/csl/values-infra-cred-stage.yaml new file mode 100644 index 0000000..0e2088b --- /dev/null +++ b/deployment/kubernetes/charts/csl/values-infra-cred-stage.yaml @@ -0,0 +1,17 @@ +infraEnv: + ELASTICSEARCH_URL: ENC[AES256_GCM,data:UhcIjQieTXbSkOK7abwlmx/26uvvFdxcmPtzEiS55kYa3Csl5ySO0Z8I0ykISqduPfTRKv4pUlU/X27bFjUWDc301V7lW1lfszdu7socsb2pwhOB13yEJZyhKW4S2zY=,iv:bdPRkw7qEcdaqumJZlO7ItbZ57G6UltgDlQRDGpdzrM=,tag:DbU6SPkpx7Xz1Pseej4aFg==,type:str] +sops: + kms: + - arn: arn:aws:kms:us-west-2:478041131377:key/3531ff74-9fc2-4088-982f-e24170377116 + created_at: "2024-01-05T08:30:23Z" + enc: AQICAHg0Er1cllAzsJBfqLJDdcEGCgPovBlIgMnXYfrsx9VtQwHpzphcnjw8Cc6Y8hFee5AuAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMPDedXMglc255in9KAgEQgDu5Q0jCEpFWDZgVLjTIZ7BSBul/qhf1FMEWpqxbk/7F9WTkyVZOsREkRZTqB8zRhvMDzLGCbvZptEL0zg== + aws_profile: "" + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2024-01-05T08:30:26Z" + mac: ENC[AES256_GCM,data:vYJAUuxUfdbqTp3oel+1pW3WjrQNz90KCaoyLlMQL9DrHglo6Yrzw94vnoPnz9Bu709XiyigPQexjwwiCOw1btZtDmz7Wg3X16955dSqXZcnNgwtCuev/UD+ctTTminTp9qc9rzy/6Kja15yLSLGso/qKtlzamY0XE9s3/4nd60=,iv:vnOjccDzk90Vke1/XlYoLi0Hw+ziVxgAIjqlcFFpy8g=,tag:LarLc0T1YaQ5w54187xfBA==,type:str] + pgp: [] + encrypted_regex: ^infraEnv + version: 3.8.1 diff --git a/deployment/kubernetes/charts/csl/values-infra-prod.yaml b/deployment/kubernetes/charts/csl/values-infra-prod.yaml new file mode 100644 index 0000000..41fd353 --- /dev/null +++ b/deployment/kubernetes/charts/csl/values-infra-prod.yaml @@ -0,0 +1,162 @@ +image: + repository: 478041131377.dkr.ecr.us-west-2.amazonaws.com/gofreight/csl-api + pullPolicy: Always + +service: + type: ClusterIP + port: 80 +ingress: + className: alb + host: denied-party.gofreight.co + annotations: + alb.ingress.kubernetes.io/load-balancer-name: old-prod-alb + alb.ingress.kubernetes.io/group.name: old-prod + alb.ingress.kubernetes.io/healthcheck-path: /health + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:478041131377:certificate/d3cd776b-ccfd-46c4-a073-d8961f44c4f4 + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS": 443}, {"HTTPS": 11999}]' + alb.ingress.kubernetes.io/ssl-redirect: "443" + +# Enable internal ingress if we want to enable CN network +internalIngress: + enabled: false + className: internal-nginx + host: denied-party.gofreight.co + annotations: + nginx.ingress.kubernetes.io/proxy-read-timeout: "60" + nginx.ingress.kubernetes.io/proxy-body-size: "2m" + +app: + replicaCount: 3 + containerPort: 3000 + annotations: {} + podSecurityContext: {} + containerSecurityContext: {} + nodeSelector: + Service: "Application" + Type: "General" + resources: + requests: + cpu: 300m + memory: 1024Mi + limits: + cpu: 300m + memory: 1024Mi + startupProbe: {} + readinessProbe: + httpGet: + path: "/health" + port: 3000 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: "/health" + port: 3000 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + migratedJob: + enabled: false + command: + - ./manage.py + - migrate + scaledObject: + enabled: false + minReplicaCount: 3 + maxReplicaCount: 10 + + triggers: + - type: prometheus + metadata: + serverAddress: http://kube-prometheus-stack-prometheus.monitoring:9090 + metricName: pod_cpu_utilization_over_pod_request + threshold: "70" + query: round(100 * sum(rate(container_cpu_usage_seconds_total{container="app",namespace=~"{{ .Release.Namespace }}",pod=~"{{ .Release.Name }}-.*"}[1m]))/avg(kube_pod_container_resource_requests{container="app",namespace=~"{{ .Release.Namespace }}",pod=~"{{ .Release.Name }}-.*",resource="cpu"})) + +cronjob: + image: + repository: 478041131377.dkr.ecr.us-west-2.amazonaws.com/gofreight/csl-data-import + pullPolicy: Always + resources: + requests: + cpu: 300m + memory: 512Mi + limits: + cpu: 300m + memory: 512Mi + +celeries: + - name: celery + enabled: false + replicaCount: 3 + command: + - sleep + - infinity + annotations: {} + podSecurityContext: {} + containerSecurityContext: {} + nodeSelector: + Service: "Application" + Type: "Memory" + resources: {} + # startupProbe: {} + # readinessProbe: + # httpGet: + # path: "/" + # port: 3000 + # scheme: HTTP + # initialDelaySeconds: 10 + # periodSeconds: 10 + # timeoutSeconds: 1 + # successThreshold: 1 + # failureThreshold: 3 + # livenessProbe: {} + scaledObject: + enabled: false + minReplicaCount: 3 + maxReplicaCount: 10 + triggers: [] + # - type: aws-cloudwatch + # metadata: + # namespace: ContainerInsights + # metricName: pod_memory_utilization_over_pod_limit + # dimensionName: ClusterName;Namespace;PodName + # dimensionValue: xxx + # minMetricValue: xxx + # targetMetricValue: xxx + # metricCollectionTime: xxx + # metricStatPeriod: xxx + # metricStat: Sum + # awsRegion: "us-west-2" + # identityOwner: operator + # - type: aws-cloudwatch + # metadata: + # namespace: ContainerInsights + # metricName: pod_cpu_utilization_over_pod_limit + # dimensionName: ClusterName;Namespace;PodName + # dimensionValue: xxx + # minMetricValue: xxx + # targetMetricValue: xxx + # metricCollectionTime: xxx + # metricStatPeriod: xxx + # metricStat: Sum + # awsRegion: "us-west-2" + # identityOwner: operator +celeryBeat: + enabled: false + command: + - sleep + - infinity + nodeSelector: + Service: "Application" + Type: "Memory" + resources: {} diff --git a/deployment/kubernetes/charts/csl/values-infra-stage.yaml b/deployment/kubernetes/charts/csl/values-infra-stage.yaml new file mode 100644 index 0000000..5ee391a --- /dev/null +++ b/deployment/kubernetes/charts/csl/values-infra-stage.yaml @@ -0,0 +1,156 @@ +image: + repository: 478041131377.dkr.ecr.us-west-2.amazonaws.com/gofreight/csl-api + pullPolicy: Always + +service: + type: ClusterIP + port: 80 +ingress: + className: alb + host: denied-party-stage.gofreight.co + annotations: + alb.ingress.kubernetes.io/load-balancer-name: old-stage-alb + alb.ingress.kubernetes.io/group.name: old-stage + alb.ingress.kubernetes.io/healthcheck-path: /health + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:478041131377:certificate/d3cd776b-ccfd-46c4-a073-d8961f44c4f4 + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS": 443}, {"HTTPS": 11999}]' + alb.ingress.kubernetes.io/ssl-redirect: "443" + +# Enable internal ingress if we want to enable CN network +internalIngress: + enabled: false + className: internal-nginx + host: denied-party-stage.gofreight.co + annotations: + nginx.ingress.kubernetes.io/proxy-read-timeout: "60" + nginx.ingress.kubernetes.io/proxy-body-size: "2m" + +app: + replicaCount: 3 + containerPort: 3000 + annotations: {} + podSecurityContext: {} + containerSecurityContext: {} + nodeSelector: + Service: "Application" + Type: "General" + resources: {} + startupProbe: {} + readinessProbe: + httpGet: + path: "/health" + port: 3000 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: "/health" + port: 3000 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + migratedJob: + enabled: false + command: + - ./manage.py + - migrate + scaledObject: + enabled: false + minReplicaCount: 3 + maxReplicaCount: 10 + + triggers: + - type: prometheus + metadata: + serverAddress: http://kube-prometheus-stack-prometheus.monitoring:9090 + metricName: pod_cpu_utilization_over_pod_request + threshold: "70" + query: round(100 * sum(rate(container_cpu_usage_seconds_total{container="app",namespace=~"{{ .Release.Namespace }}",pod=~"{{ .Release.Name }}-.*"}[1m]))/avg(kube_pod_container_resource_requests{container="app",namespace=~"{{ .Release.Namespace }}",pod=~"{{ .Release.Name }}-.*",resource="cpu"})) + +cronjob: + image: + repository: 478041131377.dkr.ecr.us-west-2.amazonaws.com/gofreight/csl-data-import + pullPolicy: Always + resources: + requests: + cpu: 300m + memory: 512Mi + limits: + cpu: 300m + memory: 512Mi + +celeries: + - name: celery + enabled: false + replicaCount: 3 + command: + - sleep + - infinity + annotations: {} + podSecurityContext: {} + containerSecurityContext: {} + nodeSelector: + Service: "Application" + Type: "Memory" + resources: {} + # startupProbe: {} + # readinessProbe: + # httpGet: + # path: "/" + # port: 3000 + # scheme: HTTP + # initialDelaySeconds: 10 + # periodSeconds: 10 + # timeoutSeconds: 1 + # successThreshold: 1 + # failureThreshold: 3 + # livenessProbe: {} + scaledObject: + enabled: false + minReplicaCount: 3 + maxReplicaCount: 10 + triggers: [] + # - type: aws-cloudwatch + # metadata: + # namespace: ContainerInsights + # metricName: pod_memory_utilization_over_pod_limit + # dimensionName: ClusterName;Namespace;PodName + # dimensionValue: xxx + # minMetricValue: xxx + # targetMetricValue: xxx + # metricCollectionTime: xxx + # metricStatPeriod: xxx + # metricStat: Sum + # awsRegion: "us-west-2" + # identityOwner: operator + # - type: aws-cloudwatch + # metadata: + # namespace: ContainerInsights + # metricName: pod_cpu_utilization_over_pod_limit + # dimensionName: ClusterName;Namespace;PodName + # dimensionValue: xxx + # minMetricValue: xxx + # targetMetricValue: xxx + # metricCollectionTime: xxx + # metricStatPeriod: xxx + # metricStat: Sum + # awsRegion: "us-west-2" + # identityOwner: operator +celeryBeat: + enabled: false + command: + - sleep + - infinity + nodeSelector: + Service: "Application" + Type: "Memory" + resources: {} diff --git a/deployment/kubernetes/charts/csl/values-prod.yaml b/deployment/kubernetes/charts/csl/values-prod.yaml new file mode 100644 index 0000000..1541275 --- /dev/null +++ b/deployment/kubernetes/charts/csl/values-prod.yaml @@ -0,0 +1,18 @@ +env: + ENVIRONMENT: production + ENC_SECRET_KEY_BASE: ENC[AES256_GCM,data:8v3v8DO4FXEzXVZgWjPtWDJL+gcvXWTis7rp2JPh8yQ=,iv:34yDnJDRg8AXzRtSERAeScGeitj/oV7/AyZ4AS6ut3s=,tag:Sq3SPPPlXtlLbNYRAt+byw==,type:str] +sops: + kms: + - arn: arn:aws:kms:us-west-2:478041131377:key/d222752c-be9f-4945-b17f-a52f694bc8ef + created_at: "2024-01-05T08:31:07Z" + enc: AQICAHjrC1N+BaDn8oOmDbPP/08kRoh4Co3gLg7rtUmicxHcsQG+LMju4t3atT+l5VQSSIXkAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMX6ajGKHE+G9Tes+EAgEQgDuXyUD1z7eZlOF6fNMQjJkBu9mUzdk8xpyizIDdE8W9iUCuocYZk9sVepHQLgfzYnbN7Ojs8bVhDtHVyw== + aws_profile: "" + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2024-01-05T08:31:08Z" + mac: ENC[AES256_GCM,data:AJ/Jh0tnT7fRM98F+LgMnpxZBk1Y6dDikN60HTiNaPdCIQOBEYUvcc6LKBhJpV2vg9sueDyGd5qIsD4tIpuTolpQYxvpQJR5+H4bTvR2GvGSCh7K/gaFVQ1V7XfFM/Golnx7NEdYBRnpIinL572kDPGk0z0GCsUwyXK0SnuRKBQ=,iv:CY4wQ+oM9efEOnpbiQxXVJyH1wp2SHTzSes1IIFKDn4=,tag:Pw7xopYxLqgrcvSyA32PrA==,type:str] + pgp: [] + encrypted_regex: ^ENC + version: 3.8.1 diff --git a/deployment/kubernetes/charts/csl/values-stage.yaml b/deployment/kubernetes/charts/csl/values-stage.yaml new file mode 100644 index 0000000..fb2856d --- /dev/null +++ b/deployment/kubernetes/charts/csl/values-stage.yaml @@ -0,0 +1,18 @@ +env: + ENVIRONMENT: staging + ENC_SECRET_KEY_BASE: ENC[AES256_GCM,data:gXrlptWWV4W1i9fNHgufj0StNWTW9yWBVxFtpD884T4=,iv:6kYfJ2gzC+AWvQ6YVNKPV5TFhQhAD8sn9TrKE1LlfK0=,tag:jcdLpUNd8Dv0zqgmyOWiew==,type:str] +sops: + kms: + - arn: arn:aws:kms:us-west-2:478041131377:key/d222752c-be9f-4945-b17f-a52f694bc8ef + created_at: "2024-01-05T08:31:08Z" + enc: AQICAHjrC1N+BaDn8oOmDbPP/08kRoh4Co3gLg7rtUmicxHcsQGn7TKhm/Igw3xgfxOVp44mAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMHcA22a1PZq5MyUI+AgEQgDvy5cw9MuEpjL2zI5pVQ/V1fwmqYcG+U9KGz6HeQiKMUPhxot2gZiD8sxM8lxBtQHS9ic70Ybltx42x4g== + aws_profile: "" + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2024-01-05T08:31:08Z" + mac: ENC[AES256_GCM,data:2if2vKqJFSi29xcYgbOocBpcvucSx9DgLDZHMngZzLPo2imYi034B6e7pAWqW1Q/Kd1oT2KR8UFOHNI2uOVo+SGlfzz1sbNMfmib5jUtdSquB23s+ZJibiqXB5uk86BkGXkEfLFF7kDBoiWnpZBMcYM/ra+PkeH9hwex1u2Zsn4=,iv:v72kvA9cFeMWOEZQ7G6W7hiz8Td7+kmXbXPJ5Sv/hO0=,tag:X+7w8bxuhC21pwPy5K4/VA==,type:str] + pgp: [] + encrypted_regex: ^ENC + version: 3.8.1 diff --git a/deployment/kubernetes/charts/csl/values-version.yaml b/deployment/kubernetes/charts/csl/values-version.yaml new file mode 100644 index 0000000..c300ecd --- /dev/null +++ b/deployment/kubernetes/charts/csl/values-version.yaml @@ -0,0 +1,2 @@ +image: + tag: "" diff --git a/docker/csl-python/Dockerfile b/docker/csl-python/Dockerfile index 886b1ea..14cc18a 100644 --- a/docker/csl-python/Dockerfile +++ b/docker/csl-python/Dockerfile @@ -1,5 +1,5 @@ ######################################## -# Backend Builder stage +# Builder stage ######################################## FROM python:3.11-bookworm AS builder # python @@ -15,11 +15,6 @@ ENV PYTHONUNBUFFERED=1 \ WORKDIR /repo -RUN set -ex \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* - COPY pyproject.toml poetry.lock ./ # Upgrade pip and install poetry @@ -32,7 +27,7 @@ RUN pip install --no-cache-dir pip==23.3.1 poetry==1.7.1 && \ find /repo/wheels/ -name '*.whl' > /repo/requirements-wheel.txt ######################################## -# Backend Final stage +# Final stage ######################################## FROM python:3.11-slim-bookworm diff --git a/docker/csl-python/entrypoint.sh b/docker/csl-python/entrypoint.sh index 0dd96c7..3a56f5e 100755 --- a/docker/csl-python/entrypoint.sh +++ b/docker/csl-python/entrypoint.sh @@ -1,6 +1,7 @@ #!/bin/sh -/usr/bin/wait-for-it.sh elastic:9200 -t 30 -- echo "Elasticsearch server is ready" +/usr/local/bin/wait-for-it.sh elastic:9200 -t 30 -- echo "Elasticsearch server is ready" +/usr/local/bin/wait-for-it.sh rails:3000 -t 60 -- echo "DB migration is ready" echo "********** Start CSL-python import script **********" python import_source.py diff --git a/docker/csl-python/import_source.py b/docker/csl-python/import_source.py index ac9afcb..54dfce2 100644 --- a/docker/csl-python/import_source.py +++ b/docker/csl-python/import_source.py @@ -1,22 +1,21 @@ import logging import os import re -import time import traceback +import warnings from datetime import datetime import requests from elasticsearch import Elasticsearch -from elasticsearch.exceptions import ConnectionError +from elasticsearch.exceptions import ConnectionError, ElasticsearchWarning -import warnings -from elasticsearch.exceptions import ElasticsearchWarning warnings.simplefilter('ignore', ElasticsearchWarning) STOP_WORDS = {'and', 'the', 'los'} COMMON_WORDS = { - 'co', 'company', 'corp', 'corporation', 'inc', 'incorporated', 'limited', 'ltd', 'mr', 'mrs', 'ms', - 'organization', 'sa', 'sas', 'llc', 'university', 'univ' + 'co', 'company', 'corp', 'corporation', 'inc', 'incorporated', 'limited', + 'ltd', 'mr', 'mrs', 'ms', 'organization', 'sa', 'sas', 'llc', 'university', + 'univ' } @@ -64,13 +63,13 @@ def has_any_common_words(name): def make_names_with_common(doc, prefix): doc[f'{prefix}_no_ws_with_common'] = doc[f'{prefix}_idx'].replace(' ', '') - doc[f'{prefix}_no_ws_rev_with_common'] = name_rev(doc[f'{prefix}_idx']).replace(' ', '') + doc[f'{prefix}_no_ws_rev_with_common'] = name_rev(doc[f'{prefix}_idx']).replace(' ', '') # noqa doc[f'{prefix}_idx'] = remove_words(doc[f'{prefix}_idx'], COMMON_WORDS) def make_alt_names_with_common(doc): doc['alt_no_ws_with_common'] = [n.replace(' ', '') for n in doc['alt_idx']] - doc['alt_no_ws_rev_with_common'] = [name_rev(n.replace(' ', '')) for n in doc['alt_idx']] + doc['alt_no_ws_rev_with_common'] = [name_rev(n.replace(' ', '')) for n in doc['alt_idx']] # noqa doc['alt_idx'] = [remove_words(n, COMMON_WORDS) for n in doc['alt_idx']] @@ -83,7 +82,7 @@ def name_rev(name): def make_full_addresses(doc): if doc.get('addresses'): for addr in doc.get('addresses'): - addr_info = [addr[k] for k in ['address', 'city', 'country', 'postal_code', 'state'] if addr[k]] + addr_info = [addr[k] for k in ['address', 'city', 'country', 'postal_code', 'state'] if addr[k]] # noqa addr['full_address'] = ', '.join(addr_info) if addr_info else None return doc @@ -104,7 +103,7 @@ def make_source_object(name, doc): def get_json_data(url): - r = requests.get(url) + r = requests.get(url, timeout=600) return r.json() @@ -115,6 +114,7 @@ def get_json_data(url): steam_handler.setLevel(logging.INFO) logger.addHandler(steam_handler) + class BaseImporter: ES_INDEX_NAME = None SOURCE_NAME = None @@ -169,7 +169,7 @@ def filter_source(self, doc) -> bool: def is_isn_source(doc) -> bool: - return f'(ISN)' in doc['source'] + return '(ISN)' in doc['source'] if __name__ == '__main__': @@ -178,20 +178,20 @@ def is_isn_source(doc) -> bool: es = Elasticsearch(hosts=es_url) - source_importers = [importer_cls(es) for importer_cls in SOURCE_IMPORTER_CLASSES] + source_importers = [importer_cls(es) for importer_cls in SOURCE_IMPORTER_CLASSES] # noqa - logger.info('Start import CSL source: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + logger.info('Start import CSL source: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S')) # noqa try: if not es.ping(): raise ConnectionError - json_data = get_json_data('https://api.trade.gov/static/consolidated_screening_list/consolidated.json') + json_data = get_json_data('https://api.trade.gov/static/consolidated_screening_list/consolidated.json') # noqa for importer in source_importers: importer.do_import(json_data) - logger.info('Finish import CSL source: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + logger.info('Finish import CSL source: ' + datetime.now().strftime('%Y-%m-%d %H:%M:%S')) # noqa except ConnectionError: logger.error('Connect ES server failed') except Exception as e: - logger.error(f'Import CSL source failed: {e}, {traceback.format_exc()}') + logger.error(f'Import CSL source failed: {e}, {traceback.format_exc()}') # noqa diff --git a/docker/csl-python/poetry.lock b/docker/csl-python/poetry.lock index 1b1ef02..4e77538 100644 --- a/docker/csl-python/poetry.lock +++ b/docker/csl-python/poetry.lock @@ -121,25 +121,40 @@ files = [ {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] +[[package]] +name = "elastic-transport" +version = "8.11.0" +description = "Transport classes and utilities shared among Python Elastic client libraries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "elastic-transport-8.11.0.tar.gz", hash = "sha256:1a6ab59b2a6f6feeef20d254babb1b27fbf0b2bc8d3cf485f5211b422290dd4c"}, + {file = "elastic_transport-8.11.0-py3-none-any.whl", hash = "sha256:dfb5d8a0cb649c159ebf4d9b1f55b7c8d00bb65687c53060b54cc9b2a7c84344"}, +] + +[package.dependencies] +certifi = "*" +urllib3 = ">=1.26.2,<3" + +[package.extras] +develop = ["aiohttp", "furo", "mock", "pytest", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests", "sphinx (>2)", "sphinx-autodoc-typehints", "trustme"] + [[package]] name = "elasticsearch" -version = "7.13.1" +version = "8.11.1" description = "Python client for Elasticsearch" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4" +python-versions = ">=3.6" files = [ - {file = "elasticsearch-7.13.1-py2.py3-none-any.whl", hash = "sha256:a09ae1de8869efa6ef2d9a0a9b9f6d9260b0c2506e83dd32bc1119a23fff49a5"}, - {file = "elasticsearch-7.13.1.tar.gz", hash = "sha256:d6bcca0b2e5665d08e6fe6fadc2d4d321affd76ce483603078fc9d3ccd2bc0f9"}, + {file = "elasticsearch-8.11.1-py3-none-any.whl", hash = "sha256:a98309cee11fef8d6750f388683e9a8005da94bdfd940b36ef85cb6cc53186c7"}, + {file = "elasticsearch-8.11.1.tar.gz", hash = "sha256:360b721324ce4bc7d554afb8acbf4942370e73c5ef8c4dad5f5ba3bb2a70eeae"}, ] [package.dependencies] -certifi = "*" -urllib3 = ">=1.21.1,<2" +elastic-transport = ">=8,<9" [package.extras] async = ["aiohttp (>=3,<4)"] -develop = ["black", "coverage", "jinja2", "mock", "pytest", "pytest-cov", "pyyaml", "requests (>=2.0.0,<3.0.0)", "sphinx (<1.7)", "sphinx-rtd-theme"] -docs = ["sphinx (<1.7)", "sphinx-rtd-theme"] requests = ["requests (>=2.4.0,<3.0.0)"] [[package]] @@ -193,4 +208,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "0bbae87f8f5a2d420fa905f4721dea53d06df6287761a1edf981c0707b73e948" +content-hash = "a95b779fdb8fcb2dd60ff4cc280df70897ba3dd212322c8e822b7268877ff663" diff --git a/docker/csl-python/pyproject.toml b/docker/csl-python/pyproject.toml index 2618a1b..2227497 100644 --- a/docker/csl-python/pyproject.toml +++ b/docker/csl-python/pyproject.toml @@ -10,7 +10,7 @@ packages = [{include = "csl_python"}] python = "^3.11" certifi = "*" chardet = "^4" -elasticsearch = "7.13.1" +elasticsearch = "*" idna = "^2" requests = "^2" urllib3 = "^1" diff --git a/docker/csl/Dockerfile b/docker/csl/Dockerfile index bb94028..cc9efb0 100644 --- a/docker/csl/Dockerfile +++ b/docker/csl/Dockerfile @@ -4,41 +4,43 @@ FROM ruby:2.6.10-bullseye as builder # Set environment variables ENV LANG=C.UTF-8 \ RAILS_ENV=production \ - BUNDLE_PATH=/bundle \ BUNDLE_JOBS=4 -# set the working directory WORKDIR /repo # copy the Gemfile and Gemfile.lock COPY Gemfile Gemfile.lock ./ # Install gems -RUN gem install bundler -v 2.4.22 && \ - bundle install --without development test +RUN gem update --system 3.2.3 && \ + gem install bundler -v 2.4.22 && \ + bundle update --bundler && \ + bundle config set --local without 'development test' && \ + bundle install --no-color # Stage 2: Create a smaller image with Debian Bullseye FROM ruby:2.6.10-slim-bullseye AS final # Set environment variables ENV LANG=C.UTF-8 \ - RAILS_ENV=production \ - BUNDLE_PATH=/bundle + RAILS_ENV=production -# Install security updates + # Install security updates RUN apt-get update \ && apt-get -y full-upgrade --no-install-recommends \ - && apt-get install -y libicu67 \ + \ + # Install dependencies for building ruby gems + && apt-get install -y libicu67=67.1-7 --no-install-recommends \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* WORKDIR /app COPY . . -COPY --from=builder /bundle /bundle -COPY ./docker/csl/* /usr/bin/ +COPY --from=builder /usr/local/bundle /usr/local/bundle +COPY ./docker/csl/*.sh /usr/bin/ -RUN bundle install --without development test && \ +RUN gem update --system 3.2.3 && \ chmod +x /usr/bin/*.sh EXPOSE 3000 diff --git a/docker/csl/entrypoint.sh b/docker/csl/entrypoint.sh index eafed2d..75538a7 100755 --- a/docker/csl/entrypoint.sh +++ b/docker/csl/entrypoint.sh @@ -1,6 +1,16 @@ #!/bin/sh -/usr/bin/wait-for-it.sh elastic:9200 -t 30 -- echo "Elasticsearch server is ready" +# Check if variable is set +if [ -z "$ELASTICSEARCH_URL" ] ; then + echo "ELASTICSEARCH_URL variable is not set" + exit 1 +fi + +# Remove http:// or https:// from ELASTICSEARCH_URL +ELASTICSEARCH_URL=${ELASTICSEARCH_URL#*//} + +# Wait for Elasticsearch to start up before doing anything. +/usr/bin/wait-for-it.sh "$ELASTICSEARCH_URL" -t 30 -- echo "Elasticsearch server is ready" # DB Migration if [ "$RECREATE_DB" = "true" ] ; then diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml deleted file mode 100644 index 3c90f1c..0000000 --- a/docker/dev/docker-compose.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: "3.3" -services: - elastic: - container_name: elasticsearch - image: docker.elastic.co/elasticsearch/elasticsearch:7.4.2 - environment: - - discovery.type=single-node - rails: - build: - context: ./../../ - dockerfile: $PWD/../csl/Dockerfile - container_name: csl - environment: - - RECREATE_DB=true - ports: - - "3000:3000" - depends_on: - - elastic - python: - build: ./../csl-python/ - container_name: csl-python - depends_on: - - elastic diff --git a/docker/dev/run-csl.sh b/docker/dev/run-csl.sh deleted file mode 100755 index 5ce54b8..0000000 --- a/docker/dev/run-csl.sh +++ /dev/null @@ -1,9 +0,0 @@ -# ======================================================================== -# Build docker image -# ======================================================================== -docker-compose -f docker-compose.yml build - -# ======================================================================== -# Run CSL -# ======================================================================== -docker-compose -f docker-compose.yml up -d diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 374e2f4..82e5095 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -28,15 +28,15 @@ services: rails: build: context: ./../ - dockerfile: $PWD/csl/Dockerfile + dockerfile: docker/csl/Dockerfile container_name: csl-api environment: - RECREATE_DB=true - - SECRET_KEY_BASE=fake-key - - ELASTICSEARCH_URL=elastic:9200 - command: ["tail", "-f", "/dev/null"] + - ENC_SECRET_KEY_BASE=fake-key + - ELASTICSEARCH_URL=http://elastic:9200 + platform: linux/amd64 volumes: - - $PWD/../:/app + - $PWD:/app ports: - 3000:3000 depends_on: @@ -44,9 +44,10 @@ services: python: build: - context: ./csl-python/ - dockerfile: $PWD/csl-python/Dockerfile + context: ./csl-python + dockerfile: Dockerfile container_name: csl-python + platform: linux/amd64 environment: - RECREATE_DB=true - ELASTICSEARCH_URL=http://elastic:9200 diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile deleted file mode 100644 index 03d8140..0000000 --- a/docker/nginx/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM nginx:1.21.0-alpine -LABEL maintainer=hardcore -RUN apk add --no-cache --upgrade bash - -RUN mkdir -p /ssl -COPY ssl-127.0.0.1/ssl.cert /ssl/ssl.cert -COPY ssl-127.0.0.1/ssl.key /ssl/ssl.key -COPY wait-for-it.sh /usr/bin/ -RUN chmod +x /usr/bin/wait-for-it.sh -COPY docker-cmd.sh /usr/bin/ -RUN chmod +x /usr/bin/docker-cmd.sh - -CMD ["/usr/bin/docker-cmd.sh"] \ No newline at end of file diff --git a/docker/nginx/conf.d/denied-party-stage.gofreight.conf b/docker/nginx/conf.d/denied-party-stage.gofreight.conf deleted file mode 100644 index be2e68f..0000000 --- a/docker/nginx/conf.d/denied-party-stage.gofreight.conf +++ /dev/null @@ -1,78 +0,0 @@ -log_format apm-stage '$remote_addr - $remote_user [$time_local] ' - '"$request" $status $body_bytes_sent ' - '"$http_referer" "$http_user_agent" ' - '"$request_time" "$upstream_connect_time" "$upstream_header_time" "$upstream_response_time"'; - -# redirect HTTP traffic to HTTPS -# server { -# resolver 8.8.8.8; - -# listen 80; -# server_name denied-party-stage.gofreight.co; -# return 301 https://$host$request_uri; -# } - -server { - # reg server_name need add resolver to avoid issues - resolver 8.8.8.8; - - listen 11999 ssl http2; - server_name denied-party-stage.gofreight.co; - - server_tokens off; - - ssl_certificate /ssl/ssl.cert; - ssl_certificate_key /ssl/ssl.key; - - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DHE+AES128:!ADH:!AECDH:!MD5; # GoFreight used cipher suite - # ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; # Mozilla recommended intermediate compatibility cipher suite - - ssl_dhparam /ssl/dhparam.pem; - - ssl_prefer_server_ciphers on; - - ssl_session_cache shared:SSL:10m; - ssl_session_timeout 1d; - - ssl_session_tickets off; - - ssl_buffer_size 4k; - - ssl_stapling on; - ssl_stapling_verify on; - - location / { - proxy_pass http://rails:3000; - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - client_max_body_size 75M; - - proxy_read_timeout 300; - proxy_send_timeout 300; - - access_log /var/log/nginx/access.log apm-stage; - - # add custom header info for debug - add_header X-HC-Proxy "CSL-PROXY" always; - add_header X-HC-Domain "denied-party-stage" always; - } - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - - ### GZip ### - gzip on; - gzip_disable "msie6"; - - gzip_vary on; - gzip_min_length 1k; - gzip_proxied any; - gzip_comp_level 4; - gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml; -} \ No newline at end of file diff --git a/docker/nginx/conf.d/denied-party.gofreight.conf b/docker/nginx/conf.d/denied-party.gofreight.conf deleted file mode 100644 index 4bd188b..0000000 --- a/docker/nginx/conf.d/denied-party.gofreight.conf +++ /dev/null @@ -1,78 +0,0 @@ -log_format apm '$remote_addr - $remote_user [$time_local] ' - '"$request" $status $body_bytes_sent ' - '"$http_referer" "$http_user_agent" ' - '"$request_time" "$upstream_connect_time" "$upstream_header_time" "$upstream_response_time"'; - -# redirect HTTP traffic to HTTPS -# server { -# resolver 8.8.8.8; - -# listen 80; -# server_name denied-party.gofreight.co; -# return 301 https://$host$request_uri; -# } - -server { - # reg server_name need add resolver to avoid issues - resolver 8.8.8.8; - - listen 11999 ssl http2; - server_name denied-party.gofreight.co; - - server_tokens off; - - ssl_certificate /ssl/ssl.cert; - ssl_certificate_key /ssl/ssl.key; - - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DHE+AES128:!ADH:!AECDH:!MD5; # GoFreight used cipher suite - # ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; # Mozilla recommended intermediate compatibility cipher suite - - ssl_dhparam /ssl/dhparam.pem; - - ssl_prefer_server_ciphers on; - - ssl_session_cache shared:SSL:10m; - ssl_session_timeout 1d; - - ssl_session_tickets off; - - ssl_buffer_size 4k; - - ssl_stapling on; - ssl_stapling_verify on; - - location / { - proxy_pass http://rails:3000; - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - client_max_body_size 75M; - - proxy_read_timeout 300; - proxy_send_timeout 300; - - access_log /var/log/nginx/access.log apm; - - # add custom header info for debug - add_header X-HC-Proxy "CSL-PROXY" always; - add_header X-HC-Domain "denied-party" always; - } - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - - ### GZip ### - gzip on; - gzip_disable "msie6"; - - gzip_vary on; - gzip_min_length 1k; - gzip_proxied any; - gzip_comp_level 4; - gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml; -} \ No newline at end of file diff --git a/docker/nginx/conf.d/health-check.conf b/docker/nginx/conf.d/health-check.conf deleted file mode 100644 index 2bf7903..0000000 --- a/docker/nginx/conf.d/health-check.conf +++ /dev/null @@ -1,12 +0,0 @@ -server { - listen 443 ssl; - server_name csl-proxy-health-check.gofreight.co; - - ssl_certificate /ssl/ssl.cert; - ssl_certificate_key /ssl/ssl.key; - - location /check { - access_log off; - return 200 'health'; - } -} \ No newline at end of file diff --git a/docker/nginx/docker-cmd.sh b/docker/nginx/docker-cmd.sh deleted file mode 100644 index 8df4d7d..0000000 --- a/docker/nginx/docker-cmd.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -set -ex - -/usr/bin/wait-for-it.sh rails:3000 -t 300 -- echo "CSL server is ready" -exec nginx -g "daemon off;" \ No newline at end of file diff --git a/docker/nginx/ssl-127.0.0.1/ssl.cert b/docker/nginx/ssl-127.0.0.1/ssl.cert deleted file mode 100644 index 954feee..0000000 --- a/docker/nginx/ssl-127.0.0.1/ssl.cert +++ /dev/null @@ -1,18 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIC+zCCAeOgAwIBAgIJAOdsAKMo36NSMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV -BAMMCTEyNy4wLjAuMTAeFw0xOTAxMTAxNzU4MDZaFw0yOTAxMDcxNzU4MDZaMBQx -EjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC -ggEBAMQ3OWtvoG+u4AmIwIfb7z22OyFs42Z3+BodZ9ZelGUi1YRdJSk4e2Anjaja -TXl956ItPHMjC4+S3O9AKARHREC6+nHffrqBwzGm1zVIPbIWFg7kSTBsfqA3VLX1 -D7hLtquBu7Brpz+2stqdBzUN2Ry3PqYYkX+gDGkJi4z96LUPPptyXeZSd9fkhyk4 -AizCh1ih79rfvGoK09yjIwG1pIHd0YoZ1g+7rbZDj9WjwcqsTrK0AjBGiTdeW0so -vZNs86hlsYGEdBvCjmU3YcwRlIkolTC8e+m/1MOnQ27gKIcUgak7UuKfHjE7mL4Q -nK5GZ+5mDxsCwEmLVE1QCV7UbGkCAwEAAaNQME4wHQYDVR0OBBYEFEFaPOO7Ebc/ -pw2euccmcHrs3WtNMB8GA1UdIwQYMBaAFEFaPOO7Ebc/pw2euccmcHrs3WtNMAwG -A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAAO9j44PHDyQxdkLyAvzzNOz -f6uSEX8rr/yiRKQt/AxGuBp5juSDI9mw7+oOzY7TB6W++RwyxjsAWggTc7hUDKly -Z6nhYKeRiR29TwRPT/Wgifswn3ih+p8ZLe0/akEWYJ4Yi9274s0I4f2jJ5OEwoDw -3XPYXkglf3+Uqv/PZ6gfMv91B0YNejJR8YfpHGUxHpoxc7+ERWvTlTEwpO3fZICE -74pePemKMx19Ib2Nhy9+XZhXFdy6KUlzMTjkyN9S8BQH8C5+W93GujdK5xjAnv93 -w1U1Kc9VpL+hl3BGRwJFjRiogGOh4p0+CwQuZ6CL9zxoZRDwe1nrvAVG5++RhqE= ------END CERTIFICATE----- diff --git a/docker/nginx/ssl-127.0.0.1/ssl.key b/docker/nginx/ssl-127.0.0.1/ssl.key deleted file mode 100644 index 1d89715..0000000 --- a/docker/nginx/ssl-127.0.0.1/ssl.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAxDc5a2+gb67gCYjAh9vvPbY7IWzjZnf4Gh1n1l6UZSLVhF0l -KTh7YCeNqNpNeX3noi08cyMLj5Lc70AoBEdEQLr6cd9+uoHDMabXNUg9shYWDuRJ -MGx+oDdUtfUPuEu2q4G7sGunP7ay2p0HNQ3ZHLc+phiRf6AMaQmLjP3otQ8+m3Jd -5lJ31+SHKTgCLMKHWKHv2t+8agrT3KMjAbWkgd3RihnWD7uttkOP1aPByqxOsrQC -MEaJN15bSyi9k2zzqGWxgYR0G8KOZTdhzBGUiSiVMLx76b/Uw6dDbuAohxSBqTtS -4p8eMTuYvhCcrkZn7mYPGwLASYtUTVAJXtRsaQIDAQABAoIBAEjysxFDPeVnoXlJ -8zJPTSGnV1J09hIJdh7XYdxr60Wybqk4K/GoQw4bhmoJRxZs/anooXm4xLqLAkKc -moQErtZV7Xhae/YA+j3CZm4zvmQwG+FLc/5hx63vBDplsoN0+qPGpq3PSC3C0bm+ -+Yf67wXjBpKfvbAyI4sFdnFjkJmHOt+nywRpmw4ucCHR0XIszPPBbDYfQpkoh85s -s/2rJsoWOlGSjmWlN/+jztm+1Gskbd36MYZmcqpqfQOWr4CxGDzRfj0XtjUV0dlq -+Q+BhWclm0Xrm4y4qlyoWsOE6nde7xrx2IzqpL55GW5TdSh/CkZep1xuUBxn0YCm -3BEvPsUCgYEA+Bp7867xWgpGj/OncJIyjWaEv80c1sZ0d5guiQCbHwi1As2Qsfq3 -FgBWYavcXVJO04Ep1kzGEKvYNHhgvyRBOOXRlAEKTeHF4lCtnmh2iuN7I2d2BYV7 -hP7s/q8xZfy2Lqc9WzOrRjxubdUng0EbMcgHIxyxW16WSj6RnZ5Mf48CgYEAynX3 -IgAbTceiKkUa24JaGzRl3+bOsE09kTOWKqtodOFjLfOtVmMLyHbK7uqgGwRULrsQ -RqIIpliNtks+5jORzfexk2903yj/Qe/xzstlnHaV6ki24Y7u2QmSVocYkybHtATf -F9ZBtq4ErSto46x7K9adJXnEQpr1ZcG0IIscWIcCgYEAjPL+2EHa+7w6ui1CQsHL -1mkXoZ91cs7r+8WJKmwdtFPOAfmm6nX07ZrjeDmax6mq/p2ylvymyXHXMnfw/qYq -J5jop5yvre9F7HzJJeVS8zSpEEQG2LwsoDHzkf0Vo0sedQKt6NJBy3Uph1xBRmAG -gxGQfHIgkDf8SdGiCFmWPCECgYB9WyETWkGNnSfZSERi2Yj1AzJ4ua4MFychunEm -Jk9c3xjLoO8YZ+cpzfPEjl1nnPyKmqRvN3BnnrjPwDNH0XJbtxuqD/6rFfVGXEy6 -H9HgWMRl9XPWoOteUsDa9nbO1docqk9/sNerRn/6wMAHCAqTkCRoycYs/0kmIhNn -evqoDQKBgDKVvYid2KPNs5bXPPdJms2JgHf2YxKt2ae+TRDwAbceuKONgFrMnM4N -FO432/qgRiWwkrcfWrBn1/kMIHMw5R0xXIM0nrb/EpIZx3ybefMtwRoOAbTwDZUn -AHmZx2IcFMFL7jTs3zdKwOolX6Te+bN6OVsSya57Cxx/EZUNEUt8 ------END RSA PRIVATE KEY----- diff --git a/docker/nginx/wait-for-it.sh b/docker/nginx/wait-for-it.sh deleted file mode 100755 index d990e0d..0000000 --- a/docker/nginx/wait-for-it.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env bash -# Use this script to test if a given TCP host/port are available - -WAITFORIT_cmdname=${0##*/} - -echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} - -wait_for() -{ - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - else - echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" - fi - WAITFORIT_start_ts=$(date +%s) - while : - do - if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then - nc -z $WAITFORIT_HOST $WAITFORIT_PORT - WAITFORIT_result=$? - else - (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 - WAITFORIT_result=$? - fi - if [[ $WAITFORIT_result -eq 0 ]]; then - WAITFORIT_end_ts=$(date +%s) - echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" - break - fi - sleep 1 - done - return $WAITFORIT_result -} - -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $WAITFORIT_QUIET -eq 1 ]]; then - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - else - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - fi - WAITFORIT_PID=$! - trap "kill -INT -$WAITFORIT_PID" INT - wait $WAITFORIT_PID - WAITFORIT_RESULT=$? - if [[ $WAITFORIT_RESULT -ne 0 ]]; then - echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - fi - return $WAITFORIT_RESULT -} - -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - WAITFORIT_hostport=(${1//:/ }) - WAITFORIT_HOST=${WAITFORIT_hostport[0]} - WAITFORIT_PORT=${WAITFORIT_hostport[1]} - shift 1 - ;; - --child) - WAITFORIT_CHILD=1 - shift 1 - ;; - -q | --quiet) - WAITFORIT_QUIET=1 - shift 1 - ;; - -s | --strict) - WAITFORIT_STRICT=1 - shift 1 - ;; - -h) - WAITFORIT_HOST="$2" - if [[ $WAITFORIT_HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - WAITFORIT_HOST="${1#*=}" - shift 1 - ;; - -p) - WAITFORIT_PORT="$2" - if [[ $WAITFORIT_PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - WAITFORIT_PORT="${1#*=}" - shift 1 - ;; - -t) - WAITFORIT_TIMEOUT="$2" - if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - WAITFORIT_TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - WAITFORIT_CLI=("$@") - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done - -if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi - -WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} -WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} -WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} -WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} - -# Check to see if timeout is from busybox? -WAITFORIT_TIMEOUT_PATH=$(type -p timeout) -WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) - -WAITFORIT_BUSYTIMEFLAG="" -if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then - WAITFORIT_ISBUSY=1 - # Check if busybox timeout uses -t flag - # (recent Alpine versions don't support -t anymore) - if timeout &>/dev/stdout | grep -q -e '-t '; then - WAITFORIT_BUSYTIMEFLAG="-t" - fi -else - WAITFORIT_ISBUSY=0 -fi - -if [[ $WAITFORIT_CHILD -gt 0 ]]; then - wait_for - WAITFORIT_RESULT=$? - exit $WAITFORIT_RESULT -else - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - wait_for_wrapper - WAITFORIT_RESULT=$? - else - wait_for - WAITFORIT_RESULT=$? - fi -fi - -if [[ $WAITFORIT_CLI != "" ]]; then - if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then - echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" - exit $WAITFORIT_RESULT - fi - exec "${WAITFORIT_CLI[@]}" -else - exit $WAITFORIT_RESULT -fi diff --git a/spec/fixtures/screening_lists/dtc/itar_debarred_parties.csv b/spec/fixtures/screening_lists/dtc/itar_debarred_parties.csv index 068efc5..a073df5 100644 --- a/spec/fixtures/screening_lists/dtc/itar_debarred_parties.csv +++ b/spec/fixtures/screening_lists/dtc/itar_debarred_parties.csv @@ -2,4 +2,4 @@ Party Name,Date Of Birth,Federal Register Notice,Notice Date,Corrected Notice,Co "Yun, Juwhan",,57 FR 1296,1/13/1992,, "Dilligas Trading Co., Inc.",,57 FR 43768,9/22/1992,57 FR 37184,8/18/1992 "Fleming, Brian Joseph (a.k.a. Brian Joseph McSulla)",,58 FR 6835,2/2/1993,, -"Hukic, Bajro (a.k.a. Bob Hukic; Bab Hucici)",,59 FR 44451,8/29/1994,, \ No newline at end of file +"Hukic, Bajro (a.k.a. Bob Hukic; Bab Hucici)",,59 FR 44451,8/29/1994,, diff --git a/spec/fixtures/screening_lists/isn/isn.csv b/spec/fixtures/screening_lists/isn/isn.csv index 3f17588..37cdef6 100644 --- a/spec/fixtures/screening_lists/isn/isn.csv +++ b/spec/fixtures/screening_lists/isn/isn.csv @@ -5,4 +5,4 @@ ISN,E.O. 13382,Amir Hossein Rahimyar,,Iran,"Vol. 78, No. 38, 02/26/13",12/13/201 ,,,,,,,, ISN,E.O. 13382,Abdul Qadeer Khan,,Pakistan,"Vol. 74, No. 11, 01/16/09",1/9/2009,Associated with the A.Q. Khan Network,http://www.state.gov/t/isn/c15231.htm ISN,Export-Import Bank Act,Abdul Qadeer Khan,,Pakistan,"Vol. 74, No. 11, 01/16/09",1/9/2009,,http://www.state.gov/t/isn/c15231.htm -ISN,Nuclear Proliferation Prevention Act,Abdul Qadeer Khan,,Pakistan,"Vol. 74, No. 11, 01/16/09",1/9/2009,,http://www.state.gov/t/isn/c15231.htm \ No newline at end of file +ISN,Nuclear Proliferation Prevention Act,Abdul Qadeer Khan,,Pakistan,"Vol. 74, No. 11, 01/16/09",1/9/2009,,http://www.state.gov/t/isn/c15231.htm diff --git a/spec/importers/screening_list/cap/results.yaml b/spec/importers/screening_list/cap/results.yaml index e2ff0dc..4cd57f6 100644 --- a/spec/importers/screening_list/cap/results.yaml +++ b/spec/importers/screening_list/cap/results.yaml @@ -85,4 +85,4 @@ - KARAMAYCITYCOMMERCIALBANK :alt_no_ws_rev: - COOPERATIVESCREDITURBANKARAMAY - - BANKCOMMERCIALCITYKARAMAY \ No newline at end of file + - BANKCOMMERCIALCITYKARAMAY diff --git a/spec/importers/screening_list/dpl/results.yaml b/spec/importers/screening_list/dpl/results.yaml index a544222..8404e62 100644 --- a/spec/importers/screening_list/dpl/results.yaml +++ b/spec/importers/screening_list/dpl/results.yaml @@ -3,8 +3,8 @@ :start_date: '1992-03-20' :end_date: '2000-10-04' :standard_order: Y - :remarks: - :federal_register_notice: + :remarks: + :federal_register_notice: :id: 33002aa2bdc2b177712d2bfd6fa5f7f700e88d23 :source: &1 :full_name: Denied Persons List (DPL) - Bureau of Industry and Security @@ -183,4 +183,4 @@ :city: PANTIN :state: :country: FR - :postal_code: \ No newline at end of file + :postal_code: diff --git a/spec/importers/screening_list/dtc/results.yaml b/spec/importers/screening_list/dtc/results.yaml index 1f8277d..fe13023 100644 --- a/spec/importers/screening_list/dtc/results.yaml +++ b/spec/importers/screening_list/dtc/results.yaml @@ -57,4 +57,4 @@ :alt_no_ws: ['BobHukic','BabHucici'] :alt_no_ws_rev: ['HukicBob','HuciciBab'] :source_list_url: https://www.pmddtc.state.gov/?id=ddtc_kb_article_page&sys_id=7188dac6db3cd30044f9ff621f961914 - :id: '47138e5f361e31c71d3e0f39be10ac3edef20b23' \ No newline at end of file + :id: '47138e5f361e31c71d3e0f39be10ac3edef20b23' diff --git a/spec/importers/screening_list/el/results.yaml b/spec/importers/screening_list/el/results.yaml index 689f88e..40bd19b 100644 --- a/spec/importers/screening_list/el/results.yaml +++ b/spec/importers/screening_list/el/results.yaml @@ -2,7 +2,7 @@ - :name: Fazal Rahim Farid :federal_register_notice: 76 FR 71867 :start_date: '2011-11-21' - :standard_order: + :standard_order: :license_requirement: For all items subject to the EAR. (See §744.11 of the EAR). :license_policy: Presumption of denial. :source_list_url: http://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/entity-list diff --git a/spec/importers/screening_list/isn/results.yaml b/spec/importers/screening_list/isn/results.yaml index d08b328..41692c4 100644 --- a/spec/importers/screening_list/isn/results.yaml +++ b/spec/importers/screening_list/isn/results.yaml @@ -2,9 +2,9 @@ - :name: 150th Aircraft Repair Plant (ARZ) (Kaliningrad) :federal_register_notice: Vol. 81, No. 128, 07/05/16 :start_date: '2016-06-28' - :remarks: + :remarks: :source_list_url: http://www.state.gov/t/isn/c15231.htm - :alt_names: + :alt_names: :country: RU :id: 66a593540d9afb462007a039dc4ba73bbf0306b8 :source: &1 @@ -20,9 +20,9 @@ - :name: Amir Hossein Rahimyar :federal_register_notice: Vol. 78, No. 38, 02/26/13 :start_date: '2012-12-13' - :remarks: + :remarks: :source_list_url: http://www.state.gov/t/isn/c15231.htm - :alt_names: + :alt_names: :country: IR :id: f862fb370c69ed50a5448840cd5aafd3dbfef568 :source: *1 @@ -38,7 +38,7 @@ :start_date: '2009-01-09' :remarks: Associated with the A.Q. Khan Network :source_list_url: http://www.state.gov/t/isn/c15231.htm - :alt_names: + :alt_names: :country: PK :id: 2d2db09c686e4829d0ef1b0b04145eec3d42cd88 :source: *1 diff --git a/spec/importers/screening_list/meu/results.yaml b/spec/importers/screening_list/meu/results.yaml index c24d7dd..716474e 100644 --- a/spec/importers/screening_list/meu/results.yaml +++ b/spec/importers/screening_list/meu/results.yaml @@ -2,7 +2,7 @@ - :name: Fazal Rahim Farid :federal_register_notice: 76 FR 71867 :start_date: '2011-11-21' - :standard_order: + :standard_order: :license_requirement: For all items subject to the EAR. (See §744.11 of the EAR). :license_policy: Presumption of denial. :source_list_url: https://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/1770 diff --git a/spec/importers/screening_list/plc/results.yaml b/spec/importers/screening_list/plc/results.yaml index 9b99739..8e2e63c 100644 --- a/spec/importers/screening_list/plc/results.yaml +++ b/spec/importers/screening_list/plc/results.yaml @@ -1,18 +1,18 @@ --- - :entity_number: '9639' - :title: - :call_sign: - :vessel_type: - :gross_tonnage: - :gross_registered_tonnage: - :vessel_flag: - :vessel_owner: - :remarks: + :title: + :call_sign: + :vessel_type: + :gross_tonnage: + :gross_registered_tonnage: + :vessel_flag: + :vessel_owner: + :remarks: :id: '9639' :source: &1 :full_name: Palestinian Legislative Council List (PLC) - Treasury Department :code: PLC - :source_list_url: + :source_list_url: :source_information_url: https://www.treasury.gov/resource-center/sanctions/Programs/Pages/pa.aspx :name: HANIYA, Ismail Abdul Salah :type: Individual @@ -76,7 +76,7 @@ :alt_no_ws: [KAFEESHEHHatemRabah, QEFEISHAHatem, ABUANEDHadjHatem] :alt_no_ws_rev: - [RabahHatemKAFEESHEH, HatemQEFEISHA, HatemHadjANEDABU] + [RabahHatemKAFEESHEH, HatemQEFEISHA, HatemHadjANEDABU] - :entity_number: '9688' :title: :call_sign: diff --git a/spec/importers/screening_list/ssi/results.yaml b/spec/importers/screening_list/ssi/results.yaml index e0313e5..944e4f1 100644 --- a/spec/importers/screening_list/ssi/results.yaml +++ b/spec/importers/screening_list/ssi/results.yaml @@ -9,14 +9,14 @@ :vessel_owner: :remarks: 'For more information on directives, please visit the following link: http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives.' :id: '17013' - :source: + :source: :full_name: Sectoral Sanctions Identifications List (SSI) - Treasury Department :code: SSI :source_list_url: :source_information_url: https://www.treasury.gov/resource-center/sanctions/SDN-List/Pages/ssi_list.aspx :name: VTB BANK OAO :type: Entity - :alt_names: + :alt_names: - JSC VTB BANK - BANK VTB, OPEN JOINT-STOCK COMPANY - VNESHTORGBANK @@ -27,13 +27,13 @@ - VNESHTORGBANK ROSSII, CLOSED JOINT-STOCK COMPANY - VTB BANK, OPEN JOINT-STOCK COMPANY - BANK VTB OAO - :programs: + :programs: - UKRAINE-EO13662 :nationalities: [] :citizenships: [] :dates_of_birth: [] :places_of_birth: [] - :addresses: + :addresses: - :city: St. Petersburg :country: RU :postal_code: '190000' @@ -52,7 +52,7 @@ :state: :address: 43, Vorontsovskaya str. :full_address: 43, Vorontsovskaya str., Moscow, RU, 109044 - :ids: + :ids: - :country: RU :expiration_date: :issue_date: @@ -175,16 +175,16 @@ :source_information_url: https://www.treasury.gov/resource-center/sanctions/SDN-List/Pages/ssi_list.aspx :name: BANK OF MOSCOW :type: Entity - :alt_names: + :alt_names: - JOINT STOCK COMMERCIAL BANK - BANK OF MOSCOW, OPEN JOINT STOCK COMPANY - AKTSIONERNY KOMMERCHESKI BANK BANK MOSKVY, OTKRYTOE AKTSIONERNOE OBSCHCHESTVO - :programs: + :programs: - UKRAINE-EO13662 :nationalities: [] :citizenships: [] :dates_of_birth: [] :places_of_birth: [] - :addresses: + :addresses: - :city: Moscow :country: RU :postal_code: '107996' @@ -197,7 +197,7 @@ :state: :address: Bld 3 8/15, Rozhdestvenka St. :full_address: Bld 3 8/15, Rozhdestvenka St., Moscow, RU, 107996 - :ids: + :ids: - :country: :expiration_date: :issue_date: @@ -270,27 +270,27 @@ :vessel_owner: :remarks: 'For more information on directives, please visit the following link: http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives.' :id: '17254' - :source: + :source: :full_name: Sectoral Sanctions Identifications List (SSI) - Treasury Department :code: SSI :source_list_url: :source_information_url: https://www.treasury.gov/resource-center/sanctions/SDN-List/Pages/ssi_list.aspx :name: AK TRANSNEFT OAO :type: Entity - :alt_names: + :alt_names: - OAO AK TRANSNEFT - AKTSIONERNAYA KOMPANIYA PO TRANSPORTUNEFTI TRANSNEFT OAO - OIL TRANSPORTING JOINT-STOCK COMPANY TRANSNEFT - TRANSNEFT, JSC - TRANSNEFT OJSC - TRANSNEFT - :programs: + :programs: - UKRAINE-EO13662 :nationalities: [] :citizenships: [] :dates_of_birth: [] :places_of_birth: [] - :addresses: + :addresses: - :city: Moscow :country: RU :postal_code: '119180' @@ -390,14 +390,14 @@ :vessel_owner: :remarks: 'For more information on directives, please visit the following link: http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives.' :id: '17260' - :source: + :source: :full_name: Sectoral Sanctions Identifications List (SSI) - Treasury Department :code: SSI :source_list_url: :source_information_url: https://www.treasury.gov/resource-center/sanctions/SDN-List/Pages/ssi_list.aspx :name: ROSTEC :type: Entity - :alt_names: + :alt_names: - STATE CORPORATION FOR ASSISTANCE TO DEVELOPMENT, PRODUCTION AND EXPORT OF ADVANCED TECHNOLOGY INDUSTRIAL PRODUCT ROSTEKHNOLOGII - STATE CORPORATION ROSTEKHNOLOGII - ROSTEC STATE CORPORATION @@ -410,7 +410,7 @@ :citizenships: [] :dates_of_birth: [] :places_of_birth: [] - :addresses: + :addresses: - :city: Moscow :country: RU :postal_code: '119048' @@ -423,7 +423,7 @@ :state: :address: 21 Gogolevsky Blvd. :full_address: 21 Gogolevsky Blvd., Moscow, RU, 119991 - :ids: + :ids: - :country: RU :expiration_date: :issue_date: diff --git a/spec/importers/screening_list/uvl/results.yaml b/spec/importers/screening_list/uvl/results.yaml index 355f6e0..3fb51f3 100644 --- a/spec/importers/screening_list/uvl/results.yaml +++ b/spec/importers/screening_list/uvl/results.yaml @@ -8,9 +8,9 @@ :source_information_url: http://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/unverified-list :addresses: - :address: Room 2135, Jingxin Building A, No Dong San Huan North Road, Beijing - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: CN :alt_names: [] :name_idx: China National Plant ImportExport @@ -26,14 +26,14 @@ :source_information_url: http://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/unverified-list :addresses: - :address: Flat A, 11/F, Adolfo Mansion, 114-116 Austin Road, Tsim Sha Tsui, Kowloon - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: HK - :address: Rm 1203, 12/F, Hip Kwan Commercial Bldg, 38 Pitt Street, Yau Ma Tei - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: HK :alt_names: [] :name_idx: Brilliance Technology @@ -49,9 +49,9 @@ :source_information_url: http://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/unverified-list :addresses: - :address: Unit 2209, 22/F, Wu Chung House, 213 Queen's Road East, Wanchai - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: HK :alt_names: [] :name_idx: Dynasense Photonics @@ -68,15 +68,15 @@ :addresses: - :address: Flat D12, 11/F, King Yip Factory Bldg, 59 King Yip Street, Kwun Tong, Kowloon - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: HK - :address: Room 603, 6/F, Hang Pont Commercial Building, 31 Tonking Street, Cheung Sha Wan, Kowloon - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: HK :alt_names: [] :name_idx: HiShine Technology HK @@ -92,9 +92,9 @@ :source_information_url: http://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/unverified-list :addresses: - :address: Rm 1013B, Well Fung Ind. Center, Ta Chuen Ping Street, Kwai Chung - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: HK :alt_names: [] :name_idx: Hong Kong Haimao InfoTec Development @@ -110,9 +110,9 @@ :source_information_url: http://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/unverified-list :addresses: - :address: Unit N, 3/F, Hopewell House, 175 Hip Wo Street, KwunTong, Kowloon - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: HK :alt_names: [] :name_idx: Lianqi HK Electronics @@ -128,9 +128,9 @@ :source_information_url: http://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/unverified-list :addresses: - :address: Unit N, 3/F, Hopewell House, 175 Hip Wo Street, KwunTong, Kowloon - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: HK :alt_names: [] :name_idx: Lion Chip Electronics @@ -146,14 +146,14 @@ :source_information_url: http://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/unverified-list :addresses: - :address: Deguninskaya Street 10, Moscow - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: RU - :address: Novoslobodskaya Str. 14/19, Moscow - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: RU :alt_names: [] :name_idx: Security 2 Business Academy @@ -167,14 +167,14 @@ :source_information_url: http://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/unverified-list :addresses: - :address: Deguninskaya Street 10, Moscow - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: RU - :address: Novoslobodskaya Str. 14/19, Moscow - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: RU :alt_names: [] :name_idx: S2BA @@ -188,14 +188,14 @@ :source_information_url: http://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/unverified-list :addresses: - :address: Deguninskaya Street 10, Moscow - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: RU - :address: Novoslobodskaya Str. 14/19, Moscow - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: RU :alt_names: [] :name_idx: Academy of Business Security @@ -209,9 +209,9 @@ :source_information_url: http://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/unverified-list :addresses: - :address: Deguninskaya Street 30, Moscow, Russia - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: RU :alt_names: [] :name_idx: T2BA @@ -225,14 +225,14 @@ :source_information_url: http://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/unverified-list :addresses: - :address: Deguninskaya Street 20, Moscow, Russia - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: RU - :address: Novoslobodskaya Str. 24/29, Moscow, Russia - :city: - :state: - :postal_code: + :city: + :state: + :postal_code: :country: RU :alt_names: - Training 2 Business Academy diff --git a/spec/lib/query/search_body_with_all.json b/spec/lib/query/search_body_with_all.json index 396fc16..48d8cae 100644 --- a/spec/lib/query/search_body_with_all.json +++ b/spec/lib/query/search_body_with_all.json @@ -42,4 +42,4 @@ } } } -} \ No newline at end of file +} diff --git a/spec/queries/screening_list/query/search_body_with_all.json b/spec/queries/screening_list/query/search_body_with_all.json index 848fd4b..89b6acc 100644 --- a/spec/queries/screening_list/query/search_body_with_all.json +++ b/spec/queries/screening_list/query/search_body_with_all.json @@ -78,4 +78,4 @@ ] } } -} \ No newline at end of file +} diff --git a/spec/queries/screening_list/query/search_body_with_countries_filter.json b/spec/queries/screening_list/query/search_body_with_countries_filter.json index f46cce7..0c5267d 100644 --- a/spec/queries/screening_list/query/search_body_with_countries_filter.json +++ b/spec/queries/screening_list/query/search_body_with_countries_filter.json @@ -50,4 +50,4 @@ } } } -} \ No newline at end of file +} diff --git a/spec/queries/screening_list/query/search_body_with_end_date.json b/spec/queries/screening_list/query/search_body_with_end_date.json index 6b5fb96..de196b3 100644 --- a/spec/queries/screening_list/query/search_body_with_end_date.json +++ b/spec/queries/screening_list/query/search_body_with_end_date.json @@ -17,4 +17,4 @@ } } } -} \ No newline at end of file +} diff --git a/spec/queries/screening_list/query/search_body_with_expiration_date.json b/spec/queries/screening_list/query/search_body_with_expiration_date.json index 728517a..5f4e7c5 100644 --- a/spec/queries/screening_list/query/search_body_with_expiration_date.json +++ b/spec/queries/screening_list/query/search_body_with_expiration_date.json @@ -17,4 +17,4 @@ } } } -} \ No newline at end of file +} diff --git a/spec/queries/screening_list/query/search_body_with_issue_date.json b/spec/queries/screening_list/query/search_body_with_issue_date.json index c542d69..9a084cc 100644 --- a/spec/queries/screening_list/query/search_body_with_issue_date.json +++ b/spec/queries/screening_list/query/search_body_with_issue_date.json @@ -17,4 +17,4 @@ } } } -} \ No newline at end of file +} diff --git a/spec/queries/screening_list/query/search_body_with_sources_filter.json b/spec/queries/screening_list/query/search_body_with_sources_filter.json index 519e201..d970c8d 100644 --- a/spec/queries/screening_list/query/search_body_with_sources_filter.json +++ b/spec/queries/screening_list/query/search_body_with_sources_filter.json @@ -16,4 +16,4 @@ } } } -} \ No newline at end of file +} diff --git a/spec/queries/screening_list/query/search_body_with_start_date.json b/spec/queries/screening_list/query/search_body_with_start_date.json index f2f53ff..02b8d94 100644 --- a/spec/queries/screening_list/query/search_body_with_start_date.json +++ b/spec/queries/screening_list/query/search_body_with_start_date.json @@ -17,4 +17,4 @@ } } } -} \ No newline at end of file +} diff --git a/spec/queries/screening_list/query/search_body_with_type_filter.json b/spec/queries/screening_list/query/search_body_with_type_filter.json index 5309d2e..71526ac 100644 --- a/spec/queries/screening_list/query/search_body_with_type_filter.json +++ b/spec/queries/screening_list/query/search_body_with_type_filter.json @@ -14,4 +14,4 @@ } } } -} \ No newline at end of file +} diff --git a/spec/support/screening_lists/cap/expected_results.json b/spec/support/screening_lists/cap/expected_results.json index 9b56d8b..7497ed7 100644 --- a/spec/support/screening_lists/cap/expected_results.json +++ b/spec/support/screening_lists/cap/expected_results.json @@ -68,4 +68,4 @@ "title": null, "type": "Entity" } -] \ No newline at end of file +] diff --git a/spec/support/screening_lists/isn/expected_results.json b/spec/support/screening_lists/isn/expected_results.json index 5dcde54..463eec2 100644 --- a/spec/support/screening_lists/isn/expected_results.json +++ b/spec/support/screening_lists/isn/expected_results.json @@ -46,4 +46,4 @@ "source_list_url": "http://www.state.gov/t/isn/c15231.htm", "start_date": "2009-01-09" } -] \ No newline at end of file +]