From 1e563e55bfe0d7c78a723429f89c8bf121dd4bae Mon Sep 17 00:00:00 2001 From: Andrew Burnes Date: Wed, 14 Feb 2024 08:54:51 -0700 Subject: [PATCH] chore: Update CI pipelines to hardened resources --- README.md | 4 - ci/docker/entrypoint.sh | 6 - ci/federalist-pipeline.yml | 318 ------------------------------------- ci/pipeline-dev.yml | 65 +++++--- ci/pipeline.yml | 118 +++++++------- docker-compose.yml | 2 +- package-lock.json | 27 ++-- package.json | 2 +- 8 files changed, 119 insertions(+), 423 deletions(-) delete mode 100644 ci/federalist-pipeline.yml diff --git a/README.md b/README.md index e14ecdf..4348ba3 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,6 @@ Some credentials in this pipeline are "compound" credentials that use the pipeli |**`((proxy-repository-path))`**|The url path to the repository|:x:| |**`((gh-access-token))`**| The Github access token|:x:| |**`((pages-proxy-((deploy-env))-site-redirects))`**|JSON array of redirect objects|:white_check_mark:| -|**`((federalist-proxy-((deploy-env))-site-redirects))`**|JSON array of redirect objects|:white_check_mark:| ### Setting up the pipeline The pipeline and each of it's instances will only need to be set once per instance to create the initial pipeline. After the pipelines are set, updates to the respective `git-branch` source will automatically set the pipeline with any updates. See the [`set_pipeline` step](https://concourse-ci.org/set-pipeline-step.html) for more information. Run the following command with the fly CLI to set a pipeline instance: @@ -119,9 +118,6 @@ $ fly -t destroy-pipeline \ -p proxy/deploy-env:production,git-branch:main ``` -### Production pages-proxy pipeline transition -We are currently migrating from Federalist to Pages. The migration includes maintaining the former "Federalist" components of the platform to smoothly transition our customers and their sites. The CI configuration for this deployment pipeline can be found in [`ci/federalist-pipeline.yml`](./ci/federalist-pipeline.yml). This pipeline will serve to manage the existing `federalist-proxy` during the transition until it can be decommissioned. - ## Notes ### When making changes In order for changes to the `nginx.conf` file or mock server to be reflected when running the tests, the dockers services must be restarted. This can be done by running `docker-compose down` before the above commands to parse the nginx.conf and run the tests. diff --git a/ci/docker/entrypoint.sh b/ci/docker/entrypoint.sh index 693bd54..15ca743 100755 --- a/ci/docker/entrypoint.sh +++ b/ci/docker/entrypoint.sh @@ -93,12 +93,6 @@ start_docker() { local docker_opts="${DOCKER_OPTS:-}" - # Pass through `--garden-mtu` from gardian container - if [[ "${docker_opts}" != *'--mtu'* ]]; then - local mtu="$(cat /sys/class/net/$(ip route get 8.8.8.8|awk '{ print $5 }')/mtu)" - docker_opts+=" --mtu ${mtu}" - fi - # Use Concourse's scratch volume to bypass the graph filesystem by default if [[ "${docker_opts}" != *'--data-root'* ]] && [[ "${docker_opts}" != *'--graph'* ]]; then docker_opts+=' --data-root /scratch/docker' diff --git a/ci/federalist-pipeline.yml b/ci/federalist-pipeline.yml deleted file mode 100644 index 2d349b8..0000000 --- a/ci/federalist-pipeline.yml +++ /dev/null @@ -1,318 +0,0 @@ ---- -############################ -# SHARED - -env-cf: &env-cf - CF_API: https://api.fr.cloud.gov - CF_USERNAME: ((production-cf-username)) - CF_PASSWORD: ((production-cf-password)) - CF_ORG: gsa-18f-federalist - CF_SPACE: production - CF_STACK: cflinuxfs4 - -node-image: &node-image - platform: linux - image_resource: - type: docker-image - source: - repository: node - tag: 14 - -cf-image: &cf-image - platform: linux - image_resource: - type: registry-image - source: - aws_access_key_id: ((ecr-aws-key)) - aws_secret_access_key: ((ecr-aws-secret)) - repository: harden-concourse-task - aws_region: us-gov-west-1 - tag: ((harden-concourse-task-tag)) - -test: &test - - in_parallel: - - get: nginx - params: {save: true} - - get: node - params: {save: true} - - in_parallel: - - task: test-mock - privileged: true - config: - platform: linux - image_resource: - type: docker-image - source: - repository: karlkfi/concourse-dcind - inputs: - - name: src - - name: nginx - - name: node - run: - dir: src - path: ci/docker/entrypoint.sh - args: - - bash - - -ceux - - | - pushd .. - docker load -i nginx/image - docker tag "$(cat nginx/image-id)" "$(cat nginx/repository):$(cat nginx/tag)" - docker load -i node/image - docker tag "$(cat node/image-id)" "$(cat node/repository):$(cat node/tag)" - popd - docker-compose -f docker-compose.yml run --no-deps app npm install - docker-compose -f docker-compose.yml run --no-deps app npm run parse - docker-compose -f docker-compose.yml run app npm test - docker-compose -f docker-compose.yml down - - task: test-integration - privileged: true - params: - DEDICATED_AWS_ACCESS_KEY_ID: ((dedicated-aws-access-key-id)) - DEDICATED_AWS_SECRET_ACCESS_KEY: ((dedicated-aws-secret-access-key)) - config: - platform: linux - image_resource: - type: docker-image - source: - repository: karlkfi/concourse-dcind - inputs: - - name: src - - name: nginx - - name: node - run: - dir: src - path: ci/docker/entrypoint.sh - args: - - bash - - -ceux - - | - pushd .. - docker load -i nginx/image - docker tag "$(cat nginx/image-id)" "$(cat nginx/repository):$(cat nginx/tag)" - docker load -i node/image - docker tag "$(cat node/image-id)" "$(cat node/repository):$(cat node/tag)" - popd - docker-compose -f docker-compose.yml run --no-deps app npm install - docker-compose -f docker-compose.yml run --no-deps app npm run parse:integration - docker-compose -f docker-compose.yml run app npm run test:integration - docker-compose -f docker-compose.yml run app npm run test:integration:website-config - docker-compose -f docker-compose.yml down - -############################ -# JOBS - -jobs: - - - name: set-pipeline - plan: - - get: src - resource: src-production - trigger: true - - set_pipeline: federalist-proxy - file: src/ci/federalist-pipeline.yml - - - name: test-pr-main - plan: - - get: src - resource: pr-main - trigger: true - version: every - - - put: src - resource: pr-main - params: - path: src - status: pending - base_context: concourse - context: test - - - do: *test - - on_failure: - put: src - resource: pr-main - params: - path: src - status: failure - base_context: concourse - context: test - - on_success: - put: src - resource: pr-main - params: - path: src - status: success - base_context: concourse - context: test - - - name: test-and-deploy-federalist-production - plan: - - get: src - resource: src-production - passed: [set-pipeline] - trigger: true - params: {depth: 1} - - put: gh-status - inputs: [src] - params: {state: pending} - - do: *test - - task: build-redirects - config: - <<: *node-image - inputs: [name: src] - outputs: [name: src] - params: - SITE_REDIRECTS: ((federalist-proxy-production-site-redirects)) - run: - dir: src - path: bash - args: [-c, npm run build-redirects] - - task: deploy - config: - <<: *cf-image - inputs: [name: src] - run: - dir: src - path: ci/tasks/deploy.sh - params: - <<: *env-cf - CF_APP_NAME: federalist-proxy - CF_MANIFEST: .cloudgov/manifest.yml - CF_VARS_FILE: .cloudgov/vars/federalist-production.yml - on_failure: - try: - task: cancel-api-deployment - config: - <<: *cf-image - inputs: [name: src] - run: - dir: src - path: ci/tasks/cancel-deployment.sh - params: - <<: *env-cf - CF_APP_NAME: federalist-proxy - on_failure: - in_parallel: - - put: gh-status - inputs: [src] - params: {state: failure} - - put: slack - params: - text: | - :x: FAILED: federalist-proxy deployment on production - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - channel: ((slack-channel)) - username: ((slack-username)) - icon_url: ((slack-icon-url)) - on_success: - in_parallel: - - put: gh-status - inputs: [src] - params: {state: success} - - put: slack - params: - text: | - :white_check_mark: SUCCESS: Successfully deployed federalist-proxy on production - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - channel: ((slack-channel)) - username: ((slack-username)) - icon_url: ((slack-icon-url)) - - - name: nightly-tasks-production - plan: - - get: src - resource: src-production - passed: [set-pipeline] - params: {depth: 1} - - get: nightly - trigger: true - - task: restage - config: - <<: *cf-image - inputs: [name: src] - run: - dir: src - path: ci/tasks/restage.sh - params: - <<: *env-cf - CF_APP_NAME: federalist-proxy - -############################ -# RESOURCES - -resources: - - - name: src-production - type: git - icon: github - source: - uri: ((git-base-url))/((proxy-repository-path)) - branch: main - commit_verification_keys: ((cloud-gov-pages-gpg-keys)) - - - name: pr-main - type: pull-request - check_every: 1m - source: - repository: ((proxy-repository-path)) - access_token: ((gh-access-token)) - base_branch: main - disable_forks: true - ignore_drafts: false - - - name: nightly - type: time - source: - start: 12:00 AM - stop: 1:00 AM - location: America/New_York - - - name: nginx - type: docker-image - source: - repository: nginx - tag: 1 - - - name: node - type: docker-image - source: - repository: node - tag: 18 - - - name: slack - type: slack-notification - source: - url: ((slack-webhook-url)) - - - name: gh-status - type: cogito - check_every: 1h - source: - owner: cloud-gov - repo: pages-proxy - access_token: ((gh-access-token)) - context_prefix: concourse - -############################ -# RESOURCE TYPES - -resource_types: - - - name: cogito - type: docker-image - check_every: 24h - source: - repository: pix4d/cogito - - - name: slack-notification - type: docker-image - source: - repository: cfcommunity/slack-notification-resource - - - name: pull-request - type: docker-image - source: - repository: teliaoss/github-pr-resource - diff --git a/ci/pipeline-dev.yml b/ci/pipeline-dev.yml index 198369c..d78d17f 100644 --- a/ci/pipeline-dev.yml +++ b/ci/pipeline-dev.yml @@ -13,10 +13,13 @@ env-cf: &env-cf node-image: &node-image platform: linux image_resource: - type: docker-image + type: registry-image source: - repository: node - tag: 18 + aws_access_key_id: ((ecr-aws-key)) + aws_secret_access_key: ((ecr-aws-secret)) + repository: pages-node-v20 + aws_region: us-gov-west-1 + tag: latest cf-image: &cf-image platform: linux @@ -33,21 +36,22 @@ test: &test - in_parallel: - get: nginx params: { save: true } - - get: node - params: { save: true } - in_parallel: - task: test-mock privileged: true config: platform: linux image_resource: - type: docker-image + type: registry-image source: - repository: karlkfi/concourse-dcind + aws_access_key_id: ((ecr-aws-key)) + aws_secret_access_key: ((ecr-aws-secret)) + repository: pages-dind-v25 + aws_region: us-gov-west-1 + tag: latest inputs: - name: src - name: nginx - - name: node run: dir: src path: ci/docker/entrypoint.sh @@ -58,8 +62,6 @@ test: &test pushd .. docker load -i nginx/image docker tag "$(cat nginx/image-id)" "$(cat nginx/repository):$(cat nginx/tag)" - docker load -i node/image - docker tag "$(cat node/image-id)" "$(cat node/repository):$(cat node/tag)" popd docker-compose -f docker-compose.yml run --no-deps app npm install docker-compose -f docker-compose.yml run --no-deps app npm run parse @@ -73,13 +75,16 @@ test: &test config: platform: linux image_resource: - type: docker-image + type: registry-image source: - repository: karlkfi/concourse-dcind + aws_access_key_id: ((ecr-aws-key)) + aws_secret_access_key: ((ecr-aws-secret)) + repository: pages-dind-v25 + aws_region: us-gov-west-1 + tag: latest inputs: - name: src - name: nginx - - name: node run: dir: src path: ci/docker/entrypoint.sh @@ -90,8 +95,6 @@ test: &test pushd .. docker load -i nginx/image docker tag "$(cat nginx/image-id)" "$(cat nginx/repository):$(cat nginx/tag)" - docker load -i node/image - docker tag "$(cat node/image-id)" "$(cat node/repository):$(cat node/tag)" popd docker-compose -f docker-compose.yml run --no-deps app npm install docker-compose -f docker-compose.yml run --no-deps app npm run parse:integration @@ -222,12 +225,6 @@ resources: repository: nginx tag: 1 - - name: node - type: docker-image - source: - repository: node - tag: 18 - - name: slack type: slack-notification source: @@ -237,12 +234,30 @@ resources: # RESOURCE TYPES resource_types: + - name: slack-notification - type: docker-image + type: registry-image source: - repository: cfcommunity/slack-notification-resource + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: slack-notification-resource + aws_region: us-gov-west-1 + tag: latest - name: pull-request - type: docker-image + type: registry-image source: - repository: teliaoss/github-pr-resource + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: github-pr-resource + aws_region: us-gov-west-1 + tag: latest + + - name: time + type: registry-image + source: + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: time-resource + aws_region: us-gov-west-1 + tag: latest diff --git a/ci/pipeline.yml b/ci/pipeline.yml index d0ee735..5e140b8 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -13,10 +13,13 @@ env-cf: &env-cf node-image: &node-image platform: linux image_resource: - type: docker-image + type: registry-image source: - repository: node - tag: 18 + aws_access_key_id: ((ecr-aws-key)) + aws_secret_access_key: ((ecr-aws-secret)) + repository: pages-node-v20 + aws_region: us-gov-west-1 + tag: latest cf-image: &cf-image platform: linux @@ -33,24 +36,25 @@ test: &test - in_parallel: - get: nginx params: {save: true} - - get: node - params: {save: true} - in_parallel: - task: test-mock privileged: true config: platform: linux image_resource: - type: docker-image + type: registry-image source: - repository: karlkfi/concourse-dcind + aws_access_key_id: ((ecr-aws-key)) + aws_secret_access_key: ((ecr-aws-secret)) + repository: pages-dind-v25 + aws_region: us-gov-west-1 + tag: latest inputs: - name: src - name: nginx - - name: node run: dir: src - path: ci/docker/entrypoint.sh + path: /bin/bash args: - bash - -ceux @@ -58,8 +62,6 @@ test: &test pushd .. docker load -i nginx/image docker tag "$(cat nginx/image-id)" "$(cat nginx/repository):$(cat nginx/tag)" - docker load -i node/image - docker tag "$(cat node/image-id)" "$(cat node/repository):$(cat node/tag)" popd docker-compose -f docker-compose.yml run --no-deps app npm install docker-compose -f docker-compose.yml run --no-deps app npm run parse @@ -73,16 +75,19 @@ test: &test config: platform: linux image_resource: - type: docker-image + type: registry-image source: - repository: karlkfi/concourse-dcind + aws_access_key_id: ((ecr-aws-key)) + aws_secret_access_key: ((ecr-aws-secret)) + repository: pages-dind-v25 + aws_region: us-gov-west-1 + tag: latest inputs: - name: src - name: nginx - - name: node run: dir: src - path: ci/docker/entrypoint.sh + path: /bin/bash args: - bash - -ceux @@ -90,8 +95,6 @@ test: &test pushd .. docker load -i nginx/image docker tag "$(cat nginx/image-id)" "$(cat nginx/repository):$(cat nginx/tag)" - docker load -i node/image - docker tag "$(cat node/image-id)" "$(cat node/repository):$(cat node/tag)" popd docker-compose -f docker-compose.yml run --no-deps app npm install docker-compose -f docker-compose.yml run --no-deps app npm run parse:integration @@ -128,7 +131,7 @@ jobs: path: src status: pending base_context: concourse - context: test + context: test-pr-proxy-((deploy-env)) - do: *test @@ -139,7 +142,7 @@ jobs: path: src status: failure base_context: concourse - context: test + context: test-pr-proxy-((deploy-env)) on_success: put: src @@ -148,7 +151,7 @@ jobs: path: src status: success base_context: concourse - context: test + context: test-pr-proxy-((deploy-env)) - name: test-and-deploy-((deploy-env)) plan: @@ -157,9 +160,13 @@ jobs: passed: [set-pipeline] trigger: true params: {depth: 1} - - put: gh-status - inputs: [src] - params: {state: pending} + - put: src + resource: pr-((git-branch)) + params: + path: src + status: pending + base_context: concourse + context: test-and-deploy-proxy-((deploy-env)) - do: *test - task: build-redirects config: @@ -198,9 +205,13 @@ jobs: CF_APP_NAME: pages-proxy-((deploy-env)) on_failure: in_parallel: - - put: gh-status - inputs: [src] - params: {state: failure} + - put: src + resource: pr-((git-branch)) + params: + path: src + status: failure + base_context: concourse + context: test-and-deploy-proxy-((deploy-env)) - put: slack params: text: | @@ -211,9 +222,13 @@ jobs: icon_url: ((slack-icon-url)) on_success: in_parallel: - - put: gh-status - inputs: [src] - params: {state: success} + - put: src + resource: pr-((git-branch)) + params: + path: src + status: success + base_context: concourse + context: test-and-deploy-proxy-((deploy-env)) - put: slack params: text: | @@ -278,44 +293,39 @@ resources: repository: nginx tag: 1 - - name: node - type: docker-image - source: - repository: node - tag: 18 - - name: slack type: slack-notification source: url: ((slack-webhook-url)) - - name: gh-status - type: cogito - check_every: 1h - source: - owner: cloud-gov - repo: pages-proxy - access_token: ((gh-access-token)) - context_prefix: concourse - ############################ # RESOURCE TYPES resource_types: - - name: cogito - type: docker-image - check_every: 24h - source: - repository: pix4d/cogito - - name: slack-notification - type: docker-image + type: registry-image source: - repository: cfcommunity/slack-notification-resource + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: slack-notification-resource + aws_region: us-gov-west-1 + tag: latest - name: pull-request - type: docker-image + type: registry-image source: - repository: teliaoss/github-pr-resource + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: github-pr-resource + aws_region: us-gov-west-1 + tag: latest + - name: time + type: registry-image + source: + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: time-resource + aws_region: us-gov-west-1 + tag: latest diff --git a/docker-compose.yml b/docker-compose.yml index 7dc6af4..1616975 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: mock-dedicated: - image: node:18 + image: node:20 volumes: - .:/app working_dir: /app diff --git a/package-lock.json b/package-lock.json index f3b4f9c..5e32260 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "supertest": "^4.0.2" }, "engines": { - "node": "18.x.x" + "node": "20.x.x" } }, "node_modules/@ungap/promise-all-settled": { @@ -575,9 +575,9 @@ } }, "node_modules/get-func-name": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", - "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", + "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==", "engines": { "node": "*" } @@ -1374,10 +1374,9 @@ } }, "node_modules/superagent/node_modules/debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", - "deprecated": "Debug versions >=3.2.0 <3.2.7 || >=4 <4.3.1 have a low-severity ReDos regression when used in a Node.js environment. It is recommended you upgrade to 3.2.7 or 4.3.1. (https://github.com/visionmedia/debug/issues/797)", + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", "dependencies": { "ms": "^2.1.1" } @@ -2033,9 +2032,9 @@ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, "get-func-name": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", - "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=" + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", + "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==" }, "get-intrinsic": { "version": "1.2.0", @@ -2598,9 +2597,9 @@ }, "dependencies": { "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", "requires": { "ms": "^2.1.1" } diff --git a/package.json b/package.json index 5ef78e6..3d1d70d 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "author": "Amir Reavis-Bey", "private": true, "engines": { - "node": "18.x.x" + "node": "20.x.x" }, "scripts": { "build-redirects": "node ./bin/build-redirects.js",