From 9ea3e6c9f1951b16621de23f9464b13d5e893684 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:27:03 +0500 Subject: [PATCH 1/8] feat: added dockerfile and docker image push workflow for program-intent-engagement --- .../push-program-intent-engagement-image.yaml | 64 +++++++++++++ .../program-intent-engagement.Dockerfile | 96 +++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 .github/workflows/push-program-intent-engagement-image.yaml create mode 100644 dockerfiles/program-intent-engagement.Dockerfile diff --git a/.github/workflows/push-program-intent-engagement-image.yaml b/.github/workflows/push-program-intent-engagement-image.yaml new file mode 100644 index 0000000..a921da4 --- /dev/null +++ b/.github/workflows/push-program-intent-engagement-image.yaml @@ -0,0 +1,64 @@ +name: Build and Push Program Intent Engagement Image + +on: + workflow_dispatch: + inputs: + branch: + description: "Target branch from which the source dockerfile from image will be sourced" + + schedule: + - cron: "0 4 * * 1-5" # UTC Time + +# Added for testing purposes. Will remove once the PR is finalised + pull_request: + branches: + - '**' + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + + steps: + - name: Get tag name + id: get-tag-name + uses: actions/github-script@v5 + with: + script: | + const tagName = "${{ github.event.inputs.branch }}" || 'latest'; + console.log('Will use tag: ' + tagName); + return tagName; + result-encoding: string + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push Dev Docker image + uses: docker/build-push-action@v6 + with: + file: ./dockerfiles/program-intent-engagement.Dockerfile + push: true + target: app + tags: edxops/program-intent-engagement-dev:${{ steps.get-tag-name.outputs.result }} + platforms: linux/amd64,linux/arm64 + + - name: Send failure notification + if: failure() + uses: dawidd6/action-send-mail@v3 + with: + server_address: email-smtp.us-east-1.amazonaws.com + server_port: 465 + username: ${{secrets.edx_smtp_username}} + password: ${{secrets.edx_smtp_password}} + subject: Push Image to docker.io/edxops failed in Program Intent Engagement + to: team-cosmonauts@edx.org + from: github-actions + body: Push Image to docker.io/edxops for Program Intent Engagement failed! For details see "github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/dockerfiles/program-intent-engagement.Dockerfile b/dockerfiles/program-intent-engagement.Dockerfile new file mode 100644 index 0000000..efb0815 --- /dev/null +++ b/dockerfiles/program-intent-engagement.Dockerfile @@ -0,0 +1,96 @@ +FROM ubuntu:focal as app +MAINTAINER sre@edx.org + + +# Packages installed: + +# language-pack-en locales; ubuntu locale support so that system utilities have a consistent +# language and time zone. + +# python; ubuntu doesnt ship with python, so this is the python we will use to run the application + +# python3-pip; install pip to install application requirements.txt files + +# libmysqlclient-dev; to install header files needed to use native C implementation for +# MySQL-python for performance gains. + +# pkg-config; mysqlclient>=2.2.0 requires pkg-config (https://github.com/PyMySQL/mysqlclient/issues/620) + +# libssl-dev; # mysqlclient wont install without this. + +# python3-dev; to install header files for python extensions; much wheel-building depends on this + +# gcc; for compiling python extensions distributed with python packages like mysql-client + +# ENV variables for Python 3.12 support +ARG PYTHON_VERSION=3.12 +ENV TZ=UTC +ENV TERM=xterm-256color +ENV DEBIAN_FRONTEND=noninteractive + +# software-properties-common is needed to setup Python 3.12 env +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + apt-add-repository -y ppa:deadsnakes/ppa + +# If you add a package here please include a comment above describing what it is used for +RUN apt-get update && apt-get -qy install --no-install-recommends \ + language-pack-en \ + locales \ + # libmysqlclient-dev header files needed to use native C implementation for MySQL-python for performance gains. + libmysqlclient-dev \ + # mysqlclient>=2.2.0 requires pkg-config (https://github.com/PyMySQL/mysqlclient/issues/620) + pkg-config \ + # mysqlclient wont install without libssl-dev + libssl-dev \ + build-essential \ + gcc \ + curl \ + python3-pip \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-distutils + + +# need to use virtualenv pypi package with Python 3.12 +RUN pip install --upgrade pip setuptools +RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} +RUN pip install virtualenv + +RUN pip install --upgrade pip setuptools +# delete apt package lists because we do not need them inflating our image +RUN rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Setup zoneinfo for Python 3.12 +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +ENV DJANGO_SETTINGS_MODULE program_intent_engagement.settings.production + +EXPOSE 18781 +RUN useradd -m --shell /bin/false app + +WORKDIR /edx/app/program-intent-engagement + +# cloning git repo +RUN curl -L https://github.com/edx/program-intent-engagement/archive/refs/heads/main.tar.gz | tar -xz --strip-components=1 + +ARG INTENT_MANAGEMENT_VENV_DIR="/edx/app/venvs/program-intent-management" +RUN virtualenv -p python${PYTHON_VERSION} --always-copy ${INTENT_MANAGEMENT_VENV_DIR} + +# Dependencies are installed as root so they cannot be modified by the application user. +RUN pip install -r requirements/production.txt + +RUN mkdir -p /edx/var/log + +# Code is owned by root so it cannot be modified by the application user. +# So we copy it before changing users. +USER app + +# Gunicorn 19 does not log to stdout or stderr by default. Once we are past gunicorn 19, the logging to STDOUT need not be specified. +CMD gunicorn --workers=2 --name program-intent-engagement -c /edx/app/program-intent-engagement/program_intent_engagement/docker_gunicorn_configuration.py --log-file - --max-requests=1000 program_intent_engagement.wsgi:application From 4fc20707a4c26546c3e7f89c035160949da9aae9 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:41:28 +0500 Subject: [PATCH 2/8] chore: Remove pull_request trigger from workflow --- .github/workflows/push-program-intent-engagement-image.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/push-program-intent-engagement-image.yaml b/.github/workflows/push-program-intent-engagement-image.yaml index a921da4..f3dceae 100644 --- a/.github/workflows/push-program-intent-engagement-image.yaml +++ b/.github/workflows/push-program-intent-engagement-image.yaml @@ -9,11 +9,6 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time -# Added for testing purposes. Will remove once the PR is finalised - pull_request: - branches: - - '**' - jobs: build-and-push-image: runs-on: ubuntu-latest From 299ba0bdfbd57f2c9cfb5121c15fe0bfea35e990 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Thu, 17 Oct 2024 13:00:04 +0500 Subject: [PATCH 3/8] perf: updated Dockerfile to optimize requirements installation and dependency caching --- .../workflows/push-program-intent-engagement-image.yaml | 6 +++++- dockerfiles/program-intent-engagement.Dockerfile | 9 ++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push-program-intent-engagement-image.yaml b/.github/workflows/push-program-intent-engagement-image.yaml index f3dceae..5294be7 100644 --- a/.github/workflows/push-program-intent-engagement-image.yaml +++ b/.github/workflows/push-program-intent-engagement-image.yaml @@ -9,6 +9,11 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time +# Added for testing purposes. Will remove once the PR is finalised + pull_request: + branches: + - '**' + jobs: build-and-push-image: runs-on: ubuntu-latest @@ -43,7 +48,6 @@ jobs: push: true target: app tags: edxops/program-intent-engagement-dev:${{ steps.get-tag-name.outputs.result }} - platforms: linux/amd64,linux/arm64 - name: Send failure notification if: failure() diff --git a/dockerfiles/program-intent-engagement.Dockerfile b/dockerfiles/program-intent-engagement.Dockerfile index efb0815..3f232fb 100644 --- a/dockerfiles/program-intent-engagement.Dockerfile +++ b/dockerfiles/program-intent-engagement.Dockerfile @@ -77,19 +77,22 @@ RUN useradd -m --shell /bin/false app WORKDIR /edx/app/program-intent-engagement -# cloning git repo -RUN curl -L https://github.com/edx/program-intent-engagement/archive/refs/heads/main.tar.gz | tar -xz --strip-components=1 +# Create required directories for requirements +RUN mkdir -p requirements ARG INTENT_MANAGEMENT_VENV_DIR="/edx/app/venvs/program-intent-management" RUN virtualenv -p python${PYTHON_VERSION} --always-copy ${INTENT_MANAGEMENT_VENV_DIR} # Dependencies are installed as root so they cannot be modified by the application user. +RUN curl -L -o requirements/production.txt https://raw.githubusercontent.com/edx/program-intent-engagement/main/requirements/production.txt RUN pip install -r requirements/production.txt RUN mkdir -p /edx/var/log +# Clone the repository +RUN curl -L https://github.com/edx/program-intent-engagement/archive/refs/heads/main.tar.gz | tar -xz --strip-components=1 + # Code is owned by root so it cannot be modified by the application user. -# So we copy it before changing users. USER app # Gunicorn 19 does not log to stdout or stderr by default. Once we are past gunicorn 19, the logging to STDOUT need not be specified. From 84421e8f9aae04fae9cdd1c089c00325cdeafb51 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Thu, 17 Oct 2024 13:13:27 +0500 Subject: [PATCH 4/8] refactor: updated dockerfile --- dockerfiles/program-intent-engagement.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerfiles/program-intent-engagement.Dockerfile b/dockerfiles/program-intent-engagement.Dockerfile index 3f232fb..25787d8 100644 --- a/dockerfiles/program-intent-engagement.Dockerfile +++ b/dockerfiles/program-intent-engagement.Dockerfile @@ -87,11 +87,11 @@ RUN virtualenv -p python${PYTHON_VERSION} --always-copy ${INTENT_MANAGEMENT_VENV RUN curl -L -o requirements/production.txt https://raw.githubusercontent.com/edx/program-intent-engagement/main/requirements/production.txt RUN pip install -r requirements/production.txt -RUN mkdir -p /edx/var/log - # Clone the repository RUN curl -L https://github.com/edx/program-intent-engagement/archive/refs/heads/main.tar.gz | tar -xz --strip-components=1 +RUN mkdir -p /edx/var/log + # Code is owned by root so it cannot be modified by the application user. USER app From a8cee9acd22b45dfb63e7b7dc6a7ecc19af418e8 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Thu, 17 Oct 2024 13:19:05 +0500 Subject: [PATCH 5/8] refactor: updated dockerfile --- dockerfiles/program-intent-engagement.Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dockerfiles/program-intent-engagement.Dockerfile b/dockerfiles/program-intent-engagement.Dockerfile index 25787d8..692d7e7 100644 --- a/dockerfiles/program-intent-engagement.Dockerfile +++ b/dockerfiles/program-intent-engagement.Dockerfile @@ -75,6 +75,9 @@ ENV DJANGO_SETTINGS_MODULE program_intent_engagement.settings.production EXPOSE 18781 RUN useradd -m --shell /bin/false app +# Clone the repository +RUN curl -L https://github.com/edx/program-intent-engagement/archive/refs/heads/main.tar.gz | tar -xz --strip-components=1 + WORKDIR /edx/app/program-intent-engagement # Create required directories for requirements @@ -87,8 +90,6 @@ RUN virtualenv -p python${PYTHON_VERSION} --always-copy ${INTENT_MANAGEMENT_VENV RUN curl -L -o requirements/production.txt https://raw.githubusercontent.com/edx/program-intent-engagement/main/requirements/production.txt RUN pip install -r requirements/production.txt -# Clone the repository -RUN curl -L https://github.com/edx/program-intent-engagement/archive/refs/heads/main.tar.gz | tar -xz --strip-components=1 RUN mkdir -p /edx/var/log From f48305a9d627adc6fcdababcc5ab64f10c3fb010 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Thu, 17 Oct 2024 13:29:40 +0500 Subject: [PATCH 6/8] chore: Remove pull_request trigger from workflow --- .github/workflows/push-program-intent-engagement-image.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/push-program-intent-engagement-image.yaml b/.github/workflows/push-program-intent-engagement-image.yaml index 5294be7..7ecd972 100644 --- a/.github/workflows/push-program-intent-engagement-image.yaml +++ b/.github/workflows/push-program-intent-engagement-image.yaml @@ -9,11 +9,6 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time -# Added for testing purposes. Will remove once the PR is finalised - pull_request: - branches: - - '**' - jobs: build-and-push-image: runs-on: ubuntu-latest From d061e7ad68c659dc5c5a00765d826975b0de18ab Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Fri, 18 Oct 2024 08:24:10 +0500 Subject: [PATCH 7/8] refactor: refactor: moved repo clone step in Dockerfile --- .github/workflows/push-program-intent-engagement-image.yaml | 5 +++++ dockerfiles/program-intent-engagement.Dockerfile | 5 ++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push-program-intent-engagement-image.yaml b/.github/workflows/push-program-intent-engagement-image.yaml index 7ecd972..5294be7 100644 --- a/.github/workflows/push-program-intent-engagement-image.yaml +++ b/.github/workflows/push-program-intent-engagement-image.yaml @@ -9,6 +9,11 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time +# Added for testing purposes. Will remove once the PR is finalised + pull_request: + branches: + - '**' + jobs: build-and-push-image: runs-on: ubuntu-latest diff --git a/dockerfiles/program-intent-engagement.Dockerfile b/dockerfiles/program-intent-engagement.Dockerfile index 692d7e7..25787d8 100644 --- a/dockerfiles/program-intent-engagement.Dockerfile +++ b/dockerfiles/program-intent-engagement.Dockerfile @@ -75,9 +75,6 @@ ENV DJANGO_SETTINGS_MODULE program_intent_engagement.settings.production EXPOSE 18781 RUN useradd -m --shell /bin/false app -# Clone the repository -RUN curl -L https://github.com/edx/program-intent-engagement/archive/refs/heads/main.tar.gz | tar -xz --strip-components=1 - WORKDIR /edx/app/program-intent-engagement # Create required directories for requirements @@ -90,6 +87,8 @@ RUN virtualenv -p python${PYTHON_VERSION} --always-copy ${INTENT_MANAGEMENT_VENV RUN curl -L -o requirements/production.txt https://raw.githubusercontent.com/edx/program-intent-engagement/main/requirements/production.txt RUN pip install -r requirements/production.txt +# Clone the repository +RUN curl -L https://github.com/edx/program-intent-engagement/archive/refs/heads/main.tar.gz | tar -xz --strip-components=1 RUN mkdir -p /edx/var/log From 4f747596cdb1d540050d841e68bf372fd6664543 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Fri, 18 Oct 2024 08:29:24 +0500 Subject: [PATCH 8/8] chore: Remove pull_request trigger from workflow --- .github/workflows/push-program-intent-engagement-image.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/push-program-intent-engagement-image.yaml b/.github/workflows/push-program-intent-engagement-image.yaml index 5294be7..7ecd972 100644 --- a/.github/workflows/push-program-intent-engagement-image.yaml +++ b/.github/workflows/push-program-intent-engagement-image.yaml @@ -9,11 +9,6 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time -# Added for testing purposes. Will remove once the PR is finalised - pull_request: - branches: - - '**' - jobs: build-and-push-image: runs-on: ubuntu-latest