From bdcd29c923dc38cf2cf8b743ec8a4f197d331281 Mon Sep 17 00:00:00 2001 From: Serhii Koropets <33310880+koropets@users.noreply.github.com> Date: Wed, 1 Dec 2021 10:59:31 +0200 Subject: [PATCH] PyPi fix (#1203) * --index-url for pip * Latest dev for gordo-dataset and gordo-client * full_requirements.txt * Try to fix Dockerfile * pip_index_url * Additional pip_index_url * Managing secrets * Test twine creds * --index-url for release.yaml * No needs to use INDEX_URL with requirements/test_requirements.txt * Revert test changes * Update full_requirements.txt --- .github/workflows/main.yml | 50 +----------------------------- .github/workflows/master-ci.yml | 4 +++ .github/workflows/release.yml | 7 ++--- Dockerfile | 6 ++-- requirements/full_requirements.txt | 6 ++-- requirements/requirements.in | 4 +-- 6 files changed, 16 insertions(+), 61 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0ff10e21f..838511750 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,56 +31,8 @@ jobs: - name: Install run: | pip install --upgrade pip==20.1.1 - pip install -r requirements/full_requirements.txt + pip install -i "${{ secrets.INDEX_URL }}" -r requirements/full_requirements.txt pip install -r requirements/test_requirements.txt - name: Test ${{ matrix.component }} run: python setup.py test${{ matrix.component }} - - - name: Save Coverage - run: | - mkdir -p /tmp/coverage-reports - mv ./coverage.xml /tmp/coverage-reports/${{ matrix.component }}-coverage.xml - mv ./junit/junit.xml /tmp/coverage-reports/${{ matrix.component }}-junit.xml - - - uses: actions/upload-artifact@v1 - with: - name: Coverage Reports - path: /tmp/coverage-reports - - upload-coverage-reports: - runs-on: ubuntu-latest - needs: test - steps: - - uses: actions/checkout@v1 - - - uses: actions/download-artifact@v1 - with: - name: Coverage Reports - path: /tmp/coverage-reports - - - name: Copy Coverage Reports - run: cp /tmp/coverage-reports/*.xml . - - - uses: codecov/codecov-action@v1 - with: - token: fc2844a2-5a6c-43ef-a758-05bc50562b14 - fail_ci_if_error: false - - build-docs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - - uses: actions/setup-python@v1 - with: - python-version: '3.7' # Version range or exact version of a Python version to use, using semvers version range syntax. - architecture: 'x64' - - - name: Install deps - run: | - pip install --upgrade pip - pip install .[docs] - - - name: Build Docs - run: make docs diff --git a/.github/workflows/master-ci.yml b/.github/workflows/master-ci.yml index 4b96322a1..30ac62991 100644 --- a/.github/workflows/master-ci.yml +++ b/.github/workflows/master-ci.yml @@ -54,6 +54,8 @@ jobs: with: context: . file: ./Dockerfile + secrets: | + pip_index_url=${{ secrets.INDEX_URL }} tags: ${{ steps.prep.outputs.base_image }} load: true @@ -71,6 +73,8 @@ jobs: file: ./Dockerfile build-args: | BASE_IMAGE=${{ steps.prep.outputs.base_image }} + secrets: | + pip_index_url=${{ secrets.INDEX_URL }} tags: ${{ steps.prep.outputs.tags_gordo_base }} labels: | org.opencontainers.image.title=gordo-base diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d2ff052a7..c4cb8c62b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,7 +25,7 @@ jobs: run: | pip install --upgrade pip pip uninstall -y aws-sam-cli pyyaml # Workaround for fixing an issue with incompatible package versions - pip install -r requirements/full_requirements.txt + pip install -i "${{ secrets.INDEX_URL }}" -r requirements/full_requirements.txt - name: Build run: | @@ -37,8 +37,5 @@ jobs: pip install twine - name: Publish - env: - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - TWINE_USERNAME: ${{ secrets.TWINE_USER }} run: | - twine upload --skip-existing dist/* + twine upload --skip-existing --repository-url "${{ secrets.PYPI_REPOSITORY }}" -u "${{ secrets.PYPI_USER }}" -p "${{ secrets.PYPI_PASSWORD }}" dist/* diff --git a/Dockerfile b/Dockerfile index 256d2e8c5..a3add7b80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,10 +38,12 @@ ENV PATH "${HOME}/.local/bin:${PATH}" # Install requirements separately for improved docker caching COPY --from=builder /code/prereq.txt . -RUN pip install --no-deps -r prereq.txt --no-cache-dir +RUN --mount=type=secret,id=pip_index_url \ + pip install -i "$(cat /run/secrets/pip_index_url)" --no-deps -r prereq.txt --no-cache-dir COPY requirements/full_requirements.txt . -RUN pip install -r full_requirements.txt --no-cache-dir +RUN --mount=type=secret,id=pip_index_url \ + pip install -i "$(cat /run/secrets/pip_index_url)" -r full_requirements.txt --no-cache-dir # Install gordo, packaged from earlier 'python setup.py sdist' COPY --from=builder /code/dist/gordo-packed.tar.gz . diff --git a/requirements/full_requirements.txt b/requirements/full_requirements.txt index fd9489cbb..e98c47756 100644 --- a/requirements/full_requirements.txt +++ b/requirements/full_requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.7 # To update, run: # -# pip-compile --output-file=full_requirements.txt mlflow_requirements.in postgres_requirements.in requirements.in +# pip-compile --no-emit-index-url --output-file=full_requirements.txt mlflow_requirements.in postgres_requirements.in requirements.in # absl-py==0.11.0 # via @@ -147,9 +147,9 @@ google-auth-oauthlib==0.4.1 # via tensorboard google-pasta==0.2.0 # via tensorflow -gordo-client==4.0.2 +gordo-client==4.1.0 # via -r requirements.in -gordo-dataset==3.1.1 +gordo-dataset==3.2.0 # via # -r requirements.in # gordo-client diff --git a/requirements/requirements.in b/requirements/requirements.in index 0a75fbb81..2b3710950 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -24,8 +24,8 @@ typing_extensions~=3.7 prometheus_client~=0.7.1 azure-identity~=1.4.0 PyYAML~=5.4 -gordo-dataset~=3.1.1 +gordo-dataset~=3.2.0 jeepney>=0.6 packaging~=20.7 pydantic>=1.7.4 -gordo-client~=4.0.2 +gordo-client~=4.1.0