PyPi fix (#1203)

* --index-url for pip

* Latest dev for gordo-dataset and gordo-client

* full_requirements.txt

* Try to fix Dockerfile

* pip_index_url

* Additional pip_index_url

* Managing secrets

* Test twine creds

* --index-url for release.yaml

* No needs to use INDEX_URL with requirements/test_requirements.txt

* Revert test changes

* Update full_requirements.txt

.github/workflows/main.yml

@@ -31,56 +31,8 @@ jobs:
       - name: Install
         run: |
           pip install --upgrade pip==20.1.1
-          pip install -r requirements/full_requirements.txt
+          pip install -i "${{ secrets.INDEX_URL }}" -r requirements/full_requirements.txt
           pip install -r requirements/test_requirements.txt
 
       - name: Test ${{ matrix.component }}
         run: python setup.py test${{ matrix.component }}
-
-      - name: Save Coverage
-        run: |
-          mkdir -p /tmp/coverage-reports
-          mv ./coverage.xml /tmp/coverage-reports/${{ matrix.component }}-coverage.xml
-          mv ./junit/junit.xml /tmp/coverage-reports/${{ matrix.component }}-junit.xml
-
-      - uses: actions/upload-artifact@v1
-        with:
-          name: Coverage Reports
-          path: /tmp/coverage-reports
-
-  upload-coverage-reports:
-    runs-on: ubuntu-latest
-    needs: test
-    steps:
-      - uses: actions/checkout@v1
-
-      - uses: actions/download-artifact@v1
-        with:
-          name: Coverage Reports
-          path: /tmp/coverage-reports
-
-      - name: Copy Coverage Reports
-        run: cp /tmp/coverage-reports/*.xml .
-
-      - uses: codecov/codecov-action@v1
-        with:
-          token: fc2844a2-5a6c-43ef-a758-05bc50562b14
-          fail_ci_if_error: false
-
-  build-docs:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-
-      - uses: actions/setup-python@v1
-        with:
-          python-version: '3.7' # Version range or exact version of a Python version to use, using semvers version range syntax.
-          architecture: 'x64'
-
-      - name: Install deps
-        run: |
-          pip install --upgrade pip
-          pip install .[docs]
-
-      - name: Build Docs
-        run: make docs

.github/workflows/master-ci.yml

@@ -54,6 +54,8 @@ jobs:
         with:
           context: .
           file: ./Dockerfile
+          secrets: |
+            pip_index_url=${{ secrets.INDEX_URL }}
           tags: ${{ steps.prep.outputs.base_image }}
           load: true
 
@@ -71,6 +73,8 @@ jobs:
           file: ./Dockerfile
           build-args: |
             BASE_IMAGE=${{ steps.prep.outputs.base_image }}
+          secrets: |
+            pip_index_url=${{ secrets.INDEX_URL }}
           tags: ${{ steps.prep.outputs.tags_gordo_base }}
           labels: |
             org.opencontainers.image.title=gordo-base

.github/workflows/release.yml

@@ -25,7 +25,7 @@ jobs:
         run: |
           pip install --upgrade pip
           pip uninstall -y aws-sam-cli pyyaml # Workaround for fixing an issue with incompatible package versions
-          pip install -r requirements/full_requirements.txt
+          pip install -i "${{ secrets.INDEX_URL }}" -r requirements/full_requirements.txt
 
       - name: Build
         run: |
@@ -37,8 +37,5 @@ jobs:
           pip install twine
 
       - name: Publish
-        env:
-          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-          TWINE_USERNAME: ${{ secrets.TWINE_USER }}
         run: |
-          twine upload --skip-existing dist/*
+          twine upload --skip-existing --repository-url "${{ secrets.PYPI_REPOSITORY }}" -u "${{ secrets.PYPI_USER }}" -p "${{ secrets.PYPI_PASSWORD }}" dist/*

Dockerfile

@@ -38,10 +38,12 @@ ENV PATH "${HOME}/.local/bin:${PATH}"
 
 # Install requirements separately for improved docker caching
 COPY --from=builder /code/prereq.txt .
-RUN pip install --no-deps -r prereq.txt --no-cache-dir
+RUN --mount=type=secret,id=pip_index_url \
+    pip install -i "$(cat /run/secrets/pip_index_url)" --no-deps -r prereq.txt --no-cache-dir
 
 COPY requirements/full_requirements.txt .
-RUN pip install -r full_requirements.txt --no-cache-dir
+RUN --mount=type=secret,id=pip_index_url \
+    pip install -i "$(cat /run/secrets/pip_index_url)" -r full_requirements.txt --no-cache-dir
 
 # Install gordo, packaged from earlier 'python setup.py sdist'
 COPY --from=builder /code/dist/gordo-packed.tar.gz .

requirements/full_requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with python 3.7
 # To update, run:
 #
-#    pip-compile --output-file=full_requirements.txt mlflow_requirements.in postgres_requirements.in requirements.in
+#    pip-compile --no-emit-index-url --output-file=full_requirements.txt mlflow_requirements.in postgres_requirements.in requirements.in
 #
 absl-py==0.11.0
     # via
@@ -147,9 +147,9 @@ google-auth-oauthlib==0.4.1
     # via tensorboard
 google-pasta==0.2.0
     # via tensorflow
-gordo-client==4.0.2
+gordo-client==4.1.0
     # via -r requirements.in
-gordo-dataset==3.1.1
+gordo-dataset==3.2.0
     # via
     #   -r requirements.in
     #   gordo-client

requirements/requirements.in

@@ -24,8 +24,8 @@ typing_extensions~=3.7
 prometheus_client~=0.7.1
 azure-identity~=1.4.0
 PyYAML~=5.4
-gordo-dataset~=3.1.1
+gordo-dataset~=3.2.0
 jeepney>=0.6
 packaging~=20.7
 pydantic>=1.7.4
-gordo-client~=4.0.2
+gordo-client~=4.1.0