Merge branch 'master' into transfer-retries-fix

.github/workflows/pulsar.yaml

@@ -3,13 +3,13 @@ on: [push, pull_request]
 jobs:
   lint:
     name: Lint
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         tox-env: [py37-lint, py37-dist, py37-docs]
     steps:
     - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v3
       with:
         python-version: 3.7
     - name: Install tox
@@ -18,7 +18,7 @@ jobs:
       run: tox -e ${{ matrix.tox-env }}
   mypy:
     name: MyPy
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         include:
@@ -30,7 +30,7 @@ jobs:
           python: 3.8
     steps:
     - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python }}
     - name: Install tox
@@ -41,52 +41,77 @@ jobs:
       run: tox -e ${{ matrix.tox-env }}
   test:
     name: Run Tests
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         include:
-        - tox-env: py36-test
+        - tox-env: py36-test-ci
           python: 3.6
         - tox-env: py36-test-unit
           python: 3.6
-        - tox-env: py37-test
+        - tox-env: py37-test-ci
           python: 3.7
         - tox-env: py37-test-unit
           python: 3.7
-        - tox-env: py38-test
+        - tox-env: py38-test-ci
           python: 3.8
         - tox-env: py38-test-unit
           python: 3.8
-        - tox-env: py39-test
+        - tox-env: py39-test-ci
           python: 3.9
         - tox-env: py39-test-unit
           python: 3.9
+        - tox-env: py39-install-wheel-no-conda
+          python: 3.9
         - tox-env: py37-install-wheel
           python: 3.7
+    services:
+      job-files:
+        image: galaxy/simple-job-files:latest
+        env:
+          JOB_FILES_ROOT_DIRECTORY: /tmp
+        ports:
+          - '8000:8000'
+        volumes:
+          - '/tmp:/tmp:rw'
     steps:
     - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python }}
     - name: Install tox
       run: pip install tox
     - name: Run setup
       run: .ci/setup_tests.sh
     - name: Run tests
-      run: export PULSAR_TEST_KEY=test_data/testkey; export TEST_WEBAPP_POST_SHUTDOWN_SLEEP=1; . local_env.sh; tox -e ${{ matrix.tox-env }}
-  # https://github.com/coverallsapp/github-action/issues/30
-  #  - name: Coveralls Parallel
-  #    uses: coverallsapp/github-action@master
-  #    with:
-  #      github-token: ${{ secrets.github_token }}
-  #      flag-name: run-${{ matrix.tox-env }}
-  #      parallel: true
-  #finish:
-  #  needs: test
-  #  runs-on: ubuntu-latest
-  #  steps:
-  #  - name: Coveralls Finished
-  #    uses: coverallsapp/github-action@master
-  #    with:
-  #      github-token: ${{ secrets.github_token }}
-  #      parallel-finished: true
+      run: . local_env.sh; tox -e ${{ matrix.tox-env }}
+      env:
+        PULSAR_TEST_KEY: "test_data/testkey"
+        PULSAR_TEST_EXTERNAL_JOB_FILES_URL: "http://localhost:8000"
+        PULSAR_TEST_EXTERNAL_JOB_FILES_DIRECTORY: "/tmp"
+        TEST_WEBAPP_POST_SHUTDOWN_SLEEP: 1
+  ## Can't get the following tests to work - they fall over on the containerized job not being able to
+  ## connect to the host running the Pulsar server for file transfers in stage out/in.
+  # tes-test:
+  #   name: Run Tests
+  # runs-on: ubuntu-20.04
+  #   strategy:
+  #     matrix:
+  #       include:
+  #       - tox-env: py37-test-funnel
+  #         python: 3.7
+  #   steps:
+  #   - uses: actions/checkout@v2
+  #   - uses: actions/setup-python@v3
+  #     with:
+  #       python-version: ${{ matrix.python }}
+  #   - name: Set up Go 1.x
+  #     uses: actions/setup-go@v2
+  #     with:
+  #       go-version: ^1.13
+  #   - name: Pre-fetch required docker containers
+  #     run: docker pull 'galaxy/pulsar-pod-staging:0.15.0.0' && docker pull 'conda/miniconda3'
+  #   - name: Install tox
+  #     run: pip install tox
+  #   - name: Run tests
+  #     run: PULSAR_TEST_INFRASTRUCTURE_HOST="_PLATFORM_AUTO_" FUNNEL_SERVER_TARGET=DEPLOY PULSAR_TES_SERVER_TARGET=http://localhost:8000/ tox -e ${{ matrix.tox-env }}

.gitignore

@@ -6,6 +6,7 @@ dist
 *.egg
 *.egg-info
 docs/_build
+docs/plantuml.jar
 cache
 *.pyc
 *~

.isort.cfg

@@ -0,0 +1,14 @@
+[settings]
+combine_as_imports=true
+extend_skip=doc/*,scripts/*
+force_alphabetical_sort_within_sections=true
+# Override force_grid_wrap value from profile=black, but black is still happy
+force_grid_wrap=2
+line_length=140
+no_lines_before=LOCALFOLDER
+profile=black
+reverse_relative=true
+skip_gitignore=true
+# Make isort run faster by skipping database
+skip_glob=database/*
+src_paths=pulsar,test

CONTRIBUTING.rst

@@ -117,8 +117,4 @@ Before you submit a pull request, check that it meets these guidelines:
 1. If the pull request adds functionality, the docs should ideally be updated.
    Put your new functionality into a function with a docstring. (Until the
    @jmchilton learns to do this consistently this is only a suggestion though.)
-2. The pull request should work for Python 2.6, 2.7, and 3.4. Check
-   https://travis-ci.org/galaxyproject/planemo/pull_requests
-   and make sure that the tests pass for all supported Python versions. The
-   tests are imperfect and Travis sometimes fails in a transient fashion so
-   this also isn't strictly required to pass.
+2. The pull request should work for Python 3.6 and later.

HISTORY.rst

@@ -6,10 +6,43 @@ History
 .. to_doc
 
 ---------------------
-0.15.0.dev0
+0.15.0.dev1
 ---------------------
 
-
+* Updated Galaxy+Pulsar container.
+* Rework container execution - generalize Kubernetes execution to allow it to work without a
+  message queue and to allow TES execution based on pydantic-tes (https://github.com/jmchilton/pydantic-tes).
+* Add documentation and diagrams for container execution scenarios.
+* Rework integration tests to use pytest more aggressively.
+* Fixes to CI to run more tests that weren't being executed because Tox was not sending
+  environment variables through to pytest.
+* Add option ``amqp_key_prefix`` to direct task queue naming while retaining simple
+  default manager names and such in container scheduling deployments.
+* Various typing and CI fixes.
+
+---------------------
+0.14.16 (2022-10-04)
+---------------------
+
+* Fix small regression related to building URLs for client action mapping.
+
+---------------------
+0.14.15 (2022-10-03)
+---------------------
+
+* Fix small regressions bugs in 0.14.14 - updating runner util code was bigger swap over
+  than it seemed.
+
+---------------------
+0.14.14 (2022-10-30)
+---------------------
+
+* Bring in updated Galaxy runner util code. `Pull Request 303`_
+* Fix recovering "lost" jobs where the job directory does not exist at
+  startup/recovery time (thanks to `@natefoo`_). `Pull Request 301`_
+* Use urlencode to encode path (thanks to `@mvdbeek`_). `Pull Request 299`_
+* Support the k8s_job_ttl_secs_after_finished option as in the Galaxy
+  Kubernetes runner (thanks to `@natefoo`_). `Pull Request 287`_
 
 ---------------------
 0.14.13 (2021-12-06)
@@ -418,8 +451,12 @@ History
 
 
 .. github_links
+.. _Pull Request 303: https://github.com/galaxyproject/pulsar/pull/303
+.. _Pull Request 301: https://github.com/galaxyproject/pulsar/pull/301
+.. _Pull Request 299: https://github.com/galaxyproject/pulsar/pull/299
 .. _Pull Request 295: https://github.com/galaxyproject/pulsar/pull/295
 .. _Pull Request 294: https://github.com/galaxyproject/pulsar/pull/294
+.. _Pull Request 287: https://github.com/galaxyproject/pulsar/pull/287
 .. _Pull Request 271: https://github.com/galaxyproject/pulsar/pull/271
 .. _Pull Request 274: https://github.com/galaxyproject/pulsar/pull/274
 .. _Pull Request 270: https://github.com/galaxyproject/pulsar/pull/270
@@ -548,4 +585,4 @@ History
 .. _@mvdbeek: https://github.com/mvdbeek
 .. _@tirkarthi: https://github.com/tirkarthi
 .. _@natefoo: https://github.com/natefoo
-.. _@nsoranzo: https://github.com/nsoranzo
+.. _@nsoranzo: https://github.com/nsoranzo

Makefile

@@ -128,6 +128,9 @@ open-rtd: docs
 open-project:
 	open $(PROJECT_URL) || xdg-open $(PROJECT_URL)
 
+format:  ## Format Python code base
+	$(IN_VENV) isort .
+
 dist: clean-build clean-pyc
 	$(IN_VENV) python setup.py sdist bdist_wheel
 	ls -l dist

dev-requirements.txt

@@ -2,8 +2,10 @@
 pycurl
 kombu
 pykube
+boto3
 
 # For testing
+simple-job-files
 pytest
 webtest
 coverage

docker/coexecutor/Dockerfile

@@ -34,6 +34,8 @@ RUN apt-get update \
 
 ADD pulsar_app-*-py2.py3-none-any.whl /pulsar_app-*-py2.py3-none-any.whl
 
-RUN pip install --no-cache-dir /pulsar_app-*-py2.py3-none-any.whl[galaxy_extended_metadata]
+RUN pip install --upgrade setuptools && pip install pyOpenSSL --upgrade && pip install cryptography --upgrade
+RUN pip install --no-cache-dir /pulsar_app-*-py2.py3-none-any.whl[galaxy_extended_metadata] && rm /pulsar_app-*-py2.py3-none-any.whl
+RUN pip install --upgrade 'importlib-metadata<5.0'
 RUN _pulsar-configure-galaxy-cvmfs
 RUN _pulsar-conda-init --conda_prefix=/pulsar_dependencies/conda

docker/coexecutor/Makefile

@@ -4,6 +4,6 @@ dist:
 	cd ../..; make dist; cp dist/pulsar*whl docker/coexecutor
 
 docker-image:
-	docker build -t 'galaxy/pulsar-pod-staging:0.14.1' .
+	docker build -t 'galaxy/pulsar-pod-staging:0.15.0.2' .
 
 all: dist docker-image