From 1d78914631f862355d217bb94a842030275e70a2 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Thu, 21 Sep 2023 16:26:06 +0100
Subject: [PATCH] Remove PYTHONPATH items from sys.path Add CYLC_PYTHONPATH
 items to sys.path

---
 changes.d/5727.break.md                       |  2 ++
 .../etc/python-job.settings                   |  4 +--
 cylc/flow/scripts/cylc.py                     | 29 +++++++++++++++++--
 tests/unit/scripts/test_cylc.py               | 28 +++++++++++++++++-
 4 files changed, 58 insertions(+), 5 deletions(-)
 create mode 100644 changes.d/5727.break.md

diff --git a/changes.d/5727.break.md b/changes.d/5727.break.md
new file mode 100644
index 00000000000..8d0a1cd2314
--- /dev/null
+++ b/changes.d/5727.break.md
@@ -0,0 +1,2 @@
+Cylc now ignores `PYTHONPATH` to make it more robust to task environments which set this value.
+If you want to add to the Cylc environment itself, e.g. to install a Cylc extension, use `CYLC_PYTHONPATH`.
\ No newline at end of file
diff --git a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings
index 15de1f8ea13..a1366917d8f 100644
--- a/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings
+++ b/cylc/flow/etc/tutorial/cylc-forecasting-workflow/etc/python-job.settings
@@ -7,6 +7,6 @@
         [[[environment]]]
             # These environment variables ensure that tasks can
             # run in the same environment as the workflow:
-            {% from "sys" import path, executable %}
-            PYTHONPATH = {{':'.join(path)}}
+            {% from "sys" import executable %}
+            PYTHONPATH = {{executable}}/../lib/python:$PYTHONPATH
             PATH = $(dirname {{executable}}):$PATH
diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py
index 4f13dc59496..c65b4c8c9ba 100644
--- a/cylc/flow/scripts/cylc.py
+++ b/cylc/flow/scripts/cylc.py
@@ -16,10 +16,35 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """cylc main entry point"""
 
-import argparse
-from contextlib import contextmanager
 import os
 import sys
+
+
+def pythonpath_manip():
+    """Stop PYTHONPATH contaminating the Cylc Environment
+
+    * Remove PYTHONPATH items from sys.path to prevent PYTHONPATH
+      contaminating the Cylc Environment.
+    * Re-add items from CYLC_PYTHONPATH to sys.path.
+
+    See Also:
+        https://github.com/cylc/cylc-flow/issues/5124
+    """
+    if 'CYLC_PYTHONPATH' in os.environ:
+        for item in os.environ['CYLC_PYTHONPATH'].split(os.pathsep):
+            abspath = os.path.abspath(item)
+            sys.path.insert(0, abspath)
+    elif 'PYTHONPATH' in os.environ:
+        for item in os.environ['PYTHONPATH'].split(os.pathsep):
+            abspath = os.path.abspath(item)
+            if abspath in sys.path:
+                sys.path.remove(abspath)
+
+
+pythonpath_manip()
+
+import argparse
+from contextlib import contextmanager
 from typing import Iterator, NoReturn, Optional, Tuple
 
 from ansimarkup import parse as cparse
diff --git a/tests/unit/scripts/test_cylc.py b/tests/unit/scripts/test_cylc.py
index f1483e9ee4a..4656aeacb52 100644
--- a/tests/unit/scripts/test_cylc.py
+++ b/tests/unit/scripts/test_cylc.py
@@ -15,14 +15,16 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
 import pkg_resources
+import sys
 from types import SimpleNamespace
 from typing import Callable
 from unittest.mock import Mock
 
 import pytest
 
-from cylc.flow.scripts.cylc import iter_commands
+from cylc.flow.scripts.cylc import iter_commands, pythonpath_manip
 
 from ..conftest import MonkeyMock
 
@@ -136,3 +138,27 @@ def test_execute_cmd(
     # the "bad" entry point should raise an exception
     with pytest.raises(ModuleNotFoundError):
         execute_cmd('bad')
+
+
+def test_pythonpath_manip(monkeypatch):
+    """pythonpath_manip removes items in PYTHONPATH from sys.path
+
+    and adds items from CYLC_PYTHONPATH
+    """
+    # If PYTHONPATH is set...
+    monkeypatch.setenv('PYTHONPATH', '/remove-from-sys.path')
+    monkeypatch.setattr('sys.path', ['/leave-alone', '/remove-from-sys.path'])
+    pythonpath_manip()
+    # ... we don't change PYTHONPATH
+    assert os.environ['PYTHONPATH'] == '/remove-from-sys.path'
+    # ... but we do remove PYTHONPATH items from sys.path, and don't remove
+    # items there not in PYTHONPATH
+    assert sys.path == ['/leave-alone']
+    # ... and store items from PYTHONPATH in CYLC_PYTHONPATH
+    assert os.environ['CYLC_PYTHONPATH'] == '/remove-from-sys.path'
+
+    # If CYLC_PYTHONPATH is set we retrieve its contents and
+    # add them to the sys.path:
+    monkeypatch.setenv('CYLC_PYTHONPATH', '/add-to-sys.path')
+    pythonpath_manip()
+    assert sys.path == ['/add-to-sys.path', '/leave-alone']