feat!: Python 3.11 Upgrade

.github/docker-compose-ci.yml

@@ -3,7 +3,11 @@ version: "2"
 services:
 
   analytics_api:
-    image: edxops/analytics-api-dev:latest
+    # Use this image once the Python version upgrade is complete
+    # image: edxops/analytics-api-dev:latest
+    build:
+      context: ../
+      dockerfile: Dockerfile
     container_name: analytics_api_testing
     volumes:
       - ..:/edx/app/analytics_api/analytics_api

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
         os:
           - ubuntu-20.04
         python-version:
-          - 3.8
+          - 3.11
         targets: [ 'quality','main.test','docs' ]
 
     steps:
@@ -40,7 +40,7 @@ jobs:
         && export TOXENV=django42 && make test.requirements tox.requirements ${{ matrix.targets }}"
 
     - name: Run Coverage
-      if: matrix.python-version == '3.8' && matrix.targets=='main.test'
+      if: matrix.python-version == '3.11' && matrix.targets=='main.test'
       uses: codecov/codecov-action@v1
       with:
         fail_ci_if_error: true

.github/workflows/mysql8-migrations.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-20.04 ]
-        python-version: [ 3.8 ]
+        python-version: [ 3.11 ]
 
     steps:
     - name: Checkout repo

.github/workflows/upgrade-python-requirements.yml

@@ -15,7 +15,7 @@ jobs:
     uses: openedx/.github/.github/workflows/upgrade-python-requirements.yml@master
     with:
       branch: ${{ github.event.inputs.branch || 'master' }}
-      python_version: "3.8"
+      python_version: "3.11"
       # optional parameters below; fill in if you'd like github or email notifications
       # user_reviewers: ""
       # team_reviewers: ""

Dockerfile

@@ -2,22 +2,41 @@ FROM ubuntu:focal as base
 
 # System requirements.
 
+# ENV variables for Python 3.11 support
+ARG PYTHON_VERSION=3.11
+ENV TZ=UTC
+ENV TERM=xterm-256color
+ENV DEBIAN_FRONTEND=noninteractive
+
+# software-properties-common is needed to setup Python 3.11 env
+RUN apt-get update && \
+  apt-get install -y software-properties-common && \
+  apt-add-repository -y ppa:deadsnakes/ppa
+
 # pkg-config; mysqlclient>=2.2.0 requires pkg-config (https://github.com/PyMySQL/mysqlclient/issues/620)
 
-RUN apt update && \
-  DEBIAN_FRONTEND=noninteractive apt-get install -qy \ 
+RUN apt-get update && \
+  apt-get install -qy \
+  build-essential \ 
   curl \
   vim \
   language-pack-en \
   build-essential \
-  python3.8-dev \
-  python3-virtualenv \
-  python3.8-distutils \
+  python${PYTHON_VERSION} \
+  python${PYTHON_VERSION}-dev \
+  python${PYTHON_VERSION}-distutils \
   libmysqlclient-dev \
   pkg-config \
   libssl-dev && \
   rm -rf /var/lib/apt/lists/*
 
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1
+
+# need to use virtualenv pypi package with Python 3.11
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION}
+RUN pip install virtualenv
+
 # Use UTF-8.
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
@@ -39,7 +58,7 @@ ENV ANALYTICS_API_CFG "/edx/etc/${ANALYTICS_API_SERVICE_NAME}.yml"
 # Working directory will be root of repo.
 WORKDIR ${ANALYTICS_API_CODE_DIR}
 
-RUN virtualenv -p python3.8 --always-copy ${ANALYTICS_API_VENV_DIR}
+RUN virtualenv -p python${PYTHON_VERSION} --always-copy ${ANALYTICS_API_VENV_DIR}
 
 # Expose canonical Analytics port
 EXPOSE 19001

analytics_data_api/v0/tests/utils.py

@@ -17,7 +17,7 @@ def flatten(dictionary, parent_key='', sep='.'):
     items = []
     for key, value in dictionary.items():
         new_key = parent_key + sep + key if parent_key else key
-        if isinstance(value, collections.MutableMapping):
+        if isinstance(value, collections.abc.MutableMapping):
             items.extend(list(flatten(value, new_key).items()))
         else:
             items.append((new_key, value))

pylintrc

@@ -30,40 +30,28 @@ load-plugins=
 # multiple time (only on the command line, not in the configuration file where
 # it should appear only once).
 disable=
-# Never going to use these
-# I0011: Locally disabling W0232
-# W0141: Used builtin function 'map'
-# W0142: Used * or ** magic
-# R0921: Abstract class not referenced
-# R0922: Abstract class is only referenced 1 times
-    I0011,W0141,W0142,R0921,R0922,
-
-   duplicate-code,
-# Django makes classes that trigger these
-# W0232: Class has no __init__ method
-    W0232,
-
-# Might use these when the code is in better shape
-# C0302: Too many lines in module
-# R0201: Method could be a function
-# R0901: Too many ancestors
-# R0902: Too many instance attributes
-# R0903: Too few public methods (1/2)
-# R0904: Too many public methods
-# R0911: Too many return statements
-# R0912: Too many branches
-# R0913: Too many arguments
-# R0914: Too many local variables
-    C0302,R0201,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914,
-# W0511: TODOs etc
-    W0511,
-# E1103: maybe no member
-    E1103,
-# C0111: missing docstring (handled by pep257)
-    C0111,
-
-# We can decide if names are invalid on our own
+    no-member,
+    locally-disabled,
+    duplicate-code,
     invalid-name,
+    consider-using-f-string,
+    raise-missing-from,
+    deprecated-decorator,
+    invalid-str-returned,
+    use-yield-from,
+    consider-using-generator,
+    use-dict-literal,
+    not-callable,
+    redundant-u-string-prefix,
+    consider-using-from-import,
+    missing-module-docstring,
+    missing-function-docstring,
+    missing-class-docstring,
+    too-many-arguments,
+    too-many-ancestors,
+    too-many-locals,
+    too-many-lines,
+    too-few-public-methods,
 
 [REPORTS]
 
@@ -74,11 +62,6 @@ output-format=text
 # Include message's id in output
 msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}"
 
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]".
-files-output=no
-
 # Tells whether to display a full report or only the messages
 reports=no
 
@@ -121,9 +104,6 @@ generated-members=
 
 [BASIC]
 
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,filter,apply,input
-
 # Regular expression which should only match correct module names
 module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
 
@@ -166,7 +146,7 @@ no-docstring-rgx=__.*__|test_.*|setUp|tearDown
 [MISCELLANEOUS]
 
 # List of note tags to take in consideration, separated by a comma.
-notes=FIXME,XXX,TODO
+notes=FIXME,XXX
 
 
 [FORMAT]
@@ -241,9 +221,6 @@ max-locals=15
 # Maximum number of return / yield for function / method body
 max-returns=6
 
-# Maximum number of branch for function / method body
-max-branchs=12
-
 # Maximum number of statements in function / method body
 max-statements=50
 

requirements/base.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    make upgrade
@@ -9,21 +9,17 @@ asgiref==3.8.1
     #   django
     #   django-cors-headers
     #   django-countries
-backports-zoneinfo==0.2.1
-    # via
-    #   django
-    #   djangorestframework
 boto==2.49.0
     # via -r requirements/base.in
-boto3==1.35.8
+boto3==1.35.15
     # via -r requirements/base.in
-botocore==1.35.8
+botocore==1.35.15
     # via
     #   boto3
     #   s3transfer
-certifi==2024.7.4
+certifi==2024.8.30
     # via requests
-cffi==1.17.0
+cffi==1.17.1
     # via
     #   cryptography
     #   pynacl
@@ -35,11 +31,11 @@ coreapi==2.3.3
     # via -r requirements/base.in
 coreschema==0.0.4
     # via coreapi
-cryptography==43.0.0
+cryptography==43.0.1
     # via
     #   django-fernet-fields-v2
     #   pyjwt
-django==4.2.15
+django==4.2.16
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
@@ -71,7 +67,7 @@ django-fernet-fields-v2==0.9
     # via edx-enterprise-data
 django-filter==24.3
     # via edx-enterprise-data
-django-model-utils==4.5.1
+django-model-utils==5.0.0
     # via
     #   edx-enterprise-data
     #   edx-rbac
@@ -113,35 +109,33 @@ edx-django-utils==5.15.0
     #   edx-drf-extensions
     #   edx-enterprise-data
     #   edx-rest-api-client
-edx-drf-extensions==10.3.0
+edx-drf-extensions==10.4.0
     # via
     #   -r requirements/base.in
     #   edx-enterprise-data
     #   edx-rbac
-edx-enterprise-data==8.11.0
+edx-enterprise-data==8.11.1
     # via -r requirements/base.in
-edx-opaque-keys==2.10.0
+edx-opaque-keys==2.11.0
     # via
     #   -r requirements/base.in
     #   edx-ccx-keys
     #   edx-drf-extensions
     #   edx-enterprise-data
 edx-rbac==1.10.0
     # via edx-enterprise-data
-edx-rest-api-client==5.7.1
+edx-rest-api-client==6.0.0
     # via
     #   -r requirements/base.in
     #   edx-enterprise-data
 factory-boy==3.3.1
     # via edx-enterprise-data
-faker==28.0.0
+faker==28.4.1
     # via factory-boy
 html5lib==1.1
     # via -r requirements/base.in
 idna==3.8
     # via requests
-importlib-metadata==8.4.0
-    # via markdown
 inflection==0.5.1
     # via drf-yasg
 itypes==1.2.0
@@ -208,8 +202,7 @@ requests==2.32.3
     #   edx-drf-extensions
     #   edx-enterprise-data
     #   edx-rest-api-client
-    #   slumber
-rules==3.4
+rules==3.5
     # via edx-enterprise-data
 s3transfer==0.10.2
     # via boto3
@@ -222,8 +215,6 @@ six==1.16.0
     #   edx-rbac
     #   html5lib
     #   python-dateutil
-slumber==0.7.1
-    # via edx-rest-api-client
 sqlparse==0.5.1
     # via django
 stevedore==5.3.0
@@ -234,7 +225,6 @@ tqdm==4.66.5
     # via -r requirements/base.in
 typing-extensions==4.12.2
     # via
-    #   asgiref
     #   django-countries
     #   edx-opaque-keys
 tzdata==2024.1
@@ -243,16 +233,14 @@ uritemplate==4.1.1
     # via
     #   coreapi
     #   drf-yasg
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
     #   botocore
     #   requests
 webencodings==0.5.1
     # via html5lib
-zipp==3.20.1
-    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements/ci.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    make upgrade

requirements/constraints.txt

@@ -21,9 +21,8 @@ elasticsearch < 7.14.0
 # elasticsearch-dsl depends on elasticsearch >=7.8.0,<8.0.0
 elasticsearch-dsl>=7.2.1,<8.0.0
 
-# Use same version of edx-lint
-pylint==2.4.4
-pylint-django==2.0.11
-
-# botocore 1.34.144 depends on urllib3<1.27 and >=1.25.4; python_version < "3.10"
+# botocore 1.35.14 has requirement urllib3<1.27,>=1.25.4; python_version < "3.10"
 urllib3<2.0.0
+
+# path>16.14.0 has removed the deprecated abspath function, which is breaking the docs build
+path<16.15.0