[CI] Use python 3.9+ everywhere (#40139)

- Upgrade the base image of https://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel to newer https://gcr.io/oss-fuzz-base/base-builder@sha256:4f3ca10accd14292556601d70e457fa85ad57180c913484427869d3379a07684, which comes with python3.11 Before: ``` Python 3.8.3 (default, Mar 12 2024, 03:21:22) [Clang 15.0.0 (https://github.com/llvm/llvm-project.git bf7f8d6fa6f460bf0a16ffe ``` After: ``` Python 3.11.13 (main, Jul 16 2025, 03:42:11) [Clang 18.1.8 (https://github.com/llvm/llvm-project.git 3b5b5c1ec4a3095ab096dd78 ``` - Refactor and improve https://us-docker.pkg.dev/grpc-testing/testing-images-public/sanity. Removed building python 3.7 from source. - Switch Windows builds to use `3.9.13` (previously `3.8.10`) - Changed minimum python version in bazel builds from 3.8 to 3.9. - Updated pylint to use python3.11. - Various minor compatibility fixes. There's a few other dependencies/settings that can be upgraded now that we're using 3.9+. I'll make them in a follow-up PR. Closes #40139 PiperOrigin-RevId: 786775922
2026-01-24 10:53:30 +08:00 · 2025-07-24 11:27:59 -07:00
parent f108df8850
commit 0497da859e
19 changed files with 70 additions and 104 deletions
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -63,7 +63,6 @@ bazel_dep(name = "rules_shell", version = "0.4.0", dev_dependency = True)
 bazel_dep(name = "rules_python", version = "0.37.1")

 PYTHON_VERSIONS = [
-    "3.8",
    "3.9",
    "3.10",
    "3.11",
--- a/2
+++ b/2
@@ -57,7 +57,7 @@ load("@com_google_protobuf//python/dist:system_python.bzl", "system_python")

 system_python(
    name = "system_python",
-    minimum_python_version = "3.8",
+    minimum_python_version = "3.9",
 )

 load("@system_python//:pip.bzl", system_pip_parse = "pip_parse")
--- a/examples/python/compression/BUILD.bazel
+++ b/examples/python/compression/BUILD.bazel
@@ -30,7 +30,7 @@ py_binary(
 py_binary(
    name = "client",
    srcs = ["client.py"],
-    imports = ["../../examples/protos"],
+    imports = ["../../protos"],
    python_version = "PY3",
    srcs_version = "PY2AND3",
    deps = [
--- a/src/python/grpcio_tests/tests/unit/_api_test.py
+++ b/src/python/grpcio_tests/tests/unit/_api_test.py
@@ -150,7 +150,7 @@ class ChannelTest(unittest.TestCase):
        threads = []
        for _ in range(_THREAD_COUNT):
            thread = threading.Thread(target=create_secure_channel)
-            thread.setDaemon(True)
+            thread.daemon = True
            thread.start()
            threads.append(thread)

@@ -172,7 +172,7 @@ class ChannelTest(unittest.TestCase):
        threads = []
        for _ in range(_THREAD_COUNT):
            thread = threading.Thread(target=create_secure_channel)
-            thread.setDaemon(True)
+            thread.daemon = True
            thread.start()
            threads.append(thread)

--- a/src/python/grpcio_tests/tests/unit/_contextvars_propagation_test.py
+++ b/src/python/grpcio_tests/tests/unit/_contextvars_propagation_test.py
@@ -173,7 +173,7 @@ class ContextVarsPropagationTest(unittest.TestCase):
            for _ in range(_THREAD_COUNT):
                q = queue.Queue()
                thread = threading.Thread(target=_run_on_thread, args=(q,))
-                thread.setDaemon(True)
+                thread.daemon = True
                thread.start()
                threads.append((thread, q))

--- a/templates/MODULE.bazel.inja
+++ b/templates/MODULE.bazel.inja
@@ -63,7 +63,6 @@ bazel_dep(name = "rules_shell", version = "0.4.0", dev_dependency = True)
 bazel_dep(name = "rules_python", version = "0.37.1")

 PYTHON_VERSIONS = [
-    "3.8",
    "3.9",
    "3.10",
    "3.11",
--- a/templates/tools/dockerfile/oss_fuzz_base.include
+++ b/templates/tools/dockerfile/oss_fuzz_base.include
@@ -1,8 +1,8 @@
 # Pinned version of the base image is used to avoid regressions caused
 # by rebuilding of this docker image. To see available versions, you can run
 # "gcloud container images list-tags gcr.io/oss-fuzz-base/base-builder"
-# This base image is built on Mar 12, 2024
-FROM gcr.io/oss-fuzz-base/base-builder@sha256:c3581153788bc49f3634fec3cd36a5d6dfd26632c4afc157fb6faf8ce3af732e
+# This base image is built on Jul 16, 2025.
+FROM gcr.io/oss-fuzz-base/base-builder@sha256:4f3ca10accd14292556601d70e457fa85ad57180c913484427869d3379a07684

 # -------------------------- WARNING --------------------------------------
 # If you are making changes to this file, consider changing
--- a/templates/tools/dockerfile/test/sanity/Dockerfile.inja
+++ b/templates/tools/dockerfile/test/sanity/Dockerfile.inja
@@ -16,7 +16,7 @@ FROM silkeh/clang:19-bookworm

 {% include "../../apt_get_basic.include" %}

-RUN apt-get update && apt-get install -y python3 python3-pip && apt-get clean
+RUN apt update && apt install -y python3 python3-pip && apt clean

 RUN python3 -m pip install --break-system-packages --upgrade google-auth==1.23.0 google-api-python-client==1.12.8 oauth2client==4.1.0

@@ -25,36 +25,27 @@ RUN python3 -m pip install --break-system-packages --upgrade google-auth==1.23.0
 #========================
 # Sanity test dependencies

-RUN apt-get update && apt-get install -y \
-      autoconf \
-      automake \
-      libtool \
-      curl \
-      shellcheck
-
-# otherwise clang-tidy will report missing <gtest/gtest.h> header
-RUN apt-get update && apt-get install -y libgtest-dev && apt-get clean
-
-RUN python3 -m pip install --break-system-packages simplejson mako virtualenv lxml six
-
-# Install prerequisites for the clang-tidy script
-RUN apt-get update && apt-get install -y jq git && apt-get clean
+# autoconf automake libtool curl shellcheck: main dependencies
+# jq git: prerequisites for the clang-tidy script
+# libgtest-dev: otherwise clang-tidy will report missing <gtest/gtest.h> header
+RUN apt install -y \
+    autoconf \
+    automake \
+    curl \
+    git\
+    jq \
+    libgtest-dev \
+    libtool \
+    python3-lxml \
+    python3-mako \
+    python3-simplejson \
+    python3-venv \
+    python3-virtualenv \
+    shellcheck \
+    && apt clean

 {% include "../../bazel.include" %}
 {% include "../../buildifier.include" %}

-# Install Python 3.7 from source for old python tests such as pytype
-# TODO(xuan): Remote this once those tests are updated to use Python 3.11
-RUN apt update && apt install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev \
-                            libnss3-dev libssl-dev libreadline-dev libffi-dev libbz2-dev
-RUN curl -O https://www.python.org/ftp/python/3.7.17/Python-3.7.17.tar.xz && \
-    tar -xf Python-3.7.17.tar.xz && \
-    cd Python-3.7.17 && \
-    ./configure && \
-    make -j 16 && \
-    make altinstall
-RUN curl https://bootstrap.pypa.io/pip/3.7/get-pip.py | python3.7
-RUN python3.7 -m pip install virtualenv
-
 # Define the default command.
 CMD ["bash"]
--- a/test/distrib/bazel/python/WORKSPACE
+++ b/test/distrib/bazel/python/WORKSPACE
@@ -29,7 +29,7 @@ load("@com_google_protobuf//python/dist:system_python.bzl", "system_python")

 system_python(
    name = "system_python",
-    minimum_python_version = "3.8",
+    minimum_python_version = "3.9",
 )

 load("@system_python//:pip.bzl", "pip_parse")
--- a/tools/bazelify_tests/dockerimage_current_versions.bzl
+++ b/tools/bazelify_tests/dockerimage_current_versions.bzl
@@ -93,7 +93,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
    "tools/dockerfile/interoptest/grpc_interop_ruby.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby@sha256:1b4160c0bf66ba79bc423e4bb75e2e8945162287ed73895b51d5cd0c0c522616",
    "tools/dockerfile/interoptest/lb_interop_fake_servers.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/lb_interop_fake_servers@sha256:ba4c19a2b8e4af7a2d112ef65e6475fb44394cadb2fe17d6f0d434c5b1849bed",
    "tools/dockerfile/test/android_ndk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/android_ndk@sha256:2f0b941ddf7c0eb913a9c81df27e28c3c227cdde00654b0076b49aff51cc0c02",
-    "tools/dockerfile/test/bazel.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel@sha256:be0289f78127b2fce83b7d6734ac186e559af4621f562b26f1c47d182ad1a156",
+    "tools/dockerfile/test/bazel.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel@sha256:814f79b30656de4627c88e53bbf1c457037748971055702a74745a5828de12fa",
    "tools/dockerfile/test/bazel_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel_arm64@sha256:a5b84a1c6845fb6a9f2816a36d305b56dba46a6db56cb1d537aadd989e71b773",
    "tools/dockerfile/test/csharp_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_debian11_arm64@sha256:53ce3ccdc74b8c264c7706f4e98830cdc1cf79c4460ed98db6122da6fc26f119",
    "tools/dockerfile/test/csharp_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_debian11_x64@sha256:021b6a99ea13597fbe09cfe129774f1e2b9d7d3ab21eabd5ff6ce0aec1f6f88b",
@@ -118,5 +118,5 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
    "tools/dockerfile/test/rbe_ubuntu2004.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004@sha256:bce8453713efcff3f4effe0d7bb0ef743ec4e772c63bac637c38d0af0e1a9ad9",
    "tools/dockerfile/test/ruby_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64@sha256:8b1c4a9d7802f843356104b7bda4809403462b12174fe87844549a9b23cc16fc",
    "tools/dockerfile/test/ruby_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64@sha256:ef468a9965ea594f7146b308e931ca96fd723a8ca7d4d80687b1761805d5e5fe",
-    "tools/dockerfile/test/sanity.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/sanity@sha256:da094c7a2020e82a42bbd88becc55a664da9ecf29a63b63ec9ec28e59e49e7d2",
+    "tools/dockerfile/test/sanity.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/sanity@sha256:afea9ca2129607dfaf03b6f2b58e5998a1bc18e094c8a4816cba462ddb9c2295",
 }
--- a/tools/distrib/docgen/_generate_python_doc.sh
+++ b/tools/distrib/docgen/_generate_python_doc.sh
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# This script is meant to be ran in Docker instance of python:3.8.
+# This script is meant to be ran in Docker instance of python:3.9.

 set -ex

--- a/tools/distrib/pylint_code.sh
+++ b/tools/distrib/pylint_code.sh
@@ -41,7 +41,7 @@ VIRTUALENV=".venv-pylint"

 # When running locally, allow to reuse existing venv.
 if [[ ! -d "${VIRTUALENV}" ]]; then
-  python3.7 -m virtualenv "${VIRTUALENV}"
+  python3.11 -m virtualenv "${VIRTUALENV}"
 fi

 source "${VIRTUALENV}/bin/activate"
--- a/tools/dockerfile/test/bazel.current_version
+++ b/tools/dockerfile/test/bazel.current_version
@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/bazel:7e7df73e8973de9ef2c75d2c633e7c43b581c1cd@sha256:be0289f78127b2fce83b7d6734ac186e559af4621f562b26f1c47d182ad1a156
+us-docker.pkg.dev/grpc-testing/testing-images-public/bazel:17f63c84b86ac41f70f86cac931fffc785a474e0@sha256:814f79b30656de4627c88e53bbf1c457037748971055702a74745a5828de12fa
--- a/tools/dockerfile/test/bazel/Dockerfile
+++ b/tools/dockerfile/test/bazel/Dockerfile
@@ -15,8 +15,8 @@
 # Pinned version of the base image is used to avoid regressions caused
 # by rebuilding of this docker image. To see available versions, you can run
 # "gcloud container images list-tags gcr.io/oss-fuzz-base/base-builder"
-# This base image is built on Mar 12, 2024
-FROM gcr.io/oss-fuzz-base/base-builder@sha256:c3581153788bc49f3634fec3cd36a5d6dfd26632c4afc157fb6faf8ce3af732e
+# This base image is built on Jul 16, 2025.
+FROM gcr.io/oss-fuzz-base/base-builder@sha256:4f3ca10accd14292556601d70e457fa85ad57180c913484427869d3379a07684

 # -------------------------- WARNING --------------------------------------
 # If you are making changes to this file, consider changing
--- a/tools/dockerfile/test/sanity.current_version
+++ b/tools/dockerfile/test/sanity.current_version
@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/sanity:207460d79c146b51349d39b171e35826526f4b0b@sha256:da094c7a2020e82a42bbd88becc55a664da9ecf29a63b63ec9ec28e59e49e7d2
+us-docker.pkg.dev/grpc-testing/testing-images-public/sanity:a68b68edde3974956ed666759043b91da10ae870@sha256:afea9ca2129607dfaf03b6f2b58e5998a1bc18e094c8a4816cba462ddb9c2295
--- a/tools/dockerfile/test/sanity/Dockerfile
+++ b/tools/dockerfile/test/sanity/Dockerfile
@@ -63,7 +63,7 @@ RUN git config --global protocol.file.allow always



-RUN apt-get update && apt-get install -y python3 python3-pip && apt-get clean
+RUN apt update && apt install -y python3 python3-pip && apt clean

 RUN python3 -m pip install --break-system-packages --upgrade google-auth==1.23.0 google-api-python-client==1.12.8 oauth2client==4.1.0

@@ -74,20 +74,24 @@ RUN mkdir /var/local/jenkins
 #========================
 # Sanity test dependencies

-RUN apt-get update && apt-get install -y \
-      autoconf \
-      automake \
-      libtool \
-      curl \
-      shellcheck
-
-# otherwise clang-tidy will report missing <gtest/gtest.h> header
-RUN apt-get update && apt-get install -y libgtest-dev && apt-get clean
-
-RUN python3 -m pip install --break-system-packages simplejson mako virtualenv lxml six
-
-# Install prerequisites for the clang-tidy script
-RUN apt-get update && apt-get install -y jq git && apt-get clean
+# autoconf automake libtool curl shellcheck: main dependencies
+# jq git: prerequisites for the clang-tidy script
+# libgtest-dev: otherwise clang-tidy will report missing <gtest/gtest.h> header
+RUN apt install -y \
+    autoconf \
+    automake \
+    curl \
+    git\
+    jq \
+    libgtest-dev \
+    libtool \
+    python3-lxml \
+    python3-mako \
+    python3-simplejson \
+    python3-venv \
+    python3-virtualenv \
+    shellcheck \
+    && apt clean

 #========================
 # Bazel installation
@@ -118,18 +122,5 @@ RUN chmod +x buildifier
 RUN mv buildifier /usr/local/bin


-# Install Python 3.7 from source for old python tests such as pytype
-# TODO(xuan): Remote this once those tests are updated to use Python 3.11
-RUN apt update && apt install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev \
-                            libnss3-dev libssl-dev libreadline-dev libffi-dev libbz2-dev
-RUN curl -O https://www.python.org/ftp/python/3.7.17/Python-3.7.17.tar.xz && \
-    tar -xf Python-3.7.17.tar.xz && \
-    cd Python-3.7.17 && \
-    ./configure && \
-    make -j 16 && \
-    make altinstall
-RUN curl https://bootstrap.pypa.io/pip/3.7/get-pip.py | python3.7
-RUN python3.7 -m pip install virtualenv
-
 # Define the default command.
 CMD ["bash"]
--- a/tools/internal_ci/helper_scripts/install_python_interpreters.ps1
+++ b/tools/internal_ci/helper_scripts/install_python_interpreters.ps1
@@ -1,5 +1,5 @@
 #!/usr/bin/env powershell
-# Install Python 3.8 for x64 and x86 in order to build wheels on Windows.
+# Install Python 3.9 for x64 and x86 in order to build wheels on Windows.

 Set-StrictMode -Version 2
 $ErrorActionPreference = 'Stop'
@@ -65,37 +65,21 @@ function Install-Python {
    Write-Host "Python $PythonVersion installed by $PythonInstaller at $PythonInstallPath."
 }

-# Python 3.8
-$Python38x86Config = @{
-    PythonVersion = "3.8.10"
-    PythonInstaller = "python-3.8.10"
-    PythonInstallPath = "C:\Python38_32bit"
-    PythonInstallerHash = "b355cfc84b681ace8908ae50908e8761"
-}
-Install-Python @Python38x86Config
-
-$Python38x64Config = @{
-    PythonVersion = "3.8.10"
-    PythonInstaller = "python-3.8.10-amd64"
-    PythonInstallPath = "C:\Python38"
-    PythonInstallerHash = "62cf1a12a5276b0259e8761d4cf4fe42"
-}
-Install-Python @Python38x64Config
-
 # Python 3.9
 $Python39x86Config = @{
-    PythonVersion = "3.9.11"
-    PythonInstaller = "python-3.9.11"
+    PythonVersion = "3.9.13"
+    PythonInstaller = "python-3.9.13"
    PythonInstallPath = "C:\Python39_32bit"
-    PythonInstallerHash = "4210652b14a030517046cdf111c09c1e"
+    PythonInstallerHash = "46c35b0a2a4325c275b2ed3187b08ac4"
 }
 Install-Python @Python39x86Config

 $Python39x64Config = @{
-    PythonVersion = "3.9.11"
-    PythonInstaller = "python-3.9.11-amd64"
+    # Keep the version in sync with the one installed by choco in prepare_build_windows.bat.
+    PythonVersion = "3.9.13"
+    PythonInstaller = "python-3.9.13-amd64"
    PythonInstallPath = "C:\Python39"
-    PythonInstallerHash = "fef52176a572efd48b7148f006b25801"
+    PythonInstallerHash = "e7062b85c3624af82079794729618eca"
 }
 Install-Python @Python39x64Config

@@ -165,4 +149,4 @@ $Python313x64Config = @{
    PythonInstallPath = "C:\Python313"
    PythonInstallerHash = "0af5ad8734962267323ef45f384de3aa"
 }
-Install-Python @Python313x64Config
+Install-Python @Python313x64Config
--- a/tools/internal_ci/helper_scripts/prepare_build_windows.bat
+++ b/tools/internal_ci/helper_scripts/prepare_build_windows.bat
@@ -16,16 +16,18 @@
 echo "!TIME!: prepare_build_windows.bat started"

@rem make sure msys binaries are preferred over cygwin binaries
-@rem set path to python3.8
+@rem set path to python3.9
@rem set path to CMake
-set PATH=C:\tools\msys64\usr\bin;C:\Python38;C:\Program Files\CMake\bin;%PATH%
+set PATH=C:\tools\msys64\usr\bin;C:\Python39;C:\Program Files\CMake\bin;%PATH%

@rem Print image ID of the windows kokoro image being used.
 cat C:\image_id.txt

+@rem install python 3.9
+choco install -y --no-progress python --version=3.9.13
+
@rem create "python3" link that normally doesn't exist
-dir C:\Python38\
-mklink C:\Python38\python3.exe C:\Python38\python.exe
+mklink C:\Python39\python3.exe C:\Python39\python.exe

 python --version
 python3 --version
--- a/tools/run_tests/helper_scripts/run_python.sh
+++ b/tools/run_tests/helper_scripts/run_python.sh
@@ -19,7 +19,7 @@ set -ex
 cd "$(dirname "$0")/../../.."

 # TODO(jtattermusch): is the $(pwd) prefix actually useful?
-PYTHON="$(pwd)/${1:-py38/bin/python}"
+PYTHON="$(pwd)/${1:-py39/bin/python}"

 ROOT=$(pwd)