[CI][Python] Upgrade deps assuming python 3.9+ (#40323)

- Regen requirements.bazel.lock with Python 3.9 - bump isort to 6.0.1 (except in pylint, which needs to be updated separately) - fix python version specifiers for black, isort and pylint, typeguard - fix default ignore patterns for isort and pylint - consistent debug info: python version, pip list - consistent virtualenv naming: `.venv-ci-*` - bazel: bump typeguard to 4.4.2 - bazel: bumped gevent to `25.9.1`, greenlet to `3.2.4` to support Python 3.13, closes #40685 - bazel: bump pyyaml for python 3.14 support - bazel: take care of temporary pins to support 3.8-based CIs Bazel RBE CIs upgraded in the following changelists, and currently run Python 3.10: - cl/845778848 - cl/845816768 Relevant testing was done in #41239. Closes #40323 PiperOrigin-RevId: 846423001
2026-01-24 10:53:30 +08:00 · 2025-12-18 14:48:35 -08:00
parent 5339da7b80
commit 75731fd2a7
12 changed files with 88 additions and 72 deletions
--- a/.pylintrc
+++ b/.pylintrc
@@ -1,4 +1,5 @@
 [MAIN]
+py-version = 3.9
 load-plugins=
 	pylint.extensions.no_self_use,
 	pylint.extensions.comparison_placement,
@@ -10,6 +11,9 @@ ignore=
 	src/python/grpcio/grpc/framework/common,
 	src/python/grpcio/grpc/framework/foundation,
 	src/python/grpcio/grpc/framework/interfaces,
+ignore-patterns=
+	.*pb2\.py,
+	.*pb2_grpc\.py,

 extension-pkg-whitelist=grpc._cython.cygrpc

--- a/.pylintrc-examples
+++ b/.pylintrc-examples
@@ -1,4 +1,5 @@
 [MAIN]
+py-version = 3.9
 load-plugins=
 	pylint.extensions.no_self_use,
 	pylint.extensions.comparison_placement,
@@ -10,6 +11,9 @@ ignore=
 	src/python/grpcio/grpc/framework/common,
 	src/python/grpcio/grpc/framework/foundation,
 	src/python/grpcio/grpc/framework/interfaces,
+ignore-patterns=
+	.*pb2\.py,
+	.*pb2_grpc\.py,

 [VARIABLES]

--- a/.pylintrc-tests
+++ b/.pylintrc-tests
@@ -1,4 +1,5 @@
 [MAIN]
+py-version = 3.9
 load-plugins=
 	pylint.extensions.no_self_use,
 	pylint.extensions.comparison_placement,
@@ -9,6 +10,9 @@ ignore=
 	src/python/grpcio_tests/tests/unit/framework,
 	src/python/grpcio_tests/tests/unit/framework/common,
 	src/python/grpcio_tests/tests/unit/framework/foundation,
+ignore-patterns=
+	.*pb2\.py,
+	.*pb2_grpc\.py,

 [VARIABLES]

--- a/grpc-style-config.toml
+++ b/grpc-style-config.toml
@@ -4,6 +4,8 @@ target-version = [
  "py39",
  "py310",
  "py311",
+  "py312",
+  "py313",
 ]
 extend-exclude = '''
 # A regex preceded with ^/ will apply only to files and directories
@@ -29,6 +31,9 @@ extend-exclude = '''

 [tool.isort]
 profile = "black"
+py_version = 39
+follow_links = false
+
 line_length = 80
 src_paths = [
    "examples/python/data_transmission",
@@ -48,6 +53,7 @@ skip_glob = [
    "*pb2*.pyi",
    "**/site-packages/**/*",
    "tools/distrib/python/xds_protos/*",
+    "**/.venv-*",
 ]
 single_line_exclusions = ["typing"]
 force_single_line = true
--- a/requirements.bazel.lock
+++ b/requirements.bazel.lock
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
 #    pip-compile --allow-unsafe --output-file=requirements.bazel.lock requirements.bazel.txt
@@ -26,13 +26,13 @@ deprecated==1.2.18
    # via
    #   opentelemetry-api
    #   opentelemetry-semantic-conventions
-gevent==24.2.1
+gevent==25.9.1
    # via -r requirements.bazel.txt
 google-auth==2.38.0
    # via -r requirements.bazel.txt
 googleapis-common-protos==1.66.0
    # via -r requirements.bazel.txt
-greenlet==3.1.1
+greenlet==3.2.4
    # via gevent
 hyperlink==21.0.0
    # via twisted
@@ -77,7 +77,7 @@ pyasn1==0.6.1
    #   rsa
 pyasn1-modules==0.4.1
    # via google-auth
-pyyaml==6.0.2
+pyyaml==6.0.3
    # via -r requirements.bazel.txt
 requests==2.32.3
    # via
@@ -89,7 +89,7 @@ tomli==2.2.1
    # via incremental
 twisted==24.11.0
    # via -r requirements.bazel.txt
-typeguard==4.2.1
+typeguard==4.4.2
    # via -r requirements.bazel.txt
 typing-extensions==4.12.2
    # via
@@ -115,8 +115,7 @@ zope-interface==7.2
    #   twisted

 # The following packages are considered to be unsafe in a requirements file:
-setuptools==77.0.1 ; python_version > "3.8" # Manually added for python > 3.8, until Bazel RBE jobs support Python >= 3.9 (Tech debt: b/427881645)
-setuptools==75.3.2 ; python_version <= "3.8"
+setuptools==77.0.1
    # via
    #   -r requirements.bazel.txt
    #   incremental
--- a/requirements.bazel.txt
+++ b/requirements.bazel.txt
@@ -16,7 +16,7 @@
 # Instructions for generation follow:
 # Use the oldest supported version of Python
 #
-#   $ docker run -it --rm -v $(pwd):/grpc python:3.8 /bin/bash
+#   $ docker run -it --rm -v $(pwd):/grpc python:3.9 /bin/bash
 #   # cd grpc
 #   # pip install pip-tools
 #   # pip-compile --allow-unsafe requirements.bazel.txt -o requirements.bazel.lock
@@ -26,17 +26,21 @@
 # When installing a new package, sometimes it's desirable to avoid unnecessary
 # package upgrades. In such case, consider the following flow instead.
 #
-#   $ python3.8 -m venv .venv-bazel
+#   $ python3.9 -m venv .venv-bazel
 #   $ source .venv-bazel/bin/activate
 #   $ pip install pip-tools
-#   $ pip install -r requirements.bazel.lock # make the change to version in requirements.bazel.txt after this
+#   $ pip install -r requirements.bazel.lock
+#
+#   Add the new package to requirements.bazel.txt
 #   $ pip-compile --no-upgrade --allow-unsafe requirements.bazel.txt -o requirements.bazel.lock
-#   $ deactivate
+#
+#   To update a single package, change its version spec requirements.bazel.txt
+#   $ pip-compile --no-upgrade --allow-unsafe requirements.bazel.txt -o requirements.bazel.lock -P package_name

 absl-py
 certifi
 chardet
-gevent
+gevent~=25.8
 google-auth
 googleapis-common-protos
 idna
@@ -47,11 +51,8 @@ opentelemetry-sdk
 protobuf
 pyyaml  # for DNS test
 requests
-setuptools>=77.0.1; python_version > "3.8"
-setuptools>=75.3.2; python_version <= "3.8"
-# Currently our CI uses Python < 3.8, hence <4.4.1 was added
-# TODO(asheshvidyut): remove the <4.4.1, when CI uses python >= 3.9
-typeguard~=4.2.0,<4.4.1
+setuptools>=77.0.1
+typeguard~=4.2
 typing-extensions==4.12.2
 twisted  # for DNS test
 urllib3
--- a/tools/distrib/black_code.sh
+++ b/tools/distrib/black_code.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright 2015 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-set -ex
+set -eux

 ACTION="${1:-}"
 [[ $ACTION == '' ]] || [[ $ACTION == '--diff' ]] || [[ $ACTION == '--check' ]]
@@ -29,9 +29,12 @@ DIRS=(
    'setup.py'
 )

-VIRTUALENV=venv_black_code
-python3 -m virtualenv $VIRTUALENV
-source $VIRTUALENV/bin/activate
+VIRTUALENV=".venv-ci-black"
+python3 -m virtualenv "${VIRTUALENV}"
+source "${VIRTUALENV}/bin/activate"
+python -VV

-python3 -m pip install black==25.1.0
-python3 -m black --config=grpc-style-config.toml $ACTION "${DIRS[@]}"
+pip install black==25.1.0
+pip list
+
+exec black --config=grpc-style-config.toml $ACTION "${DIRS[@]}"
--- a/tools/distrib/check_pytype.sh
+++ b/tools/distrib/check_pytype.sh
@@ -1,4 +1,4 @@
-#! /bin/bash -ex
+#!/usr/bin/env bash
 # Copyright 2025 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,9 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-VIRTUALENV=.venv_check_pytype
-python3.11 -m virtualenv $VIRTUALENV
-source $VIRTUALENV/bin/activate
+set -eux
+
+VIRTUALENV=".venv-ci-pytype"
+python3.11 -m virtualenv "${VIRTUALENV}"
+source "${VIRTUALENV}/bin/activate"
+python -VV

 pip install pytype==2024.10.11
+pip list
+
 pytype --output=~/.cache/pytype --config=grpc-style-config.toml
--- a/tools/distrib/isort_code.sh
+++ b/tools/distrib/isort_code.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright 2021 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-set -ex
+set -eux

 ACTION=${1:---overwrite-in-place}
 [[ $ACTION == '--overwrite-in-place' ]] || [[ $ACTION == '--diff' ]]
@@ -33,9 +33,12 @@ DIRS=(
    'setup.py'
 )

-VIRTUALENV=venv_isort_code
-python3 -m virtualenv $VIRTUALENV
-source $VIRTUALENV/bin/activate
+VIRTUALENV=".venv-ci-isort"
+python3 -m virtualenv "${VIRTUALENV}"
+source "${VIRTUALENV}/bin/activate"
+python -VV

-python3 -m pip install isort==5.9.2
-python3 -m isort $ACTION --settings-path=grpc-style-config.toml --dont-follow-links "${DIRS[@]}"
+pip install isort==6.0.1
+pip list
+
+isort $ACTION --settings-path=grpc-style-config.toml "${DIRS[@]}"
--- a/tools/distrib/pylint_code.sh
+++ b/tools/distrib/pylint_code.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright 2017 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,10 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-set -ex
-
-# NOTE(rbellevi): We ignore generated code.
-IGNORE_PATTERNS=--ignore-patterns='.*pb2\.py,.*pb2_grpc\.py'
+set -eux

 # change to root directory
 cd "$(dirname "$0")/../.."
@@ -37,34 +34,25 @@ TEST_DIRS=(
    'src/python/grpcio_tests/tests_gevent'
 )

-VIRTUALENV=".venv-pylint"
-
-# When running locally, allow to reuse existing venv.
-if [[ ! -d "${VIRTUALENV}" ]]; then
-  python3.11 -m virtualenv "${VIRTUALENV}"
-fi
-
+VIRTUALENV=".venv-ci-pylint"
+python3.11 -m virtualenv "${VIRTUALENV}"
 source "${VIRTUALENV}/bin/activate"
-python3 --version
+python -VV

 # TODO(https://github.com/grpc/grpc/issues/23394): Update Pylint.
-pip install --upgrade astroid==2.15.8 \
-  pylint==2.17.7 \
-  "isort~=5.11"
+pip install --upgrade astroid==2.15.8 pylint==2.17.7 "isort~=5.11"
+pip list

 EXIT=0
 for dir in "${DIRS[@]}"; do
-  python3 -m pylint --rcfile=.pylintrc -rn "$dir" ${IGNORE_PATTERNS}  || EXIT=1
+  pylint --rcfile=.pylintrc -rn "$dir" || EXIT=1
 done

 for dir in "${TEST_DIRS[@]}"; do
-  python3 -m pylint --rcfile=.pylintrc-tests -rn "$dir" ${IGNORE_PATTERNS} || EXIT=1
+  pylint --rcfile=.pylintrc-tests -rn "$dir" || EXIT=1
 done

-find examples/python \
-  -iname "*.py" \
-  -not -name "*_pb2.py" \
-  -not -name "*_pb2_grpc.py" \
-  | xargs python3 -m pylint --rcfile=.pylintrc-examples -rn ${IGNORE_PATTERNS}
+find examples/python -iname "*.py" -not -name "*_pb2*.py" \
+  | xargs pylint --rcfile=.pylintrc-examples -rn

 exit $EXIT
--- a/tools/distrib/ruff_code.sh
+++ b/tools/distrib/ruff_code.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright 2025 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-set -ex
+set -eux
+
+ACTION="${1:-}"
+[[ $ACTION == '' ]] || [[ $ACTION == '--fix' ]]

 # change to root directory
 cd "$(dirname "$0")/../.."
@@ -32,17 +35,12 @@ DIRS=(
   examples/python
 )

-VIRTUALENV=.venv-ruff
-python3 -m virtualenv $VIRTUALENV
-source $VIRTUALENV/bin/activate
-python3 --version
+VIRTUALENV=".venv-ci-ruff"
+python3 -m virtualenv "${VIRTUALENV}"
+source "${VIRTUALENV}/bin/activate"
+python -VV

 pip install ruff==0.12.2
+pip list

-# Check if --fix flag is provided
-RUFF_COMMAND="ruff check"
-if [[ "$1" == "--fix" ]]; then
-    RUFF_COMMAND="ruff check --fix"
-fi
-
-exec $RUFF_COMMAND "${DIRS[@]}"
+exec ruff check $ACTION "${DIRS[@]}"
--- a/tools/internal_ci/helper_scripts/prepare_build_linux_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc
@@ -22,7 +22,8 @@ echo "Kokoro pool: ${KOKORO_JOB_POOL:-undefined}"
 if [[ -f /GRPC_IMAGE_NAME ]]; then
  echo "gRPC custom VM image: $(cat /GRPC_IMAGE_NAME)"
 fi
-echo -e "System: $(uname -a)\n"
+echo -e "System: $(uname -a)"
+echo -e "Python: $(python3 -VV)\n"
 echo -e "-- OS --\n$(lsb_release -a 2>/dev/null)\n"
 echo -e "-- CPU --\n$(lscpu)\n"
 echo -e "-- Memory --\n$(lsmem --summary)\n$(free -h --si)\n"