Add some missing type libraries for mypy (#9657)

* add some missing types

* install pkg-config

* install pkg-config for docker too

* add pkg-config to plugins

* pkg-config when cryptography may need to be built

* deps cleanup

* more comments

* more tweaks

.azure-pipelines/templates/steps/tox-steps.yml

@@ -1,3 +1,5 @@
+# This does not include the dependencies needed to build cryptography. See
+# https://cryptography.io/en/latest/installation/
 steps:
   # We run brew update because we've seen attempts to install an older version
   # of a package fail. See
@@ -12,14 +14,8 @@ steps:
       set -e
       sudo apt-get update
       sudo apt-get install -y --no-install-recommends \
-        python3-dev \
-        gcc \
         libaugeas0 \
-        libssl-dev \
-        libffi-dev \
-        ca-certificates \
-        nginx-light \
-        openssl
+        nginx-light
       sudo systemctl stop nginx
       sudo sysctl net.ipv4.ip_unprivileged_port_start=0
     condition: startswith(variables['IMAGE_NAME'], 'ubuntu')

Dockerfile-dev

@@ -7,10 +7,11 @@ EXPOSE 80 443
 WORKDIR /opt/certbot/src
 
 COPY . .
+# This does not include the dependencies needed to build cryptography. See
+# https://cryptography.io/en/latest/installation/#building-cryptography-on-linux
 RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install apache2 git python3-dev \
-        python3-venv gcc libaugeas0 libssl-dev libffi-dev ca-certificates \
-        openssl nginx-light -y --no-install-recommends && \
+    DEBIAN_FRONTEND=noninteractive apt-get install apache2 git python3-venv \
+        libaugeas0 nginx-light -y --no-install-recommends && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* \
            /tmp/* \

certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py

@@ -37,7 +37,9 @@ def do_POST(self) -> None:
                                verify=False, timeout=10)
         issuer_cert = x509.load_pem_x509_certificate(request.content, default_backend())
 
-        content_len = int(self.headers.get('Content-Length'))
+        raw_content_len = self.headers.get('Content-Length')
+        assert isinstance(raw_content_len, str)
+        content_len = int(raw_content_len)
 
         ocsp_request = ocsp.load_der_ocsp_request(self.rfile.read(content_len))
         response = requests.get('{0}/cert-status-by-serial/{1}'.format(

certbot-compatibility-test/Dockerfile

@@ -1,9 +1,10 @@
 FROM debian:buster
 MAINTAINER Brad Warren <bmw@eff.org>
 
+# This does not include the dependencies needed to build cryptography. See
+# https://cryptography.io/en/latest/installation/#building-cryptography-on-linux
 RUN apt-get update && \
-    apt install python3-dev python3-venv gcc libaugeas0 libssl-dev \
-                 libffi-dev ca-certificates openssl -y
+    apt install python3-venv libaugeas0 -y
 
 WORKDIR /opt/certbot/src
 

certbot/certbot/compat/filesystem.py

@@ -274,9 +274,9 @@ def open(file_path: str, flags: int, mode: int = 0o777) -> int:  # pylint: disab
         return os.open(file_path, flags ^ os.O_CREAT ^ os.O_EXCL)
 
     # Windows: general case, we call os.open, let exceptions be thrown, then chmod if all is fine.
-    handle = os.open(file_path, flags)
+    fd = os.open(file_path, flags)
     chmod(file_path, mode)
-    return handle
+    return fd
 
 
 def makedirs(file_path: str, mode: int = 0o777) -> None:

certbot/docs/contributing.rst

@@ -41,13 +41,11 @@ Install and configure the OS system dependencies required to run Certbot.
 
    # For APT-based distributions (e.g. Debian, Ubuntu ...)
    sudo apt update
-   sudo apt install python3-dev python3-venv gcc libaugeas0 libssl-dev \
-                    libffi-dev ca-certificates openssl
+   sudo apt install python3-venv libaugeas0
    # For RPM-based distributions (e.g. Fedora, CentOS ...)
    # NB1: old distributions will use yum instead of dnf
-   # NB2: RHEL-based distributions use python3X-devel instead of python3-devel (e.g. python36-devel)
-   sudo dnf install python3-devel gcc augeas-libs openssl-devel libffi-devel \
-                    redhat-rpm-config ca-certificates openssl
+   # NB2: RHEL-based distributions use python3X instead of python3 (e.g. python38)
+   sudo dnf install python3 augeas-libs
    # For macOS installations with Homebrew already installed and configured
    # NB: If you also run `brew install python` you don't need the ~/lib
    #     directory created below, however, Certbot's Apache plugin won't work
@@ -57,6 +55,12 @@ Install and configure the OS system dependencies required to run Certbot.
    mkdir ~/lib
    ln -s $(brew --prefix)/lib/libaugeas* ~/lib
 
+.. note:: If you have trouble creating the virtual environment below, you may
+   need to install additional dependencies. See the `cryptography project's
+   site`_ for more information.
+
+.. _`cryptography project's site`: https://cryptography.io/en/latest/installation.html#building-cryptography-on-linux
+
 Set up the Python virtual environment that will host your Certbot local instance.
 
 .. code-block:: shell

certbot/setup.py

@@ -95,9 +95,11 @@ def read_file(filename, encoding='utf8'):
     'pytest-xdist',
     'setuptools',
     'tox',
+    'types-httplib2',
     'types-pyOpenSSL',
     'types-pyRFC3339',
     'types-pytz',
+    'types-pywin32',
     'types-requests',
     'types-setuptools',
     'types-six',

letstest/scripts/bootstrap_os_packages.sh

@@ -1,6 +1,9 @@
 #!/bin/sh
 #
 # Install OS dependencies for test farm tests.
+#
+# This does not include the dependencies needed to build cryptography. See
+# https://cryptography.io/en/latest/installation/#building-cryptography-on-linux
 
 set -ex  # Work even if somebody does "sh thisscript.sh".
 

snap/snapcraft.yaml

@@ -79,6 +79,7 @@ parts:
       - libffi-dev
       - python3-dev
       - cargo
+      - pkg-config
     build-environment:
       # We set this environment variable while building to try and increase the
       # stability of fetching the rust crates needed to build the cryptography

tools/docker/Dockerfile

@@ -34,6 +34,7 @@ RUN apk add --no-cache --virtual .build-deps \
         python3-dev \
         cargo \
         git \
+        pkgconfig \
     && python tools/pip_install.py --no-cache-dir \
             --editable src/acme \
             --editable src/certbot \

tools/pinning/current/pyproject.toml

@@ -66,6 +66,11 @@ cryptography = "!= 37.0.3"
 # https://github.com/python-poetry/poetry-plugin-export/issues/168 is resolved.
 poetry = "<1.3.0"
 
+# setuptools 67.5.0 deprecated pkg_resources which we still use. Let's pin it
+# back until this is fixed. Doing this work is being tracked by
+# https://github.com/certbot/certbot/issues/9606.
+setuptools = "<67.5.0"
+
 [tool.poetry.dev-dependencies]
 
 [build-system]