[query] Python 3.8, Numpy 1.24.2, Bokeh 3.x.x, fixes to plots.py, fixes to req generation

Makefile

@@ -75,13 +75,13 @@ install-dev-requirements:
 hail/python/hailtop/pinned-requirements.txt: hail/python/hailtop/requirements.txt
 	./generate-linux-pip-lockfile.sh hail/python/hailtop
 
-hail/python/pinned-requirements.txt: hail/python/requirements.txt hail/python/hailtop/pinned-requirements.txt
+hail/python/pinned-requirements.txt: hail/python/hailtop/pinned-requirements.txt hail/python/requirements.txt
 	./generate-linux-pip-lockfile.sh hail/python
 
-hail/python/dev/pinned-requirements.txt: hail/python/dev/requirements.txt hail/python/pinned-requirements.txt
+hail/python/dev/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/requirements.txt
 	./generate-linux-pip-lockfile.sh hail/python/dev
 
-gear/pinned-requirements.txt: hail/python/hailtop/pinned-requirements.txt gear/requirements.txt
+gear/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/pinned-requirements.txt hail/python/hailtop/pinned-requirements.txt gear/requirements.txt
 	./generate-linux-pip-lockfile.sh gear
 
 web_common/pinned-requirements.txt: gear/pinned-requirements.txt web_common/requirements.txt

auth/pinned-requirements.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.7
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    pip-compile --output-file=hail/auth/pinned-requirements.txt hail/auth/requirements.txt
 #
@@ -22,7 +22,7 @@ charset-normalizer==3.1.0
     #   -c hail/auth/../hail/python/pinned-requirements.txt
     #   -c hail/auth/../web_common/pinned-requirements.txt
     #   requests
-google-auth==2.17.2
+google-auth==2.17.3
     # via
     #   -c hail/auth/../gear/pinned-requirements.txt
     #   -c hail/auth/../hail/python/pinned-requirements.txt
@@ -38,13 +38,13 @@ idna==3.4
     #   requests
 oauthlib==3.2.2
     # via requests-oauthlib
-pyasn1==0.4.8
+pyasn1==0.5.0
     # via
     #   -c hail/auth/../gear/pinned-requirements.txt
     #   -c hail/auth/../hail/python/pinned-requirements.txt
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.2.8
+pyasn1-modules==0.3.0
     # via
     #   -c hail/auth/../gear/pinned-requirements.txt
     #   -c hail/auth/../hail/python/pinned-requirements.txt

batch/Dockerfile.worker

@@ -8,7 +8,7 @@ RUN hail-apt-get-install \
     xfsprogs \
     libyajl-dev # crun runtime dependency
 
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
 
 {% if global.cloud == "gcp" %}
 RUN echo "APT::Acquire::Retries \"5\";" > /etc/apt/apt.conf.d/80-retries && \
@@ -39,7 +39,7 @@ RUN hail-pip-install \
     -r batch-requirements.txt \
     pyspark==3.3.0
 
-ENV SPARK_HOME /usr/local/lib/python3.7/dist-packages/pyspark
+ENV SPARK_HOME /usr/local/lib/python3.8/dist-packages/pyspark
 ENV PATH "$PATH:$SPARK_HOME/sbin:$SPARK_HOME/bin"
 ENV PYSPARK_PYTHON python3
 

batch/pinned-requirements.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.7
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    pip-compile --output-file=hail/batch/pinned-requirements.txt hail/batch/requirements.txt
 #
@@ -27,13 +27,7 @@ async-timeout==4.0.2
     #   -c hail/batch/../web_common/pinned-requirements.txt
     #   -r hail/batch/requirements.txt
     #   aiohttp
-asynctest==0.13.0
-    # via
-    #   -c hail/batch/../gear/pinned-requirements.txt
-    #   -c hail/batch/../hail/python/pinned-requirements.txt
-    #   -c hail/batch/../web_common/pinned-requirements.txt
-    #   aiohttp
-attrs==22.2.0
+attrs==23.1.0
     # via
     #   -c hail/batch/../gear/pinned-requirements.txt
     #   -c hail/batch/../hail/python/dev/pinned-requirements.txt
@@ -70,17 +64,17 @@ multidict==6.0.4
     #   -c hail/batch/../web_common/pinned-requirements.txt
     #   aiohttp
     #   yarl
-numpy==1.21.6
+numpy==1.24.3
     # via
     #   -c hail/batch/../hail/python/dev/pinned-requirements.txt
     #   -c hail/batch/../hail/python/pinned-requirements.txt
     #   pandas
-packaging==23.0
+packaging==23.1
     # via
     #   -c hail/batch/../hail/python/dev/pinned-requirements.txt
     #   -c hail/batch/../hail/python/pinned-requirements.txt
     #   plotly
-pandas==1.3.5
+pandas==2.0.1
     # via
     #   -c hail/batch/../hail/python/pinned-requirements.txt
     #   -r hail/batch/requirements.txt
@@ -111,15 +105,14 @@ tenacity==8.2.2
     #   plotly
 typing-extensions==4.5.0
     # via
-    #   -c hail/batch/../gear/pinned-requirements.txt
     #   -c hail/batch/../hail/python/dev/pinned-requirements.txt
     #   -c hail/batch/../hail/python/pinned-requirements.txt
-    #   -c hail/batch/../web_common/pinned-requirements.txt
     #   aiodocker
-    #   aiohttp
-    #   async-timeout
-    #   yarl
-yarl==1.8.2
+tzdata==2023.3
+    # via
+    #   -c hail/batch/../hail/python/pinned-requirements.txt
+    #   pandas
+yarl==1.9.1
     # via
     #   -c hail/batch/../gear/pinned-requirements.txt
     #   -c hail/batch/../hail/python/pinned-requirements.txt

batch/requirements.txt

@@ -3,7 +3,7 @@
 -c ../gear/pinned-requirements.txt
 -c ../web_common/pinned-requirements.txt
 dictdiffer>=0.8.1,<1
-pandas>=1.3.0,<1.5.0
+pandas>=2,<3
 plotly>=5.5.0,<6
 # Worker requirements
 aiodocker>=0.17.0,<1

benchmark/python/setup.py

@@ -16,7 +16,7 @@
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: MIT License",
     ],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     install_requires=[
         'hail>=0.2',
     ],

build.yaml

@@ -154,7 +154,7 @@ steps:
       valueFrom: hail_ubuntu_image.image
     script: |
       set -ex
-      pip install pip-tools==6.9.0
+      pip install pip-tools==6.13.0
       cd /io/repo
       chmod 755 ./check_pip_requirements.sh
       ./check_pip_requirements.sh \

ci/Dockerfile.ci-utils

@@ -20,7 +20,7 @@ RUN hail-pip-install \
       -r hailtop-requirements.txt \
       -r gear-requirements.txt \
       twine==1.11.0 \
-      Jinja2==3.0.3
+      'Jinja2>3,<4'
 
 FROM golang:1.18 AS skopeo-build
 

ci/pinned-requirements.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.7
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    pip-compile --output-file=hail/ci/pinned-requirements.txt hail/ci/requirements.txt
 #
@@ -26,7 +26,7 @@ click==8.1.3
     # via
     #   -c hail/ci/../hail/python/dev/pinned-requirements.txt
     #   zulip
-cryptography==40.0.1
+cryptography==40.0.2
     # via
     #   -c hail/ci/../hail/python/pinned-requirements.txt
     #   pyjwt
@@ -41,10 +41,6 @@ idna==3.4
     #   -c hail/ci/../hail/python/pinned-requirements.txt
     #   -c hail/ci/../web_common/pinned-requirements.txt
     #   requests
-importlib-metadata==3.10.1
-    # via
-    #   -c hail/ci/../hail/python/dev/pinned-requirements.txt
-    #   click
 matrix-client==0.4.0
     # via zulip
 pycparser==2.21
@@ -65,11 +61,8 @@ requests[security]==2.28.2
     #   zulip
 typing-extensions==4.5.0
     # via
-    #   -c hail/ci/../gear/pinned-requirements.txt
     #   -c hail/ci/../hail/python/dev/pinned-requirements.txt
     #   -c hail/ci/../hail/python/pinned-requirements.txt
-    #   -c hail/ci/../web_common/pinned-requirements.txt
-    #   importlib-metadata
     #   zulip
 uritemplate==4.1.1
     # via
@@ -82,9 +75,5 @@ urllib3==1.26.15
     #   -c hail/ci/../hail/python/pinned-requirements.txt
     #   matrix-client
     #   requests
-zipp==3.15.0
-    # via
-    #   -c hail/ci/../hail/python/dev/pinned-requirements.txt
-    #   importlib-metadata
 zulip==0.8.2
     # via -r hail/ci/requirements.txt

dev-docs/compiler-team/development_tools.md

@@ -51,7 +51,7 @@ https://www.anaconda.com/download/#macos
 After installing Anaconda, you should create a new dev environment
 for Hail with:
 
-    conda create --name hail python=3.7
+    conda create --name hail python=3.8
 
 and
 

dev-docs/development_process.md

@@ -27,7 +27,7 @@ feature into smaller components.
 Before you can write code, there are some setup steps that will allow you to
 develop effectively.
 
-Hail currently supports Python version 3.7 or greater.
+Hail currently supports Python version 3.8 or greater.
 
 ```
 make install-dev-requirements

dev-docs/hail-for-new-engineers.md

@@ -154,8 +154,8 @@ We use a number of technologies:
 
 ### Services Technology
 
-We almost exclusively write services in Python 3.7. We use a number of Python packages:
-- [`asyncio`](https://docs.python.org/3.7/library/asyncio.html) for concurrency which is built on
+We almost exclusively write services in Python 3.8. We use a number of Python packages:
+- [`asyncio`](https://docs.python.org/3.8/library/asyncio.html) for concurrency which is built on
   [coroutines](https://en.wikipedia.org/wiki/Coroutine) not threads
 - [`aiohttp`](https://docs.aiohttp.org/en/stable/) for serving HTTPS requests (most services speak
   HTTPS)

dev-docs/ksync.md

@@ -79,7 +79,7 @@ ksync watch
 3. Create a spec in ~/.ksync/ksync.yaml using the create operation
 
 ```
-ksync create --local-read-only -l app=auth --name <NAMESPACE>-<APP> -n jigold $(pwd)/<APP>/ /usr/local/lib/python3.7/dist-packages/<APP>/
+ksync create --local-read-only -l app=auth --name <NAMESPACE>-<APP> -n jigold $(pwd)/<APP>/ /usr/local/lib/python3.8/dist-packages/<APP>/
 ```
 
 4. Use ksync get to make sure the pods are being watched

docker/hail-ubuntu/Dockerfile

@@ -20,8 +20,8 @@ RUN chmod 755 /bin/retry && \
          >> /etc/apt/sources.list && \
     echo 'deb-src [signed-by=/usr/share/keyrings/deadsnakes-ppa-archive-keyring.gpg] http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main' \
          >> /etc/apt/sources.list && \
-    hail-apt-get-install python3.7-minimal python3.7-dev python3.7-distutils gcc g++ && \
-    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 && \
+    hail-apt-get-install python3.8-minimal python3.8-dev python3.8-distutils gcc g++ && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 && \
     curl https://bootstrap.pypa.io/get-pip.py | python3 && \
     python3 -m pip install 'pip>=21<22' && \
     python3 -m pip check && \

docker/hailgenetics/mirror_images.sh

@@ -32,8 +32,6 @@ then
 fi
 
 images=(
-    "python-dill:3.7"
-    "python-dill:3.7-slim"
     "python-dill:3.8"
     "python-dill:3.8-slim"
     "python-dill:3.9"

docker/hailgenetics/python-dill/push.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-for version in 3.7 3.7-slim 3.8 3.8-slim 3.9 3.9-slim 3.10 3.10-slim
+for version in 3.8 3.8-slim 3.9 3.9-slim 3.10 3.10-slim
 do
     public=hailgenetics/python-dill:$version
 

docker/third-party/images.txt

@@ -7,9 +7,6 @@ grafana/grafana:9.1.4
 jupyter/scipy-notebook
 jupyter/scipy-notebook:c094bb7219f9
 moby/buildkit:v0.8.3-rootless
-python:3.7
-python:3.7-slim
-python:3.7-slim-stretch
 python:3.8
 python:3.8-slim
 python:3.9