Merge pull request #9012 from sklam/enh/py311rvsdg_pr

RVSDG-frontend
numba · Aug 10, 2023 · 54e3a6a · 54e3a6a
2 parents 19a0196 + ead8159
commit 54e3a6a
Show file tree

Hide file tree

Showing 18 changed files with 2,857 additions and 140 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -7,119 +7,15 @@ variables:
 
 jobs:
 # Mac and Linux use the same template with different matrixes
-- template: buildscripts/azure/azure-linux-macos.yml
-  parameters:
-    name: macOS
-    vmImage: macos-11
-    matrix:
-      py38_np122:
-        PYTHON: '3.8'
-        NUMPY: '1.22'
-        CONDA_ENV: 'azure_ci'
-        TEST_START_INDEX: 0
-      py311_np125:
-        PYTHON: '3.11'
-        NUMPY: '1.25'
-        CONDA_ENV: 'azure_ci'
-        TEST_THREADING: 'tbb'
-        TEST_START_INDEX: 1
-
 - template: buildscripts/azure/azure-linux-macos.yml
   parameters:
     name: Linux
     vmImage: ubuntu-20.04
     matrix:
-      py38_np122_tbb:
-        PYTHON: '3.8'
-        NUMPY: '1.22'
-        CONDA_ENV: azure_ci
-        TEST_THREADING: 'tbb'
-        TEST_START_INDEX: 2
-      py38_np122_omp:
-        PYTHON: '3.8'
-        NUMPY: '1.22'
-        CONDA_ENV: azure_ci
-        TEST_THREADING: omp
-        TEST_START_INDEX: 3
-      py38_np122_workqueue:
-        PYTHON: '3.8'
-        NUMPY: '1.22'
-        CONDA_ENV: azure_ci
-        TEST_THREADING: workqueue
-        TEST_START_INDEX: 4
-      py38_np123_svml:
-        PYTHON: '3.8'
-        NUMPY: '1.23'
-        CONDA_ENV: azure_ci
-        TEST_SVML: yes
-        TEST_START_INDEX: 5
-      py38_np124:
-        PYTHON: '3.8'
-        NUMPY: '1.24'
-        CONDA_ENV: azure_ci
-        TEST_START_INDEX: 6
-      py39_np122_cov_doc:
-        PYTHON: '3.9'
-        NUMPY: '1.22'
-        CONDA_ENV: azure_ci
-        RUN_COVERAGE: yes
-        RUN_FLAKE8: yes
-        RUN_MYPY: yes
-        BUILD_DOC: yes
-        TEST_START_INDEX: 7
-      py39_np123_typeguard:
-        PYTHON: '3.9'
-        NUMPY: '1.23'
-        CONDA_ENV: azure_ci
-        RUN_TYPEGUARD: yes
-        TEST_START_INDEX: 8
-      py39_np124:
-        PYTHON: '3.9'
-        NUMPY: '1.24'
-        CONDA_ENV: azure_ci
-        TEST_START_INDEX: 9
-      py39_np125:
-        PYTHON: '3.9'
-        NUMPY: '1.25'
-        CONDA_ENV: azure_ci
-        TEST_START_INDEX: 10
-      py310_np122:
-        PYTHON: '3.10'
-        NUMPY: '1.22'
-        CONDA_ENV: azure_ci
-        TEST_START_INDEX: 11
-      py310_np123:
-        PYTHON: '3.10'
-        NUMPY: '1.23'
-        CONDA_ENV: azure_ci
-        TEST_START_INDEX: 12
-      py310_np124:
-        PYTHON: '3.10'
-        NUMPY: '1.24'
-        CONDA_ENV: azure_ci
-        TEST_START_INDEX: 13
-      py310_np125:
-        PYTHON: '3.10'
-        NUMPY: '1.25'
-        CONDA_ENV: azure_ci
-        TEST_START_INDEX: 14
-      py311_np123:
-        PYTHON: '3.11'
-        NUMPY: '1.23'
-        CONDA_ENV: azure_ci
-        TEST_START_INDEX: 15
       py311_np124:
         PYTHON: '3.11'
         NUMPY: '1.24'
+        RUN_MYPY: yes
+        RUN_FLAKE8: yes
         CONDA_ENV: azure_ci
-        TEST_START_INDEX: 16
-      py311_np125:
-        PYTHON: '3.11'
-        NUMPY: '1.25'
-        CONDA_ENV: azure_ci
-        TEST_START_INDEX: 17
-
-- template: buildscripts/azure/azure-windows.yml
-  parameters:
-    name: Windows
-    vmImage: windows-2019
+        TEST_START_INDEX: 17
diff --git a/buildscripts/azure/azure-linux-macos.yml b/buildscripts/azure/azure-linux-macos.yml
@@ -38,8 +38,9 @@ jobs:
     - script: |
         set -e
         export PATH=$HOME/miniconda3/bin:$PATH
-        conda install -y mypy
-        mypy
+        # using conda-forge because main only have version < 1.0
+        conda create -n mypy311 -y -c conda-forge python=3.11 mypy 
+        conda run -n mypy311 mypy
       displayName: 'Mypy'
       condition: eq(variables['RUN_MYPY'], 'yes')
 

diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh
@@ -80,6 +80,9 @@ if [ "$TEST_THREADING" == "tbb" ]; then $CONDA_INSTALL "tbb>=2021.6" "tbb-devel>
 # Install typeguard
 if [ "$RUN_TYPEGUARD" == "yes" ]; then $CONDA_INSTALL "conda-forge::typeguard==3.0.1"; fi
 
+# Install RVSDG
+$PIP_INSTALL numba-rvsdg
+
 # environment dump for debug
 # echo "DEBUG ENV:"
 # echo "-------------------------------------------------------------------------"

diff --git a/buildscripts/incremental/test.sh b/buildscripts/incremental/test.sh
@@ -119,27 +119,5 @@ if [[ $(uname) == "Darwin" ]]; then
     export SDKROOT=`pwd`/MacOSX10.10.sdk
 fi
 
-# First check that the test discovery works
-python -m numba.tests.test_runtests
-
 # Now run tests based on the changes identified via git
-NUMBA_ENABLE_CUDASIM=1 $SEGVCATCH python -m numba.runtests -b -v -g -m $TEST_NPROCS -- numba.tests
-
-# List the tests found
-echo "INFO: All discovered tests:"
-python -m numba.runtests -l
-
-# Now run the Numba test suite with sharding
-# Note that coverage is run from the checkout dir to match the "source"
-# directive in .coveragerc
-echo "INFO: Running shard of discovered tests: ($TEST_START_INDEX:$TEST_COUNT)"
-if [ "$RUN_COVERAGE" == "yes" ]; then
-    export PYTHONPATH=.
-    coverage erase
-    $SEGVCATCH coverage run runtests.py -b -j "$TEST_START_INDEX:$TEST_COUNT" --exclude-tags='long_running' -m $TEST_NPROCS -- numba.tests
-elif [ "$RUN_TYPEGUARD" == "yes" ]; then
-    echo "INFO: Running with typeguard"
-    NUMBA_USE_TYPEGUARD=1 NUMBA_ENABLE_CUDASIM=1 PYTHONWARNINGS="ignore:::typeguard" $SEGVCATCH python runtests.py -b -j "$TEST_START_INDEX:$TEST_COUNT" --exclude-tags='long_running' -m $TEST_NPROCS -- numba.tests
-else
-    NUMBA_ENABLE_CUDASIM=1 $SEGVCATCH python -m numba.runtests -b -j "$TEST_START_INDEX:$TEST_COUNT" --exclude-tags='long_running' -m $TEST_NPROCS -- numba.tests
-fi
+NUMBA_USE_RVSDG_FRONTEND=1 NUMBA_CAPTURED_ERRORS=new_style NUMBA_ENABLE_CUDASIM=1 $SEGVCATCH python -m numba.runtests -b -v -m $TEST_NPROCS -- numba.tests.test_usecases
diff --git a/docs/source/reference/envvars.rst b/docs/source/reference/envvars.rst
@@ -405,6 +405,14 @@ Compilation options
 
     *Default value:* "all"
 
+.. envvar:: NUMBA_USE_RVSDG_FRONTEND
+
+   Turns on the experimental RVSDG frontend. It depends on the ``numba-rvsdg`` 
+   package and only supports Python 3.11 partially. 
+   This option will be removed when the RVSDG frontend fully replaces the 
+   old frontend.
+
+   *Default value:* 0 (Off)
 
 .. _numba-envvars-caching:
 

diff --git a/docs/upcoming_changes/9012.new_feature.rst b/docs/upcoming_changes/9012.new_feature.rst
@@ -0,0 +1,8 @@
+
+Added RVSDG-frontend
+====================
+
+This PR is a preliminary work on adding a RVSDG-frontend for processing 
+bytecode. RVSDG (Regionalized Value-State Dependence Graph) allows us to
+have a dataflow-centric view instead of a traditional SSA-CFG view. This
+allows us to simplify the compiler in the future.
diff --git a/mypy.ini b/mypy.ini
@@ -4,7 +4,7 @@
 warn_unused_configs = True
 follow_imports = silent
 show_error_context = True
-files = **/numba/core/types/*.py, **/numba/core/datamodel/*.py, **/numba/core/rewrites/*.py, **/numba/core/unsafe/*.py
+files = **/numba/core/types/*.py, **/numba/core/datamodel/*.py, **/numba/core/rewrites/*.py, **/numba/core/unsafe/*.py, **/numba/core/rvsdg_frontend/*.py, **/numba/core/rvsdg_frontend/rvsdg/*.py
 
 # Per-module options:
 # To classify a given module as Level 1, 2 or 3 it must be added both in files (variable above) and in the lists below.
@@ -50,3 +50,9 @@ ignore_missing_imports = True
 [mypy-winreg.*]
 # this can be removed after Mypy 0.78 is out with the latest typeshed
 ignore_missing_imports = True
+
+[mypy-numba_rvsdg.*]
+ignore_missing_imports = True
+
+[mypy-graphviz.*]
+ignore_missing_imports = True
diff --git a/numba/core/analysis.py b/numba/core/analysis.py
@@ -309,7 +309,7 @@ def find_branches(func_ir):
             if isinstance(branch_or_jump, ir.Branch):
                 branch = branch_or_jump
                 pred = guard(get_definition, func_ir, branch.cond.name)
-                if pred is not None and pred.op == "call":
+                if pred is not None and getattr(pred, "op", None) == "call":
                     function = guard(get_definition, func_ir, pred.func)
                     if (function is not None and
                         isinstance(function, ir.Global) and

diff --git a/numba/core/compiler.py b/numba/core/compiler.py
@@ -22,6 +22,7 @@
                                        CanonicalizeLoopEntry, LiteralUnroll,
                                        ReconstructSSA, RewriteDynamicRaises,
                                        LiteralPropagationSubPipelinePass,
+                                       RVSDGFrontend,
                                        )
 
 from numba.core.typed_passes import (NopythonTypeInference, AnnotateTypes,
@@ -661,10 +662,17 @@ def define_parfor_gufunc_pipeline(state, name="parfor_gufunc_typed"):
     def define_untyped_pipeline(state, name='untyped'):
         """Returns an untyped part of the nopython pipeline"""
         pm = PassManager(name)
-        if state.func_ir is None:
-            pm.add_pass(TranslateByteCode, "analyzing bytecode")
-            pm.add_pass(FixupArgs, "fix up args")
-        pm.add_pass(IRProcessing, "processing IR")
+        if config.USE_RVSDG_FRONTEND:
+            if state.func_ir is None:
+                pm.add_pass(RVSDGFrontend, "rvsdg frontend")
+                pm.add_pass(FixupArgs, "fix up args")
+            pm.add_pass(IRProcessing, "processing IR")
+        else:
+            if state.func_ir is None:
+                pm.add_pass(TranslateByteCode, "analyzing bytecode")
+                pm.add_pass(FixupArgs, "fix up args")
+            pm.add_pass(IRProcessing, "processing IR")
+
         pm.add_pass(WithLifting, "Handle with contexts")
 
         # inline closures early in case they are using nonlocal's

diff --git a/numba/core/config.py b/numba/core/config.py
@@ -158,6 +158,9 @@ def _readenv(name, ctor, default):
         def optional_str(x):
             return str(x) if x is not None else None
 
+        # RVSDG frontend selection
+        USE_RVSDG_FRONTEND = _readenv("NUMBA_USE_RVSDG_FRONTEND", int, 0)
+
         # developer mode produces full tracebacks, disables help instructions
         DEVELOPER_MODE = _readenv("NUMBA_DEVELOPER_MODE", int, 0)
 

diff --git a/numba/core/rvsdg_frontend/__init__.py b/numba/core/rvsdg_frontend/__init__.py