[ZEPPELIN-5525] Python vanillar interpreter doesn't' work in Python 3.8

.github/workflows/core.yml

@@ -32,6 +32,7 @@ jobs:
       fail-fast: false
       matrix:
         hadoop: [hadoop2, hadoop3]
+        python: [3.7, 3.8]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -51,12 +52,12 @@ jobs:
           key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-zeppelin-
-      - name: Setup conda environment with python 3.7 and R
+      - name: Setup conda environment with python ${{ matrix.python }} and R
         uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: python_3_with_R
-          environment-file: testing/env_python_3_with_R.yml
-          python-version: 3.7
+          environment-file: testing/env_python_${{ matrix.python }}_with_R.yml
+          python-version: ${{ matrix.python }}
           auto-activate-base: false
           channel-priority: strict
       - name: Make IRkernel available to Jupyter
@@ -298,6 +299,10 @@ jobs:
 
   spark-3-0-and-scala-2-12-and-other-interpreter:
     runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python: [ 3.7, 3.8 ]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -317,24 +322,28 @@ jobs:
           key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-zeppelin-
-      - name: Setup conda environment with python 3.7 and R
+      - name: Setup conda environment with python ${{ matrix.python }} and R
         uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: python_3_with_R
-          environment-file: testing/env_python_3_with_R.yml
-          python-version: 3.7
+          environment-file: testing/env_python_${{ matrix.python }}_with_R.yml
+          python-version: ${{ matrix.python }}
           auto-activate-base: false
       - name: Make IRkernel available to Jupyter
         run: |
           R -e "IRkernel::installspec()"
       - name: install environment
         run: |
           mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.0 -Pspark-scala-2.12 -Phadoop2 -B
-      - name: run tests
+      - name: run tests with ${{ matrix.python }}
         run: mvn test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.0 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false
 
   spark-3-1-and-scala-2-12-and-other-interpreter:
     runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python: [ 3.7, 3.8 ]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -354,19 +363,19 @@ jobs:
           key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-zeppelin-
-      - name: Setup conda environment with python 3.7 and R
+      - name: Setup conda environment with python ${{ matrix.python }} and R
         uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: python_3_with_R
-          environment-file: testing/env_python_3_with_R.yml
-          python-version: 3.7
+          environment-file: testing/env_python_${{ matrix.python }}_with_R.yml
+          python-version: ${{ matrix.python }}
           auto-activate-base: false
       - name: Make IRkernel available to Jupyter
         run: |
           R -e "IRkernel::installspec()"
       - name: install environment
         run: mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B
-      - name: run tests
+      - name: run tests with ${{ matrix.python }}
         run: mvn test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false
   test-livy-0-5-with-spark-2-2-0-under-python3:
     runs-on: ubuntu-20.04

flink/flink-scala-parent/src/test/java/org/apache/zeppelin/flink/IPyFlinkInterpreterTest.java

@@ -62,6 +62,12 @@ public class IPyFlinkInterpreterTest extends IPythonInterpreterTest {
   private LazyOpenInterpreter flinkScalaInterpreter;
 
 
+  public IPyFlinkInterpreterTest() {
+    super();
+    // disable bokeh test, because its (bokeh2) dependencies conflicts with apache-flink,
+    this.enableBokehTest = false;
+  }
+
   protected Properties initIntpProperties() {
     Properties p = new Properties();
     p.setProperty("zeppelin.pyflink.python", "python");

python/src/main/resources/python/zeppelin_python.py

@@ -77,6 +77,15 @@ def getCompletion(self, text_value):
       result = json.dumps(list(filter(lambda x : not re.match("^__.*", x), list(completionList))))
       self.interpreter.setStatementsFinished(result, False)
 
+# ast api is changed after python 3.8, see https://github.com/ipython/ipython/pull/11593
+if sys.version_info > (3,8):
+  from ast import Module
+else :
+  # mock the new API, ignore second argument
+  # see https://github.com/ipython/ipython/issues/11590
+  from ast import Module as OriginalModule
+  Module = lambda nodelist, type_ignores: OriginalModule(nodelist)
+
 host = sys.argv[1]
 port = int(sys.argv[2])
 
@@ -148,7 +157,7 @@ def getCompletion(self, text_value):
                                    [code.body[-(nhooks + 1)]] if len(code.body) > nhooks else [])
       try:
         for node in to_run_exec:
-          mod = ast.Module([node])
+          mod = Module([node], [])
           code = compile(mod, '<stdin>', 'exec')
           exec(code, _zcUserQueryNameSpace)
 
@@ -158,7 +167,7 @@ def getCompletion(self, text_value):
           exec(code, _zcUserQueryNameSpace)
 
         for node in to_run_hooks:
-          mod = ast.Module([node])
+          mod = Module([node], [])
           code = compile(mod, '<stdin>', 'exec')
           exec(code, _zcUserQueryNameSpace)
 

python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java

@@ -29,6 +29,8 @@
 import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -49,6 +51,10 @@
 
 public class IPythonInterpreterTest extends BasePythonInterpreterTest {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(IPythonInterpreterTest.class);
+
+  protected boolean enableBokehTest = true;
+
   protected Properties initIntpProperties() {
     Properties properties = new Properties();
     properties.setProperty("zeppelin.python.maxResult", "3");
@@ -215,14 +221,18 @@ public void testIPythonPlotting() throws InterpreterException, InterruptedExcept
     assertTrue("No Image Output", hasImageOutput);
     assertTrue("No Line Text", hasLineText);
 
+    if (!enableBokehTest) {
+      LOGGER.info("Bokeh test is skipped");
+      return;
+    }
+
     // bokeh
     // bokeh initialization
     context = getInterpreterContext();
     result = interpreter.interpret("from bokeh.io import output_notebook, show\n" +
         "from bokeh.plotting import figure\n" +
-        "import bkzep\n" +
-        "output_notebook(notebook_type='zeppelin')", context);
-    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+        "output_notebook()", context);
+    assertEquals(context.out.toString(), InterpreterResult.Code.SUCCESS, result.code());
     interpreterResultMessages = context.out.toInterpreterResultMessage();
 
     if (interpreterResultMessages.size() == 3) {
@@ -299,14 +309,14 @@ public void testIPythonPlotting() throws InterpreterException, InterruptedExcept
             InterpreterResult.Code.SUCCESS, result.code());
     interpreterResultMessages = context.out.toInterpreterResultMessage();
 
-    assertEquals(context.out.toString(), 5, interpreterResultMessages.size());
+    assertEquals(interpreterResultMessages.size() + ":" + context.out.toString(),
+            3, interpreterResultMessages.size());
     // the first message is the warning text message.
+    assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(0).getType());
     assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(1).getType());
     assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(2).getType());
-    assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(3).getType());
-    assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(4).getType());
     // docs_json is the source data of plotting which bokeh would use to render the plotting.
-    assertTrue(interpreterResultMessages.get(4).getData().contains("docs_json"));
+    assertTrue(interpreterResultMessages.get(2).getData().contains("docs_json"));
   }
 
 

testing/env_python_3.7_with_R.yml

@@ -0,0 +1,33 @@
+name: python_3_with_R
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - pycodestyle
+  - scipy
+  - numpy=1.19.5
+  - grpcio
+  - protobuf
+  - pandasql
+  - ipython
+  - ipykernel
+  - jupyter_client=5
+  - hvplot
+  - plotnine
+  - seaborn
+  - intake
+  - intake-parquet
+  - intake-xarray
+  - altair
+  - vega_datasets
+  - plotly
+  - pip
+  - r-base=3
+  - r-data.table
+  - r-evaluate
+  - r-base64enc
+  - r-knitr
+  - r-ggplot2
+  - r-irkernel
+  - r-shiny
+  - r-googlevis

testing/env_python_3.8_with_R.yml

@@ -0,0 +1,33 @@
+name: python_3_with_R
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - pycodestyle
+  - scipy
+  - numpy=1.19.5
+  - grpcio
+  - protobuf
+  - pandasql
+  - ipython
+  - ipykernel
+  - jupyter_client=5
+  - hvplot
+  - plotnine
+  - seaborn
+  - intake
+  - intake-parquet
+  - intake-xarray
+  - altair
+  - vega_datasets
+  - plotly
+  - pip
+  - r-base=3
+  - r-data.table
+  - r-evaluate
+  - r-base64enc
+  - r-knitr
+  - r-ggplot2
+  - r-irkernel
+  - r-shiny
+  - r-googlevis

testing/env_python_3_with_R.yml

@@ -4,30 +4,31 @@ channels:
   - defaults
 dependencies:
   - pycodestyle
-  - numpy=1
-  - pandas=0.25
-  - scipy=1
-  - grpcio=1.22.0
-  - hvplot=0.5.2
-  - protobuf=3
-  - pandasql=0.7.3
-  - ipython=7
-  - matplotlib=3
-  - ipykernel=5
+  - scipy
+  - numpy=1.19.5
+  - grpcio
+  - protobuf
+  - pandasql
+  - ipython
+  - ipykernel
   - jupyter_client=5
-  - bokeh=1.3.4
-  - panel=0.6.0
-  - holoviews=1.12.3
-  - pyyaml=3
+  - hvplot
+  - plotnine
+  - seaborn
+  - bokeh
+  - intake
+  - intake-parquet
+  - intake-xarray
+  - altair
+  - vega_datasets
+  - plotly
   - pip
-  - pip:
-    - bkzep==0.6.1
-
   - r-base=3
+  - r-data.table
   - r-evaluate
   - r-base64enc
   - r-knitr
   - r-ggplot2
   - r-irkernel
   - r-shiny
-  - r-googlevis
+  - r-googlevis

testing/env_python_3_with_R_and_tensorflow.yml

@@ -4,32 +4,32 @@ channels:
   - defaults
 dependencies:
   - pycodestyle
-  - numpy=1
-  - pandas=0.25
-  - scipy=1
-  - grpcio=1.22.0
-  - hvplot=0.5.2
-  - protobuf=3
-  - pandasql=0.7.3
-  - ipython=7
-  - matplotlib=3
-  - ipykernel=5
+  - scipy
+  - numpy=1.19.5
+  - grpcio
+  - protobuf
+  - pandasql
+  - ipython
+  - ipykernel
   - jupyter_client=5
-  - bokeh=1.3.4
-  - panel=0.6.0
-  - holoviews=1.12.3
-  - pyyaml=3
+  - hvplot
+  - plotnine
+  - seaborn
+  - bokeh
+  - intake
+  - intake-parquet
+  - intake-xarray
+  - altair
+  - vega_datasets
+  - plotly
   - pip
-  - pip:
-    - bkzep==0.6.1
-
   - r-base=3
+  - r-data.table
   - r-evaluate
   - r-base64enc
   - r-knitr
   - r-ggplot2
   - r-irkernel
   - r-shiny
   - r-googlevis
-
   - tensorflow=1.13