Skip to content

Commit

Permalink
[ZEPPELIN-5525] Python vanillar interpreter doesn't' work in Python 3.8
Browse files Browse the repository at this point in the history
  • Loading branch information
zjffdu committed Sep 26, 2021
1 parent 056f952 commit b816a53
Show file tree
Hide file tree
Showing 12 changed files with 231 additions and 124 deletions.
31 changes: 20 additions & 11 deletions .github/workflows/core.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ jobs:
fail-fast: false
matrix:
hadoop: [hadoop2, hadoop3]
python: [3.7, 3.8]
steps:
- name: Checkout
uses: actions/checkout@v2
Expand All @@ -51,12 +52,12 @@ jobs:
key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-zeppelin-
- name: Setup conda environment with python 3.7 and R
- name: Setup conda environment with python ${{ matrix.python }} and R
uses: conda-incubator/setup-miniconda@v2
with:
activate-environment: python_3_with_R
environment-file: testing/env_python_3_with_R.yml
python-version: 3.7
environment-file: testing/env_python_${{ matrix.python }}_with_R.yml
python-version: ${{ matrix.python }}
auto-activate-base: false
channel-priority: strict
- name: Make IRkernel available to Jupyter
Expand Down Expand Up @@ -298,6 +299,10 @@ jobs:

spark-3-0-and-scala-2-12-and-other-interpreter:
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
python: [ 3.7, 3.8 ]
steps:
- name: Checkout
uses: actions/checkout@v2
Expand All @@ -317,24 +322,28 @@ jobs:
key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-zeppelin-
- name: Setup conda environment with python 3.7 and R
- name: Setup conda environment with python ${{ matrix.python }} and R
uses: conda-incubator/setup-miniconda@v2
with:
activate-environment: python_3_with_R
environment-file: testing/env_python_3_with_R.yml
python-version: 3.7
environment-file: testing/env_python_${{ matrix.python }}_with_R.yml
python-version: ${{ matrix.python }}
auto-activate-base: false
- name: Make IRkernel available to Jupyter
run: |
R -e "IRkernel::installspec()"
- name: install environment
run: |
mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.0 -Pspark-scala-2.12 -Phadoop2 -B
- name: run tests
- name: run tests with ${{ matrix.python }}
run: mvn test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.0 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false

spark-3-1-and-scala-2-12-and-other-interpreter:
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
python: [ 3.7, 3.8 ]
steps:
- name: Checkout
uses: actions/checkout@v2
Expand All @@ -354,19 +363,19 @@ jobs:
key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-zeppelin-
- name: Setup conda environment with python 3.7 and R
- name: Setup conda environment with python ${{ matrix.python }} and R
uses: conda-incubator/setup-miniconda@v2
with:
activate-environment: python_3_with_R
environment-file: testing/env_python_3_with_R.yml
python-version: 3.7
environment-file: testing/env_python_${{ matrix.python }}_with_R.yml
python-version: ${{ matrix.python }}
auto-activate-base: false
- name: Make IRkernel available to Jupyter
run: |
R -e "IRkernel::installspec()"
- name: install environment
run: mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B
- name: run tests
- name: run tests with ${{ matrix.python }}
run: mvn test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false
test-livy-0-5-with-spark-2-2-0-under-python3:
runs-on: ubuntu-20.04
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,12 @@ public class IPyFlinkInterpreterTest extends IPythonInterpreterTest {
private LazyOpenInterpreter flinkScalaInterpreter;


public IPyFlinkInterpreterTest() {
super();
// disable bokeh test, because its (bokeh2) dependencies conflicts with apache-flink,
this.enableBokehTest = false;
}

protected Properties initIntpProperties() {
Properties p = new Properties();
p.setProperty("zeppelin.pyflink.python", "python");
Expand Down
13 changes: 11 additions & 2 deletions python/src/main/resources/python/zeppelin_python.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,15 @@ def getCompletion(self, text_value):
result = json.dumps(list(filter(lambda x : not re.match("^__.*", x), list(completionList))))
self.interpreter.setStatementsFinished(result, False)

# ast api is changed after python 3.8, see https://github.com/ipython/ipython/pull/11593
if sys.version_info > (3,8):
from ast import Module
else :
# mock the new API, ignore second argument
# see https://github.com/ipython/ipython/issues/11590
from ast import Module as OriginalModule
Module = lambda nodelist, type_ignores: OriginalModule(nodelist)

host = sys.argv[1]
port = int(sys.argv[2])

Expand Down Expand Up @@ -148,7 +157,7 @@ def getCompletion(self, text_value):
[code.body[-(nhooks + 1)]] if len(code.body) > nhooks else [])
try:
for node in to_run_exec:
mod = ast.Module([node])
mod = Module([node], [])
code = compile(mod, '<stdin>', 'exec')
exec(code, _zcUserQueryNameSpace)

Expand All @@ -158,7 +167,7 @@ def getCompletion(self, text_value):
exec(code, _zcUserQueryNameSpace)

for node in to_run_hooks:
mod = ast.Module([node])
mod = Module([node], [])
code = compile(mod, '<stdin>', 'exec')
exec(code, _zcUserQueryNameSpace)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.ArrayList;
Expand All @@ -49,6 +51,10 @@

public class IPythonInterpreterTest extends BasePythonInterpreterTest {

private static final Logger LOGGER = LoggerFactory.getLogger(IPythonInterpreterTest.class);

protected boolean enableBokehTest = true;

protected Properties initIntpProperties() {
Properties properties = new Properties();
properties.setProperty("zeppelin.python.maxResult", "3");
Expand Down Expand Up @@ -215,14 +221,18 @@ public void testIPythonPlotting() throws InterpreterException, InterruptedExcept
assertTrue("No Image Output", hasImageOutput);
assertTrue("No Line Text", hasLineText);

if (!enableBokehTest) {
LOGGER.info("Bokeh test is skipped");
return;
}

// bokeh
// bokeh initialization
context = getInterpreterContext();
result = interpreter.interpret("from bokeh.io import output_notebook, show\n" +
"from bokeh.plotting import figure\n" +
"import bkzep\n" +
"output_notebook(notebook_type='zeppelin')", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
"output_notebook()", context);
assertEquals(context.out.toString(), InterpreterResult.Code.SUCCESS, result.code());
interpreterResultMessages = context.out.toInterpreterResultMessage();

if (interpreterResultMessages.size() == 3) {
Expand Down Expand Up @@ -299,14 +309,14 @@ public void testIPythonPlotting() throws InterpreterException, InterruptedExcept
InterpreterResult.Code.SUCCESS, result.code());
interpreterResultMessages = context.out.toInterpreterResultMessage();

assertEquals(context.out.toString(), 5, interpreterResultMessages.size());
assertEquals(interpreterResultMessages.size() + ":" + context.out.toString(),
3, interpreterResultMessages.size());
// the first message is the warning text message.
assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(0).getType());
assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(1).getType());
assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(2).getType());
assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(3).getType());
assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(4).getType());
// docs_json is the source data of plotting which bokeh would use to render the plotting.
assertTrue(interpreterResultMessages.get(4).getData().contains("docs_json"));
assertTrue(interpreterResultMessages.get(2).getData().contains("docs_json"));
}


Expand Down
33 changes: 33 additions & 0 deletions testing/env_python_3.7_with_R.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
name: python_3_with_R
channels:
- conda-forge
- defaults
dependencies:
- pycodestyle
- scipy
- numpy=1.19.5
- grpcio
- protobuf
- pandasql
- ipython
- ipykernel
- jupyter_client=5
- hvplot
- plotnine
- seaborn
- intake
- intake-parquet
- intake-xarray
- altair
- vega_datasets
- plotly
- pip
- r-base=3
- r-data.table
- r-evaluate
- r-base64enc
- r-knitr
- r-ggplot2
- r-irkernel
- r-shiny
- r-googlevis
33 changes: 33 additions & 0 deletions testing/env_python_3.8_with_R.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
name: python_3_with_R
channels:
- conda-forge
- defaults
dependencies:
- pycodestyle
- scipy
- numpy=1.19.5
- grpcio
- protobuf
- pandasql
- ipython
- ipykernel
- jupyter_client=5
- hvplot
- plotnine
- seaborn
- intake
- intake-parquet
- intake-xarray
- altair
- vega_datasets
- plotly
- pip
- r-base=3
- r-data.table
- r-evaluate
- r-base64enc
- r-knitr
- r-ggplot2
- r-irkernel
- r-shiny
- r-googlevis
37 changes: 19 additions & 18 deletions testing/env_python_3_with_R.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,30 +4,31 @@ channels:
- defaults
dependencies:
- pycodestyle
- numpy=1
- pandas=0.25
- scipy=1
- grpcio=1.22.0
- hvplot=0.5.2
- protobuf=3
- pandasql=0.7.3
- ipython=7
- matplotlib=3
- ipykernel=5
- scipy
- numpy=1.19.5
- grpcio
- protobuf
- pandasql
- ipython
- ipykernel
- jupyter_client=5
- bokeh=1.3.4
- panel=0.6.0
- holoviews=1.12.3
- pyyaml=3
- hvplot
- plotnine
- seaborn
- bokeh
- intake
- intake-parquet
- intake-xarray
- altair
- vega_datasets
- plotly
- pip
- pip:
- bkzep==0.6.1

- r-base=3
- r-data.table
- r-evaluate
- r-base64enc
- r-knitr
- r-ggplot2
- r-irkernel
- r-shiny
- r-googlevis
- r-googlevis
36 changes: 18 additions & 18 deletions testing/env_python_3_with_R_and_tensorflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,32 +4,32 @@ channels:
- defaults
dependencies:
- pycodestyle
- numpy=1
- pandas=0.25
- scipy=1
- grpcio=1.22.0
- hvplot=0.5.2
- protobuf=3
- pandasql=0.7.3
- ipython=7
- matplotlib=3
- ipykernel=5
- scipy
- numpy=1.19.5
- grpcio
- protobuf
- pandasql
- ipython
- ipykernel
- jupyter_client=5
- bokeh=1.3.4
- panel=0.6.0
- holoviews=1.12.3
- pyyaml=3
- hvplot
- plotnine
- seaborn
- bokeh
- intake
- intake-parquet
- intake-xarray
- altair
- vega_datasets
- plotly
- pip
- pip:
- bkzep==0.6.1

- r-base=3
- r-data.table
- r-evaluate
- r-base64enc
- r-knitr
- r-ggplot2
- r-irkernel
- r-shiny
- r-googlevis

- tensorflow=1.13
Loading

0 comments on commit b816a53

Please sign in to comment.