Skip to content
Permalink
Browse files
[FLINK-27340][python][tests] Migrate flink-python to JUnit5
  • Loading branch information
snuyanzin committed May 19, 2022
1 parent a70e289 commit cf4d7553e4dd89bececbb702d0b6e79c24677110
Showing 40 changed files with 623 additions and 619 deletions.
@@ -22,7 +22,7 @@
import org.apache.flink.configuration.Configuration;
import org.apache.flink.python.PythonOptions;

import org.junit.Test;
import org.junit.jupiter.api.Test;

import java.io.File;
import java.io.IOException;
@@ -35,19 +35,18 @@

import static org.apache.flink.python.PythonOptions.PYTHON_EXECUTABLE;
import static org.apache.flink.python.PythonOptions.PYTHON_REQUIREMENTS;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.assertj.core.api.Assertions.assertThat;

/** ITCases for {@link PythonProgramOptions}. */
public class PythonProgramOptionsITCase {
class PythonProgramOptionsITCase {

/**
* It requires setting a job jar to build a {@link PackagedProgram}, and the dummy job jar used
* in this test case is available only after the packaging phase completed, so we make it as an
* ITCase.
*/
@Test
public void testConfigurePythonExecution() throws Exception {
void testConfigurePythonExecution() throws Exception {
final String[] args = {
"--python", "xxx.py",
"--pyModule", "xxx",
@@ -79,15 +78,13 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
Configuration configuration = new Configuration();
ProgramOptionsUtils.configurePythonExecution(configuration, packagedProgram);

assertEquals(
"/absolute/a.py,relative/b.py,relative/c.py",
configuration.get(PythonOptions.PYTHON_FILES));
assertEquals("d.txt#e_dir", configuration.get(PYTHON_REQUIREMENTS));
assertEquals(
"g.zip,h.zip#data,h.zip#data2", configuration.get(PythonOptions.PYTHON_ARCHIVES));
assertEquals("/usr/bin/python", configuration.get(PYTHON_EXECUTABLE));
assertArrayEquals(
new String[] {"--python", "xxx.py", "--pyModule", "xxx", "userarg1", "userarg2"},
packagedProgram.getArguments());
assertThat(configuration.get(PythonOptions.PYTHON_FILES))
.isEqualTo("/absolute/a.py,relative/b.py,relative/c.py");
assertThat(configuration.get(PYTHON_REQUIREMENTS)).isEqualTo("d.txt#e_dir");
assertThat(configuration.get(PythonOptions.PYTHON_ARCHIVES))
.isEqualTo("g.zip,h.zip#data,h.zip#data2");
assertThat(configuration.get(PYTHON_EXECUTABLE)).isEqualTo("/usr/bin/python");
assertThat(packagedProgram.getArguments())
.containsExactly("--python", "xxx.py", "--pyModule", "xxx", "userarg1", "userarg2");
}
}
@@ -23,8 +23,8 @@

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import static org.apache.flink.client.cli.CliFrontendParser.PYARCHIVE_OPTION;
import static org.apache.flink.client.cli.CliFrontendParser.PYEXEC_OPTION;
@@ -34,15 +34,14 @@
import static org.apache.flink.client.cli.CliFrontendParser.PY_OPTION;
import static org.apache.flink.python.PythonOptions.PYTHON_EXECUTABLE;
import static org.apache.flink.python.PythonOptions.PYTHON_REQUIREMENTS;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.assertj.core.api.Assertions.assertThat;

/** Test for {@link PythonProgramOptions}. */
public class PythonProgramOptionsTest {
class PythonProgramOptionsTest {
private Options options;

@Before
public void setUp() {
@BeforeEach
void setUp() {
options = new Options();
options.addOption(PY_OPTION);
options.addOption(PYFILES_OPTION);
@@ -53,7 +52,7 @@ public void setUp() {
}

@Test
public void testCreateProgramOptionsWithPythonCommandLine() throws CliArgsException {
void testCreateProgramOptionsWithPythonCommandLine() throws CliArgsException {
String[] parameters = {
"-py", "test.py",
"-pym", "test",
@@ -68,18 +67,18 @@ public void testCreateProgramOptionsWithPythonCommandLine() throws CliArgsExcept
PythonProgramOptions programOptions = (PythonProgramOptions) ProgramOptions.create(line);
Configuration config = new Configuration();
programOptions.applyToConfiguration(config);
assertEquals(
"test1.py,test2.zip,test3.egg,test4_dir", config.get(PythonOptions.PYTHON_FILES));
assertEquals("a.txt#b_dir", config.get(PYTHON_REQUIREMENTS));
assertEquals("c.zip#venv,d.zip", config.get(PythonOptions.PYTHON_ARCHIVES));
assertEquals("bin/python", config.get(PYTHON_EXECUTABLE));
assertArrayEquals(
new String[] {"--python", "test.py", "--pyModule", "test", "userarg1", "userarg2"},
programOptions.getProgramArgs());
assertThat(config.get(PythonOptions.PYTHON_FILES))
.isEqualTo("test1.py,test2.zip,test3.egg,test4_dir");
assertThat(config.get(PYTHON_REQUIREMENTS)).isEqualTo("a.txt#b_dir");
assertThat(config.get(PythonOptions.PYTHON_ARCHIVES)).isEqualTo("c.zip#venv,d.zip");
assertThat(config.get(PYTHON_EXECUTABLE)).isEqualTo("bin/python");
assertThat(programOptions.getProgramArgs())
.containsExactly(
"--python", "test.py", "--pyModule", "test", "userarg1", "userarg2");
}

@Test
public void testCreateProgramOptionsWithLongOptions() throws CliArgsException {
void testCreateProgramOptionsWithLongOptions() throws CliArgsException {
String[] args = {
"--python", "xxx.py",
"--pyModule", "xxx",
@@ -93,14 +92,13 @@ public void testCreateProgramOptionsWithLongOptions() throws CliArgsException {
PythonProgramOptions programOptions = (PythonProgramOptions) ProgramOptions.create(line);
Configuration config = new Configuration();
programOptions.applyToConfiguration(config);
assertEquals(
"/absolute/a.py,relative/b.py,relative/c.py",
config.get(PythonOptions.PYTHON_FILES));
assertEquals("d.txt#e_dir", config.get(PYTHON_REQUIREMENTS));
assertEquals("g.zip,h.zip#data,h.zip#data2", config.get(PythonOptions.PYTHON_ARCHIVES));
assertEquals("/usr/bin/python", config.get(PYTHON_EXECUTABLE));
assertArrayEquals(
new String[] {"--python", "xxx.py", "--pyModule", "xxx", "userarg1", "userarg2"},
programOptions.getProgramArgs());
assertThat(config.get(PythonOptions.PYTHON_FILES))
.isEqualTo("/absolute/a.py,relative/b.py,relative/c.py");
assertThat(config.get(PYTHON_REQUIREMENTS)).isEqualTo("d.txt#e_dir");
assertThat(config.get(PythonOptions.PYTHON_ARCHIVES))
.isEqualTo("g.zip,h.zip#data,h.zip#data2");
assertThat(config.get(PYTHON_EXECUTABLE)).isEqualTo("/usr/bin/python");
assertThat(programOptions.getProgramArgs())
.containsExactly("--python", "xxx.py", "--pyModule", "xxx", "userarg1", "userarg2");
}
}
@@ -21,61 +21,62 @@
import org.apache.flink.runtime.entrypoint.FlinkParseException;
import org.apache.flink.runtime.entrypoint.parser.CommandLineParser;

import org.junit.Test;
import org.junit.jupiter.api.Test;

import java.util.List;

import static org.junit.Assert.assertEquals;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

/** Tests for the {@link PythonDriverOptionsParserFactory}. */
public class PythonDriverOptionsParserFactoryTest {
class PythonDriverOptionsParserFactoryTest {

private static final CommandLineParser<PythonDriverOptions> commandLineParser =
new CommandLineParser<>(new PythonDriverOptionsParserFactory());

@Test
public void testPythonDriverOptionsParsing() throws FlinkParseException {
void testPythonDriverOptionsParsing() throws FlinkParseException {
final String[] args = {"--python", "xxx.py", "--input", "in.txt"};
verifyPythonDriverOptionsParsing(args);
}

@Test
public void testPymoduleOptionParsing() throws FlinkParseException {
void testPymoduleOptionParsing() throws FlinkParseException {
final String[] args = {"--pyModule", "xxx", "--input", "in.txt"};
verifyPythonDriverOptionsParsing(args);
}

@Test
public void testShortOptions() throws FlinkParseException {
void testShortOptions() throws FlinkParseException {
final String[] args = {"-py", "xxx.py", "--input", "in.txt"};
verifyPythonDriverOptionsParsing(args);
}

@Test(expected = FlinkParseException.class)
public void testMultipleEntrypointsSpecified() throws FlinkParseException {
@Test
void testMultipleEntrypointsSpecified() {
final String[] args = {"--python", "xxx.py", "--pyModule", "yyy", "--input", "in.txt"};
commandLineParser.parse(args);
assertThatThrownBy(() -> commandLineParser.parse(args))
.isInstanceOf(FlinkParseException.class);
}

@Test(expected = FlinkParseException.class)
public void testEntrypointNotSpecified() throws FlinkParseException {
@Test
void testEntrypointNotSpecified() {
final String[] args = {"--input", "in.txt"};
commandLineParser.parse(args);
assertThatThrownBy(() -> commandLineParser.parse(args))
.isInstanceOf(FlinkParseException.class);
}

private void verifyPythonDriverOptionsParsing(final String[] args) throws FlinkParseException {
final PythonDriverOptions pythonCommandOptions = commandLineParser.parse(args);

if (pythonCommandOptions.getEntryPointScript().isPresent()) {
assertEquals("xxx.py", pythonCommandOptions.getEntryPointScript().get());
assertThat(pythonCommandOptions.getEntryPointScript().get()).isEqualTo("xxx.py");
} else {
assertEquals("xxx", pythonCommandOptions.getEntryPointModule());
assertThat(pythonCommandOptions.getEntryPointModule()).isEqualTo("xxx");
}

// verify the python program arguments
final List<String> programArgs = pythonCommandOptions.getProgramArgs();
assertEquals(2, programArgs.size());
assertEquals("--input", programArgs.get(0));
assertEquals("in.txt", programArgs.get(1));
assertThat(programArgs).containsExactly("--input", "in.txt");
}
}
@@ -18,8 +18,7 @@

package org.apache.flink.client.python;

import org.junit.Assert;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import py4j.GatewayServer;

import java.io.IOException;
@@ -28,10 +27,12 @@
import java.util.List;
import java.util.concurrent.ExecutionException;

import static org.assertj.core.api.Assertions.assertThat;

/** Tests for the {@link PythonDriver}. */
public class PythonDriverTest {
class PythonDriverTest {
@Test
public void testStartGatewayServer() throws ExecutionException, InterruptedException {
void testStartGatewayServer() throws ExecutionException, InterruptedException {
GatewayServer gatewayServer = PythonEnvUtils.startGatewayServer();
try {
Socket socket = new Socket("localhost", gatewayServer.getListeningPort());
@@ -44,33 +45,25 @@ public void testStartGatewayServer() throws ExecutionException, InterruptedExcep
}

@Test
public void testConstructCommandsWithEntryPointModule() {
void testConstructCommandsWithEntryPointModule() {
List<String> args = new ArrayList<>();
args.add("--input");
args.add("in.txt");

PythonDriverOptions pythonDriverOptions = new PythonDriverOptions("xxx", null, args);
List<String> commands = PythonDriver.constructPythonCommands(pythonDriverOptions);
// verify the generated commands
Assert.assertEquals(4, commands.size());
Assert.assertEquals(commands.get(0), "-m");
Assert.assertEquals(commands.get(1), "xxx");
Assert.assertEquals(commands.get(2), "--input");
Assert.assertEquals(commands.get(3), "in.txt");
assertThat(commands).containsExactly("-m", "xxx", "--input", "in.txt");
}

@Test
public void testConstructCommandsWithEntryPointScript() {
void testConstructCommandsWithEntryPointScript() {
List<String> args = new ArrayList<>();
args.add("--input");
args.add("in.txt");

PythonDriverOptions pythonDriverOptions = new PythonDriverOptions(null, "xxx.py", args);
List<String> commands = PythonDriver.constructPythonCommands(pythonDriverOptions);
Assert.assertEquals(4, commands.size());
Assert.assertEquals(commands.get(0), "-m");
Assert.assertEquals(commands.get(1), "xxx");
Assert.assertEquals(commands.get(2), "--input");
Assert.assertEquals(commands.get(3), "in.txt");
assertThat(commands).containsExactly("-m", "xxx", "--input", "in.txt");
}
}

0 comments on commit cf4d755

Please sign in to comment.