Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions pig/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,13 @@
<forkMode>always</forkMode>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,6 @@
import org.apache.pig.backend.BackendException;
import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine;
import org.apache.pig.backend.hadoop.executionengine.Launcher;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -35,12 +31,16 @@
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;

/**
*
*/
public abstract class BasePigInterpreter extends Interpreter {

private static Logger LOGGER = LoggerFactory.getLogger(BasePigInterpreter.class);
private static final Logger LOGGER = LoggerFactory.getLogger(BasePigInterpreter.class);

protected ConcurrentHashMap<String, PigScriptListener> listenerMap = new ConcurrentHashMap<>();

Expand Down
15 changes: 9 additions & 6 deletions pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,26 @@
import org.apache.pig.PigServer;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.tools.pigscript.parser.ParseException;
import org.apache.pig.tools.pigstats.*;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.ScriptState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.*;
import java.util.Map;
import java.util.Properties;

import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;

/**
* Pig interpreter for Zeppelin.
*/
public class PigInterpreter extends BasePigInterpreter {
private static Logger LOGGER = LoggerFactory.getLogger(PigInterpreter.class);
private static final Logger LOGGER = LoggerFactory.getLogger(PigInterpreter.class);

private PigServer pigServer;
private boolean includeJobStats = false;
Expand Down
16 changes: 10 additions & 6 deletions pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,8 @@
* limitations under the License.
*/


package org.apache.zeppelin.pig;


import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.pig.PigServer;
Expand All @@ -28,8 +26,6 @@
import org.apache.pig.tools.pigscript.parser.ParseException;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.ScriptState;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -40,12 +36,20 @@
import java.util.List;
import java.util.Properties;

import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.ResultMessages;
import org.apache.zeppelin.interpreter.WrappedInterpreter;

/**
*
*/
public class PigQueryInterpreter extends BasePigInterpreter {

private static Logger LOGGER = LoggerFactory.getLogger(PigQueryInterpreter.class);
private static final Logger LOGGER = LoggerFactory.getLogger(PigQueryInterpreter.class);
private static final String MAX_RESULTS = "zeppelin.pig.maxResult";
private PigServer pigServer;
private int maxResult;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
* limitations under the License.
*/


package org.apache.zeppelin.pig;

import org.apache.pig.impl.plan.OperatorPlan;
Expand All @@ -32,8 +31,7 @@
*
*/
public class PigScriptListener implements PigProgressNotificationListener {

private static Logger LOGGER = LoggerFactory.getLogger(PigScriptListener.class);
private static final Logger LOGGER = LoggerFactory.getLogger(PigScriptListener.class);

private Set<String> jobIds = new HashSet();
private int progress;
Expand Down
24 changes: 2 additions & 22 deletions pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,40 +17,21 @@

package org.apache.zeppelin.pig;


import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.pig.PigRunner;
import org.apache.pig.backend.hadoop.executionengine.spark.plan.SparkOperator;
import org.apache.pig.backend.hadoop.executionengine.tez.TezExecType;
import org.apache.pig.tools.pigstats.InputStats;
import org.apache.pig.tools.pigstats.JobStats;
import org.apache.pig.tools.pigstats.OutputStats;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.mapreduce.MRJobStats;
import org.apache.pig.tools.pigstats.mapreduce.SimplePigStats;
import org.apache.pig.tools.pigstats.spark.SparkJobStats;
import org.apache.pig.tools.pigstats.spark.SparkPigStats;
import org.apache.pig.tools.pigstats.spark.SparkScriptState;
import org.apache.pig.tools.pigstats.tez.TezDAGStats;
import org.apache.pig.tools.pigstats.tez.TezPigScriptStats;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Field;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.List;

/**
*
*/
public class PigUtils {

private static Logger LOGGER = LoggerFactory.getLogger(PigUtils.class);
private static final Logger LOGGER = LoggerFactory.getLogger(PigUtils.class);

protected static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";

Expand All @@ -66,5 +47,4 @@ public static File createTempPigScript(String content) throws IOException {
public static File createTempPigScript(List<String> lines) throws IOException {
return createTempPigScript(StringUtils.join(lines, "\n"));
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@
* limitations under the License.
*/


package org.apache.zeppelin.pig;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.After;
import org.junit.Test;

Expand All @@ -30,9 +30,8 @@
import java.io.IOException;
import java.util.Properties;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;

public class PigInterpreterSparkTest {
private PigInterpreter pigInterpreter;
Expand Down Expand Up @@ -86,7 +85,8 @@ public void testBasics() throws IOException {
result = pigInterpreter.interpret(pigscript, context);
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'"));
assertTrue(result.message().get(0).getData().contains(
"Syntax error, unexpected symbol at or near 'a'"));

// syntax error
pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
Expand Down Expand Up @@ -133,7 +133,8 @@ public void testIncludeJobStats() throws IOException {
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertEquals(InterpreterResult.Code.ERROR, result.code());
// no job is launched, so no jobStats
assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'"));
assertTrue(result.message().get(0).getData().contains(
"Syntax error, unexpected symbol at or near 'a'"));

// execution error
pigscript = "a = load 'invalid_path';"
Expand Down
23 changes: 12 additions & 11 deletions pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,24 @@
* limitations under the License.
*/


package org.apache.zeppelin.pig;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Properties;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;

public class PigInterpreterTest {

Expand All @@ -48,7 +47,7 @@ private void setUpLocal(boolean includeJobStats) {
pigInterpreter = new PigInterpreter(properties);
pigInterpreter.open();
context = new InterpreterContext(null, "paragraph_id", null, null, null,
null, null, null, null, null, null,null, null);
null, null, null, null, null, null, null, null);
}

@After
Expand Down Expand Up @@ -89,7 +88,8 @@ public void testBasics() throws IOException {
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'"));
assertTrue(result.message().get(0).getData().contains(
"Syntax error, unexpected symbol at or near 'a'"));

// execution error
pigscript = "a = load 'invalid_path';"
Expand Down Expand Up @@ -139,7 +139,8 @@ public void testIncludeJobStats() throws IOException {
assertEquals(Code.ERROR, result.code());
// no job is launched, so no jobStats
assertTrue(!result.message().get(0).getData().contains("Counters:"));
assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'"));
assertTrue(result.message().get(0).getData().contains(
"Syntax error, unexpected symbol at or near 'a'"));

// execution error
pigscript = "a = load 'invalid_path';"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,24 @@
* limitations under the License.
*/


package org.apache.zeppelin.pig;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Properties;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;

public class PigInterpreterTezTest {

Expand Down Expand Up @@ -94,7 +93,8 @@ public void testBasics() throws IOException {
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'"));
assertTrue(result.message().get(0).getData().contains(
"Syntax error, unexpected symbol at or near 'a'"));

// syntax error
pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
Expand Down Expand Up @@ -143,7 +143,8 @@ public void testIncludeJobStats() throws IOException {
assertEquals(Code.ERROR, result.code());
// no job is launched, so no jobStats
assertTrue(!result.message().get(0).getData().contains("Vertex Stats"));
assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'"));
assertTrue(result.message().get(0).getData().contains(
"Syntax error, unexpected symbol at or near 'a'"));

// execution error
pigscript = "a = load 'invalid_path';"
Expand Down
Loading