Skip to content
Permalink
Browse files

update hadoop and pig version

  • Loading branch information...
sdpython committed Jun 16, 2017
1 parent 13b1e14 commit c5b25db3ac38abd95f3aab28de04f45efddb6b8b
Showing with 18 additions and 21 deletions.
  1. +7 −8 src/pyensae/file_helper/jython_helper.py
  2. +11 −13 src/pyensae/file_helper/pig_helper.py
@@ -9,27 +9,26 @@
import urllib
import urllib.request
from pyquickhelper.loghelper import run_cmd, noLOG
from .pig_helper import HADOOP_VERSION


def download_java_standalone(version="2.7.1b3"):
def download_java_standalone(version=HADOOP_VERSION):
"""
download the standalone jython
if it does not exists, we should version 2.5.3 by default
in order to fit the cluster's version
if it does not exists, we should version ``HADOOP_VERSION``
by default in order to fit the cluster's version
@param version ``2.5.3`` or ``2.5.4-rc1`` or ...
@param version ``HADOOP_VERSION`` or ...
@return path to it
"""

dest = "jython-standalone-%s.jar" % version
if version in ("2.5.3", "2.7.1b2", "2.7.0", "2.7.1b3"):
url = "http://repo1.maven.org/maven2/org/python/jython-standalone/{1}/{0}".format(
dest,
version)
dest, version)
else:
url = "http://search.maven.org/remotecontent?filepath=org/python/jython-standalone/{1}/{0}".format(
dest,
version)
dest, version)
this = os.path.abspath(os.path.dirname(__file__))
final = os.path.join(this, dest)
if os.path.exists(final):
@@ -13,12 +13,16 @@
from .jython_helper import get_java_cmd, get_java_path
from ..datasource.http_retrieve import download_data

PIG_VERSION = "0.16.0"
HADOOP_VERSION = "2.8.0"

def download_pig_standalone(pig_version="0.15.0", hadoop_version="2.7.2", fLOG=noLOG):

def download_pig_standalone(pig_version=PIG_VERSION,
hadoop_version=HADOOP_VERSION, fLOG=noLOG):
"""
download the standalone jython
if it does not exists, we should version 2.5.3 by default
in order to fit the cluster's version
if it does not exists, we should version ``HADOOP_VERSION``
by default in order to fit the cluster's version
@param pig_version pig_version
@param hadoop_version hadoop_version
@@ -126,7 +130,7 @@ def get_pig_jars():
and "hadoop1-runtime" not in root \
and "test" not in root \
and "h2" not in name \
and "pig-0.15.0-withouthadoop-h2" not in name:
and ("pig-" + PIG_VERSION + "-withouthadoop-h2") not in name:
res.append(os.path.join(root, name))
return res

@@ -148,15 +152,9 @@ def get_hadoop_jars():
return res


def run_pig(pigfile,
argv=None,
pig_path=None,
hadoop_path=None,
jython_path=None,
timeout=None,
logpath="logs",
pig_version="0.15.0",
hadoop_version="2.7.2",
def run_pig(pigfile, argv=None, pig_path=None, hadoop_path=None,
jython_path=None, timeout=None, logpath="logs",
pig_version=PIG_VERSION, hadoop_version=HADOOP_VERSION,
fLOG=noLOG):
"""
runs a pig script and returns the standard output and error

0 comments on commit c5b25db

Please sign in to comment.
You can’t perform that action at this time.