Skip to content
Permalink
Browse files

improve magic command for hadoop

  • Loading branch information...
sdpython committed Oct 23, 2014
1 parent f460df3 commit dccd1f05c4333a2deb3c76b7ef55f7fc6254b3c1
Showing with 82 additions and 6 deletions.
  1. +77 −2 src/pyensae/remote/magic_remote.py
  2. +5 −4 src/pyensae/remote/remote_connection.py
@@ -30,6 +30,78 @@ def get_connection(self):
raise KeyError("No opened SSH connection.")

return self.shell.user_ns["remote_ssh"]

@cell_magic
def PIG(self, line, cell = None):
"""
defines command ``%%PIG``
"""
if line in [None, ""] :
print("Usage:")
print(" %%PIG <filename>")
print("")
print("The command store the content of the cell as a local file.")
else:
filename = line.strip()
with open(filename, "w", encoding="utf8") as f :
f.write(cell.replace("\r",""))

@line_magic
def jobsubmit(self, line):
"""
defines command ``%jobsubmit``
"""
if line in [None, ""]:
print("Usage:")
print(" %jobsubmit <jobname.pig> [redirection]")
print("")
print("If redirection is specified, the standard output and error are redirected to")
print("redirection.out, redirection.err and the function does not wait.")
else:
filename = line.strip()
spl = filename.split()
filename = spl[0]
redirection = None if len(spl) == 1 else spl[1]
if not os.path.exists(filename):
raise FileNotFoundError(filename)

dest = os.path.split(filename)[-1]
ssh = self.get_connection()
ssh.upload(filename, dest)

if redirection is None:
cmd = "pig -execute -f " + dest
else:
cmd = "pig -execute -f {0} 2> {1}.err 1> {1}.out &".format(filename, redirection)

out, err = ssh.execute_command(cmd, no_exception = True)
if len(err) > 0 and (len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err):
return HTML("<pre>\n%s\n</pre>" % err)
else:
return HTML("<pre>\n%s\n</pre>" % out)

@line_magic
def jobsyntax(self, line):
"""
defines command ``%jobsyntax``
"""
if line in [None, ""]:
print("Usage:")
print(" %jobsyntax <jobname.pig>")
print("")
else:
filename = line.strip()
if not os.path.exists(filename):
raise FileNotFoundError(filename)

dest = os.path.split(filename)[-1]
ssh = self.get_connection()
ssh.upload(filename, dest)
out, err = ssh.execute_command("pig -check " + dest, no_exception = True)
if len(err) > 0 and (len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err):
return HTML("<pre>\n%s\n</pre>" % err)
else:
return HTML("<pre>\n%s\n</pre>" % out)

@line_magic
def remote_open (self, line):
@@ -85,8 +157,11 @@ def remote_cmd(self, line):
"""
ssh = self.get_connection()
ssh = self.shell.user_ns["remote_ssh"]
out, err = ssh.execute_command(line)
return HTML("<pre>{0}</pre>".format(out))
out, err = ssh.execute_command(line, no_exception = True)
if len(err) > 0 and (len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err):
return HTML("<pre>\n%s\n</pre>" % err)
else:
return HTML("<pre>\n%s\n</pre>" % out)

@line_magic
def remote_up(self, line):
@@ -41,13 +41,14 @@ def connect(self):
self.connection.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.connection.connect(self.server, username=self.username, password=self.password)

def execute_command(self, command):
def execute_command(self, command, no_exception = False):
"""
execute a command line, it raises an error
if there is an error
@param command command
@return stdout, stderr
@param command command
@param no_exception if True, do not raise any exception
@return stdout, stderr
Example of commands::
@@ -70,7 +71,7 @@ def execute_command(self, command):
if isinstance(out, bytes):
out = out.decode("utf-8")

if len(err) > 0 :
if not no_exception and len(err) > 0 :
raise Exception("unable to run: {0}\nOUT:\n{1}\nERR:\n{2}".format(command, out, err))

return out,err

0 comments on commit dccd1f0

Please sign in to comment.
You can’t perform that action at this time.