Skip to content

Commit

Permalink
fixed streaming upload in CLAM Client (addinputfile()), now using pycurl
Browse files Browse the repository at this point in the history
  • Loading branch information
@proycon
proycon committed May 8, 2013
1 parent ac5535e commit e08315a
Show file tree
Hide file tree
Showing 8 changed files with 160 additions and 94 deletions.
69 changes: 35 additions & 34 deletions clamclient.py
View file
Expand Up @@ -8,10 +8,10 @@
# http://ilk.uvt.nl/~mvgompel
# Induction for Linguistic Knowledge Research Group
# Universiteit van Tilburg
#
#
# Licensed under GPLv3
#
# This is a very generic CLAM command-line client, useable with
# This is a very generic CLAM command-line client, useable with
# any CLAM Service. It provides a simple and fairly low-level
# command-line interface.
#
Expand All @@ -26,7 +26,7 @@
from clam.common.client import CLAMClient, NotFound, PermissionDenied, ServerError, AuthRequired
from clam.common.data import ParameterCondition

VERSION = '0.5'
VERSION = '0.9'

def usage():
print >>sys.stderr, "clamclient.py [[options]] [url] [command] [command-arguments]"
Expand All @@ -50,7 +50,7 @@ def usage():
print >>sys.stderr, " input [project] - Get a list of input files"
print >>sys.stderr, " output [project] - Get a list of output files"
print >>sys.stderr, " xml - Get service specification and project list in CLAM XML"
print >>sys.stderr, " xml [project] - Get entire project state in CLAM XML"
print >>sys.stderr, " xml [project] - Get entire project state in CLAM XML"
#print >>sys.stderr, " download [project] [filename]"
#print >>sys.stderr, "\t Download the specified output file (with metadata)"
print >>sys.stderr, " upload [project] [inputtempate] [file] [[metadata]]"
Expand All @@ -60,11 +60,11 @@ def usage():
print >>sys.stderr, "\t View the metadata parameters in the requested inputtemplate"
#print >>sys.stderr, " metadata [project] [filename]"
#print >>sys.stderr, "\t View the metadata of the specified file"



if __name__ == "__main__":

username = password = None
parameters = {}
begin = 0
Expand All @@ -85,15 +85,15 @@ def usage():
elif o[0] == '-' and len(o) > 1 and o[1] != '-':
usage()
print "ERROR: Unknown option: ", o
sys.exit(2)
sys.exit(2)
elif o[:2] == '--':
if len(rawargs) > i + 1:
parameters[o[2:]] = rawargs[i+1]
begin = i+2
else:
parameters[o[2:]] = True
begin = i+1

if len(rawargs) > begin:
url = rawargs[begin]
else:
Expand All @@ -104,21 +104,21 @@ def usage():
print "ERROR: URL expected"
usage()
sys.exit(2)

client = CLAMClient(url, username,password)

if len(rawargs) > begin + 1:
if len(rawargs) > begin + 1:
command = rawargs[begin+1]
args = rawargs[begin+2:]
else:
command = 'info'
args = []

if command == 'info' and len(args) > 1:
command = 'get'

try:

try:
data = None
if command in ['info','index','projects','inputtemplates','parameters','profiles']:
data = client.index()
Expand All @@ -135,7 +135,7 @@ def usage():
elif command == 'delete' or command == 'abort':
if len(args) != 1:
print >>sys.stderr, "Expected project ID"
sys.exit(2)
sys.exit(2)
client.delete(args[0])
elif command == 'reset':
if len(args) != 1:
Expand All @@ -150,8 +150,9 @@ def usage():
elif command == 'upload':
if len(args) < 3:
print >>sys.stderr, "Expected: project inputtemplate file "
sys.exit(2)
data = client.get(args[0])
sys.exit(2)
project = args[0]
data = client.get(project)
try:
inputtemplate = data.inputtemplate(args[1])
except:
Expand All @@ -161,12 +162,12 @@ def usage():
if not os.path.isfile(filepath):
print >>sys.stderr, "File does not exist: " + filepath
sys.exit(2)
client.upload(project,inputtemplate, filepath, **parameters)
client.upload(project,inputtemplate, filepath, **parameters)
else:
print >>sys.stderr,"Unknown command: " + command
sys.exit(1)


if data:
if command == 'xml':
print data.xml
Expand All @@ -179,7 +180,7 @@ def usage():
print "\tUser: " + username
if command == 'get':
print "\tProject: " + data.project
if command in ['info','projects','index']:
if command in ['info','projects','index']:
print "Projects"
for project in data.projects:
print "\t" + project
Expand All @@ -188,7 +189,7 @@ def usage():
print "\tStatus: " + str(data.status) #TODO: nicer messages
print "\tStatus Message: " + data.statusmessage
print "\tCompletion: " + str(data.completion) + "%"
if command in ['info','profiles']:
if command in ['info','profiles']:
print "Profiles:" #TODO: Implement
for i, profile in enumerate(data.profiles):
print "\tProfile " + str(i+1)
Expand All @@ -205,49 +206,49 @@ def usage():
if command == 'inputtemplates':
print "Input templates:"
for template in data.input:
print "\t\t" + template.id + " - " + template.label
print "\t\t" + template.id + " - " + template.label
if command == 'inputtemplate':
try:
inputtemplate = data.inputtemplate(args[0])
except:
print >>sys.stderr, "No such inputtemplate"
print >>sys.stderr, "No such inputtemplate"
sys.exit(1)
print "Inputtemplate parameters:"
for parameter in inputtemplate.parameters:
print "\t\t" + str(parameter) #VERIFY: unicode support?
print "Inputtemplate converters:"
for c in inputtemplate.converters:
print "\t\t" + c.id + " - " + c.label #VERIFY: unicode support?
if command in ['info','parameters']:
if command in ['info','parameters']:
print "Global Parameters:"
for group, parameters in data.parameters:
print "\t" + group
for parameter in parameters:
print "\t\t" + str(parameter) #VERIFY: unicode support?
if command in ['get','input'] and data.input:
if command in ['get','input'] and data.input:
print "Input files:"
for f in data.input:
print "\t" + f.filename + "\t" + str(f),
if f.metadata and f.metadata.inputtemplate:
print "\t" + f.metadata.inputtemplate
else:
print
if command in ['get','output'] and data.output:
if command in ['get','output'] and data.output:
print "Output files:"
for f in data.output:
print "\t" + f.filename + "\t" + str(f),
print "\t" + f.filename + "\t" + str(f),
if f.metadata and f.metadata.provenance and f.metadata.provenance.outputtemplate_id:
print "\t" + f.metadata.provenance.outputtemplate_id
else:
print
print

except NotFound:
except NotFound:
print >>sys.stderr, "Not Found (404)"
except PermissionDenied:
except PermissionDenied:
print >>sys.stderr, "Permission Denied (403)"
except ServerError:
except ServerError:
print >>sys.stderr, "Server Error! (500)"
except AuthRequired:
print >>sys.stderr, "Authorization required (401)"
except AuthRequired:
print >>sys.stderr, "Authorization required (401)"


27 changes: 13 additions & 14 deletions clamservice.py
View file
Expand Up @@ -71,7 +71,7 @@ def __init__(self, message="forbidden"):
#web.wsgiserver.CherryPyWSGIServer.ssl_private_key = "path/to/ssl_private_key"


VERSION = '0.9.3'
VERSION = '0.9.4'

DEBUG = False

Expand Down Expand Up @@ -1217,19 +1217,18 @@ def addfile(project, filename, user, postdata, inputsource=None):
#Inputtemplate not found, send 404
printlog("Specified inputtemplate (" + postdata['inputtemplate'] + ") not found!")
raise web.webapi.NotFound("Specified inputtemplate (" + postdata['inputtemplate'] + ") not found!")

inputtemplate = ""
#See if an inputtemplate is explicitly specified in the filename
if '/' in filename.strip('/'):
raw = filename.split('/')
inputtemplate = None
for profile in settings.PROFILES:
for it in profile.input:
if it.id == raw[0]:
inputtemplate = it
break
if inputtemplate:
filename = raw[1]
if not inputtemplate:
#See if an inputtemplate is explicitly specified in the filename
if '/' in filename.strip('/'):
raw = filename.split('/')
inputtemplate = None
for profile in settings.PROFILES:
for it in profile.input:
if it.id == raw[0]:
inputtemplate = it
break
if inputtemplate:
filename = raw[1]
if not inputtemplate:
#Check if the specified filename can be uniquely associated with an inputtemplate
for profile in settings.PROFILES:
Expand Down

0 comments on commit e08315a

Please sign in to comment.