Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

singularity support! #41

Merged
merged 1 commit into from Jun 15, 2018
Merged

singularity support! #41

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
38 changes: 34 additions & 4 deletions packtivity/handlers/execution_handlers.py
View file
Open in desktop
Expand Up @@ -165,6 +165,19 @@ def docker_execution_cmdline(state,log,metadata, race_spec):
command = quoted_string
)

def singularity_execution_cmdline(state,log,metadata, race_spec):
#for running in subprocess
quoted_string = ' '.join(map(pipes.quote,race_spec['argv']))


from os.path import expanduser
home = expanduser("~")
return 'singularity exec -C -B {home}:{home} docker://{image} {command}'.format(
home = home,
image = race_spec['image'],
command = quoted_string
)

def script_argv(environment,job,log):
script = job['script']
interpreter = job['interpreter']
Expand Down Expand Up @@ -200,7 +213,9 @@ def execute_and_tail_subprocess(metadata,state,log,command_string,stdin_content
proc = None
if stdin_content:
log.debug('stdin: \n%s',stdin_content)
proc = subprocess.Popen(shlex.split(command_string),
argv = shlex.split(command_string)
log.debug('argv: %s', argv)
proc = subprocess.Popen(argv,
stdin = subprocess.PIPE,
stderr = subprocess.STDOUT,
stdout = subprocess.PIPE,
Expand All @@ -213,7 +228,11 @@ def execute_and_tail_subprocess(metadata,state,log,command_string,stdin_content

log.debug('started subprocess with pid %s. now wait to finish',proc.pid)
time.sleep(0.5)
log.debug('process children: %s',[x for x in psutil.Process(proc.pid).children(recursive = True)])

try: #some issues on some linux machines.. swallow exception
log.debug('process children: %s',[x for x in psutil.Process(proc.pid).children(recursive = True)])
except:
pass

for line in iter(proc.stdout.readline, b''):
subproclog.info(line.strip())
Expand Down Expand Up @@ -262,14 +281,25 @@ def run_containers_in_docker_runtime(state,log,metadata,race_spec):
cmdline = docker_execution_cmdline(state,log,metadata,race_spec)
execute_and_tail_subprocess(metadata,state,log,cmdline, stdin_content = race_spec['stdin'], logging_topic = 'run')

def run_containers_in_singularity_runtime(state,log,metadata,race_spec):
cmdline = singularity_execution_cmdline(state,log,metadata,race_spec)
execute_and_tail_subprocess(metadata,state,log,cmdline, stdin_content = race_spec['stdin'], logging_topic = 'run')


@executor('docker-encapsulated')
def docker_enc_handler(environment,state,job,metadata):
with logutils.setup_logging_topic(metadata,state,'step',return_logger = True) as log:
rspec = race_spec(state,environment,log,job)

log.info('rspec is\n{}'.format(json.dumps(rspec, indent = 4)))
run_containers_in_docker_runtime(state,log,metadata,rspec)
log.debug('rspec is\n{}'.format(json.dumps(rspec, indent = 4)))

runtimes = {
'docker' : run_containers_in_docker_runtime,
'singularity': run_containers_in_singularity_runtime
}

run = runtimes[os.environ.get('PACKTIVITY_CONTAINER_RUNTIME','docker')]
run(state,log,metadata,rspec)

@executor('noop-env')
def noop_env(environment,state,job,metadata):
Expand Down