Permalink
Browse files

updated sweep_make mytardis

  • Loading branch information...
ianedwardthomas committed Dec 12, 2013
1 parent 2471eb8 commit 83c3fddda3362d960c0ed4e2ff169cf7f835d5c0
@@ -249,7 +249,7 @@ def start_setup(instance, ip, settings, source, destination):
ssh.close()
logger.debug("command_out1=(%s, %s)" % (command_out, errs))
command = "cd %s; make %s" % (makefile_path, 'setupstart')
command = "cd %s; make -f Makefile %s" % (makefile_path, 'setupstart')
command_out = ''
errs = ''
logger.debug("starting command for %s" % ip)
@@ -270,7 +270,7 @@ def job_finished(ip, settings, destination):
"""
ssh = sshconnector.open_connection(ip_address=ip, settings=settings)
makefile_path = hrmcstages.get_make_path(destination)
command = "cd %s; make %s" % (makefile_path, 'setupdone')
command = "cd %s; make -f Makefile %s" % (makefile_path, 'setupdone')
command_out, _ = sshconnector.run_command_with_status(ssh, command)
if command_out:
logger.debug("command_out = %s" % command_out)
@@ -242,7 +242,7 @@ def run_task(self, ip_address, process_id, settings):
ip_address=ip)
makefile_path = hrmcstages.get_make_path(destination)
#makefile_path = settings['payload_destination']
command = "cd %s; make %s" % (makefile_path, 'startrun IDS=%s' % (
command = "cd %s; make -f Makefile %s" % (makefile_path, 'startrun IDS=%s' % (
settings['filename_for_PIDs']))
logger.debug('command_exec=%s' % command)
command_out = ''
@@ -89,6 +89,11 @@ def setup_settings(run_settings):
settings['bdp_username'] = run_settings[
'http://rmit.edu.au/schemas/bdp_userprofile']['username']
settings['directive'] = run_settings[
'http://rmit.edu.au/schemas/stages/sweep']['directive']
settings['mytardis_platform'] = run_settings['http://rmit.edu.au/schemas/input/mytardis']['mytardis_platform']
@@ -80,7 +80,7 @@ def _job_finished(self, settings, remote_path):
ip_address=settings['host'])
(scheme, host, mypath, location, query_settings) = \
hrmcstages.parse_bdpurl(encoded_d_url)
command = "cd %s; make %s" % (os.path.join(
command = "cd %s; make -f Makefile %s" % (os.path.join(
query_settings['root_path'], mypath),
'running')
command_out = ''
@@ -121,6 +121,7 @@ def process(self, run_settings):
self.experiment_id = 0
settings = setup_settings(run_settings)
self.experiment_id = settings['experiment_id']
smartconnector.info(run_settings, "1: waiting for completion")
logger.debug("settings=%s" % settings)
if self._exists(run_settings,
'http://rmit.edu.au/schemas/stages/make',
@@ -222,124 +223,147 @@ def _get_output(self, settings, source_url):
# been transferred.
raise
directive = settings['directive']
def _get_mytardis_settings(settings, bdp_username):
mytardis_url = settings['mytardis_platform']
return platform.get_platform_settings(mytardis_url, bdp_username)
mytardis_settings = _get_mytardis_settings(settings, bdp_username)
logger.debug(mytardis_settings)
# TODO: this is very domain specific
if mytardis_settings['mytardis_host']:
OUTCAR_FILE = "OUTCAR"
VALUES_FILE = "values"
outcar_url = smartconnector.get_url_with_pkey(settings,
os.path.join(dest_url, OUTCAR_FILE), is_relative_path=False)
logger.debug("outcar_url=%s" % outcar_url)
try:
outcar_content = hrmcstages.get_file(outcar_url)
except IOError, e:
logger.error(e)
toten = None
else:
toten = None
for line in outcar_content.split('\n'):
#logger.debug("line=%s" % line)
if 'e en' in line:
logger.debug("found")
try:
toten = float(line.rsplit(' ', 2)[-2])
except ValueError, e:
logger.error(e)
pass
break
logger.debug("toten=%s" % toten)
values_url = smartconnector.get_url_with_pkey(settings,
os.path.join(dest_url, VALUES_FILE), is_relative_path=False)
logger.debug("values_url=%s" % values_url)
try:
values_content = hrmcstages.get_file(values_url)
except IOError, e:
logger.error(e)
values = None
else:
values = None
try:
values = dict(json.loads(values_content))
except Exception, e:
logger.error(e)
pass
if directive == "vasp":
logger.debug("values=%s" % values)
# TODO: this is very domain specific
# FIXME: all values from map are strings initially, so need to know
# type to coerce.
num_kp = None
if 'num_kp' in values:
try:
num_kp = int(values['num_kp'])
except IndexError:
pass
except ValueError:
pass
OUTCAR_FILE = "OUTCAR"
VALUES_FILE = "values"
logger.debug("num_kp=%s" % num_kp)
outcar_url = smartconnector.get_url_with_pkey(settings,
os.path.join(dest_url, OUTCAR_FILE), is_relative_path=False)
logger.debug("outcar_url=%s" % outcar_url)
encut = None
if 'encut' in values:
try:
encut = int(values['encut'])
except IndexError:
pass
except ValueError:
pass
logger.debug("encut=%s" % encut)
def _get_exp_name_for_make(settings, url, path):
"""
Break path based on EXP_DATASET_NAME_SPLIT
"""
return str(os.sep.join(path.split(os.sep)[-2:-1]))
def _get_dataset_name_for_make(settings, url, path):
"""
Break path based on EXP_DATASET_NAME_SPLIT
"""
encut = settings['ENCUT']
numkp = settings['NUMKP']
runcounter = settings['RUNCOUNTER']
return "%s:encut=%s,num_kp=%s" % (runcounter, encut, numkp)
#return str(os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
mytardis_settings['ENCUT'] = encut
mytardis_settings['NUMKP'] = num_kp
mytardis_settings['RUNCOUNTER'] = settings['contextid']
self.experiment_id = mytardis.post_dataset(
settings=mytardis_settings,
source_url=encoded_d_url,
exp_id=self.experiment_id,
exp_name=_get_exp_name_for_make,
dataset_name=_get_dataset_name_for_make,
experiment_paramset=[],
dataset_paramset=[
make_paramset("remotemake/output", []),
make_graph_paramset("dsetgraph",
name="makedset",
graph_info={},
value_dict={"makedset/num_kp": num_kp, "makedset/encut": encut, "makedset/toten": toten}
if (num_kp is not None)
and (encut is not None)
and (toten is not None) else {},
value_keys=[]
),
]
)
outcar_content = hrmcstages.get_file(outcar_url)
except IOError, e:
logger.error(e)
toten = None
else:
toten = None
for line in outcar_content.split('\n'):
#logger.debug("line=%s" % line)
if 'e en' in line:
logger.debug("found")
try:
toten = float(line.rsplit(' ', 2)[-2])
except ValueError, e:
logger.error(e)
pass
break
logger.debug("toten=%s" % toten)
values_url = smartconnector.get_url_with_pkey(settings,
os.path.join(dest_url, VALUES_FILE), is_relative_path=False)
logger.debug("values_url=%s" % values_url)
try:
values_content = hrmcstages.get_file(values_url)
except IOError, e:
logger.error(e)
values = None
else:
values = None
try:
values = dict(json.loads(values_content))
except Exception, e:
logger.error(e)
pass
logger.debug("values=%s" % values)
# FIXME: all values from map are strings initially, so need to know
# type to coerce.
num_kp = None
if 'num_kp' in values:
try:
num_kp = int(values['num_kp'])
except IndexError:
pass
except ValueError:
pass
logger.debug("num_kp=%s" % num_kp)
encut = None
if 'encut' in values:
try:
encut = int(values['encut'])
except IndexError:
pass
except ValueError:
pass
logger.debug("encut=%s" % encut)
def _get_exp_name_for_vasp(settings, url, path):
"""
Break path based on EXP_DATASET_NAME_SPLIT
"""
return str(os.sep.join(path.split(os.sep)[-2:-1]))
def _get_dataset_name_for_vasp(settings, url, path):
"""
Break path based on EXP_DATASET_NAME_SPLIT
"""
encut = settings['ENCUT']
numkp = settings['NUMKP']
runcounter = settings['RUNCOUNTER']
return "%s:encut=%s,num_kp=%s" % (runcounter, encut, numkp)
#return str(os.sep.join(path.split(os.sep)[-EXP_DATASET_NAME_SPLIT:]))
mytardis_settings['ENCUT'] = encut
mytardis_settings['NUMKP'] = num_kp
mytardis_settings['RUNCOUNTER'] = settings['contextid']
self.experiment_id = mytardis.post_dataset(
settings=mytardis_settings,
source_url=encoded_d_url,
exp_id=self.experiment_id,
exp_name=_get_exp_name_for_vasp,
dataset_name=_get_dataset_name_for_vasp,
experiment_paramset=[],
dataset_paramset=[
make_paramset("remotemake/output", []),
make_graph_paramset("dsetgraph",
name="makedset",
graph_info={},
value_dict={"makedset/num_kp": num_kp, "makedset/encut": encut, "makedset/toten": toten}
if (num_kp is not None)
and (encut is not None)
and (toten is not None) else {},
value_keys=[]
),
]
)
elif directive == "remotemake":
def _get_exp_name_for_make(settings, url, path):
return str(os.sep.join(path.split(os.sep)[-2:-1]))
def _get_dataset_name_for_make(settings, url, path):
return str(os.sep.join(path.split(os.sep)[-1:]))
self.experiment_id = mytardis.post_dataset(
settings=mytardis_settings,
source_url=encoded_d_url,
exp_id=self.experiment_id,
exp_name=_get_exp_name_for_make,
dataset_name=_get_dataset_name_for_make,
experiment_paramset=[],
dataset_paramset=[
make_paramset("remotemake/output", [])]
)
def output(self, run_settings):
run_settings['http://rmit.edu.au/schemas/stages/make']['runs_left'] \
@@ -92,10 +92,10 @@ def process(self, run_settings):
values_map = _load_values_map(settings, map_initial_location)
logger.debug("values_map=%s" % values_map)
_upload_payload(settings, settings['payload_source'], values_map)
_upload_variations_inputs(
settings,
map_initial_location, values_map)
_upload_payload(settings, settings['payload_source'], values_map)
smartconnector.info(run_settings, "1: upload done")
@@ -101,7 +101,7 @@ def _get_dest_bdp_url(settings):
(scheme, host, mypath, location, query_settings) = \
hrmcstages.parse_bdpurl(encoded_d_url)
command = "cd %s; make %s" % (os.path.join(
command = "cd %s; make -f Makefile %s" % (os.path.join(
query_settings['root_path'],
mypath),
'startrun')
@@ -180,9 +180,20 @@ def process(self, run_settings):
run_settings[RMIT_SCHEMA + '/input/mytardis'][
'experiment_id'] = self.experiment_id
elif subdirective == "remotemake":
self.experiment_id = self._make_mytardis_exp(
run_settings=run_settings,
experiment_id=self.experiment_id,
experiment_paramset=[
make_paramset("remotemake", [])],
output_location=self.scratch_platform)
elif subdirective == "hrmc":
pass
if '%s/input/mytardis' % RMIT_SCHEMA in run_settings:
run_settings[RMIT_SCHEMA + '/input/mytardis'][
'experiment_id'] = str(self.experiment_id)
# # generate all variations
map_text = run_settings[RMIT_SCHEMA + '/input/sweep']['sweep_map']
sweep_map = json.loads(map_text)
@@ -119,7 +119,7 @@ def job_finished(self, ip_address, process_id, retry_left, settings):
ip_address=ip)
makefile_path = hrmcstages.get_make_path(destination)
#makefile_path = settings['payload_destination']
command = "cd %s; make %s" % (makefile_path, 'running') # IDS=%s' % (
command = "cd %s; make -f Makefile %s" % (makefile_path, 'running') # IDS=%s' % (
#settings['filename_for_PIDs']))
command_out = ''
@@ -262,6 +262,8 @@ def process(self, run_settings):
computation_platform_url = run_settings['http://rmit.edu.au/schemas/platform/computation']['platform_url']
comp_pltf_settings = platform.get_platform_settings(computation_platform_url, local_settings['bdp_username'])
local_settings.update(comp_pltf_settings)
comp_pltf_settings['bdp_username'] = local_settings['bdp_username']
for process in processes:
#instance_id = node.id
View
@@ -1,5 +1,5 @@
startrun:
echo hello from startrun
sh ./startrun.sh
running:
echo hello from running
sh ./running.sh

0 comments on commit 83c3fdd

Please sign in to comment.