Skip to content

Commit

Permalink
Trying to get BashApps working again
Browse files Browse the repository at this point in the history
  • Loading branch information
@awicenec
awicenec committed May 26, 2022
1 parent 8635f9e commit 9cb4386
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 66 deletions.
11 changes: 6 additions & 5 deletions daliuge-engine/dlg/apps/bash_shell_app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,8 +169,8 @@ def initialize(self, **kwargs):
self.proc = None
self._inputRedirect = self._getArg(kwargs, "input_redirection", "")
self._outputRedirect = self._getArg(kwargs, "output_redirection", "")
self._cmdLineArgs = self._getArg(kwargs, "command_line_arguments", "")
self._applicationArgs = self._getArg(kwargs, "applicationArgs", {})
self._cmdLineArgs = self._getArg(kwargs, "command_line_arguments", "")
self._argumentPrefix = self._getArg(kwargs, "argumentPrefix", "--")
self._paramValueSeparator = self._getArg(kwargs, "paramValueSeparator", " ")

Expand Down Expand Up @@ -342,17 +342,17 @@ def execute(self, data):
# @par EAGLE_START
# @param category BashShellApp
# @param tag template
# @param[in] cparam/command Command//String/readwrite/False//False/
# @param[in] aparam/command Command//String/readwrite/False//False/
# \~English The command to be executed
# @param[in] cparam/input_redirection Input Redirection//String/readwrite/False//False/
# \~English The command line argument that specifies the input into this application
# @param[in] cparam/output_redirection Output Redirection//String/readwrite/False//False/
# \~English The command line argument that specifies the output from this application
# @param[in] cparam/command_line_arguments Command Line Arguments//String/readwrite/False//False/
# @param[in] aparam/command_line_arguments Command Line Arguments//String/readwrite/False//False/
# \~English Additional command line arguments to be added to the command line to be executed
# @param[in] cparam/paramValueSeparator Param value separator/ /String/readwrite/False//False/
# @param[in] aparam/paramValueSeparator Param value separator/ /String/readwrite/False//False/
# \~English Separator character(s) between parameters on the command line
# @param[in] cparam/argumentPrefix Argument prefix/"--"/String/readwrite/False//False/
# @param[in] aparam/argumentPrefix Argument prefix/"--"/String/readwrite/False//False/
# \~English Prefix to each keyed argument on the command line
# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/
# \~English Estimated execution time
Expand Down Expand Up @@ -381,6 +381,7 @@ class BashShellApp(BashShellBase, BarrierAppDROP):
)

def run(self):
logger.debug("Parameters found: %s",self.parameters)
self._run_bash(self._inputs, self._outputs)


Expand Down
131 changes: 70 additions & 61 deletions daliuge-translator/dlg/dropmake/lg.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -634,12 +634,43 @@ def _create_test_drop_spec(self, oid, rank, kwargs) -> dropdict:
Categories.COMPONENT,
Categories.PYTHON_APP,
Categories.BRANCH,
Categories.DOCKER,
]:
# default generic component becomes "sleep and copy"
if "appclass" not in self.jd or len(self.jd["appclass"]) == 0:
app_class = "dlg.apps.simple.SleepApp"
if drop_type not in [Categories.DOCKER]:
if "appclass" not in self.jd or len(self.jd["appclass"]) == 0:
app_class = "dlg.apps.simple.SleepApp"
else:
app_class = self.jd["appclass"]
else:
app_class = self.jd["appclass"]
# deal with the Docker specific component params
app_class = "dlg.apps.dockerapp.DockerApp"
typ = DropType.APP
image = str(self.jd.get("image"))
if image == "":
raise GraphException("Missing image for Docker component '%s'" % self.text)

command = str(self.jd.get("command"))
# There ARE containers which don't need/want a command
# if command == "":
# raise GraphException("Missing command for Construct '%s'" % self.text)

kwargs["image"] = image
kwargs["command"] = command
# TODO: User inside docker should follow user of engine.
kwargs["user"] = str(self.jd.get("user", ""))
kwargs["ensureUserAndSwitch"] = self.str_to_bool(
str(self.jd.get("ensureUserAndSwitch", "0"))
)
kwargs["removeContainer"] = self.str_to_bool(
str(self.jd.get("removeContainer", "1"))
)
kwargs["additionalBindings"] = str(self.jd.get("additionalBindings", ""))
if kwargs["additionalBindings"]:
kwargs["additionalBindings"] += ","
# always mount DLG_ROOT directory. ENV variable is only known in engine
kwargs["additionalBindings"] += "${DLG_ROOT}:${DLG_ROOT}"
kwargs["portMappings"] = str(self.jd.get("portMappings", ""))
kwargs["shmSize"] = str(self.jd.get("shmSize",""))

if "execution_time" in self.jd:
execTime = int(self.jd["execution_time"])
Expand Down Expand Up @@ -706,66 +737,44 @@ def _create_test_drop_spec(self, oid, rank, kwargs) -> dropdict:
)
# add more arguments
cmds = []
for i in range(10):
k = "Arg%02d" % (i + 1,)
if k not in self.jd:
k = "arg%02d" % (i + 1,)
if k not in self.jd:
continue
v = self.jd[k]
if v is not None and len(str(v)) > 0:
cmds.append(str(v))
# add more arguments - this is the new method of adding arguments in EAGLE
# the method above (Arg**) is retained for compatibility, but eventually should be removed
for k in [
"command",
"input_redirection",
"output_redirection",
"command_line_arguments",
]:
if k in self.jd:
cmds.append(self.jd[k])
# kwargs['command'] = ' '.join(cmds)
if drop_type == Categories.MPI:
kwargs["command"] = BashCommand(cmds).to_real_command()
else:
kwargs["command"] = BashCommand(
cmds
) # TODO: Check if this actually solves a problem.
if "command" in self.jd:
cmds = [self.jd["command"]]
self._update_key_value_attributes(kwargs) # get all the other params
kwargs["command"] = BashCommand(cmds) # NOTE: Not really required anymore?
kwargs["num_cpus"] = int(self.jd.get("num_cpus", 1))
drop_spec.update(kwargs)

elif drop_type == Categories.DOCKER:
# Docker application.
app_class = "dlg.apps.dockerapp.DockerApp"
typ = DropType.APP
drop_spec = dropdict(
{"oid": oid, "type": typ, "app": app_class, "rank": rank}
)

image = str(self.jd.get("image"))
if image == "":
raise GraphException("Missing image for Construct '%s'" % self.text)

command = str(self.jd.get("command"))
# There ARE containers which don't need/want a command
# if command == "":
# raise GraphException("Missing command for Construct '%s'" % self.text)

kwargs["tw"] = int(self.jd.get("execution_time", "5"))
kwargs["image"] = image
kwargs["command"] = command
kwargs["user"] = str(self.jd.get("user", ""))
kwargs["ensureUserAndSwitch"] = self.str_to_bool(
str(self.jd.get("ensureUserAndSwitch", "0"))
)
kwargs["removeContainer"] = self.str_to_bool(
str(self.jd.get("removeContainer", "1"))
)
kwargs["additionalBindings"] = str(self.jd.get("additionalBindings", ""))
kwargs["portMappings"] = str(self.jd.get("portMappings", ""))
kwargs["shmSize"] = str(self.jd.get("shmSize", ""))
drop_spec.update(kwargs)
# elif drop_type == Categories.DOCKER:
# # Docker application.
# app_class = "dlg.apps.dockerapp.DockerApp"
# typ = DropType.APP
# drop_spec = dropdict(
# {"oid": oid, "type": typ, "app": app_class, "rank": rank}
# )

# image = str(self.jd.get("image"))
# if image == "":
# raise GraphException("Missing image for Construct '%s'" % self.text)

# command = str(self.jd.get("command"))
# # There ARE containers which don't need/want a command
# # if command == "":
# # raise GraphException("Missing command for Construct '%s'" % self.text)

# kwargs["tw"] = int(self.jd.get("execution_time", "5"))
# kwargs["image"] = image
# kwargs["command"] = command
# kwargs["user"] = str(self.jd.get("user", ""))
# kwargs["ensureUserAndSwitch"] = self.str_to_bool(
# str(self.jd.get("ensureUserAndSwitch", "0"))
# )
# kwargs["removeContainer"] = self.str_to_bool(
# str(self.jd.get("removeContainer", "1"))
# )
# kwargs["additionalBindings"] = str(self.jd.get("additionalBindings", ""))
# kwargs["portMappings"] = str(self.jd.get("portMappings", ""))
# kwargs["shmSize"] = str(self.jd.get("shmSize", ""))
# drop_spec.update(kwargs)

elif drop_type == Categories.GROUP_BY:
drop_spec = dropdict(
Expand Down

0 comments on commit 9cb4386

Please sign in to comment.