diff --git a/deps/pash b/deps/pash index 254205eb..cb63f817 160000 --- a/deps/pash +++ b/deps/pash @@ -1 +1 @@ -Subproject commit 254205eb603f3d4daf76514943bc4e6ffc22d01d +Subproject commit cb63f817d49918886aab67e72c354af62560e695 diff --git a/deps/try b/deps/try index ba6a9061..588fc456 160000 --- a/deps/try +++ b/deps/try @@ -1 +1 @@ -Subproject commit ba6a90615944203a95d5a86638447da34e539d1b +Subproject commit 588fc456d2aa11240cd9a7518bd8e1c219431f0d diff --git a/parallel-orch/analysis.py b/parallel-orch/analysis.py index 3620d28e..a01746d6 100644 --- a/parallel-orch/analysis.py +++ b/parallel-orch/analysis.py @@ -92,7 +92,8 @@ def safe_to_execute(asts: "list[AstNode]", variables: dict) -> bool: BASH_PRIMITIVES = ["break", "continue", - "return"] + "return", + "exit"] safe_cases = { diff --git a/parallel-orch/node.py b/parallel-orch/node.py index f546833a..725b6286 100644 --- a/parallel-orch/node.py +++ b/parallel-orch/node.py @@ -333,14 +333,16 @@ def start_spec_executing(self, env_file): def commit_frontier_execution(self): assert self.state == NodeState.EXECUTING - self.exec_result = ExecResult(self.exec_ctxt.process.pid, self.exec_ctxt.process.returncode) + self.exec_ctxt.process.wait() + self.exec_result = ExecResult(self.exec_ctxt.process.returncode, self.exec_ctxt.process.pid) self.gather_fs_actions() executor.commit_workspace(self.exec_ctxt.sandbox_dir) self.state = NodeState.COMMITTED def finish_spec_execution(self): assert self.state == NodeState.SPEC_EXECUTING - self.exec_result = ExecResult(self.exec_ctxt.process.pid, self.exec_ctxt.process.returncode) + self.exec_ctxt.process.wait() + self.exec_result = ExecResult(self.exec_ctxt.process.returncode, self.exec_ctxt.process.pid) self.gather_fs_actions() self.state = NodeState.SPECULATED @@ -404,7 +406,7 @@ def has_env_conflict_with(self, other_env) -> bool: "CMD_ID", "STDOUT_FILE", "DIRSTACK", "SECONDS", "TMPDIR", "UPDATED_DIRS_AND_MOUNTS", "EPOCHSECONDS", "LATEST_ENV_FILE", "TRY_COMMAND", "SRANDOM", "speculate_flag", "EXECUTION_ID", - "EPOCHREALTIME", "OLDPWD" + "EPOCHREALTIME", "OLDPWD", "exit_code", ]) diff --git a/parallel-orch/run_command.sh b/parallel-orch/run_command.sh index 9fe48f26..8e516b96 100755 --- a/parallel-orch/run_command.sh +++ b/parallel-orch/run_command.sh @@ -34,7 +34,6 @@ fi bash "${PASH_SPEC_TOP}/deps/try/try" -D "${SANDBOX_DIR}" "${PASH_SPEC_TOP}/parallel-orch/template_script_to_execute.sh" > "${STDOUT_FILE}" exit_code=$? - ## Only used for debugging # ls -R "${SANDBOX_DIR}/upperdir" 1>&2 out=`head -3 $SANDBOX_DIR/upperdir/$TRACE_FILE` @@ -42,5 +41,6 @@ out=`head -3 $SANDBOX_DIR/upperdir/$TRACE_FILE` ## Assumes "${PASH_SPEC_SCHEDULER_SOCKET}" is set and exported ## Pass the proper exit code -msg="CommandExecComplete:${CMD_ID}|Exec id:${EXECUTION_ID}|Exit code:${exit_code}|Sandbox dir:${SANDBOX_DIR}|Trace file:${TRACE_FILE}|Tempdir:${TEMPDIR}" +msg="CommandExecComplete:${CMD_ID}|Exec id:${EXECUTION_ID}|Sandbox dir:${SANDBOX_DIR}|Trace file:${TRACE_FILE}|Tempdir:${TEMPDIR}" daemon_response=$(pash_spec_communicate_scheduler_just_send "$msg") # Blocking step, daemon will not send response until it's safe to continue +(exit $exit_code) diff --git a/parallel-orch/scheduler_server.py b/parallel-orch/scheduler_server.py index 4a8ef5a5..76ea120b 100644 --- a/parallel-orch/scheduler_server.py +++ b/parallel-orch/scheduler_server.py @@ -106,7 +106,8 @@ def process_next_cmd(self): logging.info(f'Scheduler: Received daemon start message.') connection.close() elif (input_cmd.startswith("CommandExecComplete:")): - node_id, exec_id, exit_code, sandbox_dir, trace_file = self.__parse_command_exec_x(input_cmd) + node_id, exec_id, sandbox_dir, trace_file = self.__parse_command_exec_x(input_cmd) + connection.close() if self.partial_program_order.get_concrete_node(node_id).exec_id == exec_id: logging.info(f'Scheduler: Received command exec complete message - {node_id}.') self.partial_program_order.handle_complete(node_id, node_id in self.waiting_for_response, self.latest_env) @@ -141,6 +142,7 @@ def respond_to_pending_wait(self, node_id: ConcreteNodeId): ## Get the completed node info node = self.partial_program_order.get_concrete_node(node_id) msg = '{} {} {}'.format(*node.execution_outcome()) + util.debug_log(f'outcome for node {node_id} is {node.execution_outcome()}') response = success_response(msg) ## Send the response @@ -165,10 +167,9 @@ def __parse_command_exec_x(self, input_cmd: str) -> "tuple[int, int]": components = input_cmd.rstrip().split("|") command_id = ConcreteNodeId.parse(components[0].split(":")[1]) exec_id = int(components[1].split(":")[1]) - exit_code = int(components[2].split(":")[1]) - sandbox_dir = components[3].split(":")[1] - trace_file = components[4].split(":")[1] - return command_id, exec_id, exit_code, sandbox_dir, trace_file + sandbox_dir = components[2].split(":")[1] + trace_file = components[3].split(":")[1] + return command_id, exec_id, sandbox_dir, trace_file except: raise Exception(f'Parsing failure for line: {input_cmd}') diff --git a/parallel-orch/template_script_to_execute.sh b/parallel-orch/template_script_to_execute.sh index cd0496cc..11f4e2a0 100755 --- a/parallel-orch/template_script_to_execute.sh +++ b/parallel-orch/template_script_to_execute.sh @@ -1,6 +1,6 @@ #!/bin/bash -strace -y -f --seccomp-bpf --trace=fork,clone,%file -o $TRACE_FILE bash -c "source $LATEST_ENV_FILE; $CMD_STRING; source $RUNTIME_DIR/pash_declare_vars.sh $POST_EXEC_ENV" -exit_code=$? -(exit $exit_code) +# Magic, don't touch without consulting Di +RUN=$(printf 'source %s; %s; exit_code=$?; source $RUNTIME_DIR/pash_declare_vars.sh %s; exit $exit_code' "${LATEST_ENV_FILE}" "${CMD_STRING}" "${POST_EXEC_ENV}") +strace -y -f --seccomp-bpf --trace=fork,clone,%file -o $TRACE_FILE env -i bash -c "$RUN" diff --git a/parallel-orch/trace_v2.py b/parallel-orch/trace_v2.py index 68719df2..87fd7eab 100644 --- a/parallel-orch/trace_v2.py +++ b/parallel-orch/trace_v2.py @@ -2,6 +2,7 @@ import logging import os.path import sys +import util from typing import Tuple from dataclasses import dataclass @@ -44,12 +45,42 @@ class RFile: def __init__(self, fname): self.fname = os.path.normpath(fname) + def closure(self): + all_files = [self] + if not self.fname.startswith('/'): + return all_files + current_name = self.fname + i = 0 + while current_name != '/': + dir, _ = os.path.split(current_name) + all_files.append(RFile(dir)) + current_name = dir + i += 1 + if i > 15: + util.debug_log(f"{self.fname}") + return all_files + @dataclass class WFile: fname: str def __init__(self, fname): self.fname = os.path.normpath(fname) + def closure(self): + all_files = [self] + current_name = self.fname + if not self.fname.startswith('/'): + return all_files + i = 0 + while current_name != '/': + dir, _ = os.path.split(current_name) + all_files.append(RFile(dir)) + current_name = dir + i += 1 + if i > 15: + util.debug_log(f"{current_name}") + return all_files + class Context: def __init__(self): self.line_dict = {} @@ -235,6 +266,7 @@ def parse_clone(pid, args, ret, ctx): flags = flags[len('flags='):] if has_clone_fs(flags): ctx.do_clone(pid, child) + return [] def parse_symlinkat(pid, args, ret): a0, rest = args.split(sep=',', maxsplit=1) @@ -328,9 +360,12 @@ def parse_and_gather_cmd_rw_sets(trace_object) -> Tuple[set, set]: except Exception: logging.debug(l) raise ValueError("error while parsing trace") + if records is None or isinstance(records, ExitStatus): + continue if not isinstance(records, list): records = [records] - for record in records: + all_records = [r for record in records for r in record.closure()] + for record in all_records: if type(record) is RFile and record.fname != '/dev/tty': read_set.add(record.fname) elif type(record) is WFile and record.fname != '/dev/tty': diff --git a/test/misc/source_foo_equal.sh b/test/misc/source_foo_equal.sh new file mode 100644 index 00000000..a84a6b0d --- /dev/null +++ b/test/misc/source_foo_equal.sh @@ -0,0 +1 @@ +foo=$1 diff --git a/test/test_orch.sh b/test/test_orch.sh index 572b4057..2f86345f 100755 --- a/test/test_orch.sh +++ b/test/test_orch.sh @@ -309,6 +309,12 @@ test_stdout() $shell $2/test_stdout.sh } +test_if() +{ + local shell=$1 + $shell $2/test_if.sh +} + test_loop() { local shell=$1 @@ -360,6 +366,12 @@ test_local_vars_3() $shell $2/test_local_vars_3.sh } +test_local_vars_4() +{ + local shell=$1 + $shell $2/test_local_vars_4.sh +} + test_command_var_assignments_1(){ local shell=$1 $shell $2/test_command_var_assignments_1.sh @@ -417,6 +429,7 @@ if [ "$#" -eq 0 ]; then run_test test9_2 # "1 1 1 1 1 1 1 1 1 1 1 1 1" # 13 run_test test9_3 # "1 1 1 1 1 1 1 1 2 2 1 1 1" # 15 run_test test_stdout #"1 1 1 1 1 1" # 6 + run_test test_if run_test test_loop run_test test_break run_test test_network_access_1 #"1 2 2" diff --git a/test/test_scripts/test_if.sh b/test/test_scripts/test_if.sh new file mode 100644 index 00000000..ba0d7d88 --- /dev/null +++ b/test/test_scripts/test_if.sh @@ -0,0 +1,15 @@ +a=hello +if [ -z $a ]; then + echo haha +else + echo hoho +fi + +b= +if [ -z $b ]; then + echo haha +else + echo hoho +fi + + diff --git a/test/test_scripts/test_local_vars_4.sh b/test/test_scripts/test_local_vars_4.sh new file mode 100644 index 00000000..ae2f2a87 --- /dev/null +++ b/test/test_scripts/test_local_vars_4.sh @@ -0,0 +1,2 @@ +source "$MISC_SCRIPT_DIR/source_foo_equal.sh" bar +echo $foo