diff --git a/cli/popper/parser.py b/cli/popper/parser.py index ac7211d0a..17e6e3a7b 100644 --- a/cli/popper/parser.py +++ b/cli/popper/parser.py @@ -1,793 +1,176 @@ from __future__ import unicode_literals -import re -import hcl +import logging import os -import threading +import re import yaml -from copy import deepcopy -from builtins import str, dict from popper.cli import log as log -import popper.utils as pu - - -VALID_STEP_ATTRS = [ - "uses", - "args", - "needs", - "runs", - "skip_pull", - "secrets", - "env", - "name", - "next", -] - - -class threadsafe_iter_3: - """Takes an iterator/generator and makes it thread-safe by serializing call - to the `next` method of given iterator/generator.""" - - def __init__(self, it): - self.it = it - self.lock = threading.Lock() - - def __iter__(self): - return self - - def __next__(self): - with self.lock: - return self.it.__next__() - - -def threadsafe_generator(f): - """A decorator that takes a generator function and makes it thread-safe. - - Args: - f(function): Generator function - - Returns: - None - """ - - def g(*args, **kwargs): - """ - - Args: - *args(list): List of non-key worded,variable length arguments. - **kwargs(dict): List of key-worded,variable length arguments. - - Returns: - function: The thread-safe function. - """ - return threadsafe_iter_3(f(*args, **kwargs)) - - return g - - -class Workflow(object): - """Represents an immutable workflow.""" - - def __init__(self): - pass - - def parse(self, substitutions=None, allow_loose=False): - """Parse and validate a workflow. - - Args: - substitutions(list): Substitutions that are to be passed - as an arguments. (Default value = None) - allow_loose(bool): Flag if the unused variables are to be - ignored. (Default value = False) - - Returns: - None. - - """ - self.validate_workflow_block() - self.validate_step_blocks() - self.normalize() - self.parse_substitutions(substitutions, allow_loose) - self.check_for_broken_workflow() - self.complete_graph() - - def complete_graph(self): - raise NotImplementedError( - "This method is required to be implemented in the derived class." - ) - - def normalize(self): - raise NotImplementedError( - "This method is required to be implemented in the derived class." - ) +from box import Box + +from pykwalify.core import Core as YMLValidator +from pykwalify.errors import SchemaError + + +class WorkflowParser(object): + _wf_schema = { + "type": "map", + "mapping": { + "steps": { + "type": "seq", + "sequence": [ + { + "type": "map", + "mapping": { + "uses": {"type": "str", "required": True}, + "id": {"type": "str"}, + "args": {"type": "seq", "sequence": [{"type": "str"}]}, + "runs": {"type": "seq", "sequence": [{"type": "str"}]}, + "secrets": {"type": "seq", "sequence": [{"type": "str"}]}, + "skip_pull": {"type": "bool"}, + "env": { + "type": "map", + "matching-rule": "any", + "mapping": {"regex;(.+)": {"type": "str"}}, + }, + }, + } + ], + }, + }, + } @staticmethod - def new( - file, + def parse( + file=None, + wf_data=None, step=None, skipped_steps=[], substitutions=[], allow_loose=False, - include_step_dependencies=False, ): + """Returns an immutable workflow structure (a frozen Box) with 'steps'. See + WorkflowParser._wf_schema above for their structure. + """ - if not os.path.exists(file): - log.fail(f"File {file} was not found.") + if not file and not wf_data: + log.fail("Expecting 'file' or 'wf_data'") - if file.endswith(".workflow"): - wf = HCLWorkflow(file) - elif file.endswith(".yml") or file.endswith(".yaml"): - wf = YMLWorkflow(file) - else: - log.fail("Unrecognized workflow file format.") + if file: + if wf_data: + log.fail("Expecting only one of 'file' and 'wf_data'") - wf.parse(substitutions=substitutions, allow_loose=allow_loose) + if not os.path.exists(file): + log.fail(f"File {file} was not found.") - log.debug( - f"Parsed workflow:\n" - f'{yaml.dump(wf, default_flow_style=False, default_style="")}' - ) + if not file.endswith(".yml") and not file.endswith(".yaml"): + log.fail("Unrecognized workflow file format.") - wf = Workflow.skip_steps(wf, skipped_steps) - wf = Workflow.filter_step(wf, step, include_step_dependencies) + with open(file, "r") as f: + _wf_data = yaml.safe_load(f) + else: + _wf_data = wf_data - wf.check_for_unreachable_steps(skipped_steps) + # hack to silence warnings about error to fail change + logging.disable(logging.CRITICAL) - return wf + v = YMLValidator(source_data=_wf_data, schema_data=WorkflowParser._wf_schema) - @staticmethod - def format_command(params): - """A static method that formats the `runs` and `args` attributes into a - list of strings. + try: + v.validate() + except SchemaError as e: + log.fail(f"{e.msg}") - Args: - params(list/str): run or args that are being executed. + logging.disable(logging.NOTSET) - Returns: - list: List of strings of parameters. - """ - if pu.of_type(params, ["str"]): - return params.split(" ") - return params - - @threadsafe_generator - def get_stages(self): - """Generator of stages. A stages is a list of steps that can be - executed in parallel. - """ + WorkflowParser.__add_missing_ids(_wf_data) + WorkflowParser.__apply_substitutions( + _wf_data, substitutions=substitutions, allow_loose=allow_loose + ) + WorkflowParser.__skip_steps(_wf_data, skipped_steps) + WorkflowParser.__filter_step(_wf_data, step) - def resolve_intersections(stage): - """Removes steps from a stage that creates conflict between the - selected stage candidates. - - Args: - stage(set): Stage from which conflicted steps are to be - removed. - - Returns: - None - """ - steps_to_remove = set() - for a in stage: - if self.steps[a].get("next", None): - intersection = self.steps[a]["next"].intersection(stage) - if intersection: - for i in intersection: - steps_to_remove.add(i) - - for a in steps_to_remove: - stage.remove(a) - - current_stage = self.root - - while current_stage: - yield current_stage - next_stage = set() - for n in current_stage: - next_stage.update(self.steps[n].get("next", set())) - resolve_intersections(next_stage) - current_stage = next_stage - - def check_for_broken_workflow(self): - step_dependencies = set() - for a_name, a_block in self.steps.items(): - step_dependencies.update(set(a_block.get("needs", list()))) - - if self.wf_fmt == "hcl": - step_dependencies.update(set(self.resolves)) - - for step in step_dependencies: - if step not in self.steps: - log.fail(f"Step '{step}' referenced in workflow but missing") - - def validate_workflow_block(self): - """Validate the syntax of the workflow block.""" - if self.wf_fmt == "yml": - return + # create and frozen a box + wf_box = Box(_wf_data, frozen_box=True, default_box=True) - workflow_block_cnt = len(self.wf_dict.get("workflow", dict()).items()) - if workflow_block_cnt == 0: - log.fail("A workflow block must be present.") - - if workflow_block_cnt > 1: - log.fail("Cannot have more than one workflow blocks.") - - workflow_block = list(self.wf_dict["workflow"].values())[0] - for key in workflow_block.keys(): - if key not in VALID_WORKFLOW_ATTRS: - log.fail(f"Invalid workflow attribute '{key}'.") - - if not workflow_block.get("resolves", None): - log.fail("[resolves] attribute must be present in a " "workflow block.") - - if not pu.of_type(workflow_block["resolves"], ["str", "los"]): - log.fail("[resolves] attribute must be a string or a list " "of strings.") - - if workflow_block.get("on", None): - if not pu.of_type(workflow_block["on"], ["str"]): - log.fail("[on] attribute mist be a string.") - - def validate_step_blocks(self): - """Validate the syntax of the step blocks.""" - if not self.wf_dict.get("steps", None): - log.fail("At least one step block must be present.") - - for _, a_block in self.wf_dict["steps"].items(): - for key in a_block.keys(): - if key not in VALID_STEP_ATTRS: - log.fail(f"Invalid step attribute '{key}'.") - - if not a_block.get("uses", None): - log.fail("[uses] attribute must be present in step block.") - - if not pu.of_type(a_block["uses"], ["str"]): - log.fail("[uses] attribute must be a string.") - - if a_block.get("needs", None): - if not pu.of_type(a_block["needs"], ["str", "los"]): - log.fail( - "[needs] attribute must be a string or a list " "of strings." - ) - - if a_block.get("args", None): - if not pu.of_type(a_block["args"], ["str", "los"]): - log.fail( - "[args] attribute must be a string or a list " "of strings." - ) - - if a_block.get("runs", None): - if not pu.of_type(a_block["runs"], ["str", "los"]): - log.fail( - "[runs] attribute must be a string or a list " "of strings." - ) - - if a_block.get("env", None): - if not pu.of_type(a_block["env"], ["dict"]): - log.fail("[env] attribute must be a dict.") - - if a_block.get("secrets", None): - if not pu.of_type(a_block["secrets"], ["str", "los"]): - log.fail( - "[secrets] attribute must be a string or a list " "of strings." - ) - - def check_for_unreachable_steps(self, skip=None): - """Validates a workflow by checking for unreachable nodes / gaps in the - workflow. - - Args: - skip(list, optional): The list of steps to skip if applicable. - (Default value = None) - - Returns: - None - """ - if not skip or self.wf_fmt == "yml": - # noop - return + log.debug(f"Parsed workflow:\n{wf_box}") - def _traverse(entrypoint, reachable, steps): - """ - - Args: - entrypoint(set): Set containing the entry point of part of the - workflow. - reachable(set): Set containing all the reachable parts of - workflow. - steps(dict): Dictionary containing the identifier of the - workflow and its description. - - Returns: - None - """ - for node in entrypoint: - reachable.add(node) - _traverse(steps[node].get("next", []), reachable, steps) - - reachable = set() - skipped = set(self.props.get("skip_list", [])) - steps = set(map(lambda a: a[0], self.steps.items())) - - _traverse(self.root, reachable, self.steps) - - unreachable = steps - reachable - if unreachable - skipped: - if skip: - log.fail(f'Unreachable step(s): {", ".join(unreachable-skipped)}.') - else: - log.warning(f'Unreachable step(s): {", ".join(unreachable)}.') - - for a in unreachable: - self.steps.pop(a) - - def parse_substitutions(self, substitutions=None, allow_loose=False): - """ + return wf_box - Args: - substitutions(list): List of substitutions that are passed - as an arguments. - allow_loose(bool): Flag used to ignore unused substitution - variable in the workflow. - - Returns: - None + @staticmethod + def __apply_substitution(wf_element, k, v, used_registry): + if isinstance(wf_element, str): + if k in wf_element: + wf_element.replace(k, v) + used_registry[k] = 1 + + elif isinstance(wf_element, list): + # we assume list of strings + for i, e in enumerate(wf_element): + if k in e: + wf_element[i].replace(k, v) + used_registry[k] = 1 + + elif isinstance(wf_element, dict): + # we assume list of strings + for ek in wf_element: + if k in ek: + log.fail("Substitutions only allowed on keys of dictionaries") + if k in wf_element[ek]: + wf_element[ek].replace(k, v) + used_registry[k] = 1 - """ + @staticmethod + def __add_missing_ids(wf_data): + for i, step in enumerate(wf_data["steps"]): + step["id"] = step.get("id", f"{i+1}") + @staticmethod + def __apply_substitutions(wf_data, substitutions=None, allow_loose=False): if not substitutions: - # noop return - substitution_dict = dict() - - for args in substitutions: - item = args.split("=", 1) + used = {} + for substitution in substitutions: + item = substitution.split("=", 1) if len(item) < 2: raise Exception("Excepting '=' as seperator") - substitution_dict["$" + item[0]] = item[1] - for keys in substitution_dict: - if not re.match(r"\$_[A-Z0-9]+", keys): - log.fail(f"Improper substitution format: '{keys}'.") + k, v = ("$" + item[0], item[1]) - used = {} + if not re.match(r"\$_[A-Z0-9]+", k): + log.fail(f"Expecting substitution key as $_[A-Z0-9] but got '{k}'.") + + # replace in steps + for step in wf_data["steps"]: + for _, step_attr in step.items(): + Workflow._apply_substitution(step_attr, k, v, used) - for wf_name, wf_block in self.steps.items(): - - attr = wf_block.get("needs", []) - for i in range(len(attr)): - for k, v in substitution_dict.items(): - if k in attr[i]: - used[k] = 1 - attr[i] = attr[i].replace(k, v) - - attr = wf_block.get("uses", "") - for k, v in substitution_dict.items(): - if k in attr: - used[k] = 1 - wf_block["uses"] = attr.replace(k, v) - - attr = wf_block.get("args", []) - for i in range(len(attr)): - for k, v in substitution_dict.items(): - if k in attr[i]: - used[k] = 1 - attr[i] = attr[i].replace(k, v) - - attr = wf_block.get("runs", []) - for i in range(len(attr)): - for k, v in substitution_dict.items(): - if k in attr[i]: - used[k] = 1 - attr[i] = attr[i].replace(k, v) - - attr = wf_block.get("secrets", []) - for i in range(len(attr)): - for k, v in substitution_dict.items(): - if k in attr[i]: - used[k] = 1 - attr[i] = attr[i].replace(k, v) - - attr = wf_block.get("env", {}) - temp_dict = {} - for key in attr.keys(): - check_replacement = False - for k, v in substitution_dict.items(): - if k in key: - used[k] = 1 - temp_dict[v] = attr[key] - check_replacement = True - - if check_replacement is False: - temp_dict[key] = attr[key] - - for key, value in temp_dict.items(): - for k, v in substitution_dict.items(): - if k in value: - used[k] = 1 - temp_dict[key] = v - - if len(temp_dict) != 0: - wf_block["env"] = temp_dict - - if not allow_loose and len(substitution_dict) != len(used): + if not allow_loose and len(substitutions) != len(used): log.fail("Not all given substitutions are used in " "the workflow file") @staticmethod - def skip_steps(wf, skip_list=[]): - """Removes the steps to be skipped from the workflow graph and return - a new `Workflow` object. - - Args: - wf(Workflow): The workflow object to operate upon. - skip_list(list): List of steps to be skipped. - (Default value = list()) - - Returns: - Workflow : The updated workflow object. - """ + def __skip_steps(wf_data, skip_list=[]): if not skip_list: - # noop - return wf - - workflow = deepcopy(wf) - for sa_name in skip_list: - if sa_name not in workflow.steps: - log.fail(f"Referenced step '{sa_name} missing.") - sa_block = workflow.steps[sa_name] - # Clear up all connections from sa_block - sa_block.get("next", set()).clear() - del sa_block.get("needs", list())[:] - - # Handle skipping of root step's - if sa_name in workflow.root: - workflow.root.remove(sa_name) - - # Handle skipping of non-root step's - for a_name, a_block in workflow.steps.items(): - if sa_name in a_block.get("next", set()): - a_block["next"].remove(sa_name) - - if sa_name in a_block.get("needs", list()): - a_block["needs"].remove(sa_name) + return + filtered_list = [] + used = {} + for step in wf_data["steps"]: + if step["id"] in skip_list: + used[step["id"]] = 1 + continue + filtered_list.append(step) + wf_data["steps"] = filtered_list - workflow.props["skip_list"] = list(skip_list) - return workflow + if len(used) != len(skip_list): + log.fail(f"Not all skipped steps exist in the workflow.") @staticmethod - def filter_step(wf, step, with_dependencies=False): - """Filters out all steps except the one passed in the argument from - the workflow. - Args: - wf(Workflow): The workflow object to operate upon. - step(str): The step to run. - with_dependencies(bool, optional): Filter out step to - run with dependencies or not. (Default value = False) - Returns: - Workflow: The updated workflow object. - """ - - if not step: - # noop - return wf - - # Recursively generate root when an step is run - # with the `--with-dependencies` flag. - def find_root_recursively(workflow, step, required_steps): - """ - Args: - workflow(worklfow): The workflow object to operate upon. - step(str): The step to run. - required_steps(set): Set containing steps that are - to be executed. - Returns: - None - """ - required_steps.add(step) - if workflow.steps[step].get("needs", None): - for a in workflow.steps[step]["needs"]: - find_root_recursively(workflow, a, required_steps) - if not workflow.steps[a].get("next", None): - workflow.steps[a]["next"] = set() - workflow.steps[a]["next"].add(step) - else: - workflow.root.add(step) - - # The list of steps that needs to be preserved. - workflow = deepcopy(wf) - - if step not in workflow.steps: - log.fail(f"Referenced step '{step}' missing.") - - steps = set(map(lambda x: x[0], workflow.steps.items())) - - required_steps = set() - - if with_dependencies: - # Prepare the graph for running only the given step - # only with its dependencies. - find_root_recursively(workflow, step, required_steps) - - filtered_steps = steps - required_steps - - for ra in required_steps: - a_block = workflow.steps[ra] - common_steps = filtered_steps.intersection(a_block.get("next", set())) - if common_steps: - for ca in common_steps: - a_block["next"].remove(ca) - else: - # Prepare the step for its execution only. - required_steps.add(step) - - if workflow.steps[step].get("next", None): - workflow.steps[step]["next"] = set() - - if workflow.steps[step].get("needs", None): - workflow.steps[step]["needs"] = list() - - workflow.root.add(step) - - # Make the list of the steps to be removed. - steps = steps - required_steps - - # Remove the remaining steps - for a in steps: - if a in workflow.root: - workflow.root.remove(a) - workflow.steps.pop(a) - - return workflow - - -class YMLWorkflow(Workflow): - """Parse a yml based workflow and generate the workflow graph. - """ - - def __init__(self, wfile): - """Loads the workflow as a dict from the `.yml` file.""" - super(YMLWorkflow, self).__init__() - self.wf_fmt = "yml" - - if not os.path.exists(wfile): - # try to load string - self.wf_list = yaml.safe_load(wfile)["steps"] - else: - with open(wfile) as fp: - self.wf_list = yaml.safe_load(fp)["steps"] - if not self.wf_list: - return - - self.wf_dict = {"steps": dict()} - self.id_map = dict() - - for idx, step in enumerate(self.wf_list): - # If no id attribute present, make one - _id = str(step.get("id", idx + 1)) - self.wf_dict["steps"][_id] = step - self.id_map[idx + 1] = _id - step.pop("id", None) - - self.name = os.path.basename(wfile)[:-4] - - def normalize(self): - """Takes properties from the `self.wf_dict` dict and makes them - native to the `Workflow` class. Also it normalizes some of the - attributes of a parsed workflow according to the Github defined - specifications. - - For example, it changes `args`, `runs` and `secrets` attribute, - if provided as a string to a list of string by splitting around - whitespace. Also, it changes parameters like `uses` and `resolves`, - if provided as a string to a list. - - Args: - None - - Returns: - None - """ - self.on = "" - self.root = set() - self.props = dict() - self.steps = self.wf_dict["steps"] - - for a_name, a_block in self.steps.items(): - a_block["name"] = a_name - - if a_block.get("needs", None): - if pu.of_type(a_block["needs"], ["str"]): - a_block["needs"] = [a_block["needs"]] - - if a_block.get("args", None): - a_block["args"] = Workflow.format_command(a_block["args"]) - - if a_block.get("runs", None): - a_block["runs"] = Workflow.format_command(a_block["runs"]) - - if a_block.get("secrets", None): - a_block["secrets"] = Workflow.format_command(a_block["secrets"]) - - def get_containing_set(self, idx): - """Find the set from the list of sets of step dependencies, where - the step with the given index is present. - - Args: - idx(int): The index of the step to be searched. - - Returns: - set: The required set. - """ - for _set in self.dep_sets: - if self.id_map[idx] in _set: - return _set - - required_set = set() - required_set.add(self.id_map[idx]) - return required_set - - def complete_graph(self): - """Function to generate the workflow graph by adding forward edges. - - Args: - None - - Returns: - None - """ - # Connect the graph as much as possible. - for a_id, a_block in self.steps.items(): - if a_block.get("needs", None): - for a in a_block["needs"]: - if not self.steps[a].get("next", None): - self.steps[a]["next"] = set() - self.steps[a]["next"].add(a_id) - - # Generate the dependency sets. - self.dep_sets = list() - self.visited = dict() - - for a_id, a_block in self.steps.items(): - if a_block.get("next", None): - if a_block["next"] not in self.dep_sets: - self.dep_sets.append(a_block["next"]) - self.visited[tuple(a_block["next"])] = False - - if a_block.get("needs", None): - if a_block["needs"] not in self.dep_sets: - self.dep_sets.append(set(a_block["needs"])) - self.visited[tuple(a_block["needs"])] = False - - # Moving from top to bottom - for idx, id in self.id_map.items(): - step = self.steps[id] - if not step.get("next", None): - # if this is not the last step, - if idx + 1 <= len(self.steps): - curr = self.id_map[idx] - next = self.id_map[idx + 1] - # If the current step and next step is not in any - # set, - if ({curr, next} not in self.dep_sets) and ( - {next, curr} not in self.dep_sets - ): - next_set = self.get_containing_set(idx + 1) - curr_set = self.get_containing_set(idx) - - if not self.visited.get(tuple(next_set), None): - step["next"] = next_set - for nsa in next_set: - self.steps[nsa]["needs"] = [id] - self.visited[tuple(curr_set)] = True - - # Finally, generate the root. - for a_id, a_block in self.steps.items(): - if not a_block.get("needs", None): - self.root.add(a_id) - - -VALID_WORKFLOW_ATTRS = ["resolves", "on"] - - -class HCLWorkflow(Workflow): - """Parse a hcl based workflow and generate - the workflow graph. - """ - - def __init__(self, wfile): - """Loads the workflow as a dict from the `.workflow` file.""" - super(HCLWorkflow, self).__init__() - self.wf_fmt = "hcl" - - if not os.path.exists(wfile): - # try to load a string - self.wf_dict = hcl.loads(wfile) - else: - with open(wfile) as fp: - self.wf_dict = hcl.load(fp) - - if "action" in self.wf_dict: - self.wf_dict["steps"] = self.wf_dict.pop("action") - - def find_root(self, entrypoint, root): - """A GHA workflow is defined by specifying edges that point to the - previous nodes they depend on. To make the workflow easier to process, - we add forward edges. This also obtains the root nodes. - - Args: - entrypoint(list): List of nodes from where to start - generating the graph. - root(set): Set of nodes without dependencies, - that would eventually be used as root. - - Returns: - None - """ - for node in entrypoint: - if self.steps[node].get("needs", None): - for n in self.steps[node]["needs"]: - self.find_root([n], root) - if not self.steps[n].get("next", None): - self.steps[n]["next"] = set() - self.steps[n]["next"].add(node) - else: - root.add(node) - - def complete_graph(self): - """Driver function to run the recursive function - `find_root()` which adds forward edges. - - Args: - None - - Returns: - None - """ - self.find_root(self.resolves, self.root) - - def normalize(self): - """Takes properties from the `self.wf_dict` dict and makes them - native to the `Workflow` class. Also it normalizes some of the - attributes of a parsed workflow according to the Github defined - specifications. - - For example, it changes `args`, `runs` and `secrets` attribute, - if provided as a string to a list of string by splitting around - whitespace. Also, it changes parameters like `uses` and `resolves`, - if provided as a string to a list. - - Args: - None - - Returns: - None - """ - for wf_name, wf_block in self.wf_dict["workflow"].items(): - - self.name = wf_name - self.resolves = wf_block["resolves"] - self.on = wf_block.get("on", "push") - self.root = set() - self.steps = self.wf_dict["steps"] - self.props = dict() - - if pu.of_type(self.resolves, ["str"]): - self.resolves = [self.resolves] - - for a_name, a_block in self.steps.items(): - a_block["name"] = a_name - - if a_block.get("needs", None): - if pu.of_type(a_block["needs"], ["str"]): - a_block["needs"] = [a_block["needs"]] - - if a_block.get("args", None): - a_block["args"] = Workflow.format_command(a_block["args"]) - - if a_block.get("runs", None): - a_block["runs"] = Workflow.format_command(a_block["runs"]) - - if a_block.get("secrets", None): - a_block["secrets"] = Workflow.format_command(a_block["secrets"]) + def __filter_step(wf_data, filtered_step=None): + """Remove all but the given one.""" + if not filtered_step: + return + for step in wf_data["steps"]: + if step["id"] == filtered_step: + return step diff --git a/cli/setup.py b/cli/setup.py index 172a5d4b6..71fc60bab 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -17,7 +17,7 @@ "click==7.1.2", "docker==4.2.0", "GitPython==3.1.0", - "pyhcl==0.4.0", + "pykwalify==1.7.0", "python-box==4.2.3", "pyyaml==5.3.1", "spython==0.0.79", diff --git a/cli/test/fixtures/a.workflow b/cli/test/fixtures/a.workflow deleted file mode 100644 index 2495a4530..000000000 --- a/cli/test/fixtures/a.workflow +++ /dev/null @@ -1,36 +0,0 @@ -workflow "example" { -resolves = "end" -} - -action "a" { -uses = "sh" -args = "ls" -} - -action "b" { -uses = "sh" -args = "ls" -} - -action "c" { -uses = "sh" -args = "ls" -} - -action "d" { -needs = ["c"] -uses = "sh" -args = "ls" -} - -action "e" { -needs = ["d", "b", "a"] -uses = "sh" -args = "ls" -} - -action "end" { -needs = "e" -uses = "sh" -args = "ls" -} diff --git a/cli/test/fixtures/a.yml b/cli/test/fixtures/a.yml deleted file mode 100644 index 2a6cb932d..000000000 --- a/cli/test/fixtures/a.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: '1' -steps: -- id: "a" - uses: "sh" - args: "ls" - -- id: "b" - uses: "sh" - args: "ls" - -- id: "c" - uses: "sh" - args: "ls" - -- id: "d" - needs: ["c"] - uses: "sh" - args: "ls" - -- id: "e" - needs: ["d", "b", "a"] - uses: "sh" - args: "ls" - -- id: "end" - needs: "e" - uses: "sh" - args: "ls" diff --git a/cli/test/fixtures/b.workflow b/cli/test/fixtures/b.workflow deleted file mode 100644 index 9b57c28ee..000000000 --- a/cli/test/fixtures/b.workflow +++ /dev/null @@ -1,49 +0,0 @@ -workflow "example" { - resolves = ["end"] -} - -action "a" { - uses = "sh" - args = "ls" -} - -action "b" { - needs = "a" - uses = "sh" - args = "ls" -} - -action "c" { - uses = "sh" - args = "ls" -} - -action "d" { - uses = "sh" - needs = ["b", "c"] - args = "ls" -} - -action "g" { - needs = "d" - uses = "sh" - args = "ls" -} - -action "f" { - needs = "d" - uses = "sh" - args = "ls" -} - -action "h" { - needs = "g" - uses = "sh" - args = "ls" -} - -action "end" { - needs = ["h", "f"] - uses = "sh" - args = "ls" -} diff --git a/cli/test/fixtures/b.yml b/cli/test/fixtures/b.yml deleted file mode 100644 index 691e29b2f..000000000 --- a/cli/test/fixtures/b.yml +++ /dev/null @@ -1,39 +0,0 @@ -version: '1' -steps: -- id: "a" - uses: "sh" - args: "ls" - -- id: "b" - needs: "a" - uses: "sh" - args: "ls" - -- id: "c" - uses: "sh" - args: "ls" - -- id: "d" - uses: "sh" - needs: ["b", "c"] - args: "ls" - -- id: "g" - needs: "d" - uses: "sh" - args: "ls" - -- id: "f" - needs: "d" - uses: "sh" - args: "ls" - -- id: "h" - needs: "g" - uses: "sh" - args: "ls" - -- id: "end" - needs: ["h", "f"] - uses: "sh" - args: "ls" diff --git a/cli/test/fixtures/missing_dependency.workflow b/cli/test/fixtures/missing_dependency.workflow deleted file mode 100644 index 73d8cfa9c..000000000 --- a/cli/test/fixtures/missing_dependency.workflow +++ /dev/null @@ -1,10 +0,0 @@ -workflow "samples" { - resolves = ["c"] -} -action "b" { - uses = "sh" -} -action "c" { - uses = "sh" - needs = "a" -} diff --git a/cli/test/fixtures/missing_dependency.yml b/cli/test/fixtures/missing_dependency.yml deleted file mode 100644 index 2f1e74496..000000000 --- a/cli/test/fixtures/missing_dependency.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: '1' -steps: -- id: 'a' - uses: 'sh' -- id: 'b' - needs: 'a1' - uses: 'sh' diff --git a/cli/test/fixtures/ok.workflow b/cli/test/fixtures/ok.workflow deleted file mode 100644 index a2cb33066..000000000 --- a/cli/test/fixtures/ok.workflow +++ /dev/null @@ -1,11 +0,0 @@ -workflow "sample" { - resolves = ["reachable"] -} -action "reachable" { - uses = "popperized/bin/sh@master" - args = "ls" -} -action "unreachable" { - uses = "popperized/bin/sh@master" - args = ["ls -ltr"] -} diff --git a/cli/test/fixtures/ok.yaml b/cli/test/fixtures/ok.yaml deleted file mode 100644 index bca61e97c..000000000 --- a/cli/test/fixtures/ok.yaml +++ /dev/null @@ -1,4 +0,0 @@ -- uses: "popperized/bin/sh@master" - args: "ls" -- uses: "popperized/bin/sh@master" - args: ["ls -ltr"] diff --git a/cli/test/fixtures/substitutions.workflow b/cli/test/fixtures/substitutions.workflow deleted file mode 100644 index 4f9d2772f..000000000 --- a/cli/test/fixtures/substitutions.workflow +++ /dev/null @@ -1,19 +0,0 @@ -workflow "example" { - resolves = ["b"] -} - -action "a" { - uses = "$_VAR1" - args = "$_VAR2" -} - -action "b" { - needs = "a" - uses = "$_VAR1" - args = "$_VAR2" - runs = "$_VAR4" - secrets = ["$_VAR5"] - env = { - "$_VAR6" = "$_VAR7" - } -} diff --git a/cli/test/fixtures/substitutions.yml b/cli/test/fixtures/substitutions.yml deleted file mode 100644 index dc0a5f453..000000000 --- a/cli/test/fixtures/substitutions.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: '1' -steps: -- id: "a" - uses: "$_VAR1" - args: "$_VAR2" - -- id: "b" - needs: "a" - uses: "$_VAR1" - args: "$_VAR2" - runs: "$_VAR4" - secrets: ["$_VAR5"] - env: - "$_VAR6": "$_VAR7" diff --git a/cli/test/fixtures/wf_1.yml b/cli/test/fixtures/wf_1.yml deleted file mode 100644 index 6d4a1176f..000000000 --- a/cli/test/fixtures/wf_1.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: 1 -steps: -- id: one - uses: docker://alpine - args: sleep 10 - -- id: two - uses: sh - runs: ls -l diff --git a/cli/test/test_cmd_scaffold.py b/cli/test/test_cmd_scaffold.py deleted file mode 100644 index 2e9ab6de9..000000000 --- a/cli/test/test_cmd_scaffold.py +++ /dev/null @@ -1,56 +0,0 @@ -import os -import tempfile - -from click.testing import CliRunner - -from popper.cli import log -from popper.commands import cmd_scaffold, cmd_run -from popper.parser import Workflow - -from .test_common import PopperTest - - -class TestScaffold(PopperTest): - def setUp(self): - log.setLevel("CRITICAL") - - def test_scaffold(self): - - wf_dir = tempfile.mkdtemp() - runner = CliRunner() - file_loc = f"{wf_dir}/wf.yml" - - result = runner.invoke(cmd_scaffold.cli, ["-f", file_loc]) - - self.assertEqual(result.exit_code, 0) - self.assertTrue(os.path.isfile(file_loc)) - - wf = Workflow.new(file_loc) - self.assertDictEqual( - wf.steps, - { - "1": { - "uses": "popperized/bin/sh@master", - "args": ["ls"], - "name": "1", - "next": {"2"}, - }, - "2": { - "uses": "docker://alpine:3.11", - "args": ["ls"], - "name": "2", - "needs": ["1"], - }, - }, - ) - - with self.assertLogs("popper") as test_logger: - result = runner.invoke(cmd_run.cli, ["-f", file_loc, "-w", wf_dir]) - self.assertEqual(result.exit_code, 0) - self.assertTrue(len(test_logger.output)) - self.assertTrue( - "INFO:popper:Step '1' ran successfully !" in test_logger.output - ) - self.assertTrue( - "INFO:popper:Step '2' ran successfully !" in test_logger.output - ) diff --git a/cli/test/test_parser.py b/cli/test/test_parser.py index 90ea95f39..5093b1fb1 100644 --- a/cli/test/test_parser.py +++ b/cli/test/test_parser.py @@ -1,14 +1,10 @@ import unittest import os -from popper.parser import Workflow, YMLWorkflow, HCLWorkflow +from popper.parser import WorkflowParser from popper.cli import log -FIXDIR = f"{os.path.dirname(os.path.realpath(__file__))}/fixtures" - - -def _wfile(name, format): - return f"{FIXDIR}/{name}.{format}" +import logging class TestWorkflow(unittest.TestCase): @@ -19,572 +15,20 @@ def tearDown(self): log.setLevel("NOTSET") def test_new_workflow(self): - self.assertIsInstance(Workflow.new(_wfile("a", "yml")), YMLWorkflow) - self.assertIsInstance(Workflow.new(_wfile("a", "workflow")), HCLWorkflow) - - def test_missing_dependency(self): - wf = HCLWorkflow(_wfile("missing_dependency", "workflow")) - wf.normalize() - self.assertRaises(SystemExit, wf.check_for_broken_workflow) - wf = YMLWorkflow(_wfile("missing_dependency", "yml")) - wf.normalize() - self.assertRaises(SystemExit, wf.check_for_broken_workflow) - - def test_command(self): - cmd = "docker version" - res = Workflow.format_command(cmd) - self.assertEqual(res, ["docker", "version"]) - - cmd = ["docker", "version"] - res = Workflow.format_command(cmd) - self.assertEqual(res, ["docker", "version"]) - - def test_validate_workflow_block(self): - wf = HCLWorkflow( - """workflow "w1" { - resolves = ["a"] -} -workflow "w2" { - resolves = ["a"] -} -""" - ) - self.assertRaises(SystemExit, wf.validate_workflow_block) - - wf = HCLWorkflow( - """ -action "a" { - uses = "sh" -} -""" - ) - self.assertRaises(SystemExit, wf.validate_workflow_block) - - wf = HCLWorkflow( - """ -workflow "sample workflow 1" { - resolves = ["a"] - runs = ["sh", "-c", "ls"] -} -action "a" { - uses = ["sh"] -} -""" - ) - self.assertRaises(SystemExit, wf.validate_workflow_block) - - wf = HCLWorkflow( - """ -workflow "sample workflow 1" { - on = "push" -} -action "a" { - uses = ["sh"] -} -""" - ) - self.assertRaises(SystemExit, wf.validate_workflow_block) - - def test_validate_step_blocks(self): - wf = HCLWorkflow( - """workflow "sample workflow" { - resolves = "a" - }""" - ) - self.assertRaises(SystemExit, wf.validate_step_blocks) - - wf = HCLWorkflow( - """workflow "sample workflow" { - resolves = "a" -} -action "a" { - uses = "sh" - on = "push" -}""" - ) - self.assertRaises(SystemExit, wf.validate_step_blocks) - - wf = HCLWorkflow( - """workflow "sample workflow" { - resolves = "a" -} -action "a" { - args = "ls" -}""" - ) - self.assertRaises(SystemExit, wf.validate_step_blocks) - - wf = HCLWorkflow( - """workflow "sample workflow" { - resolves = "a" -} -action "a" { - uses = 1 -}""" - ) - self.assertRaises(SystemExit, wf.validate_step_blocks) - - wf = HCLWorkflow( - """workflow "sample workflow" { - resolves = "a" -} - -action "a" { - uses = "sh" - needs = 1 -}""" - ) - self.assertRaises(SystemExit, wf.validate_step_blocks) - - wf = HCLWorkflow( - """workflow "sample workflow" { - resolves = "a" -} -action "a" { - uses = "sh" - args = [1, 2, 3, 4] -} -""" - ) - self.assertRaises(SystemExit, wf.validate_step_blocks) - - wf = HCLWorkflow( - """workflow "sample workflow" { - resolves = "a" -} -action "a" { - uses = "sh" - runs = [1, 2, 3, 4] -} -""" - ) - self.assertRaises(SystemExit, wf.validate_step_blocks) - - wf = HCLWorkflow( - """workflow "sample workflow" { - resolves = "a" -} -action "a" { - uses = "sh" - secrets = { - SECRET_A = 1234, - SECRET_B = 5678 - } -} -""" - ) - self.assertRaises(SystemExit, wf.validate_step_blocks) - - wf = HCLWorkflow( - """workflow "sample workflow" { - resolves = "a" -} -action "a" { - uses = "sh" - env = [ - "SECRET_A", "SECRET_B" - ] -} -""" - ) - self.assertRaises(SystemExit, wf.validate_step_blocks) - - def test_skip_steps(self): - wf = YMLWorkflow(_wfile("a", "yml")) - wf.parse() - changed_wf = Workflow.skip_steps(wf, ["b"]) - self.assertDictEqual( - changed_wf.steps, - { - "a": {"uses": "sh", "args": ["ls"], "name": "a", "next": {"e"}}, - "b": {"uses": "sh", "args": ["ls"], "name": "b", "next": set()}, - "c": {"uses": "sh", "args": ["ls"], "name": "c", "next": {"d"}}, - "d": { - "needs": ["c"], - "uses": "sh", - "args": ["ls"], - "name": "d", - "next": {"e"}, - }, - "e": { - "needs": ["d", "a"], - "uses": "sh", - "args": ["ls"], - "name": "e", - "next": {"end"}, - }, - "end": {"needs": ["e"], "uses": "sh", "args": ["ls"], "name": "end"}, - }, - ) - - changed_wf = Workflow.skip_steps(wf, ["d", "a"]) - self.assertDictEqual( - changed_wf.steps, - { - "a": {"uses": "sh", "args": ["ls"], "name": "a", "next": set()}, - "b": {"uses": "sh", "args": ["ls"], "name": "b", "next": {"e"}}, - "c": {"uses": "sh", "args": ["ls"], "name": "c", "next": set()}, - "d": { - "needs": [], - "uses": "sh", - "args": ["ls"], - "name": "d", - "next": set(), - }, - "e": { - "needs": ["b"], - "uses": "sh", - "args": ["ls"], - "name": "e", - "next": {"end"}, - }, - "end": {"needs": ["e"], "uses": "sh", "args": ["ls"], "name": "end"}, - }, - ) - - def test_filter_step(self): - wf = YMLWorkflow(_wfile("a", "yml")) - wf.parse() - changed_wf = Workflow.filter_step(wf, "e") - self.assertSetEqual(changed_wf.root, {"e"}) - self.assertDictEqual( - changed_wf.steps, - { - "e": { - "needs": [], - "uses": "sh", - "args": ["ls"], - "name": "e", - "next": set(), - } - }, - ) - - changed_wf = Workflow.filter_step(wf, "d") - self.assertSetEqual(changed_wf.root, {"d"}) - self.assertDictEqual( - changed_wf.steps, - { - "d": { - "needs": [], - "uses": "sh", - "args": ["ls"], - "name": "d", - "next": set(), - } - }, - ) - - changed_wf = Workflow.filter_step(wf, "e", with_dependencies=True) - self.assertSetEqual(changed_wf.root, {"b", "a", "c"}) - self.assertDictEqual( - changed_wf.steps, - { - "a": {"uses": "sh", "args": ["ls"], "name": "a", "next": {"e"}}, - "b": {"uses": "sh", "args": ["ls"], "name": "b", "next": {"e"}}, - "c": {"uses": "sh", "args": ["ls"], "name": "c", "next": {"d"}}, - "d": { - "needs": ["c"], - "uses": "sh", - "args": ["ls"], - "name": "d", - "next": {"e"}, - }, - "e": { - "needs": ["d", "b", "a"], - "uses": "sh", - "args": ["ls"], - "name": "e", - "next": set(), - }, - }, - ) - - changed_wf = Workflow.filter_step(wf, "d", with_dependencies=True) - self.assertSetEqual(changed_wf.root, {"c"}) - self.assertDictEqual( - changed_wf.steps, - { - "c": {"uses": "sh", "args": ["ls"], "name": "c", "next": {"d"}}, - "d": { - "needs": ["c"], - "uses": "sh", - "args": ["ls"], - "name": "d", - "next": set(), - }, - }, - ) - - def test_check_for_unreachable_steps(self): - wf = HCLWorkflow(_wfile("a", "workflow")) - wf.parse() - changed_wf = Workflow.skip_steps(wf, ["d", "a", "b"]) - self.assertDictEqual( - changed_wf.steps, - { - "a": {"uses": "sh", "args": ["ls"], "name": "a", "next": set()}, - "b": {"uses": "sh", "args": ["ls"], "name": "b", "next": set()}, - "c": {"uses": "sh", "args": ["ls"], "name": "c", "next": set()}, - "d": { - "needs": [], - "uses": "sh", - "args": ["ls"], - "name": "d", - "next": set(), - }, - "e": { - "needs": [], - "uses": "sh", - "args": ["ls"], - "name": "e", - "next": {"end"}, - }, - "end": {"needs": ["e"], "uses": "sh", "args": ["ls"], "name": "end"}, - }, - ) - - changed_wf.check_for_unreachable_steps() - - wf = HCLWorkflow(_wfile("ok", "workflow")) - wf.parse() - wf.check_for_unreachable_steps() - - def test_get_stages(self): - wf = HCLWorkflow(_wfile("a", "workflow")) - wf.parse() - stages = list() - for stage in wf.get_stages(): - stages.append(stage) - - self.assertListEqual(stages, [{"b", "c", "a"}, {"d"}, {"e"}, {"end"}]) - - wf = YMLWorkflow(_wfile("b", "yml")) - wf.parse() - stages = list() - for stage in wf.get_stages(): - stages.append(stage) - - self.assertListEqual( - stages, [{"a", "c"}, {"b"}, {"d"}, {"g", "f"}, {"h"}, {"end"}] - ) - - def test_substitutions(self): - subs = [ - "_VAR1=sh", - "_VAR2=ls", - "_VAR4=test_env", - "_VAR5=TESTING", - "_VAR6=TESTER", - "_VAR7=TEST", - ] - wf = YMLWorkflow(_wfile("substitutions", "yml")) - wf.parse(subs, False) - self.assertDictEqual( - wf.steps, - { - "a": {"uses": "sh", "args": ["ls"], "name": "a", "next": {"b"}}, - "b": { - "needs": ["a"], - "uses": "sh", - "args": ["ls"], - "runs": ["test_env"], - "secrets": ["TESTING"], - "env": {"TESTER": "TEST"}, - "name": "b", - }, - }, - ) - - wf = YMLWorkflow(_wfile("substitutions", "yml")) - wf.parse(subs, False) - self.assertDictEqual( - wf.steps, - { - "a": {"uses": "sh", "args": ["ls"], "name": "a", "next": {"b"}}, - "b": { - "needs": ["a"], - "uses": "sh", - "args": ["ls"], - "runs": ["test_env"], - "secrets": ["TESTING"], - "env": {"TESTER": "TEST"}, - "name": "b", - }, - }, - ) - - -class TestHCLWorkflow(unittest.TestCase): - def setUp(self): - log.setLevel("CRITICAL") - - def tearDown(self): - log.setLevel("NOTSET") - - def test_load_file(self): - wf = HCLWorkflow( - """workflow "sample" { - resolves = "b" -} -action "a" { - uses = "sh" -} -action "b" { - needs = "a" - uses = "sh" -}""" - ) - self.assertEqual(wf.wf_fmt, "hcl") - self.assertDictEqual( - wf.wf_dict, - { - "workflow": {"sample": {"resolves": "b"}}, - "steps": {"a": {"uses": "sh"}, "b": {"needs": "a", "uses": "sh"}}, - }, - ) - - def test_normalize(self): - wf = HCLWorkflow( - """workflow "sample workflow" { - resolves = "a" -} -action "a" { - needs = "b" - uses = "popperized/bin/npm@master" - args = "npm --version" - secrets = "SECRET_KEY" -}""" - ) - wf.normalize() - self.assertEqual(wf.resolves, ["a"]) - self.assertEqual(wf.name, "sample workflow") - self.assertEqual(wf.on, "push") - self.assertDictEqual(wf.props, dict()) - step_a = wf.steps["a"] - self.assertEqual(step_a["name"], "a") - self.assertEqual(step_a["needs"], ["b"]) - self.assertEqual(step_a["args"], ["npm", "--version"]) - self.assertEqual(step_a["secrets"], ["SECRET_KEY"]) - - def test_complete_graph(self): - wf = HCLWorkflow(_wfile("a", "workflow")) - wf.normalize() - wf.complete_graph() - self.assertEqual(wf.name, "example") - self.assertEqual(wf.resolves, ["end"]) - self.assertEqual(wf.on, "push") - self.assertEqual(wf.props, {}) - self.assertEqual(wf.root, {"b", "c", "a"}) - - steps_dict = { - "a": {"uses": "sh", "args": ["ls"], "name": "a", "next": {"e"}}, - "b": {"uses": "sh", "args": ["ls"], "name": "b", "next": {"e"}}, - "c": {"uses": "sh", "args": ["ls"], "name": "c", "next": {"d"}}, - "d": { - "needs": ["c"], - "uses": "sh", - "args": ["ls"], - "name": "d", - "next": {"e"}, - }, - "e": { - "needs": ["d", "b", "a"], - "uses": "sh", - "args": ["ls"], - "name": "e", - "next": {"end"}, - }, - "end": {"needs": ["e"], "uses": "sh", "args": ["ls"], "name": "end"}, - } - self.assertDictEqual(wf.steps, steps_dict) - - -class TestYMLWorkflow(unittest.TestCase): - def setUp(self): - log.setLevel("CRITICAL") - - def tearDown(self): - log.setLevel("NOTSET") - - def test_load_file(self): - wf = YMLWorkflow( - """ - steps: - - id: 'a' - uses: 'sh' - - - id: 'b' - uses: 'sh' - """ - ) - self.assertEqual(wf.wf_fmt, "yml") - self.assertDictEqual( - wf.wf_dict, {"steps": {"a": {"uses": "sh"}, "b": {"uses": "sh"}}} - ) - self.assertListEqual(wf.wf_list, [{"uses": "sh"}, {"uses": "sh"}]) - self.assertDictEqual(wf.id_map, {1: "a", 2: "b"}) + wf_data = {} + self.assertRaises(SystemExit, WorkflowParser.parse, **{"wf_data": wf_data}) - def test_normalize(self): - wf = YMLWorkflow( - """ - steps: - - id: "a" - needs: "b" - uses: "popperized/bin/npm@master" - args: "npm --version" - secrets: "SECRET_KEY" - """ - ) - wf.normalize() - self.assertEqual(wf.on, "") - self.assertDictEqual(wf.props, dict()) - step_a = wf.steps["a"] - self.assertEqual(step_a["name"], "a") - self.assertEqual(step_a["needs"], ["b"]) - self.assertEqual(step_a["args"], ["npm", "--version"]) - self.assertEqual(step_a["secrets"], ["SECRET_KEY"]) + wf_data = {"unexpected": []} + self.assertRaises(SystemExit, WorkflowParser.parse, **{"wf_data": wf_data}) - def test_get_containing_set(self): - wf = YMLWorkflow(_wfile("a", "yml")) - wf.normalize() - wf.complete_graph() - set_1 = wf.get_containing_set(2) - self.assertSetEqual(set_1, {"b", "a", "d"}) + wf_data = {"steps": [{"uses": "foo",}]} + wf = WorkflowParser.parse(wf_data=wf_data) - wf = YMLWorkflow(_wfile("b", "yml")) - wf.normalize() - wf.complete_graph() - set_2 = wf.get_containing_set(3) - self.assertSetEqual(set_2, {"c", "b"}) + self.assertEqual("foo", wf.steps[0].uses) - def test_complete_graph(self): - wf = YMLWorkflow(_wfile("a", "yml")) - wf.normalize() - wf.complete_graph() - self.assertEqual(wf.name, "a") - self.assertEqual(wf.on, "") - self.assertEqual(wf.props, {}) - self.assertEqual(wf.root, {"b", "c", "a"}) - steps_dict = { - "a": {"uses": "sh", "args": ["ls"], "name": "a", "next": {"e"}}, - "b": {"uses": "sh", "args": ["ls"], "name": "b", "next": {"e"}}, - "c": {"uses": "sh", "args": ["ls"], "name": "c", "next": {"d"}}, - "d": { - "needs": ["c"], - "uses": "sh", - "args": ["ls"], - "name": "d", - "next": {"e"}, - }, - "e": { - "needs": ["d", "b", "a"], - "uses": "sh", - "args": ["ls"], - "name": "e", - "next": {"end"}, - }, - "end": {"needs": ["e"], "uses": "sh", "args": ["ls"], "name": "end"}, - } - self.assertDictEqual(wf.steps, steps_dict) +# TODO add missing tests: +# - test_add_missing_ids +# - test_apply_substitutions +# - test_skip_tests +# - test_filter_step