diff --git a/scripts/rca/common.py b/scripts/rca/common.py new file mode 100644 index 00000000..8628b777 --- /dev/null +++ b/scripts/rca/common.py @@ -0,0 +1,157 @@ +import rule +import directive +import sys +from tabulate import tabulate +sys.path.insert(0, "/usr/share/rudder-api-client/") +sys.path.insert(0, "/opt/rudder/share/python") +from rudder import RudderEndPoint, RudderError + +with open('/var/rudder/run/api-token') as ftoken: + TOKEN = ftoken.read() +RUDDER_URL="https://localhost/rudder" +endpoint = RudderEndPoint(RUDDER_URL, TOKEN, verify=False) + +## Get all variable definition +rawDirectives = endpoint.list_techniques_directives("genericVariableDefinition")["directives"] +directives = [ directive.Directive(x) for x in rawDirectives ] + +## Get all rules +rawRules = endpoint.list_rules()["rules"] +rules = [ rule.Rule(x) for x in rawRules ] + +## Get all groups +rawGroups = endpoint.list_groups()["groups"] +groups = [ x["id"] for x in rawGroups ] + + +#### Functions +""" + Return all directives defined +""" +def get_directives(): + return directives + +""" + Return all rules defined +""" +def get_rules(): + return rules + +""" + Return all groups defined +""" +def get_groups(): + return groups + +""" + Return a list with the ids of all directives applied to the target group +""" +def get_directives_applied_to(group): + appliedDirectives = set() + for iRule in rules: + if group in iRule.getIncludeTargets(): + for iDirective in iRule.directives: + appliedDirectives |= set([ x for x in directives if x.id == iDirective ]) + return list(appliedDirectives) + +""" + Sort the table to display on the given column. + toSort must respect the same input than the display function +""" +def sortToDisplay(toSort, criterion): + sortedList = [] + columnNames = [x['title'] for x in toSort] + if criterion not in columnNames: + print("Unknown column name '%s' to sort by, available columns are: %s"%(criterion, columnNames)) + exit(1) + refColumn = next(x for x in toSort if x['title'] == criterion)['value'] + for iColumn in toSort: + zipped = zip(refColumn, iColumn['value']) + zipped.sort() + sortedvalue = [j for (i, j) in zipped] + sortedList.append({ + 'title' : iColumn['title'], + 'value' : sortedvalue + }) + return sortedList + +""" + Print dict list in a fancy manner + Assume the list is following the format: + [ + { "title": "strkey1", "value" : [ "str", "str2", ... ] }, + { "tilte": "strkey2", "value" : [ "str", "str2", ... ] }, + ... + ] +""" +def dictToAsciiTable(data): + # Get maximum text length to print + lengths = [ len(max([str(i) for i in [column["title"]] + column["value"]], key=len)) for column in data ] + lengths = [50, 50, 50] + lines = [] + sep = "|" + nbElem = len(data[0]['value']) + for i in range(0, nbElem): + line = [] + for c in data: + line.append(c['value'][i]) + lines.append(line) + + headers = [] + for c in data: + headers.append(c['title']) + + # Print headers + toPrint = generateSepLine(lengths) + toPrint += generateLine(headers, lengths, sep) + for i in lines: + toPrint += generateSepLine(lengths) + toPrint += generateLine(i, lengths, sep) + toPrint += generateSepLine(lengths) + return toPrint + +def generateSepLine(lengths): + sepBar = "" + for iSep in lengths: + fieldLength = int(iSep) -1 + sepBar += "+" + "-" * fieldLength + return sepBar + "+\n" + +""" +Parse the line array to split each element in multiline +sublines: +[ + ["line 1 of A", "line 2 of A" ], + ["line 1 of B", "line 2 of B" ], + ["line 1 of C" +] +Transform it to something like: +printables: +[ + ["line 1 of A", "line 1 of B", "line 1 of C", ... ], + ["line 2 of A", " ", "line 2 of C", ... ], + [ ... ] +] +""" +def generateLine(line, lengths, sep): + sublines = [] + for i in range(len(line)): + string = str(line[i]) + width = lengths[i] -1 + sublines.append([string[x:x+width] for x in range(0, len(string), width)]) + + printables = [] + for i in range(max([ len(x) for x in sublines])): + printables.append([]) + for j in range(len(sublines)): + width = lengths[j] -1 + if i < len(sublines[j]): + subfield = '{0: <{1}}'.format(sublines[j][i], width) + else: + subfield = ' ' * width + printables[i].append(subfield.replace('\n', ' ')) + printedLine = "" + for i in printables: + printedLine += "|" + printedLine += "|".join(i) + "|\n" + return printedLine diff --git a/scripts/rca/directive.py b/scripts/rca/directive.py new file mode 100755 index 00000000..75f0d213 --- /dev/null +++ b/scripts/rca/directive.py @@ -0,0 +1,64 @@ +import json +from pprint import pprint +class Directive: + def __init__(self, data): + self.id = data["id"] + self.displayName = data["displayName"] + self.shortDescription = data["shortDescription"] + self.longDescription = data["longDescription"] + self.techniqueName = data["techniqueName"] + self.techniqueVersion = data["techniqueVersion"] + self.priority = data["priority"] + self.system = data["system"] + self.enabled = data["enabled"] + self.parameters = data["parameters"] + self.tags = data["tags"] + self.policyMode = data["policyMode"] + + def toJson(self): + data = { + "id": self.id, + "displayName" : self.displayName, + "shortDescription" : self.shortDescription, + "longDescription" : self.longDescription, + "techniqueName" : self.techniqueName, + "techniqueVersion" : self.techniqueVersion, + "priority" : self.priority, + "system" : self.system, + "enabled" : self.enabled, + "parameters" : self.parameters, + "tags" : self.tags, + "policyMode" : self.policyMode + } + + return json.dumps(data) + + def usedIn(self, rules): + results = [] + for iRule in rules: + if self.id in iRule.directives: + results.append(iRule) + return results + +class GenericVariableDefinitionDirective: + def __init__(self, directive): + self.directive = directive + + def getVariables(self): + variables = [] + for iSection in self.directive.parameters["section"]["sections"]: + for iVar in iSection["section"]["vars"]: + if iVar["var"]["name"] == "GENERIC_VARIABLE_NAME": + name = "generic_variable_definition." + iVar["var"]["value"] + elif iVar["var"]["name"] == "GENERIC_VARIABLE_CONTENT": + value = iVar["var"]["value"] + variables.append({ "name": name, "value": value }) + return variables + + def getValues(self, varName): + variables = self.getVariables() + return [ x["value"] for x in variables if x["name"] == varName ] + + def defines(self, varName): + variables = self.getVariables() + return any([ True for x in variables if x["name"] == varName ]) diff --git a/scripts/rca/gvd.py b/scripts/rca/gvd.py new file mode 100644 index 00000000..cfb01715 --- /dev/null +++ b/scripts/rca/gvd.py @@ -0,0 +1,242 @@ +from directive import GenericVariableDefinitionDirective +from rule import Rule +import common + +class Gvd(): + def __init__(self, name, value, gvdDirective): + self.name = name + self.value = value + self.directive = gvdDirective + +### GET functions ### +""" + Return a list of all GenericVariableDefinitionDirective +""" +def get_all(): + results = [] + for iDirective in common.get_directives(): + if iDirective.techniqueName == "genericVariableDefinition": + gvdDirective = GenericVariableDefinitionDirective(iDirective) + for iVar in gvdDirective.getVariables(): + gvd = Gvd(iVar['name'], iVar['value'], gvdDirective) + results.append(gvd) + return results + +""" + Return the gvdDirective with given uid +""" +def get_gvd(uid): + for iDirective in common.get_directives(): + if iDirective.id == uid: + gvdDirective = GenericVariableDefinitionDirective(iDirective) + results = [] + for iVar in gvdDirective.getVariables(): + results.append(Gvd(iVar['name'], iVar['value'], gvdDirective)) + return results + return None + +""" + Return a list of all GenericVariableDefinitionDirective applied to the target group +""" +def get_defined_in_group(group_id): + applied_to = common.get_directives_applied_to(group_id) + results = [] + for iDirective in applied_to: + if iDirective.techniqueName == "genericVariableDefinition": + gvdDirective = GenericVariableDefinitionDirective(iDirective) + for iVar in gvdDirective.getVariables(): + results.append(Gvd(iVar['name'], iVar['value'], gvdDirective)) + return results + +""" + Return a list of all GenericVariableDefinitionDirective applied in the target rule +""" +def get_defined_in_rule(rule): + if not isinstance(rule, Rule): + rule = next(x for x in common.get_rules() if x.id == rule) + results = [] + directives = common.get_directives() + for iDirId in rule.directives: + matchingDirectives = [ x for x in directives if x.id == iDirId and x.techniqueName == "genericVariableDefinition" ] + for iDirective in matchingDirectives: + gvdDirective = GenericVariableDefinitionDirective(iDirective) + for iVar in gvdDirective.getVariables(): + results.append(Gvd(iVar['name'], iVar['value'], gvdDirective)) + return results + +""" + Return a list of all GenericVariableDefinitionDirective defining the target variable +""" +def get_all_directives_defining(varName): + results = [] + for iDirective in common.get_directives(): + if iDirective.techniqueName == "genericVariableDefinition": + gvdDirective = GenericVariableDefinitionDirective(iDirective) + if gvdDirective.defines(varName): + results.append(gvdDirective) + return results + +### DISPLAY functions ### + +""" + Display a list of GenericvariableDefinitionDirective +""" +def display_gvd_list(results, criterion="Name"): + names = [] + values = [] + displayNames = [] + sortedResults = sorted(results, key=lambda gvd: gvd.name) + for iGvd in sortedResults: + names.append(iGvd.name) + values.append(iGvd.value) + displayNames.append(iGvd.directive.directive.displayName) + toDisplay = [ + { "title": "Name", "value": names }, + { "title": "Value", "value": values }, + { "title": "Directive", "value": displayNames } + ] + sortedToDisplay = common.sortToDisplay(toDisplay, criterion) + print(common.dictToAsciiTable(sortedToDisplay)) + +""" + Display all GenericVariableDefinitionDirective +""" +def display_all(criterion): + results = get_all() + display_gvd_list(results, criterion) + +""" + Display all GenericVariableDefinitionDirective applied to the target group +""" +def display_defined_in_rule(rule, criterion): + results = get_defined_in_rule(rule) + display_gvd_list(results, criterion) + +""" + Display all GenericVariableDefinitionDirective applied to the target group +""" +def display_defined_in_group(group_id, criterion): + results = get_defined_in_group(group_id) + display_gvd_list(results, criterion) + +""" + Display all GenericVariableDefinitionDirective defining the target variable +""" +def display_all_definitions_of(varName, criterion): + results = get_all_directives_defining(varName) + names = [] + values = [] + displayNames = [] + for iDirective in results: + for iVar in iDirective.getVariables(): + if iVar["name"] == varName: + names.append(iVar['name']) + values.append(iVar['value']) + displayNames.append(iDirective.directive.displayName) + toDisplay = [ + { "title": "Name", "value": names }, + { "title": "Value", "value": values }, + { "title": "Directive", "value": displayNames } + ] + print(common.dictToAsciiTable(common.sortToDisplay(toDisplay, criterion))) + +""" + Display a list of all Rules defining a variable multiple times +""" +def display_all_duplicates_in_rules(criterion): + for iRule in common.get_rules(): + allVars = [] + gvdDirectives = get_defined_in_rule(iRule) + # list all variables and store them with their directive + for iGvd in gvdDirectives: + allVars.append(iGvd.name) + + # find all duplicates + results = [] + for iGvd in gvdDirectives: + occ = sum(1 for x in allVars if x == iGvd.name) + if occ > 1: + results.append(iGvd) + if [] != results: + print(iRule.displayName) + display_gvd_list(results, criterion) + +""" + Display a list of all conflicting gvdD applied by group +""" +def display_all_duplicates_in_groups(criterion): + for iGroup in common.get_groups(): + gvdInGroup = get_defined_in_group(iGroup) + allVars = [] + # list all variables and store them with their directive + for iGvd in gvdInGroup: + allVars.append(iGvd.name) + + # find all duplicates + results = [] + for iGvd in gvdInGroup: + occ = sum(1 for x in allVars if x == iGvd.name) + if occ > 1: + results.append(iGvd) + if [] != results: + print(iGroup) + display_gvd_list(results, criterion) + +""" + Display given gvd, referenced by its uuid +""" +def display_given_uuid(uuid, criterion): + display_gvd_list(get_gvd(uuid), criterion) + +""" + Display gvd by number of definition in rules +""" +def display_by_number_of_def_in_rules(criterion='usage'): + names = [] + ids = [] + usage = [] + rules = common.get_rules() + directives = common.get_directives() + for iDirective in directives: + names.append(iDirective.displayName) + ids.append(iDirective.id) + usage.append(len(iDirective.usedIn(rules))) + + toDisplay = [ + { "title": "Name", "value": names }, + { "title": "id", "value": ids }, + { "title": "usage", "value": usage } + ] + print(common.dictToAsciiTable(common.sortToDisplay(toDisplay, criterion))) + print("With a total of %s rules found"%len(rules)) + +""" + Display gvd by number of definition in directives +""" +def display_by_number_of_def_in_directives(criterion='Defined in x directives'): + gvds = get_all() + + results = {} + for gvd in gvds: + if gvd.name not in results: + results[gvd.name] = { + 'nbDirectives': set(), + 'values': set() + } + results[gvd.name]['nbDirectives'].add(gvd.directive) + results[gvd.name]['values'].add(gvd.value) + + names = [] + nbValues = [] + nbDirectives = [] + for k in results.keys(): + names.append(k) + nbValues.append(len(results[k]['values'])) + nbDirectives.append(len(results[k]['nbDirectives'])) + + toDisplay = [ + { "title": "Name", "value": names }, + { "title": "Take x different values", "value": nbValues }, + { "title": "Defined in x directives", "value": nbDirectives } + ] + print(common.dictToAsciiTable(common.sortToDisplay(toDisplay, criterion))) diff --git a/scripts/rca/rca b/scripts/rca/rca new file mode 100755 index 00000000..f000da3a --- /dev/null +++ b/scripts/rca/rca @@ -0,0 +1,57 @@ +#!/usr/bin/python + +""" +Rudder config analyzer + +This is aimed to identify generic_variable_definition usages in the Rudder configuration. +It can find duplicates definitions, on a per rule basis, list all the gvd applied +for a given group or rule and list all directives defining a target gvd. + +gvd stand for generic_variable_definition + +Usage: + rca list-gvd [--sorted-by=] [--for-group=] + rca list-gvd [--sorted-by=] [--for-rule=] + rca list-gvd [--sorted-by=] [--defining=] + rca list-gvd [--sorted-by=] [--duplicates-by-rule] + rca list-gvd [--sorted-by=] [--duplicates-by-group] + rca list-gvd [--sorted-by=] [--uuid=] + rca list-gvd [--sorted-by=] [--by-usage] + rca list-gvd [--sorted-by=] [--by-number-of-directives] + rca list-gvd [--sorted-by=] [--by-number-of-rules] + """ + +import sys +sys.path.insert(0,"/opt/rudder/share/python") +sys.path.insert(0,"/opt/rudder/share/python/rudder-pkg") + +from docopt import docopt +import gvd + +## MAIN +if __name__ == "__main__": + arguments = docopt(__doc__) + if arguments['list-gvd']: + if arguments['--sorted-by']: + criterion = arguments['--sorted-by'] + else: + criterion = "Name" + + if arguments['--for-group']: + gvd.display_defined_in_group(arguments['--for-group'], criterion) + elif arguments['--for-rule']: + gvd.display_defined_in_rule(arguments['--for-rule'], criterion) + elif arguments['--defining']: + gvd.display_all_definitions_of(arguments['--defining'], criterion) + elif arguments['--uuid']: + gvd.display_given_uuid(arguments['--uuid'], criterion) + elif arguments['--duplicates-by-rule']: + gvd.display_all_duplicates_in_rules(criterion) + elif arguments['--duplicates-by-group']: + gvd.display_all_duplicates_in_groups(criterion) + elif arguments['--by-number-of-rules']: + gvd.display_by_number_of_def_in_rules(criterion) + elif arguments['--by-number-of-directives']: + gvd.display_by_number_of_def_in_directives(criterion) + else: + gvd.display_all(criterion) diff --git a/scripts/rca/rule.py b/scripts/rca/rule.py new file mode 100755 index 00000000..fcbaaec5 --- /dev/null +++ b/scripts/rca/rule.py @@ -0,0 +1,51 @@ +class Rule: + def __init__(self, data): + self.id = data["id"] + self.displayName = data["displayName"] + self.shortDescription = data["shortDescription"] + self.longDescription = data["longDescription"] + self.system = data["system"] + self.enabled = data["enabled"] + self.directives = data["directives"] + self.targets = data["targets"] + self.tags = data["tags"] + + """ + return a list containing all groups listed in the include part + of the target section of the current rule + + + Rule api can define the targets in the following manners: + + "targets": [ + "group:2a3a1a8c-a2a0-446f-8347-849ad2d7fb15" + ], + + + or + + "targets": [ + { + "include": { + "or": [ + "group:f0e7d338-4a7b-4451-a64b-f83dd096d00f" + ] + }, + "exclude": { + "or": [] + } + } + ], + + """ + def getIncludeTargets(self): + include = [] + for iTarget in self.targets: + if "include" in iTarget: + for iGroup in iTarget["include"]["or"]: + # remove all prefix in group id such as group:, special:, etc... + include.append(iGroup.split(":", 1)[1]) + else: + include.append(iTarget.split(":", 1)[1]) + return include +