scadtool.py

'''
    scadtool.py: A tool to create and manage libraries for OpenSCAD.

    Copyright (C) 2015  Hauke Thorenz <htho@thorenz.net>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''

if __name__ == "__main__":
    # import statements: We use pythons included batteries!
    import os
    import json
    import scadtoolLib as lib
    import argparse
    import collections

    def cmd_info_handler(args):
        lib.printConsole("PROGRESS: Collecting Information about these sources:\nPROGRESS:         {}\nPROGRESS:     recursively: '{}'\nPROGRESS:     traversing through dirs: '{}'".format(repr(args.INPUT_FILE_OR_DIR), args.recursive, args.traverse_dirs), 1)
        if args.recursive and args.traverse_dirs:
            lib.printConsole("NOTICE: You've set --recursive and --traverse-dirs. This might lead to problems if a file is referenced in a source file and found by traversing.", 1)
        scadLibrary = lib.ScadLibrary(args.INPUT_FILE_OR_DIR, args.recursive, args.traverse_dirs)

        toOutput = list()

        if args.self:
            toOutput.extend(scadLibrary.fileList)
            if args.recursive:
                toOutput.extend(scadLibrary.getReferencedFiles())

        if args.modules:
            toOutput.extend(scadLibrary.getAvailableModules())

        if args.variables:
            toOutput.extend(scadLibrary.getAvailableVariables())

        if args.functions:
            toOutput.extend(scadLibrary.getAvailableFunctions())

        if args.includes:
            toOutput.extend(scadLibrary.getIncludedFiles())

        if args.uses:
            toOutput.extend(scadLibrary.getUsedFiles())

        if args.filter:
            # TODO define a json syntax that allows searching for entities
            # based on information given in the ScadDoc.

            # toOutput.extend(scadLibrary.findEntity(description))
            pass

        outString = ""

        for out in toOutput:
            if args.as_scad:
                if isinstance(out, lib.ScadFile):
                    outString = outString + out.asScad(args.recursive) + "\n" + "\n"
                else:
                    outString = outString + out.asScad() + "\n" + "\n"
            elif args.as_json:
                if isinstance(out, lib.ScadFile):
                    outString = outString + out.asJson() + "\n" + "\n"
                else:
                    outString = outString + out.asJson() + "\n" + "\n"
            elif args.as_dump:

                if isinstance(out, lib.ScadFile):
                    outString = outString + out.asDump(args.recursive) + "\n" + "\n"
                else:
                    outString = outString + out.asDump() + "\n" + "\n"
            else:
                outString = outString + str(out) + "\n" + "\n"

        if args.as_scad:
            outFile = lib.determineOutFile(args.INPUT_FILE_OR_DIR[0], "scad.info.", "scad")
        elif args.as_json:
            outFile = lib.determineOutFile(args.INPUT_FILE_OR_DIR[0], "scad.info.", "json")
        elif args.as_dump:
            outFile = lib.determineOutFile(args.INPUT_FILE_OR_DIR[0], "scad.info.dump.", "scad")
        else:
            outFile = lib.determineOutFile(args.INPUT_FILE_OR_DIR[0], "scad.info.", "txt")

        lib.outputHelper(outString, outFile)

    def cmd_map_handler(args):
        lib.printConsole("PROGRESS: Creating a mapping...", 1)
        if args.input_file is not None:
            lib.printConsole("PROGRESS: Creating the mapping only for entities needed by '{}'".format(args.input_file), 1)
            inputFile = lib.ScadFileFromFile.buildFromFile(path=args.input_file, recursive=False, referencedFromScadFile=None)

            if inputFile.metaDataIsAutoGenerated:
                raise ValueError("'{}' did not have a @filename tag with the correct name. IS THE FILENAME TAG CORRECT? We can't build a library without knowing the dependencies.".format(args.INPUT_FILE))

            lib.printConsole(str(inputFile) + "\n", 1)
            inputFileDependencyNames = list()

            for entity in inputFile.getAvailableEntities():
                for dependency in entity.getDependencies():
                    inputFileDependencyNames.append(dependency.name)

        if os.path.isfile(args.MAPPING):
            with open(args.MAPPING, 'r') as f:
                jsonMapping = json.load(f, object_pairs_hook=collections.OrderedDict)
        else:
            jsonMapping = json.loads(args.MAPPING, object_pairs_hook=collections.OrderedDict)

        lib.printConsole("INFO: JSON-Mapping:" + lib.txt_prefix_each_line(lib.txt_pretty_print(jsonMapping), "    ") + "\n", 2)

        mappingFile = lib.ScadFile()

        for entityType, mapping in jsonMapping.items():
            lib.printConsole("INFO: MAP: entityType='{}'".format(entityType), 2)

            for sourceName, targetDescription in mapping.items():
                if isinstance(targetDescription, str):
                    lib.printConsole("INFO: MAP: targetDescription=str('{}')".format(targetDescription), 2)
                    targetDescription = {"name": targetDescription}
                if isinstance(targetDescription, dict):
                    lib.printConsole("INFO: MAP: targetDescription=dict('{}')".format(str(targetDescription)), 2)
                    targetDescription["targetSignature"] = list()
                    if "name" not in targetDescription.keys():
                        raise ValueError("The mapping for the module '{}' does not have a name. For modules, the 'name' must be specified.".format(sourceName))
                    if "arguments" in targetDescription.keys():
                        arguments = targetDescription["arguments"]
                        if isinstance(arguments, list):
                            newArguments = dict()
                            for argument in arguments:
                                if not isinstance(argument, str):
                                    raise ValueError("argument must be a json-string.")
                                newArguments[argument] = argument
                            arguments = newArguments
                        if isinstance(arguments, dict):
                            for sourceArgument, targetArgument in arguments.items():
                                if not isinstance(sourceArgument, str):
                                    raise ValueError("sourceArgument must be a json-string.")
                                if not isinstance(targetArgument, str):
                                    raise ValueError("targetArgument must be a json-string.")
                                targetDescription["targetSignature"].append("{}={}".format(targetArgument, sourceArgument))
                        else:
                            raise ValueError("arguments must either be a json-object or a json-array.")
                        targetDescription["arguments"] = arguments
                    else:
                        targetDescription["arguments"] = dict()
                else:
                    raise ValueError("targetDescription must either be a json-string or a json-object.")

                if entityType == "modules":
                    meta = lib.ScadDoc("A Mapping from '{}' to '{}'".format(sourceName, targetDescription["name"]), lib.ScadModule)
                    meta.add("module-dependency", targetDescription["name"])
                    for sourceArgument in targetDescription["arguments"].keys():
                        meta.add("argument", sourceArgument)
                    arguments = ", ".join(targetDescription["arguments"].keys())
                    content = "{}({});".format(targetDescription["name"], ", ".join(targetDescription["targetSignature"]))
                    entity = lib.ScadModule(name=sourceName, arguments=arguments, content=content, metaData=meta)
                elif entityType == "functions":
                    meta = lib.ScadDoc("A Mapping from '{}' to '{}'".format(sourceName, targetDescription["name"]), lib.ScadFunction)
                    meta.add("function-dependency", targetDescription["name"])
                    for sourceArgument in targetDescription["arguments"].keys():
                        meta.add("argument", sourceArgument)
                    arguments = ", ".join(targetDescription["arguments"].keys())
                    content = "{}({});".format(targetDescription["name"], ", ".join(targetDescription["targetSignature"]))
                    entity = lib.ScadFunction(name=sourceName, arguments=arguments, content=content, metaData=meta)
                elif entityType == "variables":
                    meta = lib.ScadDoc("A Mapping from '{}' to '{}'".format(sourceName, targetDescription["name"]), lib.ScadVariable)
                    meta.add("variable-dependency", targetDescription["name"])
                    entity = lib.ScadVariable(name=sourceName, value=targetDescription["name"], metaData=meta)

            # if there is an input file and the created entity is needed for the input file
                if args.input_file is None or entity.name in inputFileDependencyNames:
                    mappingFile.addDefinedEntity(entity)

        lib.printConsole("INFO: Mapping-Entities:\n" + lib.txt_prefix_each_line(lib.txt_pretty_print(mappingFile.getDefinedEntities()), "    ") + "\n", 2)
        outFileName = lib.determineOutFile(args.input_file, "mapping", ".scad")

        mappingFile.metaData = lib.ScadDoc("@filename: " + str(outFileName), lib.ScadFile, None)
        lib.outputHelper(mappingFile.asScad(recursive=False, excludeList=[], dummiesFirst=False), outFileName)

    def cmd_build_handler(args):
        lib.printConsole("PROGRESS: Building a library based on these sources:\nPROGRESS:         {}\nPROGRESS:     recursively: '{}'\nPROGRESS:     traversing through dirs: '{}'".format(repr(args.LIBRARY_FILE_OR_DIR), args.recursive, args.traverse_dirs), 1)
        if args.recursive and args.traverse_dirs:
            lib.printConsole("NOTICE: You've set --recursive and --traverse-dirs. This might lead to problems if a file is referenced in a source file and found by traversing.", 1)
        scadLibrary = lib.ScadLibrary(args.LIBRARY_FILE_OR_DIR, args.recursive, args.traverse_dirs)

        lib.printConsole("PROGRESS: Building the library for: '{}'".format(repr(args.INPUT_FILE)), 1)
        inputFile = lib.ScadFileFromFile.buildFromFile(path=args.INPUT_FILE, recursive=True, referencedFromScadFile=None)

        if inputFile.metaDataIsAutoGenerated:
            raise ValueError("'{}' did not have a @filename tag with the correct name. IS THE FILENAME TAG CORRECT? We can't build a library without knowing the dependencies.".format(args.INPUT_FILE))

        lib.printConsole("PROGRESS: Checking the internal structure of the input file. Trying to resolve dependencies internally...", 1)

        dependencyTree, unresolvedDependencies = inputFile.getDependencyTreeAndUnresolvedDependencies([inputFile])
        lib.printConsole("INFO: Internal Dependency Tree:\n" + lib.txt_pretty_print(dependencyTree, kvsep=" depends on: "), 2)
        lib.printConsole("INFO: Internally Unresolved Dependencies:\n" + lib.txt_pretty_print(unresolvedDependencies), 2)

        if unresolvedDependencies:  # unresolvedDependencies is not empty
            lib.printConsole("PROGRESS: Resolving the dependencies by searching the library...", 2)
            t, u = scadLibrary.findResolutions(unresolvedDependencies)
            if t is not None:
                dependencyTree.update(t)
            unresolvedDependencies = u

        if dependencyTree is None:
            lib.printConsole("""\nWARNING: The dependency tree is empty!
    This means NONE of the defined dependencies could be resolved.

    Possible Reason 0: There are no models for the given entities.
        That would be sad...

    Possible Reason 1: No dependencies are defined.
        Is there a @module-dependency tag in the input file?

    Possible Reason 2: The library is empty
        Often the --traverse (-t) flag is forgotten. This flag makes
        sure sub-directories are used to create the library.

    Possible Reason 3: The mapping is missing
        Is there a file containing the mapping to the module names used
        in the library?

    Possible Reason 4: Some includes are missing.
        I once forgot t include the file that defines the model and
        therefore all the dependencies.

    Dummies will be created...""", 1)
            dependencyTree = dict()

        lib.printConsole("INFO: Complete Dependency Tree:\n" + lib.txt_pretty_print(dependencyTree, kvsep=" depends on: "), 2)

        neededEntities = lib.ScadLibrary.reduceRedundanciesInDependencyTree(dependencyTree)

        if len(unresolvedDependencies) > 0:
            lib.printConsole("INFO: Still Unresolved Dependencies:\n" + lib.txt_pretty_print(unresolvedDependencies), 2)
            dummyResolutions = list()
            if not args.dont_create_dummies:
                lib.printConsole("INFO: Creating Dummies for the Unresolved Dependencies", 2)
                for dependency in unresolvedDependencies:
                    dummyResolutions.append(dependency.getDummyResolution())
            lib.printConsole(lib.txt_prefix_each_line(lib.txt_pretty_print(dummyResolutions), "    "), 3)
            neededEntities = neededEntities + dummyResolutions
            neededEntities = list(set(neededEntities))  # should be unnecessary as there should be no duplicates.

        # remove entities that are defined in the input file
        neededEntities = list(filter(lambda entity: entity not in inputFile.getAvailableEntities(), neededEntities))
        lib.printConsole("INFO: Entities in library:\n" + lib.txt_prefix_each_line(lib.txt_pretty_print(neededEntities), "    "), 2)

        outFileName = lib.determineOutFile(args.INPUT_FILE, "lib.", "scad")
        outScadFile = lib.ScadFile(definedEntities=neededEntities)
        if outFileName is not None:
            outScadFile.metaData.add("filename", outFileName)
        outString = outScadFile.asScad(dummiesFirst=True)

        lib.outputHelper(outString, outFileName)

    def cmd_compile_handler(args):
        lib.printConsole("PROGRESS: Compiling all references in '{}' to a single file".format(args.INPUT_FILE), 1)
        inputFile = lib.ScadFileFromFile.buildFromFile(path=args.INPUT_FILE, recursive=True, referencedFromScadFile=None)
        outFileName = lib.determineOutFile(args.INPUT_FILE, "comp.", "scad")
        outString = inputFile.asDump(recursive=True)
        lib.outputHelper(outString, outFileName)

    # Argument parsing
    parser = argparse.ArgumentParser(description="Collect and Extract Information, Manipulate and Compile .scad Files or Collections of .scad Files.")

    parser.add_argument("-v", "--verbose", action="count", default=0, help="-v -vv- -vvv increase output verbosity")
    parser.add_argument("-q", "--quiet", action="store_true", help="suppress any output except for final results.")
    parser.add_argument('-V', '--version', action='version', version="%(prog)s " + str(lib.VERSION))

    subparsers = parser.add_subparsers(dest="cmd")
    parser_info = subparsers.add_parser("info", description="Show information about the given file or set of files. You may get information about a single file or whole directories (library).")
    parser_info_group_input = parser_info.add_argument_group(title="input", description="How to handle the input files.")
    parser_info_group_input.add_argument("INPUT_FILE_OR_DIR", nargs="+", help="The files/directories that should be searched.")
    parser_info_group_input.add_argument("-t", "--traverse-dirs", action="store_true", help="If a directory is given traverse through the sub directories. (This is what you probably want to do if you are extracting information from a library file structure.)  You probably don't want to combine this with --recursive")
    parser_info_group_input.add_argument("-r", "--recursive", action="store_true", help="look for information recursively (look in included and used files). You probably don't want to combine this with --traverse-dirs")

    parser_info_group_output = parser_info.add_argument_group(title="output", description="What should the output look line?")
    parser_info_group_output.add_argument("-o", "--output", nargs="?", default=None, const="", help="write output to an .scad File instead to console. (if not defined further 'foo.scad' becomes 'foo.info.scad'.)")

    parser_info_group_output_override = parser_info_group_output.add_mutually_exclusive_group()
    parser_info_group_output_override.add_argument("--override", action="store_true", help="Override existing output files without asking.")
    parser_info_group_output_override.add_argument("--dont-override", action="store_true", help="Do not override any existing output files - Print to console instead.")
    parser_info_group_output_override.add_argument("--ask", default="true", action="store_true", help="Ask if an existing file should be overwritten. (default)")

    parser_info_group_output_type = parser_info_group_output.add_mutually_exclusive_group()
    parser_info_group_output_type.add_argument("--as-scad", action="store_true", help="give output that can be used in .scad files.")
    parser_info_group_output_type.add_argument("--as-json", action="store_true", help="give output that is json encoded. Useful for creating tools that depend on the data in the library. (NOT IMPLEMENTED YET).")
    parser_info_group_output_type.add_argument("--as-dump", action="store_true", help="dump the relevant sections from the content. If recursive, included or used sections will be copied.")

    parser_info_group_selection = parser_info.add_argument_group(title="selection", description="Which information should be extracted?")
    parser_info_group_selection.add_argument("-s", "--self", action="store_true", help="show information about the file itself.")
    parser_info_group_selection.add_argument("-m", "--modules", action="store_true", help="list the modules in the given file.")
    parser_info_group_selection.add_argument("-v", "--variables", action="store_true", help="list the variables in the given file.")
    parser_info_group_selection.add_argument("-f", "--functions", action="store_true", help="list the functions in the given file.")
    parser_info_group_selection.add_argument("-i", "--includes", action="store_true", help="list the files that are included in this file.")
    parser_info_group_selection.add_argument("-u", "--uses", action="store_true", help="list the files that are used by this file.")

    parser_info_group_filter = parser_info.add_argument_group(title="filter", description="Filter the entities. (NOT IMPLEMENTED YET!)")
    parser_info_group_filter.add_argument("--filter", help="A json string or file, that defines what to look for. (NOT IMPLEMENTED YET!)")
    parser_info_group_filter.add_argument("--with-meta", help="Only show results with the given metadata field. May be defined multiple times in order to limit the amount of results. (NOT IMPLEMENTED YET!)")
    parser_info_group_filter.add_argument("--with-meta-key-value", nargs=2, help="Only show results where the given metadata field has the given value, or item.  May be defined multiple times in order to limit the amount of results. (NOT IMPLEMENTED YET!)")

    parser_map = subparsers.add_parser("map", description="Creates a file that maps between different entity names, using a json encoded mapping file or string.")
    parser_map.add_argument("MAPPING", help="A json file or a json string that specifies name mappings for modules, variables and functions. Simple Example:" + """'{ "modules": { "moduleName" : "implementingModuleName" } }'""")
    parser_map.add_argument("-i", "--input-file", nargs="?", default=None, const="", help="If given, only the entities needed for the modules in this file are mapped.")
    parser_map.add_argument("-o", "--output", nargs="?", default=None, const="", help="write output to an .scad File instead to console. (if not defined further 'foo.scad' becomes 'foo.lib.scad'.)")
    parser_map_group_output_override = parser_map.add_mutually_exclusive_group()
    parser_map_group_output_override.add_argument("--override", action="store_true", help="Override existing output files without asking.")
    parser_map_group_output_override.add_argument("--dont-override", action="store_true", help="Do not override any existing output files - Print to console instead.")
    parser_map_group_output_override.add_argument("--ask", default="true", action="store_true", help="Ask if an existing file should be overwritten. (default)")

    parser_build = subparsers.add_parser("build", description="Builds a Library for a file: Finds all unresolved dependencies in a file and creates a so-called library file, that resolves these dependencies using models from a library (a collection of .scad files).")
    parser_build_group_input = parser_build.add_argument_group(title="input", description="How to handle the input files.")

    parser_build_group_input.add_argument("INPUT_FILE", help="The file to create the library for.")
    parser_build_group_input.add_argument("LIBRARY_FILE_OR_DIR", nargs="+", help="The files/directories that should be searched for the needed entities to create this library.")

    parser_build_group_input.add_argument("-t", "--traverse-dirs", action="store_true", help="If a directory is given traverse through the sub directories to find .scad files.")
    parser_build_group_input.add_argument("-r", "--recursive", action="store_true", help="look for entities recursively (look in included and used files).")

    parser_build_group_output = parser_build.add_argument_group(title="output", description=None)
    parser_build_group_output.add_argument("-o", "--output", nargs="?", default=None, const="", help="write output to an .scad File instead to console. (if not defined further 'foo.scad' becomes 'foo.lib.scad'.)")
    parser_build_group_output_override = parser_build_group_output.add_mutually_exclusive_group()
    parser_build_group_output_override.add_argument("--override", action="store_true", help="Override existing output files without asking.")
    parser_build_group_output_override.add_argument("--dont-override", action="store_true", help="Do not override any existing output files - Print to console instead.")
    parser_build_group_output_override.add_argument("--ask", default="true", action="store_true", help="Ask if an existing file should be overwritten. (default)")
    parser_build_group_output.add_argument("--dont-create-dummies", action="store_true", help="Don't create dummies for unresolved dependencies.")

    parser_compile = subparsers.add_parser("compile", description="Compile the referenced files to a single file. Useful for debugging, when OpenSCAD complains on line numbers you can't know.")
    parser_compile.add_argument("INPUT_FILE", help="The file to compile.")
    parser_compile.add_argument("-o", "--output", nargs="?", default=None, const="", help="write output to an .scad File instead to console. (if not defined further 'foo.scad' becomes 'foo.comp.scad'.)")
    parser_compile_group_output_override = parser_compile.add_mutually_exclusive_group()
    parser_compile_group_output_override.add_argument("--override", action="store_true", help="Override existing output files without asking.")
    parser_compile_group_output_override.add_argument("--dont-override", action="store_true", help="Do not override any existing output files - Print to console instead.")
    parser_compile_group_output_override.add_argument("--ask", default="true", action="store_true", help="Ask if an existing file should be overwritten. (default)")

    args = parser.parse_args()
    lib.args = args

    if args.cmd == "info":
        cmd_info_handler(args)
    elif args.cmd == "map":
        cmd_map_handler(args)
    elif args.cmd == "build":
        cmd_build_handler(args)
    elif args.cmd == "compile":
        cmd_compile_handler(args)
    else:
        print(parser.error("a subcommand is required."))