Merge pull request #57 from genuinelucifer/parallel

Adding parallel processing of many files.

README.markdown

@@ -44,6 +44,12 @@ For translating Objective-C headers add the `-ObjC` flag.
 
     $ dstep Foo.h -o Foo.d -ObjC
 
+For translating multiple files at once, simply pass all the files to dstep.
+In this case though, `-o` (if given) would point to output directory name.
+The directory will be created if it doesn't exist.
+
+    $ dstep Foo1.h Foo2.h Foo3.h .... FooN.h -o ./outputDirectory/
+
 Use `-h` for usage information. Any flags recognized by Clang can be used.
 
 ## Limitations/Known issues

changelog.md

@@ -7,6 +7,7 @@
 * Basic unit tests were added
 * Most of cucumber tests was replaced with D-based tests.
 * Statements are translated in original 'C' order now.
+* Multiple input files can be processed in different threads.
 
 ## Version 0.2.1
 ### New/Changed Features

dstep/driver/Application.d

@@ -8,10 +8,13 @@ module dstep.driver.Application;
 
 import std.getopt;
 import std.stdio : writeln, stderr;
-import Path = std.path : setExtension;
+import Path = std.path;
+import std.file;
+import std.parallelism;
 
 import DStack = dstack.application.Application;
 
+import mambo.arguments.Arguments;
 import mambo.core._;
 import mambo.util.Singleton;
 import mambo.util.Use;
@@ -34,12 +37,6 @@ class Application : DStack.Application
 
     private
     {
-        string[] inputFiles;
-
-        Index index;
-        TranslationUnit translationUnit;
-        DiagnosticVisitor diagnostics;
-
         Language language;
         string[] argsToRestore;
         bool helpFlag;
@@ -52,24 +49,39 @@ class Application : DStack.Application
 
         if (arguments.argument.input.hasValue)
         {
-            inputFiles ~= arguments.argument.input;
-            startConversion(inputFiles.first);
+            auto inputFiles = arguments.argument.input.values[1 .. $];
+            string outputDir = null;
+            if (arguments.output != "" && inputFiles.length > 0)
+            {
+                outputDir = arguments.output;
+                if (!exists(outputDir) || !outputDir.isDir)
+                    mkdirRecurse(outputDir);
+            }
+            foreach(string fileName; inputFiles)
+            {
+                auto conversionTask = task!startParsingFile(language, argsToRestore, fileName, true,
+                                                            arguments, compiler, compilerArgs, outputDir);
+                conversionTask.executeInNewThread();
+            }
+
+            startParsingFile(language, argsToRestore, arguments.argument.input.values[0], false, arguments,
+                             compiler, compilerArgs, outputDir);
         }
 
         else
-            startConversion("");
+            startParsingFile(language, argsToRestore, "", false, arguments, compiler, compilerArgs, null);
     }
 
     protected override void setupArguments ()
     {
         arguments.sloppy = true;
         arguments.passThrough = true;
 
-        arguments.argument("input", "input files");
+        arguments.argument("input", "input files").params(1, int.max);
 
         arguments('o', "output", "Write output to")
             .params(1)
-            .defaults(&defaultOutputFilename);
+            .defaults("");
 
         arguments('x', "language", "Treat subsequent input files as having type <language>")
             .params(1)
@@ -79,55 +91,15 @@ class Application : DStack.Application
         arguments("objective-c", "Treat source input file as Objective-C input.");
     }
 
-private:
-
-    string defaultOutputFilename ()
+    static void startParsingFile(Language lang, string[] argsToRestore, string fileName, bool createOutputFileName,
+                                 Arguments args, Compiler comp, string[] compilerArguments, string outputDir)
     {
-        return Path.setExtension(arguments.argument.input, "d");
+        auto parseFile = ParseFile(lang, argsToRestore, fileName, createOutputFileName,
+                                   args, comp, compilerArguments, outputDir);
+        parseFile.startConversion();
     }
 
-    void startConversion (string file)
-    {
-        if (arguments["objective-c"])
-            argsToRestore ~= "-ObjC";
-
-        index = Index(false, false);
-        translationUnit = TranslationUnit.parse(
-            index, 
-            file, 
-            compilerArgs,
-            compiler.extraHeaders);
-
-        // hope that the diagnostics below handle everything
-        // if (!translationUnit.isValid)
-        //     throw new DStepException("An unknown error occurred");
-
-        diagnostics = translationUnit.diagnostics;
-
-        scope (exit)
-            clean;
-
-        if (handleDiagnostics && file.any)
-        {
-            Translator.Options options;
-            options.outputFile = arguments.output;
-            options.language = language;
-
-            auto translator = new Translator(translationUnit, options);
-            translator.translate;
-        }
-    }
-
-    void clean ()
-    {
-        translationUnit.dispose;
-        index.dispose;
-    }
-
-    bool anyErrors ()
-    {
-        return diagnostics.length > 0;
-    }
+private:
 
     void handleArguments ()
     {
@@ -186,22 +158,111 @@ private:
         return remainingArgs ~ extraArgs;
     }
 
-    bool handleDiagnostics ()
+    struct ParseFile
     {
-        bool translate = true;
+        private
+        {
+            string inputFile;
+            string outputFile;
+            string outputDirectory;
+            Index index;
+            TranslationUnit translationUnit;
+            DiagnosticVisitor diagnostics;
+
+            Language language;
+            string[] argsToRestore;
+            Compiler compiler;
+            string[] compilerArgs;
+            Arguments arguments;
+        }
 
-        foreach (diag ; diagnostics)
+        this(Language language, string[] argsToRestore, string fileName, bool createOutputFileName, Arguments arguments,
+             Compiler compiler, string[] compilerArgs, string outputDir)
         {
-            auto severity = diag.severity;
+            this.language = language;
+            this.argsToRestore = argsToRestore.dup;
+            this.arguments = arguments;
+            this.compiler = compiler;
+            this.compilerArgs = compilerArgs.dup;
+
+            inputFile = fileName;
+
+            if (outputDir != null)
+                outputFile = Path.buildPath(outputDir, defaultOutputFilename());
+            else if (!createOutputFileName && arguments.output != "")
+                outputFile = arguments.output;
+            else
+                outputFile = defaultOutputFilename();
+        }
 
-            with (CXDiagnosticSeverity)
-                if (translate)
-                    translate = !(severity == CXDiagnostic_Error || severity == CXDiagnostic_Fatal);
+        void startConversion ()
+        {
+            if (arguments["objective-c"])
+                argsToRestore ~= "-ObjC";
+            index = Index(false, false);
+            translationUnit = TranslationUnit.parse(
+                index, 
+                inputFile, 
+                compilerArgs,
+                compiler.extraHeaders);
+
+            // hope that the diagnostics below handle everything
+            // if (!translationUnit.isValid)
+            //     throw new DStepException("An unknown error occurred");
+
+            diagnostics = translationUnit.diagnostics;
+
+            if (handleDiagnostics && exists(inputFile))
+            {
+                Translator.Options options;
+                options.outputFile = outputFile;
+                options.language = language;
+
+                auto translator = new Translator(translationUnit, options);
+                translator.translate;
+            }
+        }
 
-            writeln(stderr, diag.format);
+        ~this()
+        {
+            clean();
         }
 
-        return translate;
+    private:
+
+        string defaultOutputFilename ()
+        {
+            return Path.setExtension(Path.baseName(inputFile), "d");
+        }
+
+        void clean ()
+        {
+            translationUnit.dispose;
+            index.dispose;
+        }
+
+        bool anyErrors ()
+        {
+            return diagnostics.length > 0;
+        }
+
+        bool handleDiagnostics ()
+        {
+            bool translate = true;
+
+            foreach (diag ; diagnostics)
+            {
+                auto severity = diag.severity;
+
+                with (CXDiagnosticSeverity)
+                    if (translate)
+                        translate = !(severity == CXDiagnostic_Error || severity == CXDiagnostic_Fatal);
+
+                writeln(stderr, diag.format);
+            }
+
+            return translate;
+        }
     }
 
     override protected void showHelp ()
@@ -213,6 +274,7 @@ private:
         println();
         println("Options:");
         println("    -o, --output <file>          Write output to <file>.");
+        println("    -o, --output <directory>     Write all the files to <directory>, in case of multiple input files.");
         println("    -ObjC, --objective-c         Treat source input file as Objective-C input.");
         println("    -x, --language <language>    Treat subsequent input files as having type <language>.");
         println("    -h, --help                   Show this message and exit.");

test_files/multiThreadTest1.d

@@ -0,0 +1,3 @@
+extern (C):
+
+void func1 ();

test_files/multiThreadTest1.h

@@ -0,0 +1 @@
+void func1(void);

test_files/multiThreadTest2.d

@@ -0,0 +1,3 @@
+extern (C):
+
+void func2 ();

test_files/multiThreadTest2.h

@@ -0,0 +1 @@
+void func2(void);