Merge from mr-miles with the added join dsl operations

Fixed a compile error and detabify

Rhino.Etl.Dsl/EtlDslEngine.cs

@@ -18,6 +18,24 @@ public class EtlDslEngine : DslEngine
     {
         private static readonly DslFactory factory = CreateFactory();
         private readonly IDictionary<string, IList<string>> moduleNameToContainedTypes = new Dictionary<string, IList<string>>();
+        private readonly string[] _namespaces;
+
+        /// <summary>
+        /// Create the ETL DSL engine
+        /// </summary>
+        public EtlDslEngine() :  this(new string[0])
+        { }
+
+        /// <summary>
+        /// Create the ETL DSL engine, registering additional namespaces with the compiler pipeline
+        /// </summary>
+        /// <param name="additionalNamespaces">Additional namespaces to register</param>
+        public EtlDslEngine(IEnumerable<string> additionalNamespaces)
+        {
+            var namespaces = new List<string> {"Rhino.Etl.Core", "Rhino.Etl.Dsl", "Rhino.Etl.Dsl.Macros"};
+            namespaces.AddRange(additionalNamespaces);
+            _namespaces = namespaces.ToArray();
+        }
 
         /// <summary>
         /// Compile the DSL and return the resulting context
@@ -79,10 +97,7 @@ protected override void CustomizeCompiler(BooCompiler compiler, CompilerPipeline
             compiler.Parameters.References.Add(typeof(EtlProcess).Assembly);
             pipeline.Insert(1, new AutoReferenceFilesCompilerStep());
             pipeline.Insert(2, new UseModuleNameAsNamespaceIfMissing());
-            pipeline.Insert(3, new AutoImportCompilerStep(
-                "Rhino.Etl.Core",
-                "Rhino.Etl.Dsl",
-                "Rhino.Etl.Dsl.Macros"));
+            pipeline.Insert(3, new AutoImportCompilerStep(_namespaces));
 
             pipeline.InsertAfter(typeof(MacroAndAttributeExpansion), 
                 new CorrelateTypesToModuleName(moduleNameToContainedTypes));

Rhino.Etl.Dsl/Macros/AbstractClassGeneratorMacro.cs

@@ -223,10 +223,15 @@ protected void MoveConstructorArguments(
             {
                 Expression argument = macro.Arguments[i];
                 BinaryExpression assign;
+                MethodInvocationExpression call;
                 if (TryGetAssignment(argument, out assign))
                 {
                     constructor.Body.Add(assign);
                 }
+                else if(TryGetCall(argument, out call))
+                {
+                    constructor.Body.Add(call);
+                }
                 else
                 {
                     superInvocation.Arguments.Add(argument);
@@ -247,6 +252,31 @@ protected static bool TryGetAssignment(Expression expression, out BinaryExpressi
             return (assign != null && assign.Operator == BinaryOperatorType.Assign);
         }
 
+        /// <summary>
+        /// Tries to get a method call from the expression
+        /// </summary>
+        /// <param name="expression">The expression.</param>
+        /// <param name="call">The method call.</param>
+        /// <returns></returns>
+        protected static bool TryGetCall(Expression expression, out MethodInvocationExpression call)
+        {
+            call = expression as MethodInvocationExpression;
+            return (call != null && call.Target is ReferenceExpression);
+        }
+
+        /// <summary>
+        /// Add a method definition to the resultant class definition
+        /// </summary>
+        /// <param name="macro"></param>
+        /// <param name="method"></param>
+        protected void AddMethodDefinitionToClassDefinition(MacroStatement macro, Method method)
+        {
+            var members = (IList<Method>)macro["members"];
+            if (members == null)
+                macro["members"] = members = new List<Method>();
+            members.Add(method);
+        }
+
         /// <summary>
         /// Get the members collection from this macro
         /// </summary>

Rhino.Etl.Dsl/Macros/ActionMacro.cs

@@ -0,0 +1,54 @@
+using Boo.Lang.Compiler;
+using Boo.Lang.Compiler.Ast;
+using Rhino.Etl.Core;
+
+namespace Rhino.Etl.Dsl.Macros
+{
+    /// <summary>
+    /// The on part of a join, split into two parts, the first is the condition, the second
+    /// is the merge
+    /// </summary>
+    public class ActionMacro : AbstractChildMacro
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="OnMacro"/> class.
+        /// </summary>
+        public ActionMacro()
+            : base("leftjoin","rightjoin","innerjoin","fulljoin")
+        {
+        }
+
+
+        /// <summary>
+        /// Perform the actual expansion of the macro
+        /// </summary>
+        /// <param name="macro">The macro.</param>
+        /// <returns></returns>
+        protected override Statement DoExpand(MacroStatement macro)
+        {
+            if (macro.Arguments.Count != 0)
+            {
+                Errors.Add(CompilerErrorFactory.CustomError(macro.LexicalInfo,"No arguments allowed for action statement"));
+                return null;
+            }
+
+            Method mergeRowsMethod = new Method("MergeRows");
+            mergeRowsMethod.Modifiers = TypeMemberModifiers.Override;
+            mergeRowsMethod.Parameters.Add(new ParameterDeclaration("left", new SimpleTypeReference(typeof(Row).FullName)));
+            mergeRowsMethod.Parameters.Add(new ParameterDeclaration("right", new SimpleTypeReference(typeof(Row).FullName)));
+            CodeBuilder.DeclareLocal(mergeRowsMethod, "row", TypeSystemServices.Map(typeof(Row)));
+            mergeRowsMethod.Body.Add(
+                new BinaryExpression(BinaryOperatorType.Assign,
+                                     new ReferenceExpression("row"),
+                                     new MethodInvocationExpression(
+                                         AstUtil.CreateReferenceExpression(typeof(Row).FullName))
+                    )
+                );
+            mergeRowsMethod.Body.Add(macro.Body);
+            mergeRowsMethod.Body.Add(new ReturnStatement(new ReferenceExpression("row")));
+
+            ParentMethods.Add(mergeRowsMethod);
+            return null;
+        }
+    }
+}

Rhino.Etl.Dsl/Macros/FullJoinMacro.cs

@@ -0,0 +1,17 @@
+using Rhino.Etl.Core.Operations;
+
+namespace Rhino.Etl.Dsl.Macros
+{
+    /// <summary>
+    /// 
+    /// </summary>
+    public class FullJoinMacro : HashJoinMacro
+    {
+        /// <summary>
+        /// 
+        /// </summary>
+        public FullJoinMacro()
+            : base(JoinType.Full)
+        { }
+    }
+}

Rhino.Etl.Dsl/Macros/HashJoinMacro.cs

@@ -0,0 +1,110 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Boo.Lang.Compiler;
+using Boo.Lang.Compiler.Ast;
+using Rhino.Etl.Core.Operations;
+
+namespace Rhino.Etl.Dsl.Macros
+{
+    /// <summary>
+    /// Create a class based on <see cref="JoinOperation"/> and tranform the code
+    /// into a join condition
+    /// </summary>
+    public abstract class HashJoinMacro : AbstractClassGeneratorMacro<JoinOperation>
+    {
+        private readonly JoinType JoinType;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="JoinMacro"/> class.
+        /// </summary>
+        protected HashJoinMacro(JoinType joinType)
+            : base("Initialize")
+        {
+            JoinType = joinType;
+        }
+
+        /// <summary>
+        /// Expands the specified macro
+        /// </summary>
+        /// <param name="macro">The macro.</param>
+        /// <returns></returns>
+        public override Statement Expand(MacroStatement macro)
+        {
+            ICollection<StringLiteralExpression> leftKeys = null;
+            ICollection<StringLiteralExpression> rightKeys = null;
+
+            foreach(var arg in macro.Arguments)
+            {
+                BinaryExpression assignment;
+                if (!TryGetAssignment(arg, out assignment)) continue;
+
+                var rf = assignment.Left as ReferenceExpression;
+                if (rf == null) continue;
+
+                var val = assignment.Right as ArrayLiteralExpression;
+                if (val == null) continue;
+
+                if (rf.Name != "RightKeyColumns" && rf.Name != "LeftKeyColumns") continue;
+
+                var lst = Enumerable.OfType<StringLiteralExpression>(val.Items).ToList();
+                if (rf.Name == "RightKeyColumns") 
+                    rightKeys = lst;
+                else
+                    leftKeys = lst;
+            }
+
+            if (leftKeys==null || leftKeys.Count==0)
+            {
+                Errors.Add(CompilerErrorFactory.CustomError(macro.LexicalInfo, "LeftKeys must be defined for a hash join"));
+                return null;
+            }
+
+            if (rightKeys == null || rightKeys.Count==0)
+            {
+                Errors.Add(CompilerErrorFactory.CustomError(macro.LexicalInfo, "RightKeys must be defined for a hash join"));
+                return null;
+            }
+
+            if (rightKeys.Count != leftKeys.Count)
+            {
+                Errors.Add(CompilerErrorFactory.CustomError(macro.LexicalInfo, "Number of key columns must be the same for both sides of the join"));
+            }
+
+            var method = SetupJoinMethodDefinition(leftKeys, rightKeys);
+            AddMethodDefinitionToClassDefinition(macro, method);
+            return base.Expand(macro);
+        }
+
+        private Method SetupJoinMethodDefinition(IEnumerable<StringLiteralExpression> leftKeys, IEnumerable<StringLiteralExpression> rightKeys)
+        {
+            var method = new Method("SetupJoinConditions");
+            method.Modifiers = TypeMemberModifiers.Override;
+
+            CodeBuilder.DeclareLocal(method, "join", TypeSystemServices.Map(typeof(JoinOperation.JoinBuilder)));
+            ReferenceExpression joinConstructor;
+            switch (JoinType)
+            {
+                case JoinType.Full:
+                    joinConstructor = new ReferenceExpression("FullOuterJoin");
+                    break;
+                case JoinType.Left:
+                    joinConstructor = new ReferenceExpression("LeftJoin");
+                    break;
+                case JoinType.Right:
+                    joinConstructor = new ReferenceExpression("RightJoin");
+                    break;
+                case JoinType.Inner:
+                    joinConstructor = new ReferenceExpression("InnerJoin");
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException();
+            }
+
+            method.Body.Add(new BinaryExpression(BinaryOperatorType.Assign, new ReferenceExpression("join"), joinConstructor));
+            method.Body.Add(new MethodInvocationExpression(new MemberReferenceExpression(new ReferenceExpression("join"), "Left"), leftKeys.OfType<Expression>().ToArray()));
+            method.Body.Add(new MethodInvocationExpression(new MemberReferenceExpression(new ReferenceExpression("join"), "Right"), rightKeys.OfType<Expression>().ToArray()));
+            return method;
+        }
+    }
+}

Rhino.Etl.Dsl/Macros/InnerJoinMacro.cs

@@ -0,0 +1,17 @@
+using Rhino.Etl.Core.Operations;
+
+namespace Rhino.Etl.Dsl.Macros
+{
+    /// <summary>
+    /// 
+    /// </summary>
+    public class InnerJoinMacro : HashJoinMacro
+    {
+        /// <summary>
+        /// 
+        /// </summary>
+        public InnerJoinMacro()
+            : base(JoinType.Inner)
+        { }
+    }
+}

Rhino.Etl.Dsl/Macros/JoinSectionMacro.cs

@@ -1,3 +1,5 @@
+using System.Collections.Generic;
+
 namespace Rhino.Etl.Dsl.Macros
 {
     using Boo.Lang.Compiler;
@@ -14,7 +16,7 @@ public abstract class JoinSectionMacro : AbstractChildMacro
         /// Initializes a new instance of the <see cref="JoinSectionMacro"/> class.
         /// </summary>
         public JoinSectionMacro(string name)
-            : base("join")
+            : base("join", "leftjoin", "rightjoin", "innerjoin", "fulljoin")
         {
             this.name = name;
         }
@@ -27,25 +29,50 @@ public JoinSectionMacro(string name)
         protected override Statement DoExpand(MacroStatement macro)
         {
             Block body = (Block)macro.GetAncestor(NodeType.Block);
+            MacroStatement macroParent = (MacroStatement)macro.GetAncestor(NodeType.MacroStatement);
 
-            if (macro.Body.Statements.Count < 1)
+            if (macro.Body.Statements.Count < 1 && parent.Name=="join")
+            {
+                Errors.Add(CompilerErrorFactory.CustomError(macro.LexicalInfo, "Join " + name + " section must contain at least a single expression statement"));
+                return null;
+            }
+
+            if (macro.Arguments.Count == 0 && parent.Name!="join")
             {
-                Errors.Add(CompilerErrorFactory.CustomError(macro.LexicalInfo, "Join"+name+" section must contain at least a single expression statement"));
+                Errors.Add(CompilerErrorFactory.CustomError(macro.LexicalInfo, "Join " + name + " section must contain at least one key column"));
                 return null;
             }
 
+            if (macro.Arguments.Count > 0)
+            {
+                ArrayLiteralExpression ale = new ArrayLiteralExpression(macro.LexicalInfo);
+                ale.Type = new ArrayTypeReference(CodeBuilder.CreateTypeReference(typeof(string)));
+                foreach (Expression argument in macro.Arguments)
+                {
+                    ReferenceExpression expr = argument as ReferenceExpression;
+                    if (expr == null)
+                    {
+                        Errors.Add(CompilerErrorFactory.CustomError(macro.LexicalInfo,"Join " + name +" section arguments must be reference expressions. Example: " + name + " name, surname"));
+                        return null;
+                    }
+                    ale.Items.Add(new StringLiteralExpression(expr.Name));
+                }
+                var keyExpr = new BinaryExpression(BinaryOperatorType.Assign, new ReferenceExpression(name+"KeyColumns"), ale);
+                macroParent.Arguments.Add(keyExpr);
+            }
+
             foreach (Statement statement in macro.Body.Statements)
             {
                 ExpressionStatement exprStmt = statement as ExpressionStatement;
                 if(exprStmt==null)
                 {
-                    Errors.Add(CompilerErrorFactory.CustomError(macro.LexicalInfo, "Join" + name + " section can only contain expressions"));
+                    Errors.Add(CompilerErrorFactory.CustomError(macro.LexicalInfo, "Join " + name + " section can only contain expressions"));
                     return null;
                 }
                 Expression expr = exprStmt.Expression;
-                MethodInvocationExpression expression = new MethodInvocationExpression(new ReferenceExpression(name), expr);
-                body.Add(new ExpressionStatement(expression));
+                parent.Arguments.Add(new MethodInvocationExpression(new ReferenceExpression(name), expr));
             }
+
             return null;
 
         }

Rhino.Etl.Dsl/Macros/LeftJoinMacro.cs

@@ -0,0 +1,17 @@
+using Rhino.Etl.Core.Operations;
+
+namespace Rhino.Etl.Dsl.Macros
+{
+    /// <summary>
+    /// 
+    /// </summary>
+    public class LeftJoinMacro : HashJoinMacro
+    {
+        /// <summary>
+        /// 
+        /// </summary>
+        public LeftJoinMacro()
+            : base(JoinType.Left)
+        { }
+    }
+}

Rhino.Etl.Dsl/Macros/RightJoinMacro.cs

@@ -0,0 +1,17 @@
+using Rhino.Etl.Core.Operations;
+
+namespace Rhino.Etl.Dsl.Macros
+{
+    /// <summary>
+    /// 
+    /// </summary>
+    public class RightJoinMacro : HashJoinMacro
+    {
+        /// <summary>
+        /// 
+        /// </summary>
+        public RightJoinMacro()
+            : base(JoinType.Right)
+        { }
+    }
+}