Context sensitive iterators

src/Search.jl

@@ -6,15 +6,23 @@ using ..Data
 using ..Evaluation
 
 include("utils.jl")
-include("iterators.jl")
-include("priority_enumerator.jl")
+
+include("cfg_enumerator.jl")
+include("cfg_priority_enumerator.jl")
+
+include("csg_enumerator.jl")
+include("csg_priority_enumerator.jl")
+
 include("search_procedure.jl")
 
 export 
   count_expressions,
   ExpressionIterator,
   ContextFreeEnumerator,
-  ContextFreeBFSEnumerator,
+  ContextFreePriorityEnumerator,
+
+  ContextSensitiveEnumerator,
+  ContextSensitivePriorityEnumerator,
 
   search,
 

src/iterators.jl → src/cfg_enumerator.jl

@@ -4,6 +4,7 @@ abstract type ExpressionIterator end
 """
 ContextFreeEnumerator(grammar::Grammar, max_depth::Int, sym::Symbol)
 An iterator over all possible expressions of a grammar up to max_depth with start symbol sym.
+Provides no guarantees on order.
 """
 mutable struct ContextFreeEnumerator <: ExpressionIterator
     grammar::ContextFreeGrammar

src/csg_enumerator.jl

@@ -0,0 +1,224 @@
+"""
+Reduces the set of possible children of a node using the grammar's constraints
+"""
+function propagate_constraints(
+        grammar::ContextSensitiveGrammar, 
+        context::GrammarContext, 
+        child_rules::Vector{Int}
+    )
+    domain = child_rules
+
+    for propagator ∈ grammar.constraints
+        domain = propagate(propagator, context, domain)
+    end
+
+    return domain
+end
+
+mutable struct ContextSensitiveEnumerator <: ExpressionIterator
+    grammar::ContextSensitiveGrammar
+    max_depth::Int
+    sym::Symbol
+end
+
+
+function Base.iterate(iter::ContextSensitiveEnumerator)
+    init_node = RuleNode(0)  # needed for propagating constraints on the root node 
+    init_context = GrammarContext(init_node)
+
+    grammar, sym, max_depth = iter.grammar, iter.sym, iter.max_depth
+
+    # propagate constraints on the root node 
+    sym_rules = [x for x ∈ grammar[sym]]
+    sym_rules = propagate_constraints(grammar, init_context, sym_rules)
+    #node = RuleNode(grammar[sym][1])
+    node = RuleNode(sym_rules[1])
+
+    if isterminal(grammar, node)
+        return (deepcopy(node), node)
+    else
+        context = GrammarContext(node)
+        node, worked = _next_state!(node, grammar, max_depth, context)
+        while !worked
+            # increment root's rule
+            rules = [x for x in grammar[sym]]
+            rules = propagate_constraints(grammar, init_context, rules) # propagate constraints on the root node
+
+            i = something(findfirst(isequal(node.ind), rules), 0)
+            if i < length(rules)
+                node, worked = RuleNode(rules[i+1]), true
+                if !isterminal(grammar, node)
+                    node, worked = _next_state!(node, grammar, max_depth, context)
+                end
+            else
+                break
+            end
+        end
+        return worked ? (deepcopy(node), node) : nothing
+    end
+end
+
+
+function Base.iterate(iter::ContextSensitiveEnumerator, state::RuleNode)
+    grammar, max_depth = iter.grammar, iter.max_depth
+    context = GrammarContext(state)
+    node, worked = _next_state!(state, grammar, max_depth, context)
+
+    while !worked
+        # increment root's rule
+        init_node = RuleNode(0)  # needed for propagating constraints on the root node 
+        init_context = GrammarContext(init_node)
+
+        rules = [x for x ∈ grammar[iter.sym]]
+        rules = propagate_constraints(grammar, init_context, rules)
+
+        i = something(findfirst(isequal(node.ind), rules), 0)
+        if i < length(rules)
+            node, worked = RuleNode(rules[i+1]), true
+            if !isterminal(grammar, node)
+                context = GrammarContext(node)
+                node, worked = _next_state!(node, grammar, max_depth, context)
+            end
+        else
+            break
+        end
+    end
+    return worked ? (deepcopy(node), node) : nothing
+end
+
+"""
+reimplementation of cfg _next_state!
+Change: child expressions are filtered so that the constraints are not violated
+"""
+function _next_state!(node::RuleNode, grammar::ContextSensitiveGrammar, max_depth::Int, context::GrammarContext)
+
+    if max_depth < 1
+        return (node, false) # did not work
+    elseif isterminal(grammar, node)
+        # do nothing
+        if iseval(grammar, node.ind) && (node._val ≡ nothing)  # evaluate the rule
+            node._val = eval(grammar.rules[node.ind].args[2])
+        end
+        return (node, false) # cannot change leaves
+    else # !isterminal
+        # if node is not terminal and doesn't have children, expand every child
+        if isempty(node.children)  
+            if max_depth ≤ 1
+                return (node,false) # cannot expand
+            end
+
+            child_index = 1  # keep track of which child we are processing now (needed for context)
+
+            # build out the node
+            for c in child_types(grammar, node)
+                worked = false
+                i = 0
+                child = RuleNode(0)
+
+                new_context = GrammarContext(context.originalExpr, deepcopy(context.nodeLocation))
+                push!(new_context.nodeLocation, child_index)
+
+                child_rules = [x for x in grammar[c]]  # select all applicable rules
+                child_rules = propagate_constraints(grammar, new_context, child_rules)  # filter out those that violate constraints
+
+                while !worked && i < length(child_rules)
+                    i += 1
+                    child = RuleNode(child_rules[i])
+
+                    if iseval(grammar, child.ind) # if rule needs to be evaluated (_())
+                        child._val = eval(grammar.rules[child.ind].args[2])
+                    end
+
+                    worked = true
+                    if !isterminal(grammar, child)
+                        child, worked = _next_state!(child, grammar, max_depth-1, new_context)
+                    end
+                end
+                if !worked
+                    return (node, false) # did not work
+                end
+                push!(node.children, child)
+
+                child_index += 1
+            end
+            return (node, true)
+        else # not empty
+            # make one change, starting with rightmost child
+            worked = false
+            child_index = length(node.children) + 1
+            while !worked && child_index > 1
+                child_index -= 1
+                child = node.children[child_index]
+
+                new_context = GrammarContext(context.originalExpr, deepcopy(context.nodeLocation))
+                push!(new_context.nodeLocation, child_index)
+
+                child, child_worked = _next_state!(child, grammar, max_depth-1, new_context)
+                while !child_worked
+                    child_type = return_type(grammar, child)
+
+                    child_rules = [x for x in grammar[child_type]]  # get all applicable rules
+                    child_rules = propagate_constraints(grammar, new_context, child_rules)  # filter ones that violate constraints
+
+
+                    i = something(findfirst(isequal(child.ind), child_rules), 0)
+                    if i < length(child_rules)
+                        child_worked = true
+                        child = RuleNode(child_rules[i+1])
+
+                        # node needs to be evaluated
+                        if iseval(grammar, child.ind)
+                            child._val = eval(grammar.rules[child.ind].args[2])
+                        end
+
+                        if !isterminal(grammar, child)
+                            child, child_worked = _next_state!(child, grammar, max_depth-1, new_context)
+                        end
+                        node.children[child_index] = child
+                    else
+                        break
+                    end
+                end
+
+                if child_worked
+                    worked = true
+
+                    # reset remaining children
+                    for child_index2 in child_index+1 : length(node.children)
+                        c = child_types(grammar, node)[child_index2]
+                        worked = false
+                        i = 0
+                        child = RuleNode(0)
+
+                        new_context = GrammarContext(context.originalExpr, deepcopy(context.nodeLocation))
+                        push!(new_context.nodeLocation, child_index2)
+
+                        child_rules = [x for x in grammar[c]]  # take all applicable rules
+                        child_rules = propagate_constraints(grammar, new_context, child_rules)  # remove ones that violate constraints
+
+
+                        while !worked && i < length(child_rules)
+                            i += 1
+                            child = RuleNode(child_rules[i])
+
+                            if iseval(grammar, child.ind)
+                                child._val = eval(grammar.rules[child.ind].args[2])
+                            end
+
+                            worked = true
+                            if !isterminal(grammar, child)
+                                child, worked = _next_state!(child, grammar, max_depth-1, new_context)
+                            end
+                        end
+                        if !worked
+                            break
+                        end
+                        node.children[child_index2] = child
+                    end
+                end
+            end
+
+            return (node, worked)
+        end
+    end
+end

src/csg_priority_enumerator.jl

@@ -0,0 +1,129 @@
+"""
+Enumerates a context-free grammar up to a given depth in a breadth-first order.
+This means that smaller programs are returned before larger programs.
+"""
+mutable struct ContextSensitivePriorityEnumerator <: ExpressionIterator 
+    grammar::ContextSensitiveGrammar
+    max_depth::Int
+    # Assigns a priority to a (partial or complete) tree.
+    priority_function::Function
+    # Expands a partial tree.
+    expand_function::Function
+    sym::Symbol
+end 
+
+
+function Base.iterate(iter::ContextSensitivePriorityEnumerator)
+    # Priority queue with number of nodes in the program
+    pq :: PriorityQueue{RuleNode, Union{Real, Tuple{Vararg{Real}}}} = PriorityQueue()
+
+    grammar, max_depth, sym = iter.grammar, iter.max_depth, iter.sym
+    priority_function, expand_function = iter.priority_function, iter.expand_function
+
+    init_node = RuleNode(0)
+    init_context = GrammarContext(init_node)
+
+    rules = [x for x ∈ grammar[sym]]
+    rules = propagate_constraints(grammar, init_context, rules)
+
+    for r ∈ rules
+        enqueue!(pq, RuleNode(r), 1)
+    end
+    return _find_next_complete_tree(grammar, max_depth, priority_function, expand_function, pq)
+end
+
+
+function Base.iterate(iter::ContextSensitivePriorityEnumerator, pq::DataStructures.PriorityQueue)
+    grammar, max_depth = iter.grammar, iter.max_depth
+    priority_function, expand_function = iter.priority_function, iter.expand_function
+    return _find_next_complete_tree(grammar, max_depth, priority_function, expand_function, pq)
+end
+
+
+"""
+Takes a priority queue and returns the smallest AST from the grammar it can obtain from the
+queue or by (repeatedly) expanding trees that are in the queue.
+Returns nothing if there are no trees left within the depth limit.
+"""
+function _find_next_complete_tree(grammar::ContextSensitiveGrammar, max_depth::Int, priority_function::Function, expand_function::Function, pq::PriorityQueue)
+    while length(pq) ≠ 0
+        (tree, priority_value) = dequeue_pair!(pq)
+        expanded_trees = expand_function(tree, grammar, max_depth - 1, GrammarContext(tree))
+        if expanded_trees ≡ nothing
+            # Current tree is complete, it can be returned
+            return (tree, pq)
+        else
+            # Either the current tree can't be expanded due to depth 
+            # limit (no expanded trees), or the expansion was successful. 
+            # We add the potential expanded trees to the pq and move on to 
+            # the next tree in the queue.
+            for expanded_tree ∈ expanded_trees
+                # Pass the local scope to the function for calculating the priority 
+                enqueue!(pq, expanded_tree, priority_function(expanded_tree, priority_value))
+            end
+        end
+    end
+    return nothing
+end
+
+"""
+Recursive expand function used in multiple enumeration techniques.
+Expands one hole/undefined leaf of the given RuleNode tree.
+The first hole found using a DFS is expanded first.
+Returns list of new trees when expansion was succesfull.
+Returns nothing if tree is already complete (contains no holes).
+Returns empty list if the tree is partial (contains holes), 
+    but they couldn't be expanded because of the depth limit.
+"""
+function _expand(node::RuleNode, grammar::ContextSensitiveGrammar, max_depth::Int, context::GrammarContext, expand_heuristic::Function=bfs_expand_heuristic)
+    # Find any hole. Technically, the type of search doesn't matter.
+    # We use recursive DFS for memory efficiency, since depth is limited.
+    if grammar.isterminal[node.ind]
+        return nothing
+    elseif max_depth ≤ 0
+        return []
+    end
+
+    childtypes = grammar.childtypes[node.ind]
+    # This node doesn't have holes, check the children
+    if length(childtypes) == length(node.children)
+        for (child_index, child) ∈ enumerate(node.children)
+            child_context = GrammarContext(context.originalExpr, deepcopy(context.nodeLocation))
+            push!(child_context.nodeLocation, child_index)
+            expanded_child_trees = _expand(child, grammar, max_depth - 1, child_context, expand_heuristic)
+            if expanded_child_trees ≡ nothing
+                # Subtree is already complete
+                continue
+            elseif expanded_child_trees == []
+                # There is a hole that can't be expanded further, so we cannot make this 
+                # tree complete anymore. 
+                return []
+            else
+                # Hole was found and expanded
+                nodes = []
+                for expanded_tree ∈ expanded_child_trees
+                    # Copy other children of the current node
+                    children = deepcopy(node.children)
+                    # Update the child we are expanding
+                    children[child_index] = expanded_tree
+                    push!(nodes, RuleNode(node.ind, children))
+                end
+                return nodes
+            end
+
+        end
+    else # This node has an unfilled hole
+        child_type = childtypes[length(node.children) + 1]
+        nodes = []
+
+        for rule_index ∈ expand_heuristic(propagate_constraints(grammar, context, grammar[child_type]))
+            # Copy existing children of the current node
+            children = deepcopy(node.children)
+            # Add the child we are expanding
+            push!(children, RuleNode(rule_index))
+            push!(nodes, RuleNode(node.ind, children))
+        end
+        return nodes
+    end
+end
+