Add dynamic step elision

This optimization allows Knit to avoid rebuilding a file if, dynamically
during the build, it determines that the prereq has not changed. For
example, if you change a source file by modifying a comment, Knit will
re-run the compiler but may notice that the object file is unchanged and
therefore will not invoke the linker or the rest of the build. This
feature is only enabled when using hash-based detection.

When using the "steps" printer, you may notice that the build may jump
some steps. This is because those steps have been dynamically elided by
this optimization, but the total step counter that runs before the build
cannot know which steps might be elided.

rules/build.go

@@ -104,13 +104,15 @@ func (e *Executor) execNode(n *node) {
 		e.lock.Unlock()
 		return
 	}
-	ood := n.outOfDate(e.db, e.opts.Hash)
+
+	ood := n.outOfDate(e.db, e.opts.Hash, false)
 	if !e.opts.BuildAll && !n.rule.attrs.Linked && ood == UpToDate {
 		n.setDone(e.db, e.opts.NoExec, e.opts.Hash)
 		e.lock.Unlock()
 		return
 	}
 	e.lock.Unlock()
+	// fmt.Println("exec", n.rule.targets)
 
 	for _, p := range n.prereqs {
 		e.execNode(p)
@@ -124,6 +126,20 @@ func (e *Executor) execNode(n *node) {
 			p.wait()
 		}
 
+		e.lock.Lock()
+		// Cannot do dynamic step elision if hashing is disabled.
+		ood := n.outOfDate(e.db, e.opts.Hash, e.opts.Hash)
+		if !e.opts.BuildAll && !n.rule.attrs.Linked && (ood == UpToDate || ood == UpToDateDynamic) {
+			n.setDone(e.db, e.opts.NoExec, e.opts.Hash)
+			if ood == UpToDateDynamic && len(n.rule.recipe) != 0 {
+				log.Println(n.rule.targets, "elided")
+				e.step.Add(1)
+			}
+			e.lock.Unlock()
+			return
+		}
+		e.lock.Unlock()
+
 		if ood == OnlyPrereqs {
 			e.lock.Lock()
 			n.setDone(e.db, e.opts.NoExec, e.opts.Hash)

rules/graph.go

@@ -38,8 +38,8 @@ type node struct {
 	myExpPrereqs []string
 	myOutput     *file
 
-	memoized   bool
-	memoUpdate UpdateReason
+	memoized   [2]bool
+	memoUpdate [2]UpdateReason
 }
 
 type info struct {
@@ -609,6 +609,7 @@ const (
 	Untracked
 	Prereq
 	LinkedUpdate
+	UpToDateDynamic
 )
 
 func (u UpdateReason) String() string {
@@ -639,16 +640,22 @@ func (u UpdateReason) String() string {
 	panic("unreachable")
 }
 
-func (n *node) outOfDate(db *Database, hash bool) UpdateReason {
-	if !n.memoized {
-		n.memoUpdate = n.outOfDateNoMemo(db, hash)
-		n.memoized = true
+func (n *node) outOfDate(db *Database, hash, dynamic bool) UpdateReason {
+	var i int
+	if dynamic {
+		i = 0
+	} else {
+		i = 1
+	}
+	if !n.memoized[i] {
+		n.memoUpdate[i] = n.outOfDateNoMemo(db, hash, dynamic)
+		n.memoized[i] = true
 	}
-	return n.memoUpdate
+	return n.memoUpdate[i]
 }
 
 // returns true if this node should be rebuilt during the build
-func (n *node) outOfDateNoMemo(db *Database, hash bool) UpdateReason {
+func (n *node) outOfDateNoMemo(db *Database, hash bool, dynamic bool) UpdateReason {
 	// rebuild rules are always out of date
 	if n.rule.attrs.Rebuild {
 		return Rebuild
@@ -700,10 +707,13 @@ func (n *node) outOfDateNoMemo(db *Database, hash bool) UpdateReason {
 	// if a prereq is out of date, this rule is out of date
 	order := false
 	for _, p := range n.prereqs {
-		ood := p.outOfDate(db, hash)
+		ood := p.outOfDate(db, hash, false)
 		// if the only prereqs out of date are order-only, then we just run
 		// them but this rule does not need to rebuild
 		if !p.rule.attrs.Order && ood != UpToDate && ood != OnlyPrereqs {
+			if dynamic {
+				return UpToDateDynamic
+			}
 			return Prereq
 		}
 		if ood != UpToDate {
@@ -718,8 +728,8 @@ func (n *node) outOfDateNoMemo(db *Database, hash bool) UpdateReason {
 
 func (n *node) count(db *Database, full, hash bool, counted map[*info]bool) int {
 	s := 0
-	ood := n.outOfDate(db, hash)
-	if !full && n.outOfDate(db, hash) == UpToDate {
+	ood := n.outOfDate(db, hash, false)
+	if !full && ood == UpToDate {
 		return 0
 	}
 	if ood != OnlyPrereqs && len(n.rule.recipe) != 0 {

rules/tool.go

@@ -511,7 +511,7 @@ type StatusTool struct {
 }
 
 func (t *StatusTool) visit(prev UpdateReason, indent string, n *node, visited map[*node]bool) {
-	status := n.outOfDate(t.Db, t.Hash)
+	status := n.outOfDate(t.Db, t.Hash, false)
 	if n.rule.attrs.Linked && status == UpToDate && prev != UpToDate {
 		status = LinkedUpdate
 	}