Add caching to speed up bin/packwerk check and bin/packwerk update-deprecations by 85-90%

USAGE.md

@@ -77,6 +77,7 @@ Packwerk reads from the `packwerk.yml` configuration file in the root directory.
 | package_paths        | **/                                       | a single pattern or a list of patterns to find package configuration files, see: [Defining packages](#Defining-packages) |
 | custom_associations  | N/A                                       | list of custom associations, if any |
 | parallel             | true                                      | when true, fork code parsing out to subprocesses |
+| cache                | false                                     | when true, caches the results of parsing files |
 
 ### Using a custom ERB parser
 
@@ -100,6 +101,11 @@ end
 Packwerk::Parsers::Factory.instance.erb_parser_class = CustomParser
 ```
 
+## Using the cache
+Packwerk ships with an cache to help speed up file parsing. You can turn this on by setting `cache: true` in `packwerk.yml`.
+
+This will write to `tmp/cache/packwerk`.
+
 ## Validating the package system
 
 There are some criteria that an application must meet in order to have a valid package system. These criteria include having a valid autoload path cache, package definition files, and application folder structure. The dependency graph within the package system also has to be acyclic.

lib/packwerk.rb

@@ -15,6 +15,7 @@ module Packwerk
   autoload :ApplicationValidator
   autoload :AssociationInspector
   autoload :OffenseCollection
+  autoload :Cache
   autoload :Cli
   autoload :Configuration
   autoload :ConstantDiscovery
@@ -36,6 +37,7 @@ module Packwerk
   autoload :ParsedConstantDefinitions
   autoload :Parsers
   autoload :ParseRun
+  autoload :UnresolvedReference
   autoload :Reference
   autoload :ReferenceExtractor
   autoload :ReferenceOffense

lib/packwerk/cache.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+# typed: strict
+
+module Packwerk
+  class Cache
+    extend T::Sig
+
+    class CacheContents < T::Struct
+      extend T::Sig
+
+      const :file_contents_digest, String
+      const :unresolved_references, T::Array[UnresolvedReference]
+
+      sig { returns(String) }
+      def serialize
+        to_json
+      end
+
+      sig { params(serialized_cache_contents: String).returns(CacheContents) }
+      def self.deserialize(serialized_cache_contents)
+        cache_contents_json = JSON.parse(serialized_cache_contents)
+        unresolved_references = cache_contents_json["unresolved_references"].map do |json|
+          UnresolvedReference.new(
+            json["constant_name"],
+            json["namespace_path"],
+            json["relative_path"],
+            Node::Location.new(json["source_location"]["line"], json["source_location"]["column"],)
+          )
+        end
+
+        CacheContents.new(
+          file_contents_digest: cache_contents_json["file_contents_digest"],
+          unresolved_references: unresolved_references,
+        )
+      end
+    end
+
+    CACHE_SHAPE = T.type_alias do
+      T::Hash[
+        String,
+        CacheContents
+      ]
+    end
+
+    sig { params(enable_cache: T::Boolean, cache_directory: Pathname, config_path: String).void }
+    def initialize(enable_cache:, cache_directory:, config_path:)
+      @enable_cache = enable_cache
+      @cache = T.let({}, CACHE_SHAPE)
+      @files_by_digest = T.let({}, T::Hash[String, String])
+      @config_path = config_path
+      @cache_directory = cache_directory
+
+      if @enable_cache
+        create_cache_directory!
+        bust_cache_if_packwerk_yml_has_changed!
+        bust_cache_if_inflections_have_changed!
+      end
+    end
+
+    sig { void }
+    def bust_cache!
+      FileUtils.rm_rf(@cache_directory)
+    end
+
+    sig do
+      params(
+        file_path: String,
+        block: T.proc.returns(T::Array[UnresolvedReference])
+      ).returns(T::Array[UnresolvedReference])
+    end
+    def with_cache(file_path, &block)
+      return block.call unless @enable_cache
+
+      cache_location = @cache_directory.join(digest_for_string(file_path))
+      cache_contents = if cache_location.exist?
+        T.let(CacheContents.deserialize(cache_location.read),
+          CacheContents)
+      end
+      file_contents_digest = digest_for_file(file_path)
+
+      if !cache_contents.nil? && cache_contents.file_contents_digest == file_contents_digest
+        Debug.out("Cache hit for #{file_path}")
+        cache_contents.unresolved_references
+      else
+        Debug.out("Cache miss for #{file_path}")
+        unresolved_references = block.call
+        cache_contents = CacheContents.new(
+          file_contents_digest: file_contents_digest,
+          unresolved_references: unresolved_references,
+        )
+        cache_location.write(cache_contents.serialize)
+        unresolved_references
+      end
+    end
+
+    sig { params(file: String).returns(String) }
+    def digest_for_file(file)
+      digest_for_string(File.read(file))
+    end
+
+    sig { params(str: String).returns(String) }
+    def digest_for_string(str)
+      # MD5 appears to be the fastest
+      # https://gist.github.com/morimori/1330095
+      Digest::MD5.hexdigest(str)
+    end
+
+    sig { void }
+    def bust_cache_if_packwerk_yml_has_changed!
+      bust_cache_if_contents_have_changed(File.read(@config_path), :packwerk_yml)
+    end
+
+    sig { void }
+    def bust_cache_if_inflections_have_changed!
+      bust_cache_if_contents_have_changed(YAML.dump(ActiveSupport::Inflector.inflections), :inflections)
+    end
+
+    sig { params(contents: String, contents_key: Symbol).void }
+    def bust_cache_if_contents_have_changed(contents, contents_key)
+      current_digest = digest_for_string(contents)
+      cached_digest_path = @cache_directory.join(contents_key.to_s)
+      if !cached_digest_path.exist?
+        # In this case, we have nothing cached
+        # We save the current digest. This way the next time we compare current digest to cached digest,
+        # we can accurately determine if we should bust the cache
+        cached_digest_path.write(current_digest)
+        nil
+      elsif cached_digest_path.read == current_digest
+        Debug.out("#{contents_key} contents have NOT changed, preserving cache")
+      else
+        Debug.out("#{contents_key} contents have changed, busting cache")
+        bust_cache!
+        create_cache_directory!
+        cached_digest_path.write(current_digest)
+      end
+    end
+
+    sig { void }
+    def create_cache_directory!
+      FileUtils.mkdir_p(@cache_directory)
+    end
+  end
+
+  class Debug
+    extend T::Sig
+
+    sig { params(out: String).void }
+    def self.out(out)
+      if ENV["DEBUG_PACKWERK_CACHE"]
+        puts(out)
+      end
+    end
+  end
+
+  private_constant :Debug
+end

lib/packwerk/configuration.rb

@@ -34,7 +34,7 @@ def from_packwerk_config(path)
     DEFAULT_EXCLUDE_GLOBS = ["{bin,node_modules,script,tmp,vendor}/**/*"]
 
     attr_reader(
-      :include, :exclude, :root_path, :package_paths, :custom_associations, :config_path
+      :include, :exclude, :root_path, :package_paths, :custom_associations, :config_path, :cache_directory
     )
 
     def initialize(configs = {}, config_path: nil)
@@ -74,7 +74,8 @@ def initialize(configs = {}, config_path: nil)
       @package_paths = configs["package_paths"] || "**/"
       @custom_associations = configs["custom_associations"] || []
       @parallel = configs.key?("parallel") ? configs["parallel"] : true
-
+      @cache_enabled = configs.key?("cache") ? configs["cache"] : false
+      @cache_directory = Pathname.new(configs["cache_directory"] || "tmp/cache/packwerk")
       @config_path = config_path
     end
 
@@ -85,5 +86,9 @@ def load_paths
     def parallel?
       @parallel
     end
+
+    def cache_enabled?
+      !!@cache_enabled
+    end
   end
 end

lib/packwerk/constant_discovery.rb

@@ -4,7 +4,7 @@
 require "constant_resolver"
 
 module Packwerk
-  # Get information about (partially qualified) constants without loading the application code.
+  # Get information about unresolved constants without loading the application code.
   # Information gathered: Fully qualified name, path to file containing the definition, package,
   # and visibility (public/private to the package).
   #
@@ -35,9 +35,9 @@ def package_from_path(path)
     end
 
     # Analyze a constant via its name.
-    # If the name is partially qualified, we need the current namespace path to correctly infer its full name
+    # If the constant is unresolved, we need the current namespace path to correctly infer its full name
     #
-    # @param const_name [String] The constant's name, fully or partially qualified.
+    # @param const_name [String] The unresolved constant's name.
     # @param current_namespace_path [Array<String>] (optional) The namespace of the context in which the constant is
     #   used, e.g. ["Apps", "Models"] for `Apps::Models`. Defaults to [] which means top level.
     # @return [Packwerk::ConstantDiscovery::ConstantContext]

lib/packwerk/file_processor.rb

@@ -13,16 +13,24 @@ def initialize(file:)
       end
     end
 
-    def initialize(node_processor_factory:, parser_factory: nil)
+    sig do
+      params(
+        node_processor_factory: NodeProcessorFactory,
+        cache: Cache,
+        parser_factory: T.nilable(Parsers::Factory)
+      ).void
+    end
+    def initialize(node_processor_factory:, cache:, parser_factory: nil)
       @node_processor_factory = node_processor_factory
+      @cache = cache
       @parser_factory = parser_factory || Packwerk::Parsers::Factory.instance
     end
 
     sig do
       params(file_path: String).returns(
         T::Array[
           T.any(
-            Packwerk::Reference,
+            Packwerk::UnresolvedReference,
             Packwerk::Offense,
           )
         ]
@@ -31,16 +39,21 @@ def initialize(node_processor_factory:, parser_factory: nil)
     def call(file_path)
       return [UnknownFileTypeResult.new(file: file_path)] if parser_for(file_path).nil?
 
-      node = parse_into_ast(file_path)
-      return [] unless node
+      @cache.with_cache(file_path) do
+        node = parse_into_ast(file_path)
+        return [] unless node
 
-      references_from_ast(node, file_path)
+        references_from_ast(node, file_path)
+      end
     rescue Parsers::ParseError => e
       [e.result]
     end
 
     private
 
+    sig do
+      params(node: Parser::AST::Node, file_path: String).returns(T::Array[UnresolvedReference])
+    end
     def references_from_ast(node, file_path)
       references = []
 

lib/packwerk/generators/templates/packwerk.yml.erb

@@ -15,3 +15,9 @@
 # List of custom associations, if any
 # custom_associations:
 # - "cache_belongs_to"
+
+# Whether or not you want the cache enabled (disabled by default)
+# cache: true
+
+# Where you want the cache to be stored (default below)
+# cache_directory: 'tmp/cache/packwerk'

lib/packwerk/node.rb

@@ -5,7 +5,7 @@
 require "parser/ast/node"
 
 module Packwerk
-  # Convenience methods for working with AST nodes.
+  # Convenience methods for working with Parser::AST::Node nodes.
   module Node
     class TypeError < ArgumentError; end
     Location = Struct.new(:line, :column)

lib/packwerk/node_processor.rb

@@ -21,7 +21,7 @@ def initialize(reference_extractor:, filename:)
       params(
         node: Parser::AST::Node,
         ancestors: T::Array[Parser::AST::Node]
-      ).returns(T.nilable(Packwerk::Reference))
+      ).returns(T.nilable(UnresolvedReference))
     end
     def call(node, ancestors)
       return unless Node.method_call?(node) || Node.constant?(node)

lib/packwerk/node_processor_factory.rb

@@ -22,7 +22,6 @@ def for(filename:, node:)
     sig { params(node: AST::Node).returns(::Packwerk::ReferenceExtractor) }
     def reference_extractor(node:)
       ::Packwerk::ReferenceExtractor.new(
-        context_provider: context_provider,
         constant_name_inspectors: constant_name_inspectors,
         root_node: node,
         root_path: root_path,

lib/packwerk/node_visitor.rb

@@ -4,6 +4,9 @@
 module Packwerk
   # Visits all nodes of an AST, processing them using a given node processor.
   class NodeVisitor
+    extend T::Sig
+
+    sig { params(node_processor: NodeProcessor).void }
     def initialize(node_processor:)
       @node_processor = node_processor
     end

lib/packwerk/parsed_constant_definitions.rb

@@ -25,13 +25,13 @@ def local_reference?(constant_name, location: nil, namespace_path: [])
     def self.reference_qualifications(constant_name, namespace_path:)
       return [constant_name] if constant_name.start_with?("::")
 
-      fully_qualified_constant_name = "::#{constant_name}"
+      resolved_constant_name = "::#{constant_name}"
 
       possible_namespaces = namespace_path.each_with_object([""]) do |current, acc|
         acc << "#{acc.last}::#{current}" if acc.last && current
       end
 
-      possible_namespaces.map { |namespace| namespace + fully_qualified_constant_name }
+      possible_namespaces.map { |namespace| namespace + resolved_constant_name }
     end
 
     private
@@ -53,9 +53,9 @@ def collect_local_definitions_from_root(node, current_namespace_path = [])
     end
 
     def add_definition(constant_name, current_namespace_path, location)
-      fully_qualified_constant = [""].concat(current_namespace_path).push(constant_name).join("::")
+      resolved_constant = [""].concat(current_namespace_path).push(constant_name).join("::")
 
-      @local_definitions[fully_qualified_constant] = location
+      @local_definitions[resolved_constant] = location
     end
   end
 end