Add coverage for tested API operations

CHANGELOG.md

@@ -7,6 +7,31 @@ and this project adheres to [Semantic Versioning].
 
 ## [Unreleased]
 
+### Added
+
+- Add coverage for tested API operations. ([@skryukov])
+
+    ```ruby
+
+    # spec/rails_helper.rb
+
+    RSpec.configure do |config|
+      # To enable coverage, pass `coverage: :report` option,
+      # and to raise an error when an operation is not covered, pass `coverage: :strict` option:
+      config.include Skooma::RSpec[Rails.root.join("docs", "openapi.yml"), coverage: :report], type: :request
+    end
+    ```
+
+    ```shell
+    $ bundle exec rspec
+    # ...
+    OpenAPI schema /openapi.yml coverage report: 110 / 194 operations (56.7%) covered.
+    Uncovered paths:
+    GET /api/uncovered 200
+    GET /api/partially_covered 403
+    # ...
+    ```
+
 ## [0.3.0] - 2024-04-09
 
 ### Changed
@@ -39,16 +64,16 @@ and this project adheres to [Semantic Versioning].
 
 - Add support for APIs mounted under a path prefix. ([@skryukov])
 
-```ruby
-# spec/rails_helper.rb
-
-RSpec.configure do |config|
-  # ...
-  path_to_openapi = Rails.root.join("docs", "openapi.yml")
-  # pass path_prefix option if your API is mounted under a prefix:
-  config.include Skooma::RSpec[path_to_openapi, path_prefix: "/internal/api"], type: :request
-end
-```
+    ```ruby
+    # spec/rails_helper.rb
+    
+    RSpec.configure do |config|
+      # ...
+      path_to_openapi = Rails.root.join("docs", "openapi.yml")
+      # pass path_prefix option if your API is mounted under a prefix:
+      config.include Skooma::RSpec[path_to_openapi, path_prefix: "/internal/api"], type: :request
+    end
+    ```
 
 ### Changed
 

README.md

@@ -51,6 +51,10 @@ RSpec.configure do |config|
 
   # OR pass path_prefix option if your API is mounted under a prefix:
   config.include Skooma::RSpec[path_to_openapi, path_prefix: "/internal/api"], type: :request
+
+  # To enable coverage, pass `coverage: :report` option,
+  # and to raise an error when an operation is not covered, pass `coverage: :strict` option:
+  config.include Skooma::RSpec[path_to_openapi, coverage: :report], type: :request
 end
 ```
 

examples/minitest.rb

@@ -18,7 +18,8 @@
 
 describe TestApp do
   include Rack::Test::Methods
-  include Skooma::Minitest[File.join(__dir__, "openapi.yml")]
+
+  include Skooma::Minitest[File.join(__dir__, "openapi.yml"), coverage: :report]
 
   def app
     TestApp["bar"]

examples/rails_app/spec/rails_helper.rb

@@ -16,6 +16,6 @@
 
   # You can use different RSpec filters if you want to test different API descriptions.
   # Check RSpec's config.define_derived_metadata for better UX.
-  config.include Skooma::RSpec[bar_openapi, path_prefix: "/bar"], :bar_api
-  config.include Skooma::RSpec[baz_openapi, path_prefix: "/baz"], :baz_api
+  config.include Skooma::RSpec[bar_openapi, path_prefix: "/bar", coverage: :strict], :bar_api
+  config.include Skooma::RSpec[baz_openapi, path_prefix: "/baz", coverage: :strict], :baz_api
 end

examples/rspec.rb

@@ -17,7 +17,7 @@
 
 RSpec.configure do |config|
   path_to_openapi = File.join(__dir__, "openapi.yml")
-  config.include Skooma::RSpec[path_to_openapi], type: :request
+  config.include Skooma::RSpec[path_to_openapi, coverage: :strict], type: :request
 
   config.include Rack::Test::Methods, type: :request
 end

lib/skooma/coverage.rb

@@ -0,0 +1,90 @@
+module Skooma
+  class NoopCoverage
+    def track_request(*)
+    end
+
+    def report
+    end
+  end
+
+  class Coverage
+    class SimpleReport
+      def initialize(coverage)
+        @coverage = coverage
+      end
+
+      attr_reader :coverage
+
+      def report
+        puts <<~MSG
+          OpenAPI schema #{URI.parse(coverage.schema.uri.to_s).path} coverage report: #{coverage.covered_paths.count} / #{coverage.defined_paths.count} operations (#{coverage.covered_percent.round(2)}%) covered.
+          #{coverage.uncovered_paths.empty? ? "All paths are covered!" : "Uncovered paths:"}
+          #{coverage.uncovered_paths.map { |method, path, status| "#{method.upcase} #{path} #{status}" }.join("\n")}
+        MSG
+      end
+    end
+
+    def self.new(schema, mode: nil, format: nil)
+      case mode
+      when nil, false
+        NoopCoverage.new
+      when :report, :strict
+        super
+      else
+        raise ArgumentError, "Invalid coverage: #{mode}, expected :report, :strict, or false"
+      end
+    end
+
+    attr_reader :mode, :format, :defined_paths, :covered_paths, :schema
+
+    def initialize(schema, mode:, format:)
+      @schema = schema
+      @mode = mode
+      @format = format || SimpleReport
+      @defined_paths = find_defined_paths(schema)
+      @covered_paths = Set.new
+    end
+
+    def track_request(result)
+      operation = [nil, nil, nil]
+      result.collect_annotations(result.instance, keys: %w[paths responses]) do |node|
+        case node.key
+        when "paths"
+          operation[0] = node.annotation["method"]
+          operation[1] = node.annotation["current_path"]
+        when "responses"
+          operation[2] = node.annotation
+        end
+      end
+      covered_paths << operation
+    end
+
+    def uncovered_paths
+      defined_paths - covered_paths
+    end
+
+    def covered_percent
+      covered_paths.count * 100.0 / defined_paths.count
+    end
+
+    def report
+      format.new(self).report
+      exit 1 if mode == :strict && uncovered_paths.any?
+    end
+
+    private
+
+    def find_defined_paths(schema)
+      Set.new.tap do |paths|
+        schema["paths"].each do |path, path_item|
+          resolved_path_item = (path_item.key?("$ref") ? path_item.resolve_ref(path_item["$ref"]) : path_item)
+          resolved_path_item.slice("get", "post", "put", "patch", "delete", "options", "head", "trace").each do |method, operation|
+            operation["responses"]&.each do |code, _|
+              paths << [method, path, code]
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/skooma/matchers/conform_request_schema.rb

@@ -5,13 +5,17 @@
 module Skooma
   module Matchers
     class ConformRequestSchema
-      def initialize(schema, mapped_response)
-        @schema = schema
+      def initialize(skooma, mapped_response)
+        @skooma = skooma
+        @schema = skooma.schema
         @mapped_response = mapped_response
       end
 
       def matches?(*)
         @result = @schema.evaluate(@mapped_response)
+
+        @skooma.coverage.track_request(@result) if @mapped_response["response"]
+
         @result.valid?
       end
 

lib/skooma/matchers/conform_response_schema.rb

@@ -3,8 +3,8 @@
 module Skooma
   module Matchers
     class ConformResponseSchema < ConformRequestSchema
-      def initialize(schema, mapped_response, expected)
-        super(schema, mapped_response)
+      def initialize(skooma, mapped_response, expected)
+        super(skooma, mapped_response)
         @expected = expected
       end
 

lib/skooma/matchers/wrapper.rb

@@ -38,30 +38,39 @@ def response_object
 
           raise "Response object not found"
         end
+
+        def skooma_openapi_schema
+          skooma.schema
+        end
       end
 
-      def initialize(helper_methods_module, openapi_path, base_uri: "https://skoomarb.dev/", path_prefix: "")
+      def initialize(helper_methods_module, openapi_path, base_uri: "https://skoomarb.dev/", path_prefix: "", **params)
         super()
 
         registry = create_test_registry
         pathname = Pathname.new(openapi_path)
-        source_uri = "#{base_uri}#{path_prefix.delete_suffix("/")}"
+        source_uri = "#{base_uri}#{path_prefix.delete_suffix("/").delete_prefix("/")}"
         source_uri += "/" unless source_uri.end_with?("/")
         registry.add_source(
           source_uri,
           JSONSkooma::Sources::Local.new(pathname.dirname.to_s)
         )
-        schema = registry.schema(URI.parse("#{source_uri}#{pathname.basename}"), schema_class: Skooma::Objects::OpenAPI)
-        schema.path_prefix = path_prefix
+        @schema = registry.schema(URI.parse("#{source_uri}#{pathname.basename}"), schema_class: Skooma::Objects::OpenAPI)
+        @schema.path_prefix = path_prefix
+
+        @coverage = Coverage.new(@schema, mode: params[:coverage], format: params[:coverage_format])
 
         include DefaultHelperMethods
         include helper_methods_module
 
-        define_method :skooma_openapi_schema do
-          schema
+        skooma_self = self
+        define_method :skooma do
+          skooma_self
         end
       end
 
+      attr_accessor :schema, :coverage
+
       private
 
       def create_test_registry

lib/skooma/minitest.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "minitest/unit"
+
 module Skooma
   # Minitest helpers for OpenAPI schema validation
   # @example
@@ -10,19 +12,19 @@ module Skooma
   class Minitest < Matchers::Wrapper
     module HelperMethods
       def assert_conform_schema(expected_status)
-        matcher = Matchers::ConformSchema.new(skooma_openapi_schema, mapped_response, expected_status)
+        matcher = Matchers::ConformSchema.new(skooma, mapped_response, expected_status)
 
         assert matcher.matches?, -> { matcher.failure_message }
       end
 
       def assert_conform_request_schema
-        matcher = Matchers::ConformRequestSchema.new(skooma_openapi_schema, mapped_response(with_response: false))
+        matcher = Matchers::ConformRequestSchema.new(skooma, mapped_response(with_response: false))
 
         assert matcher.matches?, -> { matcher.failure_message }
       end
 
       def assert_conform_response_schema(expected_status)
-        matcher = Matchers::ConformResponseSchema.new(skooma_openapi_schema, mapped_response(with_request: false), expected_status)
+        matcher = Matchers::ConformResponseSchema.new(skooma, mapped_response(with_request: false), expected_status)
 
         assert matcher.matches?, -> { matcher.failure_message }
       end
@@ -36,6 +38,8 @@ def assert_is_valid_document(document)
 
     def initialize(openapi_path, **params)
       super(HelperMethods, openapi_path, **params)
+
+      MiniTest::Unit.after_tests { coverage.report }
     end
   end
 end

lib/skooma/objects/openapi/keywords/paths.rb

@@ -28,6 +28,8 @@ def evaluate(instance, result)
 
             return result.failure("Path #{instance["path"]} not found in schema") unless path
 
+            result.annotate({"current_path" => path})
+
             result.call(instance, path) do |subresult|
               subresult.annotate({"path_attributes" => attributes})
               path_schema.evaluate(instance, subresult)

lib/skooma/objects/path_item/keywords/base_operation.rb

@@ -13,12 +13,16 @@ def evaluate(instance, result)
             end
 
             json.evaluate(instance, result)
-            return result.success if result.passed?
 
             path_item_result = result.parent
             path_item_result = path_item_result.parent until path_item_result.key.start_with?("/")
 
-            path = path_item_result.annotation["path"]
+            paths_result = path_item_result.parent
+            paths_result.annotate(paths_result.annotation.merge("method" => key))
+
+            return result.success if result.passed?
+
+            path = paths_result.annotation["current_path"]
 
             result.failure("Path #{path}/#{key} is invalid")
           end

lib/skooma/objects/path_item/keywords/delete.rb

@@ -5,7 +5,7 @@ module Objects
     class PathItem
       module Keywords
         class Delete < BaseOperation
-          self.key = "options"
+          self.key = "delete"
           self.depends_on = %w[parameters]
           self.value_schema = :schema
           self.schema_value_class = Objects::Operation

lib/skooma/rspec.rb

@@ -10,15 +10,15 @@ module Skooma
   class RSpec < Matchers::Wrapper
     module HelperMethods
       def conform_schema(expected_status)
-        Matchers::ConformSchema.new(skooma_openapi_schema, mapped_response, expected_status)
+        Matchers::ConformSchema.new(skooma, mapped_response, expected_status)
       end
 
       def conform_response_schema(expected_status)
-        Matchers::ConformResponseSchema.new(skooma_openapi_schema, mapped_response(with_request: false), expected_status)
+        Matchers::ConformResponseSchema.new(skooma, mapped_response(with_request: false), expected_status)
       end
 
       def conform_request_schema
-        Matchers::ConformRequestSchema.new(skooma_openapi_schema, mapped_response(with_response: false))
+        Matchers::ConformRequestSchema.new(skooma, mapped_response(with_response: false))
       end
 
       def be_valid_document
@@ -28,6 +28,13 @@ def be_valid_document
 
     def initialize(openapi_path, **params)
       super(HelperMethods, openapi_path, **params)
+
+      skooma_self = self
+      ::RSpec.configure do |c|
+        c.after(:suite) do
+          at_exit { skooma_self.coverage.report }
+        end
+      end
     end
   end
 end