Hk issue 9

.circleci/config.yml

@@ -11,6 +11,7 @@ jobs:
       - run: ./bin/code-style
       - run: SPEC_VERBOSE=1 ./bin/test
       - run: SPEC_VERBOSE=1 ./bin/test-coverage
+      - run: ./bin/test-mutations
   test-crystal-0.27.0:
     <<: *test-template
     docker:

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - `--min-msi` cli argument to allow passing the suite (exiting with 0) even when there are mutants that survived. Pass as float like `--min-msi=75.0`.
+- Post MSI score to [stryker dashboard](https://dashboard.stryker-mutator.io) if env vars are set.
 
 ### Changed
 - NumberLiteralChange mutant now outputs 0 (for everything != 0) and 1 (for 0)

README.md

@@ -1,4 +1,4 @@
-[![CircleCI](https://circleci.com/gh/hanneskaeufler/crytic/tree/master.svg?style=svg)](https://circleci.com/gh/hanneskaeufler/crytic/tree/master)
+[![CircleCI](https://circleci.com/gh/hanneskaeufler/crytic/tree/master.svg?style=svg)](https://circleci.com/gh/hanneskaeufler/crytic/tree/master) ![Mutation Score](https://badge.stryker-mutator.io/github.com/hanneskaeufler/crytic/master)
 
 # crytic
 
@@ -76,6 +76,19 @@ Each following occurance of `✅` shows that a mutant has been killed, ergo that
 
 `❌ NumberLiteralSignFlip` is signaling that indeed a mutation was not detected. The diff below shows the change that was made which was not caught by the test suite.
 
+### Mutation Badge
+
+To show a badge about your mutation testing efforts like at the top of this readme you can make use of the [dashboard](https://dashboard.stryker-mutator.io) of stryker by letting crytic post  the msi score to the stryker api. To do that, make sure to have the following env vars set:
+
+```
+CIRCLE_BRANCH             => "master",
+CIRCLE_PROJECT_REPONAME   => "crytic",
+CIRCLE_PROJECT_USERNAME   => "hanneskaeufler",
+STRYKER_DASHBOARD_API_KEY => "apikey",
+```
+
+It is currently limited to work with Circle CI and assumes your project is hosted on GitHub.
+
 ### Available mutants
 
 There are many ways a code-base can be modified to introduce arbitrary failures. Crytic only provides mutators which keep the code compiling (at least in theory).

bin/test-mutations

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-./bin/crytic --subject src/crytic/io_reporter.cr spec/io_reporter_spec.cr
+crystal ./src/crytic.cr -- --min-msi=20.0 --subject src/crytic/reporter/io_reporter.cr spec/reporter/io_reporter_spec.cr

spec/mutation/fake_process_runner.cr

@@ -2,28 +2,21 @@ require "../../src/crytic/process_runner"
 
 module Crytic
   class FakeProcessRunner < ProcessRunner
-    private getter cmd
-    private getter args
-    property exit_code
-    property timeout
-    @timeout = [] of Time::Span
+    property exit_code : Int32 = 0
+    property timeout = [] of Time::Span
     @cmd = [] of String
     @args = [] of String
     @output_io = IO::Memory.new
 
     def cmd_with_args
-      cmd.zip(args).map { |c, a| "#{c} #{a}".strip }
-    end
-
-    def initialize
-      @exit_code = 0
+      @cmd.zip(@args).map { |c, a| "#{c} #{a}".strip }
     end
 
     def run(cmd : String, args : Array(String), output, error)
       @cmd << cmd
       @args << args.join(" ")
       output << @output_io.to_s
-      @exit_code
+      exit_code
     end
 
     def run(cmd : String, args : Array(String), output, error, timeout)

spec/io_reporter_spec.cr → spec/reporter/io_reporter_spec.cr

@@ -1,7 +1,7 @@
-require "../src/crytic/io_reporter"
-require "../src/crytic/mutant/number_literal_change"
-require "../src/crytic/mutation/original_result"
-require "./spec_helper"
+require "../../src/crytic/mutant/number_literal_change"
+require "../../src/crytic/mutation/original_result"
+require "../../src/crytic/reporter/io_reporter"
+require "../spec_helper"
 
 private def fake_mutant
   Crytic::Mutant::NumberLiteralChange.at(Crystal::Location.new(filename: nil, line_number: 0, column_number: 0))
@@ -11,7 +11,7 @@ private def original(exit_code = 0, output = "output")
   Crytic::Mutation::OriginalResult.new(exit_code: exit_code, output: output)
 end
 
-module Crytic
+module Crytic::Reporter
   describe IoReporter do
     describe "#report_original_result" do
       it "prints the original passing suites status" do
@@ -89,5 +89,14 @@ module Crytic
         io.to_s.should contain "Mutation Score Indicator (MSI): N/A"
       end
     end
+
+    describe "#report_msi" do
+      it "is a noop" do
+        io = IO::Memory.new
+        results = [] of Mutation::Result
+        IoReporter.new(io).report_msi(results).should eq nil
+        io.to_s.should eq ""
+      end
+    end
   end
 end

spec/reporter/stryker_badge_reporter_spec.cr

@@ -0,0 +1,58 @@
+require "../../src/crytic/mutant/number_literal_change"
+require "../../src/crytic/mutation/result"
+require "../../src/crytic/reporter/http_client"
+require "../../src/crytic/reporter/stryker_badge_reporter"
+require "../spec_helper"
+require "json"
+
+module Crytic::Reporter
+  describe StrykerBadgeReporter do
+    describe "#report_msi" do
+      it "posts to the stryker dashboard" do
+        client = FakeClient.new
+
+        StrykerBadgeReporter.new(client, {
+          "CIRCLE_BRANCH"             => "master",
+          "CIRCLE_PROJECT_REPONAME"   => "crytic",
+          "CIRCLE_PROJECT_USERNAME"   => "hanneskaeufler",
+          "STRYKER_DASHBOARD_API_KEY" => "apikey",
+        }).report_msi(results)
+
+        client.path.should eq "https://dashboard.stryker-mutator.io/api/reports"
+        client.body.should eq({
+          "apiKey"         => "apikey",
+          "branch"         => "master",
+          "mutationScore"  => 100.0,
+          "repositorySlug" => "github.com/hanneskaeufler/crytic",
+        })
+      end
+    end
+  end
+end
+
+private class FakeClient < Crytic::Reporter::HttpClient
+  def post(url : String, bbody : Hash(String, String | Float64))
+    @path = url
+    @body = bbody
+  end
+
+  def path
+    @path
+  end
+
+  def body
+    @body
+  end
+end
+
+private def fake_mutant
+  Crytic::Mutant::NumberLiteralChange.at(
+    Crystal::Location.new(filename: nil, line_number: 0, column_number: 0))
+end
+
+private def results
+  [Crytic::Mutation::Result.new(
+    status: Crytic::Mutation::Status::Covered,
+    mutant: fake_mutant,
+    diff: "")]
+end

spec/runner_spec.cr

@@ -17,5 +17,43 @@ describe Crytic::Runner do
         Crytic::Runner.new.run("./fixtures/simple/bar.cr", ["./nope_spec.cr"])
       end
     end
+
+    it "reports events in order" do
+      reporter = FakeReporter.new
+      runner = Crytic::Runner.new(
+        generator: FakeGenerator.new,
+        reporters: [reporter] of Crytic::Reporter::Reporter)
+
+      runner.run("./fixtures/simple/bar.cr", ["./fixtures/simple/bar_spec.cr"])
+
+      reporter.events.should eq ["report_original_result", "report_summary", "report_msi"]
+    end
+  end
+end
+
+private class FakeReporter < Crytic::Reporter::Reporter
+  getter events
+  @events = [] of String
+
+  def report_original_result(original_result)
+    @events << "report_original_result"
+  end
+
+  def report_result(result)
+    @events << "report_result"
+  end
+
+  def report_summary(results)
+    @events << "report_summary"
+  end
+
+  def report_msi(results)
+    @events << "report_msi"
+  end
+end
+
+private class FakeGenerator < Crytic::Generator
+  def mutations_for(source : String, specs : Array(String))
+    [] of Crytic::Mutation::Mutation
   end
 end

src/crytic.cr

@@ -1,5 +1,8 @@
-require "option_parser"
+require "./crytic/reporter/http_client"
+require "./crytic/reporter/io_reporter"
+require "./crytic/reporter/stryker_badge_reporter"
 require "./crytic/runner"
+require "option_parser"
 
 subject_source = ""
 msi_threshold = 100.0
@@ -22,8 +25,22 @@ OptionParser.parse! do |parser|
   end
 end
 
+reporters = [Crytic::Reporter::IoReporter.new(STDOUT)] of Crytic::Reporter::Reporter
+
+if ENV["STRYKER_DASHBOARD_API_KEY"]?
+  client = Crytic::Reporter::DefaultHttpClient.new
+  reporters << Crytic::Reporter::StrykerBadgeReporter.new(client, {
+    # manually map from ENV to a Hash because I am unable to conform ENV
+    # to anything that I can replace with a stub in the tests
+    "CIRCLE_BRANCH"             => ENV["CIRCLE_BRANCH"],
+    "CIRCLE_PROJECT_REPONAME"   => ENV["CIRCLE_PROJECT_REPONAME"],
+    "CIRCLE_PROJECT_USERNAME"   => ENV["CIRCLE_PROJECT_USERNAME"],
+    "STRYKER_DASHBOARD_API_KEY" => ENV["STRYKER_DASHBOARD_API_KEY"],
+  })
+end
+
 success = Crytic::Runner
-  .new(threshold: msi_threshold)
+  .new(threshold: msi_threshold, reporters: reporters)
   .run(subject_source, spec_files)
 
 exit(success ? 0 : 1)

src/crytic/generator/generator.cr

@@ -0,0 +1,5 @@
+module Crytic
+  abstract class Generator
+    abstract def mutations_for(source : String, specs : Array(String))
+  end
+end

src/crytic/generator.cr → src/crytic/generator/in_memory_generator.cr

@@ -1,5 +1,7 @@
+require "./generator"
+
 module Crytic
-  class Generator
+  class InMemoryMutationsGenerator < Generator
     MUTANT_POSSIBILITIES = [
       Mutant::AndOrSwapPossibilities.new,
       Mutant::BoolLiteralFlipPossibilities.new,

src/crytic/reporter/http_client.cr

@@ -0,0 +1,17 @@
+require "http/client"
+require "json"
+
+module Crytic::Reporter
+  abstract class HttpClient
+    abstract def post(url : String, body : Hash(String, String | Float64))
+  end
+
+  class DefaultHttpClient < HttpClient
+    def post(url : String, body : Hash(String, String | Float64))
+      HTTP::Client.post(
+        url,
+        headers: HTTP::Headers{"Content-Type" => "application/json"},
+        body: body.to_json)
+    end
+  end
+end

src/crytic/io_reporter.cr → src/crytic/reporter/io_reporter.cr

@@ -1,10 +1,11 @@
-require "./msi_calculator"
-require "./mutation/mutation"
+require "../msi_calculator"
+require "../mutation/mutation"
+require "./reporter"
 require "spec/dsl"
 
-module Crytic
+module Crytic::Reporter
   # Reports crytics output into an IO. Useful for e.g. the console output
-  class IoReporter
+  class IoReporter < Reporter
     INDENT = "    "
 
     def initialize(@io : IO, @start_time = Time.now)
@@ -59,6 +60,10 @@ module Crytic
       @io << summary.colorize(results.map(&.status.covered?).all? ? :green : :red).to_s
     end
 
+    # intentional noop
+    def report_msi(results)
+    end
+
     private def score_in_percent(results)
       return "N/A" if results.empty?
       "#{MsiCalculator.new(results).msi}%"

src/crytic/reporter/reporter.cr

@@ -0,0 +1,8 @@
+module Crytic::Reporter
+  abstract class Reporter
+    abstract def report_original_result(original_result)
+    abstract def report_result(result)
+    abstract def report_summary(results)
+    abstract def report_msi(results)
+  end
+end

src/crytic/reporter/stryker_badge_reporter.cr

@@ -0,0 +1,40 @@
+require "../msi_calculator"
+require "./http_client"
+require "./reporter"
+
+module Crytic::Reporter
+  # Sends a MSI score to the stryker dashboard
+  # See also https://infection.github.io/guide/mutation-badge.html
+  class StrykerBadgeReporter < Reporter
+    private DASHBOARD_URL = "https://dashboard.stryker-mutator.io/api/reports"
+
+    def initialize(@client : HttpClient, @env : Hash(String, String))
+    end
+
+    def report_msi(results)
+      @client.post(DASHBOARD_URL, {
+        "apiKey"         => @env["STRYKER_DASHBOARD_API_KEY"],
+        "repositorySlug" => slug,
+        "branch"         => @env["CIRCLE_BRANCH"],
+        "mutationScore"  => score(results),
+      })
+    end
+
+    def report_original_result(original_result)
+    end
+
+    def report_result(result)
+    end
+
+    def report_summary(results)
+    end
+
+    private def slug
+      "github.com/#{@env["CIRCLE_PROJECT_USERNAME"]}/#{@env["CIRCLE_PROJECT_REPONAME"]}"
+    end
+
+    private def score(results)
+      MsiCalculator.new(results).msi
+    end
+  end
+end