diff --git a/.gitignore b/.gitignore index 42cfbcc9c..4e17ae11f 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ lib_managed/ .ivyjars out/ sbt-launch.jar +example/*.class diff --git a/.travis.yml b/.travis.yml index 446b7b069..9dcdf9227 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ -sudo: true language: scala scala: @@ -6,20 +5,20 @@ scala: - 2.11.7 jdk: - - openjdk7 - oraclejdk7 - oraclejdk8 cache: directories: - $HOME/.ivy2/cache - - $HOME/.sbt/boot + - $HOME/.sbt/boot/scala-$TRAVIS_SCALA_VERSION -script: - - unset SBT_OPTS - - ./sbt ++$TRAVIS_SCALA_VERSION clean coverage test coverageReport +before_script: unset SBT_OPTS + +script: sbt clean coverage test coverageReport + +after_script: - find $HOME/.sbt -name "*.lock" | xargs rm - find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm after_success: bash <(curl -s https://codecov.io/bash) - diff --git a/README.md b/README.md index 0617a5494..10b4ec1a5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Diffy +[![GitHub license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) [![Build status](https://img.shields.io/travis/twitter/diffy/master.svg)](https://travis-ci.org/twitter/diffy) [![Coverage status](https://img.shields.io/codecov/c/github/twitter/diffy/master.svg)](https://codecov.io/github/twitter/diffy) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) @@ -19,6 +20,7 @@ the running instances. It then compares the responses, and reports any regressio from those comparisons. The premise for Diffy is that if two implementations of the service return “similar” responses for a sufficiently large and diverse set of requests, then the two implementations can be treated as equivalent and the newer implementation is regression-free. +For a more detailed analysis of Diffy checkout this [blogpost](https://blog.twitter.com/engineering/en_us/a/2015/diffy-testing-services-without-writing-tests.html). ## How does Diffy work? @@ -37,28 +39,29 @@ things: 2. Non-deterministic noise observed between the primary and secondary instances. Since both of these instances are running known-good code, you should expect responses to be in agreement. If not, your service may have non-deterministic behavior, which is to be expected. +![Diffy Topology](https://g.twimg.com/blog/blog/image/Diffy_2.png) Diffy measures how often primary and secondary disagree with each other vs. how often primary and candidate disagree with each other. If these measurements are roughly the same, then Diffy determines that there is nothing wrong and that the error can be ignored. ## How to get started? +# Running the example +The example.sh script included here builds and launches example servers as well as a diffy instance. Verify +that the following ports are available (9000, 9100, 9200, 8880, 8881, & 8888) and run `./example/run.sh start`. -First, you need to build Diffy by invoking `./sbt assembly` from your diffy directory. This will create -a diffy jar at `diffy/target/scala-2.11/diffy-server.jar`. - -Diffy comes bundled with an example.sh script that you can run to start comparing examples instances -we have already deployed online. Once your local Diffy instance is deployed, you send it a few requests -via `curl --header "Canonical-Resource: Html" localhost:8880` and `curl --header "Canonical-Resource: Json" localhost:8880/json`. You can then go to your browser at +Once your local Diffy instance is deployed, you send it a few requests +like `curl --header "Canonical-Resource: Json" localhost:8880/json?Twitter`. You can then go to your browser at [http://localhost:8888](http://localhost:8888) to see what the differences across our example instances look like. +# Digging deeper That was cool but now you want to compare old and new versions of your own service. Here’s how you can start using Diffy to compare three instances of your service: 1. Deploy your old code to `localhost:9990`. This is your primary. 2. Deploy your old code to `localhost:9991`. This is your secondary. 3. Deploy your new code to `localhost:9992`. This is your candidate. -4. Download the latest Diffy binary or build your own from the code. +4. Download the latest Diffy binary from maven central or build your own from the code using `./sbt assembly`. 5. Run the Diffy jar with following command line arguments: ``` @@ -68,27 +71,31 @@ start using Diffy to compare three instances of your service: -master.secondary=localhost:9991 \ -service.protocol=http \ -serviceName=My-Service \ - -proxy.port=:31900 \ - -admin.port=:31159 \ - -http.port=:31149 \ - -rootUrl='localhost:31149' + -proxy.port=:8880 \ + -admin.port=:8881 \ + -http.port=:8888 \ + -rootUrl='localhost:8888' ``` 6. Send a few test requests to your Diffy instance on its proxy port: ``` - curl localhost:31900/your/application/route?with=queryparams + curl localhost:8880/your/application/route?with=queryparams ``` -7. Watch the differences show up in your browser at [http://localhost:31149](http://localhost:31149). - -### Running example on docker-compose +7. Watch the differences show up in your browser at [http://localhost:8888](http://localhost:8888). -Inside the example directory you will find instructions to run a complete example with apis and diffy configured and ready to run using docker-compose. - ## FAQ's For safety reasons `POST`, `PUT`, ` DELETE ` are ignored by default . Add ` -allowHttpSideEffects=true ` to your command line arguments to enable these verbs. +## HTTPS +If you are trying to run Diffy over a HTTPS API, the config required is: + + -service.protocol=https + +And in case of the HTTPS port be different than 443: + + -https.port=123 ## License diff --git a/example.sh b/example.sh deleted file mode 100755 index 02ce266fa..000000000 --- a/example.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -java -jar ./target/scala-2.11/diffy-server.jar \ --candidate='http-candidate.herokuapp.com:80' \ --master.primary='http-primary.herokuapp.com:80' \ --master.secondary='http-secondary.herokuapp.com:80' \ --service.protocol='http' \ --serviceName='My Service' \ --proxy.port=:8880 \ --admin.port=:8881 \ --http.port=:8888 \ --rootUrl='localhost:8888' diff --git a/example/ExampleServers.java b/example/ExampleServers.java new file mode 100644 index 000000000..7dbc2719e --- /dev/null +++ b/example/ExampleServers.java @@ -0,0 +1,71 @@ +import java.net.InetSocketAddress; +import java.io.IOException; +import java.io.OutputStream; +import java.util.function.Function; +import java.util.stream.Stream; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; + +public class ExampleServers { + public static void main(String[] args) throws Exception { + int primary = Integer.parseInt(args[0]); + int secondary = Integer.parseInt(args[1]); + int candidate = Integer.parseInt(args[2]); + Thread p = new Thread(() -> bind(primary, x -> x.toLowerCase())); + Thread s = new Thread(() -> bind(secondary, x -> x.toLowerCase())); + Thread c = new Thread(() -> bind(candidate, x -> x.toUpperCase())); + p.start(); + s.start(); + c.start(); + while(true){ + Thread.sleep(10); + } + } + + public static void bind(int port, Function lambda) { + try { + HttpServer server = HttpServer.create(new InetSocketAddress(port), 0); + server.createContext( + "/json", + new Handler( + "{\"name\":\"%s\", \"timestamp\":\"%s\"}", + "application/json", + lambda)); + server.createContext( + "/html", + new Handler( + "%s%s", + "text/html", + lambda)); + server.setExecutor(null); + server.start(); + } catch (Exception exception) { + System.err.println("!!!failed to start!!!"); + } + } +} +class Handler implements HttpHandler { + private String template; + private String contentType; + private Function lambda; + public Handler(String template, String contentType, Function lambda) { + super(); + this.template = template; + this.contentType = contentType; + this.lambda = lambda; + } + + @Override + public void handle(HttpExchange t) throws IOException { + String name = lambda.apply(t.getRequestURI().getQuery()); + String response = String.format(template, name, System.currentTimeMillis()); + System.out.println(response); + t.getResponseHeaders().add("Content-Type", contentType); + t.sendResponseHeaders(200, response.length()); + OutputStream os = t.getResponseBody(); + os.write(response.getBytes()); + os.close(); + } +} \ No newline at end of file diff --git a/example/README.md b/example/README.md deleted file mode 100644 index a17cd84b4..000000000 --- a/example/README.md +++ /dev/null @@ -1,60 +0,0 @@ -This is an example of diffy. - -## Running - -To run it you will need docker and docker-compose installed. Read more about it here: https://www.docker.com/toolbox - -In this directory, run docker compose up command: - - $ docker-compose up - -It will download the needed images and start the application. You will notice that the logs for all the four services: the twitter diffy proxy, the candidate service, the primary and secondary services. -You will be able to access the web console for diffy on http://localhost:8888/ - -Now you can curl to check the differences: - - $ curl http://localhost:31900/endpoint - -In the web console you will be able to see the diffs. - -To change the services the only thing you need to do is to change the three flavor files withing the flavors directory, candidate, primary and secondary. Whatever you change there will change when you restart the docker-compose. - -ps: If you are running on mac or windows, remember to use the docker-machine/boot2docker/whereveryourundockeron ip instead of localhost, or do port forward it to localhost. - -Have fun! - -## Digging a bit more - -There are some other resources running in the same containers. You can play with it with the following curls: - - $ curl --header "Canonical-Resource: /endpoint" http://localhost:31900/endpoint - $ curl --header "Canonical-Resource: /endpoint/foo" http://localhost:31900/endpoint/foo - $ curl --header "Canonical-Resource: /endpoint/meh" http://localhst:31900/endpoint/meh - -## Applying to your own service - -As you can see in the docker-compose.yml, you can replace the container with two different versions of your service and expose the ports in a way to keep the same configuratio in both services. For example: - - candidate: - image: mycompany/my_service:new_version - ports: - - "8080" - - primary: - image: mycompany/my_service:stable_version - ports: - - "8080" - - secondary: - image: mycompany/my_service:stable_version - ports: - - "8080" - -Than run docker-compose up again. - -## What is the service being tested here? - -It is a rest-shifter service. Easy way to create simple service mocks and prototypes. -Read more: https://github.com/camiloribeiro/restshifter - -This example was originaly posted on https://github.com/camiloribeiro/dockdiffy under the Apache License, Version 2.0 (the "License"). diff --git a/example/docker-compose.yml b/example/docker-compose.yml deleted file mode 100644 index 8b6f86498..000000000 --- a/example/docker-compose.yml +++ /dev/null @@ -1,34 +0,0 @@ -diffy: - image: camiloribeiro/twitter-diffy - ports: - - "8888:8888" - - "31900:31900" - links: - - primary - - secondary - - candidate - command: java -jar ./target/scala-2.11/diffy-server.jar -candidate='candidate:8080' -master.primary='primary:8080' -master.secondary='secondary:8080' -service.protocol='http' -serviceName='Happy Service' -proxy.port=:31900 -admin.port=:8881 -http.port=:8888 -rootUrl='localhost:8888' - -candidate: - image: camiloribeiro/rest_shifter - ports: - - "9881:8080" - volumes: - - flavors/candidate:/root/.rest_shifter/flavors - command: -s - -primary: - image: camiloribeiro/rest_shifter - ports: - - "9882:8080" - volumes: - - flavors/primary:/root/.rest_shifter/flavors - command: -s - -secondary: - image: camiloribeiro/rest_shifter - ports: - - "9883:8080" - volumes: - - flavors/secondary:/root/.rest_shifter/flavors - command: -s diff --git a/example/flavors/candidate/endpointcandidate.flavor b/example/flavors/candidate/endpointcandidate.flavor deleted file mode 100644 index 6a5de151a..000000000 --- a/example/flavors/candidate/endpointcandidate.flavor +++ /dev/null @@ -1,10 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "200" -response_body = { "sizes": { "ipad": { "h": 313, "w": 621, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad" }, "ipad_retina": { "h": 626, "w": 1252, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad_retina" }, "web": { "h": 260, "w": 520, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web" }, "web_retina": { "h": 520, "w": 1040, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web_retina" }, "mobile": { "h": 160, "w": 320, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile" }, "mobile_retina": { "h": 320, "w": 640, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile_retina" }, "300x100": { "h": 100, "w": 300, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/300x100" }, "600x200": { "h": 200, "w": 600, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/600x200" }, "1500x500": { "h": 500, "w": 1500, "urls": "https://pbs.twimg.com/profile_banners/6253282/1347394302/1500x500" } } } -response_content_type = "application/json" -response_location = "" diff --git a/example/flavors/candidate/endpointfoocandidate.flavor b/example/flavors/candidate/endpointfoocandidate.flavor deleted file mode 100644 index 628f8be25..000000000 --- a/example/flavors/candidate/endpointfoocandidate.flavor +++ /dev/null @@ -1,10 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint/foo" -request_accept = "" -request_content_type = "" -response_sleep = 1 -response_status = "400" -response_body = { "foo" : "candidate", "started_at": "2015-09-18T09:22:52Z", "last_modified_at": "2015-09-18T09:22:52Z" } -response_content_type = "application/json" -response_location = "" diff --git a/example/flavors/candidate/endpointmehcandidate.flavor b/example/flavors/candidate/endpointmehcandidate.flavor deleted file mode 100644 index d541fa031..000000000 --- a/example/flavors/candidate/endpointmehcandidate.flavor +++ /dev/null @@ -1,10 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint/meh" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "200" -response_body = { "hello_meh" : "candidate" } -response_content_type = "application/json" -response_location = "" diff --git a/example/flavors/primary/endpointfooprimary.flavor b/example/flavors/primary/endpointfooprimary.flavor deleted file mode 100644 index dbc781fad..000000000 --- a/example/flavors/primary/endpointfooprimary.flavor +++ /dev/null @@ -1,10 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint/foo" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "400" -response_body = { "foo" : "primary", "started_at": "2015-09-18T09:22:51Z", "last_modified_at": "2015-09-18T09:22:51Z" } -response_content_type = "application/json" -response_location = "" diff --git a/example/flavors/primary/endpointmehprimary.flavor b/example/flavors/primary/endpointmehprimary.flavor deleted file mode 100644 index f0399ff2b..000000000 --- a/example/flavors/primary/endpointmehprimary.flavor +++ /dev/null @@ -1,10 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint/meh" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "200" -response_body = { "hello_world" : "primary" } -response_content_type = "application/json" -response_location = "" diff --git a/example/flavors/primary/endpointprimary.flavor b/example/flavors/primary/endpointprimary.flavor deleted file mode 100644 index 817fa9a8b..000000000 --- a/example/flavors/primary/endpointprimary.flavor +++ /dev/null @@ -1,10 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "200" -response_body = { "sizes": { "ipad": { "h": 313, "w": 626, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad" }, "ipad_retina": { "h": 626, "w": 1252, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad_retina" }, "web": { "h": 260, "w": 520, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web" }, "web_retina": { "h": 520, "w": 1040, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web_retina" }, "mobile": { "h": 160, "w": 320, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile" }, "mobile_retina": { "h": 320, "w": 640, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile_retina" }, "300x100": { "h": 100, "w": 300, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/300x100" }, "600x200": { "h": 200, "w": 600, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/600x200" }, "1500x500": { "h": 500, "w": 1500, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/1500x500" } } } -response_content_type = "application/json" -response_location = "" diff --git a/example/flavors/secondary/endpointfoosecondary.flavor b/example/flavors/secondary/endpointfoosecondary.flavor deleted file mode 100644 index 9f51ec6d4..000000000 --- a/example/flavors/secondary/endpointfoosecondary.flavor +++ /dev/null @@ -1,10 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint/foo" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "400" -response_body = { "foo" : "secondary", "started_at": "2015-09-18T09:22:53Z", "last_modified_at": "2015-09-18T09:22:53Z" } -response_content_type = "application/json" -response_location = "" diff --git a/example/flavors/secondary/endpointmehsecondary.flavor b/example/flavors/secondary/endpointmehsecondary.flavor deleted file mode 100644 index f4cb0fa67..000000000 --- a/example/flavors/secondary/endpointmehsecondary.flavor +++ /dev/null @@ -1,10 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint/meh" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "200" -response_body = { "hello_bar" : "secondary" } -response_content_type = "application/json" -response_location = "" diff --git a/example/flavors/secondary/endpointsecondary.flavor b/example/flavors/secondary/endpointsecondary.flavor deleted file mode 100644 index 817fa9a8b..000000000 --- a/example/flavors/secondary/endpointsecondary.flavor +++ /dev/null @@ -1,10 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "200" -response_body = { "sizes": { "ipad": { "h": 313, "w": 626, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad" }, "ipad_retina": { "h": 626, "w": 1252, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad_retina" }, "web": { "h": 260, "w": 520, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web" }, "web_retina": { "h": 520, "w": 1040, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web_retina" }, "mobile": { "h": 160, "w": 320, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile" }, "mobile_retina": { "h": 320, "w": 640, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile_retina" }, "300x100": { "h": 100, "w": 300, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/300x100" }, "600x200": { "h": 200, "w": 600, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/600x200" }, "1500x500": { "h": 500, "w": 1500, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/1500x500" } } } -response_content_type = "application/json" -response_location = "" diff --git a/example/run.sh b/example/run.sh new file mode 100755 index 000000000..df1e51522 --- /dev/null +++ b/example/run.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +if [ "$1" = "start" ]; +then + + # Build primary, secondary, and candidate servers + javac example/ExampleServers.java && \ + + # Deploy primary, secondary, and candidate servers + java -cp example ExampleServers 9000 9100 9200 & \ + + # Build diffy + ./sbt assembly && \ + + # Deploy diffy + java -jar ./target/scala-2.11/diffy-server.jar \ + -candidate='localhost:9200' \ + -master.primary='localhost:9000' \ + -master.secondary='localhost:9100' \ + -service.protocol='http' \ + -serviceName='My Service' \ + -proxy.port=:8880 \ + -admin.port=:8881 \ + -http.port=:8888 \ + -rootUrl='localhost:8888' &\ + +else + echo "Please make sure ports 9000, 9100, 9200, 8880, 8881, & 8888 are available before running \"example/run.sh start\"" +fi + diff --git a/src/main/resources/templates/dashboard.mustache b/src/main/resources/templates/dashboard.mustache index dff304360..a9a3df837 100644 --- a/src/main/resources/templates/dashboard.mustache +++ b/src/main/resources/templates/dashboard.mustache @@ -110,7 +110,6 @@ [[info.last_reset | ago]] ([[info.last_reset | formatDate]])

Thresholds

[[info.threshold_relative]]% relative, [[info.threshold_absolute]]% absolute - (more on thresholds)
@@ -261,4 +260,4 @@ - \ No newline at end of file + diff --git a/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala b/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala index cbdc46e34..84cfffb4c 100644 --- a/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala +++ b/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala @@ -26,7 +26,7 @@ object DiffyServiceModule extends TwitterModule { flag[String]("master.secondary", "secondary master serverset where known good code is deployed") val protocol = - flag[String]("service.protocol", "Service protocol, thrift or http") + flag[String]("service.protocol", "Service protocol: thrift, http or https") val clientId = flag[String]("proxy.clientId", "diffy.proxy", "The clientId to be used by the proxy service to talk to candidate, primary, and master") @@ -70,6 +70,9 @@ object DiffyServiceModule extends TwitterModule { val skipEmailsWhenNoErrors = flag[Boolean]("skipEmailsWhenNoErrors", false, "Do not send emails if there are no critical errors") + var httpsPort = + flag[String]("httpsPort", "443", "Port to be used when using HTTPS as a protocol") + @Provides @Singleton def settings = @@ -93,7 +96,8 @@ object DiffyServiceModule extends TwitterModule { rootUrl(), allowHttpSideEffects(), excludeHttpHeadersComparison(), - skipEmailsWhenNoErrors() + skipEmailsWhenNoErrors(), + httpsPort() ) @Provides diff --git a/src/main/scala/com/twitter/diffy/analysis/DifferenceCollector.scala b/src/main/scala/com/twitter/diffy/analysis/DifferenceCollector.scala index 6ceb77446..82e53e0ee 100644 --- a/src/main/scala/com/twitter/diffy/analysis/DifferenceCollector.scala +++ b/src/main/scala/com/twitter/diffy/analysis/DifferenceCollector.scala @@ -75,11 +75,12 @@ class DifferenceAnalyzer @Inject()( } } - def clear(): Future[Unit] = Future { - rawCounter.counter.clear() - noiseCounter.counter.clear() - store.clear() - } + def clear(): Future[Unit] = + Future.join( + rawCounter.counter.clear(), + noiseCounter.counter.clear(), + store.clear() + ) map { _ => () } def differencesToJson(diffs: Map[String, Difference]): Map[String, String] = diffs map { diff --git a/src/main/scala/com/twitter/diffy/analysis/JoinedDifferences.scala b/src/main/scala/com/twitter/diffy/analysis/JoinedDifferences.scala index e9890b364..c8709ddd4 100644 --- a/src/main/scala/com/twitter/diffy/analysis/JoinedDifferences.scala +++ b/src/main/scala/com/twitter/diffy/analysis/JoinedDifferences.scala @@ -15,7 +15,7 @@ object DifferencesFilterFactory { } case class JoinedDifferences @Inject() (raw: RawDifferenceCounter, noise: NoiseDifferenceCounter) { - lazy val endpoints: Future[Map[String, JoinedEndpoint]] = { + def endpoints: Future[Map[String, JoinedEndpoint]] = { raw.counter.endpoints map { _.keys } flatMap { eps => Future.collect( eps map { ep => @@ -43,9 +43,9 @@ case class JoinedEndpoint( { def differences = endpoint.differences def total = endpoint.total - lazy val fields: Map[String, JoinedField] = original map { case (path, field) => + def fields: Map[String, JoinedField] = original map { case (path, field) => path -> JoinedField(endpoint, field, noise.getOrElse(path, FieldMetadata.Empty)) - } toMap + } } case class JoinedField(endpoint: EndpointMetadata, raw: FieldMetadata, noise: FieldMetadata) { diff --git a/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala b/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala index 3e8956ee3..d221815fb 100644 --- a/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala +++ b/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala @@ -47,14 +47,15 @@ class HttpLifter(excludeHttpHeadersComparison: Boolean) { def liftRequest(req: HttpRequest): Future[Message] = { val canonicalResource = Option(req.headers.get("Canonical-Resource")) - Future.value(Message(canonicalResource, FieldMap(Map("request"-> req.toString)))) + val body = req.getContent.copy.toString(Charsets.Utf8) + Future.value(Message(canonicalResource, FieldMap(Map("request"-> req.toString, "body" -> body)))) } def liftResponse(resp: Try[HttpResponse]): Future[Message] = { Future.const(resp) flatMap { r: HttpResponse => val mediaTypeOpt: Option[MediaType] = Option(r.headers.get(HttpHeaders.CONTENT_TYPE)) map { MediaType.parse } - + val contentLengthOpt = Option(r.headers.get(HttpHeaders.CONTENT_LENGTH)) /** header supplied by macaw, indicating the controller reached **/ @@ -70,7 +71,7 @@ class HttpLifter(excludeHttpHeadersComparison: Boolean) { /** When Content-Type is set as application/json, lift as Json **/ case (Some(mediaType), _) if mediaType.is(MediaType.JSON_UTF_8) || mediaType.toString == "application/json" => { val jsonContentTry = Try { - JsonLifter.decode(r.getContent.toString(Charsets.Utf8)) + JsonLifter.decode(r.getContent.copy.toString(Charsets.Utf8)) } Future.const(jsonContentTry map { jsonContent => @@ -90,9 +91,9 @@ class HttpLifter(excludeHttpHeadersComparison: Boolean) { /** When Content-Type is set as text/html, lift as Html **/ case (Some(mediaType), _) if mediaType.is(MediaType.HTML_UTF_8) || mediaType.toString == "text/html" => { - val htmlContentTry = Try { - HtmlLifter.lift(HtmlLifter.decode(r.getContent.toString(Charsets.Utf8))) - } + val htmlContentTry = Try { + HtmlLifter.lift(HtmlLifter.decode(r.getContent.copy.toString(Charsets.Utf8))) + } Future.const(htmlContentTry map { htmlContent => val responseMap = Map( diff --git a/src/main/scala/com/twitter/diffy/proxy/DifferenceProxy.scala b/src/main/scala/com/twitter/diffy/proxy/DifferenceProxy.scala index 818fd7a23..0e36d64b0 100644 --- a/src/main/scala/com/twitter/diffy/proxy/DifferenceProxy.scala +++ b/src/main/scala/com/twitter/diffy/proxy/DifferenceProxy.scala @@ -21,11 +21,13 @@ object DifferenceProxyModule extends TwitterModule { settings.protocol match { case "thrift" => ThriftDifferenceProxy(settings, collector, joinedDifferences, analyzer) case "http" => SimpleHttpDifferenceProxy(settings, collector, joinedDifferences, analyzer) + case "https" => SimpleHttpsDifferenceProxy(settings, collector, joinedDifferences, analyzer) } } object DifferenceProxy { - val NoResponseException = Future.exception(new Exception("No responses provided by diffy")) + object NoResponseException extends Exception("No responses provided by diffy") + val NoResponseExceptionFuture = Future.exception(NoResponseException) val log = Logger(classOf[DifferenceProxy]) } @@ -97,7 +99,7 @@ trait DifferenceProxy { } } - NoResponseException + NoResponseExceptionFuture } } diff --git a/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala b/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala index e1390213d..eb17b5a29 100644 --- a/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala +++ b/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala @@ -4,11 +4,28 @@ import java.net.SocketAddress import com.twitter.diffy.analysis.{DifferenceAnalyzer, JoinedDifferences, InMemoryDifferenceCollector} import com.twitter.diffy.lifter.{HttpLifter, Message} -import com.twitter.finagle.{Http, Filter} -import com.twitter.finagle.http.{Method, Request} +import com.twitter.diffy.proxy.DifferenceProxy.NoResponseException +import com.twitter.finagle.{Service, Http, Filter} +import com.twitter.finagle.http.{Status, Response, Method, Request} import com.twitter.util.{StorageUnit, Try, Future} import org.jboss.netty.handler.codec.http.{HttpResponse, HttpRequest} +object HttpDifferenceProxy { + val okResponse = Future.value(Response(Status.Ok)) + + val noResponseExceptionFilter = + new Filter[HttpRequest, HttpResponse, HttpRequest, HttpResponse] { + override def apply( + request: HttpRequest, + service: Service[HttpRequest, HttpResponse] + ): Future[HttpResponse] = { + service(request).rescue[HttpResponse] { case NoResponseException => + okResponse + } + } + } +} + trait HttpDifferenceProxy extends DifferenceProxy { val servicePort: SocketAddress val lifter = new HttpLifter(settings.excludeHttpHeadersComparison) @@ -20,7 +37,11 @@ trait HttpDifferenceProxy extends DifferenceProxy { override def serviceFactory(serverset: String, label: String) = HttpService(Http.client.withMaxResponseSize(StorageUnit.parse("50.megabytes")).newClient(serverset, label).toService) - override lazy val server = Http.serve(servicePort, proxy) + override lazy val server = + Http.serve( + servicePort, + HttpDifferenceProxy.noResponseExceptionFilter andThen proxy + ) override def liftRequest(req: HttpRequest): Future[Message] = lifter.liftRequest(req) @@ -40,7 +61,7 @@ object SimpleHttpDifferenceProxy { val hasSideEffects = Set(Method.Post, Method.Put, Method.Delete).contains(Request(req).method) - if (hasSideEffects) DifferenceProxy.NoResponseException else svc(req) + if (hasSideEffects) DifferenceProxy.NoResponseExceptionFuture else svc(req) } } @@ -64,4 +85,31 @@ case class SimpleHttpDifferenceProxy ( Filter.identity andThenIf (!settings.allowHttpSideEffects, httpSideEffectsFilter) andThen super.proxy +} + +/** + * Alternative to SimpleHttpDifferenceProxy allowing HTTPS requests + */ +case class SimpleHttpsDifferenceProxy ( + settings: Settings, + collector: InMemoryDifferenceCollector, + joinedDifferences: JoinedDifferences, + analyzer: DifferenceAnalyzer) + extends HttpDifferenceProxy +{ + import SimpleHttpDifferenceProxy._ + + override val servicePort = settings.servicePort + + override val proxy = + Filter.identity andThenIf + (!settings.allowHttpSideEffects, httpSideEffectsFilter) andThen + super.proxy + + override def serviceFactory(serverset: String, label: String) = + HttpService( + Http.client + .withTls(serverset) + .newService(serverset+":"+settings.httpsPort, label) + ) } \ No newline at end of file diff --git a/src/main/scala/com/twitter/diffy/proxy/Settings.scala b/src/main/scala/com/twitter/diffy/proxy/Settings.scala index 7e4c6c335..7ed4c9511 100644 --- a/src/main/scala/com/twitter/diffy/proxy/Settings.scala +++ b/src/main/scala/com/twitter/diffy/proxy/Settings.scala @@ -24,6 +24,7 @@ case class Settings( rootUrl: String, allowHttpSideEffects: Boolean, excludeHttpHeadersComparison: Boolean, - skipEmailsWhenNoErrors: Boolean) + skipEmailsWhenNoErrors: Boolean, + httpsPort: String) case class Target(path: String) diff --git a/src/test/scala/com/twitter/diffy/StartupFeatureTest.scala b/src/test/scala/com/twitter/diffy/StartupFeatureTest.scala index 4d5deed2f..a4c51b43d 100644 --- a/src/test/scala/com/twitter/diffy/StartupFeatureTest.scala +++ b/src/test/scala/com/twitter/diffy/StartupFeatureTest.scala @@ -11,7 +11,7 @@ class StartupFeatureTest extends Test { twitterServer = new MainService { }, extraArgs = Seq( - "-proxy.port=:9992", + "-proxy.port=:0", "-candidate=localhost:80", "-master.primary=localhost:80", "-master.secondary=localhost:80", diff --git a/src/test/scala/com/twitter/diffy/TestHelper.scala b/src/test/scala/com/twitter/diffy/TestHelper.scala index 3fe3ffe53..4e89440ac 100644 --- a/src/test/scala/com/twitter/diffy/TestHelper.scala +++ b/src/test/scala/com/twitter/diffy/TestHelper.scala @@ -29,7 +29,8 @@ object TestHelper extends MockitoSugar { rootUrl = "test", allowHttpSideEffects = true, excludeHttpHeadersComparison = true, - skipEmailsWhenNoErrors = false + skipEmailsWhenNoErrors = false, + httpsPort = "443" ) def makeEmptyJoinedDifferences = { diff --git a/src/test/scala/com/twitter/diffy/lifter/HttpLifterSpec.scala b/src/test/scala/com/twitter/diffy/lifter/HttpLifterSpec.scala index 8d6dda46b..231652ba4 100644 --- a/src/test/scala/com/twitter/diffy/lifter/HttpLifterSpec.scala +++ b/src/test/scala/com/twitter/diffy/lifter/HttpLifterSpec.scala @@ -49,8 +49,11 @@ class HttpLifterSpec extends ParentSpec { val testException = new Exception("test exception") - def request(method: HttpMethod, uri: String): HttpRequest = - new DefaultHttpRequest(HttpVersion.HTTP_1_1, method, uri) + def request(method: HttpMethod, uri: String, body: Option[String] = None): HttpRequest = { + val req = new DefaultHttpRequest(HttpVersion.HTTP_1_1, method, uri) + body foreach { b => req.setContent(ChannelBuffers.wrappedBuffer(b.getBytes(Charsets.Utf8)))} + req + } def response(status: HttpResponseStatus, body: String): HttpResponse = { val resp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, status) @@ -77,6 +80,20 @@ class HttpLifterSpec extends ParentSpec { msg.endpoint.get should equal ("endpoint") resultFieldMap.get("request").get should equal (req.toString) } + + it("lift simple Post request") { + val lifter = new HttpLifter(false) + val requestBody = "request_body" + val req = request(HttpMethod.POST, reqUri, Some(requestBody)) + req.headers().add("Canonical-Resource", "endpoint") + + val msg = Await.result(lifter.liftRequest(req)) + val resultFieldMap = msg.result.asInstanceOf[FieldMap[String]] + + msg.endpoint.get should equal ("endpoint") + resultFieldMap.get("request").get should equal (req.toString) + resultFieldMap.get("body").get should equal (requestBody) + } } describe("LiftResponse") { diff --git a/src/universal/conf/application.ini b/src/universal/conf/application.ini index d1d8e208c..f9ae10820 100644 --- a/src/universal/conf/application.ini +++ b/src/universal/conf/application.ini @@ -6,4 +6,5 @@ -proxy.port=:8880 -admin.port=:8881 -http.port=:8888 --rootUrl=localhost:8888 \ No newline at end of file +-rootUrl=localhost:8888 +-excludeHttpHeadersComparison=true diff --git a/version.sbt b/version.sbt index d201e09df..941cfdca7 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "1.0.6-SNAPSHOT" \ No newline at end of file +version in ThisBuild := "0.0.2-SNAPSHOT" \ No newline at end of file