From 4b397a7f56149f13f92afb2ce17b7a1535f7f100 Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Wed, 25 Nov 2015 01:28:29 +0100 Subject: [PATCH 01/27] diffy : returning 200 OK instead of NoResponseException for http services --- example.sh | 6 ++-- .../com/twitter/diffy/lifter/HttpLifter.scala | 2 ++ .../twitter/diffy/proxy/DifferenceProxy.scala | 5 ++-- .../diffy/proxy/HttpDifferenceProxy.scala | 29 ++++++++++++++++--- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/example.sh b/example.sh index 02ce266fa..4c3eceae8 100755 --- a/example.sh +++ b/example.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash java -jar ./target/scala-2.11/diffy-server.jar \ --candidate='http-candidate.herokuapp.com:80' \ --master.primary='http-primary.herokuapp.com:80' \ --master.secondary='http-secondary.herokuapp.com:80' \ +-candidate='localhost:9200' \ +-master.primary='localhost:9000' \ +-master.secondary='localhost:9100' \ -service.protocol='http' \ -serviceName='My Service' \ -proxy.port=:8880 \ diff --git a/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala b/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala index 3e8956ee3..c6376cf43 100644 --- a/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala +++ b/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala @@ -55,6 +55,8 @@ class HttpLifter(excludeHttpHeadersComparison: Boolean) { val mediaTypeOpt: Option[MediaType] = Option(r.headers.get(HttpHeaders.CONTENT_TYPE)) map { MediaType.parse } +println(mediaTypeOpt) + val contentLengthOpt = Option(r.headers.get(HttpHeaders.CONTENT_LENGTH)) /** header supplied by macaw, indicating the controller reached **/ diff --git a/src/main/scala/com/twitter/diffy/proxy/DifferenceProxy.scala b/src/main/scala/com/twitter/diffy/proxy/DifferenceProxy.scala index 818fd7a23..76405c611 100644 --- a/src/main/scala/com/twitter/diffy/proxy/DifferenceProxy.scala +++ b/src/main/scala/com/twitter/diffy/proxy/DifferenceProxy.scala @@ -25,7 +25,8 @@ object DifferenceProxyModule extends TwitterModule { } object DifferenceProxy { - val NoResponseException = Future.exception(new Exception("No responses provided by diffy")) + object NoResponseException extends Exception("No responses provided by diffy") + val NoResponseExceptionFuture = Future.exception(NoResponseException) val log = Logger(classOf[DifferenceProxy]) } @@ -97,7 +98,7 @@ trait DifferenceProxy { } } - NoResponseException + NoResponseExceptionFuture } } diff --git a/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala b/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala index e1390213d..7da309138 100644 --- a/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala +++ b/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala @@ -4,11 +4,28 @@ import java.net.SocketAddress import com.twitter.diffy.analysis.{DifferenceAnalyzer, JoinedDifferences, InMemoryDifferenceCollector} import com.twitter.diffy.lifter.{HttpLifter, Message} -import com.twitter.finagle.{Http, Filter} -import com.twitter.finagle.http.{Method, Request} +import com.twitter.diffy.proxy.DifferenceProxy.NoResponseException +import com.twitter.finagle.{Service, Http, Filter} +import com.twitter.finagle.http.{Status, Response, Method, Request} import com.twitter.util.{StorageUnit, Try, Future} import org.jboss.netty.handler.codec.http.{HttpResponse, HttpRequest} +object HttpDifferenceProxy { + val okResponse = Future.value(Response(Status.Ok)) + + val noResponseExceptionFilter = + new Filter[HttpRequest, HttpResponse, HttpRequest, HttpResponse] { + override def apply( + request: HttpRequest, + service: Service[HttpRequest, HttpResponse] + ): Future[HttpResponse] = { + service(request).rescue[HttpResponse] { case NoResponseException => + okResponse + } + } + } +} + trait HttpDifferenceProxy extends DifferenceProxy { val servicePort: SocketAddress val lifter = new HttpLifter(settings.excludeHttpHeadersComparison) @@ -20,7 +37,11 @@ trait HttpDifferenceProxy extends DifferenceProxy { override def serviceFactory(serverset: String, label: String) = HttpService(Http.client.withMaxResponseSize(StorageUnit.parse("50.megabytes")).newClient(serverset, label).toService) - override lazy val server = Http.serve(servicePort, proxy) + override lazy val server = + Http.serve( + servicePort, + HttpDifferenceProxy.noResponseExceptionFilter andThen proxy + ) override def liftRequest(req: HttpRequest): Future[Message] = lifter.liftRequest(req) @@ -40,7 +61,7 @@ object SimpleHttpDifferenceProxy { val hasSideEffects = Set(Method.Post, Method.Put, Method.Delete).contains(Request(req).method) - if (hasSideEffects) DifferenceProxy.NoResponseException else svc(req) + if (hasSideEffects) DifferenceProxy.NoResponseExceptionFuture else svc(req) } } From 815812c50d063d27046310ab8f00e2433a05d38b Mon Sep 17 00:00:00 2001 From: Alexander Kazakov Date: Tue, 29 Dec 2015 18:55:39 +0300 Subject: [PATCH 02/27] Removing println --- src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala b/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala index c6376cf43..0cd1a678f 100644 --- a/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala +++ b/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala @@ -54,8 +54,6 @@ class HttpLifter(excludeHttpHeadersComparison: Boolean) { Future.const(resp) flatMap { r: HttpResponse => val mediaTypeOpt: Option[MediaType] = Option(r.headers.get(HttpHeaders.CONTENT_TYPE)) map { MediaType.parse } - -println(mediaTypeOpt) val contentLengthOpt = Option(r.headers.get(HttpHeaders.CONTENT_LENGTH)) From 712de36fe9abfd2f51e74ae9e82ef79cb7489dde Mon Sep 17 00:00:00 2001 From: Brice Jaglin Date: Thu, 24 Dec 2015 00:55:41 +0100 Subject: [PATCH 03/27] fix compatibility with compose 1.5.0+ https://github.com/docker/compose/commit/46e8e4322aa694f176c3fec5e705c5c40c704824 --- example/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/example/docker-compose.yml b/example/docker-compose.yml index 8b6f86498..74c38b5e8 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -14,7 +14,7 @@ candidate: ports: - "9881:8080" volumes: - - flavors/candidate:/root/.rest_shifter/flavors + - ./flavors/candidate:/root/.rest_shifter/flavors command: -s primary: @@ -22,7 +22,7 @@ primary: ports: - "9882:8080" volumes: - - flavors/primary:/root/.rest_shifter/flavors + - ./flavors/primary:/root/.rest_shifter/flavors command: -s secondary: @@ -30,5 +30,5 @@ secondary: ports: - "9883:8080" volumes: - - flavors/secondary:/root/.rest_shifter/flavors + - ./flavors/secondary:/root/.rest_shifter/flavors command: -s From da5764d39f4d53737a83b90b2fa7596d27627d24 Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Tue, 26 Apr 2016 14:36:30 +0530 Subject: [PATCH 04/27] modified StartupFeatureTest to bind to port 0 --- src/test/scala/com/twitter/diffy/StartupFeatureTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/scala/com/twitter/diffy/StartupFeatureTest.scala b/src/test/scala/com/twitter/diffy/StartupFeatureTest.scala index 4d5deed2f..a4c51b43d 100644 --- a/src/test/scala/com/twitter/diffy/StartupFeatureTest.scala +++ b/src/test/scala/com/twitter/diffy/StartupFeatureTest.scala @@ -11,7 +11,7 @@ class StartupFeatureTest extends Test { twitterServer = new MainService { }, extraArgs = Seq( - "-proxy.port=:9992", + "-proxy.port=:0", "-candidate=localhost:80", "-master.primary=localhost:80", "-master.secondary=localhost:80", From 3f3e2d0cf77a4ba5cc67a33eeade176e03341d4c Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Fri, 29 Apr 2016 16:07:19 +0530 Subject: [PATCH 05/27] Setting version to 0.0.2-SNAPSHOT --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index d201e09df..941cfdca7 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "1.0.6-SNAPSHOT" \ No newline at end of file +version in ThisBuild := "0.0.2-SNAPSHOT" \ No newline at end of file From 9fe7d1a2fcb4d8835e739270a6e235d83748c10a Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Wed, 11 May 2016 00:59:22 +0530 Subject: [PATCH 06/27] modified joined differences so that endpoints are recalculated on every access --- .../twitter/diffy/analysis/DifferenceCollector.scala | 11 ++++++----- .../twitter/diffy/analysis/JoinedDifferences.scala | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/scala/com/twitter/diffy/analysis/DifferenceCollector.scala b/src/main/scala/com/twitter/diffy/analysis/DifferenceCollector.scala index 6ceb77446..82e53e0ee 100644 --- a/src/main/scala/com/twitter/diffy/analysis/DifferenceCollector.scala +++ b/src/main/scala/com/twitter/diffy/analysis/DifferenceCollector.scala @@ -75,11 +75,12 @@ class DifferenceAnalyzer @Inject()( } } - def clear(): Future[Unit] = Future { - rawCounter.counter.clear() - noiseCounter.counter.clear() - store.clear() - } + def clear(): Future[Unit] = + Future.join( + rawCounter.counter.clear(), + noiseCounter.counter.clear(), + store.clear() + ) map { _ => () } def differencesToJson(diffs: Map[String, Difference]): Map[String, String] = diffs map { diff --git a/src/main/scala/com/twitter/diffy/analysis/JoinedDifferences.scala b/src/main/scala/com/twitter/diffy/analysis/JoinedDifferences.scala index e9890b364..88722035e 100644 --- a/src/main/scala/com/twitter/diffy/analysis/JoinedDifferences.scala +++ b/src/main/scala/com/twitter/diffy/analysis/JoinedDifferences.scala @@ -15,7 +15,7 @@ object DifferencesFilterFactory { } case class JoinedDifferences @Inject() (raw: RawDifferenceCounter, noise: NoiseDifferenceCounter) { - lazy val endpoints: Future[Map[String, JoinedEndpoint]] = { + def endpoints: Future[Map[String, JoinedEndpoint]] = { raw.counter.endpoints map { _.keys } flatMap { eps => Future.collect( eps map { ep => From 7ba59e2f8c50fc39e59b45540d36b57ec3c22195 Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Wed, 11 May 2016 01:18:07 +0530 Subject: [PATCH 07/27] diffy: modified aggreagted stats to defs from lazy vals to fix stale emails --- .../scala/com/twitter/diffy/analysis/JoinedDifferences.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/com/twitter/diffy/analysis/JoinedDifferences.scala b/src/main/scala/com/twitter/diffy/analysis/JoinedDifferences.scala index 88722035e..c8709ddd4 100644 --- a/src/main/scala/com/twitter/diffy/analysis/JoinedDifferences.scala +++ b/src/main/scala/com/twitter/diffy/analysis/JoinedDifferences.scala @@ -43,9 +43,9 @@ case class JoinedEndpoint( { def differences = endpoint.differences def total = endpoint.total - lazy val fields: Map[String, JoinedField] = original map { case (path, field) => + def fields: Map[String, JoinedField] = original map { case (path, field) => path -> JoinedField(endpoint, field, noise.getOrElse(path, FieldMetadata.Empty)) - } toMap + } } case class JoinedField(endpoint: EndpointMetadata, raw: FieldMetadata, noise: FieldMetadata) { From f35094560ddeb04fe9da1ece6dd8611ac15fa4ac Mon Sep 17 00:00:00 2001 From: Camilo Ribeiro Date: Thu, 12 May 2016 20:10:54 +0200 Subject: [PATCH 08/27] Fixing issue with restshifter --- example/flavors/candidate/endpointcandidate.flavor | 1 + example/flavors/candidate/endpointfoocandidate.flavor | 1 + example/flavors/candidate/endpointmehcandidate.flavor | 1 + example/flavors/primary/endpointfooprimary.flavor | 1 + example/flavors/primary/endpointmehprimary.flavor | 1 + example/flavors/primary/endpointprimary.flavor | 1 + example/flavors/secondary/endpointfoosecondary.flavor | 1 + example/flavors/secondary/endpointmehsecondary.flavor | 1 + example/flavors/secondary/endpointsecondary.flavor | 1 + 9 files changed, 9 insertions(+) diff --git a/example/flavors/candidate/endpointcandidate.flavor b/example/flavors/candidate/endpointcandidate.flavor index 6a5de151a..06c403470 100644 --- a/example/flavors/candidate/endpointcandidate.flavor +++ b/example/flavors/candidate/endpointcandidate.flavor @@ -8,3 +8,4 @@ response_status = "200" response_body = { "sizes": { "ipad": { "h": 313, "w": 621, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad" }, "ipad_retina": { "h": 626, "w": 1252, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad_retina" }, "web": { "h": 260, "w": 520, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web" }, "web_retina": { "h": 520, "w": 1040, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web_retina" }, "mobile": { "h": 160, "w": 320, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile" }, "mobile_retina": { "h": 320, "w": 640, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile_retina" }, "300x100": { "h": 100, "w": 300, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/300x100" }, "600x200": { "h": 200, "w": 600, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/600x200" }, "1500x500": { "h": 500, "w": 1500, "urls": "https://pbs.twimg.com/profile_banners/6253282/1347394302/1500x500" } } } response_content_type = "application/json" response_location = "" +response_headers = [] diff --git a/example/flavors/candidate/endpointfoocandidate.flavor b/example/flavors/candidate/endpointfoocandidate.flavor index 628f8be25..c08b0f5ba 100644 --- a/example/flavors/candidate/endpointfoocandidate.flavor +++ b/example/flavors/candidate/endpointfoocandidate.flavor @@ -8,3 +8,4 @@ response_status = "400" response_body = { "foo" : "candidate", "started_at": "2015-09-18T09:22:52Z", "last_modified_at": "2015-09-18T09:22:52Z" } response_content_type = "application/json" response_location = "" +response_headers = [] diff --git a/example/flavors/candidate/endpointmehcandidate.flavor b/example/flavors/candidate/endpointmehcandidate.flavor index d541fa031..0029837af 100644 --- a/example/flavors/candidate/endpointmehcandidate.flavor +++ b/example/flavors/candidate/endpointmehcandidate.flavor @@ -8,3 +8,4 @@ response_status = "200" response_body = { "hello_meh" : "candidate" } response_content_type = "application/json" response_location = "" +response_headers = [] diff --git a/example/flavors/primary/endpointfooprimary.flavor b/example/flavors/primary/endpointfooprimary.flavor index dbc781fad..0222f2233 100644 --- a/example/flavors/primary/endpointfooprimary.flavor +++ b/example/flavors/primary/endpointfooprimary.flavor @@ -8,3 +8,4 @@ response_status = "400" response_body = { "foo" : "primary", "started_at": "2015-09-18T09:22:51Z", "last_modified_at": "2015-09-18T09:22:51Z" } response_content_type = "application/json" response_location = "" +response_headers = [] diff --git a/example/flavors/primary/endpointmehprimary.flavor b/example/flavors/primary/endpointmehprimary.flavor index f0399ff2b..acc1d3462 100644 --- a/example/flavors/primary/endpointmehprimary.flavor +++ b/example/flavors/primary/endpointmehprimary.flavor @@ -8,3 +8,4 @@ response_status = "200" response_body = { "hello_world" : "primary" } response_content_type = "application/json" response_location = "" +response_headers = [] diff --git a/example/flavors/primary/endpointprimary.flavor b/example/flavors/primary/endpointprimary.flavor index 817fa9a8b..0e1ec52bf 100644 --- a/example/flavors/primary/endpointprimary.flavor +++ b/example/flavors/primary/endpointprimary.flavor @@ -8,3 +8,4 @@ response_status = "200" response_body = { "sizes": { "ipad": { "h": 313, "w": 626, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad" }, "ipad_retina": { "h": 626, "w": 1252, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad_retina" }, "web": { "h": 260, "w": 520, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web" }, "web_retina": { "h": 520, "w": 1040, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web_retina" }, "mobile": { "h": 160, "w": 320, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile" }, "mobile_retina": { "h": 320, "w": 640, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile_retina" }, "300x100": { "h": 100, "w": 300, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/300x100" }, "600x200": { "h": 200, "w": 600, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/600x200" }, "1500x500": { "h": 500, "w": 1500, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/1500x500" } } } response_content_type = "application/json" response_location = "" +response_headers = [] diff --git a/example/flavors/secondary/endpointfoosecondary.flavor b/example/flavors/secondary/endpointfoosecondary.flavor index 9f51ec6d4..e0d5da4f8 100644 --- a/example/flavors/secondary/endpointfoosecondary.flavor +++ b/example/flavors/secondary/endpointfoosecondary.flavor @@ -8,3 +8,4 @@ response_status = "400" response_body = { "foo" : "secondary", "started_at": "2015-09-18T09:22:53Z", "last_modified_at": "2015-09-18T09:22:53Z" } response_content_type = "application/json" response_location = "" +response_headers = [] diff --git a/example/flavors/secondary/endpointmehsecondary.flavor b/example/flavors/secondary/endpointmehsecondary.flavor index f4cb0fa67..8d482eaf1 100644 --- a/example/flavors/secondary/endpointmehsecondary.flavor +++ b/example/flavors/secondary/endpointmehsecondary.flavor @@ -8,3 +8,4 @@ response_status = "200" response_body = { "hello_bar" : "secondary" } response_content_type = "application/json" response_location = "" +response_headers = [] diff --git a/example/flavors/secondary/endpointsecondary.flavor b/example/flavors/secondary/endpointsecondary.flavor index 817fa9a8b..0e1ec52bf 100644 --- a/example/flavors/secondary/endpointsecondary.flavor +++ b/example/flavors/secondary/endpointsecondary.flavor @@ -8,3 +8,4 @@ response_status = "200" response_body = { "sizes": { "ipad": { "h": 313, "w": 626, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad" }, "ipad_retina": { "h": 626, "w": 1252, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad_retina" }, "web": { "h": 260, "w": 520, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web" }, "web_retina": { "h": 520, "w": 1040, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web_retina" }, "mobile": { "h": 160, "w": 320, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile" }, "mobile_retina": { "h": 320, "w": 640, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile_retina" }, "300x100": { "h": 100, "w": 300, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/300x100" }, "600x200": { "h": 200, "w": 600, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/600x200" }, "1500x500": { "h": 500, "w": 1500, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/1500x500" } } } response_content_type = "application/json" response_location = "" +response_headers = [] From 918c4c07f1c89dab387eb334852f5a653cf2d238 Mon Sep 17 00:00:00 2001 From: Camilo Ribeiro Date: Thu, 12 May 2016 20:33:40 +0200 Subject: [PATCH 09/27] updating docker-compose to 1.6.0++ --- example/README.md | 2 +- example/docker-compose.yml | 64 ++++++++++++++++++++------------------ 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/example/README.md b/example/README.md index a17cd84b4..f46efbdb9 100644 --- a/example/README.md +++ b/example/README.md @@ -2,7 +2,7 @@ This is an example of diffy. ## Running -To run it you will need docker and docker-compose installed. Read more about it here: https://www.docker.com/toolbox +To run it you will need docker and docker-compose (version 1.6.0+) installed. Read more about it here: https://www.docker.com/toolbox In this directory, run docker compose up command: diff --git a/example/docker-compose.yml b/example/docker-compose.yml index 74c38b5e8..0f4864306 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -1,34 +1,36 @@ -diffy: - image: camiloribeiro/twitter-diffy - ports: - - "8888:8888" - - "31900:31900" - links: - - primary - - secondary - - candidate - command: java -jar ./target/scala-2.11/diffy-server.jar -candidate='candidate:8080' -master.primary='primary:8080' -master.secondary='secondary:8080' -service.protocol='http' -serviceName='Happy Service' -proxy.port=:31900 -admin.port=:8881 -http.port=:8888 -rootUrl='localhost:8888' +version: '2' +services: + diffy: + image: camiloribeiro/twitter-diffy + ports: + - "8888:8888" + - "31900:31900" + depends_on: + - primary + - secondary + - candidate + command: java -jar ./target/scala-2.11/diffy-server.jar -candidate='candidate:8080' -master.primary='primary:8080' -master.secondary='secondary:8080' -service.protocol='http' -serviceName='Happy Service' -proxy.port=:31900 -admin.port=:8881 -http.port=:8888 -rootUrl='localhost:8888' -candidate: - image: camiloribeiro/rest_shifter - ports: - - "9881:8080" - volumes: - - ./flavors/candidate:/root/.rest_shifter/flavors - command: -s + candidate: + image: camiloribeiro/rest_shifter + ports: + - "9881:8080" + volumes: + - ./flavors/candidate:/root/.rest_shifter/flavors + command: -s -primary: - image: camiloribeiro/rest_shifter - ports: - - "9882:8080" - volumes: - - ./flavors/primary:/root/.rest_shifter/flavors - command: -s + primary: + image: camiloribeiro/rest_shifter + ports: + - "9882:8080" + volumes: + - ./flavors/primary:/root/.rest_shifter/flavors + command: -s -secondary: - image: camiloribeiro/rest_shifter - ports: - - "9883:8080" - volumes: - - ./flavors/secondary:/root/.rest_shifter/flavors - command: -s + secondary: + image: camiloribeiro/rest_shifter + ports: + - "9883:8080" + volumes: + - ./flavors/secondary:/root/.rest_shifter/flavors + command: -s From f43d446d064ca1ba8510a6109f5c216f5801db96 Mon Sep 17 00:00:00 2001 From: james_fu Date: Wed, 3 Aug 2016 11:02:19 +0800 Subject: [PATCH 10/27] Fixing the mistyped word in the README.md of example --- example/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/README.md b/example/README.md index f46efbdb9..20fa6691a 100644 --- a/example/README.md +++ b/example/README.md @@ -29,11 +29,11 @@ There are some other resources running in the same containers. You can play with $ curl --header "Canonical-Resource: /endpoint" http://localhost:31900/endpoint $ curl --header "Canonical-Resource: /endpoint/foo" http://localhost:31900/endpoint/foo - $ curl --header "Canonical-Resource: /endpoint/meh" http://localhst:31900/endpoint/meh + $ curl --header "Canonical-Resource: /endpoint/meh" http://localhost:31900/endpoint/meh ## Applying to your own service -As you can see in the docker-compose.yml, you can replace the container with two different versions of your service and expose the ports in a way to keep the same configuratio in both services. For example: +As you can see in the docker-compose.yml, you can replace the container with two different versions of your service and expose the ports in a way to keep the same configuration in both services. For example: candidate: image: mycompany/my_service:new_version From 2224d96a223dee41720eba2aa48a2311cd5fcc81 Mon Sep 17 00:00:00 2001 From: james_fu Date: Wed, 3 Aug 2016 11:03:43 +0800 Subject: [PATCH 11/27] Add mapping of the admin port for diffy container --- example/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/example/docker-compose.yml b/example/docker-compose.yml index 0f4864306..06f2eaf4f 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -5,6 +5,7 @@ services: ports: - "8888:8888" - "31900:31900" + - "8881:8881" depends_on: - primary - secondary From ca35bd96df4ee76544bdf2350ab03820ec86a90d Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Fri, 5 Aug 2016 09:54:19 +0530 Subject: [PATCH 12/27] bumped up from openjdk7 to openjdk8 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 446b7b069..43fe48910 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ scala: - 2.11.7 jdk: - - openjdk7 + - openjdk8 - oraclejdk7 - oraclejdk8 From 8b6aacc96aa9b4715ab9fd6279bcc70eacf8d860 Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Fri, 5 Aug 2016 09:58:19 +0530 Subject: [PATCH 13/27] removed openjdk from travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 43fe48910..a72c9b23d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ scala: - 2.11.7 jdk: - - openjdk8 - oraclejdk7 - oraclejdk8 From 070cae22d18a8fd4302a8538c54475fc777d4efd Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Tue, 14 Mar 2017 09:12:15 -0700 Subject: [PATCH 14/27] removed : internal diffy documentation --- src/main/resources/templates/dashboard.mustache | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/resources/templates/dashboard.mustache b/src/main/resources/templates/dashboard.mustache index dff304360..a9a3df837 100644 --- a/src/main/resources/templates/dashboard.mustache +++ b/src/main/resources/templates/dashboard.mustache @@ -110,7 +110,6 @@ [[info.last_reset | ago]] ([[info.last_reset | formatDate]])

Thresholds

[[info.threshold_relative]]% relative, [[info.threshold_absolute]]% absolute - (more on thresholds)
@@ -261,4 +260,4 @@ - \ No newline at end of file + From f976dad9267739d196dd3508213c37417c82e498 Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Fri, 7 Apr 2017 11:38:46 -0700 Subject: [PATCH 15/27] added body extraction for post requests --- .../com/twitter/diffy/lifter/HttpLifter.scala | 11 +++++----- .../twitter/diffy/lifter/HttpLifterSpec.scala | 21 +++++++++++++++++-- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala b/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala index 0cd1a678f..d221815fb 100644 --- a/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala +++ b/src/main/scala/com/twitter/diffy/lifter/HttpLifter.scala @@ -47,7 +47,8 @@ class HttpLifter(excludeHttpHeadersComparison: Boolean) { def liftRequest(req: HttpRequest): Future[Message] = { val canonicalResource = Option(req.headers.get("Canonical-Resource")) - Future.value(Message(canonicalResource, FieldMap(Map("request"-> req.toString)))) + val body = req.getContent.copy.toString(Charsets.Utf8) + Future.value(Message(canonicalResource, FieldMap(Map("request"-> req.toString, "body" -> body)))) } def liftResponse(resp: Try[HttpResponse]): Future[Message] = { @@ -70,7 +71,7 @@ class HttpLifter(excludeHttpHeadersComparison: Boolean) { /** When Content-Type is set as application/json, lift as Json **/ case (Some(mediaType), _) if mediaType.is(MediaType.JSON_UTF_8) || mediaType.toString == "application/json" => { val jsonContentTry = Try { - JsonLifter.decode(r.getContent.toString(Charsets.Utf8)) + JsonLifter.decode(r.getContent.copy.toString(Charsets.Utf8)) } Future.const(jsonContentTry map { jsonContent => @@ -90,9 +91,9 @@ class HttpLifter(excludeHttpHeadersComparison: Boolean) { /** When Content-Type is set as text/html, lift as Html **/ case (Some(mediaType), _) if mediaType.is(MediaType.HTML_UTF_8) || mediaType.toString == "text/html" => { - val htmlContentTry = Try { - HtmlLifter.lift(HtmlLifter.decode(r.getContent.toString(Charsets.Utf8))) - } + val htmlContentTry = Try { + HtmlLifter.lift(HtmlLifter.decode(r.getContent.copy.toString(Charsets.Utf8))) + } Future.const(htmlContentTry map { htmlContent => val responseMap = Map( diff --git a/src/test/scala/com/twitter/diffy/lifter/HttpLifterSpec.scala b/src/test/scala/com/twitter/diffy/lifter/HttpLifterSpec.scala index 8d6dda46b..231652ba4 100644 --- a/src/test/scala/com/twitter/diffy/lifter/HttpLifterSpec.scala +++ b/src/test/scala/com/twitter/diffy/lifter/HttpLifterSpec.scala @@ -49,8 +49,11 @@ class HttpLifterSpec extends ParentSpec { val testException = new Exception("test exception") - def request(method: HttpMethod, uri: String): HttpRequest = - new DefaultHttpRequest(HttpVersion.HTTP_1_1, method, uri) + def request(method: HttpMethod, uri: String, body: Option[String] = None): HttpRequest = { + val req = new DefaultHttpRequest(HttpVersion.HTTP_1_1, method, uri) + body foreach { b => req.setContent(ChannelBuffers.wrappedBuffer(b.getBytes(Charsets.Utf8)))} + req + } def response(status: HttpResponseStatus, body: String): HttpResponse = { val resp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, status) @@ -77,6 +80,20 @@ class HttpLifterSpec extends ParentSpec { msg.endpoint.get should equal ("endpoint") resultFieldMap.get("request").get should equal (req.toString) } + + it("lift simple Post request") { + val lifter = new HttpLifter(false) + val requestBody = "request_body" + val req = request(HttpMethod.POST, reqUri, Some(requestBody)) + req.headers().add("Canonical-Resource", "endpoint") + + val msg = Await.result(lifter.liftRequest(req)) + val resultFieldMap = msg.result.asInstanceOf[FieldMap[String]] + + msg.endpoint.get should equal ("endpoint") + resultFieldMap.get("request").get should equal (req.toString) + resultFieldMap.get("body").get should equal (requestBody) + } } describe("LiftResponse") { From 7b10b203106dd9e96957eae7a131ba9becd40164 Mon Sep 17 00:00:00 2001 From: Gabriel Tobias Date: Thu, 24 Mar 2016 15:58:54 -0700 Subject: [PATCH 16/27] Add support to HTTPS APIs --- .../twitter/diffy/DiffyServiceModule.scala | 6 ++++- .../twitter/diffy/proxy/DifferenceProxy.scala | 1 + .../diffy/proxy/HttpDifferenceProxy.scala | 25 +++++++++++++++++++ .../com/twitter/diffy/proxy/Settings.scala | 3 ++- .../scala/com/twitter/diffy/TestHelper.scala | 3 ++- 5 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala b/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala index cbdc46e34..b2c13be93 100644 --- a/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala +++ b/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala @@ -70,6 +70,9 @@ object DiffyServiceModule extends TwitterModule { val skipEmailsWhenNoErrors = flag[Boolean]("skipEmailsWhenNoErrors", false, "Do not send emails if there are no critical errors") + var httpsPort = + flag[String]("httpsPort", "443", "Port to be used when using HTTPS as a protocol") + @Provides @Singleton def settings = @@ -93,7 +96,8 @@ object DiffyServiceModule extends TwitterModule { rootUrl(), allowHttpSideEffects(), excludeHttpHeadersComparison(), - skipEmailsWhenNoErrors() + skipEmailsWhenNoErrors(), + httpsPort() ) @Provides diff --git a/src/main/scala/com/twitter/diffy/proxy/DifferenceProxy.scala b/src/main/scala/com/twitter/diffy/proxy/DifferenceProxy.scala index 76405c611..0e36d64b0 100644 --- a/src/main/scala/com/twitter/diffy/proxy/DifferenceProxy.scala +++ b/src/main/scala/com/twitter/diffy/proxy/DifferenceProxy.scala @@ -21,6 +21,7 @@ object DifferenceProxyModule extends TwitterModule { settings.protocol match { case "thrift" => ThriftDifferenceProxy(settings, collector, joinedDifferences, analyzer) case "http" => SimpleHttpDifferenceProxy(settings, collector, joinedDifferences, analyzer) + case "https" => SimpleHttpsDifferenceProxy(settings, collector, joinedDifferences, analyzer) } } diff --git a/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala b/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala index 7da309138..9d4fc0ac7 100644 --- a/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala +++ b/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala @@ -85,4 +85,29 @@ case class SimpleHttpDifferenceProxy ( Filter.identity andThenIf (!settings.allowHttpSideEffects, httpSideEffectsFilter) andThen super.proxy +} + +/** + * Alternative to SimpleHttpDifferenceProxy allowing HTTPS requests + */ +case class SimpleHttpsDifferenceProxy ( + settings: Settings, + collector: InMemoryDifferenceCollector, + joinedDifferences: JoinedDifferences, + analyzer: DifferenceAnalyzer) + extends HttpDifferenceProxy +{ + import SimpleHttpDifferenceProxy._ + + override val servicePort = settings.servicePort + + override val proxy = + Filter.identity andThenIf + (!settings.allowHttpSideEffects, httpSideEffectsFilter) andThen + super.proxy + + override def serviceFactory(serverset: String, label: String) = + HttpService(Http.client + .withTls(serverset) + .newService(serverset+":"+settings.httpsPort, label)) } \ No newline at end of file diff --git a/src/main/scala/com/twitter/diffy/proxy/Settings.scala b/src/main/scala/com/twitter/diffy/proxy/Settings.scala index 7e4c6c335..7ed4c9511 100644 --- a/src/main/scala/com/twitter/diffy/proxy/Settings.scala +++ b/src/main/scala/com/twitter/diffy/proxy/Settings.scala @@ -24,6 +24,7 @@ case class Settings( rootUrl: String, allowHttpSideEffects: Boolean, excludeHttpHeadersComparison: Boolean, - skipEmailsWhenNoErrors: Boolean) + skipEmailsWhenNoErrors: Boolean, + httpsPort: String) case class Target(path: String) diff --git a/src/test/scala/com/twitter/diffy/TestHelper.scala b/src/test/scala/com/twitter/diffy/TestHelper.scala index 3fe3ffe53..4e89440ac 100644 --- a/src/test/scala/com/twitter/diffy/TestHelper.scala +++ b/src/test/scala/com/twitter/diffy/TestHelper.scala @@ -29,7 +29,8 @@ object TestHelper extends MockitoSugar { rootUrl = "test", allowHttpSideEffects = true, excludeHttpHeadersComparison = true, - skipEmailsWhenNoErrors = false + skipEmailsWhenNoErrors = false, + httpsPort = "443" ) def makeEmptyJoinedDifferences = { From b58c8bab87baeeb42d2c0f51504386afa08159e7 Mon Sep 17 00:00:00 2001 From: Gabriel Tobias Date: Thu, 24 Mar 2016 16:10:04 -0700 Subject: [PATCH 17/27] Update README --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 0617a5494..541b214f7 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,15 @@ Inside the example directory you will find instructions to run a complete exampl ## FAQ's For safety reasons `POST`, `PUT`, ` DELETE ` are ignored by default . Add ` -allowHttpSideEffects=true ` to your command line arguments to enable these verbs. +## HTTPS +If you are trying to run Diffy over a HTTPS API, the config required is: + ``` + -service.protocol=https + ``` +And in case of the HTTPS port be different than 443: + ``` + -https.port=123 + ``` ## License From 5d1944618f7387ddfddfd99909804c69f2f14dc9 Mon Sep 17 00:00:00 2001 From: Gabriel Tobias Date: Fri, 25 Mar 2016 14:33:27 -0700 Subject: [PATCH 18/27] Update travis file, trying to fix the build --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index a72c9b23d..9dcdf9227 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ -sudo: true language: scala scala: @@ -12,13 +11,14 @@ jdk: cache: directories: - $HOME/.ivy2/cache - - $HOME/.sbt/boot + - $HOME/.sbt/boot/scala-$TRAVIS_SCALA_VERSION -script: - - unset SBT_OPTS - - ./sbt ++$TRAVIS_SCALA_VERSION clean coverage test coverageReport +before_script: unset SBT_OPTS + +script: sbt clean coverage test coverageReport + +after_script: - find $HOME/.sbt -name "*.lock" | xargs rm - find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm after_success: bash <(curl -s https://codecov.io/bash) - From b4ecc7cbe5d628dbdbba5aa7d76926ef80ad5757 Mon Sep 17 00:00:00 2001 From: Gabriel Such Date: Thu, 24 Mar 2016 16:12:58 -0700 Subject: [PATCH 19/27] Update README.md --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 541b214f7..219b33680 100644 --- a/README.md +++ b/README.md @@ -91,13 +91,12 @@ Inside the example directory you will find instructions to run a complete exampl ## HTTPS If you are trying to run Diffy over a HTTPS API, the config required is: - ``` + -service.protocol=https - ``` + And in case of the HTTPS port be different than 443: - ``` + -https.port=123 - ``` ## License From 93cb9392ceabbbb1b43aa337663e6662e7227ca5 Mon Sep 17 00:00:00 2001 From: Gabriel Tobias Date: Mon, 28 Mar 2016 10:40:17 -0700 Subject: [PATCH 20/27] Update protocol description (include https) --- src/main/scala/com/twitter/diffy/DiffyServiceModule.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala b/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala index b2c13be93..a7f18e2d6 100644 --- a/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala +++ b/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala @@ -26,7 +26,7 @@ object DiffyServiceModule extends TwitterModule { flag[String]("master.secondary", "secondary master serverset where known good code is deployed") val protocol = - flag[String]("service.protocol", "Service protocol, thrift or http") + flag[String]("service.protocol", "Service protocol, thrift, http or https") val clientId = flag[String]("proxy.clientId", "diffy.proxy", "The clientId to be used by the proxy service to talk to candidate, primary, and master") From 71055cf52ab6816dbbe6b0421e5925cdb430dd08 Mon Sep 17 00:00:00 2001 From: Gabriel Tobias Date: Mon, 28 Mar 2016 10:48:09 -0700 Subject: [PATCH 21/27] minor - update protocol description --- src/main/scala/com/twitter/diffy/DiffyServiceModule.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala b/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala index a7f18e2d6..84cfffb4c 100644 --- a/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala +++ b/src/main/scala/com/twitter/diffy/DiffyServiceModule.scala @@ -26,7 +26,7 @@ object DiffyServiceModule extends TwitterModule { flag[String]("master.secondary", "secondary master serverset where known good code is deployed") val protocol = - flag[String]("service.protocol", "Service protocol, thrift, http or https") + flag[String]("service.protocol", "Service protocol: thrift, http or https") val clientId = flag[String]("proxy.clientId", "diffy.proxy", "The clientId to be used by the proxy service to talk to candidate, primary, and master") From 807bd30d353a0ef8c8b25c52ae143c2d4270ef91 Mon Sep 17 00:00:00 2001 From: Gabriel Tobias Date: Mon, 23 May 2016 12:08:04 -0700 Subject: [PATCH 22/27] minor - code format / code review suggestions --- .../twitter/diffy/proxy/HttpDifferenceProxy.scala | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala b/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala index 9d4fc0ac7..eb17b5a29 100644 --- a/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala +++ b/src/main/scala/com/twitter/diffy/proxy/HttpDifferenceProxy.scala @@ -91,10 +91,10 @@ case class SimpleHttpDifferenceProxy ( * Alternative to SimpleHttpDifferenceProxy allowing HTTPS requests */ case class SimpleHttpsDifferenceProxy ( - settings: Settings, - collector: InMemoryDifferenceCollector, - joinedDifferences: JoinedDifferences, - analyzer: DifferenceAnalyzer) + settings: Settings, + collector: InMemoryDifferenceCollector, + joinedDifferences: JoinedDifferences, + analyzer: DifferenceAnalyzer) extends HttpDifferenceProxy { import SimpleHttpDifferenceProxy._ @@ -107,7 +107,9 @@ case class SimpleHttpsDifferenceProxy ( super.proxy override def serviceFactory(serverset: String, label: String) = - HttpService(Http.client + HttpService( + Http.client .withTls(serverset) - .newService(serverset+":"+settings.httpsPort, label)) + .newService(serverset+":"+settings.httpsPort, label) + ) } \ No newline at end of file From 42b0d8631d3b0ad34eb5e316b8ee97a2905629cd Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Mon, 13 Nov 2017 03:00:35 -0800 Subject: [PATCH 23/27] Added simple example and updated README instructions --- .gitignore | 1 + README.md | 30 ++++---- example.sh | 11 --- example/ExampleServers.java | 71 +++++++++++++++++++ example/README.md | 60 ---------------- example/docker-compose.yml | 37 ---------- .../candidate/endpointcandidate.flavor | 11 --- .../candidate/endpointfoocandidate.flavor | 11 --- .../candidate/endpointmehcandidate.flavor | 11 --- .../flavors/primary/endpointfooprimary.flavor | 11 --- .../flavors/primary/endpointmehprimary.flavor | 11 --- .../flavors/primary/endpointprimary.flavor | 11 --- .../secondary/endpointfoosecondary.flavor | 11 --- .../secondary/endpointmehsecondary.flavor | 11 --- .../secondary/endpointsecondary.flavor | 11 --- example/run.sh | 30 ++++++++ 16 files changed, 115 insertions(+), 224 deletions(-) delete mode 100755 example.sh create mode 100644 example/ExampleServers.java delete mode 100644 example/README.md delete mode 100644 example/docker-compose.yml delete mode 100644 example/flavors/candidate/endpointcandidate.flavor delete mode 100644 example/flavors/candidate/endpointfoocandidate.flavor delete mode 100644 example/flavors/candidate/endpointmehcandidate.flavor delete mode 100644 example/flavors/primary/endpointfooprimary.flavor delete mode 100644 example/flavors/primary/endpointmehprimary.flavor delete mode 100644 example/flavors/primary/endpointprimary.flavor delete mode 100644 example/flavors/secondary/endpointfoosecondary.flavor delete mode 100644 example/flavors/secondary/endpointmehsecondary.flavor delete mode 100644 example/flavors/secondary/endpointsecondary.flavor create mode 100755 example/run.sh diff --git a/.gitignore b/.gitignore index 42cfbcc9c..4e17ae11f 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ lib_managed/ .ivyjars out/ sbt-launch.jar +example/*.class diff --git a/README.md b/README.md index 219b33680..2ec3ff4fb 100644 --- a/README.md +++ b/README.md @@ -43,22 +43,22 @@ candidate disagree with each other. If these measurements are roughly the same, determines that there is nothing wrong and that the error can be ignored. ## How to get started? +# Running the example +The example.sh script included here builds and launches example servers as well as a diffy instance. Verify +that the following ports are available (9000, 9100, 9200, 8880, 8881, & 8888) and run `./example/run.sh start`. -First, you need to build Diffy by invoking `./sbt assembly` from your diffy directory. This will create -a diffy jar at `diffy/target/scala-2.11/diffy-server.jar`. - -Diffy comes bundled with an example.sh script that you can run to start comparing examples instances -we have already deployed online. Once your local Diffy instance is deployed, you send it a few requests -via `curl --header "Canonical-Resource: Html" localhost:8880` and `curl --header "Canonical-Resource: Json" localhost:8880/json`. You can then go to your browser at +Once your local Diffy instance is deployed, you send it a few requests +like `curl --header "Canonical-Resource: Json" localhost:8880/json?Twitter`. You can then go to your browser at [http://localhost:8888](http://localhost:8888) to see what the differences across our example instances look like. +# Digging deeper That was cool but now you want to compare old and new versions of your own service. Here’s how you can start using Diffy to compare three instances of your service: 1. Deploy your old code to `localhost:9990`. This is your primary. 2. Deploy your old code to `localhost:9991`. This is your secondary. 3. Deploy your new code to `localhost:9992`. This is your candidate. -4. Download the latest Diffy binary or build your own from the code. +4. Download the latest Diffy binary from maven central or build your own from the code using `./sbt assembly`. 5. Run the Diffy jar with following command line arguments: ``` @@ -68,24 +68,20 @@ start using Diffy to compare three instances of your service: -master.secondary=localhost:9991 \ -service.protocol=http \ -serviceName=My-Service \ - -proxy.port=:31900 \ - -admin.port=:31159 \ - -http.port=:31149 \ - -rootUrl='localhost:31149' + -proxy.port=:8880 \ + -admin.port=:8881 \ + -http.port=:8888 \ + -rootUrl='localhost:8888' ``` 6. Send a few test requests to your Diffy instance on its proxy port: ``` - curl localhost:31900/your/application/route?with=queryparams + curl localhost:8880/your/application/route?with=queryparams ``` -7. Watch the differences show up in your browser at [http://localhost:31149](http://localhost:31149). - -### Running example on docker-compose +7. Watch the differences show up in your browser at [http://localhost:8888](http://localhost:8888). -Inside the example directory you will find instructions to run a complete example with apis and diffy configured and ready to run using docker-compose. - ## FAQ's For safety reasons `POST`, `PUT`, ` DELETE ` are ignored by default . Add ` -allowHttpSideEffects=true ` to your command line arguments to enable these verbs. diff --git a/example.sh b/example.sh deleted file mode 100755 index 4c3eceae8..000000000 --- a/example.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -java -jar ./target/scala-2.11/diffy-server.jar \ --candidate='localhost:9200' \ --master.primary='localhost:9000' \ --master.secondary='localhost:9100' \ --service.protocol='http' \ --serviceName='My Service' \ --proxy.port=:8880 \ --admin.port=:8881 \ --http.port=:8888 \ --rootUrl='localhost:8888' diff --git a/example/ExampleServers.java b/example/ExampleServers.java new file mode 100644 index 000000000..7dbc2719e --- /dev/null +++ b/example/ExampleServers.java @@ -0,0 +1,71 @@ +import java.net.InetSocketAddress; +import java.io.IOException; +import java.io.OutputStream; +import java.util.function.Function; +import java.util.stream.Stream; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; + +public class ExampleServers { + public static void main(String[] args) throws Exception { + int primary = Integer.parseInt(args[0]); + int secondary = Integer.parseInt(args[1]); + int candidate = Integer.parseInt(args[2]); + Thread p = new Thread(() -> bind(primary, x -> x.toLowerCase())); + Thread s = new Thread(() -> bind(secondary, x -> x.toLowerCase())); + Thread c = new Thread(() -> bind(candidate, x -> x.toUpperCase())); + p.start(); + s.start(); + c.start(); + while(true){ + Thread.sleep(10); + } + } + + public static void bind(int port, Function lambda) { + try { + HttpServer server = HttpServer.create(new InetSocketAddress(port), 0); + server.createContext( + "/json", + new Handler( + "{\"name\":\"%s\", \"timestamp\":\"%s\"}", + "application/json", + lambda)); + server.createContext( + "/html", + new Handler( + "%s%s", + "text/html", + lambda)); + server.setExecutor(null); + server.start(); + } catch (Exception exception) { + System.err.println("!!!failed to start!!!"); + } + } +} +class Handler implements HttpHandler { + private String template; + private String contentType; + private Function lambda; + public Handler(String template, String contentType, Function lambda) { + super(); + this.template = template; + this.contentType = contentType; + this.lambda = lambda; + } + + @Override + public void handle(HttpExchange t) throws IOException { + String name = lambda.apply(t.getRequestURI().getQuery()); + String response = String.format(template, name, System.currentTimeMillis()); + System.out.println(response); + t.getResponseHeaders().add("Content-Type", contentType); + t.sendResponseHeaders(200, response.length()); + OutputStream os = t.getResponseBody(); + os.write(response.getBytes()); + os.close(); + } +} \ No newline at end of file diff --git a/example/README.md b/example/README.md deleted file mode 100644 index 20fa6691a..000000000 --- a/example/README.md +++ /dev/null @@ -1,60 +0,0 @@ -This is an example of diffy. - -## Running - -To run it you will need docker and docker-compose (version 1.6.0+) installed. Read more about it here: https://www.docker.com/toolbox - -In this directory, run docker compose up command: - - $ docker-compose up - -It will download the needed images and start the application. You will notice that the logs for all the four services: the twitter diffy proxy, the candidate service, the primary and secondary services. -You will be able to access the web console for diffy on http://localhost:8888/ - -Now you can curl to check the differences: - - $ curl http://localhost:31900/endpoint - -In the web console you will be able to see the diffs. - -To change the services the only thing you need to do is to change the three flavor files withing the flavors directory, candidate, primary and secondary. Whatever you change there will change when you restart the docker-compose. - -ps: If you are running on mac or windows, remember to use the docker-machine/boot2docker/whereveryourundockeron ip instead of localhost, or do port forward it to localhost. - -Have fun! - -## Digging a bit more - -There are some other resources running in the same containers. You can play with it with the following curls: - - $ curl --header "Canonical-Resource: /endpoint" http://localhost:31900/endpoint - $ curl --header "Canonical-Resource: /endpoint/foo" http://localhost:31900/endpoint/foo - $ curl --header "Canonical-Resource: /endpoint/meh" http://localhost:31900/endpoint/meh - -## Applying to your own service - -As you can see in the docker-compose.yml, you can replace the container with two different versions of your service and expose the ports in a way to keep the same configuration in both services. For example: - - candidate: - image: mycompany/my_service:new_version - ports: - - "8080" - - primary: - image: mycompany/my_service:stable_version - ports: - - "8080" - - secondary: - image: mycompany/my_service:stable_version - ports: - - "8080" - -Than run docker-compose up again. - -## What is the service being tested here? - -It is a rest-shifter service. Easy way to create simple service mocks and prototypes. -Read more: https://github.com/camiloribeiro/restshifter - -This example was originaly posted on https://github.com/camiloribeiro/dockdiffy under the Apache License, Version 2.0 (the "License"). diff --git a/example/docker-compose.yml b/example/docker-compose.yml deleted file mode 100644 index 06f2eaf4f..000000000 --- a/example/docker-compose.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: '2' -services: - diffy: - image: camiloribeiro/twitter-diffy - ports: - - "8888:8888" - - "31900:31900" - - "8881:8881" - depends_on: - - primary - - secondary - - candidate - command: java -jar ./target/scala-2.11/diffy-server.jar -candidate='candidate:8080' -master.primary='primary:8080' -master.secondary='secondary:8080' -service.protocol='http' -serviceName='Happy Service' -proxy.port=:31900 -admin.port=:8881 -http.port=:8888 -rootUrl='localhost:8888' - - candidate: - image: camiloribeiro/rest_shifter - ports: - - "9881:8080" - volumes: - - ./flavors/candidate:/root/.rest_shifter/flavors - command: -s - - primary: - image: camiloribeiro/rest_shifter - ports: - - "9882:8080" - volumes: - - ./flavors/primary:/root/.rest_shifter/flavors - command: -s - - secondary: - image: camiloribeiro/rest_shifter - ports: - - "9883:8080" - volumes: - - ./flavors/secondary:/root/.rest_shifter/flavors - command: -s diff --git a/example/flavors/candidate/endpointcandidate.flavor b/example/flavors/candidate/endpointcandidate.flavor deleted file mode 100644 index 06c403470..000000000 --- a/example/flavors/candidate/endpointcandidate.flavor +++ /dev/null @@ -1,11 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "200" -response_body = { "sizes": { "ipad": { "h": 313, "w": 621, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad" }, "ipad_retina": { "h": 626, "w": 1252, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad_retina" }, "web": { "h": 260, "w": 520, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web" }, "web_retina": { "h": 520, "w": 1040, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web_retina" }, "mobile": { "h": 160, "w": 320, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile" }, "mobile_retina": { "h": 320, "w": 640, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile_retina" }, "300x100": { "h": 100, "w": 300, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/300x100" }, "600x200": { "h": 200, "w": 600, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/600x200" }, "1500x500": { "h": 500, "w": 1500, "urls": "https://pbs.twimg.com/profile_banners/6253282/1347394302/1500x500" } } } -response_content_type = "application/json" -response_location = "" -response_headers = [] diff --git a/example/flavors/candidate/endpointfoocandidate.flavor b/example/flavors/candidate/endpointfoocandidate.flavor deleted file mode 100644 index c08b0f5ba..000000000 --- a/example/flavors/candidate/endpointfoocandidate.flavor +++ /dev/null @@ -1,11 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint/foo" -request_accept = "" -request_content_type = "" -response_sleep = 1 -response_status = "400" -response_body = { "foo" : "candidate", "started_at": "2015-09-18T09:22:52Z", "last_modified_at": "2015-09-18T09:22:52Z" } -response_content_type = "application/json" -response_location = "" -response_headers = [] diff --git a/example/flavors/candidate/endpointmehcandidate.flavor b/example/flavors/candidate/endpointmehcandidate.flavor deleted file mode 100644 index 0029837af..000000000 --- a/example/flavors/candidate/endpointmehcandidate.flavor +++ /dev/null @@ -1,11 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint/meh" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "200" -response_body = { "hello_meh" : "candidate" } -response_content_type = "application/json" -response_location = "" -response_headers = [] diff --git a/example/flavors/primary/endpointfooprimary.flavor b/example/flavors/primary/endpointfooprimary.flavor deleted file mode 100644 index 0222f2233..000000000 --- a/example/flavors/primary/endpointfooprimary.flavor +++ /dev/null @@ -1,11 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint/foo" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "400" -response_body = { "foo" : "primary", "started_at": "2015-09-18T09:22:51Z", "last_modified_at": "2015-09-18T09:22:51Z" } -response_content_type = "application/json" -response_location = "" -response_headers = [] diff --git a/example/flavors/primary/endpointmehprimary.flavor b/example/flavors/primary/endpointmehprimary.flavor deleted file mode 100644 index acc1d3462..000000000 --- a/example/flavors/primary/endpointmehprimary.flavor +++ /dev/null @@ -1,11 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint/meh" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "200" -response_body = { "hello_world" : "primary" } -response_content_type = "application/json" -response_location = "" -response_headers = [] diff --git a/example/flavors/primary/endpointprimary.flavor b/example/flavors/primary/endpointprimary.flavor deleted file mode 100644 index 0e1ec52bf..000000000 --- a/example/flavors/primary/endpointprimary.flavor +++ /dev/null @@ -1,11 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "200" -response_body = { "sizes": { "ipad": { "h": 313, "w": 626, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad" }, "ipad_retina": { "h": 626, "w": 1252, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad_retina" }, "web": { "h": 260, "w": 520, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web" }, "web_retina": { "h": 520, "w": 1040, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web_retina" }, "mobile": { "h": 160, "w": 320, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile" }, "mobile_retina": { "h": 320, "w": 640, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile_retina" }, "300x100": { "h": 100, "w": 300, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/300x100" }, "600x200": { "h": 200, "w": 600, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/600x200" }, "1500x500": { "h": 500, "w": 1500, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/1500x500" } } } -response_content_type = "application/json" -response_location = "" -response_headers = [] diff --git a/example/flavors/secondary/endpointfoosecondary.flavor b/example/flavors/secondary/endpointfoosecondary.flavor deleted file mode 100644 index e0d5da4f8..000000000 --- a/example/flavors/secondary/endpointfoosecondary.flavor +++ /dev/null @@ -1,11 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint/foo" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "400" -response_body = { "foo" : "secondary", "started_at": "2015-09-18T09:22:53Z", "last_modified_at": "2015-09-18T09:22:53Z" } -response_content_type = "application/json" -response_location = "" -response_headers = [] diff --git a/example/flavors/secondary/endpointmehsecondary.flavor b/example/flavors/secondary/endpointmehsecondary.flavor deleted file mode 100644 index 8d482eaf1..000000000 --- a/example/flavors/secondary/endpointmehsecondary.flavor +++ /dev/null @@ -1,11 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint/meh" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "200" -response_body = { "hello_bar" : "secondary" } -response_content_type = "application/json" -response_location = "" -response_headers = [] diff --git a/example/flavors/secondary/endpointsecondary.flavor b/example/flavors/secondary/endpointsecondary.flavor deleted file mode 100644 index 0e1ec52bf..000000000 --- a/example/flavors/secondary/endpointsecondary.flavor +++ /dev/null @@ -1,11 +0,0 @@ -## ! ~/.rest_shifter/flavors/hello_world.flavor -method_used = "get" -path = "/endpoint" -request_accept = "" -request_content_type = "" -response_sleep = 0 -response_status = "200" -response_body = { "sizes": { "ipad": { "h": 313, "w": 626, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad" }, "ipad_retina": { "h": 626, "w": 1252, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/ipad_retina" }, "web": { "h": 260, "w": 520, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web" }, "web_retina": { "h": 520, "w": 1040, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/web_retina" }, "mobile": { "h": 160, "w": 320, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile" }, "mobile_retina": { "h": 320, "w": 640, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/mobile_retina" }, "300x100": { "h": 100, "w": 300, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/300x100" }, "600x200": { "h": 200, "w": 600, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/600x200" }, "1500x500": { "h": 500, "w": 1500, "url": "https://pbs.twimg.com/profile_banners/6253282/1347394302/1500x500" } } } -response_content_type = "application/json" -response_location = "" -response_headers = [] diff --git a/example/run.sh b/example/run.sh new file mode 100755 index 000000000..df1e51522 --- /dev/null +++ b/example/run.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +if [ "$1" = "start" ]; +then + + # Build primary, secondary, and candidate servers + javac example/ExampleServers.java && \ + + # Deploy primary, secondary, and candidate servers + java -cp example ExampleServers 9000 9100 9200 & \ + + # Build diffy + ./sbt assembly && \ + + # Deploy diffy + java -jar ./target/scala-2.11/diffy-server.jar \ + -candidate='localhost:9200' \ + -master.primary='localhost:9000' \ + -master.secondary='localhost:9100' \ + -service.protocol='http' \ + -serviceName='My Service' \ + -proxy.port=:8880 \ + -admin.port=:8881 \ + -http.port=:8888 \ + -rootUrl='localhost:8888' &\ + +else + echo "Please make sure ports 9000, 9100, 9200, 8880, 8881, & 8888 are available before running \"example/run.sh start\"" +fi + From eea03db03bbcaef5474678041943079fcdadcb77 Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Sat, 6 Jan 2018 17:21:33 -0800 Subject: [PATCH 24/27] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2ec3ff4fb..7b98ae6e1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Diffy +[![GitHub license][license-badge]](LICENSE) [![Build status](https://img.shields.io/travis/twitter/diffy/master.svg)](https://travis-ci.org/twitter/diffy) [![Coverage status](https://img.shields.io/codecov/c/github/twitter/diffy/master.svg)](https://codecov.io/github/twitter/diffy) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) From 787bfc2b6a41062c7db63246888be97ba8314886 Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Sat, 6 Jan 2018 17:30:44 -0800 Subject: [PATCH 25/27] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b98ae6e1..640b7928d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Diffy -[![GitHub license][license-badge]](LICENSE) +[![GitHub license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) [![Build status](https://img.shields.io/travis/twitter/diffy/master.svg)](https://travis-ci.org/twitter/diffy) [![Coverage status](https://img.shields.io/codecov/c/github/twitter/diffy/master.svg)](https://codecov.io/github/twitter/diffy) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) From b348351bd68a8cc6a6b466b913601dd0d797ffc5 Mon Sep 17 00:00:00 2001 From: Puneet Khanduri Date: Mon, 8 Jan 2018 09:26:02 -0800 Subject: [PATCH 26/27] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 640b7928d..10b4ec1a5 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ the running instances. It then compares the responses, and reports any regressio from those comparisons. The premise for Diffy is that if two implementations of the service return “similar” responses for a sufficiently large and diverse set of requests, then the two implementations can be treated as equivalent and the newer implementation is regression-free. +For a more detailed analysis of Diffy checkout this [blogpost](https://blog.twitter.com/engineering/en_us/a/2015/diffy-testing-services-without-writing-tests.html). ## How does Diffy work? @@ -38,6 +39,7 @@ things: 2. Non-deterministic noise observed between the primary and secondary instances. Since both of these instances are running known-good code, you should expect responses to be in agreement. If not, your service may have non-deterministic behavior, which is to be expected. +![Diffy Topology](https://g.twimg.com/blog/blog/image/Diffy_2.png) Diffy measures how often primary and secondary disagree with each other vs. how often primary and candidate disagree with each other. If these measurements are roughly the same, then Diffy From af1cb5f79b60df5f7cdd17de9fd53c697612f250 Mon Sep 17 00:00:00 2001 From: mgifos Date: Wed, 7 Mar 2018 10:41:31 +0100 Subject: [PATCH 27/27] Exclude HTTP headers in Diffy comparison by default --- src/universal/conf/application.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/universal/conf/application.ini b/src/universal/conf/application.ini index d1d8e208c..f9ae10820 100644 --- a/src/universal/conf/application.ini +++ b/src/universal/conf/application.ini @@ -6,4 +6,5 @@ -proxy.port=:8880 -admin.port=:8881 -http.port=:8888 --rootUrl=localhost:8888 \ No newline at end of file +-rootUrl=localhost:8888 +-excludeHttpHeadersComparison=true