diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a97ce0b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# Enforce Unix newlines +*.conf text eol=lf +*.sbt text eol=lf +*.scala text eol=lf +*.sh text eol=lf +*.md text eol=lf +*.txt text eol=lf +*.yml text eol=lf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7d3a34f --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +*.class +*.log + +# sbt specific +.cache/ +.history/ +.lib/ +dist/* +target/ +lib_managed/ +src_managed/ +project/boot/ +project/plugins/project/ + +# Scala-IDE specific +.scala_dependencies +.worksheet +.idea + +# Safeguard login creds +src/main/resources/application.conf diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..3f0bc12 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,3 @@ +language: scala +scala: + - 2.10.4 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..9c23ed1 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,23 @@ +Hacking on Savage +================= +## How do I build Savage? +1. [Install sbt](http://www.scala-sbt.org/download.html) +2. Go to your `savage` directory. +3. Run `sbt compile` + +## How do I run the Savage service locally for test purposes? +**This method is not recommended for use in production deployments!** + +0. Ensure that sbt is installed (see above). +1. Go to your `savage` directory. +2. Run `sbt` +3. At the sbt prompt, enter `re-start 9090` (replace `9090` with whatever port you want the HTTP server to run on) or `re-start` (which will use the default port specified in `application.conf`). Note that running on ports <= 1024 requires root privileges (not recommended) or using port mapping. + +## How do I generate a single self-sufficient JAR that includes all of the necessary dependencies? +0. Ensure that sbt is installed (see above). +1. Go to your `savage` directory. +2. Run `sbt assembly` +3. If the build is successful, the desired JAR will be generated as `target/scala-2.10/savage-assembly-1.0.jar`. + +## Licensing +Savage is licensed under The MIT License. By contributing to Savage, you agree to license your contribution under [The MIT License](https://github.com/cvrebert/savage/blob/master/LICENSE.txt). diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a2dc8e0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +# Written against Docker v1.2.0 +FROM dockerfile/java +MAINTAINER Chris Rebert + +WORKDIR / + +RUN ["apt-get", "install", "git"] +RUN ["apt-get", "install", "openssh-client"] +RUN ["useradd", "savage"] + +ADD target/scala-2.10/savage-assembly-1.0.jar /app/server.jar +ADD git-repo /app/git-repo + +ADD ssh/id_rsa.pub /home/savage/.ssh/id_rsa.pub +ADD ssh/id_rsa /home/savage/.ssh/id_rsa + +RUN ssh-keyscan -t rsa github.com > /home/savage/.ssh/known_hosts + +RUN ["chown", "-R", "savage:savage", "/home/savage/.ssh"] +RUN ["chown", "-R", "savage:savage", "/app/git-repo"] +# chmod must happen AFTER chown, due to https://github.com/docker/docker/issues/6047 +RUN ["chmod", "-R", "go-rwx", "/home/savage/.ssh"] + +WORKDIR /app/git-repo +USER savage +CMD ["java", "-jar", "/app/server.jar", "6060"] +EXPOSE 6060 diff --git a/LICENSE b/LICENSE.txt similarity index 100% rename from LICENSE rename to LICENSE.txt diff --git a/README.md b/README.md index ea4ff68..302a037 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,124 @@ -savage +Savage ====== +[![Build Status](https://travis-ci.org/cvrebert/savage.svg?branch=master)](https://travis-ci.org/cvrebert/savage) -Service that runs Sauce Labs cross-browser JS tests on Bootstrap pull requests +Savage is a service watches for new or updated pull requests on a given GitHub repository. For each pull request, it evaluates whether the changes are "safe" (i.e. we can run a Travis CI build with them with heightened permissions without worrying about security issues) and "interesting" (i.e. would benefit from a Travis CI build with them with heightened permissions), based on which files were modified. If the pull request is "safe" and "interesting", then it initiates a Travis CI build with heightened permissions on a specified GitHub repository. When the Travis CI build completes, it posts a comment with the test results on the pull request. If the test failed, the pull requester can then revise their code to fix the problem. + +Savage's original use-case is for running Sauce Labs cross-browser JS tests on pull requests via Travis CI, while keeping the Sauce Labs access credentials secure. + +Affectionately named after an experimenter known for "busting" misconceptions, often with explosives. + +## Motivation +(Savage is general enough to be used in other situations, but the following is the specific one it was built for.) + +You're a member of a popular open source project that involves front-end Web technologies. Cool. + +Specifically, the project involves JavaScript. Because it's a serious project, you have automated cross-browser testing for your JavaScript. You happen to use [Open Sauce](https://saucelabs.com/opensauce) for this. + +Unfortunately, [due to certain limitations](http://support.saucelabs.com/entries/25614798-How-can-we-set-up-an-open-source-account-that-runs-tests-on-people-s-pull-requests-), it's not possible to do cross-browser testing on pull requests "the obvious way" via Travis CI without potentially compromising your Sauce login credentials. This means that either (a) cross-browser problems aren't discovered in pull requests until after they've already been merged (b) repo collaborators must manually initiate the cross-browser tests on pull requests (and manage the resulting branches, and possibly post comments communicating the test results). + +By automating the process of initiating Travis-based Sauce tests and posting the results, cross-browser JavaScript issues can be discovered more quickly and with less work on the part of repo collaborators. + +## How it works (for the Open Sauce use-case) +1. Use GitHub webhooks to listen for new or updated pull requests in a given GitHub repository. +2. If the pull request does not modify any JavaScript files, ignore it. +3. Ensure that no sensitive build files (e.g. `.travis.yml`, `Gruntfile.js`) have been modified. +4. Clone the pull request's branch and push it to a test repo under an autogenerated name. +5. Travis CI will automatically run a build on the new branch *under the test repo's user*. Thus, this build will have access to Travis secure environment variables; in particular, it will have access to the Sauce Labs credentials. +6. Use webhooks to track the status of the Travis build. +7. When the build finishes, post a comment to the GitHub pull request explaining the test results, and delete the corresponding branch. + +## Used by +* ~~[Bootstrap](https://github.com/twbs/bootstrap); see [$GITHUB_BOT_ACCOUNT]~~ (FUTURE) +* ~~[Video.js](https://github.com/videojs/video.js); see [$GITHUB_BOT_ACCOUNT]~~ (FUTURE, MAYBE) + +## Usage +Using Savage involves two git repos (which can both be the same repo, although that's much less secure): +* The *main repo* + * This repo is the one receiving pull requests + * Savage needs its GitHub web hook set up for this repo + * Savage does NOT need to be a Collaborator on this repo +* The *test repo* + * The repo that Savage will push test branches to + * Travis CI should be set up for this repo + * Savage needs to be a Collaborator on this repo, so that it can push branches to it and also delete branches from it + +Java 7+ is required to run Savage. For instructions on building Savage yourself, see [the Contributing docs](https://github.com/cvrebert/savage/blob/master/CONTRIBUTING.md). + +Savage accepts exactly one optional command-line argument, which is the port number to run its HTTP server on, e.g. `8080`. If you don't provide this argument, the default port specified in `application.conf` will be used. Once you've built the JAR, run e.g. `java -jar savage-assembly-1.0.jar 8080` (replace `8080` with whatever port number you want). Note that running on ports <= 1024 requires root privileges (not recommended) or using port mapping. + +When running Savage, its working directory should be a non-bare git repo which is a clone of the repo being monitored. + +Savage's GitHub webhook must be setup on the main repo that will be receivi + +Other settings live in `application.conf`. In addition to the normal Akka and Spray settings, Savage offers the following settings: +``` +savage { + // Port to run on, if not specified via the command line + default-port = 6060 + // Full name of GitHub repo to watch for new pull requests + github-repo-to-watch = "twbs/bootstrap" + // Full name of GitHub repo to push test branches to + github-test-repo = "twbs/bootstrap-tests" + // List of Unix file globs constituting the whitelist of safely editable files + whitelist = [ + "**.md", + "/bower.json", + "/composer.json", + "/fonts/**.{eot,ttf,svg,woff}", + "/less/**.less", + "/sass/**.{sass,scss}", + "/js/**.{js,html,css}", + "/dist/**.{css,js,map,eot,ttf,svg,woff}", + "/docs/**.{html,css,js,map,png,ico,xml,eot,ttf,svg,woff,swf}" + ] + // List of Unix file globs constituting the watchlist of files + // which trigger a Savage build. + // To prevent unnecessary builds, a Savage build isn't triggered + // unless the pull request affects a file that matches one of the watchlist globs. + file-watchlist = [ + "/js/**/*.js" + ] + // Prefix to use for branches that Savage pushes to the main repository. + // The branch name is generated by prefixing the pull request number with this prefix. + branch-prefix = "savage-" + // GitHub login credentials for the Savage bot to use + username = throwaway9475947 + password = XXXXXXXX + // This goes in the "Secret" field when setting up the Webhook + // in the "Webhooks & Services" part of your repo's Settings. + // This string will be converted to UTF-8 for the HMAC-SHA1 computation. + // The HMAC is used to verify that Savage is really being contacted by GitHub, + // and not by some random hacker. + github-web-hook-secret-key = abcdefg + // Used as a shared secret in a hashing scheme that's used to verify + // that Savage is really being contacted by Travis CI, + // and not by some random hacker. For how to find your Travis token, + // see http://docs.travis-ci.com/user/notifications/#Authorization-for-Webhooks + travis-token = abcdefg +} +``` + +### GitHub webhook configuration + +* Payload URL: `http://your-domain.example/savage/github` +* Content type: `application/json` +* Secret: Same as your `web-hook-secret-key` config value +* Which events would you like to trigger this webhook?: "Pull Request" + +### Travis webhook configuration +In `.travis.yml`: +``` +notifications: + webhooks: + - http://your-domain.example/savage/travis +``` + +## Acknowledgments +We all stand on the shoulders of giants and get by with a little help from our friends. Savage is written in [Scala](http://www.scala-lang.org) and built on top of: +* [Akka](http://akka.io) & [Spray](http://spray.io), for async processing & HTTP +* [Eclipse EGit GitHub library](https://github.com/eclipse/egit-github), for working with [the GitHub API](https://developer.github.com/v3/) + +## See also +* [LMVTFY](https://github.com/cvrebert/lmvtfy), Savage's sister bot who does HTML validation +* [Rorschach](https://github.com/twbs/rorschach), Savage's sister bot who sanity-checks Bootstrap pull requests diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..e01bd48 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,105 @@ +## DISCLAIMER +The author is not a security expert and this project has not been subjected to a third-party security audit. + +## Responsible disclosure; Security contact info + +The security of Savage is important to us. We encourage you to report security problems to us responsibly. + +Please report all security bugs to `savage {AT} rebertia [DOT] com`. We aim to respond (with at least an acknowledgment) within one business day. We will keep you updated on the bug's status as we work towards resolving it. + +We will disclose a problem to the public once it has been confirmed and a fix has been made available. At that point, you will be credited for your discovery in the documentation, in the release announcements, and (if applicable) in the code itself. + +As Savage currently lacks corporate backing, we are unfortunately unable to offer bounty payments at this time. + +We thank you again for helping ensure the security of Savage by responsibly reporting security problems. + +## System model + +### System operation +(Note: PR = pull request) + +``` +[GitHub] >>>(Webhook notification of new/updated PR)>>> [Savage] +* Savage verifies that the notification was really from GitHub (and not an impostor) + by verifying the HMAC-SHA1 computed using the web hook secret key previously configured with GitHub. + +[GitHub] <<<(Request details about the PR using the PR's HEAD commit's SHA)<<< [Savage] +[GitHub] >>>(Response with details about the PR)>>> [Savage] +* Savage checks list of files modified by the PR against the whitelist + * If any files are outside of the whitelist, stop further processing. + +[GitHub] <<<(Request for Git data for the PR's HEAD commit via its SHA)<<< [Savage] +[GitHub] >>>(Response with Git data for the PR's HEAD commit)>>> [Savage] +* Savage generates a new branch name using the PR number and a specified prefix + +[GitHub] >>>(Fetch refs from PR's GitHub repo)>>> [Savage] +[GitHub] <<<(Pushes new branch to test repository using the PR's HEAD commit, referenced via its SHA)<<< [Savage] +[GitHub] >>>(Notifies Travis of the test repository's newly-pushed branch)>>> [Travis CI] +* Travis CI runs the build with the privileges of the test repository + * Notably, it has access to Travis CI secure environment variables + +[Travis CI] >>>(Outcome of build)>>> [Savage] +* Savage verifies that the notification was really from Travis CI (and not an impostor) + by verifying the signature in the `Authorization` header using the secret Travis user token. + +[GitHub] <<<(Post comment on PR regarding build outcome)<<< [Savage] +[GitHub] <<<(Delete branch from test repository)<<< [Savage] +``` + +Remarks: +At no point do we use the PR's branch name directly. We also delete all fetched branches after the push is completed. This avoids maliciously crafted branch names which could be misinterpreted by other systems and also ensures that the attacker cannot change the contents of the branch out from under us, thus avoiding [TOCTTOU](http://en.wikipedia.org/wiki/Time_of_check_to_time_of_use) vulnerabilities. + +## Threat model + +### Assumptions +(These are admittedly generous.) +* We trust the machine that Savage is running on +* We trust GitHub +* We trust Travis CI +* We trust that the EGit-GitHub library communicates with GitHub securely +* We assume that the git command binaries are secure so long as they are only invoked with secure arguments +* We assume that our build scripts are secure (this is outside the scope and control of Savage itself) +* We assume that the filename whitelist is correct + +### Architecture-based threat analysis +Out of scope per our assumptions: +* Compromise of GitHub +* Compromise of Travis CI API +* Compromise of the machine on which Savage resides +* Compromise of out outbound communications with GitHub +* Allowing modification of a sensitive file due to incorrect whitelist settings + +Within scope: +* Impersonating GitHub and delivering a malicious webhook notification + * Prevented by our checking of the HMAC-SHA1 signature of the webhook payload +* Impersonating Travis and delivering a malicious webhook notification + * Prevented by our checking of the SHA-256 signature of the webhook payload +* Shell-related vulnerabilities + * Avoided by not using the shell when invoking git; we use Java's `ProcessBuilder`/`Process` instead +* Compromising the git fetch/push command via malicious input + * Avoided by checking that the relevant git-related data isn't fishy +* Compromising the git branch deletion command via malicious input + * The command involves only a Savage-generated branch name, whose computation is simple and which is checked for validity. We believe this thus avoids the vulnerability. +* Compromising the contents of the posted GitHub comment via malicious input + * Avoided by checking that the relevant data from Travis isn't fishy + +### Asset-centric threat analysis +Assets: +* Savage's GitHub credentials + * We don't believe this information is leaked by Savage itself. + * We don't believe the git commands can be induced to access the relevant configuration file that has the credentials. + * Travis deserializes the API responses as vanilla JSON; it doesn't `eval()` them; spray-json doesn't have any deserialization features that allow the execution of arbitrary code (contrast this with YAML and some of its implementations). +* Write access to the test GitHub repo + * We believe that the various checks that Savage performs on the inputs and the fact that it is only capable of performing a couple git operations prevents malicious access to the test repo. +* Commenting ability on the main GitHub repo + * Savage only uses the commit SHA and the Travis build URL in its comment text, and both of these are checked for validity/safety. +* Credentials stored in Travis secure environment variables + * Under our somewhat generous assumptions, this should be impossible. + +## Notes on securing build scripts +* Beware malicious Git input (branch names, commit messages, author info, etc.) +* Beware malicious Travis input (e.g. environment variables) +* Beware potentially-executable data files (e.g. `eval()`ing of JSON, YAML custom type deserialization hooks) +* Beware the addition of files with maliciously-chosen names +* Ensure that build scripts are absent from the whitelist +* Ensure package management control files are absent from the whitelist, to prevent the installation of malicious packages diff --git a/assembly.sbt b/assembly.sbt new file mode 100644 index 0000000..473f907 --- /dev/null +++ b/assembly.sbt @@ -0,0 +1,4 @@ +import AssemblyKeys._ + +assemblySettings + diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..5c29ddc --- /dev/null +++ b/build.sbt @@ -0,0 +1,33 @@ +name := "savage" + +version := "1.0" + +scalaVersion := "2.10.4" + +mainClass := Some("com.getbootstrap.savage.server.Boot") + +resolvers ++= Seq("snapshots", "releases").map(Resolver.sonatypeRepo) + +libraryDependencies += "org.eclipse.mylyn.github" % "org.eclipse.egit.github.core" % "2.1.5" + +libraryDependencies ++= { + val akkaV = "2.3.6" + val sprayV = "1.3.2" + Seq( + "io.spray" %% "spray-can" % sprayV, + "io.spray" %% "spray-routing" % sprayV, + "io.spray" %% "spray-testkit" % sprayV % "test", + "io.spray" %% "spray-json" % "1.3.1", + "com.typesafe.akka" %% "akka-actor" % akkaV, + "com.typesafe.akka" %% "akka-testkit" % akkaV % "test", + "org.specs2" %% "specs2" % "2.3.12" % "test" + ) +} + +scalacOptions := Seq("-unchecked", "-deprecation", "-feature", "–Xlint", "-encoding", "utf8") + +scalacOptions in Test ++= Seq("-Yrangepos") + +// parallelExecution in Test := false + +Revolver.settings diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 0000000..49c4f3e --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,3 @@ +addSbtPlugin("io.spray" % "sbt-revolver" % "0.7.2") + +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2") diff --git a/setup_droplet.sh b/setup_droplet.sh new file mode 100755 index 0000000..4694069 --- /dev/null +++ b/setup_droplet.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Step 0: You need to have copied the assembly JAR to savage/target/scala-2.10/savage-assembly-1.0.jar +# Step 0.1: You need to have the git repo checked out in ./git-repo +# Step 0.2: The user's SSH public-private keys must be at ./ssh/id_rsa and ./ssh/id_rsa.pub + +# set to Pacific Time (for @cvrebert) +# ln -sf /usr/share/zoneinfo/America/Los_Angeles /etc/localtime + +# remove useless crap +aptitude remove wpasupplicant wireless-tools +aptitude remove pppconfig pppoeconf ppp + +# setup firewall +ufw default allow outgoing +ufw default deny incoming +ufw allow ssh +ufw allow www +ufw enable +ufw status verbose + +# setup Docker; written against Docker v1.2.0 +docker build . 2>&1 | tee docker.build.log +IMAGE_ID="$(tail -n 1 docker.build.log | cut -d ' ' -f 3)" +docker run -d -p 80:6060 --name savage $IMAGE_ID diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf new file mode 100644 index 0000000..63724f9 --- /dev/null +++ b/src/main/resources/application.conf @@ -0,0 +1,45 @@ +akka { + loglevel = INFO +} + +spray.can { + server { + request-timeout = 5 s + } + + client { + user-agent-header = "TwbsSavage/0.1 (https://github.com/twbs/savage)" + request-timeout = 20 s + idle-timeout = 15 s + } + host-connector { + max-connections = 5 + max-retries = 3 + max-redirects = 3 + } +} + +savage { + default-port = 6060 + github-repo-to-watch = "cvrebert/savage-base" + github-test-repo = "cvrebert/savage-test" + whitelist = [ + "**.md", + "/bower.json", + "/composer.json", + "/fonts/**.{eot,ttf,svg,woff}", + "/less/**.less", + "/sass/**.{sass,scss}", + "/js/**.{js,html,css}", + "/dist/**.{css,js,map,eot,ttf,svg,woff}", + "/docs/**.{html,css,js,map,png,ico,xml,eot,ttf,svg,woff,swf}" + ] + file-watchlist = [ + "/js/**.js" + ] + branch-prefix = "savage-" + username = throwaway9475947 + password = XXXXXXXX + github-web-hook-secret-key = abcdefg + travis-token = abcdefg +} diff --git a/src/main/scala/com/getbootstrap/savage/PullRequestBuildResult.scala b/src/main/scala/com/getbootstrap/savage/PullRequestBuildResult.scala new file mode 100644 index 0000000..5c50765 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/PullRequestBuildResult.scala @@ -0,0 +1,10 @@ +package com.getbootstrap.savage + +import com.getbootstrap.savage.github._ + +case class PullRequestBuildResult( + prNum: PullRequestNumber, + commitSha: CommitSha, + buildUrl: String, + succeeded: Boolean +) diff --git a/src/main/scala/com/getbootstrap/savage/github/Branch.scala b/src/main/scala/com/getbootstrap/savage/github/Branch.scala new file mode 100644 index 0000000..ff52be3 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/github/Branch.scala @@ -0,0 +1,17 @@ +package com.getbootstrap.savage.github + +object Branch { + private val SafeBranchRegex = "^[0-9a-zA-Z_-]+$".r + def apply(branchName: String): Option[Branch] = { + branchName match { + case SafeBranchRegex(_*) => Some(new Branch(branchName)) + case _ => None + } + } + def unapply(branch: Branch): Option[String] = Some(branch.name) +} + +class Branch private(val name: String) extends AnyVal { + override def toString: String = s"Branch(${name})" + def asRef = s"refs/heads/${name}" +} diff --git a/src/main/scala/com/getbootstrap/savage/github/BranchDeletionRequest.scala b/src/main/scala/com/getbootstrap/savage/github/BranchDeletionRequest.scala new file mode 100644 index 0000000..4b7accb --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/github/BranchDeletionRequest.scala @@ -0,0 +1,3 @@ +package com.getbootstrap.savage.github + +case class BranchDeletionRequest(branch: Branch, commitSha: CommitSha) diff --git a/src/main/scala/com/getbootstrap/savage/github/CommitSha.scala b/src/main/scala/com/getbootstrap/savage/github/CommitSha.scala new file mode 100644 index 0000000..75f77f3 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/github/CommitSha.scala @@ -0,0 +1,15 @@ +package com.getbootstrap.savage.github + +object CommitSha { + private val ShaRegex = "^[0-9a-f]{40}$".r + def apply(sha: String): Option[CommitSha] = { + sha match { + case ShaRegex(_*) => Some(new CommitSha(sha)) + case _ => None + } + } +} + +class CommitSha private(val sha: String) extends AnyVal { + override def toString = s"CommitSha(${sha})" +} diff --git a/src/main/scala/com/getbootstrap/savage/github/GitHubActorWithLogging.scala b/src/main/scala/com/getbootstrap/savage/github/GitHubActorWithLogging.scala new file mode 100644 index 0000000..50a75bf --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/github/GitHubActorWithLogging.scala @@ -0,0 +1,10 @@ +package com.getbootstrap.savage.github + +import org.eclipse.egit.github.core.client.GitHubClient +import com.getbootstrap.savage.server.{Settings, ActorWithLogging} + +abstract class GitHubActorWithLogging extends ActorWithLogging { + protected val settings = Settings(context.system) + protected val gitHubClient = new GitHubClient() + gitHubClient.setCredentials(settings.BotUsername, settings.BotPassword) +} diff --git a/src/main/scala/com/getbootstrap/savage/github/PullRequestNumber.scala b/src/main/scala/com/getbootstrap/savage/github/PullRequestNumber.scala new file mode 100644 index 0000000..f7043cb --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/github/PullRequestNumber.scala @@ -0,0 +1,15 @@ +package com.getbootstrap.savage.github + +object PullRequestNumber { + def apply(number: Int): Option[PullRequestNumber] = { + if (number > 0) { + Some(new PullRequestNumber(number)) + } + else { + None + } + } +} +class PullRequestNumber private(val number: Int) extends AnyVal { + override def toString = s"PullRequestNumber(${number})" +} diff --git a/src/main/scala/com/getbootstrap/savage/github/PullRequestPushRequest.scala b/src/main/scala/com/getbootstrap/savage/github/PullRequestPushRequest.scala new file mode 100644 index 0000000..3aa4315 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/github/PullRequestPushRequest.scala @@ -0,0 +1,9 @@ +package com.getbootstrap.savage.github + +import org.eclipse.egit.github.core.RepositoryId + +case class PullRequestPushRequest( + origin: RepositoryId, + number: PullRequestNumber, + commitSha: CommitSha +) diff --git a/src/main/scala/com/getbootstrap/savage/github/util/package.scala b/src/main/scala/com/getbootstrap/savage/github/util/package.scala new file mode 100644 index 0000000..520378c --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/github/util/package.scala @@ -0,0 +1,30 @@ +package com.getbootstrap.savage.github + +import org.eclipse.egit.github.core._ + +package object util { + private val SafeRepoRegex = "^[0-9a-zA-Z_-]+/[0-9a-zA-Z_-]+$".r + + implicit class RichRepository(repo: Repository) { + def repositoryId: Option[RepositoryId] = { + val repoId = new RepositoryId(repo.getOwner.getLogin, repo.getName) + repo.generateId match { + case SafeRepoRegex(_*) => Some(repoId) + case _ => None + } + } + } + implicit class RichPullRequestMarker(marker: PullRequestMarker) { + def commitSha: CommitSha = CommitSha(marker.getSha).getOrElse{ throw new IllegalStateException(s"Invalid commit SHA: ${marker.getSha}") } + } + implicit class RichCommitFile(file: CommitFile) { + + } + implicit class RichPullRequest(pr: PullRequest) { + def number: PullRequestNumber = PullRequestNumber(pr.getNumber).get + } + implicit class RichRepositoryId(repoId: RepositoryId) { + def asPushRemote: String = s"git@github.com:${repoId.generateId}.git" + def asPullRemote: String = s"https://github.com/${repoId.generateId}.git" + } +} diff --git a/src/main/scala/com/getbootstrap/savage/server/ActorWithLogging.scala b/src/main/scala/com/getbootstrap/savage/server/ActorWithLogging.scala new file mode 100644 index 0000000..bfca21b --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/server/ActorWithLogging.scala @@ -0,0 +1,6 @@ +package com.getbootstrap.savage.server + +import akka.actor.Actor +import akka.actor.ActorLogging + +trait ActorWithLogging extends Actor with ActorLogging diff --git a/src/main/scala/com/getbootstrap/savage/server/Boot.scala b/src/main/scala/com/getbootstrap/savage/server/Boot.scala new file mode 100644 index 0000000..26894eb --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/server/Boot.scala @@ -0,0 +1,46 @@ +package com.getbootstrap.savage.server + +import scala.util.{Success,Failure} +import scala.concurrent.duration._ +import scala.util.Try +import akka.actor.{ActorSystem, Props} +import akka.io.IO +import spray.can.Http +import akka.pattern.ask +import akka.routing.SmallestMailboxPool +import akka.util.Timeout + + +object Boot extends App { + val arguments = args.toSeq + val argsPort = arguments match { + case Seq(portStr: String) => { + Try{ portStr.toInt } match { + case Failure(_) => { + System.err.println("USAGE: savage ") + System.exit(1) + None // dead code + } + case Success(portNum) => Some(portNum) + } + } + case Seq() => None + } + + run(argsPort) + + def run(port: Option[Int]) { + implicit val system = ActorSystem("on-spray-can") + val settings = Settings(system) + // import actorSystem.dispatcher + + val deleter = system.actorOf(SmallestMailboxPool(3).props(Props(classOf[BranchDeleter])), "branch-deleters") + val commenter = system.actorOf(SmallestMailboxPool(3).props(Props(classOf[PullRequestCommenter])), "gh-pr-commenters") + val pusher = system.actorOf(Props(classOf[PullRequestPusher]), "pr-pusher") + val prHandlers = system.actorOf(SmallestMailboxPool(3).props(Props(classOf[PullRequestEventHandler], pusher)), "pr-handlers") + val webService = system.actorOf(Props(classOf[SavageWebService], prHandlers, commenter, deleter), "savage-service") + + implicit val timeout = Timeout(15.seconds) + IO(Http) ? Http.Bind(webService, interface = "0.0.0.0", port = port.getOrElse(settings.DefaultPort)) + } +} diff --git a/src/main/scala/com/getbootstrap/savage/server/BranchDeleter.scala b/src/main/scala/com/getbootstrap/savage/server/BranchDeleter.scala new file mode 100644 index 0000000..33eb162 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/server/BranchDeleter.scala @@ -0,0 +1,41 @@ +package com.getbootstrap.savage.server + +import scala.collection.JavaConverters._ +import org.eclipse.egit.github.core.service.RepositoryService +import com.getbootstrap.savage.github.{GitHubActorWithLogging, Branch, BranchDeletionRequest} +import com.getbootstrap.savage.github.util.RichRepositoryId +import com.getbootstrap.savage.util.{SuccessfulExit, ErrorExit, SimpleSubprocess} + +class BranchDeleter extends GitHubActorWithLogging { + override def receive = { + case BranchDeletionRequest(branch, commitSha) => { + if (isSavageBranch(branch)) { + val repoService = new RepositoryService(gitHubClient) + val maybeRepoBranch = repoService.getBranches(settings.TestRepoId).asScala.find{ _.getName == branch.name } + maybeRepoBranch match { + case None => log.info(s"Nothing to delete; ${branch} does not exist in ${settings.TestRepoId}") + case Some(repoBranch) => { + val repoSha = repoBranch.getCommit.getSha + if (repoSha == commitSha.sha) { + val remote = settings.TestRepoId.asPushRemote + val process = SimpleSubprocess(Seq("git", "push", remote, ":" + branch.name)) + log.info(s"Deleting branch ${branch} from remote ${remote}") + process.run() match { + case SuccessfulExit(_) => log.info(s"Successfully deleted ${branch} in ${remote}") + case ErrorExit(exitValue, output) => log.error(s"Error deleting ${branch} in ${remote} :\nExit code: ${exitValue}\n${output}") + } + } + else { + log.info(s"Not deleting ${branch} from ${settings.TestRepoId} because commits differ; request was for ${commitSha} but current is ${repoSha}") + } + } + } + } + else { + log.error(s"Not deleting non-Savage branch : ${branch}") + } + } + } + + private def isSavageBranch(branch: Branch): Boolean = branch.name startsWith settings.BranchPrefix +} diff --git a/src/main/scala/com/getbootstrap/savage/server/GitHubPullRequestWebHooksDirectives.scala b/src/main/scala/com/getbootstrap/savage/server/GitHubPullRequestWebHooksDirectives.scala new file mode 100644 index 0000000..d61a294 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/server/GitHubPullRequestWebHooksDirectives.scala @@ -0,0 +1,22 @@ +package com.getbootstrap.savage.server + +import scala.util.{Success, Failure, Try} +import spray.routing.{Directive1, ValidationRejection} +import spray.routing.directives.{BasicDirectives, RouteDirectives} +import org.eclipse.egit.github.core.event.PullRequestPayload +import org.eclipse.egit.github.core.client.GsonUtils + +trait GitHubPullRequestWebHooksDirectives { + import RouteDirectives.reject + import BasicDirectives.provide + import HubSignatureDirectives.stringEntityMatchingHubSignature + + def authenticatedPullRequestEvent(secretKey: Array[Byte]): Directive1[PullRequestPayload] = stringEntityMatchingHubSignature(secretKey).flatMap{ entityJsonString => + Try { GsonUtils.fromJson(entityJsonString, classOf[PullRequestPayload]) } match { + case Failure(exc) => reject(ValidationRejection("JSON was either malformed or did not match expected schema!")) + case Success(payload) => provide(payload) + } + } +} + +object GitHubPullRequestWebHooksDirectives extends GitHubPullRequestWebHooksDirectives diff --git a/src/main/scala/com/getbootstrap/savage/server/HubSignatureDirectives.scala b/src/main/scala/com/getbootstrap/savage/server/HubSignatureDirectives.scala new file mode 100644 index 0000000..a0d4153 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/server/HubSignatureDirectives.scala @@ -0,0 +1,46 @@ +package com.getbootstrap.savage.server + +import scala.util.{Try,Success,Failure} +import spray.routing.{Directive1, MalformedHeaderRejection, MalformedRequestContentRejection, ValidationRejection} +import spray.routing.directives.{BasicDirectives, HeaderDirectives, RouteDirectives, MarshallingDirectives} +import com.getbootstrap.savage.util.{HmacSha1,Utf8ByteArray} + +trait HubSignatureDirectives { + import BasicDirectives.provide + import HeaderDirectives.headerValueByName + import RouteDirectives.reject + import MarshallingDirectives.{entity, as} + + private val xHubSignature = "X-Hub-Signature" + private val hubSignatureHeaderValue = headerValueByName(xHubSignature) + + val hubSignature: Directive1[Array[Byte]] = hubSignatureHeaderValue.flatMap { algoEqHex => + val bytesFromHexOption = algoEqHex.split('=') match { + case Array("sha1", hex) => Try{ javax.xml.bind.DatatypeConverter.parseHexBinary(hex) }.toOption + case _ => None + } + bytesFromHexOption match { + case Some(bytesFromHex) => provide(bytesFromHex) + case None => reject(MalformedHeaderRejection(xHubSignature, "Malformed HMAC")) + } + } + + private val bytesEntity = entity(as[Array[Byte]]) + + def stringEntityMatchingHubSignature(secretKey: Array[Byte]): Directive1[String] = hubSignature.flatMap { signature => + bytesEntity.flatMap { dataBytes => + val hmac = new HmacSha1(mac = signature, secretKey = secretKey, data = dataBytes) + if (hmac.isValid) { + dataBytes.utf8String match { + case Success(string) => provide(string) + case Failure(exc) => reject(MalformedRequestContentRejection("Request body is not valid UTF-8", Some(exc))) + } + } + else { + reject(ValidationRejection("Incorrect HMAC")) + } + } + } +} + +object HubSignatureDirectives extends HubSignatureDirectives diff --git a/src/main/scala/com/getbootstrap/savage/server/PullRequestCommenter.scala b/src/main/scala/com/getbootstrap/savage/server/PullRequestCommenter.scala new file mode 100644 index 0000000..29d6519 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/server/PullRequestCommenter.scala @@ -0,0 +1,39 @@ +package com.getbootstrap.savage.server + +import scala.util.{Failure, Success, Try} +import org.eclipse.egit.github.core.service.IssueService +import com.getbootstrap.savage.github.{GitHubActorWithLogging, PullRequestNumber} +import com.getbootstrap.savage.PullRequestBuildResult + +class PullRequestCommenter extends GitHubActorWithLogging { + private def tryToCommentOn(prNum: PullRequestNumber, commentMarkdown: String) = { + val issueService = new IssueService(gitHubClient) + Try { issueService.createComment(settings.MainRepoId, prNum.number, commentMarkdown) } + } + + override def receive = { + case PullRequestBuildResult(prNum, commitSha, buildUrl, succeeded) => { + val statusRemark = if (succeeded) { + "CONFIRMED (Tests passed)" + } + else { + "BUSTED (Tests failed)" + } + + val commentMarkdown = s""" + |Automated cross-browser testing via Sauce Labs and Travis CI shows that the changes in this pull request are + |${statusRemark} + | + |Commit: ${commitSha.sha} + |Build details: ${buildUrl} + | + |(*Please note that this is a [fully automated](https://github.com/twbs/savage) comment.*) + """.stripMargin + + tryToCommentOn(prNum, commentMarkdown) match { + case Success(comment) => log.info(s"Successfully posted comment ${comment.getUrl} for ${prNum}") + case Failure(exc) => log.error(exc, s"Error posting comment for ${prNum}") + } + } + } +} diff --git a/src/main/scala/com/getbootstrap/savage/server/PullRequestEventHandler.scala b/src/main/scala/com/getbootstrap/savage/server/PullRequestEventHandler.scala new file mode 100644 index 0000000..41f74ce --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/server/PullRequestEventHandler.scala @@ -0,0 +1,94 @@ +package com.getbootstrap.savage.server + +import java.nio.file.Path +import scala.collection.JavaConverters._ +import scala.util.{Try,Success,Failure} +import akka.actor.ActorRef +import org.eclipse.egit.github.core._ +import org.eclipse.egit.github.core.service.CommitService +import com.getbootstrap.savage.github._ +import com.getbootstrap.savage.github.util._ +import com.getbootstrap.savage.util.UnixFileSystemString + +class PullRequestEventHandler(protected val pusher: ActorRef) extends GitHubActorWithLogging { + + private def affectedFilesFor(repoId: RepositoryId, base: CommitSha, head: CommitSha): Try[Set[Path]] = { + val commitService = new CommitService(gitHubClient) + Try { commitService.compare(repoId, base.sha, head.sha) }.map { comparison => + val affectedFiles = comparison.getFiles.asScala.map{ "/" + _.getFilename }.toSet[String].map{ _.asUnixPath } + affectedFiles + } + } + + private val NormalPathRegex = "^[a-zA-Z0-9_./-]+$".r + private def isNormal(path: Path): Boolean = { + path match { + case NormalPathRegex(_) => true + case _ => false + } + } + private def areSafe(paths: Set[Path]): Boolean = { + implicit val logger = log + paths.forall{ path => isNormal(path) && settings.Whitelist.isAllowed(path) } + } + private def areInteresting(paths: Set[Path]): Boolean = { + implicit val logger = log + settings.Watchlist.anyInterestingIn(paths) + } + + private def logPrInfo(msg: String)(implicit prNum: PullRequestNumber) { + log.info(s"PR #${prNum.number} : ${msg}") + } + + override def receive = { + case pr: PullRequest => { + implicit val prNum = pr.number + val bsBase = pr.getBase + val prHead = pr.getHead + val destinationRepo = bsBase.getRepo.repositoryId + destinationRepo match { + case None => log.error(s"Received event from GitHub about irrelevant repository with unsafe name") + case Some(settings.MainRepoId) => { + val destBranch = bsBase.getRef + destBranch match { + case "master" => { + prHead.getRepo.repositoryId match { + case None => log.error(s"Received event from GitHub about repository with unsafe name") + case Some(foreignRepo) => { + val baseSha = bsBase.commitSha + val headSha = prHead.commitSha + + affectedFilesFor(foreignRepo, baseSha, headSha) match { + case Failure(exc) => { + log.error(exc, s"Could not get affected files for commits ${baseSha}...${headSha} for ${foreignRepo}") + } + case Success(affectedFiles) => { + if (areSafe(affectedFiles)) { + if (areInteresting(affectedFiles)) { + logPrInfo(s"Requesting build for safe & interesting PR") + pusher ! PullRequestPushRequest( + origin = foreignRepo, + number = pr.number, + commitSha = headSha + ) + } + else { + logPrInfo(s"Ignoring PR with no interesting file changes") + } + } + else { + logPrInfo(s"Ignoring PR with unsafe file changes") + } + } + } + } + } + } + case _ => logPrInfo(s"Ignoring since PR targets the ${destBranch} branch") + } + } + case Some(otherRepo) => log.error(s"Received event from GitHub about irrelevant repository: ${otherRepo}") + } + } + } +} diff --git a/src/main/scala/com/getbootstrap/savage/server/PullRequestPusher.scala b/src/main/scala/com/getbootstrap/savage/server/PullRequestPusher.scala new file mode 100644 index 0000000..b24bd48 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/server/PullRequestPusher.scala @@ -0,0 +1,58 @@ +package com.getbootstrap.savage.server + +import org.eclipse.egit.github.core.RepositoryId +import com.getbootstrap.savage.github._ +import com.getbootstrap.savage.github.util.RichRepositoryId +import com.getbootstrap.savage.util.{ErrorExit, SuccessfulExit, SimpleSubprocess} +import com.getbootstrap.savage.util.{RichPath,UnixFileSystemString} + +class PullRequestPusher extends GitHubActorWithLogging { + private val gitRemoteRefsDirectory = ".git/refs/remotes".asUnixPath + + override def receive = { + case PullRequestPushRequest(originRepo, prNum, commitSha) => { + if (pull(originRepo)) { + push(originRepo = originRepo, prNum = prNum, commitSha = commitSha) + } + } + } + + def pull(originRepo: RepositoryId): Boolean = { + // clobberingly fetch all branch heads into a dummy remote + SimpleSubprocess(Seq("git", "fetch", "--no-tags", "--recurse-submodules=no", originRepo.asPullRemote, "+refs/heads/*:refs/remotes/scratch/*")).run() match { + case SuccessfulExit(_) => { + log.info(s"Successfully fetched from ${originRepo}") + true + } + case ErrorExit(exitValue, output) => { + log.error(s"Error fetching from ${originRepo}:\nExit code: ${exitValue}\n${output}") + false + } + } + } + + def push(originRepo: RepositoryId, prNum: PullRequestNumber, commitSha: CommitSha): Boolean = { + val newBranch = { + val branchName = settings.BranchPrefix + prNum.number + Branch(branchName).getOrElse { + throw new SecurityException("Generated insecure branch name: ${}") + } + } + val branchSpec = s"${commitSha.sha}:${newBranch.asRef}" + val destRemote = settings.TestRepoId.asPushRemote + val success = SimpleSubprocess(Seq("git", "push", destRemote, branchSpec)).run() match { + case SuccessfulExit(_) => { + log.info(s"Successfully pushed ${commitSha} from ${originRepo} to ${destRemote} as ${newBranch}") + true + } + case ErrorExit(exitValue, output) => { + log.error(s"Error pushing ${commitSha} from ${originRepo} to ${destRemote} as ${newBranch}:\nExit code: ${exitValue}\n${output}") + false + } + } + // delete all remote refs + implicit val logger = log + gitRemoteRefsDirectory.deleteRecursively() + success + } +} diff --git a/src/main/scala/com/getbootstrap/savage/server/SavageWebService.scala b/src/main/scala/com/getbootstrap/savage/server/SavageWebService.scala new file mode 100644 index 0000000..f8c430b --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/server/SavageWebService.scala @@ -0,0 +1,86 @@ +package com.getbootstrap.savage.server + +import scala.util.{Try,Success,Failure} +import akka.actor.ActorRef +import spray.routing._ +import spray.http._ +import com.getbootstrap.savage.PullRequestBuildResult +import com.getbootstrap.savage.github.{BranchDeletionRequest, PullRequestNumber} + +class SavageWebService( + protected val pullRequestEventHandler: ActorRef, + protected val pullRequestCommenter: ActorRef, + protected val branchDeleter: ActorRef +) extends ActorWithLogging with HttpService { + import GitHubPullRequestWebHooksDirectives.authenticatedPullRequestEvent + import TravisWebHookDirectives.authenticatedTravisEvent + + private val settings = Settings(context.system) + override def actorRefFactory = context + override def receive = runRoute(theOnlyRoute) + + val theOnlyRoute = + pathPrefix("savage") { + pathEndOrSingleSlash { + get { + complete(StatusCodes.OK, "Hi! Savage is online.") + } + } ~ + path("github") { + pathEndOrSingleSlash { + post { + headerValueByName("X-Github-Event") { githubEvent => + githubEvent match { + case "ping" => { + log.info("Successfully received GitHub webhook ping.") + complete(StatusCodes.OK) + } + case "pull_request" => { + authenticatedPullRequestEvent(settings.GitHubWebHookSecretKey.toArray) { event => + event.getAction match { + case "opened" | "synchronize" => { + val pr = event.getPullRequest + if (pr.getState == "open") { + pullRequestEventHandler ! pr + complete(StatusCodes.OK) + } + else { + complete(StatusCodes.OK, s"Ignoring event about closed pull request #${pr.getId}") + } + } + case _ => complete(StatusCodes.OK, "Ignoring irrelevant action") + } + } + } + case _ => complete(StatusCodes.BadRequest, "Unexpected event type") + } + } + } + } + } ~ + path("travis") { + pathEndOrSingleSlash { + post { + authenticatedTravisEvent(travisToken = settings.TravisToken, repo = settings.TestRepoId, log = log) { event => + if (event.branchName.name.startsWith(settings.BranchPrefix)) { + Try { Integer.parseInt(event.branchName.name.stripPrefix(settings.BranchPrefix)) }.flatMap{ intStr => Try{ PullRequestNumber(intStr).get } } match { + case Failure(exc) => log.error(exc, s"Invalid Savage branch name from Travis event: ${event.branchName}") + case Success(prNum) => { + // FIXME: check event.build_url is safe + branchDeleter ! BranchDeletionRequest(event.branchName, event.commitSha) + pullRequestCommenter ! PullRequestBuildResult( + prNum = prNum, + commitSha = event.commitSha, + buildUrl = event.build_url, + succeeded = event.status.isSuccessful + ) + } + } + } + complete(StatusCodes.OK) + } + } + } + } + } +} diff --git a/src/main/scala/com/getbootstrap/savage/server/Settings.scala b/src/main/scala/com/getbootstrap/savage/server/Settings.scala new file mode 100644 index 0000000..9ae2cfd --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/server/Settings.scala @@ -0,0 +1,30 @@ +package com.getbootstrap.savage.server + +import scala.collection.JavaConverters._ +import com.typesafe.config.Config +import akka.actor.ActorSystem +import akka.actor.Extension +import akka.actor.ExtensionId +import akka.actor.ExtensionIdProvider +import akka.actor.ExtendedActorSystem +import akka.util.ByteString +import org.eclipse.egit.github.core.RepositoryId +import com.getbootstrap.savage.util.{FilePathWhitelist,FilePathWatchlist,Utf8String} + +class SettingsImpl(config: Config) extends Extension { + val MainRepoId: RepositoryId = RepositoryId.createFromId(config.getString("savage.github-repo-to-watch")) + val TestRepoId: RepositoryId = RepositoryId.createFromId(config.getString("savage.github-test-repo")) + val BotUsername: String = config.getString("savage.username") + val BotPassword: String = config.getString("savage.password") + val GitHubWebHookSecretKey: ByteString = ByteString(config.getString("savage.github-web-hook-secret-key").utf8Bytes) + val TravisToken: String = config.getString("savage.travis-token") + val DefaultPort: Int = config.getInt("savage.default-port") + val Whitelist: FilePathWhitelist = new FilePathWhitelist(config.getStringList("savage.whitelist").asScala) + val Watchlist: FilePathWatchlist = new FilePathWatchlist(config.getStringList("savage.file-watchlist").asScala) + val BranchPrefix: String = config.getString("savage.branch-prefix") +} +object Settings extends ExtensionId[SettingsImpl] with ExtensionIdProvider { + override def lookup() = Settings + override def createExtension(system: ExtendedActorSystem) = new SettingsImpl(system.settings.config) + override def get(system: ActorSystem): SettingsImpl = super.get(system) +} diff --git a/src/main/scala/com/getbootstrap/savage/server/TravisAuthDirectives.scala b/src/main/scala/com/getbootstrap/savage/server/TravisAuthDirectives.scala new file mode 100644 index 0000000..87ed804 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/server/TravisAuthDirectives.scala @@ -0,0 +1,45 @@ +package com.getbootstrap.savage.server + +import scala.util.Try +import spray.http.FormData +import spray.routing.{Directive1, MalformedHeaderRejection, MalformedRequestContentRejection, ValidationRejection} +import spray.routing.directives.{BasicDirectives, HeaderDirectives, RouteDirectives, MarshallingDirectives} +import org.eclipse.egit.github.core.RepositoryId +import com.getbootstrap.savage.util.{Sha256,Utf8String} + +trait TravisAuthDirectives { + import BasicDirectives.provide + import HeaderDirectives.headerValueByName + import RouteDirectives.reject + import MarshallingDirectives.{entity, as} + + private val authorization = "Authorization" + private val authorizationHeaderValue = headerValueByName(authorization) + + val travisAuthorization: Directive1[Array[Byte]] = authorizationHeaderValue.flatMap { hex => + Try{ javax.xml.bind.DatatypeConverter.parseHexBinary(hex) }.toOption match { + case Some(bytesFromHex) => provide(bytesFromHex) + case None => reject(MalformedHeaderRejection(authorization, "Malformed SHA-256 hex digest")) + } + } + + private val formDataEntity = entity(as[FormData]) + + def stringEntityIfTravisAuthValid(travisToken: String, repo: RepositoryId): Directive1[String] = travisAuthorization.flatMap { hash => + formDataEntity.flatMap { formData => + val plainText = repo.generateId + travisToken + val auth = new Sha256(hash = hash, plainText = plainText.utf8Bytes) + if (auth.isValid) { + formData.fields.toMap.get("payload") match { + case Some(string) => provide(string) + case None => reject(MalformedRequestContentRejection("Request body form data lacked required `payload` field")) + } + } + else { + reject(ValidationRejection("Incorrect SHA-256 hash")) + } + } + } +} + +object TravisAuthDirectives extends TravisAuthDirectives diff --git a/src/main/scala/com/getbootstrap/savage/server/TravisWebHookDirectives.scala b/src/main/scala/com/getbootstrap/savage/server/TravisWebHookDirectives.scala new file mode 100644 index 0000000..90f1e1a --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/server/TravisWebHookDirectives.scala @@ -0,0 +1,25 @@ +package com.getbootstrap.savage.server + +import scala.util.{Success, Failure, Try} +import akka.event.LoggingAdapter +import spray.routing.{Directive1, ValidationRejection} +import spray.routing.directives.{BasicDirectives, RouteDirectives} +import spray.json._ +import org.eclipse.egit.github.core.RepositoryId +import com.getbootstrap.savage.travis.{TravisJsonProtocol, TravisPayload} + +trait TravisWebHookDirectives { + import RouteDirectives.reject + import BasicDirectives.provide + import TravisAuthDirectives.stringEntityIfTravisAuthValid + import TravisJsonProtocol._ + + def authenticatedTravisEvent(travisToken: String, repo: RepositoryId, log: LoggingAdapter): Directive1[TravisPayload] = stringEntityIfTravisAuthValid(travisToken, repo).flatMap{ entityJsonString => + Try { entityJsonString.parseJson.convertTo[TravisPayload] } match { + case Failure(exc) => reject(ValidationRejection("JSON was either malformed or did not match expected schema!")) + case Success(payload) => provide(payload) + } + } +} + +object TravisWebHookDirectives extends TravisWebHookDirectives diff --git a/src/main/scala/com/getbootstrap/savage/travis/TravisJsonProtocol.scala b/src/main/scala/com/getbootstrap/savage/travis/TravisJsonProtocol.scala new file mode 100644 index 0000000..05bd43e --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/travis/TravisJsonProtocol.scala @@ -0,0 +1,7 @@ +package com.getbootstrap.savage.travis + +import spray.json._ + +object TravisJsonProtocol extends DefaultJsonProtocol { + implicit val travisPayloadFormat = jsonFormat4(TravisPayload.apply) +} diff --git a/src/main/scala/com/getbootstrap/savage/travis/TravisPayload.scala b/src/main/scala/com/getbootstrap/savage/travis/TravisPayload.scala new file mode 100644 index 0000000..67ad990 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/travis/TravisPayload.scala @@ -0,0 +1,15 @@ +package com.getbootstrap.savage.travis + +import com.getbootstrap.savage.github.{Branch, CommitSha} +import com.getbootstrap.savage.travis.build_status.BuildStatus + +case class TravisPayload( + status_message: String, + build_url: String, + branch: String, + commit: String +) { + def status: BuildStatus = BuildStatus(status_message).getOrElse{ throw new IllegalStateException(s"Invalid Travis build status message: ${status_message}") } + def commitSha: CommitSha = CommitSha(commit).getOrElse{ throw new IllegalStateException(s"Invalid commit SHA: ${commit}") } + def branchName: Branch = Branch(branch).getOrElse{ throw new IllegalStateException(s"Unsafe branch name: ${branch}") } +} diff --git a/src/main/scala/com/getbootstrap/savage/travis/build_status/BuildStatus.scala b/src/main/scala/com/getbootstrap/savage/travis/build_status/BuildStatus.scala new file mode 100644 index 0000000..98906d9 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/travis/build_status/BuildStatus.scala @@ -0,0 +1,50 @@ +package com.getbootstrap.savage.travis.build_status + +object BuildStatus { + def apply(statusMessage: String): Option[BuildStatus] = { + statusMessage match { + case Pending.StatusMessage => Some(Pending) + case Passed.StatusMessage => Some(Passed) + case Fixed.StatusMessage => Some(Fixed) + case Failed.StatusMessage => Some(Failed) + case Broken.StatusMessage => Some(Broken) + case StillFailing.StatusMessage => Some(StillFailing) + case _ => None + } + } +} +sealed trait BuildStatus { + val StatusMessage: String + def isSuccessful: Boolean +} + +// If the webhook is setup properly, we should never see a build with Pending status +object Pending extends BuildStatus { + override val StatusMessage = "Pending" + override val isSuccessful = false +} + +sealed trait Succeeded extends BuildStatus { + override val isSuccessful = true +} +sealed trait Failure extends BuildStatus { + override val isSuccessful = false +} + +object Passed extends Succeeded { + override val StatusMessage = "Passed" +} +object Fixed extends Succeeded { + override val StatusMessage = "Fixed" +} + +object Failed extends Failure { + override val StatusMessage = "Failed" +} +object Broken extends Failure { + override val StatusMessage = "Broken" +} +// Not mentioned in the Travis docs :-/ +object StillFailing extends Failure { + override val StatusMessage = "Still Failing" +} diff --git a/src/main/scala/com/getbootstrap/savage/util/DeleterFileVisitor.scala b/src/main/scala/com/getbootstrap/savage/util/DeleterFileVisitor.scala new file mode 100644 index 0000000..ee533de --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/util/DeleterFileVisitor.scala @@ -0,0 +1,39 @@ +package com.getbootstrap.savage.util + +import java.io.IOException +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.{Files, Path, SimpleFileVisitor, FileVisitResult} +import akka.event.LoggingAdapter + +class DeleterFileVisitor(private val log: LoggingAdapter) extends SimpleFileVisitor[Path] { + @throws[SecurityException] + override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { + tryToDelete(file) + FileVisitResult.CONTINUE + } + + @throws[IOException] + @throws[SecurityException] + override def postVisitDirectory(dir: Path, maybeExc: IOException): FileVisitResult = { + maybeExc match { + case null => { + tryToDelete(dir) + FileVisitResult.CONTINUE + } + case exc => throw exc + } + } + + @throws[SecurityException] + private def tryToDelete(path: Path): Boolean = { + try { + Files.deleteIfExists(path) + } + catch { + case exc:IOException => { + log.error(exc, s"Problem deleting ${path}") + false + } + } + } +} diff --git a/src/main/scala/com/getbootstrap/savage/util/FilePathWatchlist.scala b/src/main/scala/com/getbootstrap/savage/util/FilePathWatchlist.scala new file mode 100644 index 0000000..23d10c8 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/util/FilePathWatchlist.scala @@ -0,0 +1,24 @@ +package com.getbootstrap.savage.util + +import java.nio.file.Path +import akka.event.LoggingAdapter + +class FilePathWatchlist(globs: Iterable[String]) { + private val matchers = globs.toIterator.map{ _.asUnixGlob }.toVector + def isInteresting(path: Path)(implicit log: LoggingAdapter): Boolean = { + val interesting = matchers.exists{ _.matches(path) } + if (interesting) { + log.info(s"Interesting path: ${path}") + } + interesting + } + def anyInterestingIn(paths: Iterable[Path])(implicit log: LoggingAdapter) = { + paths.find{ path => isInteresting(path) } match { + case Some(path) => true + case None => { + log.info("No interesting paths found.") + false + } + } + } +} diff --git a/src/main/scala/com/getbootstrap/savage/util/FilePathWhitelist.scala b/src/main/scala/com/getbootstrap/savage/util/FilePathWhitelist.scala new file mode 100644 index 0000000..9799e05 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/util/FilePathWhitelist.scala @@ -0,0 +1,15 @@ +package com.getbootstrap.savage.util + +import java.nio.file.Path +import akka.event.LoggingAdapter + +class FilePathWhitelist(globs: Iterable[String]) { + private val matchers = globs.toIterator.map{ _.asUnixGlob }.toVector + def isAllowed(path: Path)(implicit log: LoggingAdapter): Boolean = { + var allowed = matchers.exists{ _.matches(path) } + if (!allowed) { + log.info(s"Path disallowed by whitelist: ${path}") + } + allowed + } +} diff --git a/src/main/scala/com/getbootstrap/savage/util/HmacSha1.scala b/src/main/scala/com/getbootstrap/savage/util/HmacSha1.scala new file mode 100644 index 0000000..93340ac --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/util/HmacSha1.scala @@ -0,0 +1,29 @@ +package com.getbootstrap.savage.util + +import javax.crypto.Mac +import javax.crypto.spec.SecretKeySpec +import java.security.{NoSuchAlgorithmException, InvalidKeyException, SignatureException} +import java.security.MessageDigest + +object HmacSha1 { + private val HmacSha1Algorithm = "HmacSHA1" +} + +case class HmacSha1(mac: Array[Byte], secretKey: Array[Byte], data: Array[Byte]) { + import HmacSha1.HmacSha1Algorithm + + @throws[NoSuchAlgorithmException]("if HMAC-SHA1 is not supported") + @throws[InvalidKeyException]("if the secret key is malformed") + @throws[SignatureException]("under unknown circumstances") + private lazy val correct: Array[Byte] = { + val key = new SecretKeySpec(secretKey, HmacSha1Algorithm) + val mac = Mac.getInstance(HmacSha1Algorithm) + mac.init(key) + mac.doFinal(data) + } + + lazy val isValid: Boolean = MessageDigest.isEqual(mac, correct) + + def givenHex = mac.asHexBytes + def correctHex = correct.asHexBytes +} diff --git a/src/main/scala/com/getbootstrap/savage/util/Sha256.scala b/src/main/scala/com/getbootstrap/savage/util/Sha256.scala new file mode 100644 index 0000000..4028e34 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/util/Sha256.scala @@ -0,0 +1,22 @@ +package com.getbootstrap.savage.util + +import java.security.MessageDigest +import java.security.NoSuchAlgorithmException + +object Sha256 { + private val Sha256Algorithm = "SHA-256" +} + +case class Sha256(hash: Array[Byte], plainText: Array[Byte]) { + import Sha256.Sha256Algorithm + + @throws[NoSuchAlgorithmException]("if SHA-256 is not supported") + private lazy val correct: Array[Byte] = { + MessageDigest.getInstance(Sha256Algorithm).digest(plainText) + } + + lazy val isValid: Boolean = MessageDigest.isEqual(hash, correct) + + def givenHex = hash.asHexBytes + def correctHex = correct.asHexBytes +} diff --git a/src/main/scala/com/getbootstrap/savage/util/SimpleSubprocess.scala b/src/main/scala/com/getbootstrap/savage/util/SimpleSubprocess.scala new file mode 100644 index 0000000..b56df61 --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/util/SimpleSubprocess.scala @@ -0,0 +1,37 @@ +package com.getbootstrap.savage.util + +import java.io.File + +object SimpleSubprocess { + private val devNull = new File("/dev/null") + private val rootDir = new File("/") +} +case class SimpleSubprocess(args: Seq[String]) { + val processBuilder = ( + new ProcessBuilder(args : _*) + .redirectErrorStream(true) + .redirectInput(ProcessBuilder.Redirect.from(SimpleSubprocess.devNull)) + ) + def run(): SubprocessResult = { + val process = processBuilder.start() + val output = process.getInputStream.readUntilEofAsSingleUtf8String + val exitValue = process.waitFor() + SubprocessResult(exitValue, output) + } +} + +object SubprocessResult { + def apply(exitValue: Int, output: String): SubprocessResult = { + exitValue match { + case 0 => SuccessfulExit(output) + case _ => ErrorExit(exitValue, output) + } + } +} +sealed trait SubprocessResult { + def exitValue: Int +} +case class SuccessfulExit(output: String) extends SubprocessResult { + override def exitValue = 0 +} +case class ErrorExit(exitValue: Int, output: String) extends SubprocessResult diff --git a/src/main/scala/com/getbootstrap/savage/util/package.scala b/src/main/scala/com/getbootstrap/savage/util/package.scala new file mode 100644 index 0000000..e070e7f --- /dev/null +++ b/src/main/scala/com/getbootstrap/savage/util/package.scala @@ -0,0 +1,70 @@ +package com.getbootstrap.savage + +import java.nio.charset.Charset +import java.nio.file.{FileSystems, FileSystem, Path} +import java.io.{IOException, InputStream} +import java.util.Scanner +import akka.event.LoggingAdapter +import scala.util.Try + +package object util { + val utf8Name = "UTF-8" + private val utf8 = Charset.forName(utf8Name) + + implicit class Utf8String(str: String) { + def utf8Bytes: Array[Byte] = str.getBytes(utf8) + } + + implicit class Utf8ByteArray(bytes: Array[Byte]) { + def utf8String: Try[String] = Try { new String(bytes, utf8) } + } + + private object UnixFileSystemString { + private lazy val unixFileSystem: FileSystem = { + // get a Unix-y FileSystem, or fail hard + val unixFsAbstractClass = Class.forName("sun.nio.fs.UnixFileSystem") + val systemFs = FileSystems.getDefault + if (unixFsAbstractClass isInstance systemFs) { + systemFs + } + else { + throw new SecurityException("The globbing for the editable files whitelist requires a Unix-y java.nio.file.FileSystem, but we could not obtain one.") + } + } + } + implicit class UnixFileSystemString(str: String) { + def asUnixGlob = UnixFileSystemString.unixFileSystem.getPathMatcher("glob:" + str) + def asUnixPath = UnixFileSystemString.unixFileSystem.getPath(str) + } + + implicit class RichInputStream(stream: InputStream) { + def readUntilEofAsSingleUtf8String: String = { + val scanner = new Scanner(stream, utf8Name).useDelimiter("\\A") + val string = if (scanner.hasNext) { + scanner.next() + } + else { + "" + } + scanner.close() + string + } + } + + implicit class HexByteArray(array: Array[Byte]) { + import javax.xml.bind.DatatypeConverter + def asHexBytes: String = DatatypeConverter.printHexBinary(array).toLowerCase + } + + implicit class RichPath(path: Path) { + @throws[SecurityException] + def deleteRecursively()(implicit log: LoggingAdapter) { + try { + java.nio.file.Files.walkFileTree(path, new DeleterFileVisitor(log)) + } + catch { + case exc:IOException => log.error(exc, s"Error while deleting ${path}") + } + } + } +}