Add a rule for downloading files.

Summary:
The URIs for the remote file can either be http, https, or "mvn", which is a
essentially the Buildr maven URL syntax with an optional host added, prefixed
with "mvn:".

By default, Buck disallows builds from downloading artifacts at build time. The
next step in this process will be to create a "fetch" command which will walk
the Target Graph and download all the files that it detects as being necessary.

Test Plan: buck test --all

src/com/facebook/buck/file/BUCK

@@ -0,0 +1,40 @@
+java_library(
+  name = 'downloader',
+  srcs = [
+    'Downloader.java',
+    'DownloadEvent.java',
+    'DownloadProgressEvent.java',
+    'MavenUrlDecoder.java',
+  ],
+  deps = [
+    '//lib:guava',
+    '//src/com/facebook/buck/event:event',
+    '//src/com/facebook/buck/util:exceptions',
+  ],
+  visibility = [ 'PUBLIC' ],
+)
+
+java_library(
+  name = 'file',
+  srcs = [
+    'DownloadStep.java',
+    'RemoteFile.java',
+    'RemoteFileDescription.java',
+  ],
+  deps = [
+    ':downloader',
+    '//lib:guava',
+    '//lib:jsr305',
+    '//src/com/facebook/buck/event:event',
+    '//src/com/facebook/buck/model:model',
+    '//src/com/facebook/buck/rules:build_rule',
+    '//src/com/facebook/buck/rules:rules',
+    '//src/com/facebook/buck/step/fs:fs',
+    '//src/com/facebook/buck/step:step',
+    '//src/com/facebook/buck/util:exceptions',
+    '//third-party/java/infer-annotations:infer-annotations',
+  ],
+  visibility = [
+    '//src/com/facebook/buck/rules:types',
+  ],
+)

src/com/facebook/buck/file/DownloadEvent.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2014-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.file;
+
+import com.facebook.buck.event.AbstractBuckEvent;
+import com.facebook.buck.event.BuckEvent;
+import com.google.common.base.Preconditions;
+
+import java.net.URI;
+
+public abstract class DownloadEvent extends AbstractBuckEvent {
+
+  protected URI uri;
+
+  private DownloadEvent(URI uri) {
+    this.uri = Preconditions.checkNotNull(uri);
+  }
+
+  @Override
+  protected String getValueString() {
+    return uri.toString();
+  }
+
+  @Override
+  public boolean isRelatedTo(BuckEvent event) {
+    if (!(event instanceof DownloadEvent)) {
+      return false;
+    }
+    return uri.equals(((DownloadEvent) event).uri);
+  }
+
+  public static Started started(URI uri) {
+    return new Started(uri);
+  }
+
+  public static Finished finished(URI uri) {
+    return new Finished(uri);
+  }
+
+  public static class Started extends DownloadEvent {
+    public Started(URI uri) {
+      super(uri);
+    }
+
+    @Override
+    public String getEventName() {
+      return "DownloadStarted";
+    }
+  }
+
+  public static class Finished extends DownloadEvent {
+    public Finished(URI uri) {
+      super(uri);
+    }
+
+    @Override
+    public String getEventName() {
+      return "DownloadFinished";
+    }
+  }
+}

src/com/facebook/buck/file/DownloadProgressEvent.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2014-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.file;
+
+import com.facebook.buck.event.AbstractBuckEvent;
+import com.facebook.buck.event.BuckEvent;
+import com.google.common.base.Preconditions;
+
+import java.net.URI;
+
+public class DownloadProgressEvent extends AbstractBuckEvent {
+
+  private final URI uri;
+  private final long downloadedSoFar;
+  private final String size;
+
+  public DownloadProgressEvent(URI uri, long size, long downloadedSoFar) {
+    this.uri = Preconditions.checkNotNull(uri);
+    this.size = size == -1 ? "unknown" : String.valueOf(size);
+    Preconditions.checkArgument(downloadedSoFar > 0);
+    this.downloadedSoFar = downloadedSoFar;
+  }
+
+  @Override
+  protected String getValueString() {
+    return String.format("%s -> %d/%s", uri, downloadedSoFar, size);
+  }
+
+  @Override
+  public boolean isRelatedTo(BuckEvent event) {
+    if (!(event instanceof DownloadProgressEvent)) {
+      return false;
+    }
+    return getValueString().equals(((DownloadProgressEvent) event).getValueString());
+  }
+
+  @Override
+  public String getEventName() {
+    return "DownloadProgressEvent";
+  }
+}

src/com/facebook/buck/file/DownloadStep.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2014-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.file;
+
+import com.facebook.buck.event.BuckEventBus;
+import com.facebook.buck.event.ConsoleEvent;
+import com.facebook.buck.step.ExecutionContext;
+import com.facebook.buck.step.Step;
+import com.google.common.base.Preconditions;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hashing;
+import com.google.common.io.Files;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Path;
+
+/**
+ * Download a file from a known location.
+ */
+public class DownloadStep implements Step {
+  private final URI url;
+  private final HashCode sha1;
+  private final Path output;
+  private final Downloader downloader;
+
+  public DownloadStep(Downloader downloader, URI url, HashCode sha1, Path output) {
+    this.downloader = Preconditions.checkNotNull(downloader);
+    this.url = Preconditions.checkNotNull(url);
+    this.sha1 = Preconditions.checkNotNull(sha1);
+    this.output = Preconditions.checkNotNull(output);
+  }
+
+  @Override
+  public int execute(ExecutionContext context) throws InterruptedException {
+    BuckEventBus eventBus = context.getBuckEventBus();
+    try {
+      downloader.fetch(eventBus, url, output);
+
+      HashCode readHash = Files.asByteSource(output.toFile()).hash(Hashing.sha1());
+      if (!sha1.equals(readHash)) {
+        eventBus.post(
+            ConsoleEvent.severe(
+                "Unable to download %s (hashes do not match. Expected %s, saw %s)",
+                url,
+                sha1,
+                readHash));
+        return -1;
+      }
+
+    } catch (IOException e) {
+      eventBus.post(ConsoleEvent.severe("Unable to download: %s", url));
+    }
+
+    return 0;
+  }
+
+  @Override
+  public String getShortName() {
+    return "curl";
+  }
+
+  @Override
+  public String getDescription(ExecutionContext context) {
+    return String.format("curl %s -o '%s'", url, output);
+  }
+}

src/com/facebook/buck/file/Downloader.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2014-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.file;
+
+import com.facebook.buck.event.BuckEventBus;
+import com.facebook.buck.util.HumanReadableException;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.Proxy;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Download a file over HTTP.
+ */
+public class Downloader {
+  public static final int PROGRESS_REPORT_EVERY_N_BYTES = 1000;
+  private final Optional<Proxy> proxy;
+  private final Optional<String> mavenRepo;
+
+  public Downloader(Optional<Proxy> proxy, Optional<String> mavenRepo) {
+    this.proxy = Preconditions.checkNotNull(proxy);
+    this.mavenRepo = Preconditions.checkNotNull(mavenRepo);
+  }
+
+  public void fetch(BuckEventBus eventBus, URI uri, Path output) throws IOException {
+    if ("mvn".equals(uri.getScheme())) {
+      uri = MavenUrlDecoder.toHttpUrl(mavenRepo, uri);
+    }
+
+    eventBus.post(DownloadEvent.started(uri));
+
+    try {
+      HttpURLConnection connection = createConnection(uri);
+      if (HttpURLConnection.HTTP_OK != connection.getResponseCode()) {
+        throw new HumanReadableException(
+            "Unable to download %s: %s", uri, connection.getResponseMessage());
+      }
+      long contentLength = connection.getContentLengthLong();
+      try (InputStream is = new BufferedInputStream(connection.getInputStream());
+           OutputStream os = new BufferedOutputStream(Files.newOutputStream(output))) {
+        long read = 0;
+
+        while (true) {
+          int r = is.read();
+          read++;
+          if (r == -1) {
+            break;
+          }
+          if (read % PROGRESS_REPORT_EVERY_N_BYTES == 0) {
+            eventBus.post(new DownloadProgressEvent(uri, contentLength, read));
+          }
+          os.write(r);
+        }
+      }
+    } finally {
+      eventBus.post(DownloadEvent.finished(uri));
+    }
+  }
+
+  protected HttpURLConnection createConnection(URI uri) throws IOException {
+    if (!("http".equals(uri.getScheme()) || "https".equals(uri.getScheme()))) {
+      throw new HumanReadableException(
+          "Cowardly refusing to download with unknown scheme: %s", uri);
+    }
+
+    HttpURLConnection connection;
+    if (proxy.isPresent()) {
+      connection = (HttpURLConnection) uri.toURL().openConnection(proxy.get());
+    } else {
+      connection = (HttpURLConnection) uri.toURL().openConnection();
+    }
+    connection.setConnectTimeout((int) TimeUnit.SECONDS.toMillis(20));
+    connection.setInstanceFollowRedirects(true);
+
+    return connection;
+  }
+}