Export @195097388.

tensorflow · May 2, 2018 · a4bb31d · a4bb31d
1 parent dea7ecf
commit a4bb31d
Show file tree

Hide file tree

Showing 442 changed files with 154,211 additions and 2,951 deletions.
diff --git a/research/syntaxnet/Dockerfile b/research/syntaxnet/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:16.10
+FROM ubuntu:16.04
 
 ENV SYNTAXNETDIR=/opt/tensorflow PATH=$PATH:/root/bin
 
@@ -57,10 +57,10 @@ RUN python -m pip install \
     && rm -rf /root/.cache/pip /tmp/pip*
 
 # Installs Bazel.
-RUN wget --quiet https://github.com/bazelbuild/bazel/releases/download/0.8.1/bazel-0.8.1-installer-linux-x86_64.sh \
-    && chmod +x bazel-0.8.1-installer-linux-x86_64.sh \
-    && ./bazel-0.8.1-installer-linux-x86_64.sh \
-    && rm ./bazel-0.8.1-installer-linux-x86_64.sh
+RUN wget --quiet https://github.com/bazelbuild/bazel/releases/download/0.11.1/bazel-0.11.1-installer-linux-x86_64.sh \
+    && chmod +x bazel-0.11.1-installer-linux-x86_64.sh \
+    && ./bazel-0.11.1-installer-linux-x86_64.sh \
+    && rm ./bazel-0.11.1-installer-linux-x86_64.sh
 
 COPY WORKSPACE $SYNTAXNETDIR/syntaxnet/WORKSPACE
 COPY tools/bazel.rc $SYNTAXNETDIR/syntaxnet/tools/bazel.rc

diff --git a/research/syntaxnet/README.md b/research/syntaxnet/README.md
@@ -60,10 +60,10 @@ The simplest way to get started with DRAGNN is by loading our Docker container.
 [Here](g3doc/CLOUD.md) is a tutorial for running the DRAGNN container on
 [GCP](https://cloud.google.com) (just as applicable to your own computer).
 
-### Ubuntu 16.10+ binary installation
+### Ubuntu 16.04+ binary installation
 
 _This process takes ~5 minutes, but is only compatible with Linux using GNU libc
-3.4.22 and above (e.g. Ubuntu 16.10)._
+3.4.22 and above (e.g. Ubuntu 16.04)._
 
 Binary wheel packages are provided for TensorFlow and SyntaxNet. If you do not
 need to write new binary TensorFlow ops, these should suffice.
@@ -92,9 +92,9 @@ source. You'll need to install:
 
 *   python 2.7:
     *   Python 3 support is not available yet
-*   bazel 0.5.4:
+*   bazel 0.11.1:
     *   Follow the instructions [here](http://bazel.build/docs/install.html)
-    *   Alternately, Download bazel 0.5.4 <.deb> from
+    *   Alternately, Download bazel 0.11.1 <.deb> from
         [https://github.com/bazelbuild/bazel/releases](https://github.com/bazelbuild/bazel/releases)
         for your system configuration.
     *   Install it using the command: sudo dpkg -i <.deb file>
@@ -105,14 +105,14 @@ source. You'll need to install:
 *   protocol buffers, with a version supported by TensorFlow:
     *   check your protobuf version with `pip freeze | grep protobuf`
     *   upgrade to a supported version with `pip install -U protobuf==3.3.0`
+*   autograd, with a version supported by TensorFlow:
+    *   `pip install -U autograd==1.1.13`
 *   mock, the testing package:
     *   `pip install mock`
 *   asciitree, to draw parse trees on the console for the demo:
     *   `pip install asciitree`
 *   numpy, package for scientific computing:
     *   `pip install numpy`
-*   autograd 1.1.13, for automatic differentiation (not yet compatible with autograd v1.2 rewrite):
-    *   `pip install autograd==1.1.13`
 *   pygraphviz to visualize traces and parse trees:
     *   `apt-get install -y graphviz libgraphviz-dev`
     *   `pip install pygraphviz

diff --git a/research/syntaxnet/WORKSPACE b/research/syntaxnet/WORKSPACE
@@ -1,6 +1,6 @@
 local_repository(
-  name = "org_tensorflow",
-  path = "tensorflow",
+    name = "org_tensorflow",
+    path = "tensorflow",
 )
 
 # We need to pull in @io_bazel_rules_closure for TensorFlow. Bazel design
@@ -9,22 +9,33 @@ local_repository(
 # @io_bazel_rules_closure.
 http_archive(
     name = "io_bazel_rules_closure",
-    sha256 = "25f5399f18d8bf9ce435f85c6bbf671ec4820bc4396b3022cc5dc4bc66303609",
-    strip_prefix = "rules_closure-0.4.2",
+    sha256 = "6691c58a2cd30a86776dd9bb34898b041e37136f2dc7e24cadaeaf599c95c657",
+    strip_prefix = "rules_closure-08039ba8ca59f64248bb3b6ae016460fe9c9914f",
     urls = [
-        "http://bazel-mirror.storage.googleapis.com/github.com/bazelbuild/rules_closure/archive/0.4.2.tar.gz",  # 2017-08-30
-        "https://github.com/bazelbuild/rules_closure/archive/0.4.2.tar.gz",
+        "http://bazel-mirror.storage.googleapis.com/github.com/bazelbuild/rules_closure/archive/08039ba8ca59f64248bb3b6ae016460fe9c9914f.tar.gz",
+        "https://github.com/bazelbuild/rules_closure/archive/08039ba8ca59f64248bb3b6ae016460fe9c9914f.tar.gz",
     ],
 )
 
 load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
-tf_workspace(path_prefix="", tf_repo_name="org_tensorflow")
 
-# Test that Bazel is up-to-date.
-load("@org_tensorflow//tensorflow:workspace.bzl", "check_version")
-check_version("0.4.2")
+tf_workspace(
+    path_prefix = "",
+    tf_repo_name = "org_tensorflow",
+)
+
+http_archive(
+    name = "sling",
+    sha256 = "f1ce597476cb024808ca0a371a01db9dda4e0c58fb34a4f9c4ea91796f437b10",
+    strip_prefix = "sling-e3ae9d94eb1d9ee037a851070d54ed2eefaa928a",
+    urls = [
+        "http://bazel-mirror.storage.googleapis.com/github.com/google/sling/archive/e3ae9d94eb1d9ee037a851070d54ed2eefaa928a.tar.gz",
+        "https://github.com/google/sling/archive/e3ae9d94eb1d9ee037a851070d54ed2eefaa928a.tar.gz",
+    ],
+)
 
+# Used by SLING.
 bind(
-    name = "protobuf",
-    actual = "@protobuf_archive//:protobuf",
+    name = "zlib",
+    actual = "@zlib_archive//:zlib",
 )
diff --git a/research/syntaxnet/docker-devel/Dockerfile-test b/research/syntaxnet/docker-devel/Dockerfile-test
@@ -9,3 +9,4 @@ COPY dragnn $SYNTAXNETDIR/syntaxnet/dragnn
 COPY syntaxnet $SYNTAXNETDIR/syntaxnet/syntaxnet
 COPY third_party $SYNTAXNETDIR/syntaxnet/third_party
 COPY util/utf8 $SYNTAXNETDIR/syntaxnet/util/utf8
+COPY WORKSPACE $SYNTAXNETDIR/syntaxnet/WORKSPACE
diff --git a/research/syntaxnet/docker-devel/Dockerfile-test-base b/research/syntaxnet/docker-devel/Dockerfile-test-base
@@ -1,4 +1,4 @@
-FROM ubuntu:16.10
+FROM ubuntu:16.04
 
 ENV SYNTAXNETDIR=/opt/tensorflow PATH=$PATH:/root/bin
 
@@ -57,10 +57,10 @@ RUN python -m pip install \
     && rm -rf /root/.cache/pip /tmp/pip*
 
 # Installs Bazel.
-RUN wget --quiet https://github.com/bazelbuild/bazel/releases/download/0.5.3/bazel-0.5.3-installer-linux-x86_64.sh \
-    && chmod +x bazel-0.5.3-installer-linux-x86_64.sh \
-    && JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ ./bazel-0.5.3-installer-linux-x86_64.sh \
-    && rm ./bazel-0.5.3-installer-linux-x86_64.sh
+RUN wget --quiet https://github.com/bazelbuild/bazel/releases/download/0.11.1/bazel-0.11.1-installer-linux-x86_64.sh \
+    && chmod +x bazel-0.11.1-installer-linux-x86_64.sh \
+    && JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ ./bazel-0.11.1-installer-linux-x86_64.sh \
+    && rm ./bazel-0.11.1-installer-linux-x86_64.sh
 
 COPY WORKSPACE $SYNTAXNETDIR/syntaxnet/WORKSPACE
 COPY tools/bazel.rc $SYNTAXNETDIR/syntaxnet/tools/bazel.rc
@@ -69,12 +69,9 @@ COPY tools/bazel.rc $SYNTAXNETDIR/syntaxnet/tools/bazel.rc
 # source. This makes it more convenient to re-compile DRAGNN / SyntaxNet for
 # development (though not as convenient as the docker-devel scripts).
 RUN cd $SYNTAXNETDIR/syntaxnet \
-    && git clone --branch r1.3 --recurse-submodules https://github.com/tensorflow/tensorflow \
+    && git clone --branch r1.8 --recurse-submodules https://github.com/tensorflow/tensorflow \
     && cd tensorflow \
-    # This line removes a bad archive target which causes Tensorflow install
-    # to fail.
-    && sed -i '\@https://github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz@d' tensorflow/workspace.bzl \
-    && tensorflow/tools/ci_build/builds/configured CPU \\
+    && tensorflow/tools/ci_build/builds/configured CPU \
     && cd $SYNTAXNETDIR/syntaxnet \
     && bazel build -c opt @org_tensorflow//tensorflow:tensorflow_py
 

diff --git a/research/syntaxnet/docker-devel/Dockerfile.min b/research/syntaxnet/docker-devel/Dockerfile.min
@@ -3,7 +3,7 @@
 #
 # It might be more efficient to use a minimal distribution, like Alpine. But
 # the upside of this being popular is that people might already have it.
-FROM ubuntu:16.10
+FROM ubuntu:16.04
 
 ENV SYNTAXNETDIR=/opt/tensorflow PATH=$PATH:/root/bin
 

diff --git a/research/syntaxnet/dragnn/components/stateless/BUILD b/research/syntaxnet/dragnn/components/stateless/BUILD
@@ -10,7 +10,8 @@ cc_library(
         "//dragnn/core:component_registry",
         "//dragnn/core/interfaces:component",
         "//dragnn/core/interfaces:transition_state",
-        "//dragnn/protos:data_proto",
+        "//dragnn/core/util:label",
+        "//dragnn/protos:data_proto_cc",
         "//syntaxnet:base",
     ],
     alwayslink = 1,
@@ -27,7 +28,7 @@ cc_test(
         "//dragnn/core/test:mock_transition_state",
         "//dragnn/io:sentence_input_batch",
         "//syntaxnet:base",
-        "//syntaxnet:sentence_proto",
+        "//syntaxnet:sentence_proto_cc",
         "//syntaxnet:test_main",
     ],
 )
diff --git a/research/syntaxnet/dragnn/components/stateless/stateless_component.cc b/research/syntaxnet/dragnn/components/stateless/stateless_component.cc
@@ -16,6 +16,7 @@
 #include "dragnn/core/component_registry.h"
 #include "dragnn/core/interfaces/component.h"
 #include "dragnn/core/interfaces/transition_state.h"
+#include "dragnn/core/util/label.h"
 #include "dragnn/protos/data.pb.h"
 #include "syntaxnet/base.h"
 
@@ -90,7 +91,8 @@ class StatelessComponent : public Component {
   void AdvanceFromOracle() override {
     LOG(FATAL) << "[" << name_ << "] AdvanceFromOracle not supported";
   }
-  std::vector<std::vector<int>> GetOracleLabels() const override {
+  std::vector<std::vector<std::vector<Label>>> GetOracleLabels()
+      const override {
     LOG(FATAL) << "[" << name_ << "] Method not supported";
   }
   int GetFixedFeatures(std::function<int32 *(int)> allocate_indices,
@@ -108,7 +110,15 @@ class StatelessComponent : public Component {
       float *embedding_output) override {
     LOG(FATAL) << "[" << name_ << "] Method not supported";
   }
-
+  void BulkEmbedDenseFixedFeatures(
+      const vector<const float *> &per_channel_embeddings,
+      float *embedding_output, int embedding_output_size,
+      int32 *offset_array_output, int offset_array_size) override {
+    LOG(FATAL) << "[" << name_ << "] Method not supported";
+  }
+  int BulkDenseFeatureSize() const override {
+    LOG(FATAL) << "Method not supported";
+  }
   std::vector<LinkFeatures> GetRawLinkFeatures(int channel_id) const override {
     LOG(FATAL) << "[" << name_ << "] Method not supported";
   }
@@ -118,9 +128,9 @@ class StatelessComponent : public Component {
   }
 
  private:
-  string name_;  // component name
+  string name_;         // component name
   int batch_size_ = 1;  // number of sentences in current batch
-  int beam_size_ = 1;  // maximum beam size
+  int beam_size_ = 1;   // maximum beam size
 
   // Parent states passed to InitializeData(), and passed along in GetBeam().
   std::vector<std::vector<const TransitionState *>> parent_states_;

diff --git a/research/syntaxnet/dragnn/components/syntaxnet/BUILD b/research/syntaxnet/dragnn/components/syntaxnet/BUILD
@@ -16,18 +16,20 @@ cc_library(
         "//dragnn/core:input_batch_cache",
         "//dragnn/core/interfaces:component",
         "//dragnn/core/interfaces:transition_state",
+        "//dragnn/core/util:label",
         "//dragnn/io:sentence_input_batch",
         "//dragnn/io:syntaxnet_sentence",
-        "//dragnn/protos:data_proto",
-        "//dragnn/protos:spec_proto",
-        "//dragnn/protos:trace_proto",
+        "//dragnn/protos:data_proto_cc",
+        "//dragnn/protos:spec_proto_cc",
+        "//dragnn/protos:trace_proto_cc",
         "//syntaxnet:base",
         "//syntaxnet:parser_transitions",
         "//syntaxnet:registry",
-        "//syntaxnet:sparse_proto",
+        "//syntaxnet:sparse_proto_cc",
         "//syntaxnet:task_context",
-        "//syntaxnet:task_spec_proto",
+        "//syntaxnet:task_spec_proto_cc",
         "//syntaxnet:utils",
+        "//util/utf8:unicodetext",
     ],
     alwayslink = 1,
 )
@@ -37,7 +39,7 @@ cc_library(
     srcs = ["syntaxnet_link_feature_extractor.cc"],
     hdrs = ["syntaxnet_link_feature_extractor.h"],
     deps = [
-        "//dragnn/protos:spec_proto",
+        "//dragnn/protos:spec_proto_cc",
         "//syntaxnet:base",
         "//syntaxnet:embedding_feature_extractor",
         "//syntaxnet:parser_transitions",
@@ -53,7 +55,7 @@ cc_library(
         "//dragnn/core/interfaces:cloneable_transition_state",
         "//dragnn/core/interfaces:transition_state",
         "//dragnn/io:syntaxnet_sentence",
-        "//dragnn/protos:trace_proto",
+        "//dragnn/protos:trace_proto_cc",
         "//syntaxnet:base",
         "//syntaxnet:parser_transitions",
     ],
@@ -77,7 +79,7 @@ cc_test(
         "//dragnn/core/test:mock_transition_state",
         "//dragnn/io:sentence_input_batch",
         "//syntaxnet:base",
-        "//syntaxnet:sentence_proto",
+        "//syntaxnet:sentence_proto_cc",
         "//syntaxnet:test_main",
     ],
 )
@@ -88,7 +90,7 @@ cc_test(
     deps = [
         ":syntaxnet_link_feature_extractor",
         "//dragnn/core/test:generic",
-        "//dragnn/protos:spec_proto",
+        "//dragnn/protos:spec_proto_cc",
         "//syntaxnet:task_context",
         "//syntaxnet:test_main",
     ],
@@ -105,9 +107,9 @@ cc_test(
         "//dragnn/core/test:generic",
         "//dragnn/core/test:mock_transition_state",
         "//dragnn/io:sentence_input_batch",
-        "//dragnn/protos:spec_proto",
+        "//dragnn/protos:spec_proto_cc",
         "//syntaxnet:base",
-        "//syntaxnet:sentence_proto",
+        "//syntaxnet:sentence_proto_cc",
         "//syntaxnet:test_main",
     ],
 )