[WIP] Add bazel runfiles manifest support for Windows

tensorflow/core/example/example_parser_configuration_test.cc

@@ -43,9 +43,8 @@ class ExtractExampleParserConfigurationTest : public ::testing::Test {
  protected:
   void SetUp() override {
     string proto_string;
-    string filename =
-        io::JoinPath(testing::TensorFlowSrcRoot(),
-                     "core/example/testdata/parse_example_graph_def.pbtxt");
+    string filename = testing::RunFileRelocator::GetInstance().Relocate(
+        "core/example/testdata/parse_example_graph_def.pbtxt");
     ReadFileToStringOrDie(Env::Default(), filename, &proto_string);
     protobuf::TextFormat::ParseFromString(proto_string, &graph_def_);
     session_ = CreateSession();

tensorflow/core/platform/cloud/oauth_client_test.cc

@@ -14,10 +14,10 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/platform/cloud/oauth_client.h"
-#include <fstream>
 #include <openssl/bio.h>
 #include <openssl/evp.h>
 #include <openssl/pem.h>
+#include <fstream>
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/base64.h"
@@ -90,9 +90,9 @@ TEST(OAuthClientTest, GetTokenFromRefreshTokenJson) {
 }
 
 TEST(OAuthClientTest, GetTokenFromServiceAccountJson) {
-  std::ifstream credentials(
-      io::JoinPath(io::JoinPath(testing::TensorFlowSrcRoot(), kTestData),
-                   "service_account_credentials.json"));
+  string filename = testing::RunFileRelocator::GetInstance().Relocate(
+      io::JoinPath(kTestData, "service_account_credentials.json"));
+  std::ifstream credentials(filename);
   ASSERT_TRUE(credentials.is_open());
   Json::Value json;
   Json::Reader reader;
@@ -133,9 +133,10 @@ TEST(OAuthClientTest, GetTokenFromServiceAccountJson) {
   // Check that 'signature' signs 'header_dot_claim'.
 
   // Read the serialized public key.
-  std::ifstream public_key_stream(
-      io::JoinPath(io::JoinPath(testing::TensorFlowSrcRoot(), kTestData),
-                   "service_account_public_key.txt"));
+  string public_key_filename =
+      testing::RunFileRelocator::GetInstance().Relocate(
+          io::JoinPath(kTestData, "service_account_public_key.txt"));
+  std::ifstream public_key_stream(public_key_filename);
   string public_key_serialized(
       (std::istreambuf_iterator<char>(public_key_stream)),
       (std::istreambuf_iterator<char>()));

tensorflow/core/platform/posix/test.cc

@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/core/platform/net.h"
 #include "tensorflow/core/platform/test.h"
 
+#include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/subprocess.h"
@@ -34,6 +35,13 @@ std::unique_ptr<SubProcess> CreateSubProcess(const std::vector<string>& argv) {
 }
 
 int PickUnusedPortOrDie() { return internal::PickUnusedPortOrDie(); }
+void RunFileRelocator::Init() { src_root = testing::TensorFlowSrcRoot(); }
+
+std::unique_ptr<RunFileRelocator> RunFileRelocator::m_instance;
+std::once_flag RunFileRelocator::once;
+std::string RunFileRelocator::Relocate(const string& runfile) const {
+  return io::JoinPath(src_root, runfile);
+}
 
 string TensorFlowSrcRoot() {
   // 'bazel test' sets TEST_SRCDIR, and also TEST_WORKSPACE if a new

tensorflow/core/platform/test.h

@@ -17,8 +17,10 @@ limitations under the License.
 #define TENSORFLOW_PLATFORM_TEST_H_
 
 #include <memory>
+#include <mutex>
+#include <unordered_map>
 #include <vector>
-
+#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/platform.h"
 #include "tensorflow/core/platform/subprocess.h"
@@ -38,6 +40,26 @@ limitations under the License.
 namespace tensorflow {
 namespace testing {
 
+class RunFileRelocator {
+ private:
+  static std::unique_ptr<RunFileRelocator> m_instance;
+  static std::once_flag once;
+  string src_root;
+  std::unordered_map<string, string> manifest;
+  void Init();
+
+ public:
+  // Returns the absolute path to a runfile.
+  std::string Relocate(const string& runfile) const;
+  static const RunFileRelocator& GetInstance() {
+    std::call_once(once, [] {
+      m_instance.reset(new RunFileRelocator());
+      m_instance->Init();
+    });
+    return *m_instance;
+  }
+};
+
 // Return a temporary directory suitable for temporary testing files.
 string TmpDir();
 

tensorflow/core/platform/windows/test.cc

@@ -13,9 +13,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/core/platform/net.h"
 #include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/net.h"
 
+#include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
 
@@ -29,6 +31,77 @@ std::unique_ptr<SubProcess> CreateSubProcess(const std::vector<string>& argv) {
 
 int PickUnusedPortOrDie() { return internal::PickUnusedPortOrDie(); }
 
+static void fill_manifest(std::unordered_map<string, string>& out,
+                          const std::string& line, size_t line_begin,
+                          size_t line_end) {
+  // line[line_end] == '\0' or line[line_end] == '\n'
+  if (line[line_end - 1] == '\r') {
+    --line_end;
+  }
+  size_t space_location = string::npos;
+  for (size_t i = line_begin; i != line_end; ++i) {
+    if (line[i] == ' ') {
+      if (space_location != string::npos) {
+        LOG(FATAL) << "manifest file format error, contains multiple spaces in "
+                      "one line";
+      }
+      space_location = i;
+    }
+  }
+  if (space_location == string::npos) {
+    LOG(FATAL) << "manifest file format error, contains no spaces in this line";
+  }
+  const size_t key_len = space_location - line_begin;
+  const size_t value_len = line_end - space_location - 1;
+  out[line.substr(line_begin, key_len)] =
+      line.substr(space_location + 1, value_len);
+}
+
+void RunFileRelocator::Init() {
+  size_t requiredSize;
+  getenv_s(&requiredSize, NULL, 0, "RUNFILES_MANIFEST_FILE");
+  if (requiredSize != 0) {
+    string manifest_file_path(requiredSize - 1, '\0');
+    getenv_s(&requiredSize, (char*)manifest_file_path.data(), requiredSize,
+             "RUNFILES_MANIFEST_FILE");
+    string manifest_file_content;
+    TF_CHECK_OK(ReadFileToString(Env::Default(), manifest_file_path,
+                                 &manifest_file_content));
+    size_t line_begin = 0;
+    size_t line_end;
+    while ((line_end = manifest_file_content.find('\n', line_begin)) !=
+           string::npos) {
+      fill_manifest(manifest, manifest_file_content, line_begin, line_end);
+      line_begin = line_end + 1;
+    }
+    // Now line_end == string::npos
+    if (line_begin < manifest_file_content.size()) {
+      line_end = manifest_file_content.size();
+      fill_manifest(manifest, manifest_file_content, line_begin, line_end);
+    }
+  }
+  src_root = "/tensorflow";
+  getenv_s(&requiredSize, NULL, 0, "TEST_WORKSPACE");
+  if (requiredSize == 0) {
+    return;
+  }
+  string workspace_path(requiredSize - 1, '\0');
+  getenv_s(&requiredSize, (char*)workspace_path.data(), requiredSize,
+           "TEST_WORKSPACE");
+  src_root = workspace_path + src_root;
+}
+std::unique_ptr<RunFileRelocator> RunFileRelocator::m_instance;
+std::once_flag RunFileRelocator::once;
+std::string RunFileRelocator::Relocate(const string& runfile) const {
+  // TODO: normalize path
+  auto iter = manifest.find(strings::StrCat(src_root, "/", runfile));
+  if (iter != manifest.end()) {
+    return iter->second;
+  }
+  //"using $PWD/tensorflow as TensorFlowSrcRoot() for tests."
+  return strings::StrCat("tensorflow/", runfile);
+}
+
 string TensorFlowSrcRoot() {
   // 'bazel test' and cmake set TEST_SRCDIR.
   // New versions of bazel also set TEST_WORKSPACE.