Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Overwrite the target file in HDFS Rename operation; it removes the ta… #4789

Merged
merged 1 commit into from Oct 6, 2016
Merged

Overwrite the target file in HDFS Rename operation; it removes the ta… #4789

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions tensorflow/core/platform/file_system.h
View file
Open in desktop
Expand Up @@ -80,6 +80,7 @@ class FileSystem {

virtual Status GetFileSize(const string& fname, uint64* file_size) = 0;

// Overwrites the target if it exists.
virtual Status RenameFile(const string& src, const string& target) = 0;

// Translate an URI to a filename usable by the FileSystem implementation. The
Expand Down
6 changes: 6 additions & 0 deletions tensorflow/core/platform/hadoop/hadoop_file_system.cc
View file
Open in desktop
Expand Up @@ -393,6 +393,12 @@ Status HadoopFileSystem::RenameFile(const string& src, const string& target) {
hdfsFS fs = nullptr;
TF_RETURN_IF_ERROR(Connect(src, &fs));

if (hdfs_->hdfsExists(fs, TranslateName(target).c_str()) == 0 &&
hdfs_->hdfsDelete(fs, TranslateName(target).c_str(),
/*recursive=*/0) != 0) {
return IOError(target, errno);
}

if (hdfs_->hdfsRename(fs, TranslateName(src).c_str(),
TranslateName(target).c_str()) != 0) {
return IOError(src, errno);
Expand Down
16 changes: 16 additions & 0 deletions tensorflow/core/platform/hadoop/hadoop_file_system_test.cc
View file
Open in desktop
Expand Up @@ -168,6 +168,22 @@ TEST_F(HadoopFileSystemTest, RenameFile) {
EXPECT_EQ("test", content);
}

TEST_F(HadoopFileSystemTest, RenameFile_Overwrite) {
const string fname1 =
"file://" + io::JoinPath(testing::TmpDir(), "RenameFile1");
const string fname2 =
"file://" + io::JoinPath(testing::TmpDir(), "RenameFile2");

TF_ASSERT_OK(WriteString(fname2, "test"));
EXPECT_TRUE(hdfs.FileExists(fname2));

TF_ASSERT_OK(WriteString(fname1, "test"));
TF_EXPECT_OK(hdfs.RenameFile(fname1, fname2));
string content;
TF_EXPECT_OK(ReadAll(fname2, &content));
EXPECT_EQ("test", content);
}

TEST_F(HadoopFileSystemTest, StatFile) {
const string fname = "file://" + io::JoinPath(testing::TmpDir(), "StatFile");
TF_ASSERT_OK(WriteString(fname, "test"));
Expand Down