Implement compare_db_backups "main"

Implement toplevel class that reads in two database backups and displays
diffs.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=168592124

java/google/registry/tools/BUILD

@@ -13,6 +13,7 @@ package_group(
         "//java/google/registry/eclipse",
         "//java/google/registry/testing",
         "//java/google/registry/tools",
+        "//javatests/google/registry/testing",
         "//javatests/google/registry/tools",
     ],
 )
@@ -100,3 +101,18 @@ java_binary(
         "@com_google_appengine_remote_api//:link",
     ],
 )
+
+java_binary(
+    name = "compare_db_backups",
+    srcs = [
+        "CompareDbBackups.java",
+    ],
+    create_executable = 1,
+    main_class = "google.registry.tools.CompareDbBackups",
+    deps = [
+        ":tools",
+        "@com_google_appengine_api_1_0_sdk",
+        "@com_google_guava",
+        "@com_google_protobuf_java",
+    ],
+)

java/google/registry/tools/CompareDbBackups.java

@@ -0,0 +1,67 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Sets.SetView;
+import java.io.File;
+
+/** Compare two database backups. */
+class CompareDbBackups {
+
+  public static void main(String[] args) throws Exception {
+    if (args.length != 2) {
+      System.err.println("Usage: compare_db_backups <directory1> <directory2>");
+      return;
+    }
+
+    ImmutableSet<ComparableEntity> entities1 =
+        new RecordAccumulator().readDirectory(new File(args[0])).getComparableEntitySet();
+    ImmutableSet<ComparableEntity> entities2 =
+        new RecordAccumulator().readDirectory(new File(args[1])).getComparableEntitySet();
+
+    // Calculate the entities added and removed.
+    SetView<ComparableEntity> added = Sets.difference(entities2, entities1);
+    SetView<ComparableEntity> removed = Sets.difference(entities1, entities2);
+
+    printHeader(
+        String.format("First backup: %d records", entities1.size()),
+        String.format("Second backup: %d records", entities2.size()));
+
+    if (!removed.isEmpty()) {
+      printHeader(removed.size() + " records were removed:");
+      for (ComparableEntity entity : removed) {
+        System.out.println(entity);
+      }
+    }
+
+    if (!added.isEmpty()) {
+      printHeader(added.size() + " records were added:");
+      for (ComparableEntity entity : added) {
+        System.out.println(entity);
+      }
+    }
+  }
+
+  /** Print out multi-line text in a pretty ASCII header frame. */
+  private static void printHeader(String... headerLines) {
+    System.out.println("========================================================================");
+    for (String line : headerLines) {
+      System.out.println("| " + line);
+    }
+    System.out.println("========================================================================");
+  }
+}

javatests/google/registry/tools/CompareDbBackupsTest.java

@@ -0,0 +1,80 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import google.registry.testing.AppEngineRule;
+import google.registry.tools.LevelDbFileBuilder.Property;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.PrintStream;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class CompareDbBackupsTest {
+
+  private static final int BASE_ID = 1001;
+
+  // Capture standard output.
+  private final ByteArrayOutputStream stdout = new ByteArrayOutputStream();
+
+  @Rule public final TemporaryFolder tempFs = new TemporaryFolder();
+  @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build();
+
+  @Test
+  public void testCommand() throws Exception {
+
+    // Create two directories corresponding to data dumps.
+    File dump1 = tempFs.newFolder("dump1");
+    LevelDbFileBuilder builder = new LevelDbFileBuilder(new File(dump1, "data1"));
+    builder.addEntityProto(
+        BASE_ID,
+        Property.create("eeny", 100L),
+        Property.create("meeny", 200L),
+        Property.create("miney", 300L));
+    builder.addEntityProto(
+        BASE_ID + 1,
+        Property.create("moxey", 100L),
+        Property.create("minney", 200L),
+        Property.create("motz", 300L));
+    builder.build();
+
+    File dump2 = tempFs.newFolder("dump2");
+    builder = new LevelDbFileBuilder(new File(dump2, "data2"));
+    builder.addEntityProto(
+        BASE_ID + 1,
+        Property.create("moxey", 100L),
+        Property.create("minney", 200L),
+        Property.create("motz", 300L));
+    builder.addEntityProto(
+        BASE_ID + 2,
+        Property.create("blutzy", 100L),
+        Property.create("fishey", 200L),
+        Property.create("strutz", 300L));
+    builder.build();
+
+    System.setOut(new PrintStream(stdout));
+    CompareDbBackups.main(new String[] {dump1.getCanonicalPath(), dump2.getCanonicalPath()});
+    String output = new String(stdout.toByteArray(), UTF_8);
+    assertThat(output)
+        .containsMatch("(?s)1 records were removed.*eeny.*1 records were added.*blutzy");
+  }
+}

javatests/google/registry/tools/LevelDbFileBuilder.java

@@ -0,0 +1,83 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static google.registry.tools.LevelDbLogReader.BLOCK_SIZE;
+import static google.registry.tools.LevelDbLogReader.HEADER_SIZE;
+
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.EntityTranslator;
+import com.google.auto.value.AutoValue;
+import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
+import google.registry.tools.LevelDbLogReader.ChunkType;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+/** Utility class for building a leveldb logfile. */
+final class LevelDbFileBuilder {
+  private static final String TEST_ENTITY_KIND = "TestEntity";
+
+  private final FileOutputStream out;
+  private byte[] currentBlock = new byte[BLOCK_SIZE];
+
+  // Write position in the current block.
+  private int currentPos = 0;
+
+  LevelDbFileBuilder(File file) throws FileNotFoundException {
+    out = new FileOutputStream(file);
+  }
+
+  /**
+   * Adds a record containing a new entity protobuf to the file.
+   *
+   * <p>Returns the ComparableEntity object rather than "this" so that we can check for the presence
+   * of the entity in the result set.
+   */
+  ComparableEntity addEntityProto(int id, Property... properties) throws IOException {
+    Entity entity = new Entity(TEST_ENTITY_KIND, id);
+    for (Property prop : properties) {
+      entity.setProperty(prop.name(), prop.value());
+    }
+    EntityProto proto = EntityTranslator.convertToPb(entity);
+    byte[] protoBytes = proto.toByteArray();
+    if (protoBytes.length > BLOCK_SIZE - (currentPos + HEADER_SIZE)) {
+      out.write(currentBlock);
+      currentBlock = new byte[BLOCK_SIZE];
+      currentPos = 0;
+    }
+
+    currentPos = LevelDbUtil.addRecord(currentBlock, currentPos, ChunkType.FULL, protoBytes);
+    return new ComparableEntity(entity);
+  }
+
+  /** Writes all remaining data and closes the block. */
+  void build() throws IOException {
+    out.write(currentBlock);
+    out.close();
+  }
+
+  @AutoValue
+  abstract static class Property {
+    static Property create(String name, Object value) {
+      return new AutoValue_LevelDbFileBuilder_Property(name, value);
+    }
+
+    abstract String name();
+
+    abstract Object value();
+  }
+}

javatests/google/registry/tools/LevelDbFileBuilderTest.java

@@ -0,0 +1,98 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.EntityTranslator;
+import com.google.common.collect.ImmutableList;
+import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
+import google.registry.testing.AppEngineRule;
+import google.registry.tools.LevelDbFileBuilder.Property;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class LevelDbFileBuilderTest {
+
+  public static final int BASE_ID = 1001;
+
+  @Rule public final TemporaryFolder tempFs = new TemporaryFolder();
+  @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build();
+
+  @Test
+  public void testSingleRecordWrites() throws FileNotFoundException, IOException {
+    File subdir = tempFs.newFolder("folder");
+    File logFile = new File(subdir, "testfile");
+    LevelDbFileBuilder builder = new LevelDbFileBuilder(logFile);
+    ComparableEntity entity =
+        builder.addEntityProto(
+            BASE_ID, Property.create("first", 100L), Property.create("second", 200L));
+    builder.build();
+
+    LevelDbLogReader reader = new LevelDbLogReader();
+    reader.readFrom(new FileInputStream(logFile));
+
+    ImmutableList<byte[]> records = reader.getRecords();
+    assertThat(records).hasSize(1);
+
+    // Reconstitute an entity, make sure that what we've got is the same as what we started with.
+    EntityProto proto = new EntityProto();
+    proto.parseFrom(records.get(0));
+    Entity materializedEntity = EntityTranslator.createFromPb(proto);
+    assertThat(new ComparableEntity(materializedEntity)).isEqualTo(entity);
+  }
+
+  @Test
+  public void testMultipleRecordWrites() throws FileNotFoundException, IOException {
+    File subdir = tempFs.newFolder("folder");
+    File logFile = new File(subdir, "testfile");
+    LevelDbFileBuilder builder = new LevelDbFileBuilder(logFile);
+
+    // Generate enough records to cross a block boundary.  These records end up being around 80
+    // bytes, so 1000 works.
+    ImmutableList.Builder<ComparableEntity> originalEntitiesBuilder = new ImmutableList.Builder<>();
+    for (int i = 0; i < 1000; ++i) {
+      ComparableEntity entity =
+          builder.addEntityProto(
+              BASE_ID + i, Property.create("first", 100L), Property.create("second", 200L));
+      originalEntitiesBuilder.add(entity);
+    }
+    builder.build();
+    ImmutableList<ComparableEntity> originalEntities = originalEntitiesBuilder.build();
+
+    LevelDbLogReader reader = new LevelDbLogReader();
+    reader.readFrom(new FileInputStream(logFile));
+
+    ImmutableList<byte[]> records = reader.getRecords();
+    assertThat(records).hasSize(1000);
+    int index = 0;
+    for (byte[] record : records) {
+      EntityProto proto = new EntityProto();
+      proto.parseFrom(record);
+      Entity materializedEntity = EntityTranslator.createFromPb(proto);
+      assertThat(new ComparableEntity(materializedEntity)).isEqualTo(originalEntities.get(index));
+      ++index;
+    }
+  }
+}