Skip to content
Permalink
Browse files

Add dual write for Registrar (#474)

* Add dual write for Registrar

* Use @AlsoLoad to set street field for Cloud SQL

* Change email body to use the new streetLine field

* Refactored the logic to handle street fields

* Simplify postLoad function

* Address comments

* Add a TODO to remove street

* Add test for onLoad and postLoad

* Rebase on master
  • Loading branch information
hstonec committed Feb 13, 2020
1 parent 22a879e commit b9c40648d077c99799d7fa35b7820dce3ec7989d
@@ -162,7 +162,10 @@ private static Object toMapRecursive(Object o) {
// values.
Map<String, Object> result = new LinkedHashMap<>();
for (Entry<Field, Object> entry : ModelUtils.getFieldValues(o).entrySet()) {
result.put(entry.getKey().getName(), toMapRecursive(entry.getValue()));
Field field = entry.getKey();
if (!field.isAnnotationPresent(IgnoredInDiffableMap.class)) {
result.put(field.getName(), toMapRecursive(entry.getValue()));
}
}
return result;
} else if (o instanceof Map) {
@@ -191,6 +194,12 @@ private static Object toMapRecursive(Object o) {
}
}

/** Marker to indicate that this filed should be ignored by {@link #toDiffableFieldMap}. */
@Documented
@Retention(RUNTIME)
@Target(FIELD)
protected @interface IgnoredInDiffableMap {}

/** Returns a map of all object fields (including sensitive data) that's used to produce diffs. */
@SuppressWarnings("unchecked")
public Map<String, Object> toDiffableFieldMap() {
@@ -16,20 +16,24 @@

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Strings.nullToEmpty;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.googlecode.objectify.annotation.AlsoLoad;
import com.googlecode.objectify.annotation.Ignore;
import com.googlecode.objectify.annotation.OnLoad;
import google.registry.model.Buildable;
import google.registry.model.ImmutableObject;
import google.registry.model.JsonMapBuilder;
import google.registry.model.Jsonifiable;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Stream;
import javax.persistence.Embeddable;
import javax.persistence.MappedSuperclass;
import javax.persistence.PostLoad;
import javax.persistence.Transient;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlTransient;
@@ -53,15 +57,17 @@
public class Address extends ImmutableObject implements Jsonifiable {

/** The schema validation will enforce that this has 3 lines at most. */
// TODO(shicong): Remove this field after migration. We need to figure out how to generate same
// XML from streetLine[1,2,3].
@XmlJavaTypeAdapter(NormalizedStringAdapter.class)
@Transient
List<String> street;

@Ignore String streetLine1;
@Ignore @XmlTransient @IgnoredInDiffableMap String streetLine1;

@Ignore String streetLine2;
@Ignore @XmlTransient @IgnoredInDiffableMap String streetLine2;

@Ignore String streetLine3;
@Ignore @XmlTransient @IgnoredInDiffableMap String streetLine3;

@XmlJavaTypeAdapter(NormalizedStringAdapter.class)
String city;
@@ -86,18 +92,6 @@
}
}

public String getStreetLine1() {
return streetLine1;
}

public String getStreetLine2() {
return streetLine2;
}

public String getStreetLine13() {
return streetLine3;
}

public String getCity() {
return city;
}
@@ -144,6 +138,9 @@ protected Builder(T instance) {
street == null || (!street.isEmpty() && street.size() <= 3),
"Street address must have [1-3] lines: %s", street);
getInstance().street = street;
getInstance().streetLine1 = street.get(0);
getInstance().streetLine2 = street.size() >= 2 ? street.get(1) : null;
getInstance().streetLine3 = street.size() == 3 ? street.get(2) : null;
return this;
}

@@ -171,13 +168,37 @@ protected Builder(T instance) {
}
}

@OnLoad
void setStreetForCloudSql() {
/**
* Sets {@link #streetLine1}, {@link #streetLine2} and {@link #streetLine3} after loading the
* entity from Datastore.
*
* <p>This callback method is used by Objectify to set streetLine[1,2,3] fields as they are not
* persisted in the Datastore. TODO(shicong): Delete this method after database migration.
*/
void onLoad(@AlsoLoad("street") List<String> street) {
if (street == null || street.size() == 0) {
return;
}
streetLine1 = street.get(0);
streetLine2 = street.size() >= 2 ? street.get(1) : null;
streetLine3 = street.size() >= 3 ? street.get(2) : null;
}

/**
* Sets {@link #street} after loading the entity from Cloud SQL.
*
* <p>This callback method is used by Hibernate to set {@link #street} field as it is not
* persisted in Cloud SQL. We are doing this because the street list field is exposed by Address
* class and is used everywhere in our code base. Also, setting/reading a list of strings is more
* convenient.
*/
@PostLoad
void postLoad() {
street =
streetLine1 == null
? null
: Stream.of(streetLine1, streetLine2, streetLine3)
.filter(Objects::nonNull)
.collect(toImmutableList());
}
}
@@ -29,20 +29,26 @@

@Override
public Object toEntityTypeMap(Map<String, String> map) {
return map.entrySet().stream()
.collect(
toImmutableMap(
entry -> CurrencyUnit.of(entry.getKey()),
entry ->
new BillingAccountEntry(CurrencyUnit.of(entry.getKey()), entry.getValue())));
return map == null
? null
: map.entrySet().stream()
.collect(
toImmutableMap(
entry -> CurrencyUnit.of(entry.getKey()),
entry ->
new BillingAccountEntry(
CurrencyUnit.of(entry.getKey()), entry.getValue())));
}

@Override
public Map<String, String> toDbSupportedMap(Object map) {
return ((Map<CurrencyUnit, BillingAccountEntry>) map)
.entrySet().stream()
.collect(
toImmutableMap(
entry -> entry.getKey().getCode(), entry -> entry.getValue().getAccountId()));
return map == null
? null
: ((Map<CurrencyUnit, BillingAccountEntry>) map)
.entrySet().stream()
.collect(
toImmutableMap(
entry -> entry.getKey().getCode(),
entry -> entry.getValue().getAccountId()));
}
}
@@ -0,0 +1,73 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package google.registry.schema.registrar;

import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;

import google.registry.model.registrar.Registrar;
import java.util.Optional;

/** Data access object for {@link Registrar}. */
public class RegistrarDao {

private RegistrarDao() {}

/** Persists a new or updates an existing registrar in Cloud SQL. */
public static void saveNew(Registrar registrar) {
checkArgumentNotNull(registrar, "registrar must be specified");
jpaTm().transact(() -> jpaTm().getEntityManager().persist(registrar));
}

/** Updates an existing registrar in Cloud SQL, throws excpetion if it does not exist. */
public static void update(Registrar registrar) {
checkArgumentNotNull(registrar, "registrar must be specified");
jpaTm()
.transact(
() -> {
checkArgument(
checkExists(registrar.getClientId()),
"A registrar of this id does not exist: %s.",
registrar.getClientId());
jpaTm().getEntityManager().merge(registrar);
});
}

/** Returns whether the registrar of the given id exists. */
public static boolean checkExists(String clientId) {
checkArgumentNotNull(clientId, "clientId must be specified");
return jpaTm()
.transact(
() ->
jpaTm()
.getEntityManager()
.createQuery(
"SELECT 1 FROM Registrar WHERE clientIdentifier = :clientIdentifier",
Integer.class)
.setParameter("clientIdentifier", clientId)
.setMaxResults(1)
.getResultList()
.size()
> 0);
}

/** Loads the registrar by its id, returns empty if it doesn't exist. */
public static Optional<Registrar> load(String clientId) {
checkArgumentNotNull(clientId, "clientId must be specified");
return Optional.ofNullable(
jpaTm().transact(() -> jpaTm().getEntityManager().find(Registrar.class, clientId)));
}
}
@@ -18,6 +18,7 @@
import static com.google.common.base.Predicates.isNull;
import static com.google.common.base.Strings.isNullOrEmpty;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
import static google.registry.util.RegistrarUtils.normalizeRegistrarName;
import static java.nio.charset.StandardCharsets.US_ASCII;
@@ -28,6 +29,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.flogger.FluentLogger;
import google.registry.model.registrar.Registrar;
import google.registry.model.registrar.RegistrarAddress;
import google.registry.model.registry.Registry;
@@ -53,6 +55,8 @@
/** Shared base class for commands to create or update a {@link Registrar}. */
abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand {

static final FluentLogger logger = FluentLogger.forEnclosingClass();

@Parameter(
description = "Client identifier of the registrar account",
required = true)
@@ -458,4 +462,26 @@ protected final void init() throws Exception {
stageEntityChange(oldRegistrar, newRegistrar);
}
}

@Override
protected String execute() throws Exception {
// Save registrar to Datastore and output its response
logger.atInfo().log(super.execute());

String cloudSqlMessage;
try {
jpaTm()
.transact(
() ->
getChangedEntities().forEach(newEntity -> saveToCloudSql((Registrar) newEntity)));
cloudSqlMessage =
String.format("Updated %d entities in Cloud SQL.\n", getChangedEntities().size());
} catch (Throwable t) {
cloudSqlMessage = "Unexpected error saving registrar to Cloud SQL from nomulus tool command";
logger.atSevere().withCause(t).log(cloudSqlMessage);
}
return cloudSqlMessage;
}

abstract void saveToCloudSql(Registrar registrar);
}
@@ -32,6 +32,7 @@
import com.google.common.collect.Streams;
import google.registry.config.RegistryEnvironment;
import google.registry.model.registrar.Registrar;
import google.registry.schema.registrar.RegistrarDao;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@@ -69,6 +70,11 @@ protected void initRegistrarCommand() {
registrarState = Optional.ofNullable(registrarState).orElse(ACTIVE);
}

@Override
void saveToCloudSql(Registrar registrar) {
RegistrarDao.saveNew(registrar);
}

@Nullable
@Override
Registrar getOldRegistrar(final String clientId) {
@@ -20,6 +20,7 @@
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.emptyToNull;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.util.DatastoreServiceUtils.getNameOrId;
@@ -45,9 +46,9 @@
public abstract class MutatingCommand extends ConfirmingCommand implements CommandWithRemoteApi {

/**
* A mutation of a specific entity, represented by an old and a new version of the entity.
* Storing the old version is necessary to enable checking that the existing entity has not been
* modified when applying a mutation that was created outside the same transaction.
* A mutation of a specific entity, represented by an old and a new version of the entity. Storing
* the old version is necessary to enable checking that the existing entity has not been modified
* when applying a mutation that was created outside the same transaction.
*/
private static class EntityChange {

@@ -169,8 +170,8 @@ private void executeChange(EntityChange change) {
}

/**
* Returns a set of lists of EntityChange actions to commit. Each list should be executed in
* order inside a single transaction.
* Returns a set of lists of EntityChange actions to commit. Each list should be executed in order
* inside a single transaction.
*/
private ImmutableSet<ImmutableList<EntityChange>> getCollatedEntityChangeBatches() {
ImmutableSet.Builder<ImmutableList<EntityChange>> batches = new ImmutableSet.Builder<>();
@@ -221,4 +222,11 @@ protected String prompt() {
? "No entity changes to apply."
: changedEntitiesMap.values().stream().map(Object::toString).collect(joining("\n"));
}

/** Returns the collection of the new entity in the {@link EntityChange}. */
protected ImmutableList<ImmutableObject> getChangedEntities() {
return changedEntitiesMap.values().stream()
.map(entityChange -> entityChange.newEntity)
.collect(toImmutableList());
}
}
@@ -20,6 +20,7 @@
import com.beust.jcommander.Parameters;
import google.registry.config.RegistryEnvironment;
import google.registry.model.registrar.Registrar;
import google.registry.schema.registrar.RegistrarDao;
import javax.annotation.Nullable;

/** Command to update a Registrar. */
@@ -49,4 +50,9 @@ void checkModifyAllowedTlds(@Nullable Registrar oldRegistrar) {
+ " contact.");
}
}

@Override
void saveToCloudSql(Registrar registrar) {
RegistrarDao.update(registrar);
}
}

0 comments on commit b9c4064

Please sign in to comment.
You can’t perform that action at this time.