Skip to content
Permalink
Browse files

Add initial support for persisting premium lists to Cloud SQL (#285)

* Add initial support for persisting premium lists to Cloud SQL

This adds support to the `nomulus create_premium_list` command only; support for
`nomulus update_premium_list` will be in a subsequent PR.

The design goals for this PR were:
1. Do not change the existing codepaths for premium lists at all, especially not
   on the read path.
2. Write premium lists to Cloud SQL only if requested (i.e. not by default), and
   write to Datastore first so as to not be blocked by errors with Cloud SQL.
3. Reuse existing codepaths to the maximum possible extent (e.g. don't yet
   re-implement premium list parsing; take advantage of the existing logic), but
   also ...
4. Some duplication is OK, since the existing Datastore path will be deleted
   once this migration is complete, leaving only the codepaths for Cloud SQL.

* Refactor out common logic

* Add DAO test

* Add tests for parsing premium lists

* Use containsExactly

* Code review changes

* Format

* Re-generate schema

* Fix column names

* Make some tests pass

* Add SQL migration scripts

* Fix test errors
  • Loading branch information...
CydeWeys committed Oct 8, 2019
1 parent 658f61b commit bc7f3546c73631ff241dd4406b2317d1cc1b7a58
@@ -33,8 +33,7 @@

@Override
public Timestamp convertToDatabaseColumn(CreateAutoTimestamp entity) {
DateTime dateTime =
firstNonNull(((CreateAutoTimestamp) entity).getTimestamp(), jpaTm().getTransactionTime());
DateTime dateTime = firstNonNull(entity.getTimestamp(), jpaTm().getTransactionTime());
return Timestamp.from(DateTimeUtils.toZonedDateTime(dateTime).toInstant());
}

@@ -16,20 +16,24 @@

import static com.google.common.base.Preconditions.checkState;

import google.registry.model.CreateAutoTimestamp;
import google.registry.persistence.CreateAutoTimestampConverter;
import java.math.BigDecimal;
import java.time.ZonedDateTime;
import java.util.Map;
import javax.persistence.CollectionTable;
import javax.persistence.Column;
import javax.persistence.Convert;
import javax.persistence.ElementCollection;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Index;
import javax.persistence.JoinColumn;
import javax.persistence.MapKeyColumn;
import javax.persistence.Table;
import org.joda.money.CurrencyUnit;
import org.joda.time.DateTime;

/**
* A list of premium prices for domain names.
@@ -40,47 +44,51 @@
* This is fine though, because we only use the list with the highest revisionId.
*/
@Entity
@Table(name = "PremiumList")
@Table(indexes = {@Index(columnList = "name", name = "premiumlist_name_idx")})
public class PremiumList {

@Column(nullable = false)
private String name;

@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
@Column(name = "revision_id")
@Column(nullable = false)
private Long revisionId;

@Column(name = "creation_timestamp", nullable = false)
private ZonedDateTime creationTimestamp;
@Column(nullable = false)
@Convert(converter = CreateAutoTimestampConverter.class)
private CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null);

@Column(name = "currency", nullable = false)
@Column(nullable = false)
private CurrencyUnit currency;

@ElementCollection
@CollectionTable(
name = "PremiumEntry",
joinColumns = @JoinColumn(name = "revision_id", referencedColumnName = "revision_id"))
@MapKeyColumn(name = "domain_label")
joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
@MapKeyColumn(name = "domainLabel")
@Column(name = "price", nullable = false)
private Map<String, BigDecimal> labelsToPrices;

private PremiumList(
ZonedDateTime creationTimestamp,
CurrencyUnit currency,
Map<String, BigDecimal> labelsToPrices) {
this.creationTimestamp = creationTimestamp;
private PremiumList(String name, CurrencyUnit currency, Map<String, BigDecimal> labelsToPrices) {
// TODO(mcilwain): Generate the Bloom filter and set it here.
this.name = name;
this.currency = currency;
this.labelsToPrices = labelsToPrices;
}

// Hibernate requires this default constructor.
private PremiumList() {}

// TODO(mcilwain): Change creationTimestamp to Joda DateTime.
/** Constructs a {@link PremiumList} object. */
public static PremiumList create(
ZonedDateTime creationTimestamp,
CurrencyUnit currency,
Map<String, BigDecimal> labelsToPrices) {
return new PremiumList(creationTimestamp, currency, labelsToPrices);
String name, CurrencyUnit currency, Map<String, BigDecimal> labelsToPrices) {
return new PremiumList(name, currency, labelsToPrices);
}

/** Returns the name of the premium list, which is usually also a TLD string. */
public String getName() {
return name;
}

/** Returns the ID of this revision, or throws if null. */
@@ -91,8 +99,8 @@ public Long getRevisionId() {
}

/** Returns the creation time of this revision of the premium list. */
public ZonedDateTime getCreationTimestamp() {
return creationTimestamp;
public DateTime getCreationTimestamp() {
return creationTimestamp.getTimestamp();
}

/** Returns a {@link Map} of domain labels to prices. */
@@ -0,0 +1,56 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package google.registry.schema.tld;

import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.model.transaction.TransactionManagerFactory.jpaTm;

/** Data access object class for {@link PremiumList}. */
public class PremiumListDao {

/** Persist a new premium list to Cloud SQL. */
public static void saveNew(PremiumList premiumList) {
jpaTm()
.transact(
() -> {
checkArgument(
!checkExists(premiumList.getName()),
"A premium list of this name already exists: %s.",
premiumList.getName());
jpaTm().getEntityManager().persist(premiumList);
});
}

/**
* Returns whether the premium list of the given name exists.
*
* <p>This means that at least one premium list revision must exist for the given name.
*/
public static boolean checkExists(String premiumListName) {
return jpaTm()
.transact(
() ->
jpaTm()
.getEntityManager()
.createQuery("SELECT 1 FROM PremiumList WHERE name = :name", Integer.class)
.setParameter("name", premiumListName)
.setMaxResults(1)
.getResultList()
.size()
> 0);
}

private PremiumListDao() {}
}
@@ -16,6 +16,7 @@

import static com.google.common.base.Strings.isNullOrEmpty;
import static google.registry.security.JsonHttp.JSON_SAFETY_PREFIX;
import static google.registry.tools.server.CreateOrUpdatePremiumListAction.ALSO_CLOUD_SQL_PARAM;
import static google.registry.tools.server.CreateOrUpdatePremiumListAction.INPUT_PARAM;
import static google.registry.tools.server.CreateOrUpdatePremiumListAction.NAME_PARAM;
import static google.registry.util.ListNamingUtils.convertFilePathToName;
@@ -57,6 +58,12 @@
required = true)
Path inputFile;

@Parameter(
names = {"--also_cloud_sql"},
description =
"Persist premium list to Cloud SQL in addition to Datastore; defaults to false.")
boolean alsoCloudSql;

protected AppEngineConnection connection;
protected int inputLineCount;

@@ -67,7 +74,7 @@ public void setConnection(AppEngineConnection connection) {

abstract String getCommandPath();

ImmutableMap<String, ?> getParameterMap() {
ImmutableMap<String, String> getParameterMap() {
return ImmutableMap.of();
}

@@ -88,14 +95,15 @@ protected String prompt() {

@Override
public String execute() throws Exception {
ImmutableMap.Builder<String, Object> params = new ImmutableMap.Builder<>();
ImmutableMap.Builder<String, String> params = new ImmutableMap.Builder<>();
params.put(NAME_PARAM, name);
params.put(ALSO_CLOUD_SQL_PARAM, Boolean.toString(alsoCloudSql));
String inputFileContents = new String(Files.readAllBytes(inputFile), UTF_8);
String requestBody =
Joiner.on('&').withKeyValueSeparator("=").join(
ImmutableMap.of(INPUT_PARAM, URLEncoder.encode(inputFileContents, UTF_8.toString())));

ImmutableMap<String, ?> extraParams = getParameterMap();
ImmutableMap<String, String> extraParams = getParameterMap();
if (extraParams != null) {
params.putAll(extraParams);
}
@@ -110,7 +118,7 @@ public String execute() throws Exception {

// TODO(user): refactor this behavior into a better general-purpose
// response validation that can be re-used across the new client/server commands.
String extractServerResponse(String response) {
private String extractServerResponse(String response) {
Map<String, Object> responseMap = toMap(JSONValue.parse(stripJsonPrefix(response)));

// TODO(user): consider using jart's FormField Framework.
@@ -127,7 +135,7 @@ String extractServerResponse(String response) {
}

// TODO(user): figure out better place to put this method to make it re-usable
static String stripJsonPrefix(String json) {
private static String stripJsonPrefix(String json) {
Verify.verify(json.startsWith(JSON_SAFETY_PREFIX));
return json.substring(JSON_SAFETY_PREFIX.length());
}
@@ -36,12 +36,11 @@ public String getCommandPath() {
}

@Override
ImmutableMap<String, ?> getParameterMap() {
ImmutableMap<String, String> getParameterMap() {
if (override) {
return ImmutableMap.of("override", override);
return ImmutableMap.of("override", "true");
} else {
return ImmutableMap.of();
}
}

}
@@ -14,17 +14,28 @@

package google.registry.tools.server;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.google.common.flogger.LazyArgs.lazy;

import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.flogger.FluentLogger;
import google.registry.model.registry.label.PremiumList;
import google.registry.model.registry.label.PremiumList.PremiumListEntry;
import google.registry.request.JsonResponse;
import google.registry.request.Parameter;
import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import org.joda.money.CurrencyUnit;

/**
* Abstract base class for actions that update premium lists.
*/
/** Abstract base class for actions that update premium lists. */
public abstract class CreateOrUpdatePremiumListAction implements Runnable {

private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -33,26 +44,72 @@

public static final String NAME_PARAM = "name";
public static final String INPUT_PARAM = "inputData";
public static final String ALSO_CLOUD_SQL_PARAM = "alsoCloudSql";

@Inject JsonResponse response;
@Inject @Parameter("premiumListName") String name;
@Inject @Parameter(INPUT_PARAM) String inputData;

@Inject
@Parameter("premiumListName")
String name;

@Inject
@Parameter(INPUT_PARAM)
String inputData;

@Inject
@Parameter(ALSO_CLOUD_SQL_PARAM)
boolean alsoCloudSql;

@Override
public void run() {
try {
savePremiumList();
saveToDatastore();
} catch (IllegalArgumentException e) {
logger.atInfo().withCause(e).log(
"Usage error in attempting to save premium list from nomulus tool command");
response.setPayload(ImmutableMap.of("error", e.toString(), "status", "error"));
return;
} catch (Exception e) {
logger.atSevere().withCause(e).log(
"Unexpected error saving premium list from nomulus tool command");
"Unexpected error saving premium list to Datastore from nomulus tool command");
response.setPayload(ImmutableMap.of("error", e.toString(), "status", "error"));
return;
}

if (alsoCloudSql) {
try {
saveToCloudSql();
} catch (Throwable e) {
logger.atSevere().withCause(e).log(
"Unexpected error saving premium list to Cloud SQL from nomulus tool command");
response.setPayload(ImmutableMap.of("error", e.toString(), "status", "error"));
return;
}
}
}

google.registry.schema.tld.PremiumList parseInputToPremiumList() {
List<String> inputDataPreProcessed =
Splitter.on('\n').omitEmptyStrings().splitToList(inputData);

ImmutableMap<String, PremiumListEntry> prices =
new PremiumList.Builder().setName(name).build().parse(inputDataPreProcessed);
ImmutableSet<CurrencyUnit> currencies =
prices.values().stream()
.map(e -> e.getValue().getCurrencyUnit())
.distinct()
.collect(toImmutableSet());
checkArgument(
currencies.size() == 1,
"The Cloud SQL schema requires exactly one currency, but got: %s",
ImmutableSortedSet.copyOf(currencies));
CurrencyUnit currency = Iterables.getOnlyElement(currencies);

Map<String, BigDecimal> priceAmounts =
Maps.transformValues(prices, ple -> ple.getValue().getAmount());
return google.registry.schema.tld.PremiumList.create(name, currency, priceAmounts);
}

/** Logs the premium list data at INFO, truncated if too long. */
void logInputData() {
logger.atInfo().log(
@@ -64,6 +121,9 @@ void logInputData() {
: (inputData.substring(0, MAX_LOGGING_PREMIUM_LIST_LENGTH) + "<truncated>")));
}

/** Creates a new premium list or updates an existing one. */
protected abstract void savePremiumList();
/** Saves the premium list to Datastore. */
protected abstract void saveToDatastore();

/** Saves the premium list to Cloud SQL. */
protected abstract void saveToCloudSql();
}

0 comments on commit bc7f354

Please sign in to comment.
You can’t perform that action at this time.