Permalink
Browse files

Send a plaintext link to the mapreduce console in fluent style

The link was previously being sent using a JS redirect, which doesn't work
because the endpoints that trigger mapreduces can only be hit from the command
line (because they require auth). This commit switches the link to be in
plaintext and renders the full URL instead of just the path, so that clicking it
directly from the terminal works.

This also improves how these links are sent from callsites by using a fluent
style.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=228764606
  • Loading branch information...
CydeWeys committed Jan 10, 2019
1 parent 072576e commit 765e63e7e952a90a9927c0bb9ce3ead1bb158912
Showing with 202 additions and 239 deletions.
  1. +12 −12 java/google/registry/backup/DeleteOldCommitLogsAction.java
  2. +12 −15 java/google/registry/batch/DeleteContactsAndHostsAction.java
  3. +8 −11 java/google/registry/batch/DeleteLoadTestDataAction.java
  4. +3 −3 java/google/registry/batch/DeleteProberDataAction.java
  5. +3 −3 java/google/registry/batch/ExpandRecurringBillingEventsAction.java
  6. +10 −13 java/google/registry/batch/RefreshDnsOnHostRenameAction.java
  7. +3 −3 java/google/registry/batch/ResaveAllEppResourcesAction.java
  8. +3 −3 java/google/registry/export/ExportDomainListsAction.java
  9. +56 −24 java/google/registry/mapreduce/MapreduceRunner.java
  10. +3 −4 java/google/registry/rde/RdeStagingAction.java
  11. +3 −5 java/google/registry/rde/imports/RdeContactImportAction.java
  12. +3 −5 java/google/registry/rde/imports/RdeDomainImportAction.java
  13. +3 −3 java/google/registry/rde/imports/RdeHostImportAction.java
  14. +3 −3 java/google/registry/rde/imports/RdeHostLinkAction.java
  15. +0 −8 java/google/registry/request/Response.java
  16. +0 −9 java/google/registry/request/ResponseImpl.java
  17. +1 −1 java/google/registry/tools/GenerateZoneFilesCommand.java
  18. +12 −13 java/google/registry/tools/server/GenerateZoneFilesAction.java
  19. +3 −5 java/google/registry/tools/server/KillAllCommitLogsAction.java
  20. +3 −3 java/google/registry/tools/server/KillAllEppResourcesAction.java
  21. +8 −10 java/google/registry/tools/server/RefreshDnsForAllDomainsAction.java
  22. +5 −5 java/google/registry/tools/server/ResaveAllHistoryEntriesAction.java
  23. +0 −27 java/google/registry/util/PipelineUtils.java
  24. +10 −1 javatests/google/registry/export/ExportDomainListsActionTest.java
  25. +1 −1 javatests/google/registry/rde/imports/RdeHostImportActionTest.java
  26. +1 −1 javatests/google/registry/rde/imports/RdeHostLinkActionTest.java
  27. +0 −9 javatests/google/registry/request/ResponseImplTest.java
  28. +4 −9 javatests/google/registry/testing/FakeResponse.java
  29. +1 −0 javatests/google/registry/testing/mapreduce/BUILD
  30. +17 −20 javatests/google/registry/testing/mapreduce/MapreduceTestCase.java
  31. +11 −10 javatests/google/registry/tools/server/GenerateZoneFilesActionTest.java
@@ -18,7 +18,6 @@
import static com.google.common.base.Preconditions.checkState;
import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.util.PipelineUtils.createJobPath;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;

@@ -88,17 +87,18 @@ public void run() {
"Processing asynchronous deletion of unreferenced CommitLogManifests older than %s",
deletionThreshold);

response.sendJavaScriptRedirect(createJobPath(mrRunner
.setJobName("Delete old commit logs")
.setModuleName("backend")
.setDefaultMapShards(NUM_MAP_SHARDS)
.setDefaultReduceShards(NUM_REDUCE_SHARDS)
.runMapreduce(
new DeleteOldCommitLogsMapper(deletionThreshold),
new DeleteOldCommitLogsReducer(deletionThreshold, isDryRun),
ImmutableList.of(
new CommitLogManifestInput(deletionThreshold),
EppResourceInputs.createKeyInput(EppResource.class)))));
mrRunner
.setJobName("Delete old commit logs")
.setModuleName("backend")
.setDefaultMapShards(NUM_MAP_SHARDS)
.setDefaultReduceShards(NUM_REDUCE_SHARDS)
.runMapreduce(
new DeleteOldCommitLogsMapper(deletionThreshold),
new DeleteOldCommitLogsReducer(deletionThreshold, isDryRun),
ImmutableList.of(
new CommitLogManifestInput(deletionThreshold),
EppResourceInputs.createKeyInput(EppResource.class)))
.sendLinkToMapreduceConsole(response);
}

/**
@@ -40,7 +40,6 @@
import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE;
import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE_FAILURE;
import static google.registry.model.transfer.TransferStatus.SERVER_CANCELLED;
import static google.registry.util.PipelineUtils.createJobPath;
import static java.math.RoundingMode.CEILING;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.SECONDS;
@@ -215,20 +214,18 @@ private void runMapreduce(ImmutableList<DeletionRequest> deletionRequests, Optio
try {
int numReducers =
Math.min(MAX_REDUCE_SHARDS, divide(deletionRequests.size(), DELETES_PER_SHARD, CEILING));
response.sendJavaScriptRedirect(
createJobPath(
mrRunner
.setJobName("Check for EPP resource references and then delete")
.setModuleName("backend")
.setDefaultReduceShards(numReducers)
.runMapreduce(
new DeleteContactsAndHostsMapper(deletionRequests),
new DeleteEppResourceReducer(),
ImmutableList.of(
// Add an extra shard that maps over a null domain. See the mapper code
// for why.
new NullInput<>(), EppResourceInputs.createEntityInput(DomainBase.class)),
new UnlockerOutput<Void>(lock.get()))));
mrRunner
.setJobName("Check for EPP resource references and then delete")
.setModuleName("backend")
.setDefaultReduceShards(numReducers)
.runMapreduce(
new DeleteContactsAndHostsMapper(deletionRequests),
new DeleteEppResourceReducer(),
ImmutableList.of(
// Add an extra shard that maps over a null domain. See the mapper code for why.
new NullInput<>(), EppResourceInputs.createEntityInput(DomainBase.class)),
new UnlockerOutput<Void>(lock.get()))
.sendLinkToMapreduceConsole(response);
} catch (Throwable t) {
logRespondAndUnlock(SEVERE, "Error starting mapreduce to delete contacts/hosts.", lock);
}
@@ -37,7 +37,6 @@
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.util.PipelineUtils;
import java.util.List;
import javax.inject.Inject;

@@ -83,16 +82,14 @@ public void run() {
checkState(
registryEnvironment != PRODUCTION, "This mapreduce is not safe to run on PRODUCTION.");

response.sendJavaScriptRedirect(
PipelineUtils.createJobPath(
mrRunner
.setJobName("Delete load test data")
.setModuleName("backend")
.runMapOnly(
new DeleteLoadTestDataMapper(isDryRun),
ImmutableList.of(
createEntityInput(ContactResource.class),
createEntityInput(HostResource.class)))));
mrRunner
.setJobName("Delete load test data")
.setModuleName("backend")
.runMapOnly(
new DeleteLoadTestDataMapper(isDryRun),
ImmutableList.of(
createEntityInput(ContactResource.class), createEntityInput(HostResource.class)))
.sendLinkToMapreduceConsole(response);
}

/** Provides the map method that runs for each existing contact and host entity. */
@@ -52,7 +52,6 @@
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.util.PipelineUtils;
import java.util.List;
import javax.inject.Inject;
import org.joda.time.DateTime;
@@ -87,12 +86,13 @@ public void run() {
checkState(
!Strings.isNullOrEmpty(registryAdminClientId),
"Registry admin client ID must be configured for prober data deletion to work");
response.sendJavaScriptRedirect(PipelineUtils.createJobPath(mrRunner
mrRunner
.setJobName("Delete prober data")
.setModuleName("backend")
.runMapOnly(
new DeleteProberDataMapper(getProberRoidSuffixes(), isDryRun, registryAdminClientId),
ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))));
ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))
.sendLinkToMapreduceConsole(response);
}

private ImmutableSet<String> getProberRoidSuffixes() {
@@ -28,7 +28,6 @@
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import static google.registry.util.DateTimeUtils.earliestOf;
import static google.registry.util.DomainNameUtils.getTldFromDomainName;
import static google.registry.util.PipelineUtils.createJobPath;

import com.google.appengine.tools.mapreduce.Mapper;
import com.google.appengine.tools.mapreduce.Reducer;
@@ -102,7 +101,7 @@ public void run() {
logger.atInfo().log(
"Running Recurring billing event expansion for billing time range [%s, %s).",
cursorTime, executeTime);
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Expand Recurring billing events into synthetic OneTime events.")
.setModuleName("backend")
.runMapreduce(
@@ -112,7 +111,8 @@ public void run() {
ImmutableList.of(
new NullInput<>(),
createChildEntityInput(
ImmutableSet.of(DomainResource.class), ImmutableSet.of(Recurring.class))))));
ImmutableSet.of(DomainResource.class), ImmutableSet.of(Recurring.class))))
.sendLinkToMapreduceConsole(response);
}

/** Mapper to expand {@link Recurring} billing events into synthetic {@link OneTime} events. */
@@ -27,7 +27,6 @@
import static google.registry.model.EppResourceUtils.isDeleted;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.util.DateTimeUtils.latestOf;
import static google.registry.util.PipelineUtils.createJobPath;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.logging.Level.INFO;
@@ -159,18 +158,16 @@ public void run() {

private void runMapreduce(ImmutableList<DnsRefreshRequest> refreshRequests, Optional<Lock> lock) {
try {
response.sendJavaScriptRedirect(
createJobPath(
mrRunner
.setJobName("Enqueue DNS refreshes for domains referencing renamed hosts")
.setModuleName("backend")
.setDefaultReduceShards(1)
.runMapreduce(
new RefreshDnsOnHostRenameMapper(refreshRequests, retrier),
new RefreshDnsOnHostRenameReducer(refreshRequests, lock.get(), retrier),
// Add an extra NullInput so that the reducer always fires exactly once.
ImmutableList.of(
new NullInput<>(), createEntityInput(DomainResource.class)))));
mrRunner
.setJobName("Enqueue DNS refreshes for domains referencing renamed hosts")
.setModuleName("backend")
.setDefaultReduceShards(1)
.runMapreduce(
new RefreshDnsOnHostRenameMapper(refreshRequests, retrier),
new RefreshDnsOnHostRenameReducer(refreshRequests, lock.get(), retrier),
// Add an extra NullInput so that the reducer always fires exactly once.
ImmutableList.of(new NullInput<>(), createEntityInput(DomainResource.class)))
.sendLinkToMapreduceConsole(response);
} catch (Throwable t) {
logRespondAndUnlock(
SEVERE, "Error starting mapreduce to refresh DNS for renamed hosts.", lock);
@@ -15,7 +15,6 @@
package google.registry.batch;

import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.util.PipelineUtils.createJobPath;

import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.collect.ImmutableList;
@@ -52,12 +51,13 @@

@Override
public void run() {
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Re-save all EPP resources")
.setModuleName("backend")
.runMapOnly(
new ResaveAllEppResourcesActionMapper(),
ImmutableList.of(EppResourceInputs.createKeyInput(EppResource.class)))));
ImmutableList.of(EppResourceInputs.createKeyInput(EppResource.class)))
.sendLinkToMapreduceConsole(response);
}

/** Mapper to re-save all EPP resources. */
@@ -20,7 +20,6 @@
import static google.registry.model.EppResourceUtils.isActive;
import static google.registry.model.registry.Registries.getTldsOfType;
import static google.registry.request.Action.Method.POST;
import static google.registry.util.PipelineUtils.createJobPath;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.joda.time.DateTimeZone.UTC;

@@ -78,14 +77,15 @@
public void run() {
ImmutableSet<String> realTlds = getTldsOfType(TldType.REAL);
logger.atInfo().log("Exporting domain lists for tlds %s", realTlds);
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Export domain lists")
.setModuleName("backend")
.setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS))
.runMapreduce(
new ExportDomainListsMapper(DateTime.now(UTC), realTlds),
new ExportDomainListsReducer(gcsBucket, gcsBufferSize),
ImmutableList.of(createEntityInput(DomainResource.class)))));
ImmutableList.of(createEntityInput(DomainResource.class)))
.sendLinkToMapreduceConsole(response);
}

static class ExportDomainListsMapper extends Mapper<DomainResource, String, String> {
Oops, something went wrong.

0 comments on commit 765e63e

Please sign in to comment.