diff --git a/cloud-storage/storage-transfer/README.md b/cloud-storage/storage-transfer/README.md new file mode 100644 index 00000000000..2cc0711bf72 --- /dev/null +++ b/cloud-storage/storage-transfer/README.md @@ -0,0 +1,52 @@ +# Transfer Service sample using Java + +This app creates two types of transfers using the Transfer Service tool. + +## Prerequisites + +1. Set up a project on Google Developers Console. + 1. Go to the [Developers Console](https://cloud.google.com/console) and create or select your project. + You will need the project ID later. +1. Within Developers Console, select APIs & auth > Credentials. + 1. Select Add credentials > Service account > JSON key. + 1. Set the environment variable GOOGLE_APPLICATION_CREDENTIALS to point to your JSON key. +1. Add the Storage Transfer service account, cloud-mobility@system.gserviceaccount.com as an + editor of your project. +1. Set up gcloud for application default credentials. + 1. `gcloud components update` + 1. `gcloud auth login` + 1. `gcloud config set project PROJECT_ID` + +## Transfer from Amazon S3 to Google Cloud Storage + +Creating a one-time transfer from Amazon S3 to Google Cloud Storage. +1. Set up data sink. + 1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser. +1. Set up data source. + 1. Go to AWS Management Console and create a bucket. + 1. Under Security Credentials, create an IAM User with access to the bucket. + 1. Create an Access Key for the user. Note the Access Key ID and Secret Access Key. +1. In AwsRequester.java, fill in the user-provided constants. +1. Run with `mvn compile` and + `mvn exec:java -Dexec.mainClass="com.google.cloud.storage.storagetransfer.samples.AwsRequester"` + 1. Note the job ID in the returned Transfer Job. + +## Transfer data from a standard Cloud Storage bucket to a Cloud Storage Nearline bucket + +Creating a daily transfer from a standard Cloud Storage bucket to a Cloud Storage Nearline +bucket for files untouched for 30 days. +1. Set up data sink. + 1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser. + 1. Select Nearline for Storage Class. +1. Set up data source. + 1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser. +1. In NearlineRequester.java, fill in the user-provided constants. +1. Run with `mvn compile` and + `mvn exec:java -Dexec.mainClass="com.google.cloud.storage.storagetransfer.samples.NearlineRequester"` + 1. Note the job ID in the returned Transfer Job. + +## Checking the status of a transfer + +1. In RequestChecker.java, fill in the user-provided constants. Use the Job Name you recorded earlier. +1. Run with `mvn compile` and + `mvn exec:java -Dexec.mainClass="com.google.cloud.storage.storagetransfer.samples.RequestChecker"` diff --git a/cloud-storage/storage-transfer/pom.xml b/cloud-storage/storage-transfer/pom.xml new file mode 100644 index 00000000000..9e3b119d7d0 --- /dev/null +++ b/cloud-storage/storage-transfer/pom.xml @@ -0,0 +1,75 @@ + + + + + 4.0.0 + + doc-samples + com.google.cloud + 1.0.0 + ../.. + + + com.google.storagetransfer.samples + transfersample + 0.1 + jar + + transfersample + http://maven.apache.org + + + UTF-8 + 1.6.2 + + + + + com.google.apis + google-api-services-storagetransfer + v1-rev1-1.20.0 + + + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + org.powermock + powermock-api-mockito + ${powermock.version} + test + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.7 + 1.7 + + + + + diff --git a/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/AwsRequester.java b/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/AwsRequester.java new file mode 100644 index 00000000000..f2f3536df19 --- /dev/null +++ b/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/AwsRequester.java @@ -0,0 +1,104 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.storagetransfer.samples; + +import com.google.api.services.storagetransfer.Storagetransfer; +import com.google.api.services.storagetransfer.model.AwsAccessKey; +import com.google.api.services.storagetransfer.model.AwsS3Data; +import com.google.api.services.storagetransfer.model.Date; +import com.google.api.services.storagetransfer.model.GcsData; +import com.google.api.services.storagetransfer.model.Schedule; +import com.google.api.services.storagetransfer.model.TimeOfDay; +import com.google.api.services.storagetransfer.model.TransferJob; +import com.google.api.services.storagetransfer.model.TransferSpec; + +import java.io.IOException; +import java.util.logging.Logger; + +/** + * Creates a one-off transfer job from Amazon S3 to Google Cloud Storage. + */ +public final class AwsRequester { + + private static final String JOB_DESC = "YOUR DESCRIPTION"; + private static final String PROJECT_ID = "YOUR_PROJECT_ID"; + private static final String AWS_SOURCE_NAME = "YOUR SOURCE BUCKET"; + private static final String AWS_ACCESS_KEY_ID = "YOUR_ACCESS_KEY_ID"; + private static final String AWS_SECRET_ACCESS_KEY = "YOUR_SECRET_ACCESS_KEY"; + private static final String GCS_SINK_NAME = "YOUR_SINK_BUCKET"; + + /** + * Specify times below using US Pacific Time Zone. + */ + private static final String START_DATE = "YYYY-MM-DD"; + private static final String START_TIME = "HH:MM:SS"; + + private static final Logger LOG = Logger.getLogger(AwsRequester.class.getName()); + + /** + * Creates and executes a request for a TransferJob from Amazon S3 to Cloud Storage. + * + * @return the response TransferJob if the request is successful + * @throws InstantiationException + * if instantiation fails when building the TransferJob + * @throws IllegalAccessException + * if an illegal access occurs when building the TransferJob + * @throws IOException + * if the client failed to complete the request + */ + public static TransferJob createAwsTransferJob() throws InstantiationException, + IllegalAccessException, IOException { + Date date = TransferJobUtils.createDate(START_DATE); + TimeOfDay time = TransferJobUtils.createTimeOfDay(START_TIME); + TransferJob transferJob = TransferJob.class + .newInstance() + .setDescription(JOB_DESC) + .setProjectId(PROJECT_ID) + .setTransferSpec( + TransferSpec.class + .newInstance() + .setAwsS3DataSource( + AwsS3Data.class + .newInstance() + .setBucketName(AWS_SOURCE_NAME) + .setAwsAccessKey( + AwsAccessKey.class.newInstance().setAccessKeyId(AWS_ACCESS_KEY_ID) + .setSecretAccessKey(AWS_SECRET_ACCESS_KEY))) + .setGcsDataSink(GcsData.class.newInstance().setBucketName(GCS_SINK_NAME))) + .setSchedule( + Schedule.class.newInstance().setScheduleStartDate(date).setScheduleEndDate(date) + .setStartTimeOfDay(time)).setStatus("ENABLED"); + + Storagetransfer client = TransferClientCreator.createStorageTransferClient(); + return client.transferJobs().create(transferJob).execute(); + } + + /** + * Output the contents of a successfully created TransferJob. + * + * @param args + * arguments from the command line + */ + public static void main(String[] args) { + try { + TransferJob responseT = createAwsTransferJob(); + LOG.info("Return transferJob: " + responseT.toPrettyString()); + } catch (Exception e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/NearlineRequester.java b/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/NearlineRequester.java new file mode 100644 index 00000000000..a81b40deb0f --- /dev/null +++ b/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/NearlineRequester.java @@ -0,0 +1,101 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.storagetransfer.samples; + +import com.google.api.services.storagetransfer.Storagetransfer; +import com.google.api.services.storagetransfer.model.Date; +import com.google.api.services.storagetransfer.model.GcsData; +import com.google.api.services.storagetransfer.model.ObjectConditions; +import com.google.api.services.storagetransfer.model.Schedule; +import com.google.api.services.storagetransfer.model.TimeOfDay; +import com.google.api.services.storagetransfer.model.TransferJob; +import com.google.api.services.storagetransfer.model.TransferOptions; +import com.google.api.services.storagetransfer.model.TransferSpec; + +import java.io.IOException; +import java.util.logging.Logger; + +/** + * Creates a daily transfer from a standard Cloud Storage bucket to a Cloud Storage Nearline + * bucket for files untouched for 30 days. + */ +public final class NearlineRequester { + + private static final String JOB_DESC = "YOUR DESCRIPTION"; + private static final String PROJECT_ID = "YOUR_PROJECT_ID"; + private static final String GCS_SOURCE_NAME = "YOUR_SOURCE_BUCKET"; + private static final String NEARLINE_SINK_NAME = "YOUR_SINK_BUCKET"; + + /** + * Specify times below using US Pacific Time Zone. + */ + private static final String START_DATE = "YYYY-MM-DD"; + private static final String START_TIME = "HH:MM:SS"; + + private static final Logger LOG = Logger.getLogger(AwsRequester.class.getName()); + + /** + * Creates and executes a request for a TransferJob to Cloud Storage Nearline. + * + * @return the response TransferJob if the request is successful + * @throws InstantiationException + * if instantiation fails when building the TransferJob + * @throws IllegalAccessException + * if an illegal access occurs when building the TransferJob + * @throws IOException + * if the client failed to complete the request + */ + public static TransferJob createNearlineTransferJob() throws InstantiationException, + IllegalAccessException, IOException { + Date date = TransferJobUtils.createDate(START_DATE); + TimeOfDay time = TransferJobUtils.createTimeOfDay(START_TIME); + TransferJob transferJob = TransferJob.class + .newInstance() + .setDescription(JOB_DESC) + .setProjectId(PROJECT_ID) + .setTransferSpec( + TransferSpec.class + .newInstance() + .setGcsDataSource(GcsData.class.newInstance().setBucketName(GCS_SOURCE_NAME)) + .setGcsDataSink(GcsData.class.newInstance().setBucketName(NEARLINE_SINK_NAME)) + .setObjectConditions( + ObjectConditions.class.newInstance().setMinTimeElapsedSinceLastModification("2592000s")) + .setTransferOptions( + TransferOptions.class.newInstance().setDeleteObjectsFromSourceAfterTransfer(true))) + .setSchedule(Schedule.class.newInstance().setScheduleStartDate(date) + .setStartTimeOfDay(time)) + .setStatus("ENABLED"); + + Storagetransfer client = TransferClientCreator.createStorageTransferClient(); + return client.transferJobs().create(transferJob).execute(); + } + + /** + * Output the contents of a successfully created TransferJob. + * + * @param args + * arguments from the command line + */ + public static void main(String[] args) { + try { + TransferJob responseT = createNearlineTransferJob(); + LOG.info("Return transferJob: " + responseT.toPrettyString()); + } catch (Exception e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/RequestChecker.java b/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/RequestChecker.java new file mode 100644 index 00000000000..f51397a7f3d --- /dev/null +++ b/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/RequestChecker.java @@ -0,0 +1,73 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.storagetransfer.samples; + +import com.google.api.services.storagetransfer.Storagetransfer; +import com.google.api.services.storagetransfer.model.ListOperationsResponse; + +import java.io.IOException; +import java.util.logging.Logger; + +/** + * Queries for TransferOperations associated with a specific TransferJob. A TransferJob is done when + * all of its associated TransferOperations have completed. + * + */ +public final class RequestChecker { + + private static final String PROJECT_ID = "YOUR_PROJECT_ID"; + private static final String JOB_NAME = "YOUR_JOB_NAME"; + + private static final Logger LOG = Logger.getLogger(RequestChecker.class.getName()); + + /** + * Creates and executes a query for all associated TransferOperations. + * + * @param client + * a Storagetransfer client, for interacting with the Storage Transfer API + * @param projectId + * the project to query within + * @param jobName + * the job Name of the relevant TransferJob + * @return an object containing information on associated TransferOperations + * @throws IOException + * if the client failed to complete the request + */ + public static ListOperationsResponse checkTransfer(Storagetransfer client, + String projectId, String jobName) throws IOException { + return client.transferOperations().list("transferOperations") + .setFilter("{\"project_id\": \"" + projectId + "\", \"job_names\": [\"" + jobName + "\"] }") + .execute(); + } + + /** + * Output the returned list of TransferOperations. + * + * @param args + * arguments from the command line + */ + public static void main(String[] args) { + try { + ListOperationsResponse resp = + checkTransfer(TransferClientCreator.createStorageTransferClient(), + PROJECT_ID, JOB_NAME); + LOG.info("Result of transferOperations/list: " + resp.toPrettyString()); + } catch (Exception e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/RetryHttpInitializerWrapper.java b/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/RetryHttpInitializerWrapper.java new file mode 100644 index 00000000000..3bc70b0141d --- /dev/null +++ b/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/RetryHttpInitializerWrapper.java @@ -0,0 +1,98 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.storagetransfer.samples; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.http.HttpBackOffIOExceptionHandler; +import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpUnsuccessfulResponseHandler; +import com.google.api.client.util.ExponentialBackOff; +import com.google.api.client.util.Sleeper; +import com.google.common.base.Preconditions; + +import java.io.IOException; +import java.util.logging.Logger; + +/** + * RetryHttpInitializerWrapper will automatically retry upon RPC failures, preserving the + * auto-refresh behavior of the Google Credentials. + */ +public class RetryHttpInitializerWrapper implements HttpRequestInitializer { + + private static final Logger LOG = Logger.getLogger(RetryHttpInitializerWrapper.class.getName()); + private final Credential wrappedCredential; + private final Sleeper sleeper; + private static final int MILLIS_PER_MINUTE = 60 * 1000; + + /** + * A constructor using the default Sleeper. + * + * @param wrappedCredential + * the credential used to authenticate with a Google Cloud Platform project + */ + public RetryHttpInitializerWrapper(Credential wrappedCredential) { + this(wrappedCredential, Sleeper.DEFAULT); + } + + /** + * A constructor used only for testing. + * + * @param wrappedCredential + * the credential used to authenticate with a Google Cloud Platform project + * @param sleeper + * a user-supplied Sleeper + */ + RetryHttpInitializerWrapper(Credential wrappedCredential, Sleeper sleeper) { + this.wrappedCredential = Preconditions.checkNotNull(wrappedCredential); + this.sleeper = sleeper; + } + + /** + * Initialize an HttpRequest. + * + * @param request + * an HttpRequest that should be initialized + */ + public void initialize(HttpRequest request) { + request.setReadTimeout(2 * MILLIS_PER_MINUTE); // 2 minutes read timeout + final HttpUnsuccessfulResponseHandler backoffHandler = + new HttpBackOffUnsuccessfulResponseHandler( + new ExponentialBackOff()).setSleeper(sleeper); + request.setInterceptor(wrappedCredential); + request.setUnsuccessfulResponseHandler(new HttpUnsuccessfulResponseHandler() { + public boolean handleResponse(final HttpRequest request, final HttpResponse response, + final boolean supportsRetry) throws IOException { + if (wrappedCredential.handleResponse(request, response, supportsRetry)) { + // If credential decides it can handle it, the return code or message indicated + // something specific to authentication, and no backoff is desired. + return true; + } else if (backoffHandler.handleResponse(request, response, supportsRetry)) { + // Otherwise, we defer to the judgement of our internal backoff handler. + LOG.info("Retrying " + request.getUrl().toString()); + return true; + } else { + return false; + } + } + }); + request.setIOExceptionHandler(new HttpBackOffIOExceptionHandler(new ExponentialBackOff()) + .setSleeper(sleeper)); + } +} \ No newline at end of file diff --git a/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/TransferClientCreator.java b/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/TransferClientCreator.java new file mode 100644 index 00000000000..74aaf18b0ea --- /dev/null +++ b/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/TransferClientCreator.java @@ -0,0 +1,79 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.storagetransfer.samples; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.util.Utils; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.services.storagetransfer.Storagetransfer; +import com.google.api.services.storagetransfer.StoragetransferScopes; +import com.google.common.base.Preconditions; + +import java.io.IOException; + +/** + * Create a client to make calls to Storage Transfer API. + */ +public final class TransferClientCreator { + + /** + * Create a Storage Transfer client using application default credentials and other default + * settings. + * + * @return a Storage Transfer client + * @throws IOException + * there was an error obtaining application default credentials + */ + public static Storagetransfer createStorageTransferClient() throws IOException { + HttpTransport httpTransport = Utils.getDefaultTransport(); + JsonFactory jsonFactory = Utils.getDefaultJsonFactory(); + GoogleCredential credential = GoogleCredential + .getApplicationDefault(httpTransport, jsonFactory); + return createStorageTransferClient(httpTransport, jsonFactory, credential); + } + + /** + * Create a Storage Transfer client using user-supplied credentials and other settings. + * + * @param httpTransport + * a user-supplied HttpTransport + * @param jsonFactory + * a user-supplied JsonFactory + * @param credential + * a user-supplied Google credential + * @return a Storage Transfer client + */ + public static Storagetransfer createStorageTransferClient(HttpTransport httpTransport, + JsonFactory jsonFactory, GoogleCredential credential) { + Preconditions.checkNotNull(httpTransport); + Preconditions.checkNotNull(jsonFactory); + Preconditions.checkNotNull(credential); + + // In some cases, you need to add the scope explicitly. + if (credential.createScopedRequired()) { + credential = credential.createScoped(StoragetransferScopes.all()); + } + // Please use custom HttpRequestInitializer for automatic + // retry upon failures. We provide a simple reference + // implementation in the "Retry Handling" section. + HttpRequestInitializer initializer = new RetryHttpInitializerWrapper(credential); + return new Storagetransfer.Builder(httpTransport, jsonFactory, initializer) + .setApplicationName("storagetransfer-sample").build(); + } +} \ No newline at end of file diff --git a/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/TransferJobUtils.java b/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/TransferJobUtils.java new file mode 100644 index 00000000000..a2af3951fc5 --- /dev/null +++ b/cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/TransferJobUtils.java @@ -0,0 +1,77 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.storagetransfer.samples; + +import com.google.api.services.storagetransfer.model.Date; +import com.google.api.services.storagetransfer.model.TimeOfDay; + +/** + * Utility methods for creating TransferJobs. + * + */ +public final class TransferJobUtils { + + /** + * A private constructor. + */ + private TransferJobUtils() { + + } + + /** + * Parses a Date from a string of the form "YYYY-MM-DD". + * + * @param dateString + * a string of the form "YYYY-MM-DD" + * @return a Google Date representing the desired date + * @throws NumberFormatException + * if the input string has an incorrect format + * @throws InstantiationException + * if Date object instantiation failed + * @throws IllegalAccessException + * if Date object cannot be accessed + */ + public static Date createDate(String dateString) throws NumberFormatException, + InstantiationException, IllegalAccessException { + Date date = Date.class.newInstance().setYear(Integer.decode(dateString.split("-")[0])) + .setMonth(Integer.decode(dateString.split("-")[1])) + .setDay(Integer.decode(dateString.split("-")[2])); + return date; + } + + /** + * Parses a TimeOfDay from a string of the form "HH:MM:SS". + * + * @param timeString + * a string of the form "HH:MM:SS" + * @return a TimeOfDay representing the desired time + * @throws NumberFormatException + * if the input string has an incorrect format + * @throws InstantiationException + * if Date object instantiation failed + * @throws IllegalAccessException + * if Date object cannot be accessed + */ + public static TimeOfDay createTimeOfDay(String timeString) throws NumberFormatException, + InstantiationException, IllegalAccessException { + TimeOfDay time = TimeOfDay.class.newInstance() + .setHours(Integer.decode(timeString.split(":")[0])) + .setMinutes(Integer.decode(timeString.split(":")[1])) + .setSeconds(Integer.decode(timeString.split(":")[2])); + return time; + } +} diff --git a/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/AwsRequesterTest.java b/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/AwsRequesterTest.java new file mode 100644 index 00000000000..203fde4514b --- /dev/null +++ b/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/AwsRequesterTest.java @@ -0,0 +1,78 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.storagetransfer.samples.test; + +import static org.mockito.Mockito.when; + +import com.google.api.services.storagetransfer.Storagetransfer; +import com.google.api.services.storagetransfer.Storagetransfer.TransferJobs; +import com.google.api.services.storagetransfer.Storagetransfer.TransferJobs.Create; +import com.google.api.services.storagetransfer.model.Date; +import com.google.api.services.storagetransfer.model.Schedule; +import com.google.api.services.storagetransfer.model.TimeOfDay; +import com.google.api.services.storagetransfer.model.TransferJob; +import com.google.api.services.storagetransfer.model.TransferSpec; +import com.google.cloud.storage.storagetransfer.samples.AwsRequester; +import com.google.cloud.storage.storagetransfer.samples.TransferClientCreator; +import com.google.cloud.storage.storagetransfer.samples.TransferJobUtils; + +import junit.framework.TestCase; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Matchers; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TransferJobUtils.class, TransferClientCreator.class }) +public class AwsRequesterTest extends TestCase { + + /** + * Tests whether AwsRequester executes a request to create a TransferJob. + */ + @Test + public void testTest() throws Exception { + Date date = TransferJobUtils.createDate("2000-1-1"); + TimeOfDay time = TransferJobUtils.createTimeOfDay("1:1:1"); + TransferJob dummyJob = TransferJob.class + .newInstance() + .setDescription("DUMMY DESCRIPTION") + .setProjectId("DUMMY_PROJECT_ID") + .setTransferSpec(TransferSpec.class.newInstance()) + .setSchedule( + Schedule.class.newInstance().setScheduleStartDate(date).setScheduleEndDate(date) + .setStartTimeOfDay(time)).setStatus("ENABLED"); + + PowerMockito.mockStatic(TransferClientCreator.class); + PowerMockito.mockStatic(TransferJobUtils.class); + Storagetransfer client = Mockito.mock(Storagetransfer.class); + TransferJobs jobs = Mockito.mock(TransferJobs.class); + Create create = Mockito.mock(Create.class); + + PowerMockito.when(TransferClientCreator.createStorageTransferClient()).thenReturn(client); + when(client.transferJobs()).thenReturn(jobs); + when(jobs.create(Matchers.any(TransferJob.class))).thenReturn(create); + when(create.execute()).thenReturn(dummyJob); + + TransferJob returnedJob = AwsRequester.createAwsTransferJob(); + + assertEquals(returnedJob, dummyJob); + } +} \ No newline at end of file diff --git a/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/NearlineRequesterTest.java b/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/NearlineRequesterTest.java new file mode 100644 index 00000000000..0420cde19a4 --- /dev/null +++ b/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/NearlineRequesterTest.java @@ -0,0 +1,77 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.storagetransfer.samples.test; + +import static org.mockito.Mockito.when; +import junit.framework.TestCase; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Matchers; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.google.api.services.storagetransfer.Storagetransfer; +import com.google.api.services.storagetransfer.Storagetransfer.TransferJobs; +import com.google.api.services.storagetransfer.Storagetransfer.TransferJobs.Create; +import com.google.api.services.storagetransfer.model.Date; +import com.google.api.services.storagetransfer.model.Schedule; +import com.google.api.services.storagetransfer.model.TimeOfDay; +import com.google.api.services.storagetransfer.model.TransferJob; +import com.google.api.services.storagetransfer.model.TransferSpec; +import com.google.cloud.storage.storagetransfer.samples.NearlineRequester; +import com.google.cloud.storage.storagetransfer.samples.TransferClientCreator; +import com.google.cloud.storage.storagetransfer.samples.TransferJobUtils; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TransferJobUtils.class, TransferClientCreator.class }) +public class NearlineRequesterTest extends TestCase { + + /** + * Tests whether NearlineRequester executes a request to create a TransferJob. + */ + @Test + public void testTest() throws Exception { + Date date = TransferJobUtils.createDate("2000-1-1"); + TimeOfDay time = TransferJobUtils.createTimeOfDay("1:1:1"); + TransferJob dummyJob = TransferJob.class + .newInstance() + .setDescription("DUMMY DESCRIPTION") + .setProjectId("DUMMY_PROJECT_ID") + .setTransferSpec(TransferSpec.class.newInstance()) + .setSchedule( + Schedule.class.newInstance().setScheduleStartDate(date).setScheduleEndDate(date) + .setStartTimeOfDay(time)).setStatus("ENABLED"); + + PowerMockito.mockStatic(TransferClientCreator.class); + PowerMockito.mockStatic(TransferJobUtils.class); + Storagetransfer client = Mockito.mock(Storagetransfer.class); + TransferJobs jobs = Mockito.mock(TransferJobs.class); + Create create = Mockito.mock(Create.class); + + PowerMockito.when(TransferClientCreator.createStorageTransferClient()).thenReturn(client); + when(client.transferJobs()).thenReturn(jobs); + when(jobs.create(Matchers.any(TransferJob.class))).thenReturn(create); + when(create.execute()).thenReturn(dummyJob); + + TransferJob returnedJob = NearlineRequester.createNearlineTransferJob(); + + assertEquals(returnedJob, dummyJob); + } +} \ No newline at end of file diff --git a/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/RequestCheckerTest.java b/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/RequestCheckerTest.java new file mode 100644 index 00000000000..61382119765 --- /dev/null +++ b/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/RequestCheckerTest.java @@ -0,0 +1,50 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.storagetransfer.samples.test; + +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import junit.framework.TestCase; + +import org.junit.Test; +import org.mockito.Matchers; +import org.mockito.Mockito; + +import com.google.api.services.storagetransfer.Storagetransfer; +import com.google.api.services.storagetransfer.Storagetransfer.TransferOperations; +import com.google.api.services.storagetransfer.Storagetransfer.TransferOperations.List; +import com.google.cloud.storage.storagetransfer.samples.RequestChecker; + +public class RequestCheckerTest extends TestCase { + private Storagetransfer mockClient = Mockito.mock(Storagetransfer.class); + private List mockList = Mockito.mock(List.class); + private TransferOperations mockOps = Mockito.mock(TransferOperations.class); + + /** + * Tests whether checkTransfer() makes the API call to list TransferOperations. + */ + @Test + public void testCheckTransfer() throws Exception { + when(mockClient.transferOperations()).thenReturn(mockOps); + when(mockOps.list(Matchers.anyString())).thenReturn(mockList); + when(mockList.setFilter(Matchers.anyString())).thenReturn(mockList); + + RequestChecker.checkTransfer(mockClient, "DUMMY_PROJECT_ID", "DUMMY_JOB_NAME"); + + verify(mockList).execute(); + } +} diff --git a/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/TransferClientCreatorTest.java b/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/TransferClientCreatorTest.java new file mode 100644 index 00000000000..042085b06b3 --- /dev/null +++ b/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/TransferClientCreatorTest.java @@ -0,0 +1,91 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.storagetransfer.samples.test; + +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import junit.framework.TestCase; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Matchers; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.util.Utils; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.services.storagetransfer.Storagetransfer.Builder; +import com.google.cloud.storage.storagetransfer.samples.RetryHttpInitializerWrapper; +import com.google.cloud.storage.storagetransfer.samples.TransferClientCreator; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TransferClientCreator.class, Builder.class }) +public class TransferClientCreatorTest extends TestCase { + + private Builder mockBuilder = PowerMockito.mock(Builder.class); + private GoogleCredential mockCredential = Mockito.mock(GoogleCredential.class); + private RetryHttpInitializerWrapper mockInitializer = Mockito + .mock(RetryHttpInitializerWrapper.class); + private HttpTransport httpTransport = Utils.getDefaultTransport(); + private JsonFactory jsonFactory = Utils.getDefaultJsonFactory(); + + @Before + public void setUp() throws Exception { + PowerMockito.whenNew(RetryHttpInitializerWrapper.class).withArguments(mockCredential) + .thenReturn(mockInitializer); + PowerMockito.mockStatic(Builder.class); + PowerMockito.whenNew(Builder.class).withArguments(httpTransport, jsonFactory, mockInitializer) + .thenReturn(mockBuilder); + when(mockBuilder.setApplicationName(Matchers.anyString())).thenReturn(mockBuilder); + PowerMockito.mockStatic(GoogleCredential.class); + } + + /** + * Tests whether createStorageTransferClient() makes the API call for building clients when the + * credential does not require a Scope. + */ + @Test + public void testCreateStorageTransferClientScopedRequiredFalse() throws Exception { + when(mockCredential.createScopedRequired()).thenReturn(false); + + TransferClientCreator.createStorageTransferClient(Utils.getDefaultTransport(), + Utils.getDefaultJsonFactory(), mockCredential); + + verify(mockBuilder).build(); + } + + /** + * Tests whether createStorageTransferClient() makes the API call for building clients when the + * credential requires a Scope. + */ + @Test + public void testCreateStorageTransferClientScopedRequiredTrue() throws Exception { + when(mockCredential.createScopedRequired()).thenReturn(true); + when(mockCredential.createScoped(Matchers.anyCollectionOf(String.class))).thenReturn( + mockCredential); + + TransferClientCreator.createStorageTransferClient(Utils.getDefaultTransport(), + Utils.getDefaultJsonFactory(), mockCredential); + + verify(mockBuilder).build(); + } +} diff --git a/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/TransferJobUtilsTest.java b/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/TransferJobUtilsTest.java new file mode 100644 index 00000000000..dfb03495aa5 --- /dev/null +++ b/cloud-storage/storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/TransferJobUtilsTest.java @@ -0,0 +1,65 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.storagetransfer.samples.test; + +import java.util.Random; + +import junit.framework.TestCase; + +import org.junit.Test; + +import com.google.api.services.storagetransfer.model.Date; +import com.google.api.services.storagetransfer.model.TimeOfDay; +import com.google.cloud.storage.storagetransfer.samples.TransferJobUtils; + +public class TransferJobUtilsTest extends TestCase { + private Random r = new Random(); + + /** + * Tests whether createDate() builds the correct date from a formatted String. + */ + @Test + public void testCreateDate() throws Exception { + int year = r.nextInt(2000) + 1; + int month = r.nextInt(12) + 1; + int day = r.nextInt(30) + 1; + String dateString = Integer.toString(year) + "-" + Integer.toString(month) + "-" + + Integer.toString(day); + + Date date = TransferJobUtils.createDate(dateString); + + assertEquals(date, Date.class.newInstance().setYear(year).setMonth(month).setDay(day)); + } + + /** + * Tests whether createTimeOfDay() builds the correct time from a formatted String. + */ + @Test + public void testCreateTimeOfDay() throws Exception { + int hour = r.nextInt(24); + int minute = r.nextInt(60); + int second = r.nextInt(60); + String timeString = Integer.toString(hour) + ":" + Integer.toString(minute) + ":" + + Integer.toString(second); + + TimeOfDay time = TransferJobUtils.createTimeOfDay(timeString); + + assertEquals(time, + TimeOfDay.class.newInstance().setHours(hour).setMinutes(minute).setSeconds(second)); + + } +} \ No newline at end of file diff --git a/google-checks.xml b/google-checks.xml new file mode 100644 index 00000000000..9e0166e9420 --- /dev/null +++ b/google-checks.xml @@ -0,0 +1,201 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 88d69f76990..3d4efbe1b4a 100644 --- a/pom.xml +++ b/pom.xml @@ -26,29 +26,37 @@ bigquery cloud-storage/xml-api/cmdline-sample cloud-storage/xml-api/serviceaccount-appengine-sample + cloud-storage/storage-transfer monitoring - org.apache.maven.plugins - maven-checkstyle-plugin - 2.15 - - checkstyle-checker.xml - true - true - - - - - check - - - + org.apache.maven.plugins + maven-checkstyle-plugin + 2.15 + + + com.puppycrawl.tools + checkstyle + 6.8.1 + + + + google-checks.xml + true + true + + + + + check + + + - + org.eluder.coveralls coveralls-maven-plugin 3.1.0 @@ -102,13 +110,13 @@ junit junit - 4.12-beta-1 + 4.12 test org.mockito - mockito-all - 1.9.5 + mockito-core + 2.0.28-beta test diff --git a/taskqueue/deferred/pom.xml b/taskqueue/deferred/pom.xml index 42bf08f3551..30733593c99 100644 --- a/taskqueue/deferred/pom.xml +++ b/taskqueue/deferred/pom.xml @@ -43,7 +43,7 @@ org.mockito - mockito-all + mockito-core com.google.appengine