This repository has been archived by the owner on Sep 27, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
docs(samples): add run history (#333)
- Loading branch information
Praful Makani
committed
Sep 9, 2020
1 parent
5a09efb
commit ddf567a
Showing
2 changed files
with
183 additions
and
0 deletions.
There are no files selected for viewing
49 changes: 49 additions & 0 deletions
49
samples/snippets/src/main/java/com/example/bigquerydatatransfer/RunHistory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
/* | ||
* Copyright 2020 Google LLC | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package com.example.bigquerydatatransfer; | ||
|
||
// [START bigquerydatatransfer_get_run_history] | ||
import com.google.api.gax.rpc.ApiException; | ||
import com.google.cloud.bigquery.datatransfer.v1.DataTransferServiceClient; | ||
import com.google.cloud.bigquery.datatransfer.v1.ListTransferRunsRequest; | ||
import java.io.IOException; | ||
|
||
// Sample to get run history from transfer config. | ||
public class RunHistory { | ||
|
||
public static void main(String[] args) throws IOException { | ||
// TODO(developer): Replace these variables before running the sample. | ||
String configId = "MY_CONFIG_ID"; | ||
// i.e projects/{project_id}/transferConfigs/{config_id}` or | ||
// `projects/{project_id}/locations/{location_id}/transferConfigs/{config_id}` | ||
runHistory(configId); | ||
} | ||
|
||
public static void runHistory(String configId) throws IOException { | ||
try (DataTransferServiceClient dataTransferServiceClient = DataTransferServiceClient.create()) { | ||
ListTransferRunsRequest request = | ||
ListTransferRunsRequest.newBuilder().setParent(configId).build(); | ||
dataTransferServiceClient | ||
.listTransferRuns(request) | ||
.iterateAll() | ||
.forEach(run -> System.out.print("Success! Run ID :" + run.getName() + "\n")); | ||
} catch (ApiException ex) { | ||
System.out.println("Run history not found due to error." + ex.toString()); | ||
} | ||
} | ||
} | ||
// [END bigquerydatatransfer_get_run_history] |
134 changes: 134 additions & 0 deletions
134
samples/snippets/src/test/java/com/example/bigquerydatatransfer/RunHistoryIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,134 @@ | ||
/* | ||
* Copyright 2020 Google LLC | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package com.example.bigquerydatatransfer; | ||
|
||
import static com.google.common.truth.Truth.assertThat; | ||
import static junit.framework.TestCase.assertNotNull; | ||
|
||
import com.google.cloud.bigquery.BigQuery; | ||
import com.google.cloud.bigquery.BigQueryOptions; | ||
import com.google.cloud.bigquery.DatasetInfo; | ||
import com.google.cloud.bigquery.datatransfer.v1.CreateTransferConfigRequest; | ||
import com.google.cloud.bigquery.datatransfer.v1.DataTransferServiceClient; | ||
import com.google.cloud.bigquery.datatransfer.v1.ProjectName; | ||
import com.google.cloud.bigquery.datatransfer.v1.TransferConfig; | ||
import com.google.protobuf.Struct; | ||
import com.google.protobuf.Value; | ||
import java.io.ByteArrayOutputStream; | ||
import java.io.IOException; | ||
import java.io.PrintStream; | ||
import java.util.HashMap; | ||
import java.util.Map; | ||
import java.util.UUID; | ||
import java.util.logging.Level; | ||
import java.util.logging.Logger; | ||
import org.junit.After; | ||
import org.junit.Before; | ||
import org.junit.BeforeClass; | ||
import org.junit.Test; | ||
|
||
public class RunHistoryIT { | ||
|
||
private static final Logger LOG = Logger.getLogger(GetTransferConfigInfoIT.class.getName()); | ||
private static final String ID = UUID.randomUUID().toString().substring(0, 8); | ||
private BigQuery bigquery; | ||
private ByteArrayOutputStream bout; | ||
private String name; | ||
private String displayName; | ||
private String datasetName; | ||
private PrintStream out; | ||
private PrintStream originalPrintStream; | ||
|
||
private static final String PROJECT_ID = requireEnvVar("GOOGLE_CLOUD_PROJECT"); | ||
|
||
private static String requireEnvVar(String varName) { | ||
String value = System.getenv(varName); | ||
assertNotNull( | ||
"Environment variable " + varName + " is required to perform these tests.", | ||
System.getenv(varName)); | ||
return value; | ||
} | ||
|
||
@BeforeClass | ||
public static void checkRequirements() { | ||
requireEnvVar("GOOGLE_CLOUD_PROJECT"); | ||
} | ||
|
||
@Before | ||
public void setUp() throws IOException { | ||
bout = new ByteArrayOutputStream(); | ||
out = new PrintStream(bout); | ||
originalPrintStream = System.out; | ||
System.setOut(out); | ||
|
||
displayName = "MY_SCHEDULE_NAME_TEST_" + ID; | ||
datasetName = "MY_DATASET_NAME_TEST_" + ID; | ||
// create a temporary dataset | ||
bigquery = BigQueryOptions.getDefaultInstance().getService(); | ||
bigquery.create(DatasetInfo.of(datasetName)); | ||
// create a scheduled query | ||
String query = | ||
"SELECT CURRENT_TIMESTAMP() as current_time, @run_time as intended_run_time, " | ||
+ "@run_date as intended_run_date, 17 as some_integer"; | ||
String destinationTableName = | ||
"MY_DESTINATION_TABLE_" + UUID.randomUUID().toString().substring(0, 8) + "_{run_date}"; | ||
Map<String, Value> params = new HashMap<>(); | ||
params.put("query", Value.newBuilder().setStringValue(query).build()); | ||
params.put( | ||
"destination_table_name_template", | ||
Value.newBuilder().setStringValue(destinationTableName).build()); | ||
params.put("write_disposition", Value.newBuilder().setStringValue("WRITE_TRUNCATE").build()); | ||
params.put("partitioning_field", Value.newBuilder().setStringValue("").build()); | ||
TransferConfig transferConfig = | ||
TransferConfig.newBuilder() | ||
.setDestinationDatasetId(datasetName) | ||
.setDisplayName(displayName) | ||
.setDataSourceId("scheduled_query") | ||
.setParams(Struct.newBuilder().putAllFields(params).build()) | ||
.setSchedule("every 24 hours") | ||
.build(); | ||
try (DataTransferServiceClient dataTransferServiceClient = DataTransferServiceClient.create()) { | ||
ProjectName parent = ProjectName.of(PROJECT_ID); | ||
CreateTransferConfigRequest request = | ||
CreateTransferConfigRequest.newBuilder() | ||
.setParent(parent.toString()) | ||
.setTransferConfig(transferConfig) | ||
.build(); | ||
name = dataTransferServiceClient.createTransferConfig(request).getName(); | ||
System.out.println("\nScheduled query created successfully :" + name); | ||
} | ||
} | ||
|
||
@After | ||
public void tearDown() throws IOException { | ||
// delete scheduled query that was just created | ||
DeleteScheduledQuery.deleteScheduledQuery(name); | ||
// delete a temporary dataset | ||
bigquery.delete(datasetName, BigQuery.DatasetDeleteOption.deleteContents()); | ||
|
||
// restores print statements in the original method | ||
System.out.flush(); | ||
System.setOut(originalPrintStream); | ||
LOG.log(Level.INFO, bout.toString()); | ||
} | ||
|
||
@Test | ||
public void testRunHistory() throws IOException { | ||
RunHistory.runHistory(name); | ||
assertThat(bout.toString()).contains("Success! Run ID :"); | ||
} | ||
} |