Skip to content
Permalink
Browse files

Merge pull request #199 from lofifnc/set_bq_load_job_to_write_empty

Set BigQuery load job to use write empty disposition
  • Loading branch information...
utzwestermann committed May 9, 2018
2 parents 568913a + 3f3aa05 commit fdeef25d56f15629de526344e7851ce199b91e65
Showing with 14 additions and 6 deletions.
  1. +14 −6 schedoscope-export/src/main/java/org/schedoscope/export/utils/BigQueryUtils.java
@@ -17,14 +17,12 @@

import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.bigquery.*;
import com.google.cloud.bigquery.JobInfo.WriteDisposition;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.*;
import java.util.concurrent.TimeoutException;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@@ -245,8 +243,18 @@ static public void dropTable(BigQuery bigQueryService, TableId tableId) {
* @param cloudStoragePathsToData the list of gs:// URLs to the blobs to load into the table.
*/
static public void loadTable(BigQuery bigQueryService, TableId table, List<String> cloudStoragePathsToData) {
Table t = bigQueryService.getTable(table);
Job loadJob = t.load(FormatOptions.json(), cloudStoragePathsToData);

LoadJobConfiguration configuration = LoadJobConfiguration
.newBuilder(table, cloudStoragePathsToData)
.setFormatOptions(FormatOptions.json())
.setWriteDisposition(WriteDisposition.WRITE_EMPTY)
.build();

//jobId could be used to reference the job later
JobId jobId = JobId.of(UUID.randomUUID().toString());
JobInfo jobInfo = JobInfo.of(jobId, configuration);

Job loadJob = bigQueryService.create(jobInfo);

try {
loadJob = loadJob.waitFor();

0 comments on commit fdeef25

Please sign in to comment.
You can’t perform that action at this time.