copy.go

// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigquery

import (
	"context"
	"time"

	bq "google.golang.org/api/bigquery/v2"
)

// TableCopyOperationType is used to indicate the type of operation performed by a BigQuery
// copy job.
type TableCopyOperationType string

var (
	// CopyOperation indicates normal table to table copying.
	CopyOperation TableCopyOperationType = "COPY"
	// SnapshotOperation indicates creating a snapshot from a regular table, which
	// operates as an immutable copy.
	SnapshotOperation TableCopyOperationType = "SNAPSHOT"
	// RestoreOperation indicates creating/restoring a table from a snapshot.
	RestoreOperation TableCopyOperationType = "RESTORE"
	// CloneOperation indicates creating a table clone, which creates a writeable
	// copy of a base table that is billed based on difference from the base table.
	CloneOperation TableCopyOperationType = "CLONE"
)

// CopyConfig holds the configuration for a copy job.
type CopyConfig struct {
	// Srcs are the tables from which data will be copied.
	Srcs []*Table

	// Dst is the table into which the data will be copied.
	Dst *Table

	// CreateDisposition specifies the circumstances under which the destination table will be created.
	// The default is CreateIfNeeded.
	CreateDisposition TableCreateDisposition

	// WriteDisposition specifies how existing data in the destination table is treated.
	// The default is WriteEmpty.
	WriteDisposition TableWriteDisposition

	// The labels associated with this job.
	Labels map[string]string

	// Custom encryption configuration (e.g., Cloud KMS keys).
	DestinationEncryptionConfig *EncryptionConfig

	// One of the supported operation types when executing a Table Copy jobs.  By default this
	// copies tables, but can also be set to perform snapshot or restore operations.
	OperationType TableCopyOperationType

	// Sets a best-effort deadline on a specific job.  If job execution exceeds this
	// timeout, BigQuery may attempt to cancel this work automatically.
	//
	// This deadline cannot be adjusted or removed once the job is created.  Consider
	// using Job.Cancel in situations where you need more dynamic behavior.
	//
	// Experimental: this option is experimental and may be modified or removed in future versions,
	// regardless of any other documented package stability guarantees.
	JobTimeout time.Duration
}

func (c *CopyConfig) toBQ() *bq.JobConfiguration {
	var ts []*bq.TableReference
	for _, t := range c.Srcs {
		ts = append(ts, t.toBQ())
	}
	return &bq.JobConfiguration{
		Labels: c.Labels,
		Copy: &bq.JobConfigurationTableCopy{
			CreateDisposition:                  string(c.CreateDisposition),
			WriteDisposition:                   string(c.WriteDisposition),
			DestinationTable:                   c.Dst.toBQ(),
			DestinationEncryptionConfiguration: c.DestinationEncryptionConfig.toBQ(),
			SourceTables:                       ts,
			OperationType:                      string(c.OperationType),
		},
		JobTimeoutMs: c.JobTimeout.Milliseconds(),
	}
}

func bqToCopyConfig(q *bq.JobConfiguration, c *Client) *CopyConfig {
	cc := &CopyConfig{
		Labels:                      q.Labels,
		CreateDisposition:           TableCreateDisposition(q.Copy.CreateDisposition),
		WriteDisposition:            TableWriteDisposition(q.Copy.WriteDisposition),
		Dst:                         bqToTable(q.Copy.DestinationTable, c),
		DestinationEncryptionConfig: bqToEncryptionConfig(q.Copy.DestinationEncryptionConfiguration),
		OperationType:               TableCopyOperationType(q.Copy.OperationType),
		JobTimeout:                  time.Duration(q.JobTimeoutMs) * time.Millisecond,
	}
	for _, t := range q.Copy.SourceTables {
		cc.Srcs = append(cc.Srcs, bqToTable(t, c))
	}
	return cc
}

// A Copier copies data into a BigQuery table from one or more BigQuery tables.
type Copier struct {
	JobIDConfig
	CopyConfig
	c *Client
}

// CopierFrom returns a Copier which can be used to copy data into a
// BigQuery table from one or more BigQuery tables.
// The returned Copier may optionally be further configured before its Run method is called.
func (t *Table) CopierFrom(srcs ...*Table) *Copier {
	return &Copier{
		c: t.c,
		CopyConfig: CopyConfig{
			Srcs: srcs,
			Dst:  t,
		},
	}
}

// Run initiates a copy job.
func (c *Copier) Run(ctx context.Context) (*Job, error) {
	return c.c.insertJob(ctx, c.newJob(), nil)
}

func (c *Copier) newJob() *bq.Job {
	return &bq.Job{
		JobReference:  c.JobIDConfig.createJobRef(c.c),
		Configuration: c.CopyConfig.toBQ(),
	}
}