Skip to content
Permalink
Browse files
docs: move and refresh view samples (#420)
docs: restore old view snippets

remove relative imports

docs: fix missing space in comment

sort imports
  • Loading branch information
tswast committed Dec 10, 2020
1 parent dab7af3 commit 079b6a162f6929bf801366d92f8daeb3318426c4
Showing with 311 additions and 13 deletions.
  1. +27 −0 samples/snippets/conftest.py
  2. +1 −1 samples/snippets/materialized_view.py
  3. +2 −12 samples/snippets/materialized_view_test.py
  4. +164 −0 samples/snippets/view.py
  5. +117 −0 samples/snippets/view_test.py
@@ -0,0 +1,27 @@
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from google.cloud import bigquery
import pytest


@pytest.fixture(scope="session")
def bigquery_client():
bigquery_client = bigquery.Client()
return bigquery_client


@pytest.fixture(scope="session")
def project_id(bigquery_client):
return bigquery_client.project
@@ -25,7 +25,7 @@ def create_materialized_view(override_values={}):
# To facilitate testing, we replace values with alternatives
# provided by the testing harness.
view_id = override_values.get("view_id", view_id)
base_table_id = override_values.get("base_table_id", view_id)
base_table_id = override_values.get("base_table_id", base_table_id)
# [START bigquery_create_materialized_view]
view = bigquery.Table(view_id)
view.mview_query = f"""
@@ -23,25 +23,15 @@


def temp_suffix():
return str(uuid.uuid4()).replace("-", "_")


@pytest.fixture(scope="module")
def bigquery_client():
bigquery_client = bigquery.Client()
return bigquery_client
now = datetime.datetime.now()
return f"{now.strftime('%Y%m%d%H%M%S')}_{uuid.uuid4().hex[:8]}"


@pytest.fixture(autouse=True)
def bigquery_client_patch(monkeypatch, bigquery_client):
monkeypatch.setattr(bigquery, "Client", lambda: bigquery_client)


@pytest.fixture(scope="module")
def project_id(bigquery_client):
return bigquery_client.project


@pytest.fixture(scope="module")
def dataset_id(bigquery_client):
dataset_id = f"mvdataset_{temp_suffix()}"
@@ -0,0 +1,164 @@
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


def create_view(override_values={}):
# [START bigquery_create_view]
from google.cloud import bigquery

client = bigquery.Client()

view_id = "my-project.my_dataset.my_view"
source_id = "my-project.my_dataset.my_table"
# [END bigquery_create_view]
# To facilitate testing, we replace values with alternatives
# provided by the testing harness.
view_id = override_values.get("view_id", view_id)
source_id = override_values.get("source_id", source_id)
# [START bigquery_create_view]
view = bigquery.Table(view_id)

# The source table in this example is created from a CSV file in Google
# Cloud Storage located at
# `gs://cloud-samples-data/bigquery/us-states/us-states.csv`. It contains
# 50 US states, while the view returns only those states with names
# starting with the letter 'W'.
view.view_query = f"SELECT name, post_abbr FROM `{source_id}` WHERE name LIKE 'W%'"

# Make an API request to create the view.
view = client.create_table(view)
print(f"Created {view.table_type}: {str(view.reference)}")
# [END bigquery_create_view]
return view


def get_view(override_values={}):
# [START bigquery_get_view]
from google.cloud import bigquery

client = bigquery.Client()

view_id = "my-project.my_dataset.my_view"
# [END bigquery_get_view]
# To facilitate testing, we replace values with alternatives
# provided by the testing harness.
view_id = override_values.get("view_id", view_id)
# [START bigquery_get_view]
# Make an API request to get the table resource.
view = client.get_table(view_id)

# Display view properties
print(f"Retrieved {view.table_type}: {str(view.reference)}")
print(f"View Query:\n{view.view_query}")
# [END bigquery_get_view]
return view


def update_view(override_values={}):
# [START bigquery_update_view_query]
from google.cloud import bigquery

client = bigquery.Client()

view_id = "my-project.my_dataset.my_view"
source_id = "my-project.my_dataset.my_table"
# [END bigquery_update_view_query]
# To facilitate testing, we replace values with alternatives
# provided by the testing harness.
view_id = override_values.get("view_id", view_id)
source_id = override_values.get("source_id", source_id)
# [START bigquery_update_view_query]
view = bigquery.Table(view_id)

# The source table in this example is created from a CSV file in Google
# Cloud Storage located at
# `gs://cloud-samples-data/bigquery/us-states/us-states.csv`. It contains
# 50 US states, while the view returns only those states with names
# starting with the letter 'M'.
view.view_query = f"SELECT name, post_abbr FROM `{source_id}` WHERE name LIKE 'M%'"

# Make an API request to update the query property of the view.
view = client.update_table(view, ["view_query"])
print(f"Updated {view.table_type}: {str(view.reference)}")
# [END bigquery_update_view_query]
return view


def grant_access(override_values={}):
# [START bigquery_grant_view_access]
from google.cloud import bigquery

client = bigquery.Client()

# To use a view, the analyst requires ACLs to both the view and the source
# table. Create an authorized view to allow an analyst to use a view
# without direct access permissions to the source table.
view_dataset_id = "my-project.my_view_dataset"
# [END bigquery_grant_view_access]
# To facilitate testing, we replace values with alternatives
# provided by the testing harness.
view_dataset_id = override_values.get("view_dataset_id", view_dataset_id)
# [START bigquery_grant_view_access]
# Make an API request to get the view dataset ACLs.
view_dataset = client.get_dataset(view_dataset_id)

analyst_group_email = "data_analysts@example.com"
# [END bigquery_grant_view_access]
# To facilitate testing, we replace values with alternatives
# provided by the testing harness.
analyst_group_email = override_values.get(
"analyst_group_email", analyst_group_email
)
# [START bigquery_grant_view_access]
access_entries = view_dataset.access_entries
access_entries.append(
bigquery.AccessEntry("READER", "groupByEmail", analyst_group_email)
)
view_dataset.access_entries = access_entries

# Make an API request to update the ACLs property of the view dataset.
view_dataset = client.update_dataset(view_dataset, ["access_entries"])
print(f"Access to view: {view_dataset.access_entries}")

# Group members of "data_analysts@example.com" now have access to the view,
# but they require access to the source table to use it. To remove this
# restriction, authorize the view to access the source dataset.
source_dataset_id = "my-project.my_source_dataset"
# [END bigquery_grant_view_access]
# To facilitate testing, we replace values with alternatives
# provided by the testing harness.
source_dataset_id = override_values.get("source_dataset_id", source_dataset_id)
# [START bigquery_grant_view_access]
# Make an API request to set the source dataset ACLs.
source_dataset = client.get_dataset(source_dataset_id)

view_reference = {
"projectId": "my-project",
"datasetId": "my_view_dataset",
"tableId": "my_authorized_view",
}
# [END bigquery_grant_view_access]
# To facilitate testing, we replace values with alternatives
# provided by the testing harness.
view_reference = override_values.get("view_reference", view_reference)
# [START bigquery_grant_view_access]
access_entries = source_dataset.access_entries
access_entries.append(bigquery.AccessEntry(None, "view", view_reference))
source_dataset.access_entries = access_entries

# Make an API request to update the ACLs property of the source dataset.
source_dataset = client.update_dataset(source_dataset, ["access_entries"])
print(f"Access to source: {source_dataset.access_entries}")
# [END bigquery_grant_view_access]
return view_dataset, source_dataset
@@ -0,0 +1,117 @@
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime
import uuid

from google.cloud import bigquery
import pytest

import view


def temp_suffix():
now = datetime.datetime.now()
return f"{now.strftime('%Y%m%d%H%M%S')}_{uuid.uuid4().hex[:8]}"


@pytest.fixture(autouse=True)
def bigquery_client_patch(monkeypatch, bigquery_client):
monkeypatch.setattr(bigquery, "Client", lambda: bigquery_client)


@pytest.fixture(scope="module")
def view_dataset_id(bigquery_client, project_id):
dataset_id = f"{project_id}.view_{temp_suffix()}"
bigquery_client.create_dataset(dataset_id)
yield dataset_id
bigquery_client.delete_dataset(dataset_id, delete_contents=True)


@pytest.fixture(scope="module")
def view_id(bigquery_client, view_dataset_id):
view_id = f"{view_dataset_id}.my_view"
yield view_id
bigquery_client.delete_table(view_id, not_found_ok=True)


@pytest.fixture(scope="module")
def source_dataset_id(bigquery_client, project_id):
dataset_id = f"{project_id}.view_{temp_suffix()}"
bigquery_client.create_dataset(dataset_id)
yield dataset_id
bigquery_client.delete_dataset(dataset_id, delete_contents=True)


@pytest.fixture(scope="module")
def source_table_id(bigquery_client, source_dataset_id):
source_table_id = f"{source_dataset_id}.us_states"
job_config = bigquery.LoadJobConfig(
schema=[
bigquery.SchemaField("name", "STRING"),
bigquery.SchemaField("post_abbr", "STRING"),
],
skip_leading_rows=1,
)
load_job = bigquery_client.load_table_from_uri(
"gs://cloud-samples-data/bigquery/us-states/us-states.csv",
source_table_id,
job_config=job_config,
)
load_job.result()
yield source_table_id
bigquery_client.delete_table(source_table_id, not_found_ok=True)


def test_view(capsys, view_id, view_dataset_id, source_table_id, source_dataset_id):
override_values = {
"view_id": view_id,
"source_id": source_table_id,
}
got = view.create_view(override_values)
assert source_table_id in got.view_query
out, _ = capsys.readouterr()
assert view_id in out

got = view.get_view(override_values)
assert source_table_id in got.view_query
assert "'W%'" in got.view_query
out, _ = capsys.readouterr()
assert view_id in out
assert source_table_id in out
assert "'W%'" in out

got = view.update_view(override_values)
assert source_table_id in got.view_query
assert "'M%'" in got.view_query
out, _ = capsys.readouterr()
assert view_id in out

project_id, dataset_id, table_id = view_id.split(".")
override_values = {
"analyst_group_email": "cloud-dpes-bigquery@google.com",
"view_dataset_id": view_dataset_id,
"source_dataset_id": source_dataset_id,
"view_reference": {
"projectId": project_id,
"datasetId": dataset_id,
"tableId": table_id,
},
}
view_dataset, source_dataset = view.grant_access(override_values)
assert len(view_dataset.access_entries) != 0
assert len(source_dataset.access_entries) != 0
out, _ = capsys.readouterr()
assert "cloud-dpes-bigquery@google.com" in out
assert table_id in out

0 comments on commit 079b6a1

Please sign in to comment.