Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update tableadmin #1562

Merged
merged 16 commits into from
Jul 16, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
103 changes: 43 additions & 60 deletions bigtable/tableadmin/tableadmin.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,31 +63,25 @@ def run_table_operations(project_id, instance_id, table_id):
table.create()
print 'Created table {}.'.format(table_id)

# [START List existing tables in the instance.]
# [START bigtable_list_tables]
tables = instance.list_tables()
print 'Listing tables in current project...'
if tables != []:
for tbl in tables:
print tbl.table_id
else:
print 'No table exists in current project...'
# [END List existing tables in the instance.]

# Display name of the table.
print 'Printing table metadata...'
print table.table_id
# [END bigtable_list_tables]

# [START bigtable_create_family_gc_max_age]
print 'Creating column family cf1 with with MaxAge GC Rule...'
# Create a column family with GC policy : maximum age
# where age = current time minus cell timestamp
column_family_id1 = 'cf1'

# Define the GC rule to retain data with max age of 5 days
max_age = datetime.timedelta(days=5)
max_age_rule = table.max_age_gc_rule(max_age)
print 'Created MaxAge GC rule.'
max_age_rule = table.max_age_gc_rule(datetime.timedelta(days=5))

cf1 = table.column_family(column_family_id1, max_age_rule)
cf1 = table.column_family('cf1', max_age_rule)
cf1.create()
print 'Created column family cf1 with MaxAge GC Rule.'
# [END bigtable_create_family_gc_max_age]
Expand All @@ -96,13 +90,11 @@ def run_table_operations(project_id, instance_id, table_id):
print 'Creating column family cf2 with max versions GC rule...'
# Create a column family with GC policy : most recent N versions
# where 1 = most recent version
column_family_id2 = 'cf2'

# Define the GC policy to retain only the most recent 2 versions
max_versions = 2
max_versions_rule = table.max_versions_gc_rule(max_versions)
print 'Created Max Versions GC Rule.'
max_versions_rule = table.max_versions_gc_rule(2)

cf2 = table.column_family(column_family_id2, max_versions_rule)
cf2 = table.column_family('cf2', max_versions_rule)
cf2 .create()
print 'Created column family cf2 with Max Versions GC Rule.'
# [END bigtable_create_family_gc_max_versions]
Expand All @@ -111,13 +103,13 @@ def run_table_operations(project_id, instance_id, table_id):
print 'Creating column family cf3 with union GC rule...'
# Create a column family with GC policy to drop data that matches
# at least one condition.
column_family_id3 = 'cf3'
# GC rule : Drop max age rule OR the most recent version rule.
union = [max_age_rule, max_versions_rule]
union_rule = table.gc_rule_union(union)
print 'Created Union GC Rule.'
# Define a GC rule to drop cells older than 5 days or not the
# most recent version
union_rule = table.gc_rule_union([
table.max_age_gc_rule(datetime.timedelta(days=5)),
table.max_versions_gc_rule(2)])

cf3 = table.column_family(column_family_id3, union_rule)
cf3 = table.column_family('cf3', union_rule)
cf3.create()
print 'Created column family cf3 with Union GC rule'
# [END bigtable_create_family_gc_union]
Expand All @@ -126,45 +118,33 @@ def run_table_operations(project_id, instance_id, table_id):
print 'Creating column family cf4 with Intersection GC rule...'
# Create a column family with GC policy to drop data that matches
# all conditions
column_family_id4 = 'cf4'
# GC rule: Drop cells older than 5 days AND older than the most
# recent 2 versions
intersection = [max_age_rule, max_versions_rule]
intersection_rule = table.gc_rule_intersection(union)
print 'Created Intersection GC Rule.'
intersection_rule = table.gc_rule_intersection([
table.max_age_gc_rule(datetime.timedelta(days=5)),
table.max_versions_gc_rule(2)])

cf4 = table.column_family(column_family_id4, intersection_rule)
cf4 = table.column_family('cf4', intersection_rule)
cf4.create()
print 'Created column family cf4 with Intersection GC rule.'
# [END bigtable_create_family_gc_intersection]

# [START bigtable_create_family_gc_nested]
print 'Creating column family cf5 with a Nested GC rule...'
# Create a column family with nested GC policys.
# Create a column family with nested GC policies.
# Create a nested GC rule:
# Drop cells that are either older than the 10 recent versions
# OR
# Drop cells that are older than a month AND older than the
# 2 recent versions
column_family_id5 = 'cf5'
# Drop cells that are either older than the 10 recent versions
max_versions_rule1 = table.max_versions_gc_rule(10)

# Drop cells that are older than a month AND older than
# the 2 recent versions
max_age = datetime.timedelta(days=30)
max_age_rule = table.max_age_gc_rule(max_age)
max_versions_rule2 = table.max_versions_gc_rule(2)
intersection = [max_age_rule, max_versions_rule2]
intersection_rule = table.gc_rule_intersection(intersection)

# This nesting is done with union rule since it is OR between
# the selected rules.
nest = [max_versions_rule1, intersection_rule]
nested_rule = table.gc_rule_union(nest)
print 'Created Nested GC Rule.'

cf5 = table.column_family(column_family_id5, nested_rule)
rule1 = table.max_versions_gc_rule(10)
rule2 = table.gc_rule_intersection([
table.max_age_gc_rule(datetime.timedelta(days=30)),
table.max_versions_gc_rule(2)])

nested_rule = table.gc_rule_union([rule1, rule2])

cf5 = table.column_family('cf5', nested_rule)
cf5.create()
print 'Created column family cf5 with a Nested GC rule.'
# [END bigtable_create_family_gc_nested]
Expand Down Expand Up @@ -193,23 +173,21 @@ def run_table_operations(project_id, instance_id, table_id):
# }
# [END bigtable_list_column_families]

print 'Print column family cf1 GC rule before update...'
print "Column Family: cf1"
print cf1.to_pb()

# [START bigtable_update_gc_rule]
print 'Updating column family cf1 GC rule...'
# Update the column family cf1 to update the GC rule
max_versions = 1
max_versions_rule = table.max_versions_gc_rule(max_versions)
# Create a reference to the column family with GC rule
cf1 = table.column_family(column_family_id1, max_versions_rule)
cf1 = table.column_family('cf1', table.max_versions_gc_rule(1))
cf1.update()
print 'Updated column family cf1 GC rule'
print 'Updated column family cf1 GC rule\n'
# [END bigtable_update_gc_rule]

# [START bigtable_family_get_gc_rule]
print 'Print updated column family cf1 GC rule...'
print 'Column Family:', column_family_id1
print 'GC Rule:'
print 'Print column family cf1 GC rule after update...'
print "Column Family: cf1"
print cf1.to_pb()
# [END bigtable_family_get_gc_rule]

# [START bigtable_delete_family]
print 'Delete a column family cf2...'
Expand All @@ -218,8 +196,8 @@ def run_table_operations(project_id, instance_id, table_id):
print 'Column family cf2 deleted successfully.'
# [END bigtable_delete_family]

print 'execute command python tableadmin.py delete [project_id] \
[instance_id] --table [tableName] to delete the table.'
print 'execute command "python tableadmin.py delete [project_id] \
[instance_id] --table [tableName]" to delete the table.'


def exists(instance_obj, table_id):
Expand All @@ -234,7 +212,8 @@ def exists(instance_obj, table_id):
:param instance_obj: Instance object.

:type table_id: str
:param table_id: Table id to create table.
:param table_id: Table id to identify table.

Returns bool
"""
for table in instance_obj.list_tables():
Expand All @@ -260,6 +239,9 @@ def delete_table(project_id, instance_id, table_id):
instance = client.instance(instance_id)
table = instance.table(table_id)

# [START bigtable_delete_table]
# Delete the entire table

print 'Checking if table {} exists...'.format(table_id)
if exists(instance, table_id):
print 'Table {} exists.'.format(table_id)
Expand All @@ -268,6 +250,7 @@ def delete_table(project_id, instance_id, table_id):
print 'Deleted {} table.'.format(table_id)
else:
print 'Table {} does not exists.'.format(table_id)
# [END bigtable_delete_table]


if __name__ == '__main__':
Expand Down
16 changes: 6 additions & 10 deletions bigtable/tableadmin/tableadmin_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,23 +31,19 @@ def test_run_table_operations(capsys):

run_table_operations(PROJECT, BIGTABLE_CLUSTER, table_name)
out, _ = capsys.readouterr()
assert 'Creating the {} table.'.format(table_name) in out

assert 'Creating the ' + table_name + ' table.' in out
assert 'Listing tables in current project.' in out
assert 'Creating column family cf1 with with MaxAge GC Rule' in out
assert 'Created MaxAge GC rule.' in out
assert 'Created column family cf1 with MaxAge GC Rule.' in out
assert 'Created Max Versions GC Rule.' in out
assert 'Created column family cf2 with Max Versions GC Rule.' in out
assert 'Created Union GC Rule.' in out
assert 'Created column family cf3 with Union GC rule' in out
assert 'Created Intersection GC Rule.' in out
assert 'Created column family cf4 with Intersection GC rule.' in out
assert 'Created Nested GC Rule.' in out
assert 'Created column family cf5 with a Nested GC rule.' in out
assert 'Printing Column Family and GC Rule for all column families.' in out
assert 'Updating column family cf1 GC rule...' in out
assert 'Updated column family cf1 GC rule' in out
assert 'Print updated column family cf1 GC rule...' in out
assert 'Print column family cf1 GC rule after update...' in out
assert 'Column Family: cf1' in out
assert 'max_num_versions: 1' in out
assert 'Delete a column family cf2...' in out
Expand All @@ -61,6 +57,6 @@ def test_delete_table(capsys):
delete_table(PROJECT, BIGTABLE_CLUSTER, table_name)
out, _ = capsys.readouterr()

assert 'Table {} exists.'.format(table_name) in out
assert 'Deleting {} table.'.format(table_name) in out
assert 'Deleted {} table.'.format(table_name) in out
assert 'Table ' + table_name + ' exists.' in out
assert 'Deleting ' + table_name + ' table.' in out
assert 'Deleted ' + table_name + ' table.' in out