Skip to content

Commit

Permalink
Fix Linting
Browse files Browse the repository at this point in the history
Signed-off-by: Abhinay Kathuria <abhinay@kathuria.com.au>

Signed-off-by: Abhinay Kathuria <abhinay@kathuria.com.au>
  • Loading branch information
abhinay04 authored and Abhinay Kathuria committed Nov 2, 2020
1 parent 5f6313e commit 9cc8ea5
Showing 1 changed file with 12 additions and 5 deletions.
17 changes: 12 additions & 5 deletions databuilder/extractor/bigquery_usage_extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@ def _count_usage(self) -> None: # noqa: C901
continue

email = entry['protoPayload']['authenticationInfo']['principalEmail']

# Query results can be cached and if the source tables remain untouched,
# bigquery will return it from a 24 hour cache result instead. In that
# case, referencedTables has been observed to be empty:
Expand All @@ -73,22 +72,30 @@ def _count_usage(self) -> None: # noqa: C901
refTables = job['jobStatistics'].get('referencedTables', None)
if refTables:
if 'totalTablesProcessed' in job['jobStatistics']:
self._create_records(refTables, job['jobStatistics']['totalTablesProcessed'], email, job['jobName']['jobId'])
self._create_records(
refTables,
job['jobStatistics']['totalTablesProcessed'], email,
job['jobName']['jobId'])

refViews = job['jobStatistics'].get('referencedViews', None)
if refViews:
if 'totalViewsProcessed' in job['jobStatistics']:
self._create_records(refViews, job['jobStatistics']['totalViewsProcessed'], email, job['jobName']['jobId'])
self._create_records(
refViews, job['jobStatistics']['totalViewsProcessed'],
email, job['jobName']['jobId'])

def _create_records(self, refResources, resourcesProcessed, email, jobId) -> None:
def _create_records(self, refResources, resourcesProcessed, email,
jobId) -> None:
# if email filter is provided, only the email matched with filter will be recorded.
if self.email_pattern:
if not re.match(self.email_pattern, email):
# the usage account not match email pattern
return

if len(refResources) != resourcesProcessed:
LOGGER.warn('The number of tables listed in job {job_id} is not consistent'.format(job_id=jobId))
LOGGER.warn(
'The number of tables listed in job {job_id} is not consistent'
.format(job_id=jobId))
return

for refResource in refResources:
Expand Down

0 comments on commit 9cc8ea5

Please sign in to comment.