Skip to content
This repository has been archived by the owner on Jul 17, 2023. It is now read-only.

Commit

Permalink
Docs improvement and some renaming.
Browse files Browse the repository at this point in the history
  • Loading branch information
IlyaFaer committed Feb 8, 2020
1 parent 975eb25 commit 4a5620d
Show file tree
Hide file tree
Showing 6 changed files with 95 additions and 55 deletions.
53 changes: 44 additions & 9 deletions scraper/instances.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ class Columns:
cols (list):
Dicts, each of which describes single column.
sheet_id (int): Numeric sheet's id.
sheet_id (int): Numeric sheet id.
"""

def __init__(self, cols, sheet_id):
Expand All @@ -36,6 +36,9 @@ def requests(self):
"""
Return all column requests. Title row request
must be first!
Returns:
list: Columns formatting requests.
"""
self._requests.insert(0, self._title_row_request)
return self._requests
Expand All @@ -46,13 +49,18 @@ def column_symbol(self, column):
Args:
column (str): Name of column.
Returns: Letter coordinate of column (str).
Returns:
str: Letter coordinate of the column.
"""
return string.ascii_uppercase[self.names.index(column)]

@property
def _title_row_request(self):
"""Bolding and aligning title row."""
"""Bolding and aligning title row.
Returns:
dict: Title row formatting request.
"""
request = {
"repeatCell": {
"fields": "userEnteredFormat",
Expand All @@ -77,6 +85,10 @@ def _gen_date_type_request(self, index, col):
"""
Request to set date format for column, that
designed to contain dates.
Args:
index (int): Column index.
col (dict): Column description.
"""
if col.get("type") == "date":
request = {
Expand All @@ -97,7 +109,12 @@ def _gen_date_type_request(self, index, col):
self._requests.append(request)

def _gen_align_request(self, index, col):
"""Aligning request for column."""
"""Aligning request for column.
Args:
index (int): Column index.
col (dict): Column description.
"""
if "align" in col.keys():
request = {
"repeatCell": {
Expand All @@ -117,7 +134,12 @@ def _gen_align_request(self, index, col):
self._requests.append(request)

def _gen_color_requests(self, index, col):
"""Requests to set color for specific values in cell."""
"""Requests to set color for specific values in cell.
Args:
index (int): Column index.
col (dict): Column description.
"""
if "values" in col.keys() and isinstance(col["values"], dict):
for value, color in col["values"].items():
self._requests.append(
Expand Down Expand Up @@ -145,7 +167,12 @@ def _gen_color_requests(self, index, col):
)

def _gen_size_request(self, index, col):
"""Request to set column's width."""
"""Request to set column's width.
Args:
index (int): Column index.
col (dict): Column description.
"""
if "width" in col.keys():
request = {
"updateDimensionProperties": {
Expand All @@ -163,7 +190,12 @@ def _gen_size_request(self, index, col):
self._requests.append(request)

def _gen_one_of_request(self, index, col):
"""Request to set data validation."""
"""Request to set data validation.
Args:
index (int): Column index.
col (dict): Column description.
"""
if "values" in col.keys():
if isinstance(col["values"], dict):
vals = [{"userEnteredValue": key} for key in col["values"].keys()]
Expand All @@ -190,7 +222,7 @@ def _gen_one_of_request(self, index, col):


class Row(dict):
"""Dict-like representation of single row.
"""Dict-like representation of a single row.
Args:
column_names (list): List of column names.
Expand Down Expand Up @@ -221,8 +253,11 @@ def as_list(self):

def fill_from_list(self, list_):
"""
Fill dict from list. Connections between fields
Fill dict from the list. Relations between fields
and list elements are designated by columns list.
Args:
list_ (list): List representation of the row.
"""
for index, name in enumerate(self._column_names):
if index < len(list_):
Expand Down
2 changes: 1 addition & 1 deletion scraper/pr_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def __init__(self):
# time when any PR was last updated in specific repo
self._last_pr_updates = {}

def update_config(self, in_repo_names):
def reload_config(self, in_repo_names):
"""Update list of internal repos.
Args:
Expand Down
38 changes: 25 additions & 13 deletions scraper/sheet.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,39 +66,39 @@ def update(self, ss_resource):
"""Update specified sheet with issues/PRs data."""
self._prepare_builder()

raw_new_table = self._builder.retrieve_updated()
updated_issues = self._builder.retrieve_updated()
tracked_issues = self._read(ss_resource)

to_be_deleted = []
# merging the new table into the old one
for tracked_id in tracked_issues.keys():
updated_issue = None
if tracked_id in raw_new_table:
updated_issue = raw_new_table.pop(tracked_id)
issue_obj = None
if tracked_id in updated_issues:
issue_obj = updated_issues.pop(tracked_id)
# on a first update check old (closed issues included)
# rows too in case of Scraper restarts
elif self._builder.first_update:
updated_issue = self._builder.read_issue(*tracked_id)
issue_obj = self._builder.read_issue(*tracked_id)
# if issue wasn't updated, take it's last
# version from internal index
else:
updated_issue = self._builder.get_from_index(tracked_id)
issue_obj = self._builder.get_from_index(tracked_id)

prs = self._builder.get_related_prs(tracked_id)
if updated_issue:
if issue_obj:
# update columns using fill function
for col in self._columns.names:
self._columns.fill_funcs[col](
tracked_issues[tracked_id],
updated_issue,
issue_obj,
self.name,
self._config,
prs,
False,
)

to_del = fill_funcs.to_be_deleted(
tracked_issues[tracked_id], updated_issue, prs
tracked_issues[tracked_id], issue_obj, prs
)
if to_del:
to_be_deleted.append(tracked_id)
Expand All @@ -107,7 +107,7 @@ def update(self, ss_resource):
tracked_issues.pop(id_)
self._builder.delete_from_index(id_)

self._insert_new_issues(tracked_issues, raw_new_table)
self._insert_new_issues(tracked_issues, updated_issues)
new_table, requests = self._prepare_table(tracked_issues.values())

self._format_sheet(ss_resource)
Expand All @@ -122,7 +122,7 @@ def _prepare_builder(self):
if self._builder is None:
self._builder = sheet_builder.SheetBuilder()

self._builder.update_config(self._config)
self._builder.reload_config(self._config)

def _insert(self, ss_resource, rows, start_from):
"""Write new data into this sheet.
Expand Down Expand Up @@ -210,6 +210,7 @@ def _clear_bottom(self, ss_resource, length, width):
Args:
length (int): Length of issues list.
width (int): Number of columns in range to clear.
"""
sym_range = "{sheet_name}!A{start_from}:{end}".format(
sheet_name=self.name,
Expand Down Expand Up @@ -291,7 +292,8 @@ def _build_index(table, column_names):
table (list): Lists, each of which represents single row.
column_names (list): Tracked columns names.
Returns: Dict, which values represents rows.
Returns:
dict: Index of Rows.
"""
index = {}
for row in table:
Expand All @@ -302,7 +304,17 @@ def _build_index(table, column_names):


def _gen_color_request(sheet_id, row, column, color):
"""Request, that changes color of specified cell."""
"""Request, that changes color of specified cell.
Args:
sheet_id (int): Numeric sheet id.
row (int): Number of the row to highlight with color.
column (int): Number of the column to highlight with color.
color (str): Color code.
Returns:
dict: Highlighting request.
"""
request = {
"repeatCell": {
"fields": "userEnteredFormat",
Expand Down
4 changes: 2 additions & 2 deletions scraper/sheet_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ def read_issue(self, issue_num, repo_lts):
self._issues_index[(issue_num, repo_lts)] = issue
return issue

def update_config(self, config):
def reload_config(self, config):
"""Update builder's configurations - list of tracked repos.
Args:
Expand All @@ -118,7 +118,7 @@ def update_config(self, config):
self._repo_names = config["repo_names"]
self._in_repo_names = config.get("internal_repo_names", {})

self.prs_index.update_config(self._in_repo_names)
self.prs_index.reload_config(self._in_repo_names)
self._repo_names_inverse = dict((v, k) for k, v in self._repo_names.items())

def get_related_prs(self, issue_id):
Expand Down
20 changes: 10 additions & 10 deletions scraper/spreadsheet.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def __init__(self, config, id_=None):
self._config = config
self._ss_resource = auth.authenticate()
self._id = id_ or self._create()
self._sheets = self._init_sheets()
self.sheets = self._init_sheets()

@property
def id(self):
Expand Down Expand Up @@ -81,7 +81,7 @@ def update_structure(self):

def update_all_sheets(self):
"""Update all the sheets one by one."""
for sheet_name, sheet in self._sheets.items():
for sheet_name, sheet in self.sheets.items():
logging.info("Updating sheet " + sheet_name)
try:
sheet.update(self._ss_resource)
Expand All @@ -97,7 +97,7 @@ def reload_config(self, config):
Imported config.py module with all preferences.
"""
self._config = config
for sheet_name, sheet in self._sheets.items():
for sheet_name, sheet in self.sheets.items():
sheet.reload_config(self._config.SHEETS[sheet_name])

def _init_sheets(self):
Expand Down Expand Up @@ -127,16 +127,16 @@ def _actualize_sheets(self):
resp = self._ss_resource.get(spreadsheetId=self._id).execute()
for sheet in resp["sheets"]:
props = sheet["properties"]
self._sheets[props["title"]].id = props["sheetId"]
self.sheets[props["title"]].id = props["sheetId"]
sheets_in_spreadsheet.append(props["title"])

for sheet_name, sheet in self._sheets.items():
for sheet_name, sheet in self.sheets.items():
if sheet_name not in sheets_in_spreadsheet:
to_delete.append(sheet_name)
continue

for sheet_name in to_delete:
self._sheets.pop(sheet_name)
self.sheets.pop(sheet_name)

def _build_new_sheets_requests(self, sheets_in_conf):
"""Build add-new-sheet requests for the new sheets.
Expand All @@ -150,9 +150,9 @@ def _build_new_sheets_requests(self, sheets_in_conf):
new_sheets_reqs = []

for sheet_name in sheets_in_conf:
if sheet_name not in self._sheets.keys():
self._sheets[sheet_name] = Sheet(sheet_name, self._id)
new_sheets_reqs.append(self._sheets[sheet_name].create_request)
if sheet_name not in self.sheets.keys():
self.sheets[sheet_name] = Sheet(sheet_name, self._id)
new_sheets_reqs.append(self.sheets[sheet_name].create_request)

return new_sheets_reqs

Expand All @@ -169,7 +169,7 @@ def _build_delete_sheets_requests(self, sheets_in_conf):
"""
del_sheets_reqs = []

for sheet_name, sheet in self._sheets.items():
for sheet_name, sheet in self.sheets.items():
if sheet_name not in sheets_in_conf:
del_sheets_reqs.append(sheet.delete_request)

Expand Down

0 comments on commit 4a5620d

Please sign in to comment.