Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Notion refactor #150

Merged
merged 3 commits into from
Sep 14, 2023
Merged

Notion refactor #150

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5e12747
use custom methods for all API queries
sHermanGriffiths Sep 13, 2023
1142ae0
protect against infinite recursion when handling synced blocks and co…
sHermanGriffiths Sep 13, 2023
18d638e
pass tests
sHermanGriffiths Sep 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
21 changes: 16 additions & 5 deletions n2y/blocks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -672,22 +672,33 @@ class SyncedBlock(Block):
def __init__(self, client, notion_data, page, get_children=True):
self.original = notion_data[notion_data["type"]]["synced_from"] is None
super().__init__(client, notion_data, page, get_children=self.original)
self.is_recursive = None
# Synced blocks will always have children unless not shared
# (There will always be at least one UnsupportedBlock child)
self.shared = self.has_children
self.children = self._get_synced_block_children()

def _get_synced_block_children(self):
if not self.original and self.shared:
return self.client.get_child_blocks(
self.notion_type_data["synced_from"]["block_id"],
self.page, True,
)
# This last condition is to protect against recursive synced blocks while
# still allowing synced blocks that are children of other synced blocks
# (once Notion confirms that this bug has been addressed it can be removed)
parent = self.notion_data.get('parent', None)
self.is_recursive = parent and self.notion_type_data["synced_from"][
"block_id"] == parent[parent['type']]
if not self.is_recursive:
return self.client.get_child_blocks(
self.notion_type_data["synced_from"]["block_id"],
self.page, True,
)
return self.children

def to_pandoc(self):
if not self.shared:
logger.warning('Skipping un-shared synced block (%s)', self.notion_url)
# logger.warning('Skipping un-shared synced block (%s)', self.notion_url)
return None
elif self.is_recursive:
logger.warning('Skipping recursive synced block (%s)', self.notion_url)
return None
return self.children_to_pandoc()

Expand Down
82 changes: 36 additions & 46 deletions n2y/notion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,6 @@ def get_database_pages(self, database_id, filter=None, sorts=None):
notion_pages = self.get_database_notion_pages(database_id, filter, sorts)
return [self._wrap_notion_page(np) for np in notion_pages]

@retry_api_call
def get_database_notion_pages(self, database_id, filter, sorts):
url = f"{self.base_url}databases/{database_id}/query"
request_data = {}
Expand Down Expand Up @@ -319,22 +318,18 @@ def get_block(self, block_id, page, get_children=True):
notion_block = self.get_notion_block(block_id)
return self.wrap_notion_block(notion_block, page, get_children)

@retry_api_call
def get_notion_block(self, block_id):
url = f"{self.base_url}blocks/{block_id}"
response = requests.get(url, headers=self.headers)
return self._parse_response(response)
return self._get_url(url)

def get_child_blocks(self, block_id, page, get_children):
child_notion_blocks = self.get_child_notion_blocks(block_id)
return [self.wrap_notion_block(b, page, get_children) for b in child_notion_blocks]

@retry_api_call
def get_child_notion_blocks(self, block_id):
url = f"{self.base_url}blocks/{block_id}/children"
return self._paginated_request(self._get_url, url, {})

@retry_api_call
def get_comments(self, block_id):
url = f"{self.base_url}comments"
comments = self._paginated_request(self._get_url, url, {"block_id": block_id})
Expand All @@ -347,24 +342,25 @@ def get_page_property(self, page_id, property_id):
notion_property = self.get_notion_page_property(page_id, property_id)
return self.wrap_notion_property(notion_property)

@retry_api_call
def get_notion_page_property(self, page_id, property_id):
url = f"{self.base_url}pages/{page_id}/properties/{property_id}"
response = requests.get(url, headers=self.headers)
return self._parse_response(response)
return self._get_url(url)

@retry_api_call
def create_notion_page(self, page_data):
creation_url = f'{self.base_url}pages'
response = requests.post(creation_url, headers=self.headers, json=page_data)
return self._parse_response(response)
return self._post_url(creation_url, page_data)

@retry_api_call
def _get_url(self, url, params=None):
if params is None:
def _get_url(self, url, params=None, stream=False):
if not stream and params is None:
params = {}
response = requests.get(url, headers=self.headers, params=params)
return self._parse_response(response)
response = requests.get(
url,
params=params,
stream=stream,
headers=self.headers if not stream else None
)
return self._parse_response(response, stream)

@retry_api_call
def _post_url(self, url, data=None):
Expand All @@ -373,6 +369,21 @@ def _post_url(self, url, data=None):
response = requests.post(url, headers=self.headers, json=data)
return self._parse_response(response)

@retry_api_call
def _delete_url(self, url):
response = requests.delete(
url,
headers={k: v for k, v in self.headers.items() if k != 'Content-Type'}
)
return self._parse_response(response)

@retry_api_call
def _patch_url(self, url, data=None):
if data is None:
data = {}
response = requests.patch(url, headers=self.headers, json=data)
return self._parse_response(response)

def _paginated_request(self, request_method, url, initial_params):
params = initial_params
results = []
Expand All @@ -384,7 +395,7 @@ def _paginated_request(self, request_method, url, initial_params):
else:
params["start_cursor"] = data["next_cursor"]

def _parse_response(self, response):
def _parse_response(self, response, stream=False):
"""Taken from https://github.com/ramnes/notion-sdk-py"""
try:
response.raise_for_status()
Expand All @@ -401,9 +412,8 @@ def _parse_response(self, response):
response, body["message"], code
)
raise HTTPResponseError(error.response)
return response.json()
return response.json() if not stream else response.content

@retry_api_call
def download_file(self, url, page, block_id):
"""
Download a file from a given URL into the MEDIA_ROOT.
Expand All @@ -413,8 +423,8 @@ def download_file(self, url, page, block_id):
"""
url_path = path.basename(urlparse(url).path)
_, extension = path.splitext(url_path)
request_stream = requests.get(url, stream=True)
return self.save_file(request_stream.content, page, extension, block_id)
content = self._get_url(url, stream=True)
return self.save_file(content, page, extension, block_id)

def save_file(self, content, page, extension, block_id):
block_id_chars = strip_hyphens(block_id)
Expand Down Expand Up @@ -477,7 +487,6 @@ def _notion_block_type_is(self, block, type):
def _notion_block_object_is(self, block, object):
return 'object' in block and block['object'] == object

@retry_api_call
def append_child_notion_blocks(self, block_id, children):
'''
Appends each datapoint of a list of notion_data as children to the block specified by id.
Expand Down Expand Up @@ -527,11 +536,10 @@ def _append_blocks(self, block_id, full_child_data_list, appension_history_list,
portion = child_data_list[i:portion_index_stop]
else:
portion = child_data_list[i:]
response = requests.patch(
appension_return = self._patch_url(
f"{self.base_url}blocks/{block_id}/children",
json={"children": portion}, headers=self.headers
{"children": portion}
)
appension_return = self._parse_response(response)
appension_history_list.extend(appension_return['results'])
return appension_history_list

Expand Down Expand Up @@ -594,7 +602,6 @@ def _copy_notion_database_child_database(self, parent, parent_type, child_notion
self.copy_notion_database_children(notion_children, child_database)
return child_database

@retry_api_call
def create_notion_comment(self, page_id, text_blocks_descriptors):
data = {
"rich_text": mock_rich_text_array(text_blocks_descriptors),
Expand All @@ -603,30 +610,13 @@ def create_notion_comment(self, page_id, text_blocks_descriptors):
"page_id": page_id,
},
}
response = requests.post(
f"{self.base_url}comments",
headers=self.headers,
json=data
)
return self._parse_response(response)
return self._post_url(f"{self.base_url}comments", data)

@retry_api_call
def create_notion_database(self, notion_data):
response = requests.post(
f'{self.base_url}databases',
headers=self.headers,
json=notion_data)
return self._parse_response(response)
return self._post_url(f'{self.base_url}databases', notion_data)

@retry_api_call
def delete_notion_block(self, notion_block):
block_id = notion_block['id']
headers = {**self.headers}
del headers['Content-Type']
response = requests.delete(
f"{self.base_url}blocks/{block_id}", headers=headers
)
return self._parse_response(response)
return self._delete_url(f"{self.base_url}blocks/{notion_block['id']}")

def page_class_is_in_use(self, page):
'''
Expand Down