-
Notifications
You must be signed in to change notification settings - Fork 1
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
update how database filters children and add version flag #89
Changes from 12 commits
d7bbffa
91024b6
b35ebe4
3cdd764
a358f6c
446aaf0
ee0d5e1
944dd37
279423b
b879ae1
50564a9
4b67bdb
e9f502f
c0c8560
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -30,6 +30,7 @@ def __init__(self, client, notion_data): | |
self.archived = notion_data['archived'] | ||
|
||
self._children = None | ||
self._filtered_children = {} | ||
|
||
@property | ||
def filename(self): | ||
|
@@ -41,10 +42,29 @@ def children(self): | |
self._children = self.client.get_database_pages(self.notion_id) | ||
return self._children | ||
|
||
def get_children(self): | ||
self._children = self.client.get_database_pages(self.notion_id) | ||
|
||
def children_filtered(self, filter, sort=None): | ||
if self._children is None: | ||
self._children = self.client.get_database_pages(self.notion_id, filter, sort) | ||
return self._children | ||
if not filter: | ||
return self.children | ||
else: | ||
tupled_filter = self._tuplize(filter) | ||
tupled_sort = self._tuplize(sort) | ||
if tupled_filter not in self._filtered_children: | ||
self._filtered_children[tupled_filter] = {} | ||
if tupled_sort not in self._filtered_children[tupled_filter]: | ||
self._filtered_children[tupled_filter][tupled_sort] = \ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: you can avoid the
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I did that for flake8, I tried your suggestion first but it would make the line 101 characters |
||
self.client.get_database_pages(self.notion_id, filter, sort) | ||
return self._filtered_children[tupled_filter][tupled_sort] | ||
|
||
def _tuplize(self, item): | ||
if callable(getattr(item, "items", None)): | ||
return tuple([(key, self._tuplize(val)) for (key, val) in item.items()]) | ||
elif hasattr(item, '__iter__') and type(item) is not str: | ||
return tuple([self._tuplize(i) for i in item]) | ||
else: | ||
return (item) | ||
|
||
@property | ||
def parent(self): | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,6 +2,7 @@ | |
import sys | ||
import logging | ||
import argparse | ||
import pkg_resources | ||
|
||
from n2y import notion | ||
from n2y.export import export_page, database_to_yaml, database_to_markdown_files | ||
|
@@ -27,6 +28,10 @@ def main(raw_args, access_token): | |
"--verbosity", '-v', default='INFO', | ||
help="Level to set the root logging module to", | ||
) | ||
parser.add_argument( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'd try to pull the version number out of setup.py somehow, so we don't have to update it in two places (we'll probably forget if we have to). |
||
"--version", action='version', version=pkg_resources.require("n2y")[0].version, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yay! thanks for adding this |
||
help="The version of n2y installed", | ||
) | ||
|
||
args = parser.parse_args(raw_args) | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -381,6 +381,43 @@ def save_file(self, content, page, extension): | |
temp_file.write(content) | ||
return urljoin(self.media_url, relative_filepath) | ||
|
||
def copy_notion_database_children(self, children, destination): | ||
''' | ||
copy the notion children (`children`) of one notion database to another (`destination`) | ||
''' | ||
db_children = [ | ||
{ | ||
key: value | ||
for (key, value) in child.items() | ||
if key not in [ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's extract out this array into a variable at the top or a constant somewhere |
||
'id', | ||
'url', | ||
'parent', | ||
'created_by', | ||
'created_time', | ||
'last_edited_by', | ||
'last_edited_time', | ||
] | ||
} for child in children | ||
] | ||
for page in db_children: | ||
page['parent'] = { | ||
'type': 'database_id', | ||
'database_id': destination['id'] | ||
} | ||
for key in page['properties'].keys(): | ||
del page['properties'][key]['id'] | ||
prop_type = page['properties'][key]['type'] | ||
del page['properties'][key]['type'] | ||
prop_type_info = page['properties'][key][prop_type] | ||
if isinstance(prop_type_info, dict): | ||
del page['properties'][key][prop_type]['id'] | ||
elif isinstance(prop_type_info, list) and prop_type != 'relation': | ||
for item in page['properties'][key][prop_type]: | ||
if 'id' in item: | ||
del item['id'] | ||
self.create_notion_page(page) | ||
|
||
def append_child_notion_blocks(self, block_id, children): | ||
''' | ||
Appends each datapoint of a list of notion_data as children to the block specified by id. | ||
|
@@ -409,27 +446,70 @@ def append_blocks(i1, i2, blocks, list): | |
|
||
last_i = 0 | ||
children_appended = [] | ||
parent = self.get_page_or_database(block_id) or self.get_block(block_id, None) | ||
parent_type = parent.notion_data["object"] | ||
type_is_database = lambda child: 'type' in child and child['type'] == 'child_database' | ||
object_is_database = lambda child: child['object'] == 'database' | ||
type_is_page = lambda child: 'type' in child and child['type'] == 'child_page' | ||
object_is_page = lambda child: child['object'] == 'page' | ||
for i, child in enumerate(children): | ||
if child['object'] == 'database': | ||
children_appended = append_blocks(last_i, i, children, children_appended) | ||
child_database = self.create_notion_database(child) | ||
children_appended.append(child_database) | ||
last_i = i + 1 | ||
elif child['object'] == 'page': | ||
if object_is_database(child) or type_is_database(child): | ||
children_appended = append_blocks(last_i, i, children, children_appended) | ||
child_page = self.create_notion_page(child) | ||
children_appended.append(child_page) | ||
last_i = i + 1 | ||
elif child['type'] == 'child_database': | ||
children_appended = append_blocks(last_i, i, children, children_appended) | ||
database = self.get_database(child['id']) | ||
child_database = self.create_notion_database(database.notion_data) | ||
if parent_type == 'block': | ||
logger.warning(( | ||
'Skipping database with block type parent as ' | ||
'appension is currently unsupported by Notion API' | ||
)) | ||
child_database = {} | ||
else: | ||
database = self.get_database(child['id']) | ||
child = { | ||
key: value | ||
for (key, value) in | ||
database.notion_data.items() | ||
if key not in [ | ||
'last_edited_by', | ||
'created_time', | ||
'last_edited_time', | ||
'created_by' | ||
] | ||
} | ||
child['parent'] = { | ||
'type': f'{parent_type}_id', | ||
f'{parent_type}_id': parent.notion_id | ||
} | ||
child_database = self.create_notion_database(child) | ||
if database.children: | ||
notion_children = [child.notion_data for child in database.children] | ||
self.copy_notion_database_children(notion_children, child_database) | ||
children_appended.append(child_database) | ||
last_i = i + 1 | ||
elif child['type'] == 'child_page': | ||
elif object_is_page(child) or type_is_page(child): | ||
children_appended = append_blocks(last_i, i, children, children_appended) | ||
page = self.get_page(child['id']) | ||
child_page = self.create_notion_page(page.notion_data) | ||
if parent_type == 'block': | ||
logger.warning(( | ||
'Skipping page with block type parent as ' | ||
'appension is currently unsupported by Notion API' | ||
)) | ||
child_page = {} | ||
else: | ||
page = self.get_page(child['id']) | ||
child = { | ||
key: value | ||
for (key, value) in | ||
page.notion_data.items() | ||
if key not in [ | ||
'last_edited_by', | ||
'created_time', | ||
'last_edited_time', | ||
'created_by' | ||
] | ||
} | ||
child['parent'] = { | ||
'type': f'{parent_type}_id', | ||
f'{parent_type}_id': parent.notion_id | ||
} | ||
child_page = self.create_notion_page(child) | ||
children_appended.append(child_page) | ||
last_i = i + 1 | ||
elif i == len(children) - 1: | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What was the motivation for these changes?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If the database is pulled from the cache and a filter has already been run on it, but you want it to run a different filter it won't. Thus, documents/510k was just giving me the same results as documents/dhf.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we add an intermediate variable like
loading_from_cache = filter != None
or something similar to capture why we're branching?