diff --git a/dcard/api.py b/dcard/api.py index 5400d4a..b478f76 100644 --- a/dcard/api.py +++ b/dcard/api.py @@ -1,3 +1,28 @@ API_ROOT = 'http://dcard.tw/_api' FORUMS = 'forums' POSTS = 'posts' + + +forums_url = '{api_root}/{api_forums}'.format( + api_root=API_ROOT, + api_forums=FORUMS +) + +posts_meta_url_pattern = '{api_root}/{api_forums}/{{forum}}/{api_posts}'.format( + api_root=API_ROOT, + api_forums=FORUMS, + api_posts=POSTS, +) + +post_url_pattern = '{api_root}/{api_posts}/{{post_id}}'.format( + api_root=API_ROOT, + api_posts=POSTS, +) + +post_links_url_pattern = '{post_url}/links'.format( + post_url=post_url_pattern +) + +post_comments_url_pattern = '{post_url}/comments'.format( + post_url=post_url_pattern +) diff --git a/dcard/forums.py b/dcard/forums.py index 0d63b2c..20478e2 100644 --- a/dcard/forums.py +++ b/dcard/forums.py @@ -37,32 +37,17 @@ def get_metas(self, pages=1, sort='new', callback=None): @staticmethod def get(**kwargs): - url = '{api_root}/{api_forums}'.format( - api_root=api.API_ROOT, - api_forums=api.FORUMS - ) - forums = Client.get(url) - + forums = Client.get(api.forums_url) if kwargs.get('no_school'): return [forum for forum in filter_general(forums)] - return forums - @staticmethod - def build_url(forum): - url = '{api_root}/{api_forums}/{forum}/{api_posts}'.format( - api_root=api.API_ROOT, - api_forums=api.FORUMS, - api_posts=api.POSTS, - forum=forum - ) - return url - @staticmethod def get_post_metas(forum, pages, params): logger.info(u'開始取得看板 [%s] 內文章資訊' % forum) + posts_meta_url = api.posts_meta_url_pattern.format(forum=forum) for _ in range(pages): - data = Client.get(Forum.build_url(forum), params=params) + data = Client.get(posts_meta_url, params=params) try: params['before'] = data[-1]['id'] yield data diff --git a/dcard/posts.py b/dcard/posts.py index 97f2f08..10faf1a 100644 --- a/dcard/posts.py +++ b/dcard/posts.py @@ -12,7 +12,7 @@ class Post: def __init__(self, metas): ''' - params `metas`: list of article_metas/ids, or one article_meta/id, + :params `metas`: list of article_metas/ids, or one article_meta/id, article_meta must contain `id` field ''' if isinstance(metas, list): @@ -23,26 +23,8 @@ def __init__(self, metas): self.ids = ids @staticmethod - def build_url(post_id): - post_url = '{api_root}/{api_posts}/{post_id}'.format( - api_root=api.API_ROOT, - api_posts=api.POSTS, - post_id=post_id - ) - return post_url - - @staticmethod - def get_content(post_url): - return client.sget(post_url) - - @staticmethod - def get_links(post_url): - links_url = '{post_url}/links'.format(post_url=post_url) - return client.sget(links_url) - - @staticmethod - def get_comments(post_url): - comments_url = '{post_url}/comments'.format(post_url=post_url) + def get_comments(post_id): + comments_url = api.post_comments_url_pattern.format(post_id=post_id) params = {} comments = [] @@ -56,18 +38,23 @@ def get_comments(post_url): return comments def get(self, **kwargs): - post_urls = [Post.build_url(i) for i in self.ids] crawl_links = kwargs.get('links', True) crawl_content = kwargs.get('content', True) crawl_comments = kwargs.get('comments', True) if crawl_links: - links_futures = [Post.get_links(url) for url in post_urls] + links_futures = [ + client.sget(api.post_links_url_pattern.format(post_id=post_id)) + for post_id in self.ids + ] if crawl_content: - content_futures = [Post.get_content(url) for url in post_urls] + content_futures = [ + client.sget(api.post_url_pattern.format(post_id=post_id)) + for post_id in self.ids + ] - results = [{} for _ in range(len(post_urls))] + results = [{} for _ in range(len(self.ids))] if crawl_links: for i, f in enumerate(links_futures): results[i]['links'] = f.result().json() @@ -75,7 +62,7 @@ def get(self, **kwargs): for i, f in enumerate(content_futures): results[i]['content'] = f.result().json() if crawl_comments: - for i, url in enumerate(post_urls): - results[i]['comments'] = Post.get_comments(url) + for i, post_id in enumerate(self.ids): + results[i]['comments'] = Post.get_comments(post_id) return results[0] if len(results) == 1 else results diff --git a/dcard/utils.py b/dcard/utils.py index 958b479..1857ac3 100644 --- a/dcard/utils.py +++ b/dcard/utils.py @@ -1,5 +1,3 @@ -import logging - import requests from requests_futures.sessions import FuturesSession diff --git a/tests/conftest.py b/tests/conftest.py index b2b54ea..d215dd8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,5 +14,5 @@ def forums(): @pytest.fixture(scope='module') -def article_url(): - return Dcard.posts.build_url(224341009) +def article_id(): + return 224341009 diff --git a/tests/test_dcard.py b/tests/test_dcard.py index f2b25c8..68f5869 100644 --- a/tests/test_dcard.py +++ b/tests/test_dcard.py @@ -1,4 +1,5 @@ import requests +from dcard import api from dcard import Dcard @@ -9,17 +10,20 @@ def test_valid_api_forums(forums): def test_valid_api_forum(forums): forum = forums.get('test')['alias'] - url = Dcard.forums.build_url(forum) + url = api.posts_meta_url_pattern.format(forum=forum) assert requests.get(url).ok -def test_valid_api_article(article_url): - assert requests.get(article_url).ok +def test_valid_api_article(article_id): + url = api.post_url_pattern.format(post_id=article_id) + assert requests.get(url).ok -def test_valid_api_article_links(article_url): - assert Dcard.posts.get_links(article_url).result().ok +def test_valid_api_article_links(article_id): + url = api.post_links_url_pattern.format(post_id=article_id) + assert requests.get(url).ok -def test_valid_api_article_comments(article_url): - assert Dcard.posts.get_comments(article_url) +def test_valid_api_article_comments(article_id): + url = api.post_comments_url_pattern.format(post_id=article_id) + assert requests.get(url).ok