Skip to content

Commit

Permalink
add git ignore
Browse files Browse the repository at this point in the history
  • Loading branch information
yokonsan committed Nov 3, 2017
1 parent 2dd3727 commit fc29da4
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 58 deletions.
3 changes: 3 additions & 0 deletions .gitignore
@@ -0,0 +1,3 @@
# Python:
.idea
__pycache__
58 changes: 0 additions & 58 deletions spider.py

This file was deleted.

25 changes: 25 additions & 0 deletions utils.py
Expand Up @@ -18,3 +18,28 @@ def parse_url(url):
except ConnectionError:
print('Error.')
return None

class Downloader(object):
"""
python3.5的标准库自带的async和await指令,
相当于3.5之前的 @asyncio.coroutine和yield from
提供异步抓取
"""
def __init__(self, urls):
self.urls = urls
self._htmls = []

async def download_single_page(self, url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
self._htmls.append(await resp.text())

def download(self):
loop = asyncio.get_event_loop()
tasks = [self.download_single_page(url) for url in self.urls]
loop.run_until_complete(asyncio.wait(tasks))

@property
def htmls(self):
self.download()
return self._htmls

0 comments on commit fc29da4

Please sign in to comment.