tumblr.py

# -*- coding=utf-8 -*-
import grequests
import re
import os
import sys
from time import clock
from app import db
from app.models import Context

# search for url of maxium size of a picture, which starts with '<photo-url max-width="1280">' and ends with '</photo-url>'
extractpicre = re.compile(
    r'(?<=<photo-url max-width="1280">).+?(?=</photo-url>)', flags=re.S)
extractvideore = re.compile(
    '''poster='(.*?)'[\w\W]*?/tumblr_(.*?)" type="video/mp4"''')

video_links = []
pic_links = []
vhead = 'https://vt.tumblr.com/tumblr_%s.mp4'
api_url = 'http://%s.tumblr.com/api/read?&num=50&start='
query_urls = []


def getpost(uid, query_urls):
    import requests
    url = 'http://%s.tumblr.com/api/read?&num=50' % uid
    r = requests.get(url)
    total = re.findall('<posts start="0" total="(.*?)">', r.content)[0]
    total = int(total)
    print uid + ':' + str(total)
    a = [i * 50 for i in range(total / 50 + 1)]
    ul = api_url % uid
    for i in a:
        query_url = ul + str(i)
        query_urls.append(query_url)


def run(query_urls):
    rs = [grequests.get(url) for url in query_urls]
    responses = grequests.map(rs, size=10)
    for resp in responses:
        content = resp.content
        videos = extractvideore.findall(content)
        video_links.extend([(v[0], vhead % v[1]) for v in videos])
        pic_links.extend(extractpicre.findall(content))


def write(name):
    videos = [(i[0], i[1].replace('/480', '')) for i in video_links]
    pictures = pic_links
    for url in list(set(videos)):
        poster, video = url
        data = Context.query.filter_by(id=name, urls=video).first()
        if not data:
            data = Context(id=name, urls=video, isvideo=1, poster=poster)
            db.session.add(data)
        else:
            data = Context.query.filter_by(id=name, urls=video).first()
            data.poster = poster
            db.session.add(data)
    for url in list(set(pictures)):
        dat = Context.query.filter_by(id=name, urls=url).first()
        if not dat:
            data = Context(id=name, urls=url, isvideo=0, poster=url)
            db.session.add(data)
        else:
            data = Context.query.filter_by(id=name, urls=url).first()
            data.poster = url
            data.urls = url
            db.session.add(data)
    db.session.commit()


def TumblrGet(name):
    now = clock()
    getpost(name, query_urls)
    print len(query_urls)
    parts = len(query_urls) / 50 + 1
    print parts
    for part in range(parts):
        urls = query_urls[part:(part + 1) * 50]
        run(urls)
        write(name)
    print "%sparse complete, cose %.1fs" % (name, clock() - now)
    print "pictures %d,videos %d" % (len(pic_links), len(video_links))


if __name__ == '__main__':
    name = sys.argv[1]
    name = name.strip()
    # name=raw_input()
    # now=clock()
    TumblrGet(name)
    # print u"图片%d张，视频%d部"%(len(pic_links),len(video_links))