Skip to content

Commit

Permalink
Update #749: follow last modified config, add db check and insert
Browse files Browse the repository at this point in the history
  • Loading branch information
@Nandaka
Nandaka committed Aug 1, 2020
1 parent ab5677e commit d330e56
Show file tree
Hide file tree
Showing 4 changed files with 154 additions and 31 deletions.
2 changes: 1 addition & 1 deletion PixivBrowserFactory.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -963,7 +963,7 @@ def fanboxGetPostJsonById(self, post_id, artist=None):

def sketch_get_post_by_post_id(self, post_id, artist=None):
# https://sketch.pixiv.net/api/replies/1213195054130835383.json
url = f"https://sketch.pixiv.net/api/replies/{post_id}"
url = f"https://sketch.pixiv.net/api/replies/{post_id}.json"
referer = f"https://sketch.pixiv.net/items/{post_id}"
PixivHelper.get_logger().debug('Getting sketch post detail from %s', url)
response = self.getPixivPage(url, returnParsed=False, enable_cache=True, referer=referer)
Expand Down
80 changes: 77 additions & 3 deletions PixivDBManager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ def createDatabase(self):
post_type TEXT,
last_update_date DATE
)''')
self.conn.commit()

c.execute('''CREATE TABLE IF NOT EXISTS fanbox_post_image (
post_id INTEGER,
Expand All @@ -108,6 +107,25 @@ def createDatabase(self):
)''')
self.conn.commit()

c.execute('''CREATE TABLE IF NOT EXISTS sketch_master_post (
member_id INTEGER,
post_id INTEGER PRIMARY KEY ON CONFLICT IGNORE,
title TEXT,
published_date DATE,
updated_date DATE,
post_type TEXT,
last_update_date DATE
)''')
c.execute('''CREATE TABLE IF NOT EXISTS sketch_post_image (
post_id INTEGER,
page INTEGER,
save_name TEXT,
created_date DATE,
last_update_date DATE,
PRIMARY KEY (post_id, page)
)''')
self.conn.commit()

print('done.')
except BaseException:
print('Error at createDatabase():', str(sys.exc_info()))
Expand All @@ -129,9 +147,11 @@ def dropDatabase(self):
self.conn.commit()

c.execute('''DROP TABLE IF EXISTS fanbox_master_post''')
c.execute('''DROP TABLE IF EXISTS fanbox_post_image''')
self.conn.commit()

c.execute('''DROP TABLE IF EXISTS fanbox_post_image''')
c.execute('''DROP TABLE IF EXISTS sketch_master_post''')
c.execute('''DROP TABLE IF EXISTS sketch_post_image''')
self.conn.commit()

except BaseException:
Expand Down Expand Up @@ -982,7 +1002,61 @@ def interactiveCleanUpFanbox(self):
c.close()

##########################################
# VII. Utilities #
# VII. CRUD Sketch post/image table #
##########################################

def insertSketchPost(self, post):
try:
c = self.conn.cursor()
post_id = int(post.imageId)
c.execute('''INSERT OR IGNORE INTO sketch_master_post (member_id, post_id) VALUES(?, ?)''',
(post.artist.artistId, post_id))
c.execute('''UPDATE sketch_master_post
SET title = ?,
published_date = ?,
post_type = ?,
last_update_date = ?
WHERE post_id = ?''',
(post.imageTitle, post.worksDateDateTime, post.imageMode, post.worksUpdateDateTime, post_id))
self.conn.commit()
except BaseException:
print('Error at insertSketchPost():', str(sys.exc_info()))
print('failed')
raise
finally:
c.close()

def insertSketchPostImages(self, post_id, page, save_name, created_date, last_update_date):
try:
c = self.conn.cursor()
c.execute('''INSERT OR REPLACE INTO sketch_post_image
VALUES(?, ?, ?, ?, ?)''',
(post_id, page, save_name, created_date, last_update_date))
self.conn.commit()
except BaseException:
print('Error at insertSketchPostImages():', str(sys.exc_info()))
print('failed')
raise
finally:
c.close()

def selectSketchPostByPostId(self, post_id):
try:
c = self.conn.cursor()
post_id = int(post_id)
c.execute(
'''SELECT * FROM sketch_master_post WHERE post_id = ?''',
(post_id,))
return c.fetchone()
except BaseException:
print('Error at selectSketchPostByPostId():', str(sys.exc_info()))
print('failed')
raise
finally:
c.close()

##########################################
# VIII. Utilities #
##########################################

def menu(self):
Expand Down
9 changes: 8 additions & 1 deletion PixivModelSketch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,12 @@ class SketchPost(object):
tags = None
imageUrls = []
imageResizedUrls = []
# so far only photo is supported
imageMode = ""
worksDate = ""
worksDateDateTime = None
worksUpdateDate = ""
worksUpdateDateTime = None

artist = None
dateFormat = None
Expand Down Expand Up @@ -116,12 +119,16 @@ def parse_post(self, page):
self.imageUrls.append(media["photo"]["original"]["url"])
self.imageResizedUrls.append(media["photo"]["w540"]["url"])

self.worksDateDateTime = datetime_z.parse_datetime(str(page["updated_at"]))
self.worksDateDateTime = datetime_z.parse_datetime(str(page["published_at"]))
if self._tzInfo is not None:
self.worksDateDateTime = self.worksDateDateTime.astimezone(self._tzInfo)
self.worksUpdateDateTime = datetime_z.parse_datetime(str(page["updated_at"]))
if self._tzInfo is not None:
self.worksUpdateDateTime = self.worksUpdateDateTime.astimezone(self._tzInfo)

tempDateFormat = self.dateFormat or "%m/%d/%y %H:%M" # 2/27/2018 12:31
self.worksDate = self.worksDateDateTime.strftime(tempDateFormat)
self.worksUpdateDate = self.worksUpdateDateTime.strftime(tempDateFormat)

def __str__(self):
if self.artist is not None:
Expand Down
94 changes: 68 additions & 26 deletions PixivSketchHandler.py
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,33 @@
# -*- coding: utf-8 -*-
import sys
import traceback

from colorama import Fore, Style

import PixivBrowserFactory
import PixivConstant
import PixivHelper
import PixivDownloadHandler


def process_sketch_post(caller, config, post_id):
# db = caller.__dbManager__
config.loadConfig(path=caller.configfile)
br = PixivBrowserFactory.getBrowser()

msg = Fore.YELLOW + Style.NORMAL + f'Processing Post Id: {post_id}' + Style.RESET_ALL
PixivHelper.print_and_log(None, msg)
post = br.sketch_get_post_by_post_id(post_id)
download_post(caller, config, post)

try:
post = br.sketch_get_post_by_post_id(post_id)
download_post(caller, config, post)
except Exception as ex:
if isinstance(ex, KeyboardInterrupt):
raise
caller.ERROR_CODE = getattr(ex, 'errorCode', -1)
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
PixivHelper.print_and_log('error', f'Error at process_sketch_post(): {post_id}')
PixivHelper.print_and_log('error', f'Exception: {sys.exc_info()}')


def process_sketch_artists(caller, config, artist_id, start_page=0, end_page=0):
Expand All @@ -24,28 +37,47 @@ def process_sketch_artists(caller, config, artist_id, start_page=0, end_page=0):
caller.set_console_title(title_prefix)
msg = Fore.YELLOW + Style.NORMAL + f'Processing Artist Id: {artist_id}' + Style.RESET_ALL
PixivHelper.print_and_log(None, msg)
artist = br.sketch_get_posts_by_artist_id(artist_id, end_page)

POST_PER_PAGE = 10
start_idx = POST_PER_PAGE * (start_page - 1)
end_idx = POST_PER_PAGE * (end_page)
if end_page == 0 or end_idx > len(artist.posts):
end_idx = len(artist.posts)
msg = Fore.YELLOW + Style.NORMAL + f'Processing from post #{start_idx} to #{end_idx}' + Style.RESET_ALL
PixivHelper.print_and_log(None, msg)
post_to_process = artist.posts[start_idx:end_idx]

current_post = 1
for item in post_to_process:
caller.set_console_title(f"{title_prefix} - Post {current_post} of {len(post_to_process)}")
PixivHelper.print_and_log(None, f'Post #: {current_post}')
PixivHelper.print_and_log('info', f'Post ID : {item.imageId}')
download_post(caller, config, item)
current_post = current_post + 1
try:
artist = br.sketch_get_posts_by_artist_id(artist_id, end_page)

POST_PER_PAGE = 10
start_idx = POST_PER_PAGE * (start_page - 1)
end_idx = POST_PER_PAGE * (end_page)
if end_page == 0 or end_idx > len(artist.posts):
end_idx = len(artist.posts)
msg = Fore.YELLOW + Style.NORMAL + f'Processing from post #{start_idx} to #{end_idx}' + Style.RESET_ALL
PixivHelper.print_and_log(None, msg)
post_to_process = artist.posts[start_idx:end_idx]

current_post = 1
for item in post_to_process:
caller.set_console_title(f"{title_prefix} - Post {current_post} of {len(post_to_process)}")
PixivHelper.print_and_log(None, f'Post #: {current_post}')
PixivHelper.print_and_log('info', f'Post ID : {item.imageId}')
download_post(caller, config, item)
current_post = current_post + 1
except Exception as ex:
if isinstance(ex, KeyboardInterrupt):
raise
caller.ERROR_CODE = getattr(ex, 'errorCode', -1)
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
PixivHelper.print_and_log('error', f'Error at process_sketch_artists(): {artist_id}')
PixivHelper.print_and_log('error', f'Exception: {sys.exc_info()}')


def download_post(caller, config, post):
db = caller.__dbManager__

if config.checkDBProcessHistory and not config.overwrite and not config.alwaysCheckFileSize:
result = db.selectSketchPostByPostId(post.imageId)
if result:
msg = Fore.YELLOW + Style.NORMAL + f'Skipping Post: {post.imageId} because already exists in DB and overwrite and alwaysCheckFileSize are disabled. .' + Style.RESET_ALL
PixivHelper.print_and_log(None, msg)

referer = f"https://sketch.pixiv.net/items/{post.imageId}"
current_page = 0
for url in post.imageUrls:
filename = PixivHelper.make_filename(config.filenameFormat,
post,
Expand All @@ -62,9 +94,19 @@ def download_post(caller, config, post):
PixivHelper.print_and_log(None, f'Image URL : {url}')
PixivHelper.print_and_log('info', f'Filename : {filename}')
(result, filename) = PixivDownloadHandler.download_image(caller,
url,
filename,
referer,
config.overwrite,
config.retry,
config.backupOldFile)
url,
filename,
referer,
config.overwrite,
config.retry,
config.backupOldFile,
image=post)
if result == PixivConstant.PIXIVUTIL_OK:
db.insertSketchPost(post)
db.insertSketchPostImages(post.imageId,
current_page,
filename,
post.worksDateDateTime,
post.worksUpdateDateTime)

current_page = current_page + 1

0 comments on commit d330e56

Please sign in to comment.