From 3ef32ecc8d01350cb1754cc2ac274bc1bba58acc Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 29 Nov 2017 17:07:39 +0000 Subject: [PATCH] make all log strings unicode --- ChangeLog | 7 +++++ gphotos/GoogleDriveSync.py | 28 +++++++++---------- gphotos/PicasaSync.py | 55 +++++++++++++++++++------------------- gphotos/Utils.py | 10 +++---- 4 files changed, 54 insertions(+), 46 deletions(-) diff --git a/ChangeLog b/ChangeLog index f1efe856..d704d10b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,13 @@ CHANGES ======= +* add log of version number +* handle clean up of db on kill +* mods to assist debugging + +1.0.2 +----- + * let git control version number * enable removal of items from albums on --flush-index * add --db-path sp db can go on the fastest disk (#13) diff --git a/gphotos/GoogleDriveSync.py b/gphotos/GoogleDriveSync.py index 7724fe7c..9ae34bd4 100644 --- a/gphotos/GoogleDriveSync.py +++ b/gphotos/GoogleDriveSync.py @@ -64,9 +64,9 @@ def __init__(self, root_folder, db, else: self._g_auth.LocalWebserverAuth() except InvalidConfigError: - log.error("No client secrets file found.\nPlease see " - "https://github.com/gilesknap/gphotos-sync#install" - "-and-configure") + log.error(u"No client secrets file found.\nPlease see " + u"https://github.com/gilesknap/gphotos-sync#install" + u"-and-configure") exit(1) self._googleDrive = GoogleDrive(self._g_auth) @@ -88,7 +88,7 @@ def latest_download(self): return self._latest_download def scan_folder_hierarchy(self): - log.info('Indexing Drive Folders ...') + log.info(u'Indexing Drive Folders ...') # get the root id # is this really the only way?- Find all items with root as parent # then ask the first of these for its parent id !!?? @@ -119,14 +119,14 @@ def scan_folder_hierarchy(self): parent_id = None self._db.put_drive_folder(drive_file['id'], parent_id, drive_file['title']) - log.info('Resolving paths ...') + log.info(u'Resolving paths ...') self.recurse_paths('', root_id) if len(self.folderPaths) == 1: raise ValueError( "No folders found. Please enable Google Photos in Google " "drive (see https://support.google.com/photos/answer/6156103" "). Or use one of the options --all-drive --skip-drive.") - log.info('Drive Folders scanned.') + log.info(u'Drive Folders scanned.') def recurse_paths(self, path, folder_id): self.folderPaths[folder_id] = path @@ -140,7 +140,7 @@ def recurse_paths(self, path, folder_id): def check_for_removed(self): # note for partial scans using date filters this is still OK because # for a file to exist it must have been indexed in a previous scan - log.info('Finding deleted media ...') + log.info(u'Finding deleted media ...') top_dir = os.path.join(self._root_folder, GoogleDriveMedia.MEDIA_FOLDER) for (dir_name, _, file_names) in os.walk(top_dir): for file_name in file_names: @@ -169,7 +169,7 @@ def write_media(self, media, update=True): u"createdTime <= '{0}T00:00:00') " def index_drive_media(self): - log.info('Indexing Drive Files ...') + log.info(u'Indexing Drive Files ...') if self.includeVideo: q = self.VIDEO_QUERY @@ -210,12 +210,12 @@ def index_drive_media(self): row = media.is_indexed(self._db) if not row: n += 1 - log.info("Added %d %s", n, media.local_full_path) + log.info(u"Added %d %s", n, media.local_full_path) self.write_media(media, False) if n % 1000 == 0: self._db.store() elif media.modify_date > row.ModifyDate: - log.info("Updated %d %s", n, media.local_full_path) + log.info(u"Updated %d %s", n, media.local_full_path) self.write_media(media, True) finally: # store latest date for incremental backup only if scanning all @@ -223,7 +223,7 @@ def index_drive_media(self): self._db.set_scan_dates(drive_last_date=self._latest_download) def download_drive_media(self): - log.info('Downloading Drive Files ...') + log.info(u'Downloading Drive Files ...') # noinspection PyTypeChecker for media in DatabaseMedia.get_media_by_search( self._root_folder, self._db, media_type=MediaType.DRIVE, @@ -231,7 +231,7 @@ def download_drive_media(self): if os.path.exists(media.local_full_path): if Utils.to_timestamp(media.modify_date) > \ os.path.getctime(media.local_full_path): - log.warning('{} was modified'.format( + log.warning(u'{} was modified'.format( media.local_full_path)) else: continue @@ -240,7 +240,7 @@ def download_drive_media(self): os.makedirs(media.local_folder) temp_filename = os.path.join(self._root_folder, '.temp-photo') - log.info('downloading {} ...'.format(media.local_full_path)) + log.info(u'downloading {} ...'.format(media.local_full_path)) f = self._googleDrive.CreateFile({'id': media.id}) try: Utils.retry(10, f.GetContentFile, temp_filename) @@ -253,5 +253,5 @@ def download_drive_media(self): (Utils.to_timestamp(media.modify_date), Utils.to_timestamp(media.create_date))) except ApiRequestError: - log.error('DOWNLOAD FAILURE for {}'.format( + log.error(u'DOWNLOAD FAILURE for {}'.format( media.local_full_path)) diff --git a/gphotos/PicasaSync.py b/gphotos/PicasaSync.py index 1ef31333..bcdb09ff 100644 --- a/gphotos/PicasaSync.py +++ b/gphotos/PicasaSync.py @@ -61,7 +61,7 @@ def __init__(self, credentials, root_folder, db, flush_albums): self.includeVideo = False def download_picasa_media(self): - log.info('Downloading Picasa Only Files ...') + log.info(u'Downloading Picasa Only Files ...') # noinspection PyTypeChecker for media in DatabaseMedia.get_media_by_search( self._root_folder, self._db, media_type=MediaType.PICASA, @@ -70,7 +70,7 @@ def download_picasa_media(self): if os.path.exists(media.local_full_path): continue - log.info("Downloading %s ..." % media.local_full_path) + log.info(u"Downloading %s ..." % media.local_full_path) tmp_path = os.path.join(media.local_folder, '.gphotos.tmp') if not os.path.isdir(media.local_folder): @@ -85,15 +85,15 @@ def download_picasa_media(self): (Utils.to_timestamp(media.modify_date), Utils.to_timestamp(media.create_date))) else: - log.warning("WARNING: failed to download %s", media.local_path) + log.warning(u"WARNING: failed to download %s", media.local_path) def create_album_content_links(self): - log.info("Creating album folder links to media ...") + log.info(u"Creating album folder links to media ...") # the simplest way to handle moves or deletes is to clear out all links # first, these are quickly recreated anyway links_root = os.path.join(self._root_folder, 'albums') if os.path.exists(links_root): - log.debug('removing previous album links tree') + log.debug(u'removing previous album links tree') shutil.rmtree(links_root) for (path, file_name, album_name, end_date) in \ @@ -111,22 +111,22 @@ def create_album_content_links(self): link_file = unicode(os.path.join(link_folder, file_name)) if os.path.exists(link_file): - log.error("ERROR: Name clash on link %s", link_file) + log.error(u"ERROR: Name clash on link %s", link_file) else: - log.debug('adding album link %s -> %s', full_file_name, + log.debug(u'adding album link %s -> %s', full_file_name, link_file) if not os.path.isdir(link_folder): - log.debug('new album folder %s', link_folder) + log.debug(u'new album folder %s', link_folder) os.makedirs(link_folder) os.symlink(full_file_name, link_file) - log.info("album links done.") + log.info(u"album links done.") # this will currently do nothing unless using --flush-db def check_for_removed(self): # note for partial scans using date filters this is still OK because # for a file to exist it must have been indexed in a previous scan - log.info('Finding deleted media ...') + log.info(u'Finding deleted media ...') top_dir = os.path.join(self._root_folder, PicasaMedia.MEDIA_FOLDER) for (dir_name, _, file_names) in os.walk(top_dir): for file_name in file_names: @@ -134,7 +134,7 @@ def check_for_removed(self): if not file_id: name = os.path.join(dir_name, file_name) os.remove(name) - log.warning("%s deleted", name) + log.warning(u"%s deleted", name) def match_drive_photo(self, media): sync_row = self._db.find_file_ids_dates(size=media.size, @@ -155,9 +155,10 @@ def match_drive_photo(self, media): if sync_row: return sync_row[0:1] + sync_row = self._match_by_date(media) if sync_row: - log.warning('MATCH BY DATE on %s %s', media.filename, + log.warning(u'MATCH BY DATE on %s %s', media.filename, media.modify_date) return sync_row @@ -202,12 +203,12 @@ def index_album_media(self, limit=None): contents into the db :param (int) limit: only scan this number of albums (for testing) """ - log.info('Indexing Albums ...') + log.info(u'Indexing Albums ...') albums = Utils.retry(10, self._gdata_client.GetUserFeed, limit=limit) - log.info('Album count %d', len(albums.entry)) + log.info(u'Album count %d', len(albums.entry)) if self.flush_albums: - log.debug("removing all album links from db (--flush-index)") + log.debug(u"removing all album links from db (--flush-index)") self._db.remove_all_album_files() helper = IndexAlbumHelper(self) @@ -217,7 +218,7 @@ def index_album_media(self, limit=None): helper.setup_next_album(album) if helper.skip_this_album(): continue - log.info('Album: %s, photos: %d, updated: %s, published: %s', + log.info(u'Album: %s, photos: %d, updated: %s, published: %s', album.filename, album.size, album.modify_date, album.create_date) @@ -236,7 +237,7 @@ def index_album_media(self, limit=None): if len(photos.entry) < limit: break if start_entry >= PicasaSync.ALBUM_MAX: - log.warning("LIMITING ALBUM TO %d entries", + log.warning(u"LIMITING ALBUM TO %d entries", PicasaSync.ALBUM_MAX) break if start_entry + PicasaSync.BLOCK_SIZE > PicasaSync.ALBUM_MAX: @@ -245,7 +246,7 @@ def index_album_media(self, limit=None): helper.complete_album() helper.complete_scan() - log.info('Total Album Photos in Drive %d, Picasa %d, multiples %d', + log.info(u'Total Album Photos in Drive %d, Picasa %d, multiples %d', helper.total_photos, helper.picasa_photos, helper.multiple_match_count) @@ -333,26 +334,26 @@ def index_photos(self, photos): continue self.set_album_dates(picasa_media.create_date) - log.debug('checking %s is indexed', picasa_media.local_full_path) + log.debug(u'checking %s is indexed', picasa_media.local_full_path) picasa_row = picasa_media.is_indexed(self.p._db) if picasa_row: if picasa_media.modify_date > picasa_row.ModifyDate: - log.info("Updated index for %s", + log.info(u"Updated index for %s", picasa_media.local_full_path) picasa_row_id = picasa_media.save_to_db(self.p._db, update=True) else: picasa_row_id = picasa_row.Id - log.debug("Skipped %s", picasa_media.local_full_path) + log.debug(u"Skipped %s", picasa_media.local_full_path) else: self.picasa_photos += 1 - log.info("Adding index for %d %s", self.picasa_photos, + log.info(u"Adding index for %d %s", self.picasa_photos, picasa_media.local_full_path) picasa_row_id = picasa_media.save_to_db(self.p._db) if self.picasa_photos % 1000 == 0: self.p._db.store() - log.debug('searching for drive match on {}'.format( + log.debug(u'searching for drive match on {}'.format( picasa_media.filename)) drive_rows = self.p.match_drive_photo(picasa_media) count, row = (len(drive_rows), drive_rows[0]) if drive_rows \ @@ -368,12 +369,12 @@ def index_photos(self, photos): # comparison restored if count == 0: # no match, link to the picasa file - log.info('unmatched %s will be downloaded', + log.info(u'unmatched %s will be downloaded', picasa_media.local_full_path) self.p._db.put_album_file(self.album.id, picasa_row_id) else: # store link between album and drive file - log.debug('matched to %s', drive_rows[0].Path) + log.debug(u'matched to %s', drive_rows[0].Path) self.p._db.put_album_file(self.album.id, drive_rows[0].Id) # store the link between picasa and related drive file # this also flags it as not requiring download @@ -381,7 +382,7 @@ def index_photos(self, photos): if count > 1: self.multiple_match_count += 1 - log.warning('WARNING multiple files match %s %s %s', + log.warning(u'WARNING multiple files match %s %s %s', picasa_media.orig_name, picasa_media.modify_date, picasa_media.size) @@ -394,7 +395,7 @@ def complete_album(self): EndDate=self.album_end_photo, SyncDate=Utils.date_to_string( datetime.now())) - log.debug('completed album %s', self.album.filename) + log.debug(u'completed album %s', self.album.filename) self.p._db.put_album(row) if self.album.modify_date > self.latest_download: self.latest_download = self.album.modify_date diff --git a/gphotos/Utils.py b/gphotos/Utils.py index 34c213fb..dc50d72a 100644 --- a/gphotos/Utils.py +++ b/gphotos/Utils.py @@ -55,8 +55,8 @@ def retry(count, func, *arg, **k_arg): return res except Exception as e: last_e = e - log.warning("RETRYING due to: %s", repr(e)) - log.warning("Call was: %s (%s, %s)", repr(func), arg, k_arg) + log.warning(u"RETRYING due to: %s", repr(e)) + log.warning(u"Call was: %s (%s, %s)", repr(func), arg, k_arg) time.sleep(.1) raise last_e @@ -79,7 +79,7 @@ def retry_i(count, iterator): more_data = False break except Exception as e: - log.warning("RETRYING iterator due to: %s", repr(e)) + log.warning(u"RETRYING iterator due to: %s", repr(e)) time.sleep(.1) yield last_item @@ -133,7 +133,7 @@ def string_to_date(date_string): if m: normalized = '{}-{}-{} 00:00:00'.format(*m.groups()) else: - log.warning('WARNING: time string {} illegal', date_string) + log.warning(u'WARNING: time string {} illegal', date_string) return minimum_date() return datetime.strptime(normalized, DATE_FORMAT) @@ -144,7 +144,7 @@ def timestamp_to_date(time_secs, hour_offset=0): date = datetime.fromtimestamp( int(time_secs) / 1000 + 3600 * hour_offset) except ValueError: - log.warning('WARNING: time stamp %d illegal', time_secs) + log.warning(u'WARNING: time stamp %d illegal', time_secs) date = minimum_date() return date