Skip to content

Commit

Permalink
make all log strings unicode
Browse files Browse the repository at this point in the history
  • Loading branch information
@gilesknap
gilesknap committed Nov 29, 2017
1 parent 4bf1f6f commit 3ef32ec
Show file tree
Hide file tree
Showing 4 changed files with 54 additions and 46 deletions.
7 changes: 7 additions & 0 deletions ChangeLog
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
CHANGES
=======

* add log of version number
* handle clean up of db on kill
* mods to assist debugging

1.0.2
-----

* let git control version number
* enable removal of items from albums on --flush-index
* add --db-path sp db can go on the fastest disk (#13)
Expand Down
28 changes: 14 additions & 14 deletions gphotos/GoogleDriveSync.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,9 @@ def __init__(self, root_folder, db,
else:
self._g_auth.LocalWebserverAuth()
except InvalidConfigError:
log.error("No client secrets file found.\nPlease see "
"https://github.com/gilesknap/gphotos-sync#install"
"-and-configure")
log.error(u"No client secrets file found.\nPlease see "
u"https://github.com/gilesknap/gphotos-sync#install"
u"-and-configure")
exit(1)

self._googleDrive = GoogleDrive(self._g_auth)
Expand All @@ -88,7 +88,7 @@ def latest_download(self):
return self._latest_download

def scan_folder_hierarchy(self):
log.info('Indexing Drive Folders ...')
log.info(u'Indexing Drive Folders ...')
# get the root id
# is this really the only way?- Find all items with root as parent
# then ask the first of these for its parent id !!??
Expand Down Expand Up @@ -119,14 +119,14 @@ def scan_folder_hierarchy(self):
parent_id = None
self._db.put_drive_folder(drive_file['id'], parent_id,
drive_file['title'])
log.info('Resolving paths ...')
log.info(u'Resolving paths ...')
self.recurse_paths('', root_id)
if len(self.folderPaths) == 1:
raise ValueError(
"No folders found. Please enable Google Photos in Google "
"drive (see https://support.google.com/photos/answer/6156103"
"). Or use one of the options --all-drive --skip-drive.")
log.info('Drive Folders scanned.')
log.info(u'Drive Folders scanned.')

def recurse_paths(self, path, folder_id):
self.folderPaths[folder_id] = path
Expand All @@ -140,7 +140,7 @@ def recurse_paths(self, path, folder_id):
def check_for_removed(self):
# note for partial scans using date filters this is still OK because
# for a file to exist it must have been indexed in a previous scan
log.info('Finding deleted media ...')
log.info(u'Finding deleted media ...')
top_dir = os.path.join(self._root_folder, GoogleDriveMedia.MEDIA_FOLDER)
for (dir_name, _, file_names) in os.walk(top_dir):
for file_name in file_names:
Expand Down Expand Up @@ -169,7 +169,7 @@ def write_media(self, media, update=True):
u"createdTime <= '{0}T00:00:00') "

def index_drive_media(self):
log.info('Indexing Drive Files ...')
log.info(u'Indexing Drive Files ...')

if self.includeVideo:
q = self.VIDEO_QUERY
Expand Down Expand Up @@ -210,28 +210,28 @@ def index_drive_media(self):
row = media.is_indexed(self._db)
if not row:
n += 1
log.info("Added %d %s", n, media.local_full_path)
log.info(u"Added %d %s", n, media.local_full_path)
self.write_media(media, False)
if n % 1000 == 0:
self._db.store()
elif media.modify_date > row.ModifyDate:
log.info("Updated %d %s", n, media.local_full_path)
log.info(u"Updated %d %s", n, media.local_full_path)
self.write_media(media, True)
finally:
# store latest date for incremental backup only if scanning all
if not (self.driveFileName or self.startDate):
self._db.set_scan_dates(drive_last_date=self._latest_download)

def download_drive_media(self):
log.info('Downloading Drive Files ...')
log.info(u'Downloading Drive Files ...')
# noinspection PyTypeChecker
for media in DatabaseMedia.get_media_by_search(
self._root_folder, self._db, media_type=MediaType.DRIVE,
start_date=self.startDate, end_date=self.endDate):
if os.path.exists(media.local_full_path):
if Utils.to_timestamp(media.modify_date) > \
os.path.getctime(media.local_full_path):
log.warning('{} was modified'.format(
log.warning(u'{} was modified'.format(
media.local_full_path))
else:
continue
Expand All @@ -240,7 +240,7 @@ def download_drive_media(self):
os.makedirs(media.local_folder)
temp_filename = os.path.join(self._root_folder, '.temp-photo')

log.info('downloading {} ...'.format(media.local_full_path))
log.info(u'downloading {} ...'.format(media.local_full_path))
f = self._googleDrive.CreateFile({'id': media.id})
try:
Utils.retry(10, f.GetContentFile, temp_filename)
Expand All @@ -253,5 +253,5 @@ def download_drive_media(self):
(Utils.to_timestamp(media.modify_date),
Utils.to_timestamp(media.create_date)))
except ApiRequestError:
log.error('DOWNLOAD FAILURE for {}'.format(
log.error(u'DOWNLOAD FAILURE for {}'.format(
media.local_full_path))
55 changes: 28 additions & 27 deletions gphotos/PicasaSync.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def __init__(self, credentials, root_folder, db, flush_albums):
self.includeVideo = False

def download_picasa_media(self):
log.info('Downloading Picasa Only Files ...')
log.info(u'Downloading Picasa Only Files ...')
# noinspection PyTypeChecker
for media in DatabaseMedia.get_media_by_search(
self._root_folder, self._db, media_type=MediaType.PICASA,
Expand All @@ -70,7 +70,7 @@ def download_picasa_media(self):
if os.path.exists(media.local_full_path):
continue

log.info("Downloading %s ..." % media.local_full_path)
log.info(u"Downloading %s ..." % media.local_full_path)
tmp_path = os.path.join(media.local_folder, '.gphotos.tmp')

if not os.path.isdir(media.local_folder):
Expand All @@ -85,15 +85,15 @@ def download_picasa_media(self):
(Utils.to_timestamp(media.modify_date),
Utils.to_timestamp(media.create_date)))
else:
log.warning("WARNING: failed to download %s", media.local_path)
log.warning(u"WARNING: failed to download %s", media.local_path)

def create_album_content_links(self):
log.info("Creating album folder links to media ...")
log.info(u"Creating album folder links to media ...")
# the simplest way to handle moves or deletes is to clear out all links
# first, these are quickly recreated anyway
links_root = os.path.join(self._root_folder, 'albums')
if os.path.exists(links_root):
log.debug('removing previous album links tree')
log.debug(u'removing previous album links tree')
shutil.rmtree(links_root)

for (path, file_name, album_name, end_date) in \
Expand All @@ -111,30 +111,30 @@ def create_album_content_links(self):
link_file = unicode(os.path.join(link_folder, file_name))

if os.path.exists(link_file):
log.error("ERROR: Name clash on link %s", link_file)
log.error(u"ERROR: Name clash on link %s", link_file)
else:
log.debug('adding album link %s -> %s', full_file_name,
log.debug(u'adding album link %s -> %s', full_file_name,
link_file)
if not os.path.isdir(link_folder):
log.debug('new album folder %s', link_folder)
log.debug(u'new album folder %s', link_folder)
os.makedirs(link_folder)
os.symlink(full_file_name, link_file)

log.info("album links done.")
log.info(u"album links done.")

# this will currently do nothing unless using --flush-db
def check_for_removed(self):
# note for partial scans using date filters this is still OK because
# for a file to exist it must have been indexed in a previous scan
log.info('Finding deleted media ...')
log.info(u'Finding deleted media ...')
top_dir = os.path.join(self._root_folder, PicasaMedia.MEDIA_FOLDER)
for (dir_name, _, file_names) in os.walk(top_dir):
for file_name in file_names:
file_id = self._db.get_file_by_path(dir_name, file_name)
if not file_id:
name = os.path.join(dir_name, file_name)
os.remove(name)
log.warning("%s deleted", name)
log.warning(u"%s deleted", name)

def match_drive_photo(self, media):
sync_row = self._db.find_file_ids_dates(size=media.size,
Expand All @@ -155,9 +155,10 @@ def match_drive_photo(self, media):
if sync_row:
return sync_row[0:1]


sync_row = self._match_by_date(media)
if sync_row:
log.warning('MATCH BY DATE on %s %s', media.filename,
log.warning(u'MATCH BY DATE on %s %s', media.filename,
media.modify_date)
return sync_row

Expand Down Expand Up @@ -202,12 +203,12 @@ def index_album_media(self, limit=None):
contents into the db
:param (int) limit: only scan this number of albums (for testing)
"""
log.info('Indexing Albums ...')
log.info(u'Indexing Albums ...')
albums = Utils.retry(10, self._gdata_client.GetUserFeed, limit=limit)
log.info('Album count %d', len(albums.entry))
log.info(u'Album count %d', len(albums.entry))

if self.flush_albums:
log.debug("removing all album links from db (--flush-index)")
log.debug(u"removing all album links from db (--flush-index)")
self._db.remove_all_album_files()

helper = IndexAlbumHelper(self)
Expand All @@ -217,7 +218,7 @@ def index_album_media(self, limit=None):
helper.setup_next_album(album)
if helper.skip_this_album():
continue
log.info('Album: %s, photos: %d, updated: %s, published: %s',
log.info(u'Album: %s, photos: %d, updated: %s, published: %s',
album.filename, album.size, album.modify_date,
album.create_date)

Expand All @@ -236,7 +237,7 @@ def index_album_media(self, limit=None):
if len(photos.entry) < limit:
break
if start_entry >= PicasaSync.ALBUM_MAX:
log.warning("LIMITING ALBUM TO %d entries",
log.warning(u"LIMITING ALBUM TO %d entries",
PicasaSync.ALBUM_MAX)
break
if start_entry + PicasaSync.BLOCK_SIZE > PicasaSync.ALBUM_MAX:
Expand All @@ -245,7 +246,7 @@ def index_album_media(self, limit=None):
helper.complete_album()
helper.complete_scan()

log.info('Total Album Photos in Drive %d, Picasa %d, multiples %d',
log.info(u'Total Album Photos in Drive %d, Picasa %d, multiples %d',
helper.total_photos, helper.picasa_photos,
helper.multiple_match_count)

Expand Down Expand Up @@ -333,26 +334,26 @@ def index_photos(self, photos):
continue

self.set_album_dates(picasa_media.create_date)
log.debug('checking %s is indexed', picasa_media.local_full_path)
log.debug(u'checking %s is indexed', picasa_media.local_full_path)
picasa_row = picasa_media.is_indexed(self.p._db)
if picasa_row:
if picasa_media.modify_date > picasa_row.ModifyDate:
log.info("Updated index for %s",
log.info(u"Updated index for %s",
picasa_media.local_full_path)
picasa_row_id = picasa_media.save_to_db(self.p._db,
update=True)
else:
picasa_row_id = picasa_row.Id
log.debug("Skipped %s", picasa_media.local_full_path)
log.debug(u"Skipped %s", picasa_media.local_full_path)
else:
self.picasa_photos += 1
log.info("Adding index for %d %s", self.picasa_photos,
log.info(u"Adding index for %d %s", self.picasa_photos,
picasa_media.local_full_path)
picasa_row_id = picasa_media.save_to_db(self.p._db)
if self.picasa_photos % 1000 == 0:
self.p._db.store()

log.debug('searching for drive match on {}'.format(
log.debug(u'searching for drive match on {}'.format(
picasa_media.filename))
drive_rows = self.p.match_drive_photo(picasa_media)
count, row = (len(drive_rows), drive_rows[0]) if drive_rows \
Expand All @@ -368,20 +369,20 @@ def index_photos(self, photos):
# comparison restored
if count == 0:
# no match, link to the picasa file
log.info('unmatched %s will be downloaded',
log.info(u'unmatched %s will be downloaded',
picasa_media.local_full_path)
self.p._db.put_album_file(self.album.id, picasa_row_id)
else:
# store link between album and drive file
log.debug('matched to %s', drive_rows[0].Path)
log.debug(u'matched to %s', drive_rows[0].Path)
self.p._db.put_album_file(self.album.id, drive_rows[0].Id)
# store the link between picasa and related drive file
# this also flags it as not requiring download
self.p._db.put_symlink(picasa_row_id, drive_rows[0].Id)

if count > 1:
self.multiple_match_count += 1
log.warning('WARNING multiple files match %s %s %s',
log.warning(u'WARNING multiple files match %s %s %s',
picasa_media.orig_name,
picasa_media.modify_date,
picasa_media.size)
Expand All @@ -394,7 +395,7 @@ def complete_album(self):
EndDate=self.album_end_photo,
SyncDate=Utils.date_to_string(
datetime.now()))
log.debug('completed album %s', self.album.filename)
log.debug(u'completed album %s', self.album.filename)
self.p._db.put_album(row)
if self.album.modify_date > self.latest_download:
self.latest_download = self.album.modify_date
Expand Down
10 changes: 5 additions & 5 deletions gphotos/Utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ def retry(count, func, *arg, **k_arg):
return res
except Exception as e:
last_e = e
log.warning("RETRYING due to: %s", repr(e))
log.warning("Call was: %s (%s, %s)", repr(func), arg, k_arg)
log.warning(u"RETRYING due to: %s", repr(e))
log.warning(u"Call was: %s (%s, %s)", repr(func), arg, k_arg)
time.sleep(.1)
raise last_e

Expand All @@ -79,7 +79,7 @@ def retry_i(count, iterator):
more_data = False
break
except Exception as e:
log.warning("RETRYING iterator due to: %s", repr(e))
log.warning(u"RETRYING iterator due to: %s", repr(e))
time.sleep(.1)
yield last_item

Expand Down Expand Up @@ -133,7 +133,7 @@ def string_to_date(date_string):
if m:
normalized = '{}-{}-{} 00:00:00'.format(*m.groups())
else:
log.warning('WARNING: time string {} illegal', date_string)
log.warning(u'WARNING: time string {} illegal', date_string)
return minimum_date()

return datetime.strptime(normalized, DATE_FORMAT)
Expand All @@ -144,7 +144,7 @@ def timestamp_to_date(time_secs, hour_offset=0):
date = datetime.fromtimestamp(
int(time_secs) / 1000 + 3600 * hour_offset)
except ValueError:
log.warning('WARNING: time stamp %d illegal', time_secs)
log.warning(u'WARNING: time stamp %d illegal', time_secs)
date = minimum_date()
return date

Expand Down

0 comments on commit 3ef32ec

Please sign in to comment.