Skip to content

Commit

Permalink
Update to version 1.4.0
Browse files Browse the repository at this point in the history 
Fix: Python 3 compatiblity: Replace use of "file" with "open".
Fix: Python 3 compatiblity: Sorting on the Annotations Import dialog.
Fix: Python 3 compatiblity: Completed work for tolinos and Kindles.
New: Add support for Sony devices. From @cibes.
Fix: Remove references to older calibre IPC code as that was removed in calibre 5.7.
Update: Various backend changed that should improve performance.
  • Loading branch information
@davidfor
davidfor committed Dec 26, 2020
1 parent 2617b2e commit 4e6cc5e
Show file tree
Hide file tree
Showing 17 changed files with 558 additions and 175 deletions.
2 changes: 1 addition & 1 deletion __init__.py
View file
Expand Up @@ -13,7 +13,7 @@ class AnnotationsPlugin(InterfaceActionBase):
description = 'Import annotations'
supported_platforms = ['linux', 'osx', 'windows']
author = 'David Forrester'
version = (1, 13, 0)
version = (1, 14, 0)
minimum_calibre_version = (1, 0, 0)

actual_plugin = 'calibre_plugins.annotations.action:AnnotationsAction'
Expand Down
8 changes: 8 additions & 0 deletions about.txt
View file
@@ -1,4 +1,12 @@
Version history:
1.14.0 - 26 December 2020
• Fix: Python 3 compatiblity: Replace use of "file" with "open".
• Fix: Python 3 compatiblity: Sorting on the Annotations Import dialog.
• Fix: Python 3 compatiblity: Completed work for tolinos and Kindles.
• New: Add support for Sony devices. From @Cibes.
• Fix: Remove references to older calibre IPC code as that was removed in calibre 5.7.
• Update: Various backend changed that should improve performance.

1.13.0 - 18 October 2020
• Update: Changes for Python 3 support in calibre.
• Update: Internal changes to how dialogs are handled.
Expand Down
73 changes: 47 additions & 26 deletions action.py
View file
Expand Up @@ -57,7 +57,7 @@
Logger, ProgressBar, Struct,
get_cc_mapping, get_clippings_cid, get_icon, get_pixmap, get_resource_files,
get_selected_book_mi, plugin_tmpdir,
set_cc_mapping, set_plugin_icon_resources, updateCalibreGUIView)
set_cc_mapping, set_plugin_icon_resources)
#import calibre_plugins.annotations.config as cfg
from calibre_plugins.annotations.config import plugin_prefs
from calibre_plugins.annotations.find_annotations import FindAnnotationsDialog
Expand Down Expand Up @@ -138,45 +138,48 @@ def add_annotations_to_calibre(self, book_mi, annotations_db, cid):
"""
update_field = get_cc_mapping('annotations', 'field', 'Comments')
self._log_location(update_field)
db = self.opts.gui.current_db
mi = db.get_metadata(cid, index_is_id=True)
library_db = self.opts.gui.current_db
mi = library_db.get_metadata(cid, index_is_id=True)

# Get the newly imported annotations
self._log_location("Getting new annotations as soup...")
raw_annotations = self.opts.db.annotations_to_html(annotations_db, book_mi)
self._log_location("New raw_annotations=%s" % raw_annotations)
new_soup = BeautifulSoup(self.opts.db.annotations_to_html(annotations_db, book_mi))
self._log_location("new_soup=%s" % new_soup)
new_annotation_string = None

if update_field == "Comments":
# Append merged annotations to end of existing Comments

# Any older annotations?
comments = db.comments(cid, index_is_id=True)
comments = library_db.comments(cid, index_is_id=True)
if comments is None:
comments = unicode(new_soup)
else:
# Keep comments, update annotations
comments_soup = BeautifulSoup(comments)
comments = merge_annotations_with_comments(self, cid, comments_soup, new_soup)
db.set_comment(mi.id, comments)
new_annotation_string = comments
else:
# Generate annotations formatted for custom field
# Any older annotations?
um = mi.metadata_for_field(update_field)
if mi.get_user_metadata(update_field, False)['#value#'] is None:
um['#value#'] = new_soup
# um['#value#'] = unicode(new_soup)
new_annotation_string = unicode(new_soup)
else:
# Merge new hashes into old
self._log_location("Current Annotation in library=%s" % mi.get_user_metadata(update_field, False)['#value#'])
old_soup = BeautifulSoup(mi.get_user_metadata(update_field, False)['#value#'])
self._log_location("Have old soup=%s" % old_soup)
merged_soup = merge_annotations(self, cid, old_soup, new_soup)
# um['#value#'] = unicode(merged_soup)
um['#value#'] = merged_soup
mi.set_user_metadata(update_field, um)
db.set_metadata(cid, mi, set_title=False, set_authors=False)
db.commit()
new_annotation_string = unicode(merged_soup)
# self._log_location("Have merged soup=%s" % merged_soup)

# self._log_location("Updateing GUI view")
#self.gui.library_view.select_rows([cid], using_ids=True)
updateCalibreGUIView()

self._log(" annotations updated: '%s' cid:%d " % (mi.title, cid))
return new_annotation_string

def create_menu_item(self, m, menu_text, image=None, tooltip=None, shortcut=None):
ac = self.create_action(spec=(menu_text, None, tooltip, shortcut), attr=menu_text)
Expand Down Expand Up @@ -923,6 +926,8 @@ def nuke_annotations(self):
pb.set_value(0)
pb.set_label('{:^100}'.format(_("Scanning {0} of {1}").format(0, total_books)))
pb.show()

book_ids_updated = []

for i, record in enumerate(db.data.iterall()):
mi = db.get_metadata(record[id], index_is_id=True)
Expand Down Expand Up @@ -977,13 +982,14 @@ def nuke_annotations(self):
db.set_metadata(record[id], mi, set_title=False, set_authors=False,
commit=True, force_changes=True, notify=True)

book_ids_updated.append(record[id])
pb.increment()

self.gui.library_view.model().refresh_ids(book_ids_updated)

# Hide the progress bar
pb.hide()

# Update the UI
updateCalibreGUIView()

def on_device_connection_changed(self, is_connected):
'''
Expand Down Expand Up @@ -1043,9 +1049,15 @@ def process_selected_books(self, selected_books, reader_app, annotations_db):
self._log_location()
updated_annotations = 0

library_db = self.opts.gui.current_db

# Are we collecting News clippings?
collect_news_clippings = plugin_prefs.get('cfg_news_clippings_checkbox', False)
news_clippings_destination = plugin_prefs.get('cfg_news_clippings_lineEdit', None)
update_field = get_cc_mapping('annotations', 'field', 'Comments')
self._log_location(update_field)

book_ids_updated = {}

for book_mi in selected_books[reader_app]:

Expand All @@ -1059,14 +1071,16 @@ def process_selected_books(self, selected_books, reader_app, annotations_db):
book_mi['cid'], confidence = self.generate_confidence(book_mi)

if confidence >= 3: # and False: # Uncomment this to force Kobo devices to go through the prompts.
self.add_annotations_to_calibre(book_mi, annotations_db, book_mi['cid'])
self._log(" '%s' (confidence: %d) annotations added automatically" % (book_mi['title'], confidence))
updated_annotations += 1
new_annotation_string = self.add_annotations_to_calibre(book_mi, annotations_db, book_mi['cid'])
if new_annotation_string is not None:
self._log(" '%s' (confidence: %d) annotations added automatically" % (book_mi['title'], confidence))
updated_annotations += 1
book_ids_updated[book_mi['cid']] = new_annotation_string
else:
# Low or zero confidence, confirm with user
if confidence == 0:
book_mi['cid'] = self.selected_mi.id
proposed_mi = self.opts.gui.current_db.get_metadata(int(book_mi['cid']), index_is_id=True)
proposed_mi = library_db.get_metadata(int(book_mi['cid']), index_is_id=True)
title = _('Import annotations • Mismatched metadata')
msg = ''
grey = '#ddd'
Expand Down Expand Up @@ -1121,10 +1135,8 @@ def process_selected_books(self, selected_books, reader_app, annotations_db):
det_msg = self.describe_confidence(confidence, book_mi, proposed_mi)

# Get the cover
db = self.opts.gui.current_db

cover_path = os.path.join(db.library_path,
db.path(proposed_mi.id, index_is_id=True),
cover_path = os.path.join(library_db.library_path,
library_db.path(proposed_mi.id, index_is_id=True),
'cover.jpg')
if not os.path.exists(cover_path):
cover_path = I('book.png')
Expand All @@ -1139,14 +1151,23 @@ def process_selected_books(self, selected_books, reader_app, annotations_db):
show_copy_button=False,
default_yes=True)
if d.exec_() == d.Accepted:
self.add_annotations_to_calibre(book_mi, annotations_db, book_mi['cid'])
updated_annotations += 1
new_annotation_string = self.add_annotations_to_calibre(book_mi, annotations_db, book_mi['cid'])
if new_annotation_string is not None:
self._log(" '%s' (confidence: %d) annotations added automatically" % (book_mi['title'], confidence))
updated_annotations += 1
book_ids_updated[book_mi['cid']] = new_annotation_string
self._log(" '{0}' annotations added to '{2}' with user confirmation (confidence: {1})".format(
book_mi['title'], confidence, proposed_mi.title))
else:
self._log(" NO CONFIDENCE: '%s' (confidence: %d), annotations not added to '%s'" %
(book_mi['title'], confidence, self.selected_mi.title))
self.opts.pb.increment()
if len(book_ids_updated) > 0:
debug_print("process_selected_books - Updating metadata - for column: %s number of changes=%d" % (update_field, len(book_ids_updated)))
library_db.new_api.set_field(update_field.lower(), book_ids_updated)
self._log("About to update UI for %s books" % len(book_ids_updated))
self.gui.library_view.model().refresh_ids(book_ids_updated,
current_row=self.gui.library_view.currentIndex().row())

return updated_annotations

Expand Down
20 changes: 12 additions & 8 deletions annotated_books.py
View file
Expand Up @@ -6,7 +6,7 @@
__docformat__ = 'restructuredtext en'

import operator
from time import localtime, strftime
from time import time, localtime, strftime

# calibre Python 3 compatibility.
import six
Expand Down Expand Up @@ -43,6 +43,8 @@
from calibre_plugins.annotations.config import plugin_prefs
from calibre_plugins.annotations.reader_app_support import ReaderApp

from calibre.devices.usbms.driver import debug_print

try:
debug_print("Annotations::annotated_books.py - loading translations")
load_translations()
Expand Down Expand Up @@ -155,14 +157,15 @@ def setData(self, index, value, role):
self.dataChanged.emit(index, index)
return True

def sort(self, Ncol, order):
def sort(self, Ncol, order=Qt.AscendingOrder):
"""
Sort table by given column number.
"""
self.layoutAboutToBeChanged.emit()
self.arraydata = sorted(self.arraydata, key=operator.itemgetter(Ncol))
if order == Qt.DescendingOrder:
self.arraydata.reverse()
if Ncol == self.ENABLED_COL: # Don't sort on the checkbox column.
self.arraydata = sorted(self.arraydata, key=lambda row: row[Ncol].checkState(), reverse=(order == Qt.DescendingOrder))
else:
self.arraydata = sorted(self.arraydata, key=operator.itemgetter(Ncol), reverse=(order == Qt.DescendingOrder))
self.layoutChanged.emit()


Expand Down Expand Up @@ -208,10 +211,12 @@ def __init__(self, parent, book_list, get_annotations_as_HTML, source):
enabled.setChecked(True)

# last_annotation sorts by timestamp
last_annotation_timestamp = time() if book_data['last_update'] is None else book_data['last_update']
# debug_print("AnnotatedBooksDialog::__init__ title=%s, i=%d, the_timestamp=%s" % (book_data['title'], i, the_timestamp))
last_annotation = SortableTableWidgetItem(
strftime(friendly_timestamp_format,
localtime(book_data['last_update'])),
book_data['last_update'])
localtime(last_annotation_timestamp)),
last_annotation_timestamp)

# reader_app sorts case-insensitive
reader_app = SortableTableWidgetItem(
Expand Down Expand Up @@ -284,7 +289,6 @@ def __init__(self, parent, book_list, get_annotations_as_HTML, source):
self.tv.hideColumn(self.annotations_header.index('uuid'))
self.tv.hideColumn(self.annotations_header.index('book_id'))
self.tv.hideColumn(self.annotations_header.index('genre'))
# self.tv.hideColumn(self.annotations_header.index(_('Confidence')))
self.tv.hideColumn(self.CONFIDENCE_COL)

# Set horizontal self.header props
Expand Down
38 changes: 30 additions & 8 deletions annotations.py
View file
Expand Up @@ -61,6 +61,8 @@ def __init__(self, annotation):
for p in self.all_fields:
setattr(self, p, annotation.get(p))

def __str__(self):
return '\n'.join(["%s: %s" % (field, getattr(self, field, None)) for field in self.all_fields])

class Annotations(Annotation, Logger):
'''
Expand Down Expand Up @@ -121,7 +123,6 @@ def to_HTML(self, header=''):
'''
Generate HTML with user-specified CSS, element order
'''
from calibre.ebooks.BeautifulSoup import prettify
# Retrieve CSS prefs
from calibre_plugins.annotations.appearance import default_elements
stored_css = plugin_prefs.get('appearance_css', default_elements)
Expand Down Expand Up @@ -154,13 +155,15 @@ def to_HTML(self, header=''):
</tr>
</table>'''
comments_body += re.sub(r'>\s+<', r'><', ts_css)
self._log_location("comments_body='%s'" % comments_body)

if self.annotations:
soup = BeautifulSoup(ANNOTATIONS_HEADER)
dtc = 0

# Add the annotations
for i, agroup in enumerate(sorted(self.annotations, key=self._annotation_sorter)):
self._log_location("agroup='%s'" % agroup)
location = agroup.location
if location is None:
location = ''
Expand All @@ -169,13 +172,16 @@ def to_HTML(self, header=''):

text = ''
if agroup.text:
self._log_location("agroup.text='%s'" % agroup.text)
for agt in agroup.text:
self._log_location(agt)
self._log_location("agt='%s'" % agt)
text += '<p class="highlight" style="{0}">{1}</p>'.format(text_style, agt)

note = ''
if agroup.note:
self._log_location("agroup.note='%s'" % agroup.note)
for agn in agroup.note:
self._log_location("agn='%s'" % agn)
note += '<p class="note" style="{0}">{1}</p>'.format(note_style, agn)

try:
Expand All @@ -202,8 +208,10 @@ def to_HTML(self, header=''):
try:
ka_soup = BeautifulSoup()
divTag = ka_soup.new_tag('div')
self._log_location("Used ka_soup.new_tag to create tag: %s" % divTag)
except:
divTag = Tag(BeautifulSoup(), 'div')
self._log_location("Used Tag(BeautifulSoup() to create tag: %s" % divTag)

content_args = {
'color': agroup.highlightcolor,
Expand All @@ -214,7 +222,19 @@ def to_HTML(self, header=''):
'ts_style': datetime_style.format(dt_bgcolor, dt_fgcolor),
'unix_timestamp': agroup.timestamp,
}
divTag.insert(0, BeautifulSoup(comments_body.format(**content_args)))
# self._log_location("Generated comment soup: %s" % BeautifulSoup(comments_body.format(**content_args)))
comments_body_soup = BeautifulSoup(comments_body.format(**content_args))
self._log_location("Generated comment soup: comments_body_soup=%s" % comments_body_soup)
self._log_location("Generated comment soup: comments_body_soup.body=%s" % comments_body_soup.body)
self._log_location("Generated comment soup: comments_body_soup.body.children=%s" % comments_body_soup.body.children)
self._log_location("Generated comment soup: comments_body_soup.body.contents=%s" % comments_body_soup.body.contents)
self._log_location("Generated comment soup: len(comments_body_soup.body.contents)=%s" % len(comments_body_soup.body.contents))
for i in range(0, len(comments_body_soup.body.contents)):
self._log_location("i=%s" % i)
self._log_location("comment_body_tag=%s" % comments_body_soup.body.contents[i])
while len(comments_body_soup.body.contents) > 0:
self._log_location("comment_body_tag=%s" % comments_body_soup.body.contents[0])
divTag.append(comments_body_soup.body.contents[0])
divTag['class'] = "annotation"
divTag['genre'] = ''
if agroup.genre:
Expand All @@ -223,7 +243,9 @@ def to_HTML(self, header=''):
divTag['location_sort'] = agroup.location_sort
divTag['reader'] = agroup.reader_app
divTag['style'] = ANNOTATION_DIV_STYLE
self._log_location("An annotation - divTag=%s" % divTag)
soup.div.insert(dtc, divTag)
self._log_location("Full soup after adding annotation - soup=%s" % soup)
dtc += 1
if i < len(self.annotations) - 1 and \
plugin_prefs.get('appearance_hr_checkbox', False):
Expand All @@ -232,7 +254,7 @@ def to_HTML(self, header=''):

else:
soup = BeautifulSoup(ANNOTATIONS_HEADER)
return prettify(soup)
return unicode(soup)


def merge_annotations(parent, cid, old_soup, new_soup):
Expand Down Expand Up @@ -266,10 +288,10 @@ def merge_annotations(parent, cid, old_soup, new_soup):
debug_print("Getting old annotations - old_soup after extract=", old_soup)

# Capture existing annotations
parent.opts.db.capture_content(ouas, cid, TRANSIENT_DB)
annotation_list = parent.opts.db.capture_content(ouas, cid, TRANSIENT_DB)

# Regurgitate old_soup with current CSS
regurgitated_soup = BeautifulSoup(parent.opts.db.rerender_to_html(TRANSIENT_DB, cid))
regurgitated_soup = BeautifulSoup(parent.opts.db.rerender_to_html_from_list(annotation_list))
debug_print("Getting old annotations - regurgitated_soup=", regurgitated_soup)
else:
regurgitated_soup = BeautifulSoup()
Expand Down Expand Up @@ -325,10 +347,10 @@ def merge_annotations(parent, cid, old_soup, new_soup):
ouas.extract()

# Capture existing annotations
parent.opts.db.capture_content(ouas, cid, TRANSIENT_DB)
annotation_list = parent.opts.db.capture_content(ouas, cid, TRANSIENT_DB)

# Regurgitate old_soup with current CSS
regurgitated_soup = BeautifulSoup(parent.opts.db.rerender_to_html(TRANSIENT_DB, cid))
regurgitated_soup = BeautifulSoup(parent.opts.db.rerender_to_html_from_list(annotation_list))

# Add device annotation timestamps and hashes
duas = new_soup.findAll('div', 'annotation')
Expand Down

0 comments on commit 4e6cc5e

Please sign in to comment.