diff --git a/D3Charts/DescendantIndentedTree.py b/D3Charts/DescendantIndentedTree.py index 8fab67fb5..2fb0c770d 100644 --- a/D3Charts/DescendantIndentedTree.py +++ b/D3Charts/DescendantIndentedTree.py @@ -62,6 +62,7 @@ # #------------------------------------------------------------------------ from gramps.gen.display.name import displayer as global_name_display +from gramps.gen.display.place import displayer as place_displayer from gramps.gen.errors import ReportError from gramps.gen.lib import ChildRefType from gramps.gen.plug.menu import (ColorOption, NumberOption, PersonOption, @@ -317,9 +318,7 @@ def get_date_place(self,event): date = get_date(event) place_handle = event.get_place_handle() if place_handle: - place = self.database.get_place_from_handle( - place_handle).get_title() - + place = place_displayer.display_event(self.database, event) return("%(event_abbrev)s %(date)s %(place)s" % { 'event_abbrev': event.type.get_abbreviation(), 'date' : date, diff --git a/DescendantBooks/DescendantBookReport.py b/DescendantBooks/DescendantBookReport.py index 8994f91ab..3531a8bd2 100644 --- a/DescendantBooks/DescendantBookReport.py +++ b/DescendantBooks/DescendantBookReport.py @@ -40,6 +40,7 @@ from gramps.gen.plug.menu import (NumberOption, PersonOption, BooleanOption, EnumeratedListOption, FilterOption) from gramps.gen.display.name import displayer as global_name_display +from gramps.gen.display.place import displayer as place_displayer from gramps.gen.errors import ReportError from gramps.gen.plug.report import Report from gramps.gen.plug.report import utils as ReportUtils @@ -158,8 +159,7 @@ def __date_place(self,event): date = gramps.gen.datehandler.get_date(event) place_handle = event.get_place_handle() if place_handle: - place = self.database.get_place_from_handle( - place_handle).get_title() + place = place_displayer.display_event(self.database, event) return("%(event_abbrev)s %(date)s - %(place)s" % { 'event_abbrev': event.type.get_abbreviation(), 'date' : date, diff --git a/DescendantBooks/DetailedDescendantBookReport.py b/DescendantBooks/DetailedDescendantBookReport.py index c6660141b..bb3b1fd11 100644 --- a/DescendantBooks/DetailedDescendantBookReport.py +++ b/DescendantBooks/DetailedDescendantBookReport.py @@ -38,6 +38,7 @@ # #------------------------------------------------------------------------ from gramps.gen.display.name import displayer as global_name_display +from gramps.gen.display.place import displayer as place_displayer from gramps.gen.errors import ReportError from gramps.gen.lib import FamilyRelType, Person, NoteType from gramps.gen.plug.menu import (BooleanOption, NumberOption, PersonOption, @@ -779,12 +780,7 @@ def append_event(self, event_ref, family = False): event = self.database.get_event_from_handle(event_ref.ref) date = self._get_date(event.get_date_object()) - - ph = event.get_place_handle() - if ph: - place = self.database.get_place_from_handle(ph).get_title() - else: - place = '' + place = place_displayer.display_event(self.database, event) event_name = self._get_type(event.get_type()) @@ -823,12 +819,7 @@ def write_event(self, event_ref): else: date = event.get_date_object().get_year() - ph = event.get_place_handle() - if ph: - place = self.database.get_place_from_handle(ph).get_title() - else: - place = '' - + place = place_displayer.display_event(self.database, event) self.doc.start_paragraph('DDR-EventHeader') #BOOK event_name = self._get_type(event.get_type()) diff --git a/DescendantsLines/substkw.py b/DescendantsLines/substkw.py index aab4f2f4c..2993a986a 100644 --- a/DescendantsLines/substkw.py +++ b/DescendantsLines/substkw.py @@ -51,11 +51,6 @@ # Gramps modules # #------------------------------------------------------------------------ -from gramps.gen.display.place import displayer as _pd -from gramps.gen.lib import EventType, PlaceType, Location -from gramps.gen.utils.db import get_birth_or_fallback, get_death_or_fallback -from gramps.gen.utils.location import get_main_location -from gramps.gen.const import GRAMPS_LOCALE as glocale from gramps.plugins.lib.libsubstkeyword import * import logging @@ -624,67 +619,6 @@ def replace_and_clean(self, lines, event): return new -class PlaceFormat(GenericFormat): - """ The place format class. - If no format string, the place is displayed as per preference options - otherwise, parse through a format string and put the place parts in - """ - - def __init__(self, database, _in): - self.database = database - GenericFormat.__init__(self, _in) - self.date = None - - def get_place(self, database, event): - """ A helper method for retrieving a place from an event """ - if event: - bplace_handle = event.get_place_handle() - self.date = event.date - if bplace_handle: - return database.get_place_from_handle(bplace_handle) - return None - - def _default_format(self, place): - return _pd.display(self.database, place, date=self.date) - - def parse_format(self, database, place): - """ Parse the place """ - - if self.is_blank(place): - return - - code = "elcuspn" + "oitxy" - upper = code.upper() - - main_loc = get_main_location(database, place, date=self.date) - location = Location() - location.set_street(main_loc.get(PlaceType.STREET, '')) - location.set_locality(main_loc.get(PlaceType.LOCALITY, '')) - location.set_parish(main_loc.get(PlaceType.PARISH, '')) - location.set_city(main_loc.get(PlaceType.CITY, '')) - location.set_county(main_loc.get(PlaceType.COUNTY, '')) - location.set_state(main_loc.get(PlaceType.STATE, '')) - location.set_postal_code(main_loc.get(PlaceType.STREET, '')) - location.set_country(main_loc.get(PlaceType.COUNTRY, '')) - - function = [location.get_street, - location.get_locality, - location.get_city, - location.get_county, - location.get_state, - place.get_code, - location.get_country, - - location.get_phone, - location.get_parish, - place.get_title, - place.get_longitude, - place.get_latitude - ] - - return self.generic_format(place, code, upper, function) - - # # if __name__ == '__main__': #------------------------------------------------------------------------- diff --git a/DynamicWeb/dynamicweb.py b/DynamicWeb/dynamicweb.py index 9340e0c77..f6d4377f7 100644 --- a/DynamicWeb/dynamicweb.py +++ b/DynamicWeb/dynamicweb.py @@ -167,6 +167,8 @@ DWR_VERSION_412 = (VERSION_TUPLE[0] >= 5) or ((VERSION_TUPLE[0] >= 4) and (VERSION_TUPLE[1] >= 1) and (VERSION_TUPLE[2] >= 2)) DWR_VERSION_420 = (VERSION_TUPLE[0] >= 5) or ((VERSION_TUPLE[0] >= 4) and (VERSION_TUPLE[1] >= 2)) DWR_VERSION_500 = (VERSION_TUPLE[0] >= 5) +DWR_VERSION_520 = True # TODO adjust when we have the real 5.2.x out with enhanced places + from gramps.gen.lib import (ChildRefType, Date, EventType, FamilyRelType, Name, NameType, Person, UrlType, NoteType, EventRoleType, Family, Event, Place, Source, @@ -226,6 +228,8 @@ from gramps.gen.relationship import get_relationship_calculator if (DWR_VERSION_410): from gramps.gen.utils.location import get_main_location +if DWR_VERSION_520: + from gramps.gen.utils.location import get_code if (DWR_VERSION_500): @@ -1518,13 +1522,19 @@ def _export_places(self): if (not self.inc_places): jdatas.append(jdata) continue + #if DWR_VERSION_520: # TODO + # Enhanced places support multiple types with dates + # Enhanced places support multiple attributes + # Enhanced places support multiple events + # we may want to display some of these... if DWR_VERSION_410: jdata['type'] = str(place.get_type()) else: jdata['type'] = '' jdata['names'] = [] if DWR_VERSION_410: - for pn in place.get_all_names(): + for pn in (place.get_names() if DWR_VERSION_520 else + place.get_all_names()): lang = pn.get_language() if lang != '' and pref_lang!= '' and lang != pref_lang: continue date = format_date(pn.get_date_object()) @@ -1583,7 +1593,8 @@ def _export_places(self): else: coords = ("", "") jdata['coords'] = coords - jdata['code'] = place.get_code() + jdata['code'] = (get_code(place) if DWR_VERSION_520 else + place.get_code()) # Get place notes jdata['note'] = self.get_notes_text(place) # Get place media diff --git a/ExtractCity/extractcity.py b/ExtractCity/extractcity.py index d7e4a61ce..fce45b0b1 100644 --- a/ExtractCity/extractcity.py +++ b/ExtractCity/extractcity.py @@ -51,7 +51,7 @@ from gramps.plugins.lib.libplaceimport import PlaceImport from gramps.gen.utils.location import get_main_location from gramps.gen.display.place import displayer as place_displayer -from gramps.gen.lib import PlaceType, PlaceName +from gramps.gen.lib import PlaceType, PlaceName, Attribute, AttributeType from gramps.gui.plug import tool from gramps.gui.utils import ProgressMeter @@ -610,7 +610,7 @@ def on_help_clicked(self, obj): display_help() def on_ok_clicked(self, obj): - with DbTxn(_("Extract Place data"), self.db, batch=True) as self.trans: + with DbTxn(_("Extract Place data"), self.db, batch=False) as self.trans: self.db.disable_signals() changelist = [node for node in self.iter_list if self.model.get_value(node, 0)] @@ -624,7 +624,10 @@ def on_ok_clicked(self, obj): place.set_name(PlaceName(value=row[2])) place.set_type(PlaceType.CITY) if row[4]: - place.set_code(row[4]) + attr = Attribute() + attr.set_type(AttributeType.POSTAL) + attr.set_value(row[4]) + place.add_attribute(attr) self.db.commit_place(place, self.trans) self.place_import.generate_hierarchy(self.trans) diff --git a/GetGOV/getgov.gpr.py b/GetGOV/getgov.gpr.py index 7ba7182f7..911a723fd 100644 --- a/GetGOV/getgov.gpr.py +++ b/GetGOV/getgov.gpr.py @@ -30,7 +30,7 @@ name = _("GetGOV"), description = _("Gramplet to get places from the GOV database"), status = STABLE, - version = '1.0.12', + version = '2.0.0', gramps_target_version = '5.1', fname = "getgov.py", gramplet = 'GetGOV', diff --git a/GetGOV/getgov.py b/GetGOV/getgov.py index bf5fdb354..fb9506819 100644 --- a/GetGOV/getgov.py +++ b/GetGOV/getgov.py @@ -27,8 +27,12 @@ # Python modules # #------------------------------------------------------------------------ -from urllib.request import urlopen, quote +from urllib.request import urlopen, quote, URLError from xml.dom.minidom import parseString +import socket +import re +import os +import json #------------------------------------------------------------------------ # @@ -44,10 +48,14 @@ #------------------------------------------------------------------------ from gramps.gen.plug import Gramplet from gramps.gen.db import DbTxn -from gramps.gen.lib import Place, PlaceName, PlaceType, PlaceRef, Url, UrlType +from gramps.gen.lib import (Place, PlaceName, PlaceType, PlaceRef, + PlaceGroupType as P_G, PlaceHierType as P_H, + Url, UrlType) from gramps.gen.datehandler import parser from gramps.gen.config import config from gramps.gen.display.place import displayer as _pd +from gramps.gen.utils.id import create_id +from gramps.gui.dialog import ErrorDialog #------------------------------------------------------------------------ # @@ -68,210 +76,218 @@ #------------------------------------------------------------------------ ISO_CODE_LOOKUP = { -"aar": "aa", -"abk": "ab", -"afr": "af", -"aka": "ak", -"alb": "sq", -"amh": "am", -"ara": "ar", -"arg": "an", -"arm": "hy", -"asm": "as", -"ava": "av", -"ave": "ae", -"aym": "ay", -"aze": "az", -"bak": "ba", -"bam": "bm", -"baq": "eu", -"bel": "be", -"ben": "bn", -"bih": "bh", -"bis": "bi", -"bod": "bo", -"bos": "bs", -"bre": "br", -"bul": "bg", -"bur": "my", -"cat": "ca", -"ces": "cs", -"cha": "ch", -"che": "ce", -"chi": "zh", -"chu": "cu", -"chv": "cv", -"cor": "kw", -"cos": "co", -"cre": "cr", -"cym": "cy", -"cze": "cs", -"dan": "da", -"deu": "de", -"div": "dv", -"dut": "nl", -"dzo": "dz", -"ell": "el", -"eng": "en", -"epo": "eo", -"est": "et", -"eus": "eu", -"ewe": "ee", -"fao": "fo", -"fas": "fa", -"fij": "fj", -"fin": "fi", -"fra": "fr", -"fre": "fr", -"fry": "fy", -"ful": "ff", -"geo": "ka", -"ger": "de", -"gla": "gd", -"gle": "ga", -"glg": "gl", -"glv": "gv", -"gre": "el", -"grn": "gn", -"guj": "gu", -"hat": "ht", -"hau": "ha", -"heb": "he", -"her": "hz", -"hin": "hi", -"hmo": "ho", -"hrv": "hr", -"hun": "hu", -"hye": "hy", -"ibo": "ig", -"ice": "is", -"ido": "io", -"iii": "ii", -"iku": "iu", -"ile": "ie", -"ina": "ia", -"ind": "id", -"ipk": "ik", -"isl": "is", -"ita": "it", -"jav": "jv", -"jpn": "ja", -"kal": "kl", -"kan": "kn", -"kas": "ks", -"kat": "ka", -"kau": "kr", -"kaz": "kk", -"khm": "km", -"kik": "ki", -"kin": "rw", -"kir": "ky", -"kom": "kv", -"kon": "kg", -"kor": "ko", -"kua": "kj", -"kur": "ku", -"lao": "lo", -"lat": "la", -"lav": "lv", -"lim": "li", -"lin": "ln", -"lit": "lt", -"ltz": "lb", -"lub": "lu", -"lug": "lg", -"mac": "mk", -"mah": "mh", -"mal": "ml", -"mao": "mi", -"mar": "mr", -"may": "ms", -"mkd": "mk", -"mlg": "mg", -"mlt": "mt", -"mon": "mn", -"mri": "mi", -"msa": "ms", -"mya": "my", -"nau": "na", -"nav": "nv", -"nbl": "nr", -"nde": "nd", -"ndo": "ng", -"nep": "ne", -"nld": "nl", -"nno": "nn", -"nob": "nb", -"nor": "no", -"nya": "ny", -"oci": "oc", -"oji": "oj", -"ori": "or", -"orm": "om", -"oss": "os", -"pan": "pa", -"per": "fa", -"pli": "pi", -"pol": "pl", -"por": "pt", -"pus": "ps", -"que": "qu", -"roh": "rm", -"ron": "ro", -"rum": "ro", -"run": "rn", -"rus": "ru", -"sag": "sg", -"san": "sa", -"sin": "si", -"slk": "sk", -"slo": "sk", -"slv": "sl", -"sme": "se", -"smo": "sm", -"sna": "sn", -"snd": "sd", -"som": "so", -"sot": "st", -"spa": "es", -"sqi": "sq", -"srd": "sc", -"srp": "sr", -"ssw": "ss", -"sun": "su", -"swa": "sw", -"swe": "sv", -"tah": "ty", -"tam": "ta", -"tat": "tt", -"tel": "te", -"tgk": "tg", -"tgl": "tl", -"tha": "th", -"tib": "bo", -"tir": "ti", -"ton": "to", -"tsn": "tn", -"tso": "ts", -"tuk": "tk", -"tur": "tr", -"twi": "tw", -"uig": "ug", -"ukr": "uk", -"urd": "ur", -"uzb": "uz", -"ven": "ve", -"vie": "vi", -"vol": "vo", -"wel": "cy", -"wln": "wa", -"wol": "wo", -"xho": "xh", -"yid": "yi", -"yor": "yo", -"zha": "za", -"zho": "zh", -"zul": "zu"} + "aar": "aa", + "abk": "ab", + "afr": "af", + "aka": "ak", + "alb": "sq", + "amh": "am", + "ara": "ar", + "arg": "an", + "arm": "hy", + "asm": "as", + "ava": "av", + "ave": "ae", + "aym": "ay", + "aze": "az", + "bak": "ba", + "bam": "bm", + "baq": "eu", + "bel": "be", + "ben": "bn", + "bih": "bh", + "bis": "bi", + "bod": "bo", + "bos": "bs", + "bre": "br", + "bul": "bg", + "bur": "my", + "cat": "ca", + "ces": "cs", + "cha": "ch", + "che": "ce", + "chi": "zh", + "chu": "cu", + "chv": "cv", + "cor": "kw", + "cos": "co", + "cre": "cr", + "cym": "cy", + "cze": "cs", + "dan": "da", + "deu": "de", + "div": "dv", + "dut": "nl", + "dzo": "dz", + "ell": "el", + "eng": "en", + "epo": "eo", + "est": "et", + "eus": "eu", + "ewe": "ee", + "fao": "fo", + "fas": "fa", + "fij": "fj", + "fin": "fi", + "fra": "fr", + "fre": "fr", + "fry": "fy", + "ful": "ff", + "geo": "ka", + "ger": "de", + "gla": "gd", + "gle": "ga", + "glg": "gl", + "glv": "gv", + "gre": "el", + "grn": "gn", + "guj": "gu", + "hat": "ht", + "hau": "ha", + "heb": "he", + "her": "hz", + "hin": "hi", + "hmo": "ho", + "hrv": "hr", + "hun": "hu", + "hye": "hy", + "ibo": "ig", + "ice": "is", + "ido": "io", + "iii": "ii", + "iku": "iu", + "ile": "ie", + "ina": "ia", + "ind": "id", + "ipk": "ik", + "isl": "is", + "ita": "it", + "jav": "jv", + "jpn": "ja", + "kal": "kl", + "kan": "kn", + "kas": "ks", + "kat": "ka", + "kau": "kr", + "kaz": "kk", + "khm": "km", + "kik": "ki", + "kin": "rw", + "kir": "ky", + "kom": "kv", + "kon": "kg", + "kor": "ko", + "kua": "kj", + "kur": "ku", + "lao": "lo", + "lat": "la", + "lav": "lv", + "lim": "li", + "lin": "ln", + "lit": "lt", + "ltz": "lb", + "lub": "lu", + "lug": "lg", + "mac": "mk", + "mah": "mh", + "mal": "ml", + "mao": "mi", + "mar": "mr", + "may": "ms", + "mkd": "mk", + "mlg": "mg", + "mlt": "mt", + "mon": "mn", + "mri": "mi", + "msa": "ms", + "mya": "my", + "nau": "na", + "nav": "nv", + "nbl": "nr", + "nde": "nd", + "ndo": "ng", + "nep": "ne", + "nld": "nl", + "nno": "nn", + "nob": "nb", + "nor": "no", + "nya": "ny", + "oci": "oc", + "oji": "oj", + "ori": "or", + "orm": "om", + "oss": "os", + "pan": "pa", + "per": "fa", + "pli": "pi", + "pol": "pl", + "por": "pt", + "pus": "ps", + "que": "qu", + "roh": "rm", + "ron": "ro", + "rum": "ro", + "run": "rn", + "rus": "ru", + "sag": "sg", + "san": "sa", + "sin": "si", + "slk": "sk", + "slo": "sk", + "slv": "sl", + "sme": "se", + "smo": "sm", + "sna": "sn", + "snd": "sd", + "som": "so", + "sot": "st", + "spa": "es", + "sqi": "sq", + "srd": "sc", + "srp": "sr", + "ssw": "ss", + "sun": "su", + "swa": "sw", + "swe": "sv", + "tah": "ty", + "tam": "ta", + "tat": "tt", + "tel": "te", + "tgk": "tg", + "tgl": "tl", + "tha": "th", + "tib": "bo", + "tir": "ti", + "ton": "to", + "tsn": "tn", + "tso": "ts", + "tuk": "tk", + "tur": "tr", + "twi": "tw", + "uig": "ug", + "ukr": "uk", + "urd": "ur", + "uzb": "uz", + "ven": "ve", + "vie": "vi", + "vol": "vo", + "wel": "cy", + "wln": "wa", + "wol": "wo", + "xho": "xh", + "yid": "yi", + "yor": "yo", + "zha": "za", + "zho": "zh", + "zul": "zu"} + + +COU = "#FFFF00000000" +REG = "#0000FFFFFFFF" +PLA = "#0000FFFF0000" +OTH = "#800080008000" +types_file = os.path.join(os.path.dirname(__file__), "gov_types.json") + #------------------------------------------------------------------------ # @@ -286,12 +302,17 @@ def init(self): """ Initialise the gramplet. """ + self.place_view = None root = self.__create_gui() self.gui.get_container_widget().remove(self.gui.textview) - self.gui.get_container_widget().add_with_viewport(root) + self.gui.get_container_widget().add(root) root.show_all() - self.type_dic = dict() + self.update_btn.set_sensitive(False) + self.types_scanned = False # if we need to scan for types in db + self.visited = {} # key: place handle, data: True for touched, or + # # list of [(date, hierarchy)] tuples + type_dic = {} # key: str type num, data: name def __create_gui(self): """ @@ -300,7 +321,7 @@ def __create_gui(self): vbox = Gtk.Box(orientation=Gtk.Orientation.VERTICAL) vbox.set_spacing(4) - label = Gtk.Label(_('Enter GOV-id:')) + label = Gtk.Label(label=_('Enter GOV-id:')) label.set_halign(Gtk.Align.START) self.entry = Gtk.Entry() @@ -312,115 +333,386 @@ def __create_gui(self): get.connect("clicked", self.__get_places) button_box.add(get) + self.update_btn = Gtk.Button(label=_('Update')) + self.update_btn.connect("clicked", self.__update_places) + button_box.add(self.update_btn) + vbox.pack_start(label, False, True, 0) vbox.pack_start(self.entry, False, True, 0) vbox.pack_start(button_box, False, True, 0) return vbox + def db_changed(self): + if self.dbstate.is_open(): + self.connect(self.dbstate.db, 'place-update', self.update) + self.connect_signal('Place', self.update) + else: + self.update_btn.set_sensitive(False) + self.types_scanned = False + def main(self): + if not self.place_view: + _vm = self.uistate.viewmanager + self.place_view = _vm.pages[_vm.page_lookup[( + _vm.navigator.active_cat, _vm.navigator.active_view)]] + self.place_view.selection.connect('changed', self.row_changed) + self.row_changed() + if not self.types_scanned and self.type_dic: + # scan db to see if any of the Gov_xx are in use + got_new = got_any = False + count = 0 + for place in self.dbstate.db.iter_places(): + if count % 100 == 0: + yield True + for typ in place.get_types(): + if typ.pt_id.startswith("Gov_"): + got_any = True + tup = self.type_dic[typ.pt_id[4:]] + if tup[2]: + continue + tup[2] = True + got_new = True + if not got_any: + # clear down the type usage data + for data in self.type_dic.values(): + data[2] = False + got_new = True + if got_new: + # save changes + with open(types_file, 'w', encoding='utf-8') as f_p: + json.dump((self.type_dic), f_p, + ensure_ascii=False, indent=2) + load_on_reg(None, None, None, getfile=False) + self.dbstate.db.emit("custom-type-changed") + + _gramps_id = re.compile(r' *[^\d]{0,3}(\d+){3,9}[^\d]{0,3}') + + def row_changed(self, *_selection): + """ + from the Place view + """ + selected_ids = self.place_view.selected_handles() + if not selected_ids: + return + place = self.dbstate.db.get_place_from_handle(selected_ids[0]) + # See if this is a normal Gramps ID (if not, GOV ID) + if not (self._gramps_id.match(place.gramps_id) or + place.gramps_id.startswith("GEO")): + self.update_btn.set_sensitive(True) + else: + self.update_btn.set_sensitive(False) + + def __update_places(self, _obj): """ - Called to update the display. + This is called when the update button is clicked. + The selected places are updated from GOV """ - pass + try: + self.preferred_lang = config.get('preferences.place-lang') + except AttributeError: + fmt = config.get('preferences.place-format') + pf = _pd.get_formats()[fmt] + self.preferred_lang = pf.language + if len(self.preferred_lang) != 2: + self.preferred_lang = glocale.lang[0:2] - def __get_places(self, obj): + if not self.type_dic: + # if first run, get all the place types information from GOV + if not self.__get_types(): + self.type_dic = {} + return + + selected_ids = self.place_view.selected_handles() + for hndl in selected_ids: + place = self.dbstate.db.get_place_from_handle(hndl) + # See if this is a normal Gramps ID (if not, GOV ID) + if(self._gramps_id.match(place.gramps_id) or + place.gramps_id.startswith("GEO")): + continue + self.__update_place(place) + + def __update_place(self, place): + """ + Update a single place from GOV data. Note: this will actually call + itself recursivly if an enclosing place seems out of date. It will + also call __add_place if an enclosing place is entirely missing. + + Returns: [(date, hierarchy)] list of tuples + """ + ret = self.visited.get(place.handle) + if ret is True: # we have been here before and have a loop + print("Error: Place enclosure Loop in GOV data!") + return None + elif ret is not None: # we have added or validated this place + return place.handle, ret + + # get place data from GOV + resp = self.__get_place(place.gramps_id) + if not resp: + return None + u_place, ref_list, hiers = resp + # this is tracked to detect possible loops + self.visited[place.handle] = True + # We replace the place enclosures from the found GOV data + place.set_placeref_list([]) + for ref, date in ref_list: + resp = self.__add_place(ref) + if resp is None: + return None + handle, hiers = resp + + place_ref = PlaceRef() + place_ref.ref = handle + place_ref.set_date_object(date) + for hdate, hier in hiers: + if hdate.is_empty() or date.match_exact(hdate): + hierarchy = hier + break + else: + hierarchy = P_H(P_H.ADMIN) + place_ref.set_type(hierarchy) + place.add_placeref(place_ref) + # we replace the current types from the found GOV types + place.set_types(u_place.get_types()) + # we replace the current names from the found GOV names + place.set_names(u_place.get_names()) + # we replace the current urls from the found GOV urls + place.set_url_list(u_place.get_url_list()) + with DbTxn(_('Update GOV-id place %s') % place.gramps_id, + self.dbstate.db) as self.trans: + self.dbstate.db.add_place(place, self.trans) + self.visited[place.handle] = hiers + return hiers + + def __get_places(self, _obj): + """ + Main routine called when entering a GOV ID. This looks up the place, + adds it to db, and adds any enclosing places not already present. + """ gov_id = self.entry.get_text() - to_do = [gov_id] try: - preferred_lang = config.get('preferences.place-lang') + self.preferred_lang = config.get('preferences.place-lang') except AttributeError: fmt = config.get('preferences.place-format') pf = _pd.get_formats()[fmt] - preferred_lang = pf.language - if len(preferred_lang) != 2: - preferred_lang = 'de' - visited = {} + self.preferred_lang = pf.language + if len(self.preferred_lang) != 2: + self.preferred_lang = glocale.lang[0:2] if not self.type_dic: - self.__get_types() - - with DbTxn(_('Add GOV-id place %s') % gov_id, self.dbstate.db) as trans: - while to_do: - gov_id = to_do.pop() - place = self.dbstate.db.get_place_from_gramps_id(gov_id) - if place is not None: - visited[gov_id] = (place, []) + # if first run, get all the place types information from GOV + if not self.__get_types(): + self.type_dic = {} + return + self.__add_place(gov_id) + + def __add_place(self, gov_id): + """ + This looks up and then adds a single place. During enclosure + processing, the method may get called recursivly to add missing + enclosing places. Or, if the enclosing place exists, but doesn't + have valid GOV place Types, the __update_place method might get called. + + returns place handle, [(date, hierarchy)] list of tuples + """ + place = self.dbstate.db.get_place_from_gramps_id(gov_id) + if place is not None: + ret = self.visited.get(place.handle) + if ret is True: # we have been here before and have a loop + print("Error: Place enclosure Loop in GOV data!") + return None + elif ret is not None: # we have added or validated this place + return place.handle, ret + + # need to figure out hierarchy from the place type + hiers = [] + try: + for ptype in place.get_types(): + type_date = ptype.get_date_object() + gtype = ptype.pt_id.split("Gov_")[1] + hierarchy = self.groups[self.type_dic[gtype][1]][1] + hiers.append((type_date, hierarchy)) + except (IndexError, ValueError): + # GOV placetype not found, need to perform an update + hiers = self.__update_place(place) + if hiers is None: + hiers = [] + else: + resp = self.__get_place(gov_id) + if not resp: + return None + place, ref_list, hiers = resp + # this is tracked to detect possible loops + self.visited[place.handle] = True + for ref, date in ref_list: + resp = self.__add_place(ref) + if resp is None: + return None + handle, hiers = resp + + place_ref = PlaceRef() + place_ref.ref = handle + place_ref.set_date_object(date) + for hdate, hier in hiers: + if(hdate.is_empty() or date is None or + date.match_exact(hdate)): + hierarchy = hier + break else: - place, ref_list = self.__get_place(gov_id, self.type_dic, - preferred_lang) - if place.get_name().get_value is not '': - self.dbstate.db.add_place(place, trans) - visited[gov_id] = (place, ref_list) - for ref, date in ref_list: - if (ref not in to_do) and (ref not in visited): - to_do.append(ref) - - for place, ref_list in visited.values(): - if len(ref_list) > 0: - for ref, date in ref_list: - handle = visited[ref][0].handle - place_ref = PlaceRef() - place_ref.ref = handle - place_ref.set_date_object(date) - place.add_placeref(place_ref) - self.dbstate.db.commit_place(place, trans) + hierarchy = P_H(P_H.ADMIN) + place_ref.set_type(hierarchy) + place.add_placeref(place_ref) + with DbTxn(_('Add GOV-id place %s') % gov_id, + self.dbstate.db) as self.trans: + self.dbstate.db.add_place(place, self.trans) + self.visited[place.handle] = hiers + return place.handle, hiers + + groups = { + # key is GOV group, data is tuple(group, hierarchy, priority) + 1: (P_G(P_G.REGION), P_H(P_H.ADMIN), 7, REG), # Administrative + 2: (P_G(_("Civil")), P_H(_("Civil")), 13, OTH), + 3: (P_G(P_G.REGION), P_H(P_H.RELI), 12, REG), # Religious + 4: (P_G(P_G.REGION), P_H(P_H.GEOG), 10, REG), # Geographical + 5: (P_G(P_G.REGION), P_H(P_H.CULT), 14, REG), # Cultural + 6: (P_G(P_G.REGION), P_H(P_H.JUDI), 15, REG), # Judicial + 8: (P_G(P_G.PLACE), P_H(P_H.ADMIN), 8, PLA), # Places + 9: (P_G(P_G.REGION), P_H(_("Transportation")), 11, REG), + 10: (P_G(P_G.UNPOP), P_H(P_H.ADMIN), 9, PLA), # Unpopulated Places + 13: (P_G(P_G.NONE), P_H(P_H.ADMIN), 16, OTH), # Other + 26: (P_G(P_G.COUNTRY), P_H(P_H.ADMIN), 0, COU), # ADM0 Countries + 27: (P_G(P_G.REGION), P_H(P_H.ADMIN), 1, REG), # ADM1 States + 28: (P_G(P_G.REGION), P_H(P_H.ADMIN), 2, REG), # ADM2 Counties + 29: (P_G(P_G.REGION), P_H(P_H.ADMIN), 3, REG), # ADM3 + 30: (P_G(P_G.REGION), P_H(P_H.ADMIN), 4, REG), # ADM4 + 31: (P_G(P_G.PLACE), P_H(P_H.ADMIN), 5, PLA), # ADM5 Cities, etc. + 32: (P_G(P_G.PLACE), P_H(P_H.ADMIN), 6, PLA), # ADM6 + } def __get_types(self): + """ + Get the types tables from GOV. We collect type names (for each + available language), and the GOV group, which is translated to + PlaceGroupType, as well as the hierarchy the type belongs to. + + This collects all the types. + """ type_url = 'http://gov.genealogy.net/types.owl/' - response = urlopen(type_url) - data = response.read() - dom = parseString(data) + dom = self.get_gov_data(type_url) + if not dom: + return False for group in dom.getElementsByTagName('owl:Class') : url_value = group.attributes['rdf:about'].value group_number = url_value.split('#')[1] + g_num = int(group_number.replace('group_', '')) for element in dom.getElementsByTagNameNS("http://gov.genealogy." "net/types.owl#", group_number): - type_number = element.attributes['rdf:about'].value.split('#')[1] - for pname in element.getElementsByTagName('rdfs:label'): - type_lang = pname.attributes['xml:lang'].value - type_text = pname.childNodes[0].data - self.type_dic[type_number,type_lang] = type_text + self.__do_type(dom, element, g_num) for element in dom.getElementsByTagNameNS("http://gov.genealogy." "net/ontology.owl#", 'Type'): - type_number = element.attributes['rdf:about'].value.split('#')[1] - for pname in element.getElementsByTagName('rdfs:label'): - type_lang = pname.attributes['xml:lang'].value - type_text = pname.childNodes[0].data - self.type_dic[type_number,type_lang] = type_text + self.__do_type(dom, element, g_num) + with open(types_file, 'w', encoding='utf-8') as f_p: + json.dump((self.type_dic), f_p, + ensure_ascii=False, indent=2) + load_on_reg(None, None, None, getfile=False) + self.dbstate.db.emit("custom-type-changed") + return True + + def get_gov_data(self, url): + """ + Get GOV data from web with error checking + """ + try: + with urlopen(url, timeout=20) as response: + data = response.read() + except URLError as err: + try: + txt = err.read().decode('utf-8') + except Exception: + txt = '' + ErrorDialog(_('Problem getting data from web'), + msg2=str(err) + '\n' + txt, + parent=self.uistate.window) + return None + except socket.timeout: + ErrorDialog(_('Problem getting data from web'), + msg2=_('Web request Timeout, you can try again...'), + parent=self.uistate.window) + return None - def __get_place(self, gov_id, type_dic, preferred_lang): + dom = parseString(data) + status = dom.getElementsByTagName('rdf:RDF') + if not status: + ErrorDialog(_('Problem getting data from GOV'), + parent=self.uistate.window) + return None + return dom + + def __do_type(self, _dom, element, g_num): + """ + Get the individual type datas from GOV. We collect type names (for + each available language), and the GOV group, which is translated to + PlaceGroupType, as well as the hierarchy the type belongs to. + """ + type_number = element.attributes['rdf:about'].value.split('#')[1] + langs = {} # key:lang, data:name + for pname in element.getElementsByTagName('rdfs:label'): + type_lang = pname.attributes['xml:lang'].value + type_text = pname.childNodes[0].data + langs[type_lang] = type_text[:1].upper() + type_text[1:] + groups = [g_num] + for res in element.getElementsByTagName('rdf:type'): + gx_num = res.attributes['rdf:resource'].value.split('#')[1] + if gx_num == 'Type' or 'group_' not in gx_num: + continue + groups.append(int(gx_num.replace('group_', ''))) + # we may have several groups, need to prioritize best group + group = 13 # original value for 'other' + prior = 20 # low priority + for grp in groups: + tup = self.groups.get(grp) + if not tup: + tup = self.groups[13] + if tup[2] < prior: + prior = tup[2] + group = grp + self.type_dic[type_number] = list(langs, group, False) + + def __get_place(self, gov_id): + """ + Get data on an individual place. + """ gov_url = 'http://gov.genealogy.net/semanticWeb/about/' + quote(gov_id) - response = urlopen(gov_url) - data = response.read() - - dom = parseString(data) + dom = self.get_gov_data(gov_url) + if not dom: + return None top = dom.getElementsByTagName('gov:GovObject') place = Place() + place.handle = create_id() + place.gramps_id = gov_id + place.group = None if not len(top) : - return place, [] + return place, [], P_H(P_H.ADMIN) - count = 0 + types = [] for element in top[0].getElementsByTagName('gov:hasName'): - count += 1 place_name = self.__get_hasname(element) - if count == 1: - place.set_name(place_name) - else: - if place_name.lang == preferred_lang: - place.add_alternative_name(place.get_name()) - place.set_name(place_name) - else: - place.add_alternative_name(place_name) + place.add_name(place_name) for element in top[0].getElementsByTagName('gov:hasType'): - curr_lang = place.get_name().get_language() - place_type = self.__get_hastype(element,curr_lang, type_dic, preferred_lang) - place.set_type(place_type) + place_type, group_tup = self.__get_hastype(element) + types.append((place_type, group_tup)) + types.sort(key=lambda typ: typ[0].date.get_sort_value(), reverse=True) + hiers = [] + for typ in types: + place.add_type(typ[0]) + hiers.append((typ[0].date, typ[1][1])) + place.group = types[0][1][0] for element in top[0].getElementsByTagName('gov:position'): latitude, longitude = self.__get_position(element) place.set_latitude(latitude) @@ -433,41 +725,67 @@ def __get_place(self, gov_id, type_dic, preferred_lang): url = self.__get_hasurl(element) place.add_url(url) - return place, ref_list + return place, ref_list, hiers def __get_hasname(self, element): + """ + Get one name data for a place + """ name = PlaceName() pname = element.getElementsByTagName('gov:PropertyName') if len(pname): value = pname[0].getElementsByTagName('gov:value') if len(value): - name.set_value(value[0].childNodes[0].data) + name.set_value(value[0].childNodes[0].data) language = pname[0].getElementsByTagName('gov:language') if len(language): - name.set_language(ISO_CODE_LOOKUP.get(language[0].childNodes[0].data)) + name.set_language( + ISO_CODE_LOOKUP.get(language[0].childNodes[0].data)) date = self.__get_date_range(pname[0]) name.set_date_object(date) return name - def __get_hastype(self, element, curr_lang, type_dic, preferred_lang): + def __get_hastype(self, element): + """ + get one type data for place + """ place_type = PlaceType() ptype = element.getElementsByTagName('gov:PropertyType') - if len(ptype): - value = ptype[0].getElementsByTagName('gov:type') - if len(value): - type_url = value[0].attributes['rdf:resource'].value - type_code = type_url.split('#')[1] - if tuple([type_code, curr_lang]) in type_dic: - place_type.set_from_xml_str(type_dic.get(tuple([type_code,curr_lang]),'No Type')) - elif tuple([type_code, preferred_lang]) in type_dic: - place_type.set_from_xml_str(type_dic.get(tuple([type_code,preferred_lang]),'No Type')) - elif tuple([type_code, 'de']) in type_dic: - place_type.set_from_xml_str(type_dic.get(tuple([type_code,'de']),'No Type')) - elif tuple([type_code, 'en']) in type_dic: - place_type.set_from_xml_str(type_dic.get(tuple([type_code,'en']),'No Type')) - return place_type + gnum = 8 # default to generic place if nothing found + if not len(ptype): + return place_type, self.groups[gnum] + value = ptype[0].getElementsByTagName('gov:type') + if len(value): + type_url = value[0].attributes['rdf:resource'].value + type_num = type_url.split('#')[1] + tup = self.type_dic.get(type_num, None) + pt_id = "Gov_%s" % type_num + gnum = tup[1] + if not tup: + # TODO Types list out of date? + place_type.set((pt_id, _("Unknown") + ":%s" % type_num)) + else: + for lang in (self.preferred_lang, 'de', 'en'): + t_nam = tup[0].get(lang, None) + if not t_nam: + continue + place_type.set((pt_id, t_nam)) + break + if not tup[2]: # do we have in db? + # if not, then flag as used and update the DATAMAP so + # Place Type selector sees it. + tup[2] = True + nam, nat, _ctr, col, grp, tra = PlaceType.DATAMAP[pt_id] + PlaceType.DATAMAP[pt_id] = (nam, nat, "GOV", col, grp, tra) + date = self.__get_date_range(ptype[0]) + if date: + place_type.set_date_object(date) + return place_type, self.groups[gnum] def __get_position(self, element): + """ + Get position data for place + """ latitude = '' longitude = '' point = element.getElementsByTagName('wgs84:Point') @@ -481,6 +799,9 @@ def __get_position(self, element): return (latitude, longitude) def __get_ispartof(self, element): + """ + Get one enclosure data for a place + """ ref_url = None relation = element.getElementsByTagName('gov:Relation') if len(relation): @@ -495,16 +816,22 @@ def __get_ispartof(self, element): return (ref, date) def __get_hasurl(self, element): + """ + Get one URL associated with a place + """ url = Url() pobj = element.getElementsByTagName('gov:PropertyForObject') if len(pobj): value = pobj[0].getElementsByTagName('gov:value') if len(value): - url.set_path(value[0].childNodes[0].data) - url.set_type(UrlType.WEB_HOME) + url.set_path(value[0].childNodes[0].data) + url.set_type(UrlType.WEB_HOME) return url def __get_date_range(self, element): + """ + Get date data for a place name or type + """ begin_str = None begin = element.getElementsByTagName('gov:timeBegin') if len(begin): @@ -524,3 +851,66 @@ def __get_date_range(self, element): date_str = '' return parser.parse(date_str) if date_str else None + + +def load_on_reg(_dbstate, _uistate, _plugin, getfile=True): + """ + Runs when plugin is registered. + """ + if getfile: + types_file = os.path.join(os.path.dirname(__file__), "gov_types.json") + try: + with open(types_file, encoding="utf-8") as f_p: + GetGOV.type_dic = json.load(f_p) + if len(GetGOV.type_dic) < 250: + GetGOV.type_dic = {} + return + except Exception: + GetGOV.type_dic = {} + return + # The data map (dict) contains a tuple with pt_id as key and data as tuple; + # translatable name + # native name + # color (used for map markers, I suggest picking by probable group) + # probable group (used for legacy XML import and preloading Group in + # place editor) + # gettext method (or None if standard method) + for gnum, data in GetGOV.type_dic.items(): + pt_id = "Gov_" + gnum + for lang in (glocale.lang[0:2], 'de', 'en'): + name = data[0].get(lang, None) + if not name: + continue + break + groups = GetGOV.groups[data[1]] + tup = (name, "GOV:" + gnum, groups[3], groups[0], translate_func) + PlaceType.register_placetype(pt_id, tup, "GOV" if data[2] else "") + PlaceType.update_name_map() + + +def translate_func(ptype, locale=glocale, pt_id=None): + """ + This function provides translations for the locally defined place types. + It is called by the place type display code for the GUI and reports. + + The locale parameter is an instance of a GrampsLocale. This is used to + determine the language for tranlations. (locale.lang) It is also used + as a backup translation if no local po/mo file is present. + + :param ptype: the placetype translatable string + :type ptype: str + :param locale: the backup locale + :type locale: GrampsLocale instance + :param pt_id: the placetype pt_id ("Gov_50" or similar) + :type pt_id: str + :returns: display string of the place type + :rtype: str + """ + gtype = pt_id.split("Gov_")[1] + typ_inf = GetGOV.type_dic.get(gtype) + if not typ_inf: + return _(ptype) + name = typ_inf[0].get(locale.lang) + if not name: + return _(ptype) + return name diff --git a/GraphView/graphview.py b/GraphView/graphview.py index f4d7b32fc..d97f14e8a 100644 --- a/GraphView/graphview.py +++ b/GraphView/graphview.py @@ -137,6 +137,7 @@ class GraphView(NavigationView): ('interface.graphview-show-avatars', True), ('interface.graphview-show-full-dates', False), ('interface.graphview-show-places', False), + ('interface.graphview-place-format', 0), ('interface.graphview-show-lines', 1), ('interface.graphview-show-tags', False), ('interface.graphview-highlight-home-person', True), @@ -402,6 +403,12 @@ def cb_update_show_places(self, _client, _cnxn_id, entry, _data): self.show_places = entry == 'True' self.graph_widget.populate(self.get_active()) + def cb_update_place_fmt(self, _client, _cnxn_id, _entry, _data): + """ + Called when the configuration menu changes the place setting. + """ + self.graph_widget.populate(self.get_active()) + def cb_update_show_tag_color(self, _client, _cnxn_id, entry, _data): """ Called when the configuration menu changes the show tags setting. @@ -539,6 +546,8 @@ def config_connect(self): self.cb_update_show_full_dates) self._config.connect('interface.graphview-show-places', self.cb_update_show_places) + self._config.connect('interface.graphview-place-format', + self.cb_update_place_fmt) self._config.connect('interface.graphview-show-tags', self.cb_update_show_tag_color) self._config.connect('interface.graphview-show-lines', @@ -610,6 +619,17 @@ def layout_config_panel(self, configdialog): configdialog.add_checkbox( grid, _('Show places'), row, 'interface.graphview-show-places') row += 1 + # Place format: + p_fmts = [(0, _("Default"))] + for (indx, fmt) in enumerate(place_displayer.get_formats()): + p_fmts.append((indx + 1, fmt.name)) + active = self._config.get('interface.graphview-place-format') + if active >= len(p_fmts): + active = 1 + configdialog.add_combo(grid, _('Place format'), row, + 'interface.graphview-place-format', + p_fmts, setactive=active) + row += 1 configdialog.add_checkbox( grid, _('Show tags'), row, 'interface.graphview-show-tags') @@ -1674,6 +1694,7 @@ def __init__(self, widget, view): self.transform_scale = 1 + def parse(self, ifile): """ Parse an SVG file produced by Graphviz. @@ -2095,6 +2116,8 @@ def init_dot(self): 'interface.graphview-show-full-dates') self.show_places = self.view._config.get( 'interface.graphview-show-places') + self.place_format = self.view._config.get( + 'interface.graphview-place-format') - 1 self.show_tag_color = self.view._config.get( 'interface.graphview-show-tags') spline = self.view._config.get('interface.graphview-show-lines') @@ -2746,6 +2769,7 @@ def get_person_label(self, person): else: image = '' + # get the person's name name = displayer.display_name(person.get_primary_name()) # name string should not be empty @@ -2805,6 +2829,7 @@ def get_person_label(self, person): else: birth_wraped = birth_str + # get tags table for person and add tooltip for node tag_table = '' if self.show_tag_color: @@ -2901,7 +2926,8 @@ def get_event_string(self, event): empty string """ if event: - place_title = place_displayer.display_event(self.database, event) + place_title = place_displayer.display_event(self.database, event, + fmt=self.place_format) date_object = event.get_date_object() date = '' place = '' diff --git a/PersonEverything/PersonEverything.py b/PersonEverything/PersonEverything.py index 07cfb882f..5e8963d03 100644 --- a/PersonEverything/PersonEverything.py +++ b/PersonEverything/PersonEverything.py @@ -594,13 +594,9 @@ def print_object(self, level, o): self.doc.write_text(" " + _("Type") + " : ") self.doc.end_bold() self.doc.write_text(str(place.get_type())) - self.doc.start_bold() - self.doc.write_text(" " + _("Code") + " : ") - self.doc.end_bold() - self.doc.write_text(place.get_code()) self.doc.end_paragraph() - for name in place.get_alternative_names(): + for name in place.get_names()[1:]: self.doc.start_paragraph("PE-Level%d" % min(level+1, 32)) self.doc.start_bold() self.doc.write_text(_("Alternative Name") + " : ") diff --git a/PlaceCleanup/placecleanup.glade b/PlaceCleanup/placecleanup.glade index 8b4528cda..67d637549 100644 --- a/PlaceCleanup/placecleanup.glade +++ b/PlaceCleanup/placecleanup.glade @@ -1,5 +1,5 @@ - + @@ -101,6 +101,7 @@ False The postal code for the place, if any. True + 16 2 @@ -135,8 +136,8 @@ False Check this box if you want to keep the original Postal code. start + center 5 - 0 True @@ -153,8 +154,8 @@ False Check this box if you want to keep the original Place Type. start + center 5 - 0 right True @@ -164,6 +165,55 @@ 0 + + + True + False + end + Group + + + 6 + 0 + + + + + Orig + True + True + False + Check this box if you want to keep the original Place Type. + start + right + True + + + + 7 + 0 + + + + + True + False + What type of place this is. Eg 'Country', 'City', ... . + True + + + True + 16 + True + + + + + 6 + 1 + 2 + + False @@ -237,7 +287,6 @@ center center True - 0 True @@ -255,6 +304,7 @@ center True False + 10 6 @@ -284,7 +334,6 @@ False Check this box if you want to keep the original Gramps ID. start - 0 True @@ -551,6 +600,9 @@ Select one or more rows and press 'Keep' button to toggle the 'include into plac False center-on-parent dialog + + + False diff --git a/PlaceCleanup/placecleanup.gpr.py b/PlaceCleanup/placecleanup.gpr.py index 79a5ec932..a34624e71 100644 --- a/PlaceCleanup/placecleanup.gpr.py +++ b/PlaceCleanup/placecleanup.gpr.py @@ -32,7 +32,7 @@ authors = ["Paul R. Culley"], authors_email = ["paulr2787@gmail.com"], status = STABLE, - version = '1.0.11', + version = '2.0.0', gramps_target_version = '5.1', fname = "placecleanup.py", gramplet = 'PlaceCleanup', diff --git a/PlaceCleanup/placecleanup.py b/PlaceCleanup/placecleanup.py index f76d823b2..302348a50 100644 --- a/PlaceCleanup/placecleanup.py +++ b/PlaceCleanup/placecleanup.py @@ -1,1155 +1,1215 @@ -# -*- coding: utf-8 -*- -# -# Gramps - a GTK+/GNOME based genealogy program -# -# Copyright (C) 2018 Paul Culley -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# -# Place Cleanup Gramplet. -# -# pylint: disable=attribute-defined-outside-init -#------------------------------------------------------------------------ -# -# Python modules -# -#------------------------------------------------------------------------ -from urllib.request import urlopen, URLError, quote -from xml.dom.minidom import parseString -import os -import sys -import ctypes -import locale -import socket - -#------------------------------------------------------------------------ -# -# GTK modules -# -#------------------------------------------------------------------------ -from gi.repository import Gtk - -#------------------------------------------------------------------------ -# -# Gramps modules -# -#------------------------------------------------------------------------ -from gramps.gen.merge.mergeplacequery import MergePlaceQuery -from gramps.gui.dialog import ErrorDialog, WarningDialog -from gramps.gen.plug import Gramplet -from gramps.gen.db import DbTxn -from gramps.gen.lib import Citation -from gramps.gen.lib import Place, PlaceName, PlaceType, PlaceRef, Url, UrlType -from gramps.gen.lib import Note, NoteType, Repository, RepositoryType, RepoRef -from gramps.gen.lib import StyledText, StyledTextTag, StyledTextTagType -from gramps.gen.lib import Source, SourceMediaType -from gramps.gen.datehandler import get_date -from gramps.gen.config import config -from gramps.gen.constfunc import win -from gramps.gui.display import display_url -from gramps.gui.autocomp import StandardCustomSelector -from gramps.gen.display.place import displayer as _pd -from gramps.gen.utils.location import located_in - - -#------------------------------------------------------------------------ -# -# Internationalisation -# -#------------------------------------------------------------------------ -from gramps.gen.const import GRAMPS_LOCALE as glocale -try: - _trans = glocale.get_addon_translator(__file__) -except ValueError: - _trans = glocale.translation -_ = _trans.gettext - -#------------------------------------------------------------------------ -# -# Constants -# -#------------------------------------------------------------------------ -WIKI_PAGE = ('https://gramps-project.org/wiki/index.php/' - 'Addon:PlaceCleanupGramplet') -PREFS_WIKI = ('https://gramps-project.org/wiki/index.php/' - 'Addon:PlaceCleanupGramplet#Preferences') -#------------------------------------------------------------------------- -# -# configuration -# -#------------------------------------------------------------------------- - -GRAMPLET_CONFIG_NAME = "place_cleanup_gramplet" -CONFIG = config.register_manager(GRAMPLET_CONFIG_NAME) -CONFIG.register("preferences.geo_userid", '') -CONFIG.register("preferences.web_links", True) -CONFIG.register("preferences.add_cit", True) -CONFIG.register("preferences.keep_enclosure", True) -CONFIG.register("preferences.keep_lang", "en fr de pl ru da es fi sw no") -CONFIG.load() - - -#------------------------------------------------------------------------ -# -# PlaceCleanup class -# -#------------------------------------------------------------------------ -class PlaceCleanup(Gramplet): - """ - Gramplet to cleanup places. - Can Look for place that needs attention, or work on current place. - Can search your own places, and merge current with another - Can search GeoNames data on web and load data to a place. - Data includes, Lat/Lon, enclosed by, type, postal code, and alternate - names. - """ - def init(self): - """ - Initialise the gramplet. - """ - self.keepweb = CONFIG.get("preferences.web_links") - self.addcitation = CONFIG.get("preferences.add_cit") - self.geonames_id = CONFIG.get("preferences.geo_userid") - self.keep_enclosure = CONFIG.get("preferences.keep_enclosure") - allowed_languages = CONFIG.get("preferences.keep_lang") - self.allowed_languages = allowed_languages.split() - self.incomp_hndl = '' # a last used handle for incomplete places - self.matches_warn = True # Display the 'too many matches' warning? - root = self.__create_gui() - self.gui.get_container_widget().remove(self.gui.textview) - self.gui.get_container_widget().add(root) - root.show_all() - - def __create_gui(self): - """ - Create and display the GUI components of the gramplet. - """ - self.top = Gtk.Builder() # IGNORE:W0201 - # Found out that Glade does not support translations for plugins, so - # have to do it manually. - base = os.path.dirname(__file__) - glade_file = base + os.sep + "placecleanup.glade" - # This is needed to make gtk.Builder work by specifying the - # translations directory in a separate 'domain' - try: - localedomain = "addon" - localepath = base + os.sep + "locale" - if hasattr(locale, 'bindtextdomain'): - libintl = locale - elif win(): # apparently wants strings in bytes - localedomain = localedomain.encode('utf-8') - localepath = localepath.encode('utf-8') - libintl = ctypes.cdll.LoadLibrary('libintl-8.dll') - else: # mac, No way for author to test this - libintl = ctypes.cdll.LoadLibrary('libintl.dylib') - - libintl.bindtextdomain(localedomain, localepath) - libintl.textdomain(localedomain) - libintl.bind_textdomain_codeset(localedomain, "UTF-8") - # and finally, tell Gtk Builder to use that domain - self.top.set_translation_domain("addon") - except (OSError, AttributeError): - # Will leave it in English - print("Localization of PlaceCleanup failed!") - - self.top.add_from_file(glade_file) - # the results screen items - self.results_win = self.top.get_object("results") - self.alt_store = self.top.get_object("alt_names_liststore") - self.alt_selection = self.top.get_object("alt_names_selection") - self.res_lbl = self.top.get_object("res_label") - self.find_but = self.top.get_object("find_but") - self.top.connect_signals({ - # for results screen - "on_res_ok_clicked" : self.on_res_ok_clicked, - "on_res_cancel_clicked" : self.on_res_cancel_clicked, - "on_keep_clicked" : self.on_keep_clicked, - "on_prim_clicked" : self.on_prim_clicked, - "on_disc_clicked" : self.on_disc_clicked, - "on_alt_row_activated" : self.on_alt_row_activated, - "on_latloncheck" : self.on_latloncheck, - "on_postalcheck" : self.on_postalcheck, - "on_typecheck" : self.on_typecheck, - "on_idcheck" : self.on_idcheck, - # Preferences screen item - "on_pref_help_clicked" : self.on_pref_help_clicked, - # main screen items - "on_find_clicked" : self.on_find_clicked, - "on_prefs_clicked" : self.on_prefs_clicked, - "on_select_clicked" : self.on_select_clicked, - "on_edit_clicked" : self.on_edit_clicked, - "on_next_clicked" : self.on_next_clicked, - "on_res_row_activated" : self.on_select_clicked, - "on_res_sel_changed" : self.on_res_sel_changed, - "on_title_entry_changed" : self.on_title_entry_changed, - "on_help_clicked" : self.on_help_clicked}) - # main screen items - self.res_store = self.top.get_object("res_names_liststore") - self.res_selection = self.top.get_object("res_selection") - self.mainwin = self.top.get_object("main") - return self.mainwin - - # ====================================================== - # gramplet event handlers - # ====================================================== - def on_help_clicked(self, dummy): - ''' Button: Display the relevant portion of GRAMPS manual''' - display_url(WIKI_PAGE) - - def on_res_sel_changed(self, res_sel): - """ Selecting a row in the results list """ - self.top.get_object("select_but").set_sensitive( - res_sel.get_selected()) - - def on_title_entry_changed(self, dummy): - ''' Occurs during edits of the Title box on the main screen. - we use this to reset the GeoNames search row, as the user may be - trying another search term.''' - self.reset_main() - - def on_save(self, *args, **kwargs): - CONFIG.set("preferences.geo_userid", self.geonames_id) - CONFIG.set("preferences.web_links", self.keepweb) - CONFIG.set("preferences.add_cit", self.addcitation) - CONFIG.set("preferences.keep_enclosure", self.keep_enclosure) - CONFIG.set("preferences.keep_lang", ' '.join(self.allowed_languages)) - CONFIG.save() - - def db_changed(self): - self.dbstate.db.connect('place-update', self.update) - self.main() - if not self.dbstate.db.readonly: - self.connect_signal('Place', self.update) - - def main(self): - self.reset_main() - if self.gui.get_child().get_child() == self.results_win: - self.gui.get_child().remove(self.results_win) - self.gui.get_child().add(self.mainwin) - active_handle = self.get_active('Place') - self.top.get_object("edit_but").set_sensitive(False) - self.top.get_object("find_but").set_sensitive(False) - self.top.get_object("title_entry").set_sensitive(False) - if active_handle: - self.place = self.dbstate.db.get_place_from_handle(active_handle) - self.mainwin.hide() - if self.place: - self.set_has_data(True) - title = _pd.display(self.dbstate.db, self.place) - item = self.top.get_object("title_entry") - item.set_text(title) - self.top.get_object("edit_but").set_sensitive(True) - self.top.get_object("find_but").set_sensitive(True) - self.top.get_object("title_entry").set_sensitive(True) - else: - self.set_has_data(False) - self.mainwin.show() - else: - self.set_has_data(False) - - def reset_main(self): - """ Reset the main Gui to default clear """ - self.res_store.clear() - self.res_lbl.set_text(_('No\nMatches')) - self.find_but.set_label(_("Find")) - self.start_row = 0 - self.geo_stage = False - self.top.get_object("select_but").set_sensitive(False) - - def on_find_clicked(self, dummy): - """ find a matching place. First try in the db, then try in the - GeoNames. """ - self.res_store.clear() - self.top.get_object("select_but").set_sensitive(False) - if self.geo_stage: - self.search_geo() - else: - self.geo_stage = True - item = self.top.get_object("title_entry") - title = item.get_text() - self.places = self.lookup_places_by_name(title) - for index, place in enumerate(self.places): - # make sure the place found isn't self, or a place - # enclosed by the working place - if place.handle != self.place.handle and not located_in( - self.dbstate.db, place.handle, self.place.handle): - title = _pd.display(self.dbstate.db, place) - self.res_store.append(row=(index, title, - str(place.place_type))) - if len(self.res_store) > 0: - self.res_lbl.set_text(_('%s\nLocal\nMatches') % - len(self.res_store)) - self.find_but.set_label(_("Find GeoNames")) - else: - self.search_geo() - - def search_geo(self): - """ find a matching place in the geonames, if possible """ - self.res_store.clear() - if not self.geonames_id: - ErrorDialog(_('Need to set GeoNames ID'), - msg2=_('Use the Help button for more information'), - parent=self.uistate.window) - return - # lets get a preferred language - fmt = config.get('preferences.place-format') - placef = _pd.get_formats()[fmt] - self.lang = placef.language - if len(self.lang) != 2: - self.lang = 'en' - if self.lang not in self.allowed_languages: - self.allowed_languages.append(self.lang) - # now lets search for a place in GeoNames - item = self.top.get_object("title_entry") - title = quote(item.get_text().lower().replace('co.', 'county')) - adm = self.top.get_object('adm_check').get_active() - ppl = self.top.get_object('ppl_check').get_active() - spot = self.top.get_object('spot_check').get_active() - geo_url = ( - 'http://api.geonames.org/search?q=%s' - '&maxRows=10&style=SHORT&lang=en&isNameRequired=True' - '%s%s%s&username=%s&startRow=%s' % - (title, - '&featureClass=A' if adm else '', - '&featureClass=P' if ppl else '', - '&featureClass=S' if spot else '', - self.geonames_id, self.start_row)) - dom = self.get_geo_data(geo_url) - if not dom: - return - - g_names = dom.getElementsByTagName('geoname') - if not g_names: - WarningDialog(_('No matches were found'), - msg2=_('Try changing the Title, or use the "Edit"' - ' button to finish this level of the place' - ' manually.'), - parent=self.uistate.window) - return - # let's check the total results; if too many, warn user and set up for - # another pass through the search. - value = dom.getElementsByTagName('totalResultsCount') - if value: - totalresults = int(value[0].childNodes[0].data) - self.res_lbl.set_text(_("%s\nGeoNames\nMatches") % totalresults) - if totalresults > 10: - self.start_row += 10 - if self.matches_warn: - self.matches_warn = False - WarningDialog( - _('%s matches were found') % totalresults, - msg2=_('Only 10 matches are shown.\n' - 'To see additional results, press the' - ' search button again.\n' - 'Or try changing the Title with' - ' more detail, such as a country.'), - parent=self.uistate.window) - index = 0 - self.places = [] - for g_name in g_names: - # got a match, now get its hierarchy for display. - value = g_name.getElementsByTagName('geonameId') - geoid = value[0].childNodes[0].data - value = g_name.getElementsByTagName('fcl') - fcl = value[0].childNodes[0].data - value = g_name.getElementsByTagName('fcode') - _type = fcl + ':' + value[0].childNodes[0].data - geo_url = ('http://api.geonames.org/hierarchy?geonameId=%s' - '&lang=%s&username=%s' % - (geoid, self.lang, self.geonames_id)) - hier = self.get_geo_data(geo_url) - if not hier: - return - h_names = hier.getElementsByTagName('geoname') - h_name_list = [] - h_geoid_list = [] - for h_name in h_names: - value = h_name.getElementsByTagName('fcl') - fcl = value[0].childNodes[0].data - if fcl not in 'APS': - # We don't care about Earth or continent (yet) - continue - value = h_name.getElementsByTagName('name') - h_name_list.append(value[0].childNodes[0].data) - value = h_name.getElementsByTagName('geonameId') - h_geoid_list.append('GEO' + value[0].childNodes[0].data) - # make sure that this place isn't already enclosed by our place. - bad = self.place.gramps_id in h_geoid_list[:-1] - # assemble a title for the result - h_name_list.reverse() - h_geoid_list.reverse() - if bad: - title = ('' + _(', ').join(h_name_list) + '') - else: - title = _(', ').join(h_name_list) - row = (index, title, _type) - self.res_store.append(row=row) - self.places.append(('GEO' + geoid, title, h_geoid_list, - h_name_list, bad)) - index += 1 - while Gtk.events_pending(): - Gtk.main_iteration() - - def get_geo_data(self, geo_url): - """ Get GeoNames data from web with error checking """ - print(geo_url) - try: - with urlopen(geo_url, timeout=20) as response: - data = response.read() - except URLError as err: - try: - txt = err.read().decode('utf-8') - except: - txt = '' - ErrorDialog(_('Problem getting data from web'), - msg2=str(err) +'\n' + txt, - parent=self.uistate.window) - return None - except socket.timeout: - ErrorDialog(_('Problem getting data from web'), - msg2=_('Web request Timeout, you can try again...'), - parent=self.uistate.window) - return None - - dom = parseString(data) - status = dom.getElementsByTagName('status') - if status: - err = status[0].getAttribute("message") - ErrorDialog(_('Problem getting data from GeoNames'), - msg2=err, - parent=self.uistate.window) - return None - return dom - - def on_next_clicked(self, dummy): - """ find a incomplete place in the db, if possible """ - self.reset_main() - place = self.find_an_incomplete_place() - if place: - self.set_active('Place', place.handle) - - def on_select_clicked(self, *dummy): - """ If the selected place is mergable, merge it, otherwise Open - completion screen """ - model, _iter = self.res_selection.get_selected() - if not _iter: - return - (index, ) = model.get(_iter, 0) - place = self.places[index] - if not isinstance(place, Place): - # we have a geoname_id - if place[4]: - return - # check if we might already have it in db - t_place = self.dbstate.db.get_place_from_gramps_id(place[0]) - if not t_place or t_place.handle == self.place.handle: - # need to process the GeoNames ID for result - self.gui.get_child().remove(self.mainwin) - self.gui.get_child().add(self.results_win) - if not self.geoparse(*place): - return - self.res_gui() - return - else: - # turns out we already have this place, under different name! - place = t_place - # we have a Gramps Place, need to merge - if place.handle == self.place.handle: - # found self, nothing to do. - return - if(located_in(self.dbstate.db, place.handle, self.place.handle) or - located_in(self.dbstate.db, self.place.handle, place.handle)): - # attempting to create a place loop, not good! - ErrorDialog(_('Place cycle detected'), - msg2=_("One of the places you are merging encloses " - "the other!\n" - "Please choose another place."), - parent=self.uistate.window) - return - # lets clean up the place name - self.place.name.value = self.place.name.value.split(',')[0].strip() - place_merge = MergePlaceQuery(self.dbstate, place, self.place) - place_merge.execute() - # after merge we should select merged result - self.set_active('Place', place.handle) - - adm_table = { - # note the True/False in the following indicates the certainty that the - # entry is correct. If it is only sometimes correct, and the name - # might have a different type embedded in it, then use False. - 'US': {'ADM1': (PlaceType.STATE, True), - 'ADM2': (PlaceType.COUNTY, False), - 'ADM3': (PlaceType.TOWN, False)}, - 'CA': {'ADM1': (PlaceType.PROVINCE, True), - 'ADM2': (PlaceType.REGION, False)}, - 'GB': {'ADM1': (PlaceType.COUNTRY, True), - 'ADM2': (PlaceType.REGION, True), - 'ADM3': (PlaceType.COUNTY, False), - 'ADM4': (PlaceType.BOROUGH, False)}, - 'FR': {'ADM1': (PlaceType.REGION, True), - 'ADM2': (PlaceType.DEPARTMENT, True)}, - 'DE': {'ADM1': (PlaceType.STATE, True), - 'ADM2': (PlaceType.COUNTY, False), - 'ADM3': ('Amt', False)}} - - def geoparse(self, geoid, title, h_geoid_list, h_name_list, *dummy): - """ get data for place and parse out g_name dom structure into the - NewPlace structure """ - geo_url = ('http://api.geonames.org/get?geonameId=%s&style=FULL' - '&username=%s' % (geoid.replace('GEO', ''), - self.geonames_id)) - dom = self.get_geo_data(geo_url) - if not dom: - return False - - g_name = dom.getElementsByTagName('geoname')[0] - self.newplace = NewPlace(title) - self.newplace.geoid = geoid - self.newplace.gramps_id = geoid - value = g_name.getElementsByTagName('lat') - self.newplace.lat = str(value[0].childNodes[0].data) - value = g_name.getElementsByTagName('lng') - self.newplace.long = str(value[0].childNodes[0].data) - value = g_name.getElementsByTagName('toponymName') - topname = value[0].childNodes[0].data - new_place = PlaceName() - new_place.set_value(topname) - new_place.set_language("") - # make sure we have the topname in the names list and default to - # primary - self.newplace.add_name(new_place) - self.newplace.name = new_place - # lets parse the alternative names - alt_names = g_name.getElementsByTagName('alternateName') - for a_name in alt_names: - pattr = a_name.getAttribute("lang") - value = a_name.childNodes[0].data - if pattr == "post": - if self.newplace.code: - self.newplace.code += " " + value - else: - self.newplace.code = value - elif pattr == "link": - url = Url() - url.set_path(value) - url.set_description(value) - url.set_type(UrlType(UrlType.WEB_HOME)) - self.newplace.links.append(url) - elif pattr not in ['iata', 'iaco', 'faac', 'wkdt', 'unlc']: - new_place = PlaceName() - new_place.set_language(pattr) - new_place.set_value(value) - self.newplace.add_name(new_place) - if a_name.hasAttribute('isPreferredName') and ( - pattr and pattr == self.lang): - # if not preferred lang, we use topo name, otherwise - # preferred name for lang - self.newplace.name = new_place - # Try to deduce PlaceType: - # If populated place, set as City. Long description could over-ride - # Parse long description, looking for keyword (Region, County, ...) - # Top-level must be a country. - # Children of USA are States. - # Children of Canada are Provinces. - # - value = g_name.getElementsByTagName('fcl') - fcl = value[0].childNodes[0].data - value = g_name.getElementsByTagName('fcode') - fcode = value[0].childNodes[0].data - value = g_name.getElementsByTagName('countryCode') - countrycode = value[0].childNodes[0].data - self.newplace.place_type = PlaceType(PlaceType.UNKNOWN) - ptype = PlaceType() - # scan thorough names looking for name portion that matches a Placetype - for name in self.newplace.names: - for tname in name.value.split(' '): - ptype.set_from_xml_str(tname.capitalize()) - if ptype != PlaceType.CUSTOM: - self.newplace.place_type = ptype - break - # see if it is a translated PlaceType - ptype.set(tname.capitalize()) - if ptype != PlaceType.CUSTOM: - self.newplace.place_type = ptype - break - # see if it is an already added custom type - cust_types = self.dbstate.db.get_place_types() - if tname.capitalize() in cust_types: - self.newplace.place_type = ptype - break - else: - # Continue if the inner loop wasn't broken. - continue - # Inner loop was broken, break the outer. - break - if fcl == 'P': - self.newplace.place_type = PlaceType(PlaceType.CITY) - elif fcode == 'PRSH': - self.newplace.place_type = PlaceType(PlaceType.PARISH) - elif 'PCL' in fcode: - self.newplace.place_type = PlaceType(PlaceType.COUNTRY) - elif 'ADM' in fcode: - if countrycode in self.adm_table: - _ptype = self.adm_table[countrycode].get(fcode[:4]) - if _ptype and (_ptype[1] or - self.newplace.place_type.is_default()): - self.newplace.place_type = PlaceType(_ptype[0]) - # save a parent for enclosing - if len(h_geoid_list) > 1: - # we have a parent - self.newplace.parent_names = h_name_list[1:] - self.newplace.parent_ids = h_geoid_list[1:] - return True - - def on_edit_clicked(self, dummy): - """User wants to jump directly to the results view to finish off - the place, possibly because a place was not found""" -# if ',' in self.place.name.value: -# name = self.place.name.value -# else: - name = self.place.name.value - self.newplace = NewPlace(name) - names = name.split(',') - names = [name.strip() for name in names] - self.newplace.name = PlaceName() - self.newplace.name.value = names[0] - self.newplace.gramps_id = self.place.gramps_id - self.newplace.lat = self.place.lat - self.newplace.long = self.place.long - self.newplace.code = self.place.code - if self.place.place_type == PlaceType.UNKNOWN: - self.newplace.place_type = PlaceType(PlaceType.UNKNOWN) - if any(i.isdigit() for i in self.newplace.name.value): - self.newplace.place_type = PlaceType(PlaceType.STREET) - ptype = PlaceType() - for tname in self.newplace.name.value.split(' '): - # see if it is an English PlaceType - ptype.set_from_xml_str(tname.capitalize()) - if ptype != PlaceType.CUSTOM: - self.newplace.place_type = ptype - break - # see if it is a translated PlaceType - ptype.set(tname.capitalize()) - if ptype != PlaceType.CUSTOM: - self.newplace.place_type = ptype - break - # see if it is an already added custom type - cust_types = self.dbstate.db.get_place_types() - if tname.capitalize() in cust_types: - self.newplace.place_type = ptype - else: - self.newplace.place_type = self.place.place_type - self.newplace.add_name(self.newplace.name) - self.newplace.add_name(self.place.name) - self.newplace.add_names(self.place.alt_names) - if self.place.placeref_list: - # If we already have an enclosing place, use it. - parent = self.dbstate.db.get_place_from_handle( - self.place.placeref_list[0].ref) - self.newplace.parent_ids = [parent.gramps_id] - elif len(names) > 1: - # we have an enclosing place, according to the name string - self.newplace.parent_names = names[1:] - self.gui.get_child().remove(self.mainwin) - self.gui.get_child().add(self.results_win) - self.res_gui() - -# Results view - - def res_gui(self): - """ Fill in the results display with values from new place.""" - self.alt_store.clear() - # Setup sort on 'Inc" column so Primary is a top with checks next - self.alt_store.set_sort_func(0, inc_sort, None) - self.alt_store.set_sort_column_id(0, 0) - # Merge old name and alt names into new set - self.newplace.add_name(self.place.name) - self.newplace.add_names(self.place.alt_names) - # Fill in ohter fields - self.top.get_object('res_title').set_text(self.newplace.title) - self.top.get_object('primary').set_text(self.newplace.name.value) - self.on_idcheck() - self.on_latloncheck() - self.on_postalcheck() - self.on_typecheck() - # Fill in names list - for index, name in enumerate(self.newplace.names): - if self.newplace.name == name: - inc = 'P' - elif name.lang in self.allowed_languages or ( - name.lang == 'abbr' or name.lang == 'en' or not name.lang): - inc = '\u2714' # Check mark - else: - inc = '' - row = (inc, name.value, name.lang, get_date(name), index) - self.alt_store.append(row=row) - -# Results dialog items - - def on_res_ok_clicked(self, dummy): - """ Accept changes displayed and commit to place. - Also find or create a new enclosing place from parent. """ - # do the names - namelist = [] - for row in self.alt_store: - if row[0] == 'P': - self.place.name = self.newplace.names[row[4]] - elif row[0] == '\u2714': - namelist.append(self.newplace.names[row[4]]) - self.place.alt_names = namelist - # Lat/lon/ID/code/type - self.place.lat = self.top.get_object('latitude').get_text() - self.place.long = self.top.get_object('longitude').get_text() - self.place.gramps_id = self.top.get_object('grampsid').get_text() - self.place.code = self.top.get_object('postal').get_text() - self.place.place_type.set(self.type_combo.get_values()) - # Add in URLs if wanted - if self.keepweb: - for url in self.newplace.links: - self.place.add_url(url) - # Enclose in the next level place - next_place = False - parent = None - if not self.keep_enclosure or not self.place.placeref_list: - if self.newplace.parent_ids: - # we might have a parent with geo id 'GEO12345' - parent = self.dbstate.db.get_place_from_gramps_id( - self.newplace.parent_ids[0]) - if not parent and self.newplace.parent_names: - # make one, will have to be examined/cleaned later - parent = Place() - parent.title = ', '.join(self.newplace.parent_names) - name = PlaceName() - name.value = parent.title - parent.name = name - parent.gramps_id = self.newplace.parent_ids[0] - with DbTxn(_("Add Place (%s)") % parent.title, - self.dbstate.db) as trans: - self.dbstate.db.add_place(parent, trans) - next_place = True - if parent: - if located_in(self.dbstate.db, parent.handle, - self.place.handle): - # attempting to create a place loop, not good! - ErrorDialog(_('Place cycle detected'), - msg2=_("The place you chose is enclosed in the" - " place you are workin on!\n" - "Please cancel and choose another " - "place."), - parent=self.uistate.window) - return - # check to see if we already have the enclosing place - already_there = False - for pref in self.place.placeref_list: - if parent.handle == pref.ref: - already_there = True - break - if not already_there: - placeref = PlaceRef() - placeref.set_reference_handle(parent.handle) - self.place.set_placeref_list([placeref]) - # Add in Citation/Source if wanted - if self.addcitation and self.newplace.geoid: - src_hndl = self.find_or_make_source() - cit = Citation() - cit.set_reference_handle(src_hndl) - cit.set_page("GeoNames ID: %s" % - self.newplace.geoid.replace('GEO', '')) - with DbTxn(_("Add Citation (%s)") % "GeoNames", - self.dbstate.db) as trans: - self.dbstate.db.add_citation(cit, trans) - self.place.add_citation(cit.handle) - # We're finally ready to commit the updated place - with DbTxn(_("Edit Place (%s)") % self.place.title, - self.dbstate.db) as trans: - self.dbstate.db.commit_place(self.place, trans) - # Jump to enclosing place to clean it if necessary - if next_place: - self.set_active('Place', parent.handle) - self.place = parent - # if geoparse fails, leave us at main view - if self.newplace.parent_ids and \ - self.geoparse(self.newplace.parent_ids[0], - _(", ").join(self.newplace.parent_names), - self.newplace.parent_ids, - self.newplace.parent_names): - # geoparse worked, lets put up the results view - self.gui.get_child().remove(self.mainwin) - self.gui.get_child().add(self.results_win) - self.res_gui() - return - self.reset_main() - if self.gui.get_child().get_child() == self.results_win: - self.gui.get_child().remove(self.results_win) - self.gui.get_child().add(self.mainwin) - - def on_res_cancel_clicked(self, dummy): - """ Cancel operations on this place. """ - self.gui.get_child().remove(self.results_win) - self.gui.get_child().add(self.mainwin) - - def on_keep_clicked(self, dummy): - """ Keep button clicked. Mark selected names rows to keep. """ - model, rows = self.alt_selection.get_selected_rows() - for row in rows: - if model[row][0] == 'P': - continue - model[row][0] = '\u2714' - - def on_prim_clicked(self, dummy): - """ Primary button clicked. Mark first row in selection as Primary - name, any previous primary as keep """ - model, rows = self.alt_selection.get_selected_rows() - if not rows: - return - # Clear prior primary - for row in model: - if row[0] == 'P': - row[0] = '\u2714' - # mark new one. - self.top.get_object('primary').set_text(model[rows[0]][1]) - model[rows[0]][0] = 'P' - - def on_disc_clicked(self, dummy): - """ Discard button clicked. Unmark selected rows. """ - model, rows = self.alt_selection.get_selected_rows() - for row in rows: - if model[row][0] == 'P': - continue - model[row][0] = '' - - def on_alt_row_activated(self, *dummy): - """ Toggle keep status for selected row. Seems this only works for - last selected row.""" - model, rows = self.alt_selection.get_selected_rows() - for row in rows: - if model[row][0] == 'P': - continue - if model[row][0] == '': - model[row][0] = '\u2714' - else: - model[row][0] = '' - - def on_latloncheck(self, *dummy): - """ Check toggled; if active, load lat/lon from original place, else - use lat/lon from gazetteer """ - obj = self.top.get_object("latloncheck") - if not dummy: - # inititlization - obj.set_sensitive(True) - obj.set_active(False) - place = self.newplace - if self.place.lat and self.place.long: - if obj.get_active(): - place = self.place - else: - obj.set_sensitive(False) - self.top.get_object('latitude').set_text(place.lat) - self.top.get_object('longitude').set_text(place.long) - - def on_postalcheck(self, *dummy): - """ Check toggled; if active, load postal from original place, else - use postal from gazetteer """ - obj = self.top.get_object("postalcheck") - if not dummy: - # inititlization - obj.set_sensitive(True) - obj.set_active(False) - place = self.newplace - if self.place.code: - if obj.get_active(): - place = self.place - obj.set_sensitive(True) - else: - obj.set_sensitive(False) - self.top.get_object('postal').set_text(place.code) - - def on_typecheck(self, *dummy): - """ Check toggled; if active, load type from original place, else - use type from gazetteer """ - obj = self.top.get_object("typecheck") - combo = self.top.get_object('place_type') - additional = sorted(self.dbstate.db.get_place_types(), - key=lambda s: s.lower()) - self.type_combo = StandardCustomSelector(PlaceType().get_map(), combo, - PlaceType.CUSTOM, - PlaceType.UNKNOWN, - additional) - if not dummy: - # inititlization - obj.set_sensitive(True) - obj.set_active(False) - place = self.newplace - if(self.place.place_type and - self.place.place_type != PlaceType.UNKNOWN): - if obj.get_active(): - place = self.place - else: - obj.set_sensitive(False) - self.type_combo.set_values((int(place.place_type), - str(place.place_type))) - - def on_idcheck(self, *dummy): - """ Check toggled; if active, load gramps_id from original place, else - use geonamesid from gazetteer """ - obj = self.top.get_object("idcheck") - if not dummy: - # inititlization - obj.set_sensitive(True) - obj.set_active(False) - place = self.newplace - if self.place.gramps_id: - if obj.get_active(): - place = self.place - else: - obj.set_sensitive(False) - self.top.get_object('grampsid').set_text(place.gramps_id) - - def find_or_make_source(self): - """ Find or create a source. - returns handle to source.""" - for hndl in self.dbstate.db.get_source_handles(): - if self.dbstate.db.get_raw_source_data(hndl)[2] == 'GeoNames': - return hndl - # No source found, lets add one with associated repo and note - repo = Repository() - repo.set_name("www.geonames.org") - rtype = RepositoryType(RepositoryType.WEBSITE) - repo.set_type(rtype) - url = Url() - url.set_path('http://www.geonames.org/') - url.set_description(_('GeoNames web site')) - url.set_type(UrlType(UrlType.WEB_HOME)) - repo.add_url(url) - url = Url() - url.set_path('marc@geonames.org') - url.set_description(_('GeoNames author')) - url.set_type(UrlType(UrlType.EMAIL)) - repo.add_url(url) - - note_txt = StyledText(_( - 'GeoNames was founded by Marc Wick. You can reach him at ')) - note_txt += StyledText('marc@geonames.org' + '\n') - note_txt += StyledText(_( - 'GeoNames is a project of Unxos GmbH, Weingartenstrasse 8,' - ' 8708 Männedorf, Switzerland.\nThis work is licensed under a ')) - note_txt += linkst( - _('Creative Commons Attribution 3.0 License'), - 'https://creativecommons.org/licenses/by/3.0/legalcode') - - new_note = Note() - new_note.set_styledtext(note_txt) - new_note.set_type(NoteType.REPO) - src = Source() - src.title = 'GeoNames' - src.author = 'Marc Wick' - repo_ref = RepoRef() - mtype = SourceMediaType(SourceMediaType.ELECTRONIC) - repo_ref.set_media_type(mtype) - with DbTxn(_("Add Souce/Repo/Note (%s)") % "GeoNames", - self.dbstate.db) as trans: - - self.dbstate.db.add_note(new_note, trans) - repo.add_note(new_note.get_handle()) - self.dbstate.db.add_repository(repo, trans) - repo_ref.set_reference_handle(repo.handle) - src.add_repo_reference(repo_ref) - self.dbstate.db.add_source(src, trans) - return src.handle - -# Preferences dialog items - - def on_prefs_clicked(self, dummy): - """ Button: display preference dialog """ - top = self.top.get_object("pref_dialog") - top.set_transient_for(self.uistate.window) - parent_modal = self.uistate.window.get_modal() - if parent_modal: - self.uistate.window.set_modal(False) - keepweb = self.top.get_object("keepweb") - keepweb.set_active(self.keepweb) - addcit = self.top.get_object("addcitation") - addcit.set_active(self.addcitation) - # for some reason you can only set the radiobutton to True - self.top.get_object( - "enc_radio_but_keep" if self.keep_enclosure - else "enc_radio_but_repl").set_active(True) - geoid = self.top.get_object("geonames_id_entry") - geoid.set_text(self.geonames_id) - keepalt = self.top.get_object("keep_alt_entry") - keepalt.set_text(' '.join(self.allowed_languages)) - top.show() - top.run() - if self.uistate.window and parent_modal: - self.uistate.window.set_modal(True) - self.geonames_id = geoid.get_text() - self.addcitation = addcit.get_active() - self.keepweb = keepweb.get_active() - self.keep_enclosure = self.top.get_object( - "enc_radio_but_keep").get_active() - self.allowed_languages = keepalt.get_text().split() - top.hide() - - def on_pref_help_clicked(self, dummy): - ''' Button: Display the relevant portion of GRAMPS manual''' - display_url(PREFS_WIKI) - - def lookup_places_by_name(self, search_name): - """ In local db. Only completed places are matched. - We may want to try some better matching algorithms, possibly - something from difflib""" - search_name = search_name.lower().split(',') - places = [] - for place in self.dbstate.db.iter_places(): - if (place.get_type() != PlaceType.UNKNOWN and - (place.get_type() == PlaceType.COUNTRY or - (place.get_type() != PlaceType.COUNTRY and - place.get_placeref_list()))): - # valid place, get all its names - for name in place.get_all_names(): - if name.get_value().lower() == search_name[0]: - places.append(place) - break - return places - - def find_an_incomplete_place(self): - """ in our db. Will return with a place (and active set to place) - or None if no incomplete places, in which case active will be the same. - Will also find unused places, and offer to delete.""" - p_hndls = self.dbstate.db.get_place_handles() - if not p_hndls: - return None # in case there aren't any - # keep handles in an order to avoid inconsistant - # results when db returns them in different orders. - p_hndls.sort() - # try to find the handle after the previously scanned handle in the - # list. - found = False - for indx, hndl in enumerate(p_hndls): - if hndl > self.incomp_hndl: - found = True - break - if not found: - indx = 0 - # now, starting from previous place, look for incomplete place - start = indx - while True: - hndl = p_hndls[indx] - place_data = self.dbstate.db.get_raw_place_data(hndl) - p_type = place_data[8][0] # place_type - refs = list(self.dbstate.db.find_backlink_handles(hndl)) - if(p_type == PlaceType.UNKNOWN or - not refs or - p_type != PlaceType.COUNTRY and - not place_data[5]): # placeref_list - # need to get view to this place... - self.set_active("Place", hndl) - self.incomp_hndl = hndl - if not refs: - WarningDialog( - _('This Place is not used!'), - msg2=_('You should delete it, or, if it contains ' - 'useful notes or other data, use the Find to ' - 'merge it into a valid place.'), - parent=self.uistate.window) - return self.dbstate.db.get_place_from_handle(hndl) - indx += 1 - if indx == len(p_hndls): - indx = 0 - if indx == start: - break - return None - - -class NewPlace(): - """ structure to store data about a found place""" - def __init__(self, title): - self.title = title - self.gramps_id = '' - self.lat = '' - self.long = '' - self.code = '' - self.place_type = None - self.names = [] # all names, including alternate, acts like a set - self.name = PlaceName() - self.links = [] - self.geoid = '' - self.parent_ids = [] # list of gramps_ids in hierarchical order - self.parent_names = [] # list of names in hierarchical order - - def add_name(self, name): - """ Add a name to names list without repeats """ - if ',' in name.value: - name.value = name.value.split(',')[0] - return - if name not in self.names: - self.names.append(name) - - def add_names(self, names): - """ Add names to names list without repeats """ - for name in names: - self.add_name(name) - -#------------------------------------------------------------------------ -# -# Functions -# -#------------------------------------------------------------------------ - - -def linkst(text, url): - """ Return text as link styled text - """ - tags = [StyledTextTag(StyledTextTagType.LINK, url, [(0, len(text))])] - return StyledText(text, tags) - - -def inc_sort(model, row1, row2, user_data): - value1 = model.get_value(row1, 0) - value2 = model.get_value(row2, 0) - if value1 == value2: - return 0 - if value1 == 'P': - return -1 - if value2 == 'P': - return 1 - if value2 > value1: - return 1 - else: - return -1 - -# if found a matching place in our db, merge -# if not, lookup on web, present list of candidates -# start with simple search -# extend each result with heirarchy query -# Web button, if pressed again, more answers. - -# Alternative name Lang field: -# unlc:Un/locodes -# post:for postal codes -# iata,icao,faac: for airport codes -# fr_1793:French Revolution names -# abbr:abbreviation -# link: to a website (mostly to wikipedia) -# wkdt: for the wikidata id -# otherwise 2/3 char ISO639 lang code -# empty: just an alternative name -# Alternative name other fields: -# isPreferredName="true" -# isShortName="true" -# isColloquial="true" -# isHistoric="true" +# -*- coding: utf-8 -*- +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2018 Paul Culley +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# +# Place Cleanup Gramplet. +# +# pylint: disable=attribute-defined-outside-init +#------------------------------------------------------------------------ +# +# Python modules +# +#------------------------------------------------------------------------ +from urllib.request import urlopen, URLError, quote +from xml.dom.minidom import parseString +import os +import ctypes +import locale +import socket + +#------------------------------------------------------------------------ +# +# GTK modules +# +#------------------------------------------------------------------------ +from gi.repository import Gtk + +#------------------------------------------------------------------------ +# +# Gramps modules +# +#------------------------------------------------------------------------ +from gramps.gen.merge.mergeplacequery import MergePlaceQuery +from gramps.gui.dialog import ErrorDialog, WarningDialog +from gramps.gen.plug import Gramplet +from gramps.gen.db import DbTxn +from gramps.gen.lib import (Attribute, AttributeType, Citation, Note, NoteType, + Place, PlaceAbbrev, PlaceAbbrevType, + PlaceGroupType as P_G, PlaceName, + PlaceRef, PlaceType, + Repository, RepositoryType, RepoRef, + Source, SourceMediaType, + StyledText, StyledTextTag, StyledTextTagType, + Url, UrlType) +from gramps.gen.datehandler import get_date +from gramps.gen.config import config +from gramps.gen.lib.const import IDENTICAL, EQUAL +from gramps.gen.constfunc import win +from gramps.gui.display import display_url +from gramps.gui.widgets.placetypeselector import PlaceTypeSelector +from gramps.gui.autocomp import StandardCustomSelector +from gramps.gen.display.place import displayer as _pd +from gramps.gen.utils.location import (located_in) + + +#------------------------------------------------------------------------ +# +# Internationalisation +# +#------------------------------------------------------------------------ +from gramps.gen.const import GRAMPS_LOCALE as glocale +try: + _trans = glocale.get_addon_translator(__file__) +except ValueError: + _trans = glocale.translation +_ = _trans.gettext + +#------------------------------------------------------------------------ +# +# Constants +# +#------------------------------------------------------------------------ +WIKI_PAGE = ('https://gramps-project.org/wiki/index.php/' + 'Addon:PlaceCleanupGramplet') +PREFS_WIKI = ('https://gramps-project.org/wiki/index.php/' + 'Addon:PlaceCleanupGramplet#Preferences') +#------------------------------------------------------------------------- +# +# configuration +# +#------------------------------------------------------------------------- + +GRAMPLET_CONFIG_NAME = "place_cleanup_gramplet" +CONFIG = config.register_manager(GRAMPLET_CONFIG_NAME) +CONFIG.register("preferences.geo_userid", '') +CONFIG.register("preferences.web_links", True) +CONFIG.register("preferences.add_cit", True) +CONFIG.register("preferences.keep_enclosure", True) +CONFIG.register("preferences.keep_lang", "en fr de pl ru da es fi sw no") +CONFIG.load() + + +#------------------------------------------------------------------------ +# +# PlaceCleanup class +# +#------------------------------------------------------------------------ +class PlaceCleanup(Gramplet): + """ + Gramplet to cleanup places. + Can Look for place that needs attention, or work on current place. + Can search your own places, and merge current with another + Can search GeoNames data on web and load data to a place. + Data includes, Lat/Lon, enclosed by, type, postal code, and alternate + names. + """ + def init(self): + """ + Initialise the gramplet. + """ + self.keepweb = CONFIG.get("preferences.web_links") + self.addcitation = CONFIG.get("preferences.add_cit") + self.geonames_id = CONFIG.get("preferences.geo_userid") + self.keep_enclosure = CONFIG.get("preferences.keep_enclosure") + allowed_languages = CONFIG.get("preferences.keep_lang") + self.allowed_languages = allowed_languages.split() + self.incomp_hndl = '' # a last used handle for incomplete places + self.matches_warn = True # Display the 'too many matches' warning? + root = self.__create_gui() + self.gui.get_container_widget().remove(self.gui.textview) + self.gui.get_container_widget().add(root) + root.show_all() + + def __create_gui(self): + """ + Create and display the GUI components of the gramplet. + """ + self.top = Gtk.Builder() # IGNORE:W0201 + # Found out that Glade does not support translations for plugins, so + # have to do it manually. + base = os.path.dirname(__file__) + glade_file = base + os.sep + "placecleanup.glade" + # This is needed to make gtk.Builder work by specifying the + # translations directory in a separate 'domain' + try: + localedomain = "addon" + localepath = base + os.sep + "locale" + if hasattr(locale, 'bindtextdomain'): + libintl = locale + elif win(): # apparently wants strings in bytes + localedomain = localedomain.encode('utf-8') + localepath = localepath.encode('utf-8') + libintl = ctypes.cdll.LoadLibrary('libintl-8.dll') + else: # mac, No way for author to test this + libintl = ctypes.cdll.LoadLibrary('libintl.dylib') + + libintl.bindtextdomain(localedomain, localepath) + libintl.textdomain(localedomain) + libintl.bind_textdomain_codeset(localedomain, "UTF-8") + # and finally, tell Gtk Builder to use that domain + self.top.set_translation_domain("addon") + except (OSError, AttributeError): + # Will leave it in English + print("Localization of PlaceCleanup failed!") + + self.top.add_from_file(glade_file) + # the results screen items + self.results_win = self.top.get_object("results") + self.alt_store = self.top.get_object("alt_names_liststore") + self.alt_selection = self.top.get_object("alt_names_selection") + self.res_lbl = self.top.get_object("res_label") + self.find_but = self.top.get_object("find_but") + self.top.connect_signals({ + # for results screen + "on_res_ok_clicked" : self.on_res_ok_clicked, + "on_res_cancel_clicked" : self.on_res_cancel_clicked, + "on_keep_clicked" : self.on_keep_clicked, + "on_prim_clicked" : self.on_prim_clicked, + "on_disc_clicked" : self.on_disc_clicked, + "on_alt_row_activated" : self.on_alt_row_activated, + "on_latloncheck" : self.on_latloncheck, + "on_postalcheck" : self.on_postalcheck, + "on_typecheck" : self.on_typecheck, + "on_groupcheck" : self.on_groupcheck, + "on_idcheck" : self.on_idcheck, + # Preferences screen item + "on_pref_help_clicked" : self.on_pref_help_clicked, + # main screen items + "on_find_clicked" : self.on_find_clicked, + "on_prefs_clicked" : self.on_prefs_clicked, + "on_select_clicked" : self.on_select_clicked, + "on_edit_clicked" : self.on_edit_clicked, + "on_next_clicked" : self.on_next_clicked, + "on_res_row_activated" : self.on_select_clicked, + "on_res_sel_changed" : self.on_res_sel_changed, + "on_title_entry_changed" : self.on_title_entry_changed, + "on_help_clicked" : self.on_help_clicked}) + # main screen items + self.res_store = self.top.get_object("res_names_liststore") + self.res_selection = self.top.get_object("res_selection") + self.mainwin = self.top.get_object("main") + return self.mainwin + + # ====================================================== + # gramplet event handlers + # ====================================================== + def on_help_clicked(self, _dummy): + ''' Button: Display the relevant portion of GRAMPS manual''' + display_url(WIKI_PAGE) + + def on_res_sel_changed(self, res_sel): + """ Selecting a row in the results list """ + self.top.get_object("select_but").set_sensitive( + res_sel.get_selected()) + + def on_title_entry_changed(self, _dummy): + ''' Occurs during edits of the Title box on the main screen. + we use this to reset the GeoNames search row, as the user may be + trying another search term.''' + self.reset_main() + + def on_save(self, *args, **kwargs): + CONFIG.set("preferences.geo_userid", self.geonames_id) + CONFIG.set("preferences.web_links", self.keepweb) + CONFIG.set("preferences.add_cit", self.addcitation) + CONFIG.set("preferences.keep_enclosure", self.keep_enclosure) + CONFIG.set("preferences.keep_lang", ' '.join(self.allowed_languages)) + CONFIG.save() + + def db_changed(self): + self.dbstate.db.connect('place-update', self.update) + self.main() + if not self.dbstate.db.readonly: + self.connect_signal('Place', self.update) + + def main(self): + self.reset_main() + if self.gui.get_child().get_child() == self.results_win: + self.gui.get_child().remove(self.results_win) + self.gui.get_child().add(self.mainwin) + active_handle = self.get_active('Place') + self.top.get_object("edit_but").set_sensitive(False) + self.top.get_object("find_but").set_sensitive(False) + self.top.get_object("title_entry").set_sensitive(False) + if active_handle: + self.place = self.dbstate.db.get_place_from_handle(active_handle) + self.mainwin.hide() + if self.place: + self.set_has_data(True) + title = _pd.display(self.dbstate.db, self.place, fmt=0) + item = self.top.get_object("title_entry") + item.set_text(title) + self.top.get_object("edit_but").set_sensitive(True) + self.top.get_object("find_but").set_sensitive(True) + self.top.get_object("title_entry").set_sensitive(True) + else: + self.set_has_data(False) + self.mainwin.show() + else: + self.set_has_data(False) + + def reset_main(self): + """ Reset the main Gui to default clear """ + self.res_store.clear() + self.res_lbl.set_text(_('No\nMatches')) + self.find_but.set_label(_("Find")) + self.start_row = 0 + self.geo_stage = False + self.top.get_object("select_but").set_sensitive(False) + + def on_find_clicked(self, _dummy): + """ find a matching place. First try in the db, then try in the + GeoNames. """ + self.res_store.clear() + self.top.get_object("select_but").set_sensitive(False) + if self.geo_stage: + self.search_geo() + else: + self.geo_stage = True + item = self.top.get_object("title_entry") + title = item.get_text() + self.places = self.lookup_places_by_name(title) + for index, place in enumerate(self.places): + # make sure the place found isn't self, or a place + # enclosed by the working place + if place.handle != self.place.handle and not located_in( + self.dbstate.db, place.handle, self.place.handle): + title = _pd.display(self.dbstate.db, place) + self.res_store.append(row=( + index, title, str(place.get_type()))) + if len(self.res_store) > 0: + self.res_lbl.set_text(_('%s\nLocal\nMatches') % + len(self.res_store)) + self.find_but.set_label(_("Find GeoNames")) + else: + self.search_geo() + + def search_geo(self): + """ find a matching place in the geonames, if possible """ + self.res_store.clear() + if not self.geonames_id: + ErrorDialog(_('Need to set GeoNames ID'), + msg2=_('Use the Help button for more information'), + parent=self.uistate.window) + return + # lets get a preferred language + fmt = config.get('preferences.place-format') + placef = _pd.get_formats()[fmt] + self.lang = placef.language + if len(self.lang) != 2: + self.lang = 'en' + if self.lang not in self.allowed_languages: + self.allowed_languages.append(self.lang) + # now lets search for a place in GeoNames + item = self.top.get_object("title_entry") + title = quote(item.get_text().lower().replace('co.', 'county')) + adm = self.top.get_object('adm_check').get_active() + ppl = self.top.get_object('ppl_check').get_active() + spot = self.top.get_object('spot_check').get_active() + geo_url = ( + 'http://api.geonames.org/search?q=%s' + '&maxRows=10&style=SHORT&lang=en&isNameRequired=True' + '%s%s%s&username=%s&startRow=%s' % + (title, + '&featureClass=A' if adm else '', + '&featureClass=P' if ppl else '', + '&featureClass=S' if spot else '', + self.geonames_id, self.start_row)) + dom = self.get_geo_data(geo_url) + if not dom: + return + + g_names = dom.getElementsByTagName('geoname') + if not g_names: + WarningDialog(_('No matches were found'), + msg2=_('Try changing the Title, or use the "Edit"' + ' button to finish this level of the place' + ' manually.'), + parent=self.uistate.window) + return + # let's check the total results; if too many, warn user and set up for + # another pass through the search. + value = dom.getElementsByTagName('totalResultsCount') + if value: + totalresults = int(value[0].childNodes[0].data) + self.res_lbl.set_text(_("%s\nGeoNames\nMatches") % totalresults) + if totalresults > 10: + self.start_row += 10 + if self.matches_warn: + self.matches_warn = False + WarningDialog( + _('%s matches were found') % totalresults, + msg2=_('Only 10 matches are shown.\n' + 'To see additional results, press the' + ' search button again.\n' + 'Or try changing the Title with' + ' more detail, such as a country.'), + parent=self.uistate.window) + index = 0 + self.places = [] + for g_name in g_names: + # got a match, now get its hierarchy for display. + value = g_name.getElementsByTagName('geonameId') + geoid = value[0].childNodes[0].data + value = g_name.getElementsByTagName('fcl') + fcl = value[0].childNodes[0].data + value = g_name.getElementsByTagName('fcode') + _type = fcl + ':' + value[0].childNodes[0].data + geo_url = ('http://api.geonames.org/hierarchy?geonameId=%s' + '&lang=%s&username=%s' % + (geoid, self.lang, self.geonames_id)) + hier = self.get_geo_data(geo_url) + if not hier: + return + h_names = hier.getElementsByTagName('geoname') + h_name_list = [] + h_geoid_list = [] + for h_name in h_names: + value = h_name.getElementsByTagName('fcl') + fcl = value[0].childNodes[0].data + if fcl not in 'APS': + # We don't care about Earth or continent (yet) + continue + value = h_name.getElementsByTagName('name') + h_name_list.append(value[0].childNodes[0].data) + value = h_name.getElementsByTagName('geonameId') + h_geoid_list.append('GEO' + value[0].childNodes[0].data) + # make sure that this place isn't already enclosed by our place. + bad = self.place.gramps_id in h_geoid_list[:-1] + # assemble a title for the result + h_name_list.reverse() + h_geoid_list.reverse() + if bad: + title = ('' + _(', ').join(h_name_list) + '') + else: + title = _(', ').join(h_name_list) + row = (index, title, _type) + self.res_store.append(row=row) + self.places.append(('GEO' + geoid, title, h_geoid_list, + h_name_list, bad)) + index += 1 + while Gtk.events_pending(): + Gtk.main_iteration() + + def get_geo_data(self, geo_url): + """ Get GeoNames data from web with error checking """ + # print(geo_url) + try: + with urlopen(geo_url, timeout=20) as response: + data = response.read() + except URLError as err: + try: + txt = err.read().decode('utf-8') + except: + txt = '' + ErrorDialog(_('Problem getting data from web'), + msg2=str(err) + '\n' + txt, + parent=self.uistate.window) + return None + except socket.timeout: + ErrorDialog(_('Problem getting data from web'), + msg2=_('Web request Timeout, you can try again...'), + parent=self.uistate.window) + return None + + dom = parseString(data) + status = dom.getElementsByTagName('status') + if status: + err = status[0].getAttribute("message") + ErrorDialog(_('Problem getting data from GeoNames'), + msg2=err, + parent=self.uistate.window) + return None + return dom + + def on_next_clicked(self, _dummy): + """ find a incomplete place in the db, if possible """ + self.reset_main() + place = self.find_an_incomplete_place() + if place: + self.set_active('Place', place.handle) + + def on_select_clicked(self, *dummy): + """ If the selected place is mergable, merge it, otherwise Open + completion screen """ + model, _iter = self.res_selection.get_selected() + if not _iter: + return + (index, ) = model.get(_iter, 0) + place = self.places[index] + if not isinstance(place, Place): + # we have a geoname_id + if place[4]: + return + # check if we might already have it in db + t_place = self.dbstate.db.get_place_from_gramps_id(place[0]) + if not t_place or t_place.handle == self.place.handle: + # need to process the GeoNames ID for result + self.gui.get_child().remove(self.mainwin) + self.gui.get_child().add(self.results_win) + if not self.geoparse(*place): + return + self.res_gui() + return + else: + # turns out we already have this place, under different name! + place = t_place + # we have a Gramps Place, need to merge + if place.handle == self.place.handle: + # found self, nothing to do. + return + if(located_in(self.dbstate.db, place.handle, self.place.handle) or + located_in(self.dbstate.db, self.place.handle, place.handle)): + # attempting to create a place loop, not good! + ErrorDialog(_('Place cycle detected'), + msg2=_("One of the places you are merging encloses " + "the other!\n" + "Please choose another place."), + parent=self.uistate.window) + return + # lets clean up the place name + name = self.place.get_name() + name.value = name.value.split(',')[0].strip() + place_merge = MergePlaceQuery(self.dbstate, place, self.place) + place_merge.execute() + # after merge we should select merged result + self.set_active('Place', place.handle) + + adm_table = { + # note the True/False in the following indicates the certainty that the + # entry is correct. If it is only sometimes correct, and the name + # might have a different type embedded in it, then use False. + 'US': {'ADM1': ('State', True), + 'ADM2': ('County', False), + 'ADM3': ('Town', False)}, + 'CA': {'ADM1': ('Province', True), + 'ADM2': ('Region', False)}, + 'GB': {'ADM1': ('Country', True), + 'ADM2': ('Region', True), + 'ADM3': ('County', False), + 'ADM4': ('Borough', False)}, + 'FR': {'ADM1': ('Region', True), + 'ADM2': ('Department', True)}, + 'DE': {'ADM1': ('State', True), + 'ADM2': ('County', False), + 'ADM3': ('Amt', False)}} + + def geoparse(self, geoid, title, h_geoid_list, h_name_list, *dummy): + """ get data for place and parse out g_name dom structure into the + NewPlace structure """ + geo_url = ('http://api.geonames.org/get?geonameId=%s&style=FULL' + '&username=%s' % (geoid.replace('GEO', ''), + self.geonames_id)) + dom = self.get_geo_data(geo_url) + if not dom: + return False + + g_name = dom.getElementsByTagName('geoname')[0] + self.newplace = NewPlace(title) + self.newplace.geoid = geoid + self.newplace.gramps_id = geoid + value = g_name.getElementsByTagName('lat') + self.newplace.lat = str(value[0].childNodes[0].data) + value = g_name.getElementsByTagName('lng') + self.newplace.long = str(value[0].childNodes[0].data) + value = g_name.getElementsByTagName('toponymName') + topname = value[0].childNodes[0].data + pl_name = PlaceName() + pl_name.set_value(topname) + pl_name.set_language("") + # make sure we have the topname in the names list and default to + # primary + self.newplace.add_name(pl_name) + self.newplace.name = pl_name + # lets parse the alternative names + alt_names = g_name.getElementsByTagName('alternateName') + for a_name in alt_names: + pattr = a_name.getAttribute("lang") + value = a_name.childNodes[0].data + if pattr == "post": + attr = Attribute() + attr.set_type(AttributeType.POSTAL) + attr.set_value(value) + self.newplace.postals.append(attr) + elif pattr == "abbr": + # deal with abbreviation + abbr = PlaceAbbrev(value=value) + abbr.set_type(PlaceAbbrevType.ABBR) + self.newplace.abbrs.append(abbr) + elif pattr == "link": + url = Url() + url.set_path(value) + url.set_description(value) + url.set_type(UrlType(UrlType.WEB_HOME)) + self.newplace.links.append(url) + elif pattr not in ['iata', 'iaco', 'faac', 'wkdt', 'unlc']: + pl_name = PlaceName() + pl_name.set_language(pattr) + pl_name.set_value(value) + self.newplace.add_name(pl_name) + if a_name.hasAttribute('isPreferredName') and ( + pattr and pattr == self.lang): + # if not preferred lang, we use topo name, otherwise + # preferred name for lang + self.newplace.name = pl_name + # Try to deduce PlaceType: + # If populated place, set as City. Long description could over-ride + # Parse long description, looking for keyword (Region, County, ...) + # Top-level must be a country. + # Children of USA are States. + # Children of Canada are Provinces. + # + value = g_name.getElementsByTagName('fcl') + fcl = value[0].childNodes[0].data + value = g_name.getElementsByTagName('fcode') + fcode = value[0].childNodes[0].data + value = g_name.getElementsByTagName('countryCode') + if value[0].childNodes: + countrycode = value[0].childNodes[0].data + else: + countrycode = '' + self.newplace.place_type = PlaceType(PlaceType.UNKNOWN) + # scan thorough names looking for name portion that matches a Placetype + for name in self.newplace.names: + for tname in name.value.split(' '): + tname = tname.lower() + if tname in PlaceType.str_to_pt_id: # is it in the DATAMAP? + self.newplace.place_type.set( + PlaceType.str_to_pt_id[tname]) + break + else: + # Continue if the inner loop wasn't broken. + continue + # Inner loop was broken, break the outer. + break + if fcl == 'P': + # this is likely a city + self.newplace.place_type.set("City") + self.newplace.group = P_G.PLACE + elif fcode == 'PRSH': + self.newplace.place_type.set("Parish") + elif 'PCL' in fcode: + # this is definitely a country + self.newplace.place_type.set("Country") + abbrev = PlaceAbbrev(value=countrycode) + abbrev.set_type(PlaceAbbrevType.ISO3166) + self.newplace.name.add_abbrev(abbrev) + self.newplace.group = P_G.COUNTRY + elif 'ADM' in fcode: + self.newplace.group = P_G.REGION + # 'ADMx' if no other available + self.newplace.place_type.set(fcode[:4]) + # see if we recognize country and get type from the ADMx and that + if countrycode in self.adm_table: + _ptype = self.adm_table[countrycode].get(fcode[:4]) + if _ptype and (_ptype[1] or + self.newplace.place_type == PlaceType.UNKNOWN): + self.newplace.place_type.set(_ptype[0]) + # store abbreviations on the preferred place name + self.newplace.name.get_abbrevs().extend(self.newplace.abbrs) + if self.newplace.name.get_abbrevs(): + for name in self.newplace.names: + lang = name.get_language() + if not lang or self.lang == lang: + name.set_abbrevs(self.newplace.name.get_abbrevs()) + # save a parent for enclosing + if len(h_geoid_list) > 1: + # we have a parent + self.newplace.parent_names = h_name_list[1:] + self.newplace.parent_ids = h_geoid_list[1:] + return True + + def on_edit_clicked(self, dummy): + """User wants to jump directly to the results view to finish off + the place, possibly because a place was not found""" +# if ',' in self.place.name.value: +# name = self.place.name.value +# else: + name = self.place.get_names()[0].value + self.newplace = NewPlace(name) + names = name.split(',') + names = [name.strip() for name in names] + self.newplace.name = PlaceName() + self.newplace.name.value = names[0] + self.newplace.gramps_id = self.place.gramps_id + self.newplace.lat = self.place.lat + self.newplace.long = self.place.long + if self.place.get_type() == PlaceType.UNKNOWN: + self.newplace.place_type = PlaceType(PlaceType.UNKNOWN) + if any(i.isdigit() for i in self.newplace.name.value): + self.newplace.place_type.set("Street") + for tname in self.newplace.name.value.split(' '): + tname = tname.lower() + if tname in PlaceType.str_to_pt_id: # is it in the DATAMAP? + self.newplace.place_type.set(PlaceType.str_to_pt_id[tname]) + break + else: + ptype = self.place.get_type() + self.newplace.place_type = ptype + self.newplace.add_name(self.newplace.name) + self.newplace.add_names(self.place.get_names()) + self.newplace.group = self.place.group + if self.place.placeref_list: + # If we already have an enclosing place, use it. + parent = self.dbstate.db.get_place_from_handle( + self.place.placeref_list[0].ref) + self.newplace.parent_ids = [parent.gramps_id] + elif len(names) > 1: + # we have an enclosing place, according to the name string + self.newplace.parent_names = names[1:] + self.gui.get_child().remove(self.mainwin) + self.gui.get_child().add(self.results_win) + self.res_gui() + +# Results view + + def res_gui(self): + """ Fill in the results display with values from new place.""" + self.alt_store.clear() + # Setup sort on 'Inc" column so Primary is a top with checks next + self.alt_store.set_sort_func(0, inc_sort, None) + self.alt_store.set_sort_column_id(0, 0) + self.newplace.add_names(self.place.get_names()) + # Fill in other fields + self.top.get_object('res_title').set_text(self.newplace.title) + self.top.get_object('primary').set_text(self.newplace.name.value) + self.on_idcheck() + self.on_latloncheck() + self.on_postalcheck() + self.on_typecheck() + self.on_groupcheck() + # Fill in names list + for index, name in enumerate(self.newplace.names): + if self.newplace.name == name: + inc = 'P' + elif name.lang in self.allowed_languages or ( # TODO + name.lang == 'abbr' or name.lang == 'en' or not name.lang): + inc = '\u2714' # Check mark + else: + inc = '' + row = (inc, name.value, name.lang, get_date(name), index) + self.alt_store.append(row=row) + +# Results dialog items + + def on_res_ok_clicked(self, dummy): + """ Accept changes displayed and commit to place. + Also find or create a new enclosing place from parent. """ + # do the names + namelist = [] + for row in self.alt_store: + if row[0] == 'P': + namelist.insert(0, self.newplace.names[row[4]]) + elif row[0] == '\u2714': + namelist.append(self.newplace.names[row[4]]) + self.place.set_names(namelist) + # Lat/lon/ID/code/type + self.place.lat = self.top.get_object('latitude').get_text() + self.place.long = self.top.get_object('longitude').get_text() + self.place.gramps_id = self.top.get_object('grampsid').get_text() + attrs = self.place.get_attribute_list()[:] + for addendum in self.newplace.postals: + for attr in attrs: + equi = attr.is_equivalent(addendum) + if equi == IDENTICAL: + break + elif equi == EQUAL: + attr.merge(addendum) + break + else: + self.place.add_attribute(addendum) + # retrieve Place type from combo and commit/set to place + self.place.set_type(self.ptype) + self.place.group.set(self.group_combo.get_values()) + # Add in URLs if wanted + if self.keepweb: + for url in self.newplace.links: + self.place.add_url(url) + # Enclose in the next level place + next_place = False + parent = None + if not self.keep_enclosure or not self.place.placeref_list: + if self.newplace.parent_ids: + # we might have a parent with geo id 'GEO12345' + parent = self.dbstate.db.get_place_from_gramps_id( + self.newplace.parent_ids[0]) + if not parent and self.newplace.parent_names: + # make one, will have to be examined/cleaned later + parent = Place() + parent.title = ', '.join(self.newplace.parent_names) + name = PlaceName() + name.value = parent.title + parent.add_name(name) + parent.gramps_id = self.newplace.parent_ids[0] + parent.set_type(PlaceType.UNKNOWN) + with DbTxn(_("Add Place (%s)") % parent.title, + self.dbstate.db) as trans: + self.dbstate.db.add_place(parent, trans) + next_place = True + if parent: + if located_in(self.dbstate.db, parent.handle, + self.place.handle): + # attempting to create a place loop, not good! + ErrorDialog(_('Place cycle detected'), + msg2=_("The place you chose is enclosed in the" + " place you are workin on!\n" + "Please cancel and choose another " + "place."), + parent=self.uistate.window) + return + # check to see if we already have the enclosing place + already_there = False + for pref in self.place.placeref_list: + if parent.handle == pref.ref: + already_there = True + break + if not already_there: + placeref = PlaceRef() + placeref.set_reference_handle(parent.handle) + placeref.set_type_for_place(self.place) + self.place.set_placeref_list([placeref]) + # Add in Citation/Source if wanted + if self.addcitation and self.newplace.geoid: + src_hndl = self.find_or_make_source() + cit = Citation() + cit.set_reference_handle(src_hndl) + cit.set_page("GeoNames ID: %s" % + self.newplace.geoid.replace('GEO', '')) + with DbTxn(_("Add Citation (%s)") % "GeoNames", + self.dbstate.db) as trans: + self.dbstate.db.add_citation(cit, trans) + self.place.add_citation(cit.handle) + # We're finally ready to commit the updated place + with DbTxn(_("Edit Place (%s)") % self.place.title, + self.dbstate.db) as trans: + self.dbstate.db.commit_place(self.place, trans) + # Jump to enclosing place to clean it if necessary + if next_place: + self.set_active('Place', parent.handle) + self.place = parent + # if geoparse fails, leave us at main view + if self.newplace.parent_ids and \ + self.geoparse(self.newplace.parent_ids[0], + _(", ").join(self.newplace.parent_names), + self.newplace.parent_ids, + self.newplace.parent_names): + # geoparse worked, lets put up the results view + self.gui.get_child().remove(self.mainwin) + self.gui.get_child().add(self.results_win) + self.res_gui() + return + self.reset_main() + if self.gui.get_child().get_child() == self.results_win: + self.gui.get_child().remove(self.results_win) + self.gui.get_child().add(self.mainwin) + + def on_res_cancel_clicked(self, dummy): + """ Cancel operations on this place. """ + self.gui.get_child().remove(self.results_win) + self.gui.get_child().add(self.mainwin) + + def on_keep_clicked(self, dummy): + """ Keep button clicked. Mark selected names rows to keep. """ + model, rows = self.alt_selection.get_selected_rows() + for row in rows: + if model[row][0] == 'P': + continue + model[row][0] = '\u2714' + + def on_prim_clicked(self, dummy): + """ Primary button clicked. Mark first row in selection as Primary + name, any previous primary as keep """ + model, rows = self.alt_selection.get_selected_rows() + if not rows: + return + # Clear prior primary + for row in model: + if row[0] == 'P': + row[0] = '\u2714' + # mark new one. + self.top.get_object('primary').set_text(model[rows[0]][1]) + model[rows[0]][0] = 'P' + + def on_disc_clicked(self, dummy): + """ Discard button clicked. Unmark selected rows. """ + model, rows = self.alt_selection.get_selected_rows() + for row in rows: + if model[row][0] == 'P': + continue + model[row][0] = '' + + def on_alt_row_activated(self, *dummy): + """ Toggle keep status for selected row. Seems this only works for + last selected row.""" + model, rows = self.alt_selection.get_selected_rows() + for row in rows: + if model[row][0] == 'P': + continue + if model[row][0] == '': + model[row][0] = '\u2714' + else: + model[row][0] = '' + + def on_latloncheck(self, *dummy): + """ Check toggled; if active, load lat/lon from original place, else + use lat/lon from gazetteer """ + obj = self.top.get_object("latloncheck") + if not dummy: + # inititlization + obj.set_sensitive(True) + obj.set_active(False) + place = self.newplace + if self.place.lat and self.place.long: + if obj.get_active(): + place = self.place + else: + obj.set_sensitive(False) + self.top.get_object('latitude').set_text(place.lat) + self.top.get_object('longitude').set_text(place.long) + + def on_postalcheck(self, *dummy): + """ Check toggled; if active, load postal from original place, else + use postal from gazetteer """ + obj = self.top.get_object("postalcheck") + if not dummy: + # inititlization + obj.set_sensitive(True) + obj.set_active(False) + attrs = self.newplace.postals + if self.get_postals(self.place.get_attribute_list()): + if obj.get_active(): + attrs = self.place.get_attribute_list() + obj.set_sensitive(True) + else: + obj.set_sensitive(False) + self.top.get_object('postal').set_text(self.get_postals(attrs)) + + def get_postals(self, attrs): + """ Create a postal codes string from attributes of place """ + postals = '' + for attr in attrs: + if attr.type == AttributeType.POSTAL: + postals += (', ' if postals else '') + attr.value + return postals + + def on_typecheck(self, *dummy): + """ Check toggled; if active, load type from original place, else + use type from gazetteer """ + obj = self.top.get_object("typecheck") + if not dummy: + # inititlization + obj.set_sensitive(True) + obj.set_active(False) + p_type = self.newplace.place_type + if self.place.get_type() != PlaceType.UNKNOWN: + if obj.get_active(): + p_type = self.place.get_type() + else: + obj.set_sensitive(False) + self.ptype = PlaceType(p_type) + combo = self.top.get_object('place_type') + self.type_combo = PlaceTypeSelector(self.dbstate, combo, self.ptype) + + def on_groupcheck(self, *dummy): + """ Check toggled; if active, load group from original place, else + use group from gazetteer """ + obj = self.top.get_object("groupcheck") + if not dummy: + # inititlization + obj.set_sensitive(True) + obj.set_active(False) + p_group = self.newplace.group + if self.place.group != P_G.NONE: + if obj.get_active(): + p_group = self.place.group + else: + obj.set_sensitive(False) + self.pgroup = P_G(p_group) + additional = sorted(self.dbstate.db.get_placegroup_types(), + key=lambda s: s.lower()) + self.group_combo = StandardCustomSelector( + P_G().get_map(), + self.top.get_object('place_group'), + P_G.CUSTOM, P_G.NONE, additional) + self.group_combo.set_values((int(self.pgroup), + str(self.pgroup))) + + def on_idcheck(self, *dummy): + """ Check toggled; if active, load gramps_id from original place, else + use geonamesid from gazetteer """ + obj = self.top.get_object("idcheck") + if not dummy: + # inititlization + obj.set_sensitive(True) + obj.set_active(False) + place = self.newplace + if self.place.gramps_id: + if obj.get_active(): + place = self.place + else: + obj.set_sensitive(False) + self.top.get_object('grampsid').set_text(place.gramps_id) + + def find_or_make_source(self): + """ Find or create a source. + returns handle to source.""" + for hndl in self.dbstate.db.get_source_handles(): + if self.dbstate.db.get_raw_source_data(hndl)[2] == 'GeoNames': + return hndl + # No source found, lets add one with associated repo and note + repo = Repository() + repo.set_name("www.geonames.org") + rtype = RepositoryType(RepositoryType.WEBSITE) + repo.set_type(rtype) + url = Url() + url.set_path('http://www.geonames.org/') + url.set_description(_('GeoNames web site')) + url.set_type(UrlType(UrlType.WEB_HOME)) + repo.add_url(url) + url = Url() + url.set_path('marc@geonames.org') + url.set_description(_('GeoNames author')) + url.set_type(UrlType(UrlType.EMAIL)) + repo.add_url(url) + + note_txt = StyledText(_( + 'GeoNames was founded by Marc Wick. You can reach him at ')) + note_txt += StyledText('marc@geonames.org' + '\n') + note_txt += StyledText(_( + 'GeoNames is a project of Unxos GmbH, Weingartenstrasse 8,' + ' 8708 Männedorf, Switzerland.\nThis work is licensed under a ')) + note_txt += linkst( + _('Creative Commons Attribution 3.0 License'), + 'https://creativecommons.org/licenses/by/3.0/legalcode') + + new_note = Note() + new_note.set_styledtext(note_txt) + new_note.set_type(NoteType.REPO) + src = Source() + src.title = 'GeoNames' + src.author = 'Marc Wick' + repo_ref = RepoRef() + mtype = SourceMediaType(SourceMediaType.ELECTRONIC) + repo_ref.set_media_type(mtype) + with DbTxn(_("Add Souce/Repo/Note (%s)") % "GeoNames", + self.dbstate.db) as trans: + + self.dbstate.db.add_note(new_note, trans) + repo.add_note(new_note.get_handle()) + self.dbstate.db.add_repository(repo, trans) + repo_ref.set_reference_handle(repo.handle) + src.add_repo_reference(repo_ref) + self.dbstate.db.add_source(src, trans) + return src.handle + +# Preferences dialog items + + def on_prefs_clicked(self, dummy): + """ Button: display preference dialog """ + top = self.top.get_object("pref_dialog") + top.set_transient_for(self.uistate.window) + parent_modal = self.uistate.window.get_modal() + if parent_modal: + self.uistate.window.set_modal(False) + keepweb = self.top.get_object("keepweb") + keepweb.set_active(self.keepweb) + addcit = self.top.get_object("addcitation") + addcit.set_active(self.addcitation) + # for some reason you can only set the radiobutton to True + self.top.get_object( + "enc_radio_but_keep" if self.keep_enclosure + else "enc_radio_but_repl").set_active(True) + geoid = self.top.get_object("geonames_id_entry") + geoid.set_text(self.geonames_id) + keepalt = self.top.get_object("keep_alt_entry") + keepalt.set_text(' '.join(self.allowed_languages)) + top.show() + top.run() + if self.uistate.window and parent_modal: + self.uistate.window.set_modal(True) + self.geonames_id = geoid.get_text() + self.addcitation = addcit.get_active() + self.keepweb = keepweb.get_active() + self.keep_enclosure = self.top.get_object( + "enc_radio_but_keep").get_active() + self.allowed_languages = keepalt.get_text().split() + top.hide() + + def on_pref_help_clicked(self, dummy): + ''' Button: Display the relevant portion of GRAMPS manual''' + display_url(PREFS_WIKI) + + def lookup_places_by_name(self, search_name): + """ In local db. Only completed places are matched. + We may want to try some better matching algorithms, possibly + something from difflib""" + search_name = search_name.lower() + if ',' in search_name: + search_name = search_name.split(',')[0] + places = [] + for place in self.dbstate.db.iter_places(): + if (place.get_type() != PlaceType.UNKNOWN and + (place.group == P_G.COUNTRY or + (place.group != P_G.COUNTRY and + place.get_placeref_list()))): + # valid place, get all its names + for name in place.get_names(): + if name.get_value().lower() == search_name: + places.append(place) + break + return places + + def find_an_incomplete_place(self): + """ in our db. Will return with a place (and active set to place) + or None if no incomplete places, in which case active will be the same. + Will also find unused places, and offer to delete.""" + p_hndls = self.dbstate.db.get_place_handles() + if not p_hndls: + return None # in case there aren't any + # keep handles in an order to avoid inconsistant + # results when db returns them in different orders. + p_hndls.sort() + # try to find the handle after the previously scanned handle in the + # list. + found = False + for indx, hndl in enumerate(p_hndls): + if hndl > self.incomp_hndl: + found = True + break + if not found: + indx = 0 + # now, starting from previous place, look for incomplete place + start = indx + while True: + hndl = p_hndls[indx] + place_data = self.dbstate.db.get_raw_place_data(hndl) + p_type = place_data[7] # place_types + if p_type: # if there is at least one + p_type = p_type[0][0] # pt_id for first place type + else: + p_type = PlaceType.UNKNOWN + refs = list(self.dbstate.db.find_backlink_handles(hndl)) + p_group = place_data[17] + if(p_type == PlaceType.UNKNOWN or + not refs or + not (p_group == P_G.COUNTRY or p_type == "Country") and + not place_data[5]): # placeref_list + # need to get view to this place... + self.set_active("Place", hndl) + self.incomp_hndl = hndl + if not refs: + WarningDialog( + _('This Place is not used!'), + msg2=_('You should delete it, or, if it contains ' + 'useful notes or other data, use the Find to ' + 'merge it into a valid place.'), + parent=self.uistate.window) + return self.dbstate.db.get_place_from_handle(hndl) + indx += 1 + if indx == len(p_hndls): + indx = 0 + if indx == start: + break + return None + + +class NewPlace(): + """ structure to store data about a found place""" + def __init__(self, title): + self.title = title + self.gramps_id = '' + self.lat = '' + self.long = '' + self.group = '' + self.postals = [] + self.place_type = None # a PlaceType + self.names = [] # all names, including alternate, acts like a set + self.name = PlaceName() + self.abbrs = [] + self.links = [] + self.geoid = '' + self.parent_ids = [] # list of gramps_ids in hierarchical order + self.parent_names = [] # list of names in hierarchical order + + def add_name(self, name): + """ Add a name to names list without repeats """ + if ',' in name.value: + name.value = name.value.split(',')[0] + return + for c_name in self.names: + if name.value == c_name.value and name.lang == c_name.lang: + return + self.names.append(name) + + def add_names(self, names): + """ Add names to names list without repeats """ + for name in names: + self.add_name(name) + +#------------------------------------------------------------------------ +# +# Functions +# +#------------------------------------------------------------------------ + + +def linkst(text, url): + """ Return text as link styled text + """ + tags = [StyledTextTag(StyledTextTagType.LINK, url, [(0, len(text))])] + return StyledText(text, tags) + + +def inc_sort(model, row1, row2, _user_data): + value1 = model.get_value(row1, 0) + value2 = model.get_value(row2, 0) + if value1 == value2: + return 0 + if value1 == 'P': + return -1 + if value2 == 'P': + return 1 + if value2 > value1: + return 1 + else: + return -1 + +# if found a matching place in our db, merge +# if not, lookup on web, present list of candidates +# start with simple search +# extend each result with heirarchy query +# Web button, if pressed again, more answers. + +# Alternative name Lang field: +# unlc:Un/locodes +# post:for postal codes +# iata,icao,faac: for airport codes +# fr_1793:French Revolution names +# abbr:abbreviation +# link: to a website (mostly to wikipedia) +# wkdt: for the wikidata id +# otherwise 2/3 char ISO639 lang code +# empty: just an alternative name +# Alternative name other fields: +# isPreferredName="true" +# isShortName="true" +# isColloquial="true" +# isHistoric="true" diff --git a/PlaceCompletion/PlaceCompletion.gpr.py b/PlaceCompletion/PlaceCompletion.gpr.py index c47a8b5ff..93d49f399 100644 --- a/PlaceCompletion/PlaceCompletion.gpr.py +++ b/PlaceCompletion/PlaceCompletion.gpr.py @@ -27,6 +27,7 @@ version = '0.0.36', gramps_target_version = "5.1", status = STABLE, +include_in_listing = False, fname = 'PlaceCompletion.py', authors = ["B. Malengier", "Mathieu MD"], diff --git a/PlaceTypes/placetype_nl.gpr.py b/PlaceTypes/placetype_nl.gpr.py new file mode 100644 index 000000000..3b7a2c1dc --- /dev/null +++ b/PlaceTypes/placetype_nl.gpr.py @@ -0,0 +1,41 @@ +# encoding:utf-8 +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2020 Paul Culley +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + + +#------------------------------------------------------------------------ +# +# Common Placetypes +# +#------------------------------------------------------------------------ +register( + GENERAL, + category='PLACETYPES', + id='pt_nl', + name="Netherlands PlaceType values", + description=_("Provides a library of Netherlands PlaceType values."), + version = '1.0.2', + status=STABLE, + fname='placetype_nl.py', + authors=["The Gramps project"], + authors_email=["http://gramps-project.org"], + load_on_reg=True, + gramps_target_version='5.1', +) diff --git a/PlaceTypes/placetype_nl.py b/PlaceTypes/placetype_nl.py new file mode 100644 index 000000000..3326a5ba5 --- /dev/null +++ b/PlaceTypes/placetype_nl.py @@ -0,0 +1,127 @@ +# encoding:utf-8 +# +# Gramps - a GTK+/GNOME based genealogy program - Records plugin +# +# Copyright (C) 2020 Paul Culley +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +#------------------------------------------------------------------------ +# +# Standard Python modules +# +#------------------------------------------------------------------------ +import logging +#------------------------------------------------------------------------ +# +# Gramps modules +# +#------------------------------------------------------------------------ +from gramps.gen.lib.placegrouptype import PlaceGroupType as P_G +from gramps.gen.lib.placetype import PlaceType +from gramps.gen.const import GRAMPS_LOCALE as glocale +_ = glocale.translation.sgettext +LOG = logging.getLogger() + + +# _T_ is a gramps-defined keyword -- see po/update_po.py and po/genpot.sh +def _T_(value): # enable deferred translations (see Python docs 22.1.3.4) + return value + + +try: + _trans = glocale.get_addon_translator(__file__) +except ValueError: + _trans = glocale.translation +_ = _trans.sgettext + +_report_trans = None +_report_lang = None + + +def translate_func(ptype, locale=None): + """ + This function provides translations for the locally defined place types. + It is called by the place type display code for the GUI and reports. + + The locale parameter is an instance of a GrampsLocale. This is used to + determine the language for tranlations. (locale.lang) It is also used + as a backup translation if no local po/mo file is present. + + :param ptype: the placetype translatable string + :type ptype: str + :param locale: the backup locale + :type locale: GrampsLocale instance + :returns: display string of the place type + :rtype: str + """ + global _report_lang, _report_trans + if locale is None or locale is glocale: + # using GUI language. + return _(ptype) + if locale.lang == _report_lang: + # We already created this locale, so use the previous version + # this will speed up reports in an alternate language + return _report_trans(ptype) + # We need to create a new language specific addon translator instance + try: + _r_trans = glocale.get_addon_translator( + __file__, languages=(locale.lang, )) + except ValueError: + _r_trans = glocale.translation + _report_trans = _r_trans.sgettext + _report_lang = locale.lang + return _report_trans(ptype) + + +# The data map (dict) contains a tuple with key as a handle and data as tuple; +# translatable name +# native name +# color (used for map markers, I suggest picking by probable group) +# probable group (used for legacy XML import and preloading Group in place +# editor) +# gettext method (or None if standard method) +DATAMAP = { + # add the common "Country" to the NL menu + "Country" : (_T_("Country"), "Country", + "#FFFF00000000", P_G(P_G.COUNTRY), None), + "nl_Province" : (_T_("nl|Province"), "Provincie", + "#0000FFFFFFFF", P_G(P_G.REGION), translate_func), + "nl_Municipality" : (_T_("nl|Municipality"), "Gemeente", + "#0000FFFFFFFF", P_G(P_G.REGION), translate_func), + "nl_City" : (_T_("nl|City"), "Stad", + "#0000FFFF0000", P_G(P_G.PLACE), translate_func), + "nl_Place" : (_T_("nl|Place"), "Plaats", + "#0000FFFF0000", P_G(P_G.PLACE), translate_func), + "nl_Village" : (_T_("nl|Village"), "Dorp", + "#0000FFFF0000", P_G(P_G.PLACE), translate_func), +} + + +def load_on_reg(_dbstate, _uistate, _plugin): + """ + Runs when plugin is registered. + """ + for hndl, tup in DATAMAP.items(): + # for these Netherlands elements, the category is 'NL' + # For larger regions of several countries, any descriptive text can be + # used (Holy Roman Empire) + # the register function returns True if the handle is a duplcate + # good idea to check this. + duplicate = PlaceType.register_placetype(hndl.capitalize(), tup, "NL") + if duplicate and hndl.startswith("nl_"): + LOG.debug("Duplicate handle %s detected; please fix", hndl) + PlaceType.update_name_map() diff --git a/PlaceTypes/po/de-local.po b/PlaceTypes/po/de-local.po new file mode 100644 index 000000000..87f7b6d08 --- /dev/null +++ b/PlaceTypes/po/de-local.po @@ -0,0 +1,54 @@ +# German translation for Gramps +# This file is distributed under the same license as the Gramps package. +# translation of de.po to Deutsch +# +# +# Anton Huber , 2005,2006. +# Sebastian Vöcking , 2005. +# Sebastian Vöcking , 2005. +# Martin Hawlisch , 2005, 2006. +# Alex Roitman , 2006. +# Mirko Leonhäuser , 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019. +# Alois Pöttker , 2017. +msgid "" +msgstr "" +"Project-Id-Version: de\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2020-06-11 17:06-0500\n" +"PO-Revision-Date: 2019-10-19 16:49+0200\n" +"Last-Translator: Mirko Leonhäuser \n" +"Language-Team: German \n" +"Language: de\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"X-Generator: Lokalize 18.12.3\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: Placetypes/placetype_nl.gpr.py:33 +msgid "Provides a library of Netherlands PlaceType values." +msgstr "Bietet eine Bibliothek mit Niederlande PlaceType-Werten." + +#: Placetypes/placetype_nl.py:96 +msgid "Country" +msgstr "Land" + +#: Placetypes/placetype_nl.py:98 +msgid "nl|Province" +msgstr "Provinz" + +#: Placetypes/placetype_nl.py:100 +msgid "nl|Municipality" +msgstr "Gemeinde" + +#: Placetypes/placetype_nl.py:102 +msgid "nl|City" +msgstr "Stadt" + +#: Placetypes/placetype_nl.py:104 +msgid "nl|Place" +msgstr "Platz" + +#: Placetypes/placetype_nl.py:106 +msgid "nl|Village" +msgstr "Dorf" diff --git a/PlaceTypes/po/template.pot b/PlaceTypes/po/template.pot new file mode 100644 index 000000000..f56c610fb --- /dev/null +++ b/PlaceTypes/po/template.pot @@ -0,0 +1,46 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2020-06-11 17:13-0500\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#: PlaceTypes/placetype_nl.gpr.py:33 +msgid "Provides a library of Netherlands PlaceType values." +msgstr "" + +#: PlaceTypes/placetype_nl.py:96 +msgid "Country" +msgstr "" + +#: PlaceTypes/placetype_nl.py:98 +msgid "nl|Province" +msgstr "" + +#: PlaceTypes/placetype_nl.py:100 +msgid "nl|Municipality" +msgstr "" + +#: PlaceTypes/placetype_nl.py:102 +msgid "nl|City" +msgstr "" + +#: PlaceTypes/placetype_nl.py:104 +msgid "nl|Place" +msgstr "" + +#: PlaceTypes/placetype_nl.py:106 +msgid "nl|Village" +msgstr "" diff --git a/Sqlite/ExportSql.py b/Sqlite/ExportSql.py index c9fcc0ab0..0447c7282 100644 --- a/Sqlite/ExportSql.py +++ b/Sqlite/ExportSql.py @@ -52,6 +52,7 @@ # #------------------------------------------------------------------------ from gramps.gen.utils.id import create_id +from gramps.gen.lib import PlaceType from gramps.gen.const import GRAMPS_LOCALE as glocale from gramps.gui.plug.export import WriterOptionBox # don't remove, used!!! try: @@ -186,13 +187,8 @@ def makeDB(db, callback): handle CHARACTER(25) PRIMARY KEY, gid CHARACTER(25), title TEXT, - value TEXT, - the_type0 INTEGER, - the_type1 TEXT, - code TEXT, long TEXT, lat TEXT, - lang TEXT, change INTEGER, private BOOLEAN);""") count += 1 @@ -202,7 +198,9 @@ def makeDB(db, callback): db.query("""CREATE TABLE place_ref ( handle CHARACTER(25) PRIMARY KEY, from_place_handle CHARACTER(25), - to_place_handle CHARACTER(25));""") + to_place_handle CHARACTER(25), + h_type0 INTEGER, + h_type1 TEXT);""") count += 1 callback(100 * count / total) @@ -215,6 +213,25 @@ def makeDB(db, callback): count += 1 callback(100 * count / total) + db.query("""drop table place_abbrev;""") + db.query("""CREATE TABLE place_abbrev ( + handle CHARACTER(25) PRIMARY KEY, + from_handle CHARACTER(25), + value INTEGER, + abbr_type0 INTEGER, + abbr_type1 TEXT);""") + count += 1 + callback(100 * count / total) + + db.query("""drop table place_type;""") + db.query("""CREATE TABLE place_type ( + handle CHARACTER(25) PRIMARY KEY, + from_handle CHARACTER(25), + value INTEGER, + t_str TEXT);""") + count += 1 + callback(100 * count / total) + db.query("""drop table event;""") db.query("""CREATE TABLE event ( handle CHARACTER(25) PRIMARY KEY, @@ -470,18 +487,41 @@ def close(self): self.db.close() -def export_alt_place_name_list(db, handle, alt_place_name_list): - for place_name in alt_place_name_list: +def export_place_name_list(db, handle, place_name_list): + for place_name in place_name_list: export_place_name(db, handle, place_name) def export_place_name(db, handle, place_name): # alt_place_name_list = [('Ohio', None, ''), ...] [(value, date, lang)...] - (value, date, lang) = place_name + (value, date, lang, abbr_list, citation_list) = place_name ref_handle = create_id() db.query("insert into place_name (handle, from_handle, value, lang)" " VALUES (?, ?, ?, ?);", ref_handle, handle, value, lang) export_date(db, "place_name", ref_handle, date) + export_citation_list(db, "place_name", ref_handle, citation_list) + export_place_abbr_list(db, ref_handle, abbr_list) + + +def export_place_abbr_list(db, handle, place_abbr_list): + for place_abbr in place_abbr_list: + (value, a_type) = place_abbr + ref_handle = create_id() + db.query("insert into place_abbrev (handle, from_handle, value," + " abbr_type0, abbr_type1)" + " VALUES (?, ?, ?, ?, ?);", + ref_handle, handle, value, a_type[0], a_type[1]) + + +def export_place_type_list(db, handle, place_type_list): + for place_type in place_type_list: + (value, date, citation_list) = place_type + ref_handle = create_id() + db.query("insert into place_type (handle, from_handle, value, t_str)" + " VALUES (?, ?, ?, ?);", + ref_handle, handle, value, PlaceType(value).xml_str()) + export_date(db, "place_type", ref_handle, date) + export_citation_list(db, "place_type", ref_handle, citation_list) def export_place_ref_list(db, handle, place_ref_list): @@ -492,12 +532,14 @@ def export_place_ref_list(db, handle, place_ref_list): def export_place_ref(db, handle, place_ref): - (to_place_handle, date) = place_ref + (to_place_handle, date, citation_list, htype) = place_ref ref_handle = create_id() db.query("insert into place_ref" - " (handle, from_place_handle, to_place_handle) VALUES (?, ?, ?);", - ref_handle, handle, to_place_handle) + " (handle, from_place_handle, to_place_handle, h_type0, h_type1)" + " VALUES (?, ?, ?, ?, ?);", + ref_handle, handle, to_place_handle, htype[0], htype[1]) export_date(db, "place_ref", ref_handle, date) + export_citation_list(db, "place_ref", ref_handle, citation_list) def export_location_list(db, from_type, from_handle, locations): @@ -1159,56 +1201,42 @@ def exportData(database, filename, user, option_box): continue (handle, gid, title, long, lat, place_ref_list, - place_name, - alt_place_name_list, - place_type, - code, + name_list, + type_list, + eventref_list, alt_location_list, urls, media_list, citation_list, note_list, - change, tag_list, private) = place.serialize() - - value, date, lang = place_name + change, tag_list, private, + attribute_list) = place.serialize() db.query("""INSERT INTO place ( handle, gid, title, - value, - the_type0, - the_type1, - code, long, lat, - lang, change, - private) values (?,?,?,?,?,?,?,?,?,?,?,?);""", - handle, gid, title, value, - place_type[0], place_type[1], - code, - long, lat, - lang, - change, private) + private) values (?,?,?,?,?,?,?);""", + handle, gid, title, long, lat, change, private) - export_date(db, "place", handle, date) + export_place_name_list(db, handle, name_list) + export_place_type_list(db, handle, type_list) export_url_list(db, "place", handle, urls) export_media_ref_list(db, "place", handle, media_list) export_citation_list(db, "place", handle, citation_list) export_list(db, "place", handle, "note", note_list) export_list(db, "place", handle, "tag", tag_list) - - #1. alt_place_name_list = [('Ohio', None, ''), ...] - # [(value, date, lang)...] - #2. place_ref_list = Enclosed by: [('4ECKQCWCLO5YIHXEXC', None)] - # [(handle, date)...] - - export_alt_place_name_list(db, handle, alt_place_name_list) export_place_ref_list(db, handle, place_ref_list) + export_attribute_list(db, "place", handle, attribute_list) # But we need to link these: export_location_list(db, "place_alt", handle, alt_location_list) + # Event Reference information + for event_ref in eventref_list: + export_event_ref(db, "place", handle, event_ref) count += 1 callback(100 * count / total) diff --git a/Sqlite/ImportSql.py b/Sqlite/ImportSql.py index 57ae22a87..a19b61ac6 100644 --- a/Sqlite/ImportSql.py +++ b/Sqlite/ImportSql.py @@ -45,7 +45,8 @@ #------------------------------------------------------------------------- from gramps.gen.lib import (Person, Family, Note, Media, Place, Citation, Source, Tag, Event, Repository, Name, Location, - PlaceName) + PlaceType) +from gramps.gen.lib.placetype import DM_NAME from gramps.gen.db import DbTxn from gramps.gen.const import GRAMPS_LOCALE as glocale try: @@ -187,7 +188,7 @@ def get_child_ref_list(self, sql, from_type, from_handle): (frel0, frel1), (mrel0, mrel1))) return retval - def get_datamap_list(self, sql, from_type, from_handle): + def get_datamap_list(self, sql, _from_type, from_handle): datamap = [] rows = sql.query("select * from datamap where from_handle = ?;", from_handle) @@ -312,7 +313,7 @@ def pack_lds(self, sql, data): return (citation_list, note_list, date, type_, place, famc, temple, status, bool(private)) - def pack_surnames(self, sql, data): + def pack_surnames(self, _sql, data): (_handle, surname, prefix, @@ -355,7 +356,7 @@ def pack_repository_ref(self, sql, data): (source_media_type0, source_media_type1), bool(private)) - def pack_url(self, sql, data): + def pack_url(self, _sql, data): (_handle, path, desc, @@ -451,7 +452,7 @@ def pack_name(self, sql, data): (name_type0, name_type1), group_as, sort_as, display_as, call, nick, famnick) - def pack_location(self, sql, data, with_parish): + def pack_location(self, _sql, data, with_parish): (_handle, street, locality, city, county, state, country, postal, phone, parish) = data if with_parish: @@ -476,7 +477,7 @@ def get_place_from_handle(self, sql, ref_handle): " returned %d records." % (ref_handle, len(place_row))) return '' - def get_alt_place_name_list(self, sql, handle): + def get_place_name_list(self, sql, handle): place_name_list = sql.query( """select * from place_name where from_handle = ?;""", handle) retval = [] @@ -484,7 +485,19 @@ def get_alt_place_name_list(self, sql, handle): ref_handle, handle, value, lang = place_name_data date_handle = self.get_link(sql, "place_name", ref_handle, "date") date = self.get_date(sql, date_handle) - retval.append((value, date, lang)) + abbr_list = self.get_place_abbr_list(sql, ref_handle) + citation_list = self.get_citation_list(sql, "place_name", + ref_handle) + retval.append((value, date, lang, abbr_list, citation_list)) + return retval + + def get_place_abbr_list(self, sql, handle): + place_abbr_list = sql.query( + """select * from place_abbrev where from_handle = ?;""", handle) + retval = [] + for place_abbr_data in place_abbr_list: + _r_handle, handle, value, abbr_type0, abbr_type1 = place_abbr_data + retval.append((value, (abbr_type0, abbr_type1))) return retval def get_place_ref_list(self, sql, handle): @@ -493,11 +506,45 @@ def get_place_ref_list(self, sql, handle): place_ref_list = sql.query( """select * from place_ref where from_place_handle = ?;""", handle) retval = [] - for place_ref_data in place_ref_list: - ref_handle, handle, to_place_handle = place_ref_data + for ref_data in place_ref_list: + ref_handle, handle, to_place_handle, h_type0, h_type1 = ref_data date_handle = self.get_link(sql, "place_ref", ref_handle, "date") date = self.get_date(sql, date_handle) - retval.append((to_place_handle, date)) + citation_list = self.get_citation_list(sql, "place_ref", + ref_handle) + retval.append((to_place_handle, date, citation_list, + (h_type0, h_type1))) + return retval + + def get_place_type_list(self, sql, handle): + place_type_list = sql.query( + """select * from place_type where from_handle = ?;""", + handle) + retval = [] + for place_type_data in place_type_list: + ref_handle, handle, type0, type1 = place_type_data + date_handle = self.get_link(sql, "place_type", ref_handle, "date") + date = self.get_date(sql, date_handle) + citation_list = self.get_citation_list(sql, "place_type", + ref_handle) + retval.append((type0, date, citation_list)) + if type0 in PlaceType.DATAMAP: + # if the number is already there, we are done + continue + if type0 < PlaceType.CUSTOM: + # number is not definitive, check for already there by name + for tup in PlaceType.DATAMAP.values(): + if type1.lower() == tup[DM_NAME].lower(): + break + else: + PlaceType.DATAMAP[type0] = (type1, + PlaceType.G_PLACE, # groups + True) # visible + else: + # not found, so store the new definition + PlaceType.DATAMAP[type0] = (type1, + PlaceType.G_PLACE, # groups + True) # visible return retval def get_main_location(self, sql, from_handle, with_parish): @@ -784,8 +831,7 @@ def _process(self, count, total, sql): places = sql.query("""select * from place;""") for place in places: count += 1 - (handle, gid, title, value, the_type0, the_type1, code, long, lat, - lang, change, private) = place + (handle, gid, title, long, lat, change, private) = place # We could look this up by "place_main", but we have the handle: #main_loc = self.get_main_location(sql, handle, with_parish=True) @@ -796,14 +842,15 @@ def _process(self, count, total, sql): citation_list = self.get_citation_list(sql, "place", handle) note_list = self.get_note_list(sql, "place", handle) tags = self.get_links(sql, "place", handle, "tag") - place_type = (the_type0, the_type1) - alt_place_name_list = self.get_alt_place_name_list(sql, handle) + place_type_list = self.get_place_type_list(sql, handle) + place_name_list = self.get_place_name_list(sql, handle) place_ref_list = self.get_place_ref_list(sql, handle) + eventref_list = self.get_event_ref_list(sql, "place", handle) + attr_list = self.get_attribute_list(sql, "place", handle) data = (handle, gid, title, long, lat, place_ref_list, - PlaceName(value=value, lang=lang).serialize(), - alt_place_name_list, place_type, code, alt_loc_list, - urls, media_list, citation_list, note_list, - change, tags, private) + place_name_list, place_type_list, eventref_list, + alt_loc_list, urls, media_list, citation_list, note_list, + change, tags, private, attr_list) g_plac = Place() g_plac.unserialize(data) self.db.commit_place(g_plac, self.trans) @@ -900,3 +947,4 @@ def importData(db, filename, user): g = SQLReader(db, filename, user) g.process() g.cleanup() + return _("Import finished...") diff --git a/Sqlite/Sqlite.gpr.py b/Sqlite/Sqlite.gpr.py index b35048d78..f9026f4eb 100644 --- a/Sqlite/Sqlite.gpr.py +++ b/Sqlite/Sqlite.gpr.py @@ -5,6 +5,7 @@ version = '1.1.2', gramps_target_version = "5.1", status = STABLE, # tested with python 3, need to review unicode usage + include_in_listing = False, fname = 'ImportSql.py', import_function = 'importData', extension = "sql" @@ -17,6 +18,7 @@ version = '1.1.2', gramps_target_version = "5.1", status = STABLE, # tested with python 3 but still gives errors + include_in_listing = False, fname = 'ExportSql.py', export_function = 'exportData', extension = "sql", diff --git a/TypeCleanup/type_cleanup.py b/TypeCleanup/type_cleanup.py index 57078a1c9..186081d4e 100644 --- a/TypeCleanup/type_cleanup.py +++ b/TypeCleanup/type_cleanup.py @@ -50,7 +50,7 @@ from gramps.gui.managedwindow import ManagedWindow from gramps.gui.utils import ProgressMeter from gramps.gui.utils import edit_object -from gramps.gui.widgets import MonitoredDataType +from gramps.gui.widgets import MonitoredDataType, PlaceTypeSelector from gramps.gen.lib.attrtype import AttributeType from gramps.gen.lib.eventroletype import EventRoleType @@ -61,6 +61,8 @@ from gramps.gen.lib.nametype import NameType from gramps.gen.lib.notetype import NoteType from gramps.gen.lib.placetype import PlaceType +from gramps.gen.lib.placehiertype import PlaceHierType +from gramps.gen.lib.placegrouptype import PlaceGroupType from gramps.gen.lib.repotype import RepositoryType from gramps.gen.lib.srcattrtype import SrcAttributeType from gramps.gen.lib.srcmediatype import SourceMediaType @@ -106,6 +108,8 @@ class TypeCleanup(tool.Tool, ManagedWindow): ("name_types", _("Name Types"), NameType, None), ("note_types", _("Note Types"), NoteType, None), ("place_types", _("Place Types"), PlaceType, None), + ("placehier_types", _("Place Hierarchy Types"), PlaceHierType, None), + ("placegroup_types", _("Place Group Types"), PlaceGroupType, None), ("repository_types", _("Repository Types"), RepositoryType, None), ("source_attributes", _("Source Attributes"), SrcAttributeType, None), ("source_media_types", _("Source Media Types"), SourceMediaType, None), @@ -252,11 +256,11 @@ def model_load(self): for (attr, name, _obj, _srcobj) in self.t_table: # 99 is indicator that row is a title row row = (name, 0, 99) - iter_ = self.model.append(None, row) # get custom types from db types = getattr(self.db, attr, None) if types is None: continue + iter_ = self.model.append(None, row) for indx, cust_type in enumerate(types): # update right model row = (cust_type, indx, r_indx) @@ -294,6 +298,12 @@ def do_recurse(self, obj, obj_class, hndl): # save a reference to the original primary object self.types_dict[obj.__class__][obj.string].append( (obj_class, hndl)) + elif isinstance(obj, PlaceType): + if(obj.__class__ in self.types_dict and # one of monitored types + str(obj) in self.types_dict[obj.__class__]): # custom type + # save a reference to the original primary object + self.types_dict[obj.__class__][str(obj)].append( + (obj_class, hndl)) def button_press(self, _view, path, _column): """ @@ -360,9 +370,13 @@ def selection_changed(self, selection): self.combo.get_child().set_width_chars(40) self.combo.show() self.cbox.add(self.combo) - self.type_name = MonitoredDataType(self.combo, self.set_obj, - self.get_obj, self.db.readonly, - self.get_cust_types()) + if isinstance(self.obj, PlaceType): + self.type_name = PlaceTypeSelector( + self.dbstate, self.combo, self.obj, changed=self.ptchanged) + else: + self.type_name = MonitoredDataType(self.combo, self.set_obj, + self.get_obj, self.db.readonly, + self.get_cust_types()) def get_cust_types(self): """ creates the custom types list for the MonitoredDataType """ @@ -375,6 +389,10 @@ def set_obj(self, val): if '' != str(self.obj) != self.name: self.ren_btn.set_sensitive(True) + def ptchanged(self): + if '' != str(self.obj) != self.name: + self.ren_btn.set_sensitive(True) + def get_obj(self): """ Allow MonitoredDataType to see our temporary GrampsType object """ return self.obj @@ -404,6 +422,7 @@ def rename(self, _button): # remove the old custom type from db types = getattr(self.db, self.t_table[self.r_indx][0], None) types.remove(self.name) + self.db.emit("custom-type-changed") # reload the gui self.model_load() @@ -416,7 +435,7 @@ def mod_recurse(self, mod_obj, type_obj): for item in mod_obj.__dict__.values(): self.mod_recurse(item, type_obj) if isinstance(mod_obj, self.t_table[self.r_indx][2]): # if right type - if(mod_obj.string == self.name): # and right value + if(str(mod_obj) == self.name): # and right value mod_obj.set(type_obj) def delete(self, _button): @@ -441,6 +460,7 @@ def delete(self, _button): # actually remove from the db types = getattr(self.db, attr, None) types.remove(self.name) + self.db.emit("custom-type-changed") self.model.remove(self.iter_) diff --git a/lxml/etreeGramplet.gpr.py b/lxml/etreeGramplet.gpr.py index da4adc0a9..001671b07 100644 --- a/lxml/etreeGramplet.gpr.py +++ b/lxml/etreeGramplet.gpr.py @@ -6,10 +6,10 @@ register(GRAMPLET, id="etree Gramplet", - name=_("etree"), + name=_("etree Gramplet"), description = _("Gramplet for testing etree with Gramps XML"), - status = STABLE, - version = '1.0.10', + status = STABLE, # not yet tested with python 3 + version = '1.1.0', gramps_target_version = "5.1", include_in_listing = False, height = 400, diff --git a/lxml/etreeGramplet.py b/lxml/etreeGramplet.py index 255e9fc13..fb6342527 100644 --- a/lxml/etreeGramplet.py +++ b/lxml/etreeGramplet.py @@ -63,7 +63,7 @@ -NAMESPACE = '{http://gramps-project.org/xml/1.7.1/}' +NAMESPACE = '{http://gramps-project.org/xml/1.8.0/}' #------------------------------------------------------------------------- # diff --git a/lxml/grampsxml.dtd b/lxml/grampsxml.dtd index 15128fd4a..1acfe670e 100644 --- a/lxml/grampsxml.dtd +++ b/lxml/grampsxml.dtd @@ -1,6 +1,6 @@ - @@ -59,10 +58,10 @@ DATABASE tags --> - - + + - @@ -123,7 +122,7 @@ GENDER has values of M, F, or U. - - - + @@ -221,7 +220,7 @@ EVENT - - + - - + - + + + + + + + + + - + + + + + + + @@ -437,7 +465,7 @@ SHARED ELEMENTS quality (estimated|calculated) #IMPLIED cformat CDATA #IMPLIED dualdated (0|1) #IMPLIED - newyear CDATA #IMPLIED + newyear CDATA #IMPLIED > @@ -447,7 +475,7 @@ SHARED ELEMENTS quality (estimated|calculated) #IMPLIED cformat CDATA #IMPLIED dualdated (0|1) #IMPLIED - newyear CDATA #IMPLIED + newyear CDATA #IMPLIED > @@ -457,7 +485,7 @@ SHARED ELEMENTS quality (estimated|calculated) #IMPLIED cformat CDATA #IMPLIED dualdated (0|1) #IMPLIED - newyear CDATA #IMPLIED + newyear CDATA #IMPLIED > @@ -530,16 +558,11 @@ SHARED ELEMENTS > - - - - +> \ No newline at end of file diff --git a/lxml/grampsxml.rng b/lxml/grampsxml.rng index 4888d331f..cfd815d74 100644 --- a/lxml/grampsxml.rng +++ b/lxml/grampsxml.rng @@ -1,6 +1,6 @@ -