models_load.py

# models_load.py

import logging

from sqlalchemy import insert

from .models import (MACROAREA, CLASSIFICATION,
                     Languoid,
                     languoid_macroarea, Macroarea,
                     languoid_country, Country,
                     Link, Timespan,
                     Source, SourceProvider,
                     Bibfile, Bibitem,
                     Altname, AltnameProvider,
                     Trigger, Identifier, IdentifierSite,
                     ClassificationComment, ClassificationRef,
                     Endangerment, EndangermentSource,
                     EthnologueComment,
                     IsoRetirement, IsoRetirementChangeTo)

__all__ = ['load']


log = logging.getLogger(__name__)


def load(languoids, conn):
    insert_lang = insert(Languoid, bind=conn).execute

    macroareas = sorted(MACROAREA)
    log.debug('insert macroareas: %r', macroareas)
    insert(Macroarea, bind=conn).execute([{'name': n} for n in macroareas])

    lang_ma = languoid_macroarea.insert(bind=conn).execute

    insert_country = insert(Country, bind=conn).execute
    lang_country = languoid_country.insert(bind=conn).execute

    def unseen_countries(countries, _seen={}):
        for c in countries:
            id_, name = (c[k] for k in ('id', 'name'))
            try:
                assert _seen[id_] == name
            except KeyError:
                _seen[id_] = name
                yield c

    insert_link = insert(Link, bind=conn).execute

    insert_timespan = insert(Timespan, bind=conn).execute

    insert_source = insert(Source, bind=conn).execute
    insert_sourceprovider = insert(SourceProvider, bind=conn).execute

    class SourceProviderMap(dict):

        def __missing__(self, name):
            log.debug('insert new source provider: %r', name)
            id, = insert_sourceprovider(name=name).inserted_primary_key
            self[name] = id
            return id

    sourceprovider_ids = SourceProviderMap()

    insert_bibfile = insert(Bibfile, bind=conn).execute
    insert_bibitem = insert(Bibitem, bind=conn).execute

    class BibfileMap(dict):

        def __missing__(self, name):
            log.debug('insert new bibfile: %r', name)
            id, = insert_bibfile(name=name).inserted_primary_key
            self[name] = id
            return id

    bibfile_ids = BibfileMap()

    class BibitemMap(dict):

        def __missing__(self, key):
            bibfile_name, bibkey = key
            id, = insert_bibitem(bibfile_id=bibfile_ids[bibfile_name],
                                 bibkey=bibkey).inserted_primary_key
            self[key] = id
            return id

        def pop_params(self, params):
            return self[params.pop('bibfile'), params.pop('bibkey')]

    bibitem_ids = BibitemMap()

    insert_altname = insert(Altname, bind=conn).execute
    insert_altnameprovider = insert(AltnameProvider, bind=conn).execute

    class AltnameProviderMap(dict):

        def __missing__(self, name):
            log.debug('insert new altname provider: %r', name)
            id, = insert_altnameprovider(name=name).inserted_primary_key
            self[name] = id
            return id

    altnameprovider_ids = AltnameProviderMap()

    insert_trigger = insert(Trigger, bind=conn).execute

    insert_ident = insert(Identifier, bind=conn).execute
    insert_identsite = insert(IdentifierSite, bind=conn).execute

    class IdentifierSiteMap(dict):

        def __missing__(self, name):
            log.debug('insert new identifier site: %r', name)
            id, = insert_identsite(name=name).inserted_primary_key
            self[name] = id
            return id

    identifiersite_ids = IdentifierSiteMap()

    insert_comment = insert(ClassificationComment, bind=conn).execute
    insert_ref = insert(ClassificationRef, bind=conn).execute

    insert_enda = insert(Endangerment, bind=conn).execute
    insert_enda_source = insert(EndangermentSource, bind=conn).execute

    class EndangermentSourceMap(dict):

        @staticmethod
        def params_to_key(params):
            return tuple(sorted(params.items()))

        def __missing__(self, key):
            params = dict(key)
            if params.get('bibfile') is not None:
                params['bibitem_id'] = bibitem_ids.pop_params(params)
            log.debug('insert new endangerment_source: %r', params)
            id, = insert_enda_source(**params).inserted_primary_key
            self[key] = id
            return id

    es_ids = EndangermentSourceMap()

    insert_el = insert(EthnologueComment, bind=conn).execute

    insert_ir = insert(IsoRetirement, bind=conn).execute
    insert_irct = insert(IsoRetirementChangeTo, bind=conn).execute

    for _, l in languoids:
        lid = l['id']

        macroareas = l.pop('macroareas')
        countries = l.pop('countries')
        links = l.pop('links')
        timespan = l.pop('timespan', None)

        sources = l.pop('sources', None)
        altnames = l.pop('altnames', None)
        triggers = l.pop('triggers', None)
        identifier = l.pop('identifier', None)
        classification = l.pop('classification', None)
        endangerment = l.pop('endangerment', None)
        hh_ethnologue_comment = l.pop('hh_ethnologue_comment', None)
        iso_retirement = l.pop('iso_retirement', None)

        insert_lang(l)

        if macroareas:
            lang_ma([{'languoid_id': lid, 'macroarea_name': ma}
                     for ma in macroareas])

        if countries:
            new_countries = list(unseen_countries(countries))
            if new_countries:
                ids = [n['id'] for n in new_countries]
                log.debug('insert new countries: %r', ids)

                insert_country(new_countries)

            lang_country([{'languoid_id': lid, 'country_id': c['id']}
                          for c in countries])

        if links:
            insert_link([dict(languoid_id=lid, ord=i, **link)
                         for i, link in enumerate(links, 1)])

        if timespan:
            insert_timespan(languoid_id=lid, **timespan)

        if sources is not None:
            for provider, data in sources.items():
                provider_id = sourceprovider_ids[provider]
                insert_source([dict(languoid_id=lid,
                                    provider_id=provider_id,
                                    bibitem_id=bibitem_ids.pop_params(s), **s)
                               for s in data])

        if altnames is not None:
            for provider, names in altnames.items():
                provider_id = altnameprovider_ids[provider]
                half, full = groups = ([], [])
                for n in names:
                    r = dict(languoid_id=lid, provider_id=provider_id, **n)
                    if 'lang' not in r:
                        half.append(r)
                    elif not r['lang']:  # lang = r.get('lang') or server_default
                        r.pop('lang')
                        half.append(r)
                    else:
                        full.append(r)
                for rows in groups:
                    if rows:
                        insert_altname(rows)

        if triggers is not None:
            insert_trigger([{'languoid_id': lid, 'field': field,
                             'trigger': t, 'ord': i}
                            for field, triggers in triggers.items()
                            for i, t in enumerate(triggers, 1)])

        if identifier is not None:
            insert_ident([dict(languoid_id=lid,
                               site_id=identifiersite_ids[site],
                               identifier=i)
                          for site, i in identifier.items()])

        if classification is not None:
            for c, value in classification.items():
                isref, kind = CLASSIFICATION[c]
                if isref:
                    insert_ref([dict(languoid_id=lid, kind=kind,
                                     bibitem_id=bibitem_ids.pop_params(r),
                                     ord=i, **r)
                                for i, r in enumerate(value, 1)])
                else:
                    insert_comment(languoid_id=lid, kind=kind, comment=value)

        if endangerment is not None:
            source = es_ids.params_to_key(endangerment.pop('source'))
            insert_enda(languoid_id=lid,
                        source_id=es_ids[source],
                        **endangerment)

        if hh_ethnologue_comment is not None:
            insert_el(languoid_id=lid, **hh_ethnologue_comment)

        if iso_retirement is not None:
            change_to = iso_retirement.pop('change_to')
            insert_ir(languoid_id=lid, **iso_retirement)

            if change_to:
                insert_irct([{'languoid_id': lid, 'code': c, 'ord': i}
                             for i, c in enumerate(change_to, 1)])