From 2b02fa6c6fceddb5c256e7784358092de3c5ea3c Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Fri, 29 Mar 2024 13:14:35 -0700 Subject: [PATCH 01/12] Resolving rebase conflict Signed-off-by: pfeairheller --- src/keri/app/cli/commands/rename.py | 17 ++- src/keri/app/habbing.py | 163 ++++++++-------------------- src/keri/db/basing.py | 44 +------- src/keri/db/versioning.py | 24 ++++ tests/app/test_habbing.py | 34 ++---- tests/app/test_signify.py | 4 +- tests/core/test_keystate.py | 4 +- tests/db/test_basing.py | 4 +- 8 files changed, 102 insertions(+), 192 deletions(-) create mode 100644 src/keri/db/versioning.py diff --git a/src/keri/app/cli/commands/rename.py b/src/keri/app/cli/commands/rename.py index 60fff68ef..a12420650 100644 --- a/src/keri/app/cli/commands/rename.py +++ b/src/keri/app/cli/commands/rename.py @@ -45,12 +45,19 @@ def rename(tymth, tock=0.0, **opts): try: with existing.existingHab(name=name, alias=alias, base=base, bran=bran) as (hby, hab): - habord = hab.db.habs.get(keys=alias) - hab.db.habs.put(keys=newAlias, - val=habord) - hab.db.habs.rem(keys=alias) + if hby.habByName(newAlias) is not None: + print(f"{newAlias} is already in use") - print(f"Hab {alias} renamed to {newAlias}") + if (prefixer := hab.db.names.get(keys=("", name))) is not None: + + habord = hab.db.habs.get(keys=prefixer.qb64) + habord.name = name + hab.db.habs.pin(keys=habord.hid, + val=habord) + hab.db.names.pin(keys=("", name), val=prefixer) + hab.db.names.rem(keys=("", alias)) + + print(f"Hab {alias} renamed to {newAlias}") except ConfigurationError as e: print(f"identifier prefix for {name} does not exist, incept must be run first", ) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 4fec262fa..795a8000a 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -226,7 +226,6 @@ def __init__(self, *, name='test', base="", temp=False, self.psr = parsing.Parser(framed=True, kvy=self.kvy, rvy=self.rvy, exc=self.exc, local=True) self.habs = {} # empty .habs - self.namespaces = {} # empty .namespaces self._signator = None self.inited = False @@ -327,86 +326,46 @@ def loadHabs(self): self.reconfigure() # pre hab load reconfiguration groups = [] - for name, habord in self.db.habs.getItemIter(): - name = ".".join(name) # detupleize the database key name + for prefix, habord in self.db.habs.getItemIter(): pre = habord.hid # create Hab instance and inject dependencies if habord.mid and not habord.sid: hab = GroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=pre, temp=self.temp, smids=habord.smids) + name=habord.name, pre=pre, temp=self.temp, smids=habord.smids) groups.append(habord) elif habord.sid and not habord.mid: hab = SignifyHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=habord.sid) + name=habord.name, pre=habord.sid) elif habord.sid and habord.mid: hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=pre) + name=habord.name, pre=pre) groups.append(habord) else: hab = Hab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=pre, temp=self.temp) + name=habord.name, pre=pre, temp=self.temp) # Rules for acceptance: # It is accepted into its own local KEL even if it has not been fully # witnessed and if delegated, its delegator has not yet sealed it if not hab.accepted and not habord.mid: raise kering.ConfigurationError(f"Problem loading Hab pre=" - f"{pre} name={name} from db.") + f"{pre} name={habord.name} from db.") # read in config file and process any oobis or endpoints for hab hab.inited = True self.habs[hab.pre] = hab - for keys, habord in self.db.nmsp.getItemIter(): - ns = keys[0] - name = ".".join(keys[1:]) # detupleize the database key name - pre = habord.hid - - # create Hab instance and inject dependencies - if habord.mid and not habord.sid: - hab = GroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, - rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, ns=ns, pre=pre, temp=self.temp, smids=habord.smids) - groups.append(habord) - elif habord.sid and not habord.mid: - hab = SignifyHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, - rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, ns=ns, pre=habord.sid) - elif habord.sid and habord.mid: - hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, - rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=habord.sid) - groups.append(habord) - else: - hab = Hab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, - rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, ns=ns, pre=pre, temp=self.temp) - - # Rules for acceptance: - # It is accepted into its own local KEL even if it has not been fully - # witnessed and if delegated, its delegator has not yet sealed it - if not hab.accepted and not habord.mid: - raise kering.ConfigurationError(f"Problem loading Hab pre=" - f"{pre} name={name} from db.") - - # read in config file and process any oobis or endpoints for hab - hab.inited = True - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - # Populate the participant hab after loading all habs for habord in groups: self.habs[habord.hid].mhab = self.habs[habord.mid] self.reconfigure() # post hab load reconfiguration - def makeHab(self, name, ns=None, cf=None, **kwa): """Make new Hab with name, pre is generated from **kwa @@ -437,13 +396,7 @@ def makeHab(self, name, ns=None, cf=None, **kwa): hab.make(**kwa) - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab def makeGroupHab(self, group, mhab, smids, rmids=None, ns=None, **kwa): @@ -514,13 +467,7 @@ def makeGroupHab(self, group, mhab, smids, rmids=None, ns=None, **kwa): name=group, ns=ns, mhab=mhab, smids=smids, rmids=rmids, temp=self.temp) hab.make(**kwa) # finish making group hab with injected pass throughs - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab def joinGroupHab(self, pre, group, mhab, smids, rmids=None, ns=None): @@ -568,6 +515,8 @@ def joinGroupHab(self, pre, group, mhab, smids, rmids=None, ns=None): hab.pre = pre habord = basing.HabitatRecord(hid=hab.pre, + name=self.name, + domain=ns, mid=mhab.pre, smids=smids, rmids=rmids) @@ -576,13 +525,7 @@ def joinGroupHab(self, pre, group, mhab, smids, rmids=None, ns=None): hab.prefixes.add(pre) hab.inited = True - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab def makeSignifyHab(self, name, ns=None, **kwa): @@ -592,13 +535,7 @@ def makeSignifyHab(self, name, ns=None, **kwa): name=name, ns=ns, temp=self.temp) hab.make(**kwa) # finish making group hab with injected pass throughs - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab def makeSignifyGroupHab(self, name, mhab, ns=None, **kwa): @@ -608,13 +545,8 @@ def makeSignifyGroupHab(self, name, mhab, ns=None, **kwa): name=name, mhab=mhab, ns=ns, temp=self.temp) hab.make(**kwa) # finish making group hab with injected pass throughs - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab + self.habs[hab.pre] = hab return hab def joinSignifyGroupHab(self, pre, name, mhab, smids, rmids=None, ns=None): @@ -663,6 +595,8 @@ def joinSignifyGroupHab(self, pre, name, mhab, smids, rmids=None, ns=None): hab.pre = pre habord = basing.HabitatRecord(hid=hab.pre, sid=mhab.pre, + name=name, + domain=ns, smids=smids, rmids=rmids) @@ -670,21 +604,19 @@ def joinSignifyGroupHab(self, pre, name, mhab, smids, rmids=None, ns=None): hab.prefixes.add(pre) hab.inited = True - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab - def deleteHab(self, name): - hab = self.habByName(name) + def deleteHab(self, name, ns=None): + hab = self.habByName(name, ns=ns) if not hab: return False - if not self.db.habs.rem(keys=(name,)): + if not self.db.habs.rem(keys=(hab.pre,)): + return False + + ns = "" if ns is None else ns + if not self.db.names.rem(keys=(ns, name)): return False del self.habs[hab.pre] @@ -760,7 +692,7 @@ def prefixes(self): def habByPre(self, pre): """ - Returns the Hab instance from .habs or .namespace + Returns the Hab instance from .habs or None including the default namespace. Args: @@ -769,15 +701,10 @@ def habByPre(self, pre): Returns: Hab: Hab instance for the aid pre or None """ - hab = None if pre in self.habs: - hab = self.habs[pre] - else: - for nsp in self.namespaces.values(): - if pre in nsp: - hab = nsp[pre] + return self.habs[pre] - return hab + return None def habByName(self, name, ns=None): """ @@ -790,13 +717,11 @@ def habByName(self, name, ns=None): ns (str): optional namespace of hab """ - if ns is not None: - if (habord := self.db.nmsp.get(keys=(ns, name))) is not None: - habs = self.namespaces[ns] - return habs[habord.hid] if habord.hid in habs else None - - elif (habord := self.db.habs.get(name)) is not None: - return self.habs[habord.hid] if habord.hid in self.habs else None + ns = "" if ns is None else ns + if (prefixer := self.db.names.get(keys=(ns, name))) is not None: + pre = prefixer.qb64 + if pre in self.habs: + return self.habs[pre] return None @@ -1131,12 +1056,11 @@ def make(self, DnD, code, data, delpre, estOnly, isith, verfers, nsith, digers, return serder def save(self, habord): - if self.ns is None: - self.db.habs.pin(keys=self.name, - val=habord) - else: - self.db.nmsp.put(keys=(self.ns, self.name), - val=habord) + self.db.habs.pin(keys=self.pre, + val=habord) + ns = "" if self.ns is None else self.ns + self.db.names.put(keys=(ns, self.name), + val=coring.Prefixer(qb64=self.pre)) def reconfigure(self): """Apply configuration from config file managed by .cf. to this Hab. @@ -2349,7 +2273,7 @@ def make(self, *, secrecies=None, iridx=0, code=coring.MtrDex.Blake3_256, dcode= self.mgr.move(old=opre, new=self.pre) # move to incept event pre # may want db method that updates .habs. and .prefixes together - habord = basing.HabitatRecord(hid=self.pre) + habord = basing.HabitatRecord(hid=self.pre, name=self.name, domain=self.ns) if not hidden: self.save(habord) @@ -2437,7 +2361,7 @@ def make(self, *, serder, sigers, **kwargs): self.processEvent(serder, sigers) - habord = basing.HabitatRecord(hid=self.pre, sid=self.pre) + habord = basing.HabitatRecord(hid=self.pre, sid=self.pre, name=self.name, domain=self.ns) self.save(habord) self.inited = True @@ -2596,7 +2520,7 @@ def make(self, *, serder, sigers, **kwargs): self.processEvent(serder, sigers) - habord = basing.HabitatRecord(hid=self.pre, mid=self.mhab.pre, sid=self.pre) + habord = basing.HabitatRecord(hid=self.pre, mid=self.mhab.pre, sid=self.pre, name=self.name, domain=self.ns) self.save(habord) self.inited = True @@ -2622,6 +2546,10 @@ def processEvent(self, serder, sigers): raise kering.ValidationError(f"Improper Habitat event type={serder.ked['t']} for " f"pre={self.pre}.") + def rotate(self, *, smids=None, rmids=None, serder=None, sigers=None, **kwargs): + # TODO: save smids and rmids here + super(SignifyGroupHab, self).rotate(serder=serder, sigers=sigers, **kwargs) + class GroupHab(BaseHab): """ @@ -2785,6 +2713,8 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, habord = basing.HabitatRecord(hid=self.pre, mid=self.mhab.pre, + name=self.name, + domain=self.ns, smids=self.smids, rmids=self.rmids) @@ -2793,7 +2723,8 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, self.inited = True - def rotate(self, serder=None, **kwargs): + def rotate(self, smids=None, rmids=None, serder=None, **kwargs): + # TODO: save smids and rmids here if serder is None: return super(GroupHab, self).rotate(**kwargs) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index d8bf0039e..f3021f501 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -274,6 +274,8 @@ class HabitatRecord: # baser.habs """ hid: str # hab own identifier prefix qb64 + name: str | None = None + domain: str | None = None mid: str | None = None # group member identifier qb64 when hid is group smids: list | None = None # group signing member ids when hid is group rmids: list | None = None # group rotating member ids when hid is group @@ -953,11 +955,8 @@ def reopen(self, **kwa): self.habs = koming.Komer(db=self, subkey='habs.', schema=HabitatRecord, ) - - # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix - self.nmsp = koming.Komer(db=self, - subkey='nmsp.', - schema=HabitatRecord, ) + # habitat name database mapping (domain,name) as key to Prefixer + self.names = subing.CesrSuber(db=self, subkey='names.', klas=coring.Prefixer, sep="^") # SAD support datetime stamps and signatures indexed and not-indexed # all sad sdts (sad datetime serializations) maps said to date-time @@ -1173,7 +1172,6 @@ def reopen(self, **kwa): return self.env - def reload(self): """ Reload stored prefixes and Kevers from .habs @@ -1200,28 +1198,6 @@ def reload(self): for keys in removes: # remove bare .habs records self.habs.rem(keys=keys) - # Load namespaced Habs - removes = [] - for keys, data in self.nmsp.getItemIter(): - if (ksr := self.states.get(keys=data.hid)) is not None: - try: - kever = eventing.Kever(state=ksr, - db=self, - local=True) - except kering.MissingEntryError as ex: # no kel event for keystate - removes.append(keys) # remove from .habs - continue - self.kevers[kever.prefixer.qb64] = kever - self.prefixes.add(kever.prefixer.qb64) - if data.mid: # group hab - self.groups.add(data.hid) - elif data.mid is None: # in .habs but no corresponding key state and not a group so remove - removes.append(keys) # no key state or KEL event for .hab record - - for keys in removes: # remove bare .habs records - self.nmsp.rem(keys=keys) - - def clean(self): """ Clean database by creating re-verified cleaned cloned copy @@ -1259,17 +1235,12 @@ def clean(self): for keys, val in self.habs.getItemIter(): if val.hid in copy.kevers: # only copy habs that verified copy.habs.put(keys=keys, val=val) + ns = "" if val.domain is None else val.domain + copy.names.put(keys=(ns, val.name), val=coring.Prefixer(qb64=val.hid)) copy.prefixes.add(val.hid) if val.mid: # a group hab copy.groups.add(val.hid) - # ToDo XXXX - # is this obsolete? Should this be removed or should this be - # be the Signator name not the default name of the Habery? - if not copy.habs.get(keys=(self.name,)): - raise ValueError("Error cloning habs, missing orig name={}." - "".format(self.name)) - # clone .ends and .locs databases for keys, val in self.ends.getItemIter(): exists = False # only copy if entries in both .ends and .locs @@ -1315,7 +1286,6 @@ def clean(self): if os.path.exists(copy.path): shutil.rmtree(copy.path) - def clonePreIter(self, pre, fn=0): """ Returns iterator of first seen event messages with attachments for the @@ -1504,8 +1474,6 @@ def findAnchoringSeal(self, pre, seal, sn=0): return srdr return None - - def signingMembers(self, pre: str): """ Find signing members of a multisig group aid. diff --git a/src/keri/db/versioning.py b/src/keri/db/versioning.py new file mode 100644 index 000000000..d1e6b21c7 --- /dev/null +++ b/src/keri/db/versioning.py @@ -0,0 +1,24 @@ +# -*- encoding: utf-8 -*- +""" +keri.db.dbing module + + +""" +from keri.db import koming +from keri.db.basing import HabitatRecord + + +class Upgrader: + + def __init__(self, db): + self.db = db + + # habitat application state keyed by habitat name, includes prefix + self.habs = koming.Komer(db=self.db, + subkey='habs.', + schema=HabitatRecord, ) + + # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix + self.nmsp = koming.Komer(db=self.db, + subkey='nmsp.', + schema=HabitatRecord, ) diff --git a/tests/app/test_habbing.py b/tests/app/test_habbing.py index ed35dd295..9b23675d4 100644 --- a/tests/app/test_habbing.py +++ b/tests/app/test_habbing.py @@ -727,8 +727,7 @@ def test_namespaced_habs(): nshab = hby.makeHab(name="test2", ns="agent") assert nshab.pre == "EErXOolQNmKrTMKfXdQ1sj8YsgZZe4wMXZwsX-j1V6Dd" - assert len(hby.habs) == 1 - assert len(hby.namespaces) == 1 + assert len(hby.habs) == 2 assert len(hby.prefixes) == 2 found = hby.habByName(name="test2") @@ -742,11 +741,8 @@ def test_namespaced_habs(): nshab = hby.makeHab(name="test.3", ns="agent") assert nshab.pre == "EG5FUOzW_KKVB8JGlNGoZAADDC8cZ6Jt079nLEaFnYcg" - assert len(hby.habs) == 1 - assert len(hby.namespaces) == 1 + assert len(hby.habs) == 3 assert len(hby.prefixes) == 3 - ns = hby.namespaces['agent'] - assert len(ns) == 2 # '.' characters not allowed in namespace names with pytest.raises(kering.ConfigurationError): @@ -773,14 +769,9 @@ def test_namespaced_habs(): assert pre in hby.db.kevers # read through cache assert pre in hby.db.prefixes - assert len(hby.habs) == 2 + assert len(hby.habs) == 5 assert len(hby.db.prefixes) == 5 - agent = hby.namespaces["agent"] - assert len(agent) == 2 - ctrl = hby.namespaces["controller"] - assert len(ctrl) == 1 - found = hby.habByName(name=name) assert found.pre == opre found = hby.habByName(name="test.1") @@ -840,10 +831,10 @@ def test_hab_by_pre(): # Only habs in default namespace are in hby.habs assert hab1.pre in hby.habs assert hab2.pre in hby.habs - assert hab3.pre not in hby.habs - assert hab4.pre not in hby.habs - assert hab5.pre not in hby.habs - assert hab6.pre not in hby.habs + assert hab3.pre in hby.habs + assert hab4.pre in hby.habs + assert hab5.pre in hby.habs + assert hab6.pre in hby.habs assert hby.habByPre("EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3") is None @@ -854,17 +845,6 @@ def test_hab_by_pre(): assert hby.habByPre(pre=hab5.pre) == hab5 assert hby.habByPre(pre=hab6.pre) == hab6 - assert "one" in hby.namespaces - assert hab3.pre in hby.namespaces["one"] - assert hab4.pre in hby.namespaces["one"] - assert hab1.pre not in hby.namespaces["one"] - assert hab2.pre not in hby.namespaces["one"] - assert "two" in hby.namespaces - assert hab5.pre in hby.namespaces["two"] - assert hab6.pre in hby.namespaces["two"] - assert hab1.pre not in hby.namespaces["two"] - assert hab2.pre not in hby.namespaces["two"] - def test_postman_endsfor(): with habbing.openHby(name="test", temp=True, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby, \ diff --git a/tests/app/test_signify.py b/tests/app/test_signify.py index 4afaa4647..1d9a7d08f 100644 --- a/tests/app/test_signify.py +++ b/tests/app/test_signify.py @@ -71,7 +71,7 @@ def test_remote_salty_hab(): assert [verfer.qb64 for verfer in kever.verfers] == keys assert [diger.qb64 for diger in kever.ndigers] == nxt - habord = remote.db.habs.get(name) + habord = remote.db.habs.get(hab.pre) assert habord.hid == "EHeU-ldGfJhxceV9BTq38HdFUoasoWEcYATiyZCcDH7N" assert habord.sid == "EHeU-ldGfJhxceV9BTq38HdFUoasoWEcYATiyZCcDH7N" @@ -113,7 +113,7 @@ def test_remote_salty_hab(): assert [verfer.qb64 for verfer in kever.verfers] == keys1 assert [diger.qb64 for diger in kever.ndigers] == nxt1 - habord = remote.db.habs.get(name) + habord = remote.db.habs.get(hab.pre) assert habord.hid == "EHeU-ldGfJhxceV9BTq38HdFUoasoWEcYATiyZCcDH7N" assert habord.sid == "EHeU-ldGfJhxceV9BTq38HdFUoasoWEcYATiyZCcDH7N" diff --git a/tests/core/test_keystate.py b/tests/core/test_keystate.py index 40c2c40d3..a9160a10f 100644 --- a/tests/core/test_keystate.py +++ b/tests/core/test_keystate.py @@ -163,9 +163,9 @@ def test_keystate(mockHelpingNowUTC): bamHab = bamHby.makeHab(name="bam", isith='1', icount=1, transferable=True) # Set Wes has Bam's watcher - habr = bamHab.db.habs.get("bam") + habr = bamHab.db.habs.get(bamHab.pre) habr.watchers = [wesHab.pre] - bamHab.db.habs.pin("bam", habr) + bamHab.db.habs.pin(bamHab.pre, habr) bamRtr = routing.Router() bamRvy = routing.Revery(db=bamHby.db, rtr=bamRtr) diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 791e88095..575342f21 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -1753,7 +1753,7 @@ def test_clean_baser(): assert natHab.db.env.stat()['entries'] <= 96 #68 # verify name pre kom in db - data = natHab.db.habs.get(keys=natHab.name) + data = natHab.db.habs.get(keys=natHab.pre) assert data.hid == natHab.pre # add garbage event to corrupt database @@ -1812,7 +1812,7 @@ def test_clean_baser(): assert state.f == '6' # verify name pre kom in db - data = natHab.db.habs.get(keys=natHab.name) + data = natHab.db.habs.get(keys=natHab.pre) assert data.hid == natHab.pre From 694e3066add4f458a648dc39479229c47f17a9b6 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Thu, 29 Feb 2024 15:17:52 -0800 Subject: [PATCH 02/12] Change names database to Suber, add check for Hab with existing name. Signed-off-by: pfeairheller --- src/keri/app/cli/commands/rename.py | 6 +++--- src/keri/app/habbing.py | 10 ++++++---- src/keri/db/basing.py | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/keri/app/cli/commands/rename.py b/src/keri/app/cli/commands/rename.py index a12420650..af0fb0cf6 100644 --- a/src/keri/app/cli/commands/rename.py +++ b/src/keri/app/cli/commands/rename.py @@ -48,13 +48,13 @@ def rename(tymth, tock=0.0, **opts): if hby.habByName(newAlias) is not None: print(f"{newAlias} is already in use") - if (prefixer := hab.db.names.get(keys=("", name))) is not None: + if (pre := hab.db.names.get(keys=("", name))) is not None: - habord = hab.db.habs.get(keys=prefixer.qb64) + habord = hab.db.habs.get(keys=pre) habord.name = name hab.db.habs.pin(keys=habord.hid, val=habord) - hab.db.names.pin(keys=("", name), val=prefixer) + hab.db.names.pin(keys=("", name), val=pre) hab.db.names.rem(keys=("", alias)) print(f"Hab {alias} renamed to {newAlias}") diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 795a8000a..f146aca88 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -718,8 +718,7 @@ def habByName(self, name, ns=None): """ ns = "" if ns is None else ns - if (prefixer := self.db.names.get(keys=(ns, name))) is not None: - pre = prefixer.qb64 + if (pre := self.db.names.get(keys=(ns, name))) is not None: if pre in self.habs: return self.habs[pre] @@ -1059,8 +1058,11 @@ def save(self, habord): self.db.habs.pin(keys=self.pre, val=habord) ns = "" if self.ns is None else self.ns - self.db.names.put(keys=(ns, self.name), - val=coring.Prefixer(qb64=self.pre)) + if self.db.names.get(keys=(ns, self.name)) is not None: + raise ValueError("AID already exists with that name") + + self.db.names.pin(keys=(ns, self.name), + val=self.pre) def reconfigure(self): """Apply configuration from config file managed by .cf. to this Hab. diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index f3021f501..14daa5ab1 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -956,7 +956,7 @@ def reopen(self, **kwa): subkey='habs.', schema=HabitatRecord, ) # habitat name database mapping (domain,name) as key to Prefixer - self.names = subing.CesrSuber(db=self, subkey='names.', klas=coring.Prefixer, sep="^") + self.names = subing.Suber(db=self, subkey='names.', sep="^") # SAD support datetime stamps and signatures indexed and not-indexed # all sad sdts (sad datetime serializations) maps said to date-time @@ -1236,7 +1236,7 @@ def clean(self): if val.hid in copy.kevers: # only copy habs that verified copy.habs.put(keys=keys, val=val) ns = "" if val.domain is None else val.domain - copy.names.put(keys=(ns, val.name), val=coring.Prefixer(qb64=val.hid)) + copy.names.put(keys=(ns, val.name), val=val.hid) copy.prefixes.add(val.hid) if val.mid: # a group hab copy.groups.add(val.hid) From 7c89fcc7f96e67bbffc86ec9541d22eb296f3bd9 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Fri, 1 Mar 2024 12:42:31 -0800 Subject: [PATCH 03/12] Moving the tracking of signing members and rotation members from tracking public keys to static tracking of smids and rmids in the HabitatRecord in the database for the group Habs. Signed-off-by: pfeairheller --- .../demo/basic/demo-witness-async-script.sh | 22 ++++++------ src/keri/app/habbing.py | 30 +++++++++++++--- src/keri/core/eventing.py | 5 --- src/keri/db/basing.py | 35 ++++--------------- tests/app/test_grouping.py | 8 ++--- 5 files changed, 46 insertions(+), 54 deletions(-) diff --git a/scripts/demo/basic/demo-witness-async-script.sh b/scripts/demo/basic/demo-witness-async-script.sh index 57885a7cb..26b6aac89 100755 --- a/scripts/demo/basic/demo-witness-async-script.sh +++ b/scripts/demo/basic/demo-witness-async-script.sh @@ -13,34 +13,34 @@ function isSuccess() { } # CREATE DATABASE AND KEYSTORE -kli init --name witness-test --base "${KERI_TEMP_DIR}" --nopasscode +kli init --name async-witness-test --base "${KERI_TEMP_DIR}" --nopasscode isSuccess # RESOLVE WITNESS OOBIs -kli oobi resolve --name witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wan --oobi http://127.0.0.1:5643/oobi/BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM/controller +kli oobi resolve --name async-witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wan --oobi http://127.0.0.1:5643/oobi/BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM/controller isSuccess -kli oobi resolve --name witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wil --oobi http://127.0.0.1:5642/oobi/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha/controller +kli oobi resolve --name async-witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wil --oobi http://127.0.0.1:5642/oobi/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha/controller isSuccess -kli oobi resolve --name witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wes --oobi http://127.0.0.1:5644/oobi/BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX/controller +kli oobi resolve --name async-witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wes --oobi http://127.0.0.1:5644/oobi/BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX/controller isSuccess ## INCEPT AND PROPOGATE EVENTS AND RECEIPTS TO WITNESSES -kli incept --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --file "${KERI_DEMO_SCRIPT_DIR}/data/trans-wits-sample.json" +kli incept --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --file "${KERI_DEMO_SCRIPT_DIR}/data/trans-wits-sample.json" isSuccess -kli incept --name witness-test --base "${KERI_TEMP_DIR}" --alias inquisitor --file "${KERI_DEMO_SCRIPT_DIR}/data/inquisitor-sample.json" +kli incept --name async-witness-test --base "${KERI_TEMP_DIR}" --alias inquisitor --file "${KERI_DEMO_SCRIPT_DIR}/data/inquisitor-sample.json" isSuccess -kli status --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits +kli status --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits -kli rotate --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --witness-cut BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX +kli rotate --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --witness-cut BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX isSuccess -kli status --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits +kli status --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits -kli rotate --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --witness-add BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX +kli rotate --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --witness-add BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX isSuccess -kli status --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits +kli status --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits echo 'Test Complete' \ No newline at end of file diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index f146aca88..2dd9c594f 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -2512,8 +2512,11 @@ def replyEndRole(self, cid, role=None, eids=None, scheme=""): class SignifyGroupHab(SignifyHab): - def __init__(self, mhab=None, **kwa): + def __init__(self, smids, mhab=None, rmids=None, **kwa): self.mhab = mhab + self.smids = smids # group signing member aids in this group hab + self.rmids = rmids or smids # group rotating member aids in this group hab + super(SignifyGroupHab, self).__init__(**kwa) def make(self, *, serder, sigers, **kwargs): @@ -2549,9 +2552,19 @@ def processEvent(self, serder, sigers): f"pre={self.pre}.") def rotate(self, *, smids=None, rmids=None, serder=None, sigers=None, **kwargs): - # TODO: save smids and rmids here + + if (habord := self.db.habs.get(keys=(self.pre,))) is None: + raise kering.ValidationError(f"Missing HabitatRecord for pre={self.pre}") + super(SignifyGroupHab, self).rotate(serder=serder, sigers=sigers, **kwargs) + self.smids = smids + self.rmids = rmids + habord.smids = smids + habord.rmids = rmids + + self.db.habs.pin(keys=(self.pre,), val=habord) + class GroupHab(BaseHab): """ @@ -2633,11 +2646,10 @@ def __init__(self, smids, mhab=None, rmids=None, **kwa): """ self.mhab = mhab # local participant Hab of this group hab self.smids = smids # group signing member aids in this group hab - self.rmids = rmids # group rotating member aids in this group hab + self.rmids = rmids or smids # group rotating member aids in this group hab super(GroupHab, self).__init__(**kwa) - def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, nsith=None, toad=None, wits=None, delpre=None, estOnly=False, DnD=False, merfers, migers=None, data=None): @@ -2726,11 +2738,13 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, self.inited = True def rotate(self, smids=None, rmids=None, serder=None, **kwargs): - # TODO: save smids and rmids here if serder is None: return super(GroupHab, self).rotate(**kwargs) + if (habord := self.db.habs.get(keys=(self.pre,))) is None: + raise kering.ValidationError(f"Missing HabitatRecord for pre={self.pre}") + # sign handles group hab with .mhab case sigers = self.sign(ser=serder.raw, verfers=serder.verfers, rotated=True) @@ -2745,6 +2759,12 @@ def rotate(self, smids=None, rmids=None, serder=None, **kwargs): raise kering.ValidationError("Improper Habitat rotation for " "pre={self.pre}.") from ex + self.smids = smids + self.rmids = rmids + habord.smids = smids + habord.rmids = rmids + self.db.habs.pin(keys=(self.pre,), val=habord) + return msg def sign(self, ser, verfers=None, indexed=True, rotated=False, indices=None, ondices=None): diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index d0c063aec..1d3e860f8 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2967,11 +2967,6 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, esr = basing.EventSourceRecord(local=local) self.db.esrs.put(keys=dgkeys, val=esr) - val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) - for verfer in (serder.verfers if serder.verfers is not None else []): - self.db.pubs.add(keys=(verfer.qb64,), val=val) - for diger in (serder.ndigers if serder.ndigers is not None else []): - self.db.digs.add(keys=(diger.qb64,), val=val) if first: # append event dig to first seen database in order if seqner and saider: # delegation for authorized delegated or issued event couple = seqner.qb64b + saider.qb64b diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 14daa5ab1..8fc1cc1b0 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1486,22 +1486,10 @@ def signingMembers(self, pre: str): list: qb64 identifier prefixes of signing members for provided aid """ - members = [] - if pre not in self.kevers: - return members - - kever = self.kevers[pre] - for verfer in kever.verfers: - if (couples := self.pubs.get(keys=(verfer.qb64,))) is None: - continue - - for couple in couples: - prefixer, seqner = couple - if prefixer.qb64 != pre: # Rule out aid being queried - members.append(prefixer.qb64) - - return members + if (habord := self.habs.get(keys=(pre,))) is None: + return None + return habord.smids def rotationMembers(self, pre: str): """ Find rotation members of a multisig group aid. @@ -1514,21 +1502,10 @@ def rotationMembers(self, pre: str): Returns: list: qb64 identifier prefixes of rotation members for provided aid """ - members = [] - if pre not in self.kevers: - return members - - kever = self.kevers[pre] - for diger in kever.ndigers: - if (couples := self.digs.get(keys=(diger.qb64,))) is None: - continue - - for couple in couples: - prefixer, seqner = couple - if prefixer.qb64 != pre: # Rule out aid being queried - members.append(prefixer.qb64) + if (habord := self.habs.get(keys=(pre,))) is None: + return None - return members + return habord.rmids def fullyWitnessed(self, serder): """ Verify the witness threshold on the event diff --git a/tests/app/test_grouping.py b/tests/app/test_grouping.py index 5504c5781..fca1bc91d 100644 --- a/tests/app/test_grouping.py +++ b/tests/app/test_grouping.py @@ -662,11 +662,11 @@ def test_multisig_rotate(mockHelpingNowUTC): exn, atc = grouping.multisigRotateExn(ghab=ghab1, smids=ghab1.smids, rmids=ghab1.rmids, rot=rot) assert exn.ked["r"] == '/multisig/rot' - assert exn.saidb == b'ENfCk9DUUck6Ixe6cYnbCbJfIsisA3H4kHPwm5Z-2Tf8' + assert exn.saidb == b'EAE5RLyDb_3W8fUOzDOHWwpMAvHePaz8Jpz1XWL0b0lQ' assert atc == (b'-FABEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba0AAAAAAAAAAAAAAA' - b'AAAAAAAAEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba-AABAADChiAf' - b'iExAQ2ETkzzf7MOubXV9mL-r6fPsOI4yn348yeE5dXqdI7ddn5-wnPwNVjqqKkDp' - b'xlOEFYRiBQEbwZQC') + b'AAAAAAAAEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba-AABAACzqlzb' + b'yRO_M5Kp6nyN1K5JZ9s9D4UAreliSs4OiyCe_y9KnDeP65tub3DW5hUKlVQWnPv4' + b'TrOBHk2vJNOEJtAF') data = exn.ked["a"] assert data["smids"] == ghab1.smids From f24fceda4be5f8700fcfb98282e1c4a43fb774d8 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Fri, 1 Mar 2024 14:29:57 -0800 Subject: [PATCH 04/12] Two fixes to ensure proper tracking of smids and rmids. Signed-off-by: pfeairheller --- src/keri/app/habbing.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 2dd9c594f..53059ffb5 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -538,11 +538,11 @@ def makeSignifyHab(self, name, ns=None, **kwa): self.habs[hab.pre] = hab return hab - def makeSignifyGroupHab(self, name, mhab, ns=None, **kwa): + def makeSignifyGroupHab(self, name, mhab, smids, rmids=None, ns=None, **kwa): # create group Hab in this Habery hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, mhab=mhab, ns=ns, temp=self.temp) + name=name, mhab=mhab, smids=smids, rmids=rmids, ns=ns, temp=self.temp) hab.make(**kwa) # finish making group hab with injected pass throughs @@ -590,7 +590,7 @@ def joinSignifyGroupHab(self, pre, name, mhab, smids, rmids=None, ns=None): # create group Hab in this Habery hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, mhab=mhab, ns=ns, temp=self.temp) + name=name, mhab=mhab, smids=smids, rmids=rmids, ns=ns, temp=self.temp) hab.pre = pre habord = basing.HabitatRecord(hid=hab.pre, @@ -2525,7 +2525,8 @@ def make(self, *, serder, sigers, **kwargs): self.processEvent(serder, sigers) - habord = basing.HabitatRecord(hid=self.pre, mid=self.mhab.pre, sid=self.pre, name=self.name, domain=self.ns) + habord = basing.HabitatRecord(hid=self.pre, mid=self.mhab.pre, smids=self.smids, rmids=self.rmids, + sid=self.pre, name=self.name, domain=self.ns) self.save(habord) self.inited = True From 1fe95f771311521d9c3278626c7497ca9a6fabd4 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Tue, 5 Mar 2024 14:52:06 -0800 Subject: [PATCH 05/12] First working draft of a migration framework Signed-off-by: pfeairheller --- scripts/demo/basic/clean.sh | 46 +++++++++ src/keri/app/cli/commands/clean.py | 66 +++++++++++++ src/keri/app/cli/commands/migrate/__init__.py | 0 src/keri/app/cli/commands/migrate/list.py | 65 +++++++++++++ src/keri/app/cli/commands/migrate/run.py | 58 ++++++++++++ src/keri/app/cli/commands/migrate/show.py | 62 +++++++++++++ src/keri/app/cli/common/existing.py | 5 +- src/keri/app/habbing.py | 1 - src/keri/db/basing.py | 93 ++++++++++++++----- src/keri/db/migrating.py | 45 +++++++++ src/keri/db/migrations/__init__.py | 0 src/keri/db/migrations/rekey_habs.py | 64 +++++++++++++ 12 files changed, 479 insertions(+), 26 deletions(-) create mode 100755 scripts/demo/basic/clean.sh create mode 100644 src/keri/app/cli/commands/clean.py create mode 100644 src/keri/app/cli/commands/migrate/__init__.py create mode 100644 src/keri/app/cli/commands/migrate/list.py create mode 100644 src/keri/app/cli/commands/migrate/run.py create mode 100644 src/keri/app/cli/commands/migrate/show.py create mode 100644 src/keri/db/migrating.py create mode 100644 src/keri/db/migrations/__init__.py create mode 100644 src/keri/db/migrations/rekey_habs.py diff --git a/scripts/demo/basic/clean.sh b/scripts/demo/basic/clean.sh new file mode 100755 index 000000000..2862075d9 --- /dev/null +++ b/scripts/demo/basic/clean.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +echo "Testing clean" + +function isSuccess() { + ret=$? + if [ $ret -ne 0 ]; then + echo "Error $ret" + exit $ret + fi +} + +# CREATE DATABASE AND KEYSTORE +kli init --name nat --base "${KERI_TEMP_DIR}" --nopasscode +isSuccess + +# RESOLVE WITNESS OOBIs +kli oobi resolve --name nat --base "${KERI_TEMP_DIR}" --oobi-alias wan --oobi http://127.0.0.1:5643/oobi/BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM/controller +isSuccess +kli oobi resolve --name nat --base "${KERI_TEMP_DIR}" --oobi-alias wil --oobi http://127.0.0.1:5642/oobi/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha/controller +isSuccess +kli oobi resolve --name nat --base "${KERI_TEMP_DIR}" --oobi-alias wes --oobi http://127.0.0.1:5644/oobi/BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX/controller +isSuccess + +## INCEPT AND PROPOGATE EVENTS AND RECEIPTS TO WITNESSES +kli incept --name nat --base "${KERI_TEMP_DIR}" --receipt-endpoint --alias nat --file "${KERI_DEMO_SCRIPT_DIR}/data/trans-wits-sample.json" +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess +kli rotate --name nat --base "${KERI_TEMP_DIR}" --receipt-endpoint --alias nat --witness-cut BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX +isSuccess +kli rotate --name nat --base "${KERI_TEMP_DIR}" --receipt-endpoint --alias nat --witness-add BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess + +kli clean --name nat --base "${KERI_TEMP_DIR}" +isSuccess + +kli status --name nat --alias nat diff --git a/src/keri/app/cli/commands/clean.py b/src/keri/app/cli/commands/clean.py new file mode 100644 index 000000000..a7b6e1226 --- /dev/null +++ b/src/keri/app/cli/commands/clean.py @@ -0,0 +1,66 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing + +from keri.app.cli.common import existing +from keri.db import migrating + +logger = help.ogler.getLogger() + + +def handler(args): + """ + Launch KERI database initialization + + Args: + args(Namespace): arguments object from command line + """ + clean = CleanDoer(args) + return [clean] + + +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser.set_defaults(handler=handler, + transferable=True) + +# Parameters for basic structure of database +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--temp', '-t', help='create a temporary keystore, used for testing', default=False) + +# Parameters for Manager creation +# passcode => bran +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +class CleanDoer(doing.Doer): + + def __init__(self, args): + self.args = args + super(CleanDoer, self).__init__() + + def recur(self, tyme): + + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran, temp=self.args.temp) + print("Migrating...") + migrator = migrating.Migrator(db=hby.db) + migrator.migrate() + print("Finished") + + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran, temp=self.args.temp) + + print("Database open, performing clean...") + hby.db.clean() + print("Finished.") + + return True diff --git a/src/keri/app/cli/commands/migrate/__init__.py b/src/keri/app/cli/commands/migrate/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/keri/app/cli/commands/migrate/list.py b/src/keri/app/cli/commands/migrate/list.py new file mode 100644 index 000000000..ebe9722ae --- /dev/null +++ b/src/keri/app/cli/commands/migrate/list.py @@ -0,0 +1,65 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing +from prettytable import PrettyTable + +from keri.app.cli.common import existing +from keri.db import migrating + +logger = help.ogler.getLogger() + + +def handler(args): + """ + Launch KERI database initialization + + Args: + args(Namespace): arguments object from command line + """ + lister = ListDoer(args) + return [lister] + + +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser.set_defaults(handler=handler, + transferable=True) + +# Parameters for basic structure of database +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--temp', '-t', help='create a temporary keystore, used for testing', default=False) + +# Parameters for Manager creation +# passcode => bran +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +class ListDoer(doing.Doer): + + def __init__(self, args): + self.args = args + super(ListDoer, self).__init__() + + def recur(self, tyme): + tab = PrettyTable() + tab.field_names = ["Num", "Name", "Date Completed"] + tab.align["Name"] = "l" + + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran, temp=self.args.temp) + + migrator = migrating.Migrator(db=hby.db) + for idx, (name, dater) in enumerate(migrator.complete()): + date = dater.datetime.strftime("%Y-%m-%d %H:%M") if dater is not None else "Not Run" + tab.add_row((f"{idx+1}", f"{name}", date)) + + print(tab) + return True diff --git a/src/keri/app/cli/commands/migrate/run.py b/src/keri/app/cli/commands/migrate/run.py new file mode 100644 index 000000000..f4ae6e33c --- /dev/null +++ b/src/keri/app/cli/commands/migrate/run.py @@ -0,0 +1,58 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing + +from keri.app.cli.common import existing +from keri.db import migrating + +logger = help.ogler.getLogger() + + +def handler(args): + """ + Launch KERI database initialization + + Args: + args(Namespace): arguments object from command line + """ + clean = CleanDoer(args) + return [clean] + + +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser.set_defaults(handler=handler, + transferable=True) + +# Parameters for basic structure of database +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--temp', '-t', help='create a temporary keystore, used for testing', default=False) + +# Parameters for Manager creation +# passcode => bran +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +class CleanDoer(doing.Doer): + + def __init__(self, args): + self.args = args + super(CleanDoer, self).__init__() + + def recur(self, tyme): + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran, temp=self.args.temp) + print("Migrating...") + migrator = migrating.Migrator(db=hby.db) + migrator.migrate() + print("Finished") + + return True diff --git a/src/keri/app/cli/commands/migrate/show.py b/src/keri/app/cli/commands/migrate/show.py new file mode 100644 index 000000000..3ce45c2ca --- /dev/null +++ b/src/keri/app/cli/commands/migrate/show.py @@ -0,0 +1,62 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing + +from keri.app.cli.common import existing +from keri.db import migrating + +logger = help.ogler.getLogger() + + +def handler(args): + """ + Launch KERI database initialization + + Args: + args(Namespace): arguments object from command line + """ + clean = CleanDoer(args) + return [clean] + + +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser.set_defaults(handler=handler, + transferable=True) + +# Parameters for basic structure of database +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--temp', '-t', help='create a temporary keystore, used for testing', default=False) +parser.add_argument('--migration', '-m', help='migration name', required=True) + + +# Parameters for Manager creation +# passcode => bran +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +class CleanDoer(doing.Doer): + + def __init__(self, args): + self.args = args + super(CleanDoer, self).__init__() + + def recur(self, tyme): + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran, temp=self.args.temp) + + migrator = migrating.Migrator(db=hby.db) + [(name, dater)] = migrator.complete(name=self.args.migration) + date = dater.datetime.strftime("%Y-%m-%d %H:%M") if dater is not None else "Not Run" + + print(f"{self.args.migration} -> {date}") + + return True diff --git a/src/keri/app/cli/common/existing.py b/src/keri/app/cli/common/existing.py index 287d4e033..ec0f16e3c 100644 --- a/src/keri/app/cli/common/existing.py +++ b/src/keri/app/cli/common/existing.py @@ -12,7 +12,7 @@ from keri.app import habbing, keeping -def setupHby(name, base="", bran=None, cf=None): +def setupHby(name, base="", bran=None, cf=None, temp=False): """ Create Habery off of existing directory Parameters: @@ -20,6 +20,7 @@ def setupHby(name, base="", bran=None, cf=None): base(str): optional base directory prefix bran(str): optional passcode if the Habery was created encrypted cf (Configer): optional configuration for loading reference data + temp (bool): True means create database in /tmp Returns: Habery: the configured habery @@ -27,7 +28,7 @@ def setupHby(name, base="", bran=None, cf=None): """ ks = keeping.Keeper(name=name, base=base, - temp=False, + temp=temp, cf=cf, reopen=True) aeid = ks.gbls.get('aeid') diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 53059ffb5..eba307ea5 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -313,7 +313,6 @@ def setup(self, *, seed=None, aeid=None, bran=None, pidx=None, algo=None, self.loadHabs() self.inited = True - def loadHabs(self): """Load Habs instance from db diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 8fc1cc1b0..91b8c9f14 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -34,7 +34,7 @@ from hio.base import doing -from . import dbing, koming, subing +from . import dbing, koming, subing, migrating from .. import kering from ..core import coring, eventing, parsing, serdering @@ -931,6 +931,8 @@ def reopen(self, **kwa): self.ldes = self.env.open_db(key=b'ldes.', dupsort=True) self.qnfs = self.env.open_db(key=b'qnfs.', dupsort=True) + self.migs = subing.CesrSuber(db=self, subkey="migs.", klas=coring.Dater) + # event source local (protected) or non-local (remote not protected) self.esrs = koming.Komer(db=self, schema=EventSourceRecord, @@ -945,6 +947,7 @@ def reopen(self, **kwa): # events as ordered by first seen ordinals self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=coring.Seqner) # Kever state made of KeyStateRecord key states + # TODO: clean self.states = koming.Komer(db=self, schema=KeyStateRecord, subkey='stts.') @@ -977,6 +980,7 @@ def reopen(self, **kwa): # all reply messages. Maps reply said to serialization. Replys are # versioned sads ( with version string) so use Serder to deserialize and # use .sdts, .ssgs, and .scgs for datetimes and signatures + # TODO: clean self.rpys = subing.SerderSuber(db=self, subkey='rpys.') # all reply escrows indices of partially signed reply messages. Maps @@ -987,6 +991,7 @@ def reopen(self, **kwa): # auth AuthN/AuthZ by controller at cid of endpoint provider at eid # maps key=cid.role.eid to val=said of end reply + # TODO: clean self.eans = subing.CesrSuber(db=self, subkey='eans.', klas=coring.Saider) # auth AuthN/AuthZ by endpoint provider at eid of location at scheme url @@ -1005,6 +1010,7 @@ def reopen(self, **kwa): schema=LocationRecord, ) # index of last retrieved message from witness mailbox + # TODO: clean self.tops = koming.Komer(db=self, subkey='witm.', schema=TopicsRecord, ) @@ -1022,6 +1028,7 @@ def reopen(self, **kwa): klas=(coring.Seqner, coring.Saider)) # completed group multisig + # TODO: clean self.cgms = subing.CesrSuber(db=self, subkey='cgms.', klas=coring.Saider) @@ -1029,50 +1036,58 @@ def reopen(self, **kwa): self.epse = subing.SerderSuber(db=self, subkey="epse.") # exchange messages + # TODO: clean self.exns = subing.SerderSuber(db=self, subkey="exns.") # Forward pointer to a provided reply message + # TODO: clean self.erpy = subing.CesrSuber(db=self, subkey="erpy.", klas=coring.Saider) - # exchange messages - self.sxns = subing.SerderSuber(db=self, subkey="sxns.") - # exchange message signatures + # TODO: clean self.esigs = subing.CesrIoSetSuber(db=self, subkey='esigs.', klas=coring.Siger) # exchange message signatures + # TODO: clean self.ecigs = subing.CatCesrIoSetSuber(db=self, subkey='ecigs.', klas=(coring.Verfer, coring.Cigar)) # exchange pathed attachments + # TODO: clean self.epath = subing.IoSetSuber(db=self, subkey=".epath") # accepted signed 12-word challenge response exn messages keys by prefix of signer + # TODO: clean self.chas = subing.CesrIoSetSuber(db=self, subkey='chas.', klas=coring.Saider) # successfull signed 12-word challenge response exn messages keys by prefix of signer + # TODO: clean self.reps = subing.CesrIoSetSuber(db=self, subkey='reps.', klas=coring.Saider) # authorzied well known OOBIs + # TODO: clean self.wkas = koming.IoSetKomer(db=self, subkey='wkas.', schema=WellKnownAuthN) # KSN support datetime stamps and signatures indexed and not-indexed # all ksn kdts (key state datetime serializations) maps said to date-time + # TODO: clean self.kdts = subing.CesrSuber(db=self, subkey='kdts.', klas=coring.Dater) # all key state messages. Maps key state said to serialization. ksns are # KeyStateRecords so use ._asdict or ._asjson as appropriate # use .kdts, .ksgs, and .kcgs for datetimes and signatures + # TODO: clean self.ksns = koming.Komer(db=self, schema=KeyStateRecord, subkey='ksns.') - #self.ksns = subing.SerderSuber(db=self, subkey='ksns.') # key state SAID database for successfully saved key state notices # maps key=(prefix, aid) to val=said of key state + # TODO: clean self.knas = subing.CesrSuber(db=self, subkey='knas.', klas=coring.Saider) # config loaded oobis to be processed asynchronously, keyed by oobi URL + # TODO: clean self.oobis = koming.Komer(db=self, subkey='oobis.', schema=OobiRecord, @@ -1091,52 +1106,62 @@ def reopen(self, **kwa): sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Resolved OOBIs (those that have been processed successfully for this database. + # TODO: clean self.roobi = koming.Komer(db=self, subkey='roobi.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Well known OOBIs that are to be used for mfa against a resolved OOBI. + # TODO: clean self.woobi = koming.Komer(db=self, subkey='woobi.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Well known OOBIs that are to be used for mfa against a resolved OOBI. + # TODO: clean self.moobi = koming.Komer(db=self, subkey='moobi.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Multifactor well known OOBI auth records to process. Keys by controller URL + # TODO: clean self.mfa = koming.Komer(db=self, subkey='mfa.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Resolved multifactor well known OOBI auth records. Keys by controller URL + # TODO: clean self.rmfa = koming.Komer(db=self, subkey='rmfa.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # JSON schema SADs keys by the SAID + # TODO: clean self.schema = subing.SchemerSuber(db=self, subkey='schema.') # Field values for contact information for remote identifiers. Keyed by prefix/field + # TODO: clean self.cfld = subing.Suber(db=self, subkey="cfld.") # Global settings for the Habery environment self.hbys = subing.Suber(db=self, subkey='hbys.') # Signed contact data, keys by prefix + # TODO: clean self.cons = subing.Suber(db=self, subkey="cons.") # Transferable signatures on contact data + # TODO: clean self.ccigs = subing.CesrSuber(db=self, subkey='ccigs.', klas=coring.Cigar) # Chunked image data for contact information for remote identifiers + # TODO: clean self.imgs = self.env.open_db(key=b'imgs.') # Delegation escrow dbs # @@ -1146,26 +1171,17 @@ def reopen(self, **kwa): # delegated unanchored escrow self.dune = subing.SerderSuber(db=self, subkey='dune.') - # completed group multisig + # completed group delegated AIDs + # TODO: clean self.cdel = subing.CesrSuber(db=self, subkey='cdel.', klas=coring.Saider) - # siging public keys mapped to the AID and event seq no they appeared in so - # can look up member event designating as signing keys - # updated by Kever.logEvent. - self.pubs = subing.CatCesrIoSetSuber(db=self, subkey="pubs.", - klas=(coring.Prefixer, coring.Seqner)) - - # next key digests mapped to the AID and event seq no they appeared in so - # can look up event designating as next signing key digest - # updated by Kever.logEvent. - self.digs = subing.CatCesrIoSetSuber(db=self, subkey="digs.", - klas=(coring.Prefixer, coring.Seqner)) - # multisig sig embed payload SAID mapped to containing exn messages across group multisig participants + # TODO: clean self.meids = subing.CesrIoSetSuber(db=self, subkey="meids.", klas=coring.Saider) # multisig sig embed payload SAID mapped to group multisig participants AIDs + # TODO: clean self.maids = subing.CesrIoSetSuber(db=self, subkey="maids.", klas=coring.Prefixer) self.reload() @@ -1177,6 +1193,10 @@ def reload(self): Reload stored prefixes and Kevers from .habs """ + # Check migrations to see if this database is up to date. Error otherwise + if not migrating.Migrator(db=self).current(): + raise kering.DatabaseError("Database migrations must be run.") + removes = [] for keys, data in self.habs.getItemIter(): if (ksr := self.states.get(keys=data.hid)) is not None: @@ -1230,6 +1250,33 @@ def clean(self): for msg in self.cloneAllPreIter(): # clone into copy psr.parseOne(ims=msg) + # This is the list of non-set based databases that are not created as part of event processing. + # for now we are just copying them from self to copy without worrying about being able to + # reprocess them. We need a more secure method in the future + unsecured = ["hbys", "schema", "states", "rpys", "eans", "tops", "cgms", "exns", "erpy", + "kdts", "ksns", "knas", "oobis", "roobi", "woobi", "moobi", "mfa", "rmfa", + "cfld", "cons", "ccigs", "cdel", "migs"] + + for name in unsecured: + srcdb = getattr(self, name) + cpydb = getattr(copy, name) + for keys, val in srcdb.getItemIter(): + cpydb.put(keys=keys, val=val) + + # This is the list of set based databases that are not created as part of event processing. + # for now we are just copying them from self to copy without worrying about being able to + # reprocess them. We need a more secure method in the future + sets = ["esigs", "ecigs", "epath", "chas", "reps", "wkas", "meids", "maids"] + for name in sets: + srcdb = getattr(self, name) + cpydb = getattr(copy, name) + for keys, val in srcdb.getItemIter(): + cpydb.add(keys=keys, val=val) + + # Insecure raw imgs database copy. + for (key, val) in self.getAllItemIter(self.imgs): + copy.imgs.setVal(key=key, val=val) + # clone .habs habitat name prefix Komer subdb # copy.habs = koming.Komer(db=copy, schema=HabitatRecord, subkey='habs.') # copy for keys, val in self.habs.getItemIter(): @@ -1242,15 +1289,15 @@ def clean(self): copy.groups.add(val.hid) # clone .ends and .locs databases - for keys, val in self.ends.getItemIter(): + for (cid, role, eid), val in self.ends.getItemIter(): exists = False # only copy if entries in both .ends and .locs for scheme in ("https", "http", "tcp"): # all supported schemes - lval = self.locs.get(keys=(val.eid, scheme)) - if lval and lval.cid == keys[0] and lval.role == keys[1]: + lval = self.locs.get(keys=(eid, scheme)) + if lval: exists = True # loc with matching cid and rol - copy.locs.put(keys=(val.eid, scheme), val=lval) + copy.locs.put(keys=(eid, scheme), val=lval) if exists: # only copy end if has at least one matching loc - copy.ends.put(keys=keys, vals=[val]) + copy.ends.put(keys=(cid, role, eid), val=val) # remove own db directory replace with clean clone copy if os.path.exists(self.path): diff --git a/src/keri/db/migrating.py b/src/keri/db/migrating.py new file mode 100644 index 000000000..b2c173523 --- /dev/null +++ b/src/keri/db/migrating.py @@ -0,0 +1,45 @@ +import importlib +import sys + +from keri.core import coring + +MIGRATIONS = ["rekey_habs"] + + +class Migrator: + + def __init__(self, db): + self.db = db + + def migrate(self): + for migration in MIGRATIONS: + if self.db.migs.get(keys=(migration,)) is not None: + continue + + modName = f"keri.db.migrations.{migration}" + mod = importlib.import_module(modName) + try: + sys.stdout.write(f"Running migration {modName}... ") + mod.migrate(self.db) + print("done.") + except Exception as e: + print(f"Abandoning migratoin {migration} with error: {e}") + return + + self.db.migs.pin(keys=(migration,), val=coring.Dater()) + + def current(self): + return self.db.migs.get(MIGRATIONS[-1]) is not None + + def complete(self, name=None): + migrations = [] + if not name: + for mig in MIGRATIONS: + dater = self.db.migs.get(keys=(mig,)) + migrations.append((mig, dater)) + else: + if name not in MIGRATIONS: + raise ValueError(f"No migration named {name}") + migrations.append((name, self.db.migs.get(keys=(name,)))) + + return migrations diff --git a/src/keri/db/migrations/__init__.py b/src/keri/db/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/keri/db/migrations/rekey_habs.py b/src/keri/db/migrations/rekey_habs.py new file mode 100644 index 000000000..9275a25e2 --- /dev/null +++ b/src/keri/db/migrations/rekey_habs.py @@ -0,0 +1,64 @@ +from dataclasses import dataclass, field, asdict + +from keri.db import koming, basing + + +@dataclass +class OldHabitatRecord: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + hid (str): identifier prefix of hab qb64 + mid (str | None): group member identifier qb64 when hid is group + smids (list | None): group signing member identifiers qb64 when hid is group + rmids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + + + """ + hid: str # hab own identifier prefix qb64 + mid: str | None = None # group member identifier qb64 when hid is group + smids: list | None = None # group signing member ids when hid is group + rmids: list | None = None # group rotating member ids when hid is group + sid: str | None = None # Signify identifier qb64 when hid is Signify + watchers: list[str] = field(default_factory=list) # id prefixes qb64 of watchers + + +def migrate(db): + habs = koming.Komer(db=db, + subkey='habs.', + schema=OldHabitatRecord, ) + + # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix + nmsp = koming.Komer(db=db, + subkey='nmsp.', + schema=OldHabitatRecord, ) + + habords = dict() + for name, habord in habs.getItemIter(): + name = ".".join(name) # detupleize the database key name + nhabord = basing.HabitatRecord(**asdict(habord)) + nhabord.name = name + habords[habord.hid] = nhabord + + habs.trim() + + for keys, habord in nmsp.getItemIter(): + ns = keys[0] + name = ".".join(keys[1:]) # detupleize the database key name + nhabord = basing.HabitatRecord(**asdict(habord)) + nhabord.name = name + nhabord.domain = ns + habords[habord.hid] = nhabord + + nmsp.trim() + + for pre, habord in habords.items(): + print(pre) + print(habord) + db.habs.pin(keys=(pre,), val=habord) + ns = "" if habord.domain is None else habord.domain + print(ns) + print(habord.name) + db.names.pin(keys=(ns, habord.name), val=pre) From cf336d7bee2155ae916a35712905adb80c95838f Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Mon, 11 Mar 2024 07:29:58 -0700 Subject: [PATCH 06/12] Adding version table. Signed-off-by: pfeairheller --- src/keri/app/cli/commands/multisig/join.py | 3 --- src/keri/db/basing.py | 1 + src/keri/db/migrating.py | 8 ++++++-- src/keri/db/versioning.py | 24 ---------------------- 4 files changed, 7 insertions(+), 29 deletions(-) delete mode 100644 src/keri/db/versioning.py diff --git a/src/keri/app/cli/commands/multisig/join.py b/src/keri/app/cli/commands/multisig/join.py index dfca28c15..e52dae7a1 100644 --- a/src/keri/app/cli/commands/multisig/join.py +++ b/src/keri/app/cli/commands/multisig/join.py @@ -146,9 +146,6 @@ def incept(self, attrs): """ Incept group multisig """ - if True: - return True - smids = attrs["smids"] # change body mids for group member ids rmids = attrs["rmids"] if "rmids" in attrs else None ked = attrs["ked"] diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 91b8c9f14..862288741 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -932,6 +932,7 @@ def reopen(self, **kwa): self.qnfs = self.env.open_db(key=b'qnfs.', dupsort=True) self.migs = subing.CesrSuber(db=self, subkey="migs.", klas=coring.Dater) + self.vers = subing.Suber(db=self, subkey="vers.") # event source local (protected) or non-local (remote not protected) self.esrs = koming.Komer(db=self, diff --git a/src/keri/db/migrating.py b/src/keri/db/migrating.py index b2c173523..687252a49 100644 --- a/src/keri/db/migrating.py +++ b/src/keri/db/migrating.py @@ -3,7 +3,11 @@ from keri.core import coring -MIGRATIONS = ["rekey_habs"] +MIGRATIONS = [ + ("1", ["rekey_habs"]) +] + +VERSION = "1" class Migrator: @@ -23,7 +27,7 @@ def migrate(self): mod.migrate(self.db) print("done.") except Exception as e: - print(f"Abandoning migratoin {migration} with error: {e}") + print(f"\nAbandoning migratoin {migration} with error: {e}") return self.db.migs.pin(keys=(migration,), val=coring.Dater()) diff --git a/src/keri/db/versioning.py b/src/keri/db/versioning.py deleted file mode 100644 index d1e6b21c7..000000000 --- a/src/keri/db/versioning.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- encoding: utf-8 -*- -""" -keri.db.dbing module - - -""" -from keri.db import koming -from keri.db.basing import HabitatRecord - - -class Upgrader: - - def __init__(self, db): - self.db = db - - # habitat application state keyed by habitat name, includes prefix - self.habs = koming.Komer(db=self.db, - subkey='habs.', - schema=HabitatRecord, ) - - # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix - self.nmsp = koming.Komer(db=self.db, - subkey='nmsp.', - schema=HabitatRecord, ) From c8fa849cfc8ee718bf672071ecb18f2df948754a Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Fri, 15 Mar 2024 06:00:23 -0700 Subject: [PATCH 07/12] Creating a VERSION file for databases and tracking current version of software. Signed-off-by: pfeairheller --- src/keri/app/cli/commands/init.py | 1 + src/keri/db/basing.py | 2 +- src/keri/db/dbing.py | 12 ++++++++++++ src/keri/db/migrating.py | 10 +++++----- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/keri/app/cli/commands/init.py b/src/keri/app/cli/commands/init.py index aa8479ec6..f66f936fb 100644 --- a/src/keri/app/cli/commands/init.py +++ b/src/keri/app/cli/commands/init.py @@ -12,6 +12,7 @@ import keri.app.oobiing from keri.app import habbing, configing, oobiing, connecting from keri.app.keeping import Algos +from keri.db import migrating from keri.kering import ConfigurationError from keri.vdr import credentialing diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 862288741..5bc4b2ac1 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1195,7 +1195,7 @@ def reload(self): """ # Check migrations to see if this database is up to date. Error otherwise - if not migrating.Migrator(db=self).current(): + if not migrating.Migrator(db=self).current(self.version): raise kering.DatabaseError("Database migrations must be run.") removes = [] diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index f186a0b7d..1e9ae8fbb 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -53,12 +53,14 @@ from typing import Union import lmdb +from hio.help.helping import ocfn from ordered_set import OrderedSet as oset from hio.base import filing from hio.base import filing +import keri from ..kering import MaxON # maximum ordinal number for seqence or first seen from ..help import helping @@ -383,6 +385,16 @@ def reopen(self, readonly=False, **kwa): # creates files data.mdb and lock.mdb in .dbDirPath self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=104857600, mode=self.perm, readonly=self.readonly) + + fver = ocfn(path=f"{self.path}/VERSION", perm=self.perm) + + if ver := fver.read(): + self.version = ver + else: + self.version = keri.__version__ + fver.write(self.version) + fver.close() + self.opened = True if opened and self.env else False return self.opened diff --git a/src/keri/db/migrating.py b/src/keri/db/migrating.py index 687252a49..af9574f48 100644 --- a/src/keri/db/migrating.py +++ b/src/keri/db/migrating.py @@ -1,14 +1,13 @@ import importlib import sys +import keri from keri.core import coring MIGRATIONS = [ - ("1", ["rekey_habs"]) + ("1.1.0", ["rekey_habs"]) ] -VERSION = "1" - class Migrator: @@ -32,8 +31,9 @@ def migrate(self): self.db.migs.pin(keys=(migration,), val=coring.Dater()) - def current(self): - return self.db.migs.get(MIGRATIONS[-1]) is not None + def current(self, version): + return version == keri.__version__ + # return self.db.migs.get(MIGRATIONS[-1]) is not None def complete(self, name=None): migrations = [] From be6f0b3b74e4991906774e362a04acc0e9df3747 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Tue, 26 Mar 2024 12:37:54 -0700 Subject: [PATCH 08/12] getting lastest version from database. Signed-off-by: pfeairheller --- src/keri/db/dbing.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 1e9ae8fbb..9150d9570 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -348,6 +348,7 @@ def __init__(self, readonly=False, **kwa): """ self.env = None + self.version = None self.readonly = True if readonly else False super(LMDBer, self).__init__(**kwa) @@ -386,18 +387,29 @@ def reopen(self, readonly=False, **kwa): self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=104857600, mode=self.perm, readonly=self.readonly) - fver = ocfn(path=f"{self.path}/VERSION", perm=self.perm) - - if ver := fver.read(): - self.version = ver - else: + self.version = self.getVersion() + if self.version is None: self.version = keri.__version__ - fver.write(self.version) - fver.close() + self.setVersion(keri.__version__) self.opened = True if opened and self.env else False return self.opened + def getVersion(self): + with self.env.begin() as txn: + cursor = txn.cursor() + version = cursor.get(b'__version__') + return version.decode("utf-8") if version is not None else None + + def setVersion(self, val): + if hasattr(val, "encode"): + val = val.encode("utf-8") # convert str to bytes + + with self.env.begin(write=True) as txn: + cursor = txn.cursor() + version = cursor.replace(b'__version__', val) + return version + def close(self, clear=False): """ From 1b132892371583f7001771aabc79225211ef4448 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Tue, 2 Apr 2024 11:22:50 -0700 Subject: [PATCH 09/12] Getting migrations to work across all scenarios. Signed-off-by: pfeairheller --- src/keri/app/cli/commands/clean.py | 4 +- src/keri/app/cli/commands/init.py | 16 ++++- src/keri/app/cli/commands/migrate/list.py | 7 +- src/keri/app/cli/commands/migrate/run.py | 25 ++++--- src/keri/app/cli/commands/migrate/show.py | 4 +- src/keri/app/cli/commands/version.py | 23 +++++- src/keri/app/cli/common/existing.py | 3 +- src/keri/db/basing.py | 76 +++++++++++++++++++- src/keri/db/dbing.py | 88 ++++++++++++++++------- src/keri/db/migrating.py | 49 ------------- 10 files changed, 192 insertions(+), 103 deletions(-) delete mode 100644 src/keri/db/migrating.py diff --git a/src/keri/app/cli/commands/clean.py b/src/keri/app/cli/commands/clean.py index a7b6e1226..7afc752bd 100644 --- a/src/keri/app/cli/commands/clean.py +++ b/src/keri/app/cli/commands/clean.py @@ -9,7 +9,6 @@ from hio.base import doing from keri.app.cli.common import existing -from keri.db import migrating logger = help.ogler.getLogger() @@ -52,8 +51,7 @@ def recur(self, tyme): hby = existing.setupHby(name=self.args.name, base=self.args.base, bran=self.args.bran, temp=self.args.temp) print("Migrating...") - migrator = migrating.Migrator(db=hby.db) - migrator.migrate() + hby.db.migrate() print("Finished") hby = existing.setupHby(name=self.args.name, base=self.args.base, diff --git a/src/keri/app/cli/commands/init.py b/src/keri/app/cli/commands/init.py index f66f936fb..58e11d796 100644 --- a/src/keri/app/cli/commands/init.py +++ b/src/keri/app/cli/commands/init.py @@ -5,14 +5,17 @@ """ import argparse import getpass +import os +import sys from hio import help from hio.base import doing import keri.app.oobiing -from keri.app import habbing, configing, oobiing, connecting +from keri import kering +from keri.app import habbing, configing, oobiing from keri.app.keeping import Algos -from keri.db import migrating +from keri.db import basing from keri.kering import ConfigurationError from keri.vdr import credentialing @@ -93,6 +96,15 @@ def initialize(self, tymth, tock=0.0): else: break + db = basing.Baser(name=name, + base=base, + temp=temp, + reopen=False) + + if db.exists(name=name, base=base, temp=temp): + print("Database already exists, exiting") + sys.exit(-1) + kwa = dict() kwa["salt"] = args.salt kwa["bran"] = bran diff --git a/src/keri/app/cli/commands/migrate/list.py b/src/keri/app/cli/commands/migrate/list.py index ebe9722ae..198533fd9 100644 --- a/src/keri/app/cli/commands/migrate/list.py +++ b/src/keri/app/cli/commands/migrate/list.py @@ -10,7 +10,6 @@ from prettytable import PrettyTable from keri.app.cli.common import existing -from keri.db import migrating logger = help.ogler.getLogger() @@ -56,10 +55,10 @@ def recur(self, tyme): hby = existing.setupHby(name=self.args.name, base=self.args.base, bran=self.args.bran, temp=self.args.temp) - migrator = migrating.Migrator(db=hby.db) - for idx, (name, dater) in enumerate(migrator.complete()): + for idx, (name, dater) in enumerate(hby.db.complete()): + print(name, dater) date = dater.datetime.strftime("%Y-%m-%d %H:%M") if dater is not None else "Not Run" - tab.add_row((f"{idx+1}", f"{name}", date)) + tab.add_row((f"{idx + 1}", f"{name}", date)) print(tab) return True diff --git a/src/keri/app/cli/commands/migrate/run.py b/src/keri/app/cli/commands/migrate/run.py index f4ae6e33c..b45f4e9aa 100644 --- a/src/keri/app/cli/commands/migrate/run.py +++ b/src/keri/app/cli/commands/migrate/run.py @@ -5,11 +5,13 @@ """ import argparse +import keri from hio import help from hio.base import doing +from keri import kering from keri.app.cli.common import existing -from keri.db import migrating +from keri.db import basing logger = help.ogler.getLogger() @@ -21,7 +23,7 @@ def handler(args): Args: args(Namespace): arguments object from command line """ - clean = CleanDoer(args) + clean = MigrateDoer(args) return [clean] @@ -41,18 +43,25 @@ def handler(args): dest="bran", default=None) -class CleanDoer(doing.Doer): +class MigrateDoer(doing.Doer): def __init__(self, args): self.args = args - super(CleanDoer, self).__init__() + super(MigrateDoer, self).__init__() def recur(self, tyme): - hby = existing.setupHby(name=self.args.name, base=self.args.base, - bran=self.args.bran, temp=self.args.temp) + db = basing.Baser(name=self.args.name, + base=self.args.base, + temp=self.args.temp, + reopen=False) + + try: + db.reopen() + except kering.DatabaseError: + pass + print("Migrating...") - migrator = migrating.Migrator(db=hby.db) - migrator.migrate() + db.migrate() print("Finished") return True diff --git a/src/keri/app/cli/commands/migrate/show.py b/src/keri/app/cli/commands/migrate/show.py index 3ce45c2ca..b8b9c6060 100644 --- a/src/keri/app/cli/commands/migrate/show.py +++ b/src/keri/app/cli/commands/migrate/show.py @@ -9,7 +9,6 @@ from hio.base import doing from keri.app.cli.common import existing -from keri.db import migrating logger = help.ogler.getLogger() @@ -53,8 +52,7 @@ def recur(self, tyme): hby = existing.setupHby(name=self.args.name, base=self.args.base, bran=self.args.bran, temp=self.args.temp) - migrator = migrating.Migrator(db=hby.db) - [(name, dater)] = migrator.complete(name=self.args.migration) + [(name, dater)] = hby.db.complete(name=self.args.migration) date = dater.datetime.strftime("%Y-%m-%d %H:%M") if dater is not None else "Not Run" print(f"{self.args.migration} -> {date}") diff --git a/src/keri/app/cli/commands/version.py b/src/keri/app/cli/commands/version.py index 5ea6b775a..98cc81c5a 100644 --- a/src/keri/app/cli/commands/version.py +++ b/src/keri/app/cli/commands/version.py @@ -8,18 +8,35 @@ from hio.base import doing import keri +from keri.app.cli.common import existing parser = argparse.ArgumentParser(description='Print version of KLI') parser.set_defaults(handler=lambda args: handler(args)) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=False, + default=None) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran def handler(args): - return [doing.doify(version)] + kwa = dict(args=args) + return [doing.doify(version, **kwa)] -def version(tymth, tock=0.0): +def version(tymth, tock=0.0, **opts): """ Command line version handler """ _ = (yield tock) - print(keri.__version__) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + + print(f"Library version: {keri.__version__}") + + if name is not None: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + print(f"Database version: {hby.db.version}") diff --git a/src/keri/app/cli/common/existing.py b/src/keri/app/cli/common/existing.py index ec0f16e3c..dd730df95 100644 --- a/src/keri/app/cli/common/existing.py +++ b/src/keri/app/cli/common/existing.py @@ -47,7 +47,8 @@ def setupHby(name, base="", bran=None, cf=None, temp=False): retries += 1 hby = habbing.Habery(name=name, base=base, bran=bran, cf=cf, free=True) break - except (kering.AuthError, ValueError): + except (kering.AuthError, ValueError) as e: + print(e) if retries >= 3: raise kering.AuthError("too many attempts") print("Valid passcode required, try again...") diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 5bc4b2ac1..4a725fb7b 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -18,7 +18,7 @@ So only need to set dupsort first time opened each other opening does not need to call it """ - +import importlib import os import shutil from collections import namedtuple @@ -30,11 +30,13 @@ import cbor2 as cbor import msgpack import lmdb +import semver from ordered_set import OrderedSet as oset from hio.base import doing -from . import dbing, koming, subing, migrating +import keri +from . import dbing, koming, subing from .. import kering from ..core import coring, eventing, parsing, serdering @@ -46,6 +48,10 @@ logger = help.ogler.getLogger() +MIGRATIONS = [ + ("1.1.0", ["rekey_habs"]) +] + # ToDo XXXX maybe ''' @@ -1195,7 +1201,7 @@ def reload(self): """ # Check migrations to see if this database is up to date. Error otherwise - if not migrating.Migrator(db=self).current(self.version): + if not self.current: raise kering.DatabaseError("Database migrations must be run.") removes = [] @@ -1219,6 +1225,70 @@ def reload(self): for keys in removes: # remove bare .habs records self.habs.rem(keys=keys) + def migrate(self): + for (version, migrations) in MIGRATIONS: + # Check to see if this is for an older version + if self.version is not None and semver.compare(version, self.version) != 1: + continue + + for migration in migrations: + modName = f"keri.db.migrations.{migration}" + if self.migs.get(keys=(migration,)) is not None: + continue + + mod = importlib.import_module(modName) + try: + print(f"running migration {modName}") + mod.migrate(self) + except Exception as e: + print(f"\nAbandoning migration {migration} with error: {e}") + return + + self.migs.pin(keys=(migration,), val=coring.Dater()) + + self.version = keri.__version__ + + @property + def current(self): + """ Current property determines if we are at the current database migration state. + + If the database version matches the library version return True + If the current database version is behind the current library version, check for migrations + - If there are migrations to run, return False + - If there are no migrations to run, reset database version to library version and return True + If the current database version is ahead of the current library version, raise exception + + """ + if self.version == keri.__version__: + return True + + # If database version is ahead of library version, throw exception + if self.version is not None and semver.compare(self.version, keri.__version__) == 1: + raise kering.ConfigurationError( + f"Database version={self.version} is ahead of library version={keri.__version__}") + + last = MIGRATIONS[-1] + # If we aren't at latest version, but there are no outstanding migrations, reset version to latest + if self.migs.get(keys=(last[1][0],)) is not None: + return True + + # We have migrations to run + return False + + def complete(self, name=None): + migrations = [] + if not name: + for version, migs in MIGRATIONS: + for mig in migs: + dater = self.migs.get(keys=(mig,)) + migrations.append((mig, dater)) + else: + if name not in MIGRATIONS: + raise ValueError(f"No migration named {name}") + migrations.append((name, self.migs.get(keys=(name,)))) + + return migrations + def clean(self): """ Clean database by creating re-verified cleaned cloned copy diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 9150d9570..a0246f38e 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -53,19 +53,13 @@ from typing import Union import lmdb -from hio.help.helping import ocfn -from ordered_set import OrderedSet as oset - -from hio.base import filing - +from ordered_set import OrderedSet as oset from hio.base import filing import keri from ..kering import MaxON # maximum ordinal number for seqence or first seen - from ..help import helping -#MaxON = int("f"*32, 16) # largest possible ordinal number, sequence or first seen ProemSize = 32 # does not include trailing separator MaxProem = int("f"*(ProemSize), 16) SuffixSize = 32 # does not include trailing separator @@ -310,7 +304,6 @@ class LMDBer(filing.Filer): Perm = stat.S_ISVTX | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR # 0o1700==960 MaxNamedDBs = 96 - def __init__(self, readonly=False, **kwa): """ Setup main database directory at .dirpath. @@ -347,12 +340,12 @@ def __init__(self, readonly=False, **kwa): False means open database in read/write mode """ + self.env = None - self.version = None + self._version = None self.readonly = True if readonly else False super(LMDBer, self).__init__(**kwa) - def reopen(self, readonly=False, **kwa): """ Open if closed or close and reopen if opened or create and open if not @@ -378,6 +371,7 @@ def reopen(self, readonly=False, **kwa): readonly (bool): True means open database in readonly mode False means open database in read/write mode """ + exists = self.exists(name=self.name, base=self.base, temp=self.temp) opened = super(LMDBer, self).reopen(**kwa) if readonly is not None: self.readonly = readonly @@ -387,29 +381,56 @@ def reopen(self, readonly=False, **kwa): self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=104857600, mode=self.perm, readonly=self.readonly) - self.version = self.getVersion() - if self.version is None: + self.opened = True if opened and self.env else False + + if self.opened and not self.readonly and not exists: self.version = keri.__version__ - self.setVersion(keri.__version__) - self.opened = True if opened and self.env else False return self.opened - def getVersion(self): - with self.env.begin() as txn: - cursor = txn.cursor() - version = cursor.get(b'__version__') - return version.decode("utf-8") if version is not None else None + def exists(self, name="", base="", temp=None, headDirPath=None, clean=False, filed=False, fext=None): + temp = True if temp else False - def setVersion(self, val): - if hasattr(val, "encode"): - val = val.encode("utf-8") # convert str to bytes + if temp: + return False - with self.env.begin(write=True) as txn: - cursor = txn.cursor() - version = cursor.replace(b'__version__', val) - return version + # use class defaults here so can use makePath for other dirs and files + if headDirPath is None: + headDirPath = self.HeadDirPath + if fext is None: + fext = self.Fext + + tailDirPath = self.CleanTailDirPath if clean else self.TailDirPath + + if filed: + root, ext = os.path.splitext(name) + if not ext: + name = f"{name}.{fext}" + + path = os.path.abspath( + os.path.expanduser( + os.path.join(headDirPath, + tailDirPath, + base, + name))) + + return os.path.exists(path) + + @property + def version(self): + if self._version is None: + self._version = self.getVer() + + return self._version + + @version.setter + def version(self, val): + if hasattr(val, "decode"): + val = val.decode("utf-8") # convert bytes to str + + self._version = val + self.setVer(self._version) def close(self, clear=False): """ @@ -425,8 +446,21 @@ def close(self, clear=False): self.env = None - return(super(LMDBer, self).close(clear=clear)) + return super(LMDBer, self).close(clear=clear) + + def getVer(self): + with self.env.begin() as txn: + cursor = txn.cursor() + version = cursor.get(b'__version__') + return version.decode("utf-8") if version is not None else None + def setVer(self, val): + if hasattr(val, "encode"): + val = val.encode("utf-8") # convert str to bytes + + with self.env.begin(write=True) as txn: + cursor = txn.cursor() + cursor.replace(b'__version__', val) # For subdbs with no duplicate values allowed at each key. (dupsort==False) def putVal(self, db, key, val): diff --git a/src/keri/db/migrating.py b/src/keri/db/migrating.py deleted file mode 100644 index af9574f48..000000000 --- a/src/keri/db/migrating.py +++ /dev/null @@ -1,49 +0,0 @@ -import importlib -import sys - -import keri -from keri.core import coring - -MIGRATIONS = [ - ("1.1.0", ["rekey_habs"]) -] - - -class Migrator: - - def __init__(self, db): - self.db = db - - def migrate(self): - for migration in MIGRATIONS: - if self.db.migs.get(keys=(migration,)) is not None: - continue - - modName = f"keri.db.migrations.{migration}" - mod = importlib.import_module(modName) - try: - sys.stdout.write(f"Running migration {modName}... ") - mod.migrate(self.db) - print("done.") - except Exception as e: - print(f"\nAbandoning migratoin {migration} with error: {e}") - return - - self.db.migs.pin(keys=(migration,), val=coring.Dater()) - - def current(self, version): - return version == keri.__version__ - # return self.db.migs.get(MIGRATIONS[-1]) is not None - - def complete(self, name=None): - migrations = [] - if not name: - for mig in MIGRATIONS: - dater = self.db.migs.get(keys=(mig,)) - migrations.append((mig, dater)) - else: - if name not in MIGRATIONS: - raise ValueError(f"No migration named {name}") - migrations.append((name, self.db.migs.get(keys=(name,)))) - - return migrations From f35264789450a3c0358f9f00a24ef3370d871b59 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Wed, 3 Apr 2024 10:37:51 -0700 Subject: [PATCH 10/12] Update to latest hio, python and other out of date deps. Signed-off-by: pfeairheller --- setup.py | 9 +++++---- src/keri/app/cli/commands/init.py | 9 --------- src/keri/db/dbing.py | 33 ++----------------------------- 3 files changed, 7 insertions(+), 44 deletions(-) diff --git a/setup.py b/setup.py index c38653b49..ea3f57799 100644 --- a/setup.py +++ b/setup.py @@ -68,16 +68,16 @@ keywords=[ # eg: 'keyword1', 'keyword2', 'keyword3', ], - python_requires='>=3.12.1', + python_requires='>=3.12.2', install_requires=[ 'lmdb>=1.3.0', 'pysodium>=0.7.12', 'blake3>=0.3.1', 'msgpack>=1.0.4', - 'cbor2>=5.4.3', + 'cbor2>=5.6.2', 'multidict>=6.0.2', 'ordered-set>=4.1.0', - 'hio>=0.6.9', + 'hio>=0.6.11', 'multicommand>=1.0.0', 'jsonschema>=4.17.0', 'falcon>=3.1.0', @@ -87,7 +87,8 @@ 'mnemonic>=0.20', 'PrettyTable>=3.5.0', 'http_sfv>=0.9.8', - 'cryptography>=39.0.2' + 'cryptography>=42.0.5', + 'semver>=3.0.2' ], extras_require={ }, diff --git a/src/keri/app/cli/commands/init.py b/src/keri/app/cli/commands/init.py index 58e11d796..a4eedbefd 100644 --- a/src/keri/app/cli/commands/init.py +++ b/src/keri/app/cli/commands/init.py @@ -96,15 +96,6 @@ def initialize(self, tymth, tock=0.0): else: break - db = basing.Baser(name=name, - base=base, - temp=temp, - reopen=False) - - if db.exists(name=name, base=base, temp=temp): - print("Database already exists, exiting") - sys.exit(-1) - kwa = dict() kwa["salt"] = args.salt kwa["bran"] = bran diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index a0246f38e..2038bde7f 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -371,7 +371,7 @@ def reopen(self, readonly=False, **kwa): readonly (bool): True means open database in readonly mode False means open database in read/write mode """ - exists = self.exists(name=self.name, base=self.base, temp=self.temp) + exists = self.exists(name=self.name, base=self.base) opened = super(LMDBer, self).reopen(**kwa) if readonly is not None: self.readonly = readonly @@ -383,40 +383,11 @@ def reopen(self, readonly=False, **kwa): self.opened = True if opened and self.env else False - if self.opened and not self.readonly and not exists: + if self.opened and not self.readonly and (not exists or self.temp): self.version = keri.__version__ return self.opened - def exists(self, name="", base="", temp=None, headDirPath=None, clean=False, filed=False, fext=None): - temp = True if temp else False - - if temp: - return False - - # use class defaults here so can use makePath for other dirs and files - if headDirPath is None: - headDirPath = self.HeadDirPath - - if fext is None: - fext = self.Fext - - tailDirPath = self.CleanTailDirPath if clean else self.TailDirPath - - if filed: - root, ext = os.path.splitext(name) - if not ext: - name = f"{name}.{fext}" - - path = os.path.abspath( - os.path.expanduser( - os.path.join(headDirPath, - tailDirPath, - base, - name))) - - return os.path.exists(path) - @property def version(self): if self._version is None: From 333084d73e0719c3bd6ab34d2b7230344100d454 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Wed, 3 Apr 2024 15:08:32 -0700 Subject: [PATCH 11/12] Updated doc strings Signed-off-by: pfeairheller --- src/keri/db/basing.py | 20 +++++++++++++++++++- src/keri/db/dbing.py | 26 ++++++++++++++++++++++++++ src/keri/db/migrations/rekey_habs.py | 21 ++++++++++++++++----- 3 files changed, 61 insertions(+), 6 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 4a725fb7b..7b324194c 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1226,6 +1226,15 @@ def reload(self): self.habs.rem(keys=keys) def migrate(self): + """ Run all migrations required + + Run all migrations that are required from the current version of database up to the current version + of the software that have not already been run. + + Sets the version of the database to the current version of the software after successful completion + of required migrations + + """ for (version, migrations) in MIGRATIONS: # Check to see if this is for an older version if self.version is not None and semver.compare(version, self.version) != 1: @@ -1276,6 +1285,15 @@ def current(self): return False def complete(self, name=None): + """ Returns list of tuples of migrations completed with date of completion + + Parameters: + name(str): optional name of migration to check completeness + + Returns: + list: tuples of migration,date of completed migration names and the date of completion + + """ migrations = [] if not name: for version, migs in MIGRATIONS: @@ -1283,7 +1301,7 @@ def complete(self, name=None): dater = self.migs.get(keys=(mig,)) migrations.append((mig, dater)) else: - if name not in MIGRATIONS: + if name not in MIGRATIONS or not self.migs.get(keys=(name,)): raise ValueError(f"No migration named {name}") migrations.append((name, self.migs.get(keys=(name,)))) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 2038bde7f..ddb0ce4e0 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -390,6 +390,14 @@ def reopen(self, readonly=False, **kwa): @property def version(self): + """ Return the version of database stored in __version__ key. + + This value is read through cached in memory + + Returns: + str: the version of the database or None if not set in the database + + """ if self._version is None: self._version = self.getVer() @@ -397,6 +405,12 @@ def version(self): @version.setter def version(self, val): + """ Set the version of the database in memory and in the __version__ key + + Parameters: + val (str): The new semver formatted version of the database + + """ if hasattr(val, "decode"): val = val.decode("utf-8") # convert bytes to str @@ -420,12 +434,24 @@ def close(self, clear=False): return super(LMDBer, self).close(clear=clear) def getVer(self): + """ Returns the value of the the semver formatted version in the __version__ key in this database + + Returns: + str: semver formatted version of the database + + """ with self.env.begin() as txn: cursor = txn.cursor() version = cursor.get(b'__version__') return version.decode("utf-8") if version is not None else None def setVer(self, val): + """ Set the version of the database in the __version__ key + + Parameters: + val (str): The new semver formatted version of the database + + """ if hasattr(val, "encode"): val = val.encode("utf-8") # convert str to bytes diff --git a/src/keri/db/migrations/rekey_habs.py b/src/keri/db/migrations/rekey_habs.py index 9275a25e2..e21dfdb9f 100644 --- a/src/keri/db/migrations/rekey_habs.py +++ b/src/keri/db/migrations/rekey_habs.py @@ -26,6 +26,18 @@ class OldHabitatRecord: # baser.habs def migrate(db): + """ Re-key habs migration for changing the key for .habs and introducing the .names database + + This migrations performs the following: + 1. Rekey .habs from name (alias) to the AID of the Hab + 2. Add Name and domain to the HabitatRecord for all Habs + 3. Populate the .names index as (ns, name) -> AID + 4. Remove the .nmsp namespaced Habs database (replaced within .habs and .names now) + + Parameters: + db(Baser): Baser database object on which to run the migration + + """ habs = koming.Komer(db=db, subkey='habs.', schema=OldHabitatRecord, ) @@ -36,6 +48,7 @@ def migrate(db): schema=OldHabitatRecord, ) habords = dict() + # Update Hab records from .habs with name for name, habord in habs.getItemIter(): name = ".".join(name) # detupleize the database key name nhabord = basing.HabitatRecord(**asdict(habord)) @@ -44,6 +57,7 @@ def migrate(db): habs.trim() + # Update Hab records from .nmsp with name and domain (ns) for keys, habord in nmsp.getItemIter(): ns = keys[0] name = ".".join(keys[1:]) # detupleize the database key name @@ -52,13 +66,10 @@ def migrate(db): nhabord.domain = ns habords[habord.hid] = nhabord - nmsp.trim() + nmsp.trim() # remove existing records + # Rekey .habs and create .names index for pre, habord in habords.items(): - print(pre) - print(habord) db.habs.pin(keys=(pre,), val=habord) ns = "" if habord.domain is None else habord.domain - print(ns) - print(habord.name) db.names.pin(keys=(ns, habord.name), val=pre) From bfc3ca76cdd313eaa8acabbc688f0bf044999e73 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Wed, 3 Apr 2024 15:18:14 -0700 Subject: [PATCH 12/12] Update docker version in workflow. Signed-off-by: pfeairheller --- .github/workflows/python-app-ci.yml | 12 ++++++------ scripts/demo/basic/multisig.sh | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python-app-ci.yml b/.github/workflows/python-app-ci.yml index 98751e757..1534a51c7 100644 --- a/.github/workflows/python-app-ci.yml +++ b/.github/workflows/python-app-ci.yml @@ -21,10 +21,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.12.1 + - name: Set up Python 3.12.2 uses: actions/setup-python@v2 with: - python-version: 3.12.1 + python-version: 3.12.2 - name: Install dependencies run: | python -m pip install --upgrade pip @@ -47,10 +47,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.12.1 + - name: Set up Python 3.12.2 uses: actions/setup-python@v2 with: - python-version: 3.12.1 + python-version: 3.12.2 - name: Install dependencies run: | python -m pip install --upgrade pip @@ -68,10 +68,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.12.1 + - name: Set up Python 3.12.2 uses: actions/setup-python@v2 with: - python-version: 3.12.1 + python-version: 3.12.2 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/scripts/demo/basic/multisig.sh b/scripts/demo/basic/multisig.sh index e6c6a8000..78006672d 100755 --- a/scripts/demo/basic/multisig.sh +++ b/scripts/demo/basic/multisig.sh @@ -6,11 +6,11 @@ kli init --name multisig1 --base "${KERI_TEMP_DIR}" --salt 0ACDEyMzQ1Njc4OWxtbm9aBc --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis kli incept --name multisig1 --base "${KERI_TEMP_DIR}" --alias multisig1 --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-1-sample.json -kli ends add --name multisig1 --base "${KERI_TEMP_DIR}" --alias multisig1 --eid BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM --role mailbox +#kli ends add --name multisig1 --base "${KERI_TEMP_DIR}" --alias multisig1 --eid BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM --role mailbox kli init --name multisig2 --base "${KERI_TEMP_DIR}" --salt 0ACDEyMzQ1Njc4OWdoaWpsaw --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis kli incept --name multisig2 --base "${KERI_TEMP_DIR}" --alias multisig2 --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-2-sample.json -kli ends add --name multisig2 --base "${KERI_TEMP_DIR}" --alias multisig2 --eid BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX --role mailbox +#kli ends add --name multisig2 --base "${KERI_TEMP_DIR}" --alias multisig2 --eid BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX --role mailbox kli oobi resolve --name multisig1 --base "${KERI_TEMP_DIR}" --oobi-alias multisig2 --oobi http://127.0.0.1:5642/oobi/EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 kli oobi resolve --name multisig2 --base "${KERI_TEMP_DIR}" --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4