Make cortex, view, and layer idens to be unique

synapse/cortex.py

@@ -765,6 +765,8 @@ async def __anit__(self, dirn, conf=None):
         self.axon = None  # type: s_axon.AxonApi
         self.axready = asyncio.Event()
 
+        self.view = None  # The default/main view
+
         # generic fini handler for the Cortex
         self.onfini(self._onCoreFini)
 
@@ -787,12 +789,12 @@ async def __anit__(self, dirn, conf=None):
 
         # Initialize our storage and views
         await self._initCoreAxon()
+
+        await self._migrateViewsLayers()
         await self._initCoreLayers()
-        await self._checkLayerModels()
         await self._initCoreViews()
+        await self._checkLayerModels()
         await self._initCoreQueues()
-        # our "main" view has the same iden as we do
-        self.view = self.views.get(self.iden)
 
         self.provstor = await s_provenance.ProvStor.anit(self.dirn)
         self.onfini(self.provstor.fini)
@@ -1625,13 +1627,93 @@ def addLayerCtor(self, name, ctor):
 
     async def _initCoreViews(self):
 
+        defiden = self.cellinfo.get('defaultview')
+
         for iden, node in await self.hive.open(('cortex', 'views')):
             view = await s_view.View.anit(self, node)
             self.views[iden] = view
+            if iden == defiden:
+                self.view = view
+
+        # if we have no views, we are initializing.  Add a default main view and layer.
+        if not self.views:
+            layr = await self.addLayer()
+            iden = s_common.guid()
+            view = await self.addView(iden, 'root', (layr.iden,))
+            await self.cellinfo.set('defaultview', iden)
+            self.view = view
+
+    async def _moveDict(self, oldpath, newpath):
+        '''
+        Move a dict-only hive node from old path to new path, doing nothing if old path doesn't exist
+
+        Returns:
+            True if the old path was present
+        '''
+        if not await self.hive.exists(oldpath):
+            return False
+
+        oldnode = await self.hive.open(oldpath)
+        oldinfo = await oldnode.dict()
+
+        newnode = await self.hive.open(newpath)
+        newinfo = await newnode.dict()
+
+        for name, valu in oldinfo.items():
+            await newinfo.set(name, valu)
+
+        await oldnode.pop(())
+        return True
+
+    async def _migrateViewsLayers(self):
+        '''
+        Move directories and idens to current scheme where cortex, views, and layers all have unique idens
+
+        Note that this changes directories and hive data, not existing View or Layer objects
+        '''
+        # pre-hive -> hive layer directory migration first
+        self._migrOrigLayer()
+
+        defiden = self.cellinfo.get('defaultview')
+        if defiden is not None:
+            # No need for migration; we're up-to-date
+            return
+
+        oldlayriden = self.iden
+        newlayriden = s_common.guid()
+
+        oldviewiden = self.iden
+        newviewiden = s_common.guid()
+
+        moved = await self._moveDict(('cortex', 'views', oldviewiden), ('cortex', 'views', newviewiden))
+        if moved:
+            logger.info('Migrated view from duplicate iden %s to new iden %s', oldviewiden, newviewiden)
+        else:
+            # No view info present; this is a fresh cortex
+            return
+
+        # Move view/layer metadata
+        moved = await self._moveDict(('cortex', 'layers', oldlayriden), ('cortex', 'layers', newlayriden))
+        if moved:
+            logger.info('Migrated layer from duplicate iden %s to new iden %s', oldlayriden, newlayriden)
 
-        # if we have no views, we are initializing.  add the main view.
-        if self.views.get(self.iden) is None:
-            await self.addView(self.iden, 'root', (self.iden,))
+        # Move layer data
+        oldpath = os.path.join(self.dirn, 'layers', oldlayriden)
+        newpath = os.path.join(self.dirn, 'layers', newlayriden)
+        try:
+            os.rename(oldpath, newpath)
+        except OSError:
+            pass
+
+        # Replace all views' references to old layer iden with new layer iden
+        node = await self.hive.open(('cortex', 'views'))
+        for iden, viewnode in node:
+            info = await viewnode.dict()
+            layers = info.get('layers')
+            newlayers = [newlayriden if layr == oldlayriden else layr for layr in layers]
+            await info.set('layers', newlayers)
+
+        await self.cellinfo.set('defaultview', newviewiden)
 
     async def addView(self, iden, owner, layers):
 
@@ -1649,22 +1731,20 @@ async def addView(self, iden, owner, layers):
     async def delView(self, iden):
         '''
         Delete a cortex view by iden.
+
+        Note:
+            This does not delete any of the view's layers
         '''
-        if iden == self.iden:
+        if iden == self.view.iden:
             raise s_exc.SynErr(mesg='cannot delete the main view')
 
         view = self.views.pop(iden, None)
         if view is None:
             raise s_exc.NoSuchView(iden=iden)
 
-        layeriden = view.iden if view.parent is not None and view.layers[0].iden == view.iden else None
-
         await self.hive.pop(('cortex', 'views', iden))
         await view.fini()
 
-        if layeriden is not None:
-            await self.delLayer(iden)
-
     async def delLayer(self, iden):
         layr = self.layers.get(iden, None)
         if layr is None:
@@ -1687,18 +1767,25 @@ async def setViewLayers(self, layers, iden=None):
             layers ([str]): A top-down list of of layer guids
             iden (str): The view iden (defaults to default view).
         '''
-        if iden is None:
-            iden = self.iden
-
-        view = self.views.get(iden)
+        view = self.getView(iden)
         if view is None:
             raise s_exc.NoSuchView(iden=iden)
 
         await view.setLayers(layers)
 
     def getLayer(self, iden=None):
+        '''
+        Get a Layer object.
+
+        Args:
+            iden (str): The layer iden to retrieve.
+
+        Returns:
+            Layer: A Layer object.
+        '''
         if iden is None:
-            iden = self.iden
+            return self.view.layers[0]
+
         return self.layers.get(iden)
 
     def getView(self, iden=None):
@@ -1712,17 +1799,18 @@ def getView(self, iden=None):
             View: A View object.
         '''
         if iden is None:
-            iden = self.iden
+            return self.view
+
         return self.views.get(iden)
 
     async def addLayer(self, **info):
         '''
         Add a Layer to the cortex.
 
         Args:
-            iden (str): optional iden. default: guid() )
-            type (str): optional type. default: lmdb )
-            owner (str): optional owner. default: root )
+            iden (str): optional iden. default: guid()
+            type (str): optional type. default: lmdb
+            owner (str): optional owner. default: root
             config (dict): type specific config options
         '''
         iden = info.pop('iden', None)
@@ -1783,12 +1871,6 @@ async def _initCoreLayers(self):
         for iden, node in node:
             await self._layrFromNode(node)
 
-        self._migrOrigLayer()
-
-        if self.layers.get(self.iden) is None:
-            # we have no layers.  initialize the default layer.
-            await self.addLayer(iden=self.iden)
-
     def _migrOrigLayer(self):
 
         oldpath = os.path.join(self.dirn, 'layers', '000-default')
@@ -2296,12 +2378,9 @@ def _viewFromOpts(self, opts):
             return self.view
 
         viewiden = opts.get('view')
-        if viewiden is None:
-            return self.view
-        else:
-            view = self.views.get(viewiden)
-            if view is None:
-                raise s_exc.NoSuchView(iden=viewiden)
+        view = self.getView(viewiden)
+        if view is None:
+            raise s_exc.NoSuchView(iden=viewiden)
 
         return view
 

synapse/lib/cell.py

@@ -446,6 +446,11 @@ async def __anit__(self, dirn, conf=None, readonly=False):
 
         await self._initCellHttp()
 
+        # self.cellinfo, a HiveDict for general purpose persistent storage
+        node = await self.hive.open(('cellinfo', ))
+        self.cellinfo = await node.dict()
+        self.onfini(node)
+
         self._health_funcs = []
         self.addHealthFunc(self._cellHealth)
 

synapse/lib/hive.py

@@ -220,6 +220,9 @@ async def get(self, full):
 
         return node.valu
 
+    async def exists(self, full):
+        return full in self.nodes
+
     def dir(self, full):
         '''
         List subnodes of the given Hive path.
@@ -310,9 +313,9 @@ async def _getHiveNode(self, full):
             step = node.kids.get(name)
             if step is None:
                 step = await self._initNodePath(node, path, None)
-                #print('STEP: %r %r' % (path, step))
+                # print('STEP: %r %r' % (path, step))
                 # hive add events alert the *parent* path of edits
-                #await node.fire('hive:add', path=path[:-1], name=name, valu=None)
+                # await node.fire('hive:add', path=path[:-1], name=name, valu=None)
 
             node = step
 

synapse/lib/view.py

@@ -256,7 +256,7 @@ async def fork(self, **layrinfo):
         '''
         writlayr = await self.core.addLayer(**layrinfo)
 
-        viewiden = writlayr.iden
+        viewiden = s_common.guid()
         owner = layrinfo.get('owner', 'root')
         layeridens = [writlayr.iden] + [l.iden for l in self.layers]
 

synapse/tests/test_cortex.py

@@ -3380,22 +3380,22 @@ async def test_view_setlayers(self):
             async with self.getTestCore(dirn=path00) as core00:
                 self.len(1, await core00.eval('[ test:str=core00 ]').list())
 
-                iden00 = core00.getCellIden()
+                iden00 = core00.getLayer().iden
 
             async with self.getTestCore(dirn=path01) as core01:
 
                 self.len(1, await core01.eval('[ test:str=core01 ]').list())
                 # Add a lmdb layer with core00's iden
                 await core01.addLayer(iden=iden00)
-                iden01 = core01.getCellIden()
+                iden01 = core01.getLayer().iden
                 # Set the default view for core01 to have a read layer with
                 # the iden from core00.
                 await core01.setViewLayers((iden01, iden00))
 
-            src = s_common.gendir(path00, 'layers', iden00)
-            dst = s_common.gendir(path01, 'layers', iden00)
             # Blow away the old layer at the destination and replace it
             # with our layer from core00
+            src = s_common.gendir(path00, 'layers', iden00)
+            dst = s_common.gendir(path01, 'layers', iden00)
             shutil.rmtree(dst)
             shutil.copytree(src, dst)
 
@@ -3411,10 +3411,9 @@ async def test_layers_missing_ctor(self):
                 nodes = await core.nodes('[test:str=woot]')
                 self.len(1, nodes)
 
-                cell_iden = core.getCellIden()
                 # Add the layer to the cortex and insert it into the default view stack
                 await core.addLayer(iden=iden)
-                await core.setViewLayers((cell_iden, iden))
+                await core.setViewLayers([layr.iden for layr in core.view.layers] + [iden])
 
                 # Modify the layer type
                 await core.hive.set(('cortex', 'layers', iden, 'type'), 'newp')
@@ -3742,15 +3741,11 @@ async def test_cortex_delLayerView(self):
             view2 = await core.view.fork()
 
             viewiden = view2.iden
-            layriden = view2.layers[0].iden
 
             # Can't delete a view twice
             await core.delView(viewiden)
             await self.asyncraises(s_exc.NoSuchView, core.delView(viewiden))
 
-            # A forked view deletes its write layer
-            await self.asyncraises(s_exc.NoSuchLayer, core.delLayer(layriden))
-
     async def test_cortex_cronjob_perms(self):
         async with self.getTestCore() as realcore:
             async with realcore.getLocalProxy() as core:

synapse/tests/test_lib_remotelayer.py

@@ -143,7 +143,7 @@ async def test_cortex_remote_config(self):
             rem1 = await core0.auth.addUser('remuser1')
 
             await rem1.setPasswd('beep')
-            await rem1.addRule((True, ('layer:lift', core0.iden)))
+            await rem1.addRule((True, ('layer:lift', core0.getLayer().iden)))
 
             # make a test:str node
             nodes = await core0.eval('[test:str=woot]').list()

synapse/tests/test_tools_backup.py

@@ -37,6 +37,7 @@ def compare_dirs(self, dir1, dir2, skipfns=None):
     async def test_backup(self):
 
         async with self.getTestCore() as core:
+            layriden = core.getLayer().iden
 
             await core.fini()  # Avoid having the same DB open twice
 
@@ -49,4 +50,4 @@ async def test_backup(self):
                 fpset = self.compare_dirs(core.dirn, dirn2, skipfns=['lock.mdb'])
 
                 # We expect the data.mdb file to be in the fpset
-                self.isin(f'/layers/{core.iden}/layer.lmdb/data.mdb', fpset)
+                self.isin(f'/layers/{layriden}/layer.lmdb/data.mdb', fpset)