SYN-7653: Migration for invalid it:sec:cpe nodes

.coveragerc

@@ -1,3 +1,13 @@
 [report]
 omit =
     */synapse/tests/test_*
+
+; Uncomment this section to enable code coverage of storm files in the
+; storm_dirs directory listed below. This is disabled by default right now
+; because it's pretty intensive and imposes a large perf hit on the already slow
+; tests.
+;[run]
+;plugins = synapse.utils.stormcov
+
+[synapse.utils.stormcov]
+storm_dirs = synapse/assets/storm

changes/0d926b9f9e5e7d7c790015c2495dd8d8.yaml

@@ -0,0 +1,25 @@
+---
+desc: Migrate invalid ``it:sec:cpe`` nodes if possible. Expanding on the
+  migration helpers added in v2.170.0, this migration will attempt to fix and
+  migrate invalid ``it:sec:cpe`` nodes.
+
+  Migration of these nodes will only be successful if one of the CPE2.3 (primary
+  property) or the CPE2.2 (``:v2_2``) strings are valid CPEs.  If both CPE
+  strings are invalid, the node will be removed from the cortex and stored in a
+  queue (``model_0_2_28:nodes``). All references to these invalid nodes will
+  also be removed and stored in a separate queue (``model_0_2_28:nodes:refs``).
+  Finally, all edges to these invalid nodes will be removed and stored in a
+  queue (``model_0_2_28:nodes:edges``).
+
+  The structure of items in these queues are opaque. The intent is for Power-Ups
+  to be able to process these queues in an attempt to fix the invalid nodes on a
+  per Power-Up basis (the idea being that Power-Up data vendors probably make
+  the same mistake consistently).
+
+  During migration or removal of invalid ``it:sec:cpe`` nodes, referencing nodes
+  with readonly properties will be removed and stored in the
+  ``model_0_2_28:nodes:refs`` queue. We are unable to automatically migrate
+  these nodes due to the dynamic nature of their construction.
+prs: []
+type: migration
+...

changes/78a597dcd9eb9803a3cc240410eab395.yaml

@@ -0,0 +1,6 @@
+---
+desc: Added ``opts`` field to ``model:type`` Storm type. This field contains the property
+  type options as defined in the data model.
+prs: []
+type: feat
+...

changes/88b706497043eda897376f772af55b88.yaml

@@ -0,0 +1,7 @@
+---
+desc: 'Updated Storm coverage tracker to support ``pragma: no cover`` for ignoring
+  single lines of code and ``pragma: no cover start``/``pragma: no cover stop`` for
+  ignoring multi-line blocks of Storm code.'
+prs: []
+type: feat
+...

changes/cb62250eded6427c7bcfce4602b867d7.yaml

@@ -0,0 +1,6 @@
+---
+desc: Storm lists now have a ``remove`` method that can be used to remove a single
+  item from the list without having to iterate through the list.
+prs: []
+type: feat
+...

synapse/assets/__init__.py

@@ -0,0 +1,22 @@
+import os
+
+import synapse.common as s_common
+
+dirname = os.path.dirname(__file__)
+
+def getStorm(*names):
+    '''
+    Return a storm file from the synapse storm folder.
+
+    Example:
+
+        text = storm.get('migrate.storm')
+        await core.callStorm(text)
+
+    Example #2:
+        text = storm.get('migrations', 'model-0.2.28.storm')
+        await core.callStorm(text)
+    '''
+    with s_common.genfile(dirname, 'storm', *names) as fp:
+        text = fp.read()
+    return text.decode('utf8')