From 5490f5826db4ced5248bebacae75f0e5b6d36174 Mon Sep 17 00:00:00 2001 From: Michael Stewart Date: Wed, 3 Aug 2022 15:09:18 -0700 Subject: [PATCH] update notes add promiscuous mode --- README.md | 23 +++++++++------ figtion/__init__.py | 65 ++++++++++++++++++++++++++----------------- setup.py | 2 +- tests/test_figtion.py | 52 ++++++++++++++++++++++++++++------ 4 files changed, 98 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 6b9e196..0c5886b 100755 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ .---. .-. |~~~| - |_| |~~~|--. - .---!~| .--| C |--| - |===| |--|%%| f | | - | | |__| | g | | - |===| |==| | | | + |~| |~~~|--. + .---!-| .--| C |--| + |===|*|--|%%| f | | + | |*|__| | g | | + |===|*|==| | | | | |_|__| |~~~|__| |===|~|--|%%|~~~|--| `---^-^--^--^---`--' hjw @@ -42,6 +42,8 @@ A simple configuration interface with text file support This will print either '[_www.bestsite.web_](.)' or the value of 'my server' in `~/conf.yml` if it is something else. +*defaults* strictly defines the schema. Only keys present in *defaults* from a serial file will be retained. If you want to risk unspecified input keys and load everything from the YAML file, you can either omit the *defaults* parameter or set `promiscuous=True` when constructing `Config`. + ### Config Secrets When you want a public config file and a separate secret one. @@ -57,15 +59,18 @@ To keep secret encrypted "at rest", set a secret key environment variable *FIGKE This will print the value of `'password'`, which is stored in `./creds.yml` and not `./conf.yml`. If the value of `'password'` is changed in either YAML file, the password will be updated in `./creds.yml` and masked from `./conf.yml` the next time the class is loaded in Python. If a secret key is present via environment variable *FIGKEY*, the values in `./creds.yml` will be encrypted using that key. The dictionary object returned for `cfg` contains the true value. -If you want everything treated as secret, only provide a `secretpath`: +If you want everything treated as secret, provide a `secretpath` and omit `filepath`: + + cfg = figtion.Config(secretpath='./creds.yml') + +In this case, no call to `mask` is needed and everything is encrypted at rest. - cfg = figtion.Config(defaults=defaults,secretpath='./creds.yml') +#### Encryption Details -In this case no call to `mask` is needed, everything is encrypted at rest. +This uses the *pynacl* bindings to the *libsodium* library, which uses [the XSalsa20 algorithm](https://libsodium.gitbook.io/doc/advanced/stream_ciphers/xsalsa20) for encryption. The encryption key provided by the *FIGKEY* environment variable is truncated to a 32-byte string. ## Roadmap * 0.9 - secrets store in separate location * 1.0 - secrets store in encrypted location - * 1.0.1 - easy support for all-secret config * 1.1 - automatic/dynamic reloading of YAML files diff --git a/figtion/__init__.py b/figtion/__init__.py index 737a2b2..cc05cad 100755 --- a/figtion/__init__.py +++ b/figtion/__init__.py @@ -11,26 +11,27 @@ class Config(dict): def filepath(self): return self._filepath - def __init__(self,description = None, filepath = None, defaults = None, secretpath = None, verbose=True): + def __init__(self,description = None, filepath = None, defaults = None, secretpath = None, verbose=True, promiscuous=False): self.description = description if description else "configurations" self._filepath = filepath - self._intered = None + self._interred = None self._masks = {} self._verbose=verbose self._allsecret = description == _MASK_FLAG + self._promiscuous = promiscuous or (not defaults) if secretpath: - if not _filepath: + if not filepath: self._filepath = secretpath self._allsecret = True else: - self._intered = Config(filepath=secretpath,description=_MASK_FLAG) + self._interred = Config(filepath=secretpath,description=_MASK_FLAG,promiscuous=True) ### Precedence of YAML over defaults if defaults: self.update(defaults) - if filepath: - self.load(self._verbose) + if self._filepath: + self.load() def dump(self,filepath=None): """ Serialize to YAML """ @@ -66,9 +67,10 @@ def dump(self,filepath=None): def _recursive_strict_update(self,a,b): """ Update only items from 'b' which already have a key in 'a'. This defines behavior when there is a "schema change". - a is used for the input dictionary - b is used for the serialized YAML file: + a corresponds to canon schema. + b corresponds to serialized (potentially outdated) YAML file: * only items defined in 'a' are kept + * 'promiscuous' mode: items defined in 'b' are also kept * values present in 'b' are given priority """ if not a: @@ -78,11 +80,12 @@ def _recursive_strict_update(self,a,b): return for key in b.keys(): - if key in a.keys(): - if isinstance(b[key],dict): - self._recursive_strict_update(a[key],b[key]) - else: - a[key] = b[key] + if isinstance(b[key],dict): + if not key in a.keys(): + a[key] = {} + self._recursive_strict_update(a[key],b[key]) + elif key in a.keys() or self._promiscuous: + a[key] = b[key] def _getcipherkey(self): """ return cipherkey environment variable forced to 32-bit bytestring @@ -142,32 +145,32 @@ def mask(self,cfg_key,mask='*****'): """ Good for sensitive credentials. Mask is serialized to `self.filepath`. True value serialized to `self.secretpath`. """ - if self._intered is None: + if self._interred is None: raise Exception('Cannot mask without a secretpath serializing path.') self._masks[cfg_key] = mask if self._nestread(cfg_key) != mask: - self._intered[cfg_key] = self._nestread(cfg_key) + self._interred[cfg_key] = self._nestread(cfg_key) self._unmask() def _mask(self): if self._masks: for key,mask in self._masks.items(): - self._intered[key] = self._nestread(key) + self._interred[key] = self._nestread(key) self._nestupdate(key,mask) - self._intered.update({'_masks':self._masks}) - self._intered.dump() + self._interred.update({'_masks':self._masks}) + self._interred.dump() def _unmask(self): """ resolve hierarchy: {new_val > interred > mask} """ - if not self._intered: + if not self._interred: return - self._intered.load() + self._interred.load() try: - self._masks.update(self._intered.pop('_masks')) + self._masks.update(self._interred.pop('_masks')) except KeyError: pass @@ -175,14 +178,26 @@ def _unmask(self): current = self._nestread(key) if current != mask: - self._intered[key] = current - self._intered.dump() # write to protected YAML - self.dump() # write to external YAML + self._interred[key] = current + self._interred.dump() # write to protected YAML + self.dump() # write to external YAML try: - self._nestupdate(key,self._intered[key]) + self._nestupdate(key,self._interred[key]) except KeyError: pass + def __repr__(self): + str = ('secret ' if self._allsecret else '') + f"config reading from {self._filepath}" + if self._interred: + str+= f"\nsecrets stored in {self._interred._filepath}" + if self._promiscuous: + str+= "\npromiscuous mode" + if self._verbose: + str+= "\nverbose mode" + str += "\nValues:\n" + str += super().__repr__() + return str + with open(_Path(_os.path.abspath(_os.path.dirname(__file__))) / '__doc__','r') as _f: __doc__ = _f.read() diff --git a/setup.py b/setup.py index c7e4b3c..69e111c 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ long_description = f.read() name = 'figtion' -version = '1.0.1' +version = '1.0.3' ### include README as main package docfile from shutil import copyfile diff --git a/tests/test_figtion.py b/tests/test_figtion.py index 324eec7..7dbafe0 100644 --- a/tests/test_figtion.py +++ b/tests/test_figtion.py @@ -74,15 +74,6 @@ def test_encrypted_update(self): intered = figtion.Config(description=figtion._MASK_FLAG,filepath=self.secretpath) assert(intered['password'] == 'supersecret') - def test_missing_encryption_key(self): - os.environ["FIGKEY"] = "" - - try: - fig = figtion.Config(defaults=self.defaults,filepath=self.confpath,secretpath=self.secretpath) - except Exception as e: - assert( type(e) == OSError ) - assert( str(e).startswith("Missing the encryption key for file")) - def test_unencrypted_serialization(self): os.environ["FIGKEY"] = "" @@ -97,6 +88,28 @@ def test_unencrypted_serialization(self): newfig = figtion.Config(filepath=self.openpath) assert( newfig['password'] == self.defaults['password'] ) + def test_explicit_promiscuous_mode(self): + fig = figtion.Config(promiscuous=True,filepath=self.confpath) + + assert(fig['my server'] == 'www.bestsite.web') + assert(fig['number of nodes'] == 5) + assert( len(fig._masks) == 0 ) + + def test_implicit_promiscuous_mode(self): + fig = figtion.Config(filepath=self.confpath) + + assert(fig['my server'] == 'www.bestsite.web') + assert(fig['number of nodes'] == 5) + assert( len(fig._masks) == 0 ) + + def test_promiscuous_with_secret(self): + fig = figtion.Config(promiscuous=True,filepath=self.confpath,secretpath=self.secretpath) + + assert(fig['my server'] == 'www.bestsite.web') + assert(fig['number of nodes'] == 5) + assert(fig['password'] == self.defaults['password'] ) + assert( len(fig._masks) == 1 ) + def test_only_secret(self): fig = figtion.Config(defaults=self.defaults,secretpath=self.secretpath) @@ -110,3 +123,24 @@ def test_only_secret(self): except Exception as e: assert( type(e) == OSError ) assert( str(e).startswith("Missing the encryption key for file")) + + def test_only_secret_explicit_promiscuous(self): + fig = figtion.Config(promiscuous=True,secretpath=self.secretpath) + + assert( len(fig._masks) == 0 ) + assert( fig['password'] == self.defaults['password'] ) + + def test_only_secret_implicit_promiscuous(self): + fig = figtion.Config(secretpath=self.secretpath) + + assert( len(fig._masks) == 0 ) + assert( fig['password'] == self.defaults['password'] ) + + def test_missing_encryption_key(self): + os.environ["FIGKEY"] = "" + + try: + fig = figtion.Config(defaults=self.defaults,filepath=self.confpath,secretpath=self.secretpath) + except Exception as e: + assert( type(e) == OSError ) + assert( str(e).startswith("Missing the encryption key for file"))