Keep newline/tab escapes in quoted strings (fix certificate serializi…

…ng) (#296)

Limit backslash escape removal for quoted newlines

Keep escape backslash for newline/tab characters. Parsing still
differs from shell syntax but keeps string structure intact.

Example certificate data
    VAR="---BEGIN---\r\n---END---"
now becomes
    "---BEGIN---\r\n---END---"
instead of
    "---BEGIN---rn---END---"

Co-authored-by: Serghei Iakovlev <egrep@protonmail.ch>

CHANGELOG.rst

@@ -7,6 +7,9 @@ and this project adheres to `Semantic Versioning <https://semver.org/spec/v2.0.0
 
 `v0.8.0`_ - 00-Unreleased-2021
 ------------------------------
+Fixed
++++++
+- Keep newline/tb escaped in quoted strings
 
 
 `v0.7.0`_ - 11-September-2021

docs/tips.rst

@@ -91,15 +91,54 @@ You can use something like this to handle similar cases.
 Multiline value
 ===============
 
-You can set a multiline variable value:
+To get multiline value pass ``multiline=True`` to ```str()```.
+
+.. note::
+
+   You shouldn't escape newline/tab characters yourself if you want to preserve
+   the formatting.
+
+The following example demonstrates the above:
+
+**.env file**:
+
+.. code-block:: shell
+
+   # .env file contents
+   UNQUOTED_CERT=---BEGIN---\r\n---END---
+   QUOTED_CERT="---BEGIN---\r\n---END---"
+   ESCAPED_CERT=---BEGIN---\\n---END---
+
+**settings.py file**:
 
 .. code-block:: python
 
-   # MULTILINE_TEXT=Hello\\nWorld
-   >>> print env.str('MULTILINE_TEXT', multiline=True)
-   Hello
-   World
+   # settings.py file contents
+   import environ
+
+
+   env = environ.Env()
+
+   print(env.str('UNQUOTED_CERT', multiline=True))
+   # ---BEGIN---
+   # ---END---
+
+   print(env.str('UNQUOTED_CERT', multiline=False))
+   # ---BEGIN---\r\n---END---
+
+   print(env.str('QUOTED_CERT', multiline=True))
+   # ---BEGIN---
+   # ---END---
+
+   print(env.str('QUOTED_CERT', multiline=False))
+   # ---BEGIN---\r\n---END---
+
+   print(env.str('ESCAPED_CERT', multiline=True))
+   # ---BEGIN---\
+   # ---END---
 
+   print(env.str('ESCAPED_CERT', multiline=False))
+   # ---BEGIN---\\n---END---
 
 Proxy value
 ===========

environ/environ.py

@@ -180,7 +180,7 @@ def str(self, var, default=NOTSET, multiline=False):
         """
         value = self.get_value(var, cast=str, default=default)
         if multiline:
-            return value.replace('\\n', '\n')
+            return re.sub(r'(\\r)?\\n', r'\n', value)
         return value
 
     def unicode(self, var, default=NOTSET):
@@ -769,6 +769,13 @@ def read_env(cls, env_file=None, **overrides):
 
         logger.debug('Read environment variables from: {}'.format(env_file))
 
+        def _keep_escaped_format_characters(match):
+            """Keep escaped newline/tabs in quoted strings"""
+            escaped_char = match.group(1)
+            if escaped_char in 'rnt':
+                return '\\' + escaped_char
+            return escaped_char
+
         for line in content.splitlines():
             m1 = re.match(r'\A(?:export )?([A-Za-z_0-9]+)=(.*)\Z', line)
             if m1:
@@ -778,7 +785,8 @@ def read_env(cls, env_file=None, **overrides):
                     val = m2.group(1)
                 m3 = re.match(r'\A"(.*)"\Z', val)
                 if m3:
-                    val = re.sub(r'\\(.)', r'\1', m3.group(1))
+                    val = re.sub(r'\\(.)', _keep_escaped_format_characters,
+                                 m3.group(1))
                 cls.ENVIRON.setdefault(key, str(val))
 
         # set defaults

tests/fixtures.py

@@ -31,6 +31,8 @@ class FakeEnv:
     def generate_data(cls):
         return dict(STR_VAR='bar',
                     MULTILINE_STR_VAR='foo\\nbar',
+                    MULTILINE_QUOTED_STR_VAR='---BEGIN---\\r\\n---END---',
+                    MULTILINE_ESCAPED_STR_VAR='---BEGIN---\\\\n---END---',
                     INT_VAR='42',
                     FLOAT_VAR='33.3',
                     FLOAT_COMMA_VAR='33,3',

tests/test_env.py

@@ -57,6 +57,10 @@ def test_contains(self):
             ('STR_VAR', 'bar', False),
             ('MULTILINE_STR_VAR', 'foo\\nbar', False),
             ('MULTILINE_STR_VAR', 'foo\nbar', True),
+            ('MULTILINE_QUOTED_STR_VAR', '---BEGIN---\\r\\n---END---', False),
+            ('MULTILINE_QUOTED_STR_VAR', '---BEGIN---\n---END---', True),
+            ('MULTILINE_ESCAPED_STR_VAR', '---BEGIN---\\\\n---END---', False),
+            ('MULTILINE_ESCAPED_STR_VAR', '---BEGIN---\\\n---END---', True),
         ],
     )
     def test_str(self, var, val, multiline):

tests/test_env.txt

@@ -33,6 +33,8 @@ INT_VAR=42
 STR_LIST_WITH_SPACES= foo,  bar
 STR_VAR=bar
 MULTILINE_STR_VAR=foo\nbar
+MULTILINE_QUOTED_STR_VAR="---BEGIN---\r\n---END---"
+MULTILINE_ESCAPED_STR_VAR=---BEGIN---\\n---END---
 INT_LIST=42,33
 CYRILLIC_VAR=фуубар
 INT_TUPLE=(42,33)