more integration tests and bugfixes in mail handling

docs/users/queries.rst

@@ -111,11 +111,8 @@ The query::
 will set the client name to `my-client` and return all as-sets named
 `AS-EXAMPLE`.
 
-Supported flags
-^^^^^^^^^^^^^^^
-* ``-k`` activates keepalive mode. The connection will be kept open
-  after a query has been sent. Queries are answered in the order they were
-  submitted.
+Queries
+^^^^^^^
 * ``-l``, ``-L``, ``-M`` and ``-x`` search for `route` or `route6` objects.
   The differences are:
 
@@ -132,17 +129,7 @@ Supported flags
   ``-i origin AS23456`` finds all objects with an `origin` attribute set to
   `AS23456`. In attributes that contain multiple values, one of their values
   must match the value in the query.
-* ``-s <sources>`` and ``-a`` set the sources used for queries. ``-s``
-  restricts all responses to a specified list of sources,
-  comma-separated, e.g. ``-s RIPE,NTTCOM``. ``-a`` enables all sources.
-  This persists across queries.
-* ``-T <object-classes>`` restricts a query to certain object classes,
-  comma-separated. This does not persist across queries.
 * ``-t <object-class>`` returns the template for a particular object class.
-* ``-K`` restricts the output to primary key fields and the `members` and
-  `mp-members` attributes.
-* ``-V <free-text>`` identifies the client querying IRRd. Optional, but may
-  be helpful when debugging issues.
 * ``-g`` returns an NRTM response, used for mirroring. See the
   :doc:`mirroring documentation </users/mirroring>`.
 * Any other (part of) the query is interpreted as a free text search:
@@ -155,6 +142,22 @@ Supported flags
     on the primary key of an object, or a `person` or `role` where their
     name includes the search string, case insensitive.
 
+Supported flags
+^^^^^^^^^^^^^^^
+* ``-k`` activates keepalive mode. The connection will be kept open
+  after a query has been sent. Queries are answered in the order they were
+  submitted.
+* ``-s <sources>`` and ``-a`` set the sources used for queries. ``-s``
+  restricts all responses to a specified list of sources,
+  comma-separated, e.g. ``-s RIPE,NTTCOM``. ``-a`` enables all sources.
+  This persists across queries.
+* ``-T <object-classes>`` restricts a query to certain object classes,
+  comma-separated. This does not persist across queries.
+* ``-K`` restricts the output to primary key fields and the `members` and
+  `mp-members` attributes.
+* ``-V <free-text>`` identifies the client querying IRRd. Optional, but may
+  be helpful when debugging issues.
+
 The ``-F`` and ``-r`` flags are accepted but ignored, as IRRd does not support
 recursion.
 

irrd/conf/__init__.py

@@ -231,7 +231,7 @@ def _check_staging_config(self) -> List[str]:
             if not SOURCE_NAME_RE.match(name):
                 errors.append(f'Invalid source name: {name}')
 
-            nrtm_mirror = details.get('nrtm_host') and details.get('nrtm_port') and details.get('import_serial_source')
+            nrtm_mirror = details.get('nrtm_host') and details.get('import_serial_source')
             if details.get('keep_journal') and not (nrtm_mirror or details.get('authoritative')):
                 errors.append(f'Setting keep_journal for source {name} can not be enabled unless either authoritative '
                               f'is enabled, or all three of nrtm_host, nrtm_port and import_serial_source.')
@@ -243,7 +243,7 @@ def _check_staging_config(self) -> List[str]:
                 errors.append(f'Setting authoritative for source {name} can not be enabled when either '
                               f'nrtm_host or import_source are set.')
 
-            if not details.get('nrtm_port', '43').isnumeric():
+            if not str(details.get('nrtm_port', '43')).isnumeric():
                 errors.append(f'Setting nrtm_port for source {name} must be a number.')
             if not details.get('import_timer', '0').isnumeric():
                 errors.append(f'Setting import_timer for source {name} must be a number.')

irrd/conf/test_conf.py

@@ -174,6 +174,7 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir):
                     'TESTDB2': {
                         'authoritative': True,
                         'nrtm_host': '192.0.2.1',
+                        'nrtm_port': 'aaa',
                     },
                     'TESTDB3': {
                         'authoritative': True,
@@ -204,6 +205,7 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir):
         assert 'Setting nrtm_host for source TESTDB can not be enabled without setting import_serial_source.' in str(ce)
         assert 'Setting authoritative for source TESTDB2 can not be enabled when either nrtm_host or import_source are set.' in str(ce)
         assert 'Setting authoritative for source TESTDB3 can not be enabled when either nrtm_host or import_source are set.' in str(ce)
+        assert 'Setting nrtm_port for source TESTDB2 must be a number.' in str(ce)
         assert 'Setting import_timer for source TESTDB must be a number.' in str(ce)
         assert 'Setting export_timer for source TESTDB must be a number.' in str(ce)
         assert 'Invalid source name: lowercase' in str(ce)

irrd/integration_tests/run.py

@@ -12,12 +12,11 @@
 import textwrap
 import yaml
 from alembic import command, config
-from typing import List
 
 from irrd.conf import config_init, PASSWORD_HASH_DUMMY_VALUE
 from irrd.utils.rpsl_samples import (SAMPLE_MNTNER, SAMPLE_PERSON, SAMPLE_KEY_CERT, SIGNED_PERSON_UPDATE_VALID,
                                      SIGNED_PERSON_UPDATE_INVALID, object_sample_mapping)
-from irrd.utils.whois_client import whois_query
+from irrd.utils.whois_client import whois_query, whois_query_irrd
 from .data import EMAIL_SMTP_PORT, EMAIL_DISCARD_MSGS_COMMAND, EMAIL_RETURN_MSGS_COMMAND, EMAIL_SEPARATOR, EMAIL_END
 
 IRRD_ROOT_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))
@@ -242,7 +241,6 @@ def test_irrd_integration(self, tmpdir):
         messages = self._retrieve_mails()
         assert len(messages) == 1
         mail_text = self._extract_message_body(messages[0])
-        print(mail_text)
         assert messages[0]['Subject'] == 'SUCCESS: my subject'
         assert messages[0]['From'] == 'from@example.com'
         assert messages[0]['To'] == 'Sasha <sasha@example.com>'
@@ -262,9 +260,7 @@ def test_irrd_integration(self, tmpdir):
         assert 'unįcöde tæst 🌈🦄' in mntner_text
         assert 'PERSON-TEST' in mntner_text
 
-        # After two seconds, a new export should have been generated by irrd #1,
-        # loaded by irrd #2, and the objects should be available in irrd #2
-        # This is also the first instance of an object with unicode chars
+        # This is the first instance of an object with unicode chars
         # appearing on the NRTM stream.
         time.sleep(2)
         mntner_text = whois_query('127.0.0.1', self.port_whois2, 'TEST-MNT')
@@ -273,6 +269,55 @@ def test_irrd_integration(self, tmpdir):
         assert 'unįcöde tæst 🌈🦄' in mntner_text
         assert 'PERSON-TEST' in mntner_text
 
+        for port in self.port_whois1, self.port_whois2:
+            query_result = whois_query_irrd('127.0.0.1', port, '!gAS65537')
+            assert query_result == '192.0.2.0/24'
+            query_result = whois_query_irrd('127.0.0.1', port, '!6AS65537')
+            assert query_result == '2001:db8::/48'
+            query_result = whois_query_irrd('127.0.0.1', port, '!iRS-TEST')
+            assert query_result == '192.0.2.0/24 2001:db8::/48'
+            query_result = whois_query_irrd('127.0.0.1', port, '!iAS-SETTEST')
+            assert query_result == 'AS2602 AS42909 AS49624 AS51966'
+            query_result = whois_query_irrd('127.0.0.1', port, '!maut-num,as65537')
+            assert 'AS65537' in query_result
+            assert 'TEST-AS' in query_result
+            query_result = whois_query_irrd('127.0.0.1', port, '!oTEST-MNT')
+            assert 'AS65537' in query_result
+            assert 'TEST-AS' in query_result
+            assert 'AS65536 - AS65538' in query_result
+            assert 'rtrs-settest' in query_result
+            query_result = whois_query_irrd('127.0.0.1', port, '!r192.0.2.0/24')
+            assert 'example route' in query_result
+            query_result = whois_query_irrd('127.0.0.1', port, '!r192.0.2.0/25,l')
+            assert 'example route' in query_result
+            query_result = whois_query_irrd('127.0.0.1', port, '!r192.0.2.0/24,L')
+            assert 'example route' in query_result
+            query_result = whois_query_irrd('127.0.0.1', port, '!r192.0.2.0/23,M')
+            assert 'example route' in query_result
+            query_result = whois_query_irrd('127.0.0.1', port, '!r192.0.2.0/24,o')
+            assert query_result == 'AS65537'
+            query_result = whois_query('127.0.0.1', port, '-x 192.0.02.0/24')
+            assert 'example route' in query_result
+            query_result = whois_query('127.0.0.1', port, '-l 192.0.02.0/25')
+            assert 'example route' in query_result
+            query_result = whois_query('127.0.0.1', port, '-L 192.0.02.0/24')
+            assert 'example route' in query_result
+            query_result = whois_query('127.0.0.1', port, '-M 192.0.02.0/23')
+            assert 'example route' in query_result
+            query_result = whois_query('127.0.0.1', port, '-i member-of RS-test')
+            assert 'example route' in query_result
+            query_result = whois_query('127.0.0.1', port, '-T route6 -i member-of RS-TEST')
+            assert 'No entries found for the selected source(s)' in query_result
+            query_result = whois_query('127.0.0.1', port, 'dashcare')
+            assert 'ROLE-TEST' in query_result
+
+        query_result = whois_query_irrd('127.0.0.1', self.port_whois1, '!j-*')
+        assert query_result == 'TEST:Y:1-24:24'
+        # irrd #2 missed the first two updates from NRTM, as they were done at
+        # the same time and loaded from the full export
+        query_result = whois_query_irrd('127.0.0.1', self.port_whois2, '!j-*')
+        assert query_result == 'TEST:Y:3-24:24'
+
     def _start_mailserver(self):
         self.pidfile_mailserver = str(self.tmpdir) + '/mailserver.pid'
         self.logfile_mailserver = str(self.tmpdir) + '/mailserver.log'

irrd/mirroring/tests/test_mirror_runners_import.py

@@ -120,9 +120,10 @@ def test_run_import_ftp(self, monkeypatch, config_override):
         ]
 
     def test_run_import_local_file(self, monkeypatch, config_override, tmpdir):
-        tmp_import_source1 = tmpdir + '/source1.rpsl'
-        with open(tmp_import_source1, 'w') as fh:
-            fh.write('source1')
+        tmp_import_source1 = tmpdir + '/source1.rpsl.gz'
+        with open(tmp_import_source1, 'wb') as fh:
+            # gzipped data, contains 'source1'
+            fh.write(b64decode('H4sIAE4CfFsAAyvOLy1KTjUEAE5Fj0oHAAAA'))
         tmp_import_source2 = tmpdir + '/source2.rpsl'
         with open(tmp_import_source2, 'w') as fh:
             fh.write('source2')
@@ -250,7 +251,7 @@ def test_missing_source_settings(self, monkeypatch, config_override):
         config_override({
             'sources': {
                 'TEST': {
-                    'nrtm_host': '192.0.2.1',
+                    'nrtm_port': '4343',
                 }
             }
         })

irrd/rpsl/fields.py

@@ -190,7 +190,7 @@ class RPSLRouteSetMemberField(RPSLTextField):
           - ^[integer]
           - ^[integer]-[integer]
     """
-    keep_case = False
+    keep_case = True
 
     def __init__(self, ip_version: Optional[int], *args, **kwargs) -> None:
         if ip_version and ip_version not in [4, 6]:

irrd/rpsl/tests/test_rpsl_objects.py

@@ -427,7 +427,7 @@ def test_parse(self):
         assert obj.__class__ == RPSLRouteSet
         assert not obj.messages.errors()
         assert obj.pk() == "RS-TEST"
-        assert obj.parsed_data['mp-members'] == ['2001:DB8::/48']
+        assert obj.parsed_data['mp-members'] == ['2001:db8::/48']
         assert obj.render_rpsl_text() == rpsl_text.replace('2001:0dB8::/48', '2001:db8::/48')
 
 

irrd/utils/rpsl_samples.py

@@ -557,6 +557,7 @@
 descr:          the route attribute should have the extra zero removed,
 +               but this value should not: 192.0.02.0/24
 origin:         AS65537
+member-of:      RS-TEST
 mnt-by:         TEST-MNT
 changed:        2009-10-15T09:32:17Z
 source:         TEST