[4.1.x] Fix #546 - Make NRTM parser reject partial content (#547)

(cherry picked from commit 8395e85)

irrd/mirroring/parsers.py

@@ -306,15 +306,23 @@ def __init__(self, source: str, nrtm_data: str, database_handler: DatabaseHandle
     def _split_stream(self, data: str) -> None:
         """Split a stream into individual operations."""
         paragraphs = split_paragraphs_rpsl(data, strip_comments=False)
+        last_comment_seen = ''
 
         for paragraph in paragraphs:
             if self._handle_possible_start_line(paragraph):
                 continue
             elif paragraph.startswith('%') or paragraph.startswith('#'):
-                continue  # pragma: no cover -- falsely detected as not run by coverage library
+                last_comment_seen = paragraph
             elif paragraph.startswith('ADD') or paragraph.startswith('DEL'):
                 self._handle_operation(paragraph, paragraphs)
 
+        if self.nrtm_source and last_comment_seen.upper().strip() != f'%END {self.source}':
+            msg = f'NRTM stream error for {self.source}: last paragraph expected to be ' \
+                  f'"%END {self.source}", but is actually {last_comment_seen.upper().strip()}'
+            logger.error(msg)
+            self.database_handler.record_mirror_error(self.source, msg)
+            raise ValueError(msg)
+
         if self._current_op_serial > self.last_serial and self.version != '3':
             msg = f'NRTM stream error for {self.source}: expected operations up to and including serial ' \
                   f'{self.last_serial}, last operation was {self._current_op_serial}'

irrd/mirroring/tests/nrtm_samples.py

@@ -32,6 +32,8 @@
 
 inetnum: 192.0.2.0 - 192.0.2.255
 source: TEST
+
+%END TEST
 """
 
 SAMPLE_NRTM_V1_TOO_MANY_ITEMS = """
@@ -49,6 +51,8 @@
 
 inetnum: 192.0.2.0 - 192.0.2.255
 source: TEST
+
+%END TEST
 """
 SAMPLE_NRTM_INVALID_VERSION = """%START Version: 99 TEST 11012700-11012700"""
 
@@ -101,6 +105,8 @@
 person: NRTM test
 address: NowhereLand
 source: TEST
+
+%END TEST
 """
 
 SAMPLE_NRTM_INVALID_NO_START_LINE = """
@@ -117,3 +123,5 @@
 
 %END TEST
 """
+
+SAMPLE_NRTM_V3_NO_END = """%START Version: 3 TEST 11012700-11012701"""

irrd/mirroring/tests/test_parsers.py

@@ -15,7 +15,7 @@
     SAMPLE_ROLE, SAMPLE_RTR_SET)
 from irrd.utils.test_utils import flatten_mock_calls
 from .nrtm_samples import (SAMPLE_NRTM_V3, SAMPLE_NRTM_V1, SAMPLE_NRTM_V1_TOO_MANY_ITEMS,
-                           SAMPLE_NRTM_INVALID_VERSION,
+                           SAMPLE_NRTM_INVALID_VERSION, SAMPLE_NRTM_V3_NO_END,
                            SAMPLE_NRTM_V3_SERIAL_GAP, SAMPLE_NRTM_V3_INVALID_MULTIPLE_START_LINES,
                            SAMPLE_NRTM_INVALID_NO_START_LINE, SAMPLE_NRTM_V3_SERIAL_OUT_OF_ORDER)
 from ..parsers import NRTMStreamParser, MirrorFileImportParser, MirrorUpdateFileImportParser
@@ -372,6 +372,18 @@ def test_test_parse_nrtm_invalid_no_start_line(self):
         assert mock_dh.mock_calls[0][1][0] == 'TEST'
         assert error_msg in mock_dh.mock_calls[0][1][1]
 
+    def test_test_parse_nrtm_no_end(self):
+        mock_dh = Mock()
+        with pytest.raises(ValueError) as ve:
+            NRTMStreamParser('TEST', SAMPLE_NRTM_V3_NO_END, mock_dh)
+
+        error_msg = 'last paragraph expected to be'
+        assert error_msg in str(ve.value)
+        assert len(mock_dh.mock_calls) == 1
+        assert mock_dh.mock_calls[0][0] == 'record_mirror_error'
+        assert mock_dh.mock_calls[0][1][0] == 'TEST'
+        assert error_msg in mock_dh.mock_calls[0][1][1]
+
     def _assert_valid(self, parser: NRTMStreamParser):
         assert parser.operations[0].operation == DatabaseOperation.add_or_update
         assert parser.operations[0].serial == 11012700