From 67a9ecb2f0bd59c7d3fd561f39d49dcfd9b67d60 Mon Sep 17 00:00:00 2001 From: Ties de Kock Date: Wed, 12 Aug 2026 19:15:32 +0200 Subject: [PATCH 1/5] Add recovering parser --- CHANGELOG.md | 1 + README.md | 29 ++ src/bin/main.rs | 113 +++++- src/lib.rs | 29 ++ src/parser/iters/mod.rs | 17 + src/parser/iters/recovery.rs | 753 +++++++++++++++++++++++++++++++++++ src/parser/mod.rs | 5 + 7 files changed, 946 insertions(+), 1 deletion(-) create mode 100644 src/parser/iters/recovery.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6025678..8f159ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ All notable changes to this project will be documented in this file. #### Test data * **Historical RIPE regression fixtures**: Added original RRC00 update and bview gzip files from 1999 and January 2000 as repository-only, offline integration fixtures. +* **Opt-in MRT framing recovery**: Added `into_recovering_record_iter` and CLI `--recover` support for salvaging records after damaged legacy Type-5 or BGP4MP framing. Recovery validates a three-record chain, uses exact embedded BGP headers as BGP4MP anchors, and reports every skipped decompressed byte range as a typed gap event. #### Examples diff --git a/README.md b/README.md index ba389a4..070c63c 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,29 @@ match process_mrt_file("http://example.com/updates.bz2") { } ``` +**Recovering After Damaged MRT Framing** + +Recovery is opt-in and never reconstructs a damaged record. It reports the skipped decompressed +byte range before resuming at a conservatively validated record chain. + +```rust +use bgpkit_parser::{BgpkitParser, RecoveryConfig, RecoveryEvent}; + +fn recover(path: &str) -> Result<(), Box> { + let parser = BgpkitParser::new(path)?; + for event in parser.into_recovering_record_iter(RecoveryConfig::default()) { + match event? { + RecoveryEvent::Item(record) => println!("{}", record), + RecoveryEvent::Gap(gap) => eprintln!( + "skipped bytes {}..{}: {}", + gap.start_offset, gap.end_offset, gap.cause + ), + } + } + Ok(()) +} +``` + ### Advanced Examples #### Parsing Real-time Data Streams @@ -481,6 +504,7 @@ Options: --pretty Pretty-print JSON output -e, --elems-count Count BGP elems -r, --records-count Count MRT records + --recover Recover after damaged MRT framing and report skipped byte ranges on stderr -o, --origin-asn Filter by origin AS Number -f, --filter Generic filter expression (key=value or key!=value) -p, --prefix Filter by network prefix @@ -526,6 +550,11 @@ bgpkit-parser --json updates.20211001.0000.bz2 > output.json bgpkit-parser -e updates.20211001.0000.bz2 ``` +#### Recover records after damaged framing +```bash +bgpkit-parser --recover -e damaged-updates.gz +``` + #### Cache remote files for faster repeated access ```bash bgpkit-parser -c ~/.bgpkit-cache http://example.com/updates.mrt.bz2 diff --git a/src/bin/main.rs b/src/bin/main.rs index 74f62de..3a7b971 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -4,7 +4,9 @@ use std::io::Write; use std::net::IpAddr; use std::path::PathBuf; -use bgpkit_parser::{BgpElem, BgpkitParser, Elementor}; +use bgpkit_parser::{ + BgpElem, BgpkitParser, Elementor, Filter, Filterable, RecoveryConfig, RecoveryEvent, +}; use clap::{Parser, ValueEnum}; use ipnet::IpNet; @@ -72,6 +74,10 @@ struct Opts { #[clap(short, long)] records_count: bool, + /// Recover after damaged MRT framing and report skipped byte ranges on stderr + #[clap(long)] + recover: bool, + #[clap(flatten)] filters: Filters, } @@ -248,6 +254,22 @@ fn main() { opts.format }; + if opts.recover { + let filters = parser.filters().to_vec(); + if let Err(error) = run_recovering( + parser, + &filters, + output_format, + opts.level, + opts.elems_count, + opts.records_count, + ) { + eprintln!("{error}"); + std::process::exit(1); + } + return; + } + match (opts.elems_count, opts.records_count) { (true, true) => { let mut elementor = Elementor::new(); @@ -296,6 +318,95 @@ fn main() { } } +fn run_recovering( + parser: BgpkitParser, + filters: &[Filter], + output_format: OutputFormat, + output_level: OutputLevel, + elems_count_requested: bool, + records_count_requested: bool, +) -> Result<(), String> { + let mut stdout = std::io::stdout(); + let mut elementor = Elementor::new(); + let mut records_count = 0usize; + let mut elems_count = 0usize; + let mut elem_index = 0usize; + let mut gap_count = 0usize; + let mut skipped_bytes = 0u64; + + for event in parser.into_recovering_record_iter(RecoveryConfig::default()) { + match event.map_err(|error| error.to_string())? { + RecoveryEvent::Gap(gap) => { + gap_count += 1; + skipped_bytes += gap.skipped_bytes(); + eprintln!( + "recovered MRT framing: skipped bytes {}..{} ({} bytes, {:?}, {} confirming records): {}", + gap.start_offset, + gap.end_offset, + gap.skipped_bytes(), + gap.evidence, + gap.confirmed_records, + gap.cause + ); + } + RecoveryEvent::Item(record) => { + records_count += 1; + let needs_elems = elems_count_requested + || matches!(output_level, OutputLevel::Elems) && !records_count_requested; + let elems = if needs_elems { + elementor + .record_to_elems(record.clone()) + .into_iter() + .filter(|elem| elem.match_filters(filters)) + .collect::>() + } else { + Vec::new() + }; + elems_count += elems.len(); + + if elems_count_requested || records_count_requested { + continue; + } + match output_level { + OutputLevel::Elems => { + for elem in elems { + let output = format_elem(&elem, output_format, elem_index); + elem_index += 1; + write_recovery_output(&mut stdout, &output)?; + } + } + OutputLevel::Records => { + let output = format_record(&record, output_format); + write_recovery_output(&mut stdout, &output)?; + } + } + } + } + } + + match (elems_count_requested, records_count_requested) { + (true, true) => { + println!("total records: {records_count}"); + println!("total elems: {elems_count}"); + } + (false, true) => println!("total records: {records_count}"), + (true, false) => println!("total elems: {elems_count}"), + (false, false) => {} + } + eprintln!("recovery summary: {gap_count} gaps, {skipped_bytes} bytes skipped"); + Ok(()) +} + +fn write_recovery_output(stdout: &mut std::io::Stdout, output: &str) -> Result<(), String> { + if let Err(error) = writeln!(stdout, "{output}") { + if error.kind() == std::io::ErrorKind::BrokenPipe { + return Ok(()); + } + return Err(error.to_string()); + } + Ok(()) +} + fn format_elem(elem: &BgpElem, format: OutputFormat, index: usize) -> String { match format { OutputFormat::Json => { diff --git a/src/lib.rs b/src/lib.rs index 291840d..facf60c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -236,6 +236,29 @@ match process_mrt_file("http://example.com/updates.bz2") { } ``` +**Recovering After Damaged MRT Framing** + +Recovery is opt-in and never reconstructs a damaged record. It reports the skipped decompressed +byte range before resuming at a conservatively validated record chain. + +```no_run +use bgpkit_parser::{BgpkitParser, RecoveryConfig, RecoveryEvent}; + +fn recover(path: &str) -> Result<(), Box> { + let parser = BgpkitParser::new(path)?; + for event in parser.into_recovering_record_iter(RecoveryConfig::default()) { + match event? { + RecoveryEvent::Item(record) => println!("{}", record), + RecoveryEvent::Gap(gap) => eprintln!( + "skipped bytes {}..{}: {}", + gap.start_offset, gap.end_offset, gap.cause + ), + } + } + Ok(()) +} +``` + ## Advanced Examples ### Parsing Real-time Data Streams @@ -477,6 +500,7 @@ Options: --pretty Pretty-print JSON output -e, --elems-count Count BGP elems -r, --records-count Count MRT records + --recover Recover after damaged MRT framing and report skipped byte ranges on stderr -o, --origin-asn Filter by origin AS Number -f, --filter Generic filter expression (key=value or key!=value) -p, --prefix Filter by network prefix @@ -522,6 +546,11 @@ bgpkit-parser --json updates.20211001.0000.bz2 > output.json bgpkit-parser -e updates.20211001.0000.bz2 ``` +### Recover records after damaged framing +```bash +bgpkit-parser --recover -e damaged-updates.gz +``` + ### Cache remote files for faster repeated access ```bash bgpkit-parser -c ~/.bgpkit-cache http://example.com/updates.mrt.bz2 diff --git a/src/parser/iters/mod.rs b/src/parser/iters/mod.rs index 3f5ffc5..154c94e 100644 --- a/src/parser/iters/mod.rs +++ b/src/parser/iters/mod.rs @@ -14,6 +14,7 @@ pub mod default; mod diagnostic; pub mod fallible; mod raw; +mod recovery; mod route; mod update; @@ -22,6 +23,10 @@ pub use default::{ElemIterator, RecordIterator}; pub use diagnostic::{DiagnosticEvent, DiagnosticIterator}; pub use fallible::{FallibleElemIterator, FallibleRecordIterator}; pub use raw::RawRecordIterator; +pub use recovery::{ + RecoveringRecordIterator, RecoveryConfig, RecoveryError, RecoveryEvent, RecoveryEvidence, + RecoveryGap, +}; pub use route::{FallibleRouteIterator, RouteIterator}; pub use update::{ Bgp4MpUpdate, FallibleUpdateIterator, LegacyBgpUpdate, MrtUpdate, TableDumpV2Entry, @@ -75,6 +80,18 @@ impl BgpkitParser { RawRecordIterator::new(self) } + /// Creates an opt-in iterator that reports skipped byte ranges while recovering MRT framing. + /// + /// Recovery never reconstructs a damaged record. It scans for a structurally valid boundary, + /// confirms a chain of records, emits [`RecoveryEvent::Gap`], and then resumes normal parsing. + /// Offsets in recovery events refer to the decompressed MRT byte stream. + pub fn into_recovering_record_iter( + self, + config: RecoveryConfig, + ) -> RecoveringRecordIterator { + RecoveringRecordIterator::new(self, config) + } + /// Creates an iterator over BGP announcements from MRT data. /// /// This iterator yields `MrtUpdate` items from both UPDATES files (BGP4MP messages) diff --git a/src/parser/iters/recovery.rs b/src/parser/iters/recovery.rs new file mode 100644 index 0000000..1a79147 --- /dev/null +++ b/src/parser/iters/recovery.rs @@ -0,0 +1,753 @@ +//! Opt-in MRT framing recovery. +//! +//! Recovery is deliberately separate from the default iterators. It never attempts to +//! reconstruct a damaged record: bytes are skipped until a conservatively validated chain of +//! records is found, and the skipped range is reported as a [`RecoveryEvent::Gap`]. + +use crate::models::{Bgp4MpType, EntryType, MrtMessage, MrtRecord, TableDumpV2Message}; +use crate::parser::mrt::messages::bgp4mp::uses_zebra_compat; +use crate::parser::mrt::mrt_header::parse_common_header_with_bytes; +use crate::parser::mrt::mrt_record::parse_mrt_record_with_zebra_compat; +use crate::parser::{BgpkitParser, Elementor, ParserError, ParserErrorWithBytes}; +use crate::Filterable; +use bytes::Bytes; +use std::fmt::{Display, Formatter}; +use std::io::{self, Read}; + +const DEFAULT_MAX_SCAN_BYTES: usize = 1024 * 1024; +const DEFAULT_CONFIRMATION_RECORDS: u8 = 3; +const MAX_RECOVERY_RECORD_LEN: u32 = 65_599; + +/// Settings for opt-in MRT framing recovery. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RecoveryConfig { + max_scan_bytes: usize, + confirmation_records: u8, +} + +impl Default for RecoveryConfig { + fn default() -> Self { + Self { + max_scan_bytes: DEFAULT_MAX_SCAN_BYTES, + confirmation_records: DEFAULT_CONFIRMATION_RECORDS, + } + } +} + +impl RecoveryConfig { + /// Set the maximum number of bytes searched after a damaged record. + pub const fn with_max_scan_bytes(mut self, max_scan_bytes: usize) -> Self { + self.max_scan_bytes = max_scan_bytes; + self + } + + /// Set the number of consecutive records required to confirm a recovered boundary. + /// + /// A value of zero is treated as one. + pub const fn with_confirmation_records(mut self, confirmation_records: u8) -> Self { + self.confirmation_records = confirmation_records; + self + } + + pub const fn max_scan_bytes(&self) -> usize { + self.max_scan_bytes + } + + pub const fn confirmation_records(&self) -> u8 { + self.confirmation_records + } +} + +/// Evidence used to validate the first record following a recovered gap. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[non_exhaustive] +pub enum RecoveryEvidence { + /// A deprecated MRT Type-5 record and its confirmation chain parsed structurally. + LegacyMrtChain, + /// A BGP4MP message contained an exact embedded BGP marker and length. + BgpMarkerChain, + /// A BGP4MP state-change record, which has no embedded BGP message header. + Bgp4MpStateChangeChain, +} + +/// A byte range discarded while restoring MRT record framing. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct RecoveryGap { + /// Inclusive offset in the decompressed MRT byte stream. + pub start_offset: u64, + /// Exclusive offset in the decompressed MRT byte stream. + pub end_offset: u64, + /// Error raised while parsing at `start_offset`. + pub cause: String, + /// Structural evidence used to accept `end_offset` as a new boundary. + pub evidence: RecoveryEvidence, + /// Number of consecutive records validated at the recovered boundary. + pub confirmed_records: u8, +} + +impl RecoveryGap { + pub const fn skipped_bytes(&self) -> u64 { + self.end_offset - self.start_offset + } +} + +/// An item produced by a recovering iterator. +#[derive(Debug)] +pub enum RecoveryEvent { + Item(T), + Gap(RecoveryGap), +} + +/// A framing error for which no sufficiently strong recovery boundary was found. +#[derive(Debug)] +pub struct RecoveryError { + pub offset: u64, + pub scanned_bytes: u64, + pub error: ParserErrorWithBytes, +} + +impl Display for RecoveryError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "MRT recovery failed at decompressed offset {} after scanning {} bytes: {}", + self.offset, self.scanned_bytes, self.error + ) + } +} + +impl std::error::Error for RecoveryError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.error) + } +} + +/// Iterator over parsed MRT records and explicit recovery gaps. +pub struct RecoveringRecordIterator { + reader: ReplayReader, + config: RecoveryConfig, + filters: Vec, + elementor: Elementor, + show_warnings: bool, + warned_zebra_compat: bool, + finished: bool, +} + +impl RecoveringRecordIterator { + pub(crate) fn new(parser: BgpkitParser, config: RecoveryConfig) -> Self { + Self { + reader: ReplayReader::new(parser.reader), + config, + filters: parser.filters, + elementor: Elementor::new(), + show_warnings: parser.options.show_warnings, + warned_zebra_compat: parser.options.warned_zebra_compat, + finished: false, + } + } +} + +impl Iterator for RecoveringRecordIterator { + type Item = Result, RecoveryError>; + + fn next(&mut self) -> Option { + if self.finished { + return None; + } + + loop { + let record_start = self.reader.position(); + match parse_mrt_record_with_zebra_compat(&mut self.reader) { + Ok((record, used_zebra_compat)) => { + self.reader.discard_before_current(); + if used_zebra_compat && self.show_warnings && !self.warned_zebra_compat { + log::warn!( + "recovered shortened Zebra BGP4MP records with missing envelope fields; substituting IPv4 zero addresses and interface index 0 (further occurrences for this parser will not be logged)" + ); + self.warned_zebra_compat = true; + } + + if self.filters.is_empty() { + return Some(Ok(RecoveryEvent::Item(record))); + } + if let MrtMessage::TableDumpV2Message(TableDumpV2Message::PeerIndexTable(_)) = + &record.message + { + let _ = self.elementor.record_to_elems(record.clone()); + return Some(Ok(RecoveryEvent::Item(record))); + } + let elems = self.elementor.record_to_elems(record.clone()); + if elems.iter().any(|elem| elem.match_filters(&self.filters)) { + return Some(Ok(RecoveryEvent::Item(record))); + } + } + Err(error) if matches!(error.error, ParserError::EofExpected) => { + self.finished = true; + return None; + } + Err(error) if is_non_eof_io_error(&error.error) => { + self.finished = true; + return Some(Err(RecoveryError { + offset: record_start, + scanned_bytes: 0, + error, + })); + } + Err(error) => match self.find_recovery(record_start, &error) { + Ok(Some((end_offset, evidence, confirmed_records))) => { + let gap = RecoveryGap { + start_offset: record_start, + end_offset, + cause: error.to_string(), + evidence, + confirmed_records, + }; + self.reader + .move_to(end_offset) + .expect("accepted recovery offset remains buffered"); + self.reader.discard_before_current(); + return Some(Ok(RecoveryEvent::Gap(gap))); + } + Ok(None) => { + self.finished = true; + return Some(Err(RecoveryError { + offset: record_start, + scanned_bytes: self + .reader + .buffered_end() + .saturating_sub(record_start) + .min(self.config.max_scan_bytes as u64), + error, + })); + } + Err(io_error) => { + self.finished = true; + return Some(Err(RecoveryError { + offset: record_start, + scanned_bytes: self.reader.position().saturating_sub(record_start), + error: ParserErrorWithBytes::from(ParserError::IoError(io_error)), + })); + } + }, + } + } + } +} + +impl RecoveringRecordIterator { + fn find_recovery( + &mut self, + failed_start: u64, + _error: &ParserErrorWithBytes, + ) -> io::Result> { + let confirmations = self.config.confirmation_records.max(1); + for distance in 1..=self.config.max_scan_bytes { + let candidate_offset = failed_start + distance as u64; + if !self.reader.move_to(candidate_offset)? { + return Ok(None); + } + if let Some((evidence, confirmed)) = + validate_chain(&mut self.reader, candidate_offset, confirmations)? + { + return Ok(Some((candidate_offset, evidence, confirmed))); + } + } + Ok(None) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StreamFamily { + Legacy, + Bgp4Mp, +} + +struct Candidate { + family: StreamFamily, + evidence: RecoveryEvidence, +} + +fn validate_chain( + reader: &mut ReplayReader, + offset: u64, + required: u8, +) -> io::Result> { + if !reader.move_to(offset)? { + return Ok(None); + } + let mut family = None; + let mut evidence = None; + let mut confirmed = 0u8; + + while confirmed < required { + let record_start = reader.position(); + let Some(candidate) = read_candidate(reader)? else { + reader.move_to(record_start)?; + if confirmed > 0 && reader.at_clean_eof()? { + return Ok(Some((evidence.expect("set with first record"), confirmed))); + } + return Ok(None); + }; + if family.is_some_and(|expected| expected != candidate.family) { + return Ok(None); + } + family.get_or_insert(candidate.family); + evidence.get_or_insert(candidate.evidence); + confirmed += 1; + + if confirmed < required && reader.at_clean_eof()? { + return Ok(Some((evidence.expect("set with first record"), confirmed))); + } + } + + Ok(Some(( + evidence.expect("required is at least one"), + confirmed, + ))) +} + +fn read_candidate(reader: &mut ReplayReader) -> io::Result> { + let parsed_header = match parse_common_header_with_bytes(reader) { + Ok(header) => header, + Err(error) => return parser_error_as_candidate(error), + }; + let header = parsed_header.header; + + let family = match header.entry_type { + EntryType::BGP if legacy_header_is_plausible(header.entry_subtype, header.length) => { + StreamFamily::Legacy + } + EntryType::BGP4MP | EntryType::BGP4MP_ET + if Bgp4MpType::try_from(header.entry_subtype).is_ok() + && header.length <= MAX_RECOVERY_RECORD_LEN => + { + StreamFamily::Bgp4Mp + } + _ => return Ok(None), + }; + + if header + .microsecond_timestamp + .is_some_and(|value| value >= 1_000_000) + { + return Ok(None); + } + + let mut body = vec![0u8; header.length as usize]; + if let Err(error) = reader.read_exact(&mut body) { + return if error.kind() == io::ErrorKind::UnexpectedEof { + Ok(None) + } else { + Err(error) + }; + } + let raw_record = crate::RawMrtRecord { + common_header: header, + header_bytes: parsed_header.raw_bytes, + message_bytes: Bytes::from(body), + }; + + let evidence = match family { + StreamFamily::Legacy => raw_record + .clone() + .parse() + .ok() + .map(|_| RecoveryEvidence::LegacyMrtChain), + StreamFamily::Bgp4Mp => strict_bgp4mp_evidence(&raw_record), + }; + Ok(evidence.map(|evidence| Candidate { family, evidence })) +} + +fn parser_error_as_candidate(error: ParserError) -> io::Result> { + match error { + ParserError::IoError(error) | ParserError::EofError(error) + if error.kind() != io::ErrorKind::UnexpectedEof => + { + Err(error) + } + _ => Ok(None), + } +} + +fn legacy_header_is_plausible(subtype: u16, length: u32) -> bool { + match subtype { + 1 => (16..=65_535).contains(&length), + 3 => length == 10, + 5 => (22..=4_089).contains(&length), + 6 => (14..=65_535).contains(&length), + 7 => length == 12, + _ => false, + } +} + +fn strict_bgp4mp_evidence(raw_record: &crate::RawMrtRecord) -> Option { + let msg_type = Bgp4MpType::try_from(raw_record.common_header.entry_subtype).ok()?; + if matches!( + msg_type, + Bgp4MpType::StateChange | Bgp4MpType::StateChangeAs4 + ) { + let body = &raw_record.message_bytes; + if uses_zebra_compat(raw_record.common_header.entry_subtype, body) { + if body.len() != 8 { + return None; + } + } else { + let asn_pair_len = if matches!(msg_type, Bgp4MpType::StateChange) { + 4 + } else { + 8 + }; + let afi_offset = asn_pair_len + 2; + let afi = u16::from_be_bytes(body.get(afi_offset..afi_offset + 2)?.try_into().ok()?); + let address_len = match afi { + 1 => 4, + 2 => 16, + _ => return None, + }; + if body.len() != asn_pair_len + 4 + address_len * 2 + 4 { + return None; + } + } + raw_record.clone().parse().ok()?; + return Some(RecoveryEvidence::Bgp4MpStateChangeChain); + } + + let body = &raw_record.message_bytes; + let asn_pair_len = match msg_type { + Bgp4MpType::Message + | Bgp4MpType::MessageLocal + | Bgp4MpType::MessageAddpath + | Bgp4MpType::MessageLocalAddpath => 4, + Bgp4MpType::MessageAs4 + | Bgp4MpType::MessageAs4Local + | Bgp4MpType::MessageAs4Addpath + | Bgp4MpType::MessageLocalAs4Addpath => 8, + Bgp4MpType::StateChange | Bgp4MpType::StateChangeAs4 => unreachable!(), + }; + + let marker_offset = if uses_zebra_compat(raw_record.common_header.entry_subtype, body) { + asn_pair_len + } else { + let afi_offset = asn_pair_len + 2; + let afi = u16::from_be_bytes(body.get(afi_offset..afi_offset + 2)?.try_into().ok()?); + let address_len = match afi { + 1 => 4, + 2 => 16, + _ => return None, + }; + asn_pair_len + 4 + address_len * 2 + }; + + let bgp_header = body.get(marker_offset..marker_offset + 19)?; + if bgp_header[..16] != [0xff; 16] { + return None; + } + let bgp_length = u16::from_be_bytes([bgp_header[16], bgp_header[17]]) as usize; + let bgp_type = bgp_header[18]; + if !(19..=65_535).contains(&bgp_length) + || !(1..=4).contains(&bgp_type) + || marker_offset + bgp_length != body.len() + { + return None; + } + match bgp_type { + 1 if bgp_length > 4_096 => return None, + 2 if bgp_length < 23 => return None, + 3 if bgp_length < 21 => return None, + 4 if bgp_length != 19 => return None, + _ => {} + } + + raw_record.clone().parse().ok()?; + Some(RecoveryEvidence::BgpMarkerChain) +} + +fn is_non_eof_io_error(error: &ParserError) -> bool { + matches!( + error, + ParserError::IoError(error) | ParserError::EofError(error) + if error.kind() != io::ErrorKind::UnexpectedEof + ) +} + +struct ReplayReader { + inner: R, + data: Vec, + start: usize, + cursor: usize, + base_offset: u64, +} + +impl ReplayReader { + fn new(inner: R) -> Self { + Self { + inner, + data: Vec::new(), + start: 0, + cursor: 0, + base_offset: 0, + } + } + + fn position(&self) -> u64 { + self.base_offset + (self.cursor - self.start) as u64 + } + + fn buffered_end(&self) -> u64 { + self.base_offset + (self.data.len() - self.start) as u64 + } + + fn discard_before_current(&mut self) { + self.base_offset = self.position(); + self.start = self.cursor; + if self.start >= 64 * 1024 && self.start * 2 >= self.data.len() { + self.data.drain(..self.start); + self.cursor -= self.start; + self.start = 0; + } + } +} + +impl ReplayReader { + fn move_to(&mut self, offset: u64) -> io::Result { + if offset < self.base_offset { + return Ok(false); + } + while offset > self.buffered_end() { + self.cursor = self.data.len(); + let remaining = (offset - self.buffered_end()).min(8_192) as usize; + let mut scratch = vec![0u8; remaining]; + let read = self.read(&mut scratch)?; + if read == 0 { + return Ok(false); + } + } + self.cursor = self.start + (offset - self.base_offset) as usize; + Ok(true) + } + + fn at_clean_eof(&mut self) -> io::Result { + let position = self.position(); + let mut byte = [0u8; 1]; + let read = self.read(&mut byte)?; + self.move_to(position)?; + Ok(read == 0) + } +} + +impl Read for ReplayReader { + fn read(&mut self, output: &mut [u8]) -> io::Result { + if output.is_empty() { + return Ok(0); + } + if self.cursor < self.data.len() { + let count = output.len().min(self.data.len() - self.cursor); + output[..count].copy_from_slice(&self.data[self.cursor..self.cursor + count]); + self.cursor += count; + return Ok(count); + } + + let read = self.inner.read(output)?; + self.data.extend_from_slice(&output[..read]); + self.cursor += read; + Ok(read) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::models::{Asn, Bgp4MpEnum, Bgp4MpMessage, BgpMessage, CommonHeader}; + use bytes::{BufMut, BytesMut}; + use std::io::Cursor; + use std::net::{IpAddr, Ipv4Addr}; + + fn legacy_state(timestamp: u32) -> Vec { + let mut bytes = BytesMut::new(); + bytes.put_u32(timestamp); + bytes.put_u16(EntryType::BGP as u16); + bytes.put_u16(3); + bytes.put_u32(10); + bytes.put_u16(64512); + bytes.put_slice(&[192, 0, 2, 1]); + bytes.put_u16(1); + bytes.put_u16(2); + bytes.to_vec() + } + + fn bgp4mp_keepalive(timestamp: u32) -> Vec { + MrtRecord { + common_header: CommonHeader { + timestamp, + microsecond_timestamp: None, + entry_type: EntryType::BGP4MP, + entry_subtype: Bgp4MpType::Message as u16, + length: 0, + }, + message: MrtMessage::Bgp4Mp(Bgp4MpEnum::Message(Bgp4MpMessage { + msg_type: Bgp4MpType::Message, + peer_asn: Asn::new_16bit(64_496), + local_asn: Asn::new_16bit(64_497), + interface_index: 0, + peer_ip: IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)), + local_ip: IpAddr::V4(Ipv4Addr::new(192, 0, 2, 2)), + bgp_message: BgpMessage::KeepAlive, + })), + } + .encode() + .unwrap() + .to_vec() + } + + fn decode_hex(input: &str) -> Vec { + input + .as_bytes() + .chunks_exact(2) + .map(|pair| { + let text = std::str::from_utf8(pair).unwrap(); + u8::from_str_radix(text, 16).unwrap() + }) + .collect() + } + + #[test] + fn recovers_at_three_record_legacy_chain() { + let first = legacy_state(100); + let mut input = first.clone(); + input.extend_from_slice(&[0xde, 0xad, 0xbe, 0xef, 0x01]); + input.extend_from_slice(&legacy_state(101)); + input.extend_from_slice(&legacy_state(102)); + input.extend_from_slice(&legacy_state(103)); + + let parser = BgpkitParser::from_reader(Cursor::new(input)); + let events = parser + .into_recovering_record_iter(RecoveryConfig::default()) + .collect::, _>>() + .unwrap(); + + assert_eq!(events.len(), 5); + assert!(matches!(events[0], RecoveryEvent::Item(_))); + let RecoveryEvent::Gap(gap) = &events[1] else { + panic!("expected recovery gap") + }; + assert_eq!(gap.start_offset, first.len() as u64); + assert_eq!(gap.end_offset, first.len() as u64 + 5); + assert_eq!(gap.confirmed_records, 3); + assert_eq!(gap.evidence, RecoveryEvidence::LegacyMrtChain); + assert!(events[2..] + .iter() + .all(|event| matches!(event, RecoveryEvent::Item(_)))); + } + + #[test] + fn rejects_chain_shorter_than_configured_before_non_eof_garbage() { + let mut input = vec![0xff; 12]; + input.extend_from_slice(&legacy_state(101)); + input.extend_from_slice(&legacy_state(102)); + input.extend_from_slice(&[1, 2, 3]); + + let parser = BgpkitParser::from_reader(Cursor::new(input)); + let result = parser + .into_recovering_record_iter(RecoveryConfig::default()) + .next() + .expect("one error"); + assert!(result.is_err()); + } + + #[test] + fn accepts_short_confirmation_chain_at_clean_eof() { + let mut input = vec![0xff; 12]; + input.extend_from_slice(&legacy_state(101)); + input.extend_from_slice(&legacy_state(102)); + + let parser = BgpkitParser::from_reader(Cursor::new(input)); + let events = parser + .into_recovering_record_iter(RecoveryConfig::default()) + .collect::, _>>() + .unwrap(); + let RecoveryEvent::Gap(gap) = &events[0] else { + panic!("expected gap") + }; + assert_eq!(gap.confirmed_records, 2); + } + + #[test] + fn recovers_bgp4mp_using_exact_embedded_bgp_headers() { + let first = bgp4mp_keepalive(100); + let mut input = first.clone(); + input.extend_from_slice(&[0xde, 0xad, 0xbe, 0xef]); + input.extend_from_slice(&bgp4mp_keepalive(101)); + input.extend_from_slice(&bgp4mp_keepalive(102)); + input.extend_from_slice(&bgp4mp_keepalive(103)); + + let parser = BgpkitParser::from_reader(Cursor::new(input)); + let events = parser + .into_recovering_record_iter(RecoveryConfig::default()) + .collect::, _>>() + .unwrap(); + + let RecoveryEvent::Gap(gap) = &events[1] else { + panic!("expected recovery gap") + }; + assert_eq!(gap.start_offset, first.len() as u64); + assert_eq!(gap.end_offset, first.len() as u64 + 4); + assert_eq!(gap.evidence, RecoveryEvidence::BgpMarkerChain); + assert_eq!(gap.confirmed_records, 3); + } + + #[test] + fn rejects_bgp4mp_candidate_with_inexact_embedded_length() { + let encoded = bgp4mp_keepalive(100); + let header = CommonHeader { + timestamp: 100, + microsecond_timestamp: None, + entry_type: EntryType::BGP4MP, + entry_subtype: Bgp4MpType::Message as u16, + length: (encoded.len() - 12) as u32, + }; + let mut body = encoded[12..].to_vec(); + // 16-bit ASN/IPv4 BGP4MP envelope is 16 bytes; corrupt the BGP length field. + body[32..34].copy_from_slice(&20u16.to_be_bytes()); + let raw = crate::RawMrtRecord { + common_header: header, + header_bytes: Bytes::copy_from_slice(&encoded[..12]), + message_bytes: Bytes::from(body), + }; + assert!(strict_bgp4mp_evidence(&raw).is_none()); + } + + #[test] + fn recovers_exact_supplied_ripe_damage_prefix() { + // Exact first 264 decompressed bytes of rrc00/2000.03/updates.20000325.0345.gz. + // The fifth MRT header begins at 188 and is missing its final two length bytes; the + // following valid Type-5 state-change record starts at 198. + const PREFIX: &str = "38dc366700050001000000181200cb25ff7e316ec100000100081818d81018cb1297000038dc3667000500070000000c0201c041b803316ec100000138dc366800050001000000341200cb25ff7e316ec100000100000020400101004002120208120012a909d4193d04d7070802bd02c1400304cb25ff7e18cc5c5438dc366800050001000000341200cb25ff7e316ec100000100000020400101004002120208120012a909c10cf822a2220d220d22ad400304cb25ff7e18c28d1f38dc366800050001000038dc3671000500030000000a0b6281fa00e80003000238dc3671000500030000000a0b6281fa00e80002000338dc3675000500030000000a1200cb25ff7e00020003"; + let parser = BgpkitParser::from_reader(Cursor::new(decode_hex(PREFIX))); + let events = parser + .into_recovering_record_iter(RecoveryConfig::default()) + .collect::, _>>() + .unwrap(); + + let gaps = events + .iter() + .filter_map(|event| match event { + RecoveryEvent::Gap(gap) => Some(gap), + RecoveryEvent::Item(_) => None, + }) + .collect::>(); + assert_eq!(gaps.len(), 1); + assert_eq!(gaps[0].start_offset, 188); + assert_eq!(gaps[0].end_offset, 198); + assert_eq!(gaps[0].confirmed_records, 3); + assert_eq!(gaps[0].evidence, RecoveryEvidence::LegacyMrtChain); + assert_eq!( + events + .iter() + .filter(|event| matches!(event, RecoveryEvent::Item(_))) + .count(), + 7 + ); + } +} diff --git a/src/parser/mod.rs b/src/parser/mod.rs index e486c81..07a411b 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -295,6 +295,11 @@ impl BgpkitParser> { } impl BgpkitParser { + /// Return the filters currently configured on this parser. + pub fn filters(&self) -> &[Filter] { + &self.filters + } + pub(crate) fn warn_zebra_compat_once(&mut self) { if self.options.show_warnings && !self.options.warned_zebra_compat { warn!( From 7ad3b79ad31bb79affceb0011412b0cf0f15a93f Mon Sep 17 00:00:00 2001 From: Ties de Kock Date: Wed, 12 Aug 2026 19:31:07 +0200 Subject: [PATCH 2/5] fix: Verify re-syncing/recovery using a fixture --- Cargo.toml | 1 + src/parser/iters/recovery.rs | 44 ----------------- tests/fixtures/ripe/README.md | 4 +- .../rrc00/2000.03/updates.20000325.0345.gz | Bin 0 -> 28128 bytes tests/recovery.rs | 46 ++++++++++++++++++ 5 files changed, 50 insertions(+), 45 deletions(-) create mode 100644 tests/fixtures/ripe/rrc00/2000.03/updates.20000325.0345.gz create mode 100644 tests/recovery.rs diff --git a/Cargo.toml b/Cargo.toml index b6ca253..9208107 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ exclude = [ "tests/fixtures/ripe/**", "tests/legacy_ripe_2000.rs", "tests/quagga_bgp_states.rs", + "tests/recovery.rs", ] [[bin]] diff --git a/src/parser/iters/recovery.rs b/src/parser/iters/recovery.rs index 1a79147..8a9450d 100644 --- a/src/parser/iters/recovery.rs +++ b/src/parser/iters/recovery.rs @@ -601,17 +601,6 @@ mod tests { .to_vec() } - fn decode_hex(input: &str) -> Vec { - input - .as_bytes() - .chunks_exact(2) - .map(|pair| { - let text = std::str::from_utf8(pair).unwrap(); - u8::from_str_radix(text, 16).unwrap() - }) - .collect() - } - #[test] fn recovers_at_three_record_legacy_chain() { let first = legacy_state(100); @@ -717,37 +706,4 @@ mod tests { }; assert!(strict_bgp4mp_evidence(&raw).is_none()); } - - #[test] - fn recovers_exact_supplied_ripe_damage_prefix() { - // Exact first 264 decompressed bytes of rrc00/2000.03/updates.20000325.0345.gz. - // The fifth MRT header begins at 188 and is missing its final two length bytes; the - // following valid Type-5 state-change record starts at 198. - const PREFIX: &str = "38dc366700050001000000181200cb25ff7e316ec100000100081818d81018cb1297000038dc3667000500070000000c0201c041b803316ec100000138dc366800050001000000341200cb25ff7e316ec100000100000020400101004002120208120012a909d4193d04d7070802bd02c1400304cb25ff7e18cc5c5438dc366800050001000000341200cb25ff7e316ec100000100000020400101004002120208120012a909c10cf822a2220d220d22ad400304cb25ff7e18c28d1f38dc366800050001000038dc3671000500030000000a0b6281fa00e80003000238dc3671000500030000000a0b6281fa00e80002000338dc3675000500030000000a1200cb25ff7e00020003"; - let parser = BgpkitParser::from_reader(Cursor::new(decode_hex(PREFIX))); - let events = parser - .into_recovering_record_iter(RecoveryConfig::default()) - .collect::, _>>() - .unwrap(); - - let gaps = events - .iter() - .filter_map(|event| match event { - RecoveryEvent::Gap(gap) => Some(gap), - RecoveryEvent::Item(_) => None, - }) - .collect::>(); - assert_eq!(gaps.len(), 1); - assert_eq!(gaps[0].start_offset, 188); - assert_eq!(gaps[0].end_offset, 198); - assert_eq!(gaps[0].confirmed_records, 3); - assert_eq!(gaps[0].evidence, RecoveryEvidence::LegacyMrtChain); - assert_eq!( - events - .iter() - .filter(|event| matches!(event, RecoveryEvent::Item(_))) - .count(), - 7 - ); - } } diff --git a/tests/fixtures/ripe/README.md b/tests/fixtures/ripe/README.md index ad4371c..425b21e 100644 --- a/tests/fixtures/ripe/README.md +++ b/tests/fixtures/ripe/README.md @@ -10,6 +10,7 @@ must not download data. | `rrc00/1999.12/updates.19991214.1621.gz` | | 305,644 bytes | `8e4b0ed378464f68397d60e2775a310117b23ffe6dc7f79fbb0d5039a743c91e` | | `rrc00/2000.01/updates.20000102.2014.gz` | | 32,455 bytes | `e15119ada15bed7b524f9cef91a2ae1002f28e65360b32a8f40531d76fdc0a5f` | | `rrc00/2000.01/bview.20000111.0032.gz` | | 2,932,008 bytes | `1456fd58551374c6222c3f2f88606bac327ffdb3bdd931dd150fabae68800009` | +| `rrc00/2000.03/updates.20000325.0345.gz` | | 28,128 bytes | `f5acf0a6d5bc2610c05350c75b2eaf4e3bc100701836e6cf004fbcf635f3fc78` | | `rrc01/2000.11/updates.20001104.0124.gz` | | 8,815 bytes | `89985919e4a1f8726b1e1a6391b12ce1c5576f9d066d84e729414d922e4052f1` | | `rrc03/2010.02/updates.20100227.1600.first-796-records.gz` | | 12,953 bytes | `cca59f359818c3dacc18014e50ce3b656cbc4d86be97cbfbdd6771047a2e71fd` | | `rrc15/2010.02/updates.20100227.1600.gz` | | 213,264 bytes | `68e03340030f3c44a9eb3dd22baa7739f910466cd82ee844fc8c0147f86b7028` | @@ -17,7 +18,8 @@ must not download data. The September and December 1999 and January 2000 update files use deprecated MRT Type 5 BGP records. The December fixture includes one BGP OPEN and one BGP -NOTIFY record. The bview uses early TABLE_DUMP records that batch many entries +NOTIFY record. The March 2000 fixture contains two damaged MRT boundaries and +is retained to test opt-in framing recovery. The bview uses early TABLE_DUMP records that batch many entries and declare each physical record four bytes shorter than the bytes written by the historical MRT producer. The rrc01 update file contains shortened BGP4MP state-change and OPEN records diff --git a/tests/fixtures/ripe/rrc00/2000.03/updates.20000325.0345.gz b/tests/fixtures/ripe/rrc00/2000.03/updates.20000325.0345.gz new file mode 100644 index 0000000000000000000000000000000000000000..bd34ea0ead16a679a6fafd85ec58ebbffe564109 GIT binary patch literal 28128 zcmYJab95)q(>5I2wr$&XHnz@^7`*zi_Pk3zP-BT%Ujk1NiXaJcFVG&&%_ zHR6o3tT2GU6Bd;r`M!Z7*2gW+PBXDtDQV9%#=E$rk_`!t@gx^HGYp1$0S_efP z;NDmL=l)#CVF775S6gjn^w>peJ15!d$HeEeh5D-wFJJrjnoqST2niL7Xda=FM%AP zLoTagKUQ>NKDAt8`4R)Y2=5<=fJh|-dRec8|0%O%QCV#E599wV8!YHW<^~1EiRg-rB&fpX{YMsB`ST%NOV=0E+Uf`38=EjEH?rulcRb5 zbx@NRDNtj($c#~~1)8o5Q>`&9_!j*0FR{YCP}_90)LO@;x#j}U*%d`r#5n)2hx#A1 z@tonN|E$;y%8nxiW>$c7A+ekOe)`|b;Admh+NzvmIZ^uG zKux#l|8c(cf5;VS{)f{3rveX?krtc$AK)FT|CBv>(Z5t~l>IL?Rv#(R=Nc@P2LDI8 zWs*w&Del?F!h^#*o9kLAU-!N_-4o}ss0k0ix z-0)t(z>JE-3>0$;Ny0H*zW3`%K_g&e!Yh>cCzQry;)h42$@U`fMhz>@j96kj62qJ0 zRUT+uqg6N`v4GSioq7zVu|QYgC8C1y5E>P?#mU{hP35M(NQg&t0?Ufhy9N_nzy5^1 zfU^|S-BDm*`i{ozW_o*?jqumO2b{H+B~bsKumc+`aPIE-Z09GZB7SCaZcUCzo=zs8 z9Qv+RkpL$jw*&I{q&sfzblaDRaXwL<(8Bj^A8{deDXJu2BjuYII5Un{~h?7xwhI}eqHR)MA!@hRTP zwUFbI<^?hCQ&Cpdw1?g7Ki`>2;g>tL9*>py6*m;pNr(xlsIae(PIaZO&^uL`Q25#t zTGCrxwr_jnPz}1DaU3k-X~J~V$g+qk{a>r84}WCVfm8ZGVxzaZCA zf~>>T*5*!;=NOIVJuIx-6-;YJ57lJl;P^DYsoc<00pIq&OO)5SB;05gb1uiM)7;i_K!03(ewpHV-r<-Le0I8)+n5 zFqmpkFbamVkXhY0AL|};_i26Oy!eO)eN7D{Ej@mxZ^HWWXh`^F$GHt+aZN7uUk@e$ zOb$ffhr^(>!ZrV=uu1(CUN#HvRE!oRtN6eEi*6da_9Q#6=tO?g2_d!?QwhQcDl$%= zpDY_Evp>f`llo>GbXXPZfx=QKF=k**8sNDso^xgiViyl&7pRj?7Ii2XAQP@&c1>0w zSCc81i@$SiiV+=tAB zIv>^g2Yjq8=q+8cw}uYWi8DZ5$Z0f=O(3#rGn|N6sDftnr5Pr&MOFv$%jW@=J4fDz z3eyra9!?UZP4gHFntn74%!+chIY@MDvKZ&K7~?q6gaw7wkhp2KogY$UPSc5s;IjIC zF6tQR5JnY93MzjJggVbZOCM0-mxHfOs8X*?I_opxFZjav13E-Dx|6S9j3PNov6~i{ zYF4oOT7r%SFdVHE6lIZyEDuw4h(+hVJg(y-tZ>0u)-CL(RWJ+9rORntu+ia!U7R{Il`qKifWLsc~Ll^pMnkA~~sgCTHYG+SYa|^(1l-mPgZHQDO(u zs34-P%N9@0Nm^;+BPUd&lK+P?@=q)tm`B{7nmEahWw-vD6`^PUKS7WFm*8*o-l^e# zYHq-}p?(wtIuY(6MvdZu<2i+6{X4<$IYxPjZae}0s&O_oC_a>%%~G}<5YSrF=O9(Y zypZc3xQc<{_Q|vC{Z_S}B||qf;5}V3`FAK--(p6*va-Fg#NoL|S`Y6XlfHr#35#Pu zqobDRhg5l&%m_B9HWbSav~!?&!CJLJ3!nx3Ne`2M?N$N z1hrhzEeM|;MgCbAT7lX$L(t5V}SiXXA{y zDSzy~|6Q-@b+H`82hs?9XhER zQT;jP;IH~5VsM8#;1%EBv3sr+BHym)<~JJ}r`q-`lcDnOV{3?%8|cI|-_f zbCjQbTBb<^;T;kut&hiwv#HTJ3_To3kErj2e3!;?X_mH8mc8H6@d9ra@9E>U1iFWu z!j;hG_t}X!1Or(&rSz|c0XE#ZN2aSfw<+xrJlt@jROkc4mx?StSD4zIX=Qemc;ZIAn8vZ z+)=%~L6+CA-*Ex9WD#NC{MoN#hFlHpHB4_sm_CSp&1q{ptLV#MLl6sujByBaUX z4Gi^~&pYpCJ=_NlFJT+y%^!Capo6b`o(fqAC}?=Z_E+RD7Eg$?5|w|+6D#$9C0BOW zGpYGh7@BmOn!q$6H%xs9n`~>$e67=dQ<5-ol&LcrEh;rBk-^Rgyd zt{cNKc`EZccuODdVsg)Q>--447+X$ED2-F9locf_)>s~Rk4XgWE{Ecus%3z3(2`=Y zvO=PWzQi106h#OUpwf+Bd(SBe96vQqnQbf~Y>I6gB~EIg${**Ej3vTKHG1PqtHbp( zJbLE2gaQNBuInW>#vFeR*CAbTC7|`jR{+?i7Ho5mv4*nf`m^rt!kVgdUgVnu2FA0o z8t#2mDp%HK75!so^noeP3z`*HzqBJLXupB%e6q=KHQwox+M}uZ(QTE4X`|+oD?aI3 zpRXa#fZxy#SuNeDOg!AN?9`VFTC(Xr^>8H4%tr$iA|vC$vq!O1x$ilS9Cr1#T2fbni449wksnO2_ZmmuSGc&VXR6-=uGma}V} zBV;irY2heWtOcs9G!rYa6RV)BbuO6emSm2tIV(q0j74SHG8|zwb)*xUTD_wG=x9a` zS)#GVp*INgVwO;66cutwJk*hm-NqG-jasJ@DvBpf%>*tXJbW@p{F=lCKVFcN)3Aiq z0UyuSxywY=?L`MBqM%rnn$4uE#EMlRL8>-7oy}^`N2+CqCh3yj5?UdrXde7D{d0C9*3a zA9810oHRV43#~`{6zjmN=BRnMF|z41{2uXl#=B87a*ufY!oH*)!E3YTaom9Op{xTW zRd5TkZA!7ll1PViHAcf(O~|w;xY`j{y1Kcn=BneV%|08-dpcJ9U30F@L$S8+r7?~3$| z{%tvM>0P%8HIsOF1hIN-c?2^GY3axaL=Pv{cqczK=r6P4yi^Jb_QKo)5uX{f$1Bv13EFHk*x-BO!@9;}6Btr> zLIhtYws_Mte$((ao->S8cC1JLWyw~Z;gezbTDJBxk%if_oa5nAH%=Vc6`!d-4RSbv z!Q6{A4&MQn@AW9g3kjI+>JMy$Ay!xdM)%I+Hao6Gyqliqztnr*HbdLi@!DJK+W)4w zEM$W%*&qR(6Iw7j%cs~-K$5Yd$7At7SGY@lbx8?M@LYY z+PD3nr`Ot)VFQxVr5kVM-7nf`a_v@MR>+&FBxhfTx zjfyY@0kkm+LNU)SER!;*CwZko&QcqWM}3vfEw3vHx)gYvc{2_?V!fV4SN$pRLk|=u z98r`8utd>3K@9-+5MuBJjo|^xb3COtkrSvQ<<5VYDz{pX#VRMCGi-&9No#( z636LL$M3;9;?{cUC48`#gFkNrbi>lbozYLH!CXN-xix8XaqvaC%YC{6y)Ai45Z<_Yw?%7xIZ_$>U z7V1;gRu7hwa>F8nIS$}$4rQu2+byvL3es$-F0;p7*LVZmf}ub$6*%9rXe1M(LMx9%QRw&?Y2$D2M7%ZO#CF1XQ+=>vT!?4rM7R@wZ(R9A3u&iIzC=2F#YD3SAzx5i zE`>z&9m=tBUc)pk1g8LG1n@VQLz~2EaEj^fP%>O+;ljDYY;lpRQ2cE=Kp5v-EFNVg ztJ`{L&K#Aur? z9;~Zju@4*0^FgKH9lEf1jSM!{P1>$r+!Eh_6N?I`ROv@XBfX5Qv8q@Q+~>db#1<}G zcUv5JMax)LW{R?mACt5E4=3)9c%mthX-TQ*8-a|iucqp>s|@v87=FpkQDn~B&to8~ zcydauv)y()+_z@EKk4lwW|O8{3B#cu65tokw*x`lUb>OFVL5XTq69iDcelj4vSwMU z;iGBXDC?dRR0Mj}4{=wG<{+@~D$(i0mJY4Q;V$Dr#Ee_53I7F;0t``3jPA#A7# znOR&0u@dwdK_57e1g&TxzX(vgrn_(G9TfTYJ4xQx)#-lER7QbK-tMRQ{#swg_6sZj zifi=(Sib8ZhiB6g+kh7@Yw)q!fVn{GI7(*mV%4~BUEId%dOk%@dr3T9FDgzgAu9^{ z$n+QoZY8vNdXRqndy(PIP2gPTWwRglgjB+mDsP&sSbL?&%;ba3{q4d{^OF`{Mzn-8iv`ZK6)pkqlpOAlI+SF zxH25={)(lPf<^wlMOYjuGvRaaUtK`b`m11mBg4h->`1RaK;8mCd5FCypu<0V4*{#` zE3$0yErBa#mPGVK!#gBipT02!8+GkZ?O$&t4h>;9mKp)QGc<1Yi6BErC z5nRYTF5BERRVggh^7ysBG&aImd<*OdoTx5Utbc68q0~@lcUa;VxC)EAq==0j0xDEcz>;7#ixIF$q);muaC;cVP_8;$$+fr1U4NiEJ3~ze4 zH#You$(L88G&DP9vUNKGIqKB#*ftIVP8v?%ls+;ZJQz6$>_ZOLHp-J9> zd2!+Cri2qAy-E|)dx$v!BN5&0@&R28J#AbMaRa-<;%?i+Vw0wXB#lVd7E2p(UJby5 zScfj42leX4OjFLz^eNQV^vO=aeR4$X-$#C2(yt7w9MY9M22pb!34tc{&?!^Wye(T1 z3)NRS^`G0K!mk@`9~1Jb8}}+p%B6YmhX&tXp}yG(B|h3Ve$z)XD? zN&7@x)MD^LFS0?80j*VqmeRsYnD6nvdwV^!X=(m86^^(<#f-3ZUuY=A*N_?0af0>E z5PcdUsVZzf4TVWeW*W`5rYacfrIF{up}I(f_uH$JOQ#w2@G3rdPs9C0uG-eL(tDD) z6U1Ft?~?w^`;GF$N|pKP*V z;`DpQQe+hkyM?3kqJRB)+75q%)5l**fcW0gZHN4) zud&uxM^-K%ZvbsoAOa`3R!fJ9y5K;j`8mX~gxqz=$DaV_FXK}7@@T9vb4KwCz6gwN zSKaeT@pJO4OouaTdTpLJGI6_%SR_;lqBiNU5`*YFJuK<_jDL`re;nYi`V5o_nS<2`yfmAE7GpLwwKWrC8DZKxdwlda1FM- zp9tW@dgEp8i&4Vyi4hT9My7eg%Ml*iDslO^N$7@KDW_wnQG3qy)~!W7f`RAyuJ{7w zkf)>@poY%wy4QTH*_V6LgY9s z6DM7ncM~6@@D%*3*BTH)uj91u6a5*O+0zqHDE)UnCLMlnK`NC{J1{yMkuZhrdLQ*Q zCy25P`o8bP`Cu7CXio4oCgRv$ZZaZ2^b;C9KXDTWk|&5V4kEvFhRVGV8Rsd5nL;}- z8Fi5g9Y!t-55>q5HCY^GF1yY@~*eblIIrl>T9->Fd= zER9Gt0a5}+-s^PfLSLRB#MfLi_*qh}-E%3bsecz^6YWpA-75xZD;GS`lyjDo+($%0 zJ@@g?G>k#CQsrd|`a+omh1Mi#NP!Z_74=2%vJm;jjJut0IE+2qlAI+}_meObNJa22 zaL{p*A}nv4MyWzvs1VgM({WOSvF*E*@r=yaw8l)yCA5!<(-XGk;;T`GIs@MP(R7Lh z@&{;hlQtIXDnaSMYeOImci9_~wZg#fiYE)p(`$b#33;4T>|;tQb3z-0w(mm4mabfk zm9aJ~9fF$D!Q!kL`(dz+M;!STcIi+4&=HfSFU7DJzYS9~_UHYMRyNL9BY+uB%81z~ zamCY%apH{?-(UsBU$fSymgFcjsaP29S=2md+a@<>^zKQA`8%?tuK`h|t{Aj6$>-vj&2Kdu9=ljDs$tOlyn9~N%AafQaNb8Q zk6jL2@Q3YdR7Mik=g9sFmZc)9nbBCe@qF(cn&O%U4S6a8BpHR4&Ul~ydj}0;%uO)r z(mJ6^!16A{N9F7y+&d*q(U!&A_#r@=9(=kk%E^~%SW;F!BHaoS(|d)MqH4#Z!ZIzQ zMISIB{^2!Oi*%(Y_RWrr5?VD1r!JYZDKt5{>2v z{1qEVQi~PozTUS@Znu&y64PF+UuEl9v9v~Q8?b7I;Hq!r8P<3fxT7U#sg?Aw0SWs` zmt|f&2`CR57_x1mVAnlPy_?;9?Dmr_&9_L`3Lt@P};7t2Z-1S682(1!PE~_6%~| z2VGx{Mp#vf8g2wQG=yiGb1N=ZhxRz5#S5r!6gVM`DGB-fFhAX)GPhcPJZ(Tj6}3;=4{g(LHwD1j`wcdjLrr7sye9cpt5YI+ z^|T;4rha4K^${i{dlP_{S%48xxs4!N)jBbAO+SpSF%D(i2u#JXdUywxrHT|i1QBt& zKn^<;H9Ci0u1Ud)5SJ45_{Opy`_Vu`;^GKsqj#v%Q29,}YEw+Ga4B8!W-Sj}s@ z3I+6D6~anj2}~5M*bzc}$4tdq^6$1NnqI0N0URVOl6!2Q|0?&&Vg=pV6Wv z8|tB7x%DH^S$2|=t>sIz!!>}t4Deffin=$ocDELwDCGP^d*Af$7Cs{URjVYbIGbvY zmT83(<&1Jm>3%Ek4$@jvgYy8EpAD|Db?&&&s6)V{q~059W2H33LLm;1@R`G?`g;de z4y!j+z*`$0A>U)F)8Ub6f1LLU_-q2Yt}>kQt8?ViUciw4MmtY^f_GH&(&L1J?|L0y z-z2)Y%c($D#36q3{%Q$)Rc&P}<7(T_>mF#sz@X>teg3buYpU}!D@mNuWLZ(!bKu+J z%UPBFyHiF3{UA5cd~>X&I5f6PyfgCGbblMqAq<92am2`gjj=xAe2!@nB!F?h-kJoV znYGBvOsY%DCvbfGd|CX9z9)#X?J02xnN4R@-KW7{fWe7N@c}p}tLs9|z>c^#T{|pT zzIGWn*%^=hhH5aKgyER0-zAVr)t{fVa3ImiY<&#xP*2h9)BYEjUfgFgVgR69vdn@| zzdk7cM2AAB18V{NKBDA~%!10k&B14~nl5`ME1FJX9_aqhay1!$De8D3zbjmpW3hSW zC+{Rc#c{QL7wz7oa%47-(h#$u)%Jrg{nntH>@D;!T5FO(tR|<&bWN@Bb7D5{XqoER zJl?qq1OV5)^Q3I2z7&aZi$&E%k!FafMr9PGq~0?^#1<3-%5b9H97t{X3#ngxI$p+8 zrYYY_)lBKoKL170XVMny9;V4?SBj_2o6LoY9$VTt1K5)Hcp$* z;CU9XW!=)xzM0Xzfba}QuWze_RT_V@KU8#L|BZJ%k=@ZAJ&4(bVu1JEuj7w&j`Q80 z6*1WJyr?)MUDD0p!gof#)%2y~^UI>mAO9Q! zai_7s`88EB{CUmxH8p1crzboN-0vsthZse z9u9gTk@d%Q&EhlRlcn8{W02hkBnOfQ9`9-w+E}#iv1++VZADD7zkzBQV`UkgQSGUb?#@Aen%|#uwFD?13T{9Lg;#T-Gc8k`u&vP zMK1BrDx%M6^V)60gBY(CK*xH6QkA4jO*q5$+^{e@H>!&zcGcJI+*$1YERc3AJ?fjm zOMb|?QhiSYzxHt@e}%_>NB5_;@}XmrPu;@pEaSs7=&}(jLwcSV7y+_v@Tj#Mm2VIo zIMvx7w#u^QP0od8{Jjo^Y{c%VbJ=ATbbO>tjp?qKG;3vkj^PIzpar)KAywKeaZWe! znwts#d%bqj!Z2;`FEa+M7&m-2S~siTlE}q%e=TPhV8vCC*b%oHF%`oiBdH* zir`+qT|R_R(%Dhme>|WSn#+%E+IFeF=RbZB!2UUTq4TmmCYKS6QYXDC)VxqUcYg!~ zd~K$R%yyPU@9k>3m$bq$_+IK_oG^drko(+}Q`mY<5ra&E|7l4*V*joLzE-2|7ZJcdj6c>6$^q?z`l7g@eg zF&{`68$ZZ~M;cw9d)tiO^MRB0^l!Bcm!WU^yaK4vrChM!N*4q6I$~hpZn&StSTm7b zum$MAWw0nPqzbzjTkF9I2o!}lIFrg7YDXzUXXS2*25#;wWipc`{l(^)1?@$H1b@jb zYPX+~O##MvlI1h})A5h83)#u?LQv%R5wYx?al&E(&9@+B0izL8v3sWWk*Tj>fgUCz zS^TK+Y54cG^ecTgMWc?F;yCnFK;q(OG!uIs-JN)y7$Q3&imcKDYvCeaIKA}L%x;B8 z{5lc4AgSC`(YID?W}cDCLxnP)wQs2W)Pb++GnPU*b!pUfvEWq3cUc3b3xj2AW7;4D z2DO@rRA8|qS2XjEb-Iv6iJ+%QxSd@h?@4w1lJ=#|!3tbgR@S5z(OE|%8UHAac-73V zl?z~Q{MSCKtqh-h=z0j>7vSW<3nwLD@94REm{%2zbvHvYlXBObT@_O`>;AI8IyZt< zxgn9bq^#|%fS;`HVO5lQYiY>~6Y7#@HI%N$DM}ZZi5;PqTB#F;8B~r;#O&r!=`aF) z`8YyYVhszpH-%<+cpKgrjN-cxi!|P;)p%2Eyu+wZ#3MwBdSP3B$}a~{Va^xUkLAdJ z;DXmb>8zaRSf}Kp7FS*v8BVJgSpB$h#t!^TSNjwYw6tV>wDGAd;*(?GwZ#?yr2S8bpjkV(Pg(K8-T87 zVh*y~M)l_n(7kKj`xkkyy$714->zo@=fS6B)9cuCLkBSI9#vcJF0+pAyF~iD-5VaE zrX7Rz`9(GYpQ)p=4sZSTbC@FG)!w*YajO2RCODA-Y_RvS>o%?swX^d(w`Rt;M#s$b zHfMMvQUY9e0-unw;7ijh~!k<5CwxYp`S2!i+9Rlx!GMO!KMiJ}0 zAh7C8Y-MTi$qxL(b-ns?;*!(ATxMq+$+ky>ttKtHyKY#_=GEWXr_YrUxoaUJPSMNS z&CDMSN+>Pd`6u1q2WX4NwYojxSS<;!HUg~z$Elp{S0!8tgz~0mI|4JuA?dKKGX|OC z+=YG^{9cQYgo#=u5WWhEPbZ!sH;J@Ll`lcmfCh9NdgKPAclP=p>T0n=Dsk(Q{U9M; z9U1US1g?c$p6*iH71AI8hQtaf;`eh}*gC=I6$eRQSvS}cT@eCEOLZh1mq5K*^l{2N zVSXz=TT;G;3Wy&v&3^|W+<9s`Spgj3Wi@aT#YS!XC zeI4t>3nb`{g9sWf^^k=Mg2pm&)W2Ff=SoyCSI@I zU5~XOUSA=ylm_%4U;*}t)La@j6SJ_$0Hr(}-H|(`oS>94+8*3T@TH~WsVHe95RCv) zrhqEynLxr^N;XBmP@)oG`|73RVWC7^5HFHF@kti3+|XJHiCnE@A^nQ{1y|lvG}Gi` zfH?+tO_dOD>%=c2Nnk&}kTN)N)Leu^Zy*nmQMEQ+mS$VYh`;4k^E58V_1>?62WT#A z@bP^s11|XM{!VdXjd<&#j&5wDdjkXxPmUm%B0d&`AjWXM2h;W(5Pmo*c)X zg- z*gJ1KaU`QsYd6FuGRFbY7)4JWrj}Q|ZbzX^oon(D?+(n5`0*vNm6ED-otc+|gTtTz zn*%LLC(*jYz9aRz&ETej@|cso5p#$O2QT)ZBnQ!<-s@T1jlEN@FuW>;iZscTe~;)P zZQ9qA(2pjCUnxgILQ0kyy00FO{Ep-lSZ^B{REcuztMm;%tnVecP-8-c#-hZw)Xj7P z%M@?v<@wiXKI0@+m129Ql-5;*B`D1hT?`u>q`Fj=B(RY#OqqB4nk2Zs{gJ{P0n!y> zA^#S8<9$Z8K!*fbPsCmh+`zgiSrrp5p%*SWWQ$-KiyeqR3uX1Ky z>jLTt!rgVF;5GV{z+CqVN262L>yFahN8P%lx&Y1h%})4pu;om|bKctM`I44T|Cg4} zx`~#LXPG5$ljnxBRu_jFK+tWesiX0I5?#w@Ik0;B-@f!sWVa7T)r^Oi`b`H<)l3fk zx{a6@brs=S;!>Jk_7Y@i#PX7wfiK2_?Dgri!M;yZ?K>WGrt=^^B!f(Lx6FFVcf0b; zcY+kQEC|~uRSu9}`jkvy_8pf9kZFp{JtW0k5_9r>Y~o)kF=-LJ{J9O~w7NAd!$oz# z%K3__OtjyP9k8?=ZiY(%(b0BqBBJ5 z*r!RRt^%hU7&<@ludnCCO z2@P7V=k81qrkKb z_ORsy2|RV_YNcL$>1r+H6_85{TxVzge>^^ma$e$DEj?WiMS_->PZMBYuHIFgN4oWX z=*fH*q?{ILE+n%vmU!)lLFiw=+MswNsy)KA!GyFDHE^DwZjlkhbP{>|t}oC=R`Qe0 zRt^OXnIAbb?Q1-DN2D0aqc?zgT zkFDmz*UYO9Hn1eXH1{d^0l6`*Kx}%Xd)(2R66(FPPR_K-Wy9sanK$K2gA-r z2{GAY{&^hrM|kQ|hg&C=c}n#Qi>)7S8CasNvKtgBjZ&W7-);&#tnTMN=FiwH`vP zx8;sfB22D77@AjCB$H+m=(pZS_V_^-99lO z;;I$V8Np}{r%u*y0Xk}Oxmqmyk=O!?cbAq z9@fViBdJw08|)CbSVYLZs@Q=_p;DfUYdE6qk9?Rtj`>RtKrXBu!IgjuMdynY?$u^c z4y%VE>iV;H5`fL1P7;VOi!9dZH^_a>dI_}Q@0De{LNt`tUYMFk^C?S zHBvo$mOnUlw+K`_T-Z(R9Uae$uGIN?MBc>U-`RIoAnb0|`}-3oJho;wNchp`_D!d? zSWy-~iwR=rMsEgA7KqmOk$kD!piT`hzKcnB_iVCZ$i6Py*&cT+oZqQ_%w~qL;8~8p z{#e3{?#lh!K{_D3?i{)U#ika_Tu25z@1e)z+Cw4BF&2?6t;jo{XrOf4yh7aP#)DUUgLIv>ke_z_p|H!4 z{_zOq`c~G2lwl;3Ytn}&|~Jf6eq zcV##0*2HoeI|0CQ{Nn=b0AQtYb}vbDvWciK9zDP50rMh?$~WJZpa z+aU1#ki`-Z@I1YRwV6bWb}oOIg3JZ&guMJ}e=#RP<|k=M3iY+Zg;BwY0^>mDszq*f zEnD$qyYs=n66;i+miF=M6z{%6P;`Sm+o!fQREDtdn)h&&XjgyXh_#tz zZ;cPNHHtJFSeSuxC!B(MagwsZ@0z4&J5W+2AdaoJqE85cF4rg>s9H6=IEJl)U^mFn zn=`6Al6W#Ihy`vJI;$^^WV}zer6pbx^|t7QDzZ{^8s*)>jaG@29vz!vdnf84`sgw-Q9h1;zuJkx+fO)8p#8=>EioCx)ljB`3qKbfDSOq7EMmSaHy|f8bc>k z2w9f=ORB7r+&CeUkV`4{waswHWqy5-=K7_C?rq&Y_|N@X~E)tcy} zS(YfI=X1%Kn&emQ_tRW;ghFBRAh8wxh^F#yTDOzu!D!O8U@*8S%EEGbf#pvSe1LG} zm4Le{Z{S+k+Visgc)THA+qzPb1hf)31g*GA8U^{mtmrbdh}N{p>V1$0%U56~F8>~C z%J_GLP{*3+I%1AH6yY59 znYF@(S@BUeZMf+2lM5YhF@sz2K>zO-PVieU(&FPO*x3c!u{GTS0~5J?HeCWRU=Ehu zHh(N$ZQM&wp})l39-k$?^H{C7BawC2Q6E(}MFGbW=E5kILiz}DOy%IIj!VQnPTZdJ z3O(X;^usBUK()8heW7w9=;I|sH0B;0Cr{dD)U<=_y6O~Eef(*SFHZr^)JpvmrH0S~ zN@vEL;n6IJuJAfjd_ofa+Ox$uBmoj;<{G({E0iR%`jArv6ldN_8<)TJFa-Mj}fvDp+t z@!QW$J!IBEG8mVPqH1NURZ%;+-i1b@3aFycZpW=h?7V93TSJ!5fZi{si2#IGmJ}u1 zvvl(vnMI?>r%M-+p+iUdDx?y#U_eU1)n`AFGzC!d%(*wmv6Jp7Wt;vP42ch%Ogz67A`*aRvJj#6m24GY_y0NExryq+NkIBkn_otUXY`R<6x7wv}BG zP0|hzb}Y~LZ+x{xE}N85Fc--{PgbeqpfkU&+GmsIAIR+)6cA_9uUd*+G4>)W!!Prm zm(?{Bw6-3Xz|c$BBvqgf(o&}n3aIcEmH(>kZn`@Y>Tz8PI=KJE})~1eU)Mln4JIhyknU7)B$~>k3|1bVgOz$pG1Cw?BU0k4y`6 zeMHb}F^D0`Hz^ZM=-SY#Ibo@CNov-{g%{)RSQY}}JvEGT(bY;tNR*V#zjxlhfEEd}>zZxiXD@=8ognZd5=BpJBC%bvjZ9nYEH z`MB3Y=*Z9A(ct-ZhLA+FTR-<`gZLOFU-q372{S5^#$hMiG_HwIJXCp$RZiOM5=8eR>SwPeZcmEc4x`q{O?@_~n+nh3CQe%V}W5zB2PI!e@XWBZ6g^I33F@g+?s8 zJPA*=J6A@e9~)nuUh_cd97|lR1V5sX9puwfCJ~ER_mD5&+gnR8lQ|^$=c=g&eU5=l z;+OUfPe+O;M3)XdnA&CpUI`k3?g6V%rKLYt#a8$|zOW{sW;CMWei{BTsvZz}k{MF%wV;+OqvIo(fsi$K$BFWRkfe}yqIO|ON|44WED7Phco_pUCyW~7_gG1QuaxDE-FVbgPE&( z6=o-%CN>a zSM(h<@nqLuh+<9kFn}Ybx;u$t#@GJ{CsBd($l5haq1r^lcj?j{H0t>C1!pJnsUiN3DHfvaRa>bzc00M>N@&U3kW_-D&P zgBv}-2khbr%owe|$OE(PW+`NAipjLyqiNd?c(FV+{lt%H6V~PY3D>=Mzq{hq5#*Wp2b~@ix%BK8Zti0mLv?Q>sj=B zNTiXB1J-1sC{|8WOk@jFVR?#;Fw^yF&B*E$Na6b88Jf?Ike4lPFJxnp5P#?}Gvz-> zHc~Zo4P54c>iBQ|Qej3X+_v*!ezF|oRxJaDCd~P?H$K>kwy*4J5gXHDu!|DJ8f7E3 zGHvNylIYk0li@Jpf9k=-y8{050kxm)W@@L5_jGD|TtA3u6urpDvt+Jigd5DBcRn0K zH7)Kwh^aFEek+OR^zxxqB)3b!?8{*dX55aF2AY}Eej-!VTIo~LeYVz&$TybyLTGZs z6H>+bVQq+&d$U{mS7?%$cWL65hC#gO}$-mGba3P^lImZCPr5(BriUYP1Av)VL5=NB>t^&|kWs z95fmQf9pg{-Vpv@i}{Lj)9qYyqOQes;Hq{O zl(C%gJy+eB++z8d0Q55I%lM7=bt1wg_^zJYak7f!$*bg+43qZ?q9R~A z)iCkSku8k{NK{-{=cjcYq!>p;W~Xi{Z{2t&g&OClZrn~mQAJ2fwS88j81hiNf?%+J9+{@W!v4@L<_gTdHY1d@(=+Vy?Kg%Qno}>CNS13$WJIPc*n}_gFRt9 zwa|?vn|v)9G^|vxRf&c=%Y~~ZFwlk6ZC&A?)wc2%Cz2I#9HuiS+u8Qq#T>+Y$U^Z% znE6~|jYjt@IVIB5O2*}G4#Xvd#%L9HA#uWLcj@{=M?AD+WMI;z9@jIgA+D^VRYe0P z!YYzXc)?yC`U(bb(=sbPs!s(xs0mn|2OHisT|l>#nCUl(tz%Z+;#I>gxa4glwKyIR zP-M5j2is%1zW;#{GF@}SgN|_F7}LW@8+bFvfvyJmF_4dhQL#}I83;ei&&G=n&%Fmfxtayh1mjotr8?56b8aWsQs z4L3|bj^g=a2RKHwIJu>J*fZ|}G<}s&xR*Qv(Lmu}={?J>kY!z0Z>{$0J9tG5njl& zSv&T*G;LTmlw!ES3kOP=?)tpKV;s?&7-K#k|M6RvXv;l}fnBHydkm`(vkhcO!lWnI zOM;aol+`H}RX6W+9StVPYxKc6^X$0-5;2Lu(7bQ5n*{H04WteU9Uq49mb=sb)}TXHWA6O#c`{GQ{MIS ze;e=DaZHtbj>+*?d}I0AdA5;~Jmjj91bgju*iS!&XH>{fVe38s9sU3IcXCbWhnXrW zh20uy^JR(lBTDPtxi$wuY4kGNGXmrT%L(gHn$-HUVpR&Xz8X=ITZ65?L(3G1@3dzE z>^Rn__rfDV%7@`=VL2%q!1H-ooUrCJBmMcFa|)7+8?wcX(;Gaqa`Kh^W5 zzQF#N6D1Gjgf$S9@y#geteoOHEBOvJoWqf*A-2t9na2y)AZga3@gTDpFZ|* zQ!|cT2t`2p1(^^&s$SNrRb97lfYmuxJ*P?RIPH~+c2*E6WXep@JZ=TWNKAv%M$5x0VpPA`#I ziuI2qR+n2)mSmB;im(rz6$tUBbSvLkf}~DX26seO>^Z>{u;h6(^dH!6c>-552zraG zr+k+OF<<=HB2A&h;FV^a+$%&ijCTO5{MWeGcOw!#27_N$f&Cu$sUawL+(oMaNq4i+ zt)9_sG!#rg6FL_Jl~Hp!;Q5~FCA*mi zuX(Vc=ajIW9Femwh9JpGP_U)B_>LLeB11}iSwrnYS!Uk7!o zlO3-YQFq=R=UXQ6y(&_$==!TRmok#0z>SvGc0%J!_VN%3RKfxAcVv4G_h1O&Kl#mz zdiI!2Q>Is+rQxqVSFcgWQUi5|elThjwo->bDF$9Jogh_9k;CJao*pnm0}+6j$UVm= z?g`9dfimVmDl3LnOv%La{Tci?`ruiH0EuN4B zgDA4MB%I$ycKFBtMk0z4_eqxiU(WE!7X{&24AV$0s3jo*g_4LAJuYN~!J1eOWYPat zpgE?q{;O&>(Fw6HwGZx2l`Vb>5xLW*l%Oa4H2$A-U?-n@Z1gpmzyJPBfwlsSTuHWp z|7gMslF~g87{mAnkcw(8N@`pu&CByqys$V&7YBypE_Qtj($9@t9OqK_7c?R$v|-lF2qmV%mET!179Bm+`8Sl1Y|!Qm`IkS$K|ek*ERvUoi1s zp21;xO?yRu1Fwa^-W1i8r%VC{I-c|!TjsCfM+Pre)42ZWjOVIOi-tgPUoKsx^cTIG zR2Xl+N%mip;a=R1U*q?YBTGeduVQ*|`5(T-JEl`?1ue>@anw~=?A{;AKN%Y@P)1bP zpXqOhWgJGF;I)B+!(5=vkQ(0BduSAXkeWl1e|2?{?w39CSs2oLUnpOlr>3OUOg>pY zfszjj^01h)^sD|!!Dh3%3iw{%5P1drGgj6RQL2CfnO%coJ2`9f?FmoITRJs6&m?}S zccjL*2AyNW)kx!!y6P#XU|TVg$!r_BHpNS`~b6CgQq| zN3ZGq^6H`!*3ZsJWn?5ri%i;J%W2TlMyO@KAiH{nojUizCbJ5;Y=E+;ZF4G`m)3dKFjbJ*?72@rh2KM4_*vS5JtX#A{+I!M zL`;CKOl*&TmK-}$NHZ8>Fwcl>_*LQ)jQhUH@A5s8|FJ(^;l|9FISjkpTe;Q3f^VEb4?I zD#9OX*)CRAsJQ{(=E~cJ2{0^b+Do#*51SqfmU#sZ-w@B^li3mhkMOPLjF`}K;On&a z?4Q>pj^D((pNzO}UwkB96u535%ks0Rmhtn!l8v9$p9MV;*ADdQIue;Un3J5Hy;-(7zE;wy>L9lwn7cuxtFh zNVo&*C4#9a04-r1B`x}sLzgnjg6q5m34QGklrOS)%7S_a^{bB)oz;1yS5E)k`p1%IJyL(<}= zwG#R?IQHzy=EC(j6hu9kRaM|kqnZfEK9O%8%_s83y4#7UQJRbWq_fH&aAN^x(cC6g zs*n6b*XfJ{9x=a{jZ{prW(=&|myxL{?$X%e{;{I-WJo(Jr;P!^KUD$NGFksRH;OH{ zl;54b%fhnPc-E`J=YLb!xN{ZIHLr!HRdx26Smu{@qt?f~r`YIzeTFz+w8kq=_6sSu zBqw%fwYja^flLW*4UHG|`~I38`Al-(@Wlce@Nq zhYN`Q`;Y<_9)A(^o6zNr>z^&#`eW02=p*Z+jLpz=54FfPWB@46ihDmIRJxIiQMq^;!>y zwfO%-|3HZ8@x7=Ho7Y_FHiC{H*DTdp-d68gQRMYee98VOb~YI5D%ugK8ptZK0+*oG zUX8BFnO>T`q4QtWeqV1HZ+^S`8&4gW9YN#;U{kQD&mTt9s5IO>wIIEa^Ygp+^r8Op z9)F+h!t~H(akh0rA1)~5;%nTc;S}@kEcU!;xaG^g!=>nzp`{O}AbA}4XVpWpE0&=a z$tVDK7(aH0Y36wWF{5(hk@q`sfF%8a8!c~9f^x(YGg>xG0Q{azqhX5W5;v#Aa?K_DBwE5ea? zm%P63_LG8`M*6tVaVO@9;Mn}~*7-2W0Lrkivgt;HH$yYXwao-vA}A^l5OgB zgSLyyv?!V+bXH8HWq9F`3mxA*-xwY#sU6?!H8uZU=S`aQt!j>f)zd2b&|O-~9mc== z2s_$yW>Qr7IjopAulL%rx?682zSO(4{>9n%2kvPUI5niqwG=!bB|-d2Q&=`P`o|sm z;5mEle!FHLEr84CR5&pk$&nVXjGD!!w)TVK|3)FHjkcUx+_rV}>Q&lv3gl!*4&Mat zn<#n;(d{XgkE;N&xXRQrf(uLNx@0Q-OCRa#-pP8BMOm)da|ahgt#sFkTFv8(LmGZe zwt=2$96R}ccxg7f>~b$b`?r5zaeKO3T5tHET+Rt+Xo;}D3b5hq#y+u`a)|4>nPXp| z4-qyn<^lFN3Ah!Fn60;Us9)KKbSU#o%mZR#k@A6XU8SUP*KkxqBgbBhCeMd`hBXR@ zt8ZC19q#@IzpKF(-E(aZ3)F6BZ+9bh=>A|8pT`qlaV`%gY_piLXH4OD3@WQBNBIlf z53WA_{+NH`nBv^@w?yEK3ds6In1)Zxx`QZ%@R08P8mU@7pSi_Bwhg!IS=+tNv{3x# z!bcul@n^4QF_TTvsheUw8Y>fYZpPwJANu3XF7U;qC(#K(13*oH9EJ7hvy<#5(z_Vp z5|m8=?nIWRG1P&J+d~op9Pg>(P*%Z%M2rIneadj+`n`%wuVU&vi#CacVs&8!?dx;V zPRZYDBfq#Vv9aW>N9X)VbszfGX);RmHsAvo!3`rL=xDFd@k!N0y@#THW;gumdV3Hx z@#!+IOXy>FjPo%4KtaIZ|EV357;%aNruOh{Q`l-sWpSnXt4_=XuCus#`a-&8r_pp2 zGUaS-lUC!l-}szh%D z#s3p<;Czw_Sat8cX^O|?W|ww`%iR7Qg{y~~67P3`A-(sHL)v?)VbMckTgT}Ktj&Pb zZNV<9nwkiH{7!|hnfDjn`faXljk?Zs2!)>z*%M-teYv-WHVZ=xuem^7%WWckhW?uB z(rK8WpfiuocHSjT?2q*|ZQk2XH5=H?Cps}mRVzA;ZCi5Ai?w5+loMp$utKLKG0xh4 zhj7{!n|#eXw_iQi{a-i_{Nt)nD5sR(sz@#emAbF$(n{Dan(N#UEu5;{9}jeM_~td! zuK92t!^K1OjPOY!im(H;KaG!Dw|m_46k1FC3fUz# z)5kCR3G52T1){E2Giyq!a~EZwA<2Bl{Vbw6)zfdzzc(=nN)yR~y{tc{RS z(CpH@KL6jruD^YPTU5iA{7N8XfD>4F$?JG7oqk;%c3(%;<3HYUSj)BhV;`KH3nGNg z?ToJv+qKi#_EEa8oi_pK)}O*|KMHh{$09a!Y_3r4 z_DK?=_wphEeT&CS=CG+?zT%CTXCi%ijFm~;H_I%=74QDrlsovV;2)9axAgv2JYO6~ zA>CG)$%$UDPIvNhh1kP*)Azu;OKz(vzB42G#UM7CsLfRufhg08LfdfQN=Uuq|%lxtq4&^J>5f-^sF|&(csi80QXY6jrhIT zbk!0ki*K#XgFF}oMV5XtR-FRC!R?X>TL8 zsxJ@5KR3{vQ(5)Jul#j^>hAI^y-QeJ%~8xS<~2@On&9%=yn4t@^`TUU7U(N!3iowT zYhf4Zn1z&8rD0+(%FUtlBFTs1S#CYTAuJ}jg(9;-G zN5KXT9)u~?`)-`lF0W7Lf{bos+3|;l0mmN6tC&}nrr%fhE;mVE!H3s znu_T6!hT}9pgvOjsJqaFY(b^5&$*hTN@Lj)kLMZ9n6}|ibve|vsmf#wpkk-eNSxP0 zXaAh+Ah0aX?XkbaJf>;u*T61{VgXj2kVHeA&18)pBs<)c1e+qAyseAOHOuWkg41l? zB|v=p5yzqJF~zq#EzmzqN@ee|N{=u^a>1nQV%V`C4U8ZEHT>5~*Lw}KW8dmA+K^O% z+gkmigGZ)_wD^HVGe#$)*8>w{lYI;NDS};p^9AxB?OC9KF@0PeScMFnpM^E7eA1emUM-=qhuuY z36HWZn?bePw8k?26S8SI)ewztpQ?8^&T|9zgdb~QjUDT8_Ebuy@wBvnsDWnf7eKfj z^y@Oh7&ndq()pGNMg>ly)~>6Vpjg3s)X=zbBfv{Ycg2vT=yPxA3zgaQZM7WTCmIM7 z>TzNysRqT3I{y}radQ($40031W*|Q>6=FrRj_KWKTb1ga`>PC4i?VZJoBlE+5vqM) zb9l(oWC%z(z_xN7CD@(<-e$BCDhkYa%8x81%#dW$-Gu2*EA{IV=k%`qn3CX3Piya< z@0W?`dPPGV6?Bcymrki-o*!DA$zyGz7Y`x70T5NxD4GWi@#Ci_8t?POrpe%Ycr~Kd zHrtFF!0|A7s0n!~$XBuI{2C3zU(;?_mGMAZV_ra}lsSS$NK` zfN&vqAHUto?+d-*Fwaqa;=qv)+eXl)Bj-@^7&X-S&!Kea^64&VqwX%!lR~BU#t2gyh0|L(80uc<7)HI(& zQKxV#aJYL@%B`ABcwNmY_DSWL8QG!BHW z-^cjwFWheI#X zH@9ruO@JMm${B7%Taww)tq`{8pou7OGVprmYt)xa|iS2 za7MQkXh?0W>SVsjTslSat{j-y0!Jj`kDoPea)=6*3vY7P9%4L*HCG;DkP~12GxG%w z+^m%P@Pg=a^(T48dA-5k#Z|QZh#k@V|L{pg=xM)A-EjfV)rt+Nx?3-~hNv-&`km}O zPnXKgrrJ@&5k^?POfJGG9G+Av1da<0WmGWgRd}?W3ai|K5RRp>yOF9>l0hy%N?U~Oc!5m`Mu6h5h zg&(wwHp2N@%n{7^OrC}i)VTM}QKOsfKnrcx4L8-n@w~~>Wgmxq+6}0WOp88@J~#cj z>Vr=sA}P#KRp~3$6@mUI)Fx1iAvQU%7uhGKUZj~eN6=i!J~M`-xdvR2sl~6upEsl= z8y%8XvY>!%Pm%)$r`{)EyRCvV0n%T_8~5u*D2{aDG+{IQKWvVP|dJvt_rTL3>3w|ps%{DvBG zly1$gPg@(e6plS@^mWYYSRB ztE#bsi-ct(z#I`$sVL^Hpe@~1vJ?styc0qhuA57eJz>1|{pyZU3sP^8Ni-unPIr~T z1iT`_cmLPUZLD~G>^yJq1f;O&GP`K;Kojz4*P$O+$eow1n=EU)0~Qb6;AxF8lsc$A zv_8`Tk0KE()6;s6=YK|;?s|^l_i#olY)ICcQOE#?2JYKzL;RbdfA0KiGbTwf9|w%o zAtm6Fb`ecj)99l72a4+1{SZ~yySl(k(mkNbDLaNyS%DUBMIn5E``<^x6@A%5KSbL2 z&KgFG+q*F5R;B3YnT^+OnEP5>)+q@T1B!l0jf|=Cu(zzUT1_v6dT=YvPhD%R=16Py zM>?|d8?+WvhlXVRh)dC%t#|~?)Nu*BEhKY0J7*AYo3HTGMn|n)2n-LD7D;%sPQw$f zasE$=-$YVlUBF{r$LphRyW;BLz1-%tt^1h3KL-wNv1}G?9rlsm@GYX!UpyAR@U93`I7|FB|Ut1Mzr&1}- zNwj}o=+u*;8V#q50q&RPiY)P(3+GRbAH2*fgx&6PhW_ z_GxEzPF?Gb7COZv56R`QTWLT&LX4o=GH5jbnd9KH2}&#Mh4V;W3h^^s@e#ZZh{oRG zj0`)cknsFYCKdmSB4)qezdM}*jOLgc1>0=l2UZqM!?>m?~X;vR$)(%Q0eFr zG$Cz4j&#0Kw=J>NUkt{d7rueGQg@5=e) zC4(eLUQ23*lWf!+#VEI9v<5W5auObyx@0>=2Gm;SK|O|5hYKXCD2b6Z>oJfy?FJ=& zcY;20FuKcGVsaQPT9?cy5~?6Zb5PRC51UATL{a06U%-d#BD1ino@{xn$|==JBNud9 zebZ-*YEcK>#jYAQDZVnfq>{;=fu+2QOLJt4ZC1Z$*|oPeI-L_7EEtZC&=dL0)KO*W zR%eX6MN~{-84OzZ1giRJG})Z>ll;E#dYPGk!9_M^)}aq@)qkpnY~qa-c9KY2)>>lp ze4~ZEow{6(;ik|eD5paowxuhtPF`1QCKWS>cqt`9ce*&r%kjF)$1*++cqg+z4df$! zb>CX!Q}~?CE^4?t-?HgJ9I4?pfqq`?WomE6tl&iy zURexUKEx{MZ7VTTESEIhapQ2D*Dm*uj2Op$s5K`t1kT@v*0e`M_{Vssu7%w$g6s^~ z3+{64DMeS=>ea#+-gxd{*h8B z*-BZu*V~+O&k%^X8Cgt)P*GI5tZ+Jnb!tO8m&WfP%1cY{+$(n#{tBRmPsN`aqQB@bLj&#q2^65n*c%xdkxh&d0&Y-__syYiuvdFHR9n4uSTPjZ73ui9cZoLbM^VVMG~TdF3cE)}my`#bW^glV^Eit|Yhj=wZOzTx?2Es%qpXr#y%>tMK==XuSeOl_RuPeh#0Hl$}O z#%;~YmP`HIQd9`XaNiR41FS5n(O>}~`7qtLR|Fm=QcVqD5x;P^wkY7+jliSt4=Y6m zUY?b`6mL!ccx~a>A`SZCO=wB%DpsSm<3@60E~GdNH%OM2BP2Mtd93OQu;%x2r-Sle zs;ah+1YB1tKXI{K5r#pZUEor4=%xZuu4^2Th6FTQKBhI zlYa9iMM@Mb$vF(R1X==T|%O)uNUK0M=B zmyq(+&w?J{ksVJ%IDK1cnbYAW9r_!5$1&r@#PO1Zw(k;HoE+Dudh)K7(P*{BPA-W& zxn@1~d9D18Rx{NzrFpBTLC!Zw+R26#cm2?yJ1PDMKB$ns%qWE5Qk@*I|yH}Che93!)^f!L5&NGewb7wNa89BUEFwSD*y^fS_#bLy2>&1MX`;@U_!Zr> zb6yqGuo;xU_0x`YgqKqi|1g09>o{fEZL3}Qwhhnu zr=89}{FH5s-j(F<6+e~1wK%-l{+r19ZCPGAi`hODI1L;+mo6+A6r6;+u)*&{9m zHAW5)YvUE4VF0&KTv~)hXYQ?O8fsYUzO(50WEi#Rxnw7NOXL?fpqt0L{TtocHD6WZ z&YU!=c3RV)TplV_bglT2)zsk?f>72iJAdt~WWZ<82RB&L^R^YV*4H20-SG};m}=6b zo-L3Afe589u?s8HeP4r@T zU`FUj6SfdW?#P?1w(chvYQ)9or+uDH($^DSPqZ3q7c{&h=m-`9whSIq(^MzXKEPEv z$oeK|7o4eZST`R}NKa@8531A*>tHX&je*6SbQ#!xiwa%^VF`tw$d9jtZgkK^Q0#Ai Oz~::new(); + let mut elementor = Elementor::new(); + + for event in BgpkitParser::new(&source) + .unwrap() + .into_recovering_record_iter(RecoveryConfig::default()) + { + match event.unwrap() { + RecoveryEvent::Item(record) => { + records += 1; + elements += elementor.record_to_elems(record).len(); + } + RecoveryEvent::Gap(gap) => gaps.push(gap), + } + } + + assert_eq!(records, 2_580); + assert_eq!(elements, 5_378); + assert_eq!(gaps.len(), 2); + assert_eq!(gaps.iter().map(RecoveryGap::skipped_bytes).sum::(), 24); + assert_eq!( + gaps.iter() + .map(|gap| ( + gap.start_offset, + gap.end_offset, + gap.evidence, + gap.confirmed_records, + )) + .collect::>(), + vec![ + (188, 198, RecoveryEvidence::LegacyMrtChain, 3), + (4_554, 4_568, RecoveryEvidence::LegacyMrtChain, 3), + ] + ); +} From 04db8ee3a5bc940927f3ca445eae0463d68d256d Mon Sep 17 00:00:00 2001 From: Ties de Kock Date: Wed, 12 Aug 2026 21:24:01 +0200 Subject: [PATCH 3/5] Incorporate first round of feedback --- src/bin/main.rs | 126 +++++++++++++++-------------------- src/parser/iters/default.rs | 20 +----- src/parser/iters/mod.rs | 24 ++++++- src/parser/iters/recovery.rs | 73 ++++++++++---------- src/parser/mod.rs | 18 +++-- 5 files changed, 129 insertions(+), 132 deletions(-) diff --git a/src/bin/main.rs b/src/bin/main.rs index 3a7b971..6635854 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -254,78 +254,50 @@ fn main() { opts.format }; - if opts.recover { - let filters = parser.filters().to_vec(); - if let Err(error) = run_recovering( - parser, + let filters = parser.filters().to_vec(); + let result = if opts.recover { + run_records( + parser + .into_recovering_record_iter(RecoveryConfig::default()) + .map(|event| event.map_err(|error| error.to_string())), &filters, output_format, opts.level, opts.elems_count, opts.records_count, - ) { - eprintln!("{error}"); - std::process::exit(1); - } - return; - } - - match (opts.elems_count, opts.records_count) { - (true, true) => { - let mut elementor = Elementor::new(); - let (mut records_count, mut elems_count) = (0, 0); - for record in parser.into_record_iter() { - records_count += 1; - elems_count += elementor.record_to_elems(record).len(); - } - println!("total records: {records_count}"); - println!("total elems: {elems_count}"); - } - (false, true) => { - println!("total records: {}", parser.into_record_iter().count()); - } - (true, false) => { - println!("total elems: {}", parser.into_elem_iter().count()); - } - (false, false) => { - let mut stdout = std::io::stdout(); - - match opts.level { - OutputLevel::Elems => { - for (index, elem) in parser.into_elem_iter().enumerate() { - let output_str = format_elem(&elem, output_format, index); - if let Err(e) = writeln!(stdout, "{}", output_str) { - if e.kind() != std::io::ErrorKind::BrokenPipe { - eprintln!("{e}"); - } - std::process::exit(1); - } - } - } - OutputLevel::Records => { - for record in parser.into_record_iter() { - let output_str = format_record(&record, output_format); - if let Err(e) = writeln!(stdout, "{}", output_str) { - if e.kind() != std::io::ErrorKind::BrokenPipe { - eprintln!("{e}"); - } - std::process::exit(1); - } - } - } - } - } + true, + ) + } else { + run_records( + parser + .into_record_iter() + .map(|record| Ok(RecoveryEvent::Item(record))), + &filters, + output_format, + opts.level, + opts.elems_count, + opts.records_count, + false, + ) + }; + if let Err(error) = result { + eprintln!("{error}"); + std::process::exit(1); } } -fn run_recovering( - parser: BgpkitParser, +fn run_records( + events: I, filters: &[Filter], output_format: OutputFormat, output_level: OutputLevel, elems_count_requested: bool, records_count_requested: bool, -) -> Result<(), String> { + report_recovery: bool, +) -> Result<(), String> +where + I: IntoIterator, String>>, +{ let mut stdout = std::io::stdout(); let mut elementor = Elementor::new(); let mut records_count = 0usize; @@ -334,8 +306,8 @@ fn run_recovering( let mut gap_count = 0usize; let mut skipped_bytes = 0u64; - for event in parser.into_recovering_record_iter(RecoveryConfig::default()) { - match event.map_err(|error| error.to_string())? { + for event in events { + match event? { RecoveryEvent::Gap(gap) => { gap_count += 1; skipped_bytes += gap.skipped_bytes(); @@ -354,11 +326,15 @@ fn run_recovering( let needs_elems = elems_count_requested || matches!(output_level, OutputLevel::Elems) && !records_count_requested; let elems = if needs_elems { - elementor - .record_to_elems(record.clone()) - .into_iter() - .filter(|elem| elem.match_filters(filters)) - .collect::>() + let elems = elementor.record_to_elems(record.clone()); + if records_count_requested { + elems + } else { + elems + .into_iter() + .filter(|elem| elem.match_filters(filters)) + .collect() + } } else { Vec::new() }; @@ -372,12 +348,16 @@ fn run_recovering( for elem in elems { let output = format_elem(&elem, output_format, elem_index); elem_index += 1; - write_recovery_output(&mut stdout, &output)?; + if !write_output(&mut stdout, &output)? { + return Ok(()); + } } } OutputLevel::Records => { let output = format_record(&record, output_format); - write_recovery_output(&mut stdout, &output)?; + if !write_output(&mut stdout, &output)? { + return Ok(()); + } } } } @@ -393,18 +373,20 @@ fn run_recovering( (true, false) => println!("total elems: {elems_count}"), (false, false) => {} } - eprintln!("recovery summary: {gap_count} gaps, {skipped_bytes} bytes skipped"); + if report_recovery { + eprintln!("recovery summary: {gap_count} gaps, {skipped_bytes} bytes skipped"); + } Ok(()) } -fn write_recovery_output(stdout: &mut std::io::Stdout, output: &str) -> Result<(), String> { +fn write_output(stdout: &mut std::io::Stdout, output: &str) -> Result { if let Err(error) = writeln!(stdout, "{output}") { if error.kind() == std::io::ErrorKind::BrokenPipe { - return Ok(()); + return Ok(false); } return Err(error.to_string()); } - Ok(()) + Ok(true) } fn format_elem(elem: &BgpElem, format: OutputFormat, index: usize) -> String { diff --git a/src/parser/iters/default.rs b/src/parser/iters/default.rs index a10334c..0edb466 100644 --- a/src/parser/iters/default.rs +++ b/src/parser/iters/default.rs @@ -3,7 +3,7 @@ Default iterator implementations that skip errors and return successfully parsed */ use crate::error::ParserError; use crate::models::*; -use crate::parser::iters::write_mrt_core_dump; +use crate::parser::iters::{record_matches_filters, write_mrt_core_dump}; use crate::parser::BgpkitParser; use crate::{Elementor, Filterable}; use log::{error, warn}; @@ -42,24 +42,10 @@ impl Iterator for RecordIterator { loop { return match self.parser.next_record() { Ok(v) => { - // if None, the reaches EoF. - let filters = &self.parser.filters; - if filters.is_empty() { + if record_matches_filters(&v, &self.parser.filters, &mut self.elementor) { Some(v) } else { - if let MrtMessage::TableDumpV2Message(TableDumpV2Message::PeerIndexTable( - _, - )) = &v.message - { - let _ = self.elementor.record_to_elems(v.clone()); - return Some(v); - } - let elems = self.elementor.record_to_elems(v.clone()); - if elems.iter().any(|e| e.match_filters(&self.parser.filters)) { - Some(v) - } else { - continue; - } + continue; } } Err(e) => { diff --git a/src/parser/iters/mod.rs b/src/parser/iters/mod.rs index 154c94e..10a86df 100644 --- a/src/parser/iters/mod.rs +++ b/src/parser/iters/mod.rs @@ -36,11 +36,33 @@ pub use update::{ use crate::models::BgpElem; use crate::models::{MrtMessage, MrtRecord, TableDumpV2Message}; use crate::parser::BgpkitParser; -use crate::Elementor; use crate::RawMrtRecord; +use crate::{Elementor, Filter, Filterable}; use std::io::Read; use std::path::Path; +#[inline] +pub(crate) fn record_matches_filters( + record: &MrtRecord, + filters: &[Filter], + elementor: &mut Elementor, +) -> bool { + if filters.is_empty() { + return true; + } + if matches!( + &record.message, + MrtMessage::TableDumpV2Message(TableDumpV2Message::PeerIndexTable(_)) + ) { + let _ = elementor.record_to_elems(record.clone()); + return true; + } + elementor + .record_to_elems(record.clone()) + .iter() + .any(|element| element.match_filters(filters)) +} + pub(crate) fn write_mrt_core_dump(enabled: bool, bytes: Option>) { write_mrt_core_dump_to_path(enabled, bytes, "mrt_core_dump"); } diff --git a/src/parser/iters/recovery.rs b/src/parser/iters/recovery.rs index 8a9450d..c9c8d39 100644 --- a/src/parser/iters/recovery.rs +++ b/src/parser/iters/recovery.rs @@ -4,12 +4,12 @@ //! reconstruct a damaged record: bytes are skipped until a conservatively validated chain of //! records is found, and the skipped range is reported as a [`RecoveryEvent::Gap`]. -use crate::models::{Bgp4MpType, EntryType, MrtMessage, MrtRecord, TableDumpV2Message}; +use crate::models::{Bgp4MpType, EntryType, MrtRecord}; +use crate::parser::iters::record_matches_filters; use crate::parser::mrt::messages::bgp4mp::uses_zebra_compat; use crate::parser::mrt::mrt_header::parse_common_header_with_bytes; use crate::parser::mrt::mrt_record::parse_mrt_record_with_zebra_compat; -use crate::parser::{BgpkitParser, Elementor, ParserError, ParserErrorWithBytes}; -use crate::Filterable; +use crate::parser::{BgpkitParser, Elementor, ParserError, ParserErrorWithBytes, ParserOptions}; use bytes::Bytes; use std::fmt::{Display, Formatter}; use std::io::{self, Read}; @@ -130,8 +130,7 @@ pub struct RecoveringRecordIterator { config: RecoveryConfig, filters: Vec, elementor: Elementor, - show_warnings: bool, - warned_zebra_compat: bool, + options: ParserOptions, finished: bool, } @@ -142,8 +141,7 @@ impl RecoveringRecordIterator { config, filters: parser.filters, elementor: Elementor::new(), - show_warnings: parser.options.show_warnings, - warned_zebra_compat: parser.options.warned_zebra_compat, + options: parser.options, finished: false, } } @@ -162,24 +160,11 @@ impl Iterator for RecoveringRecordIterator { match parse_mrt_record_with_zebra_compat(&mut self.reader) { Ok((record, used_zebra_compat)) => { self.reader.discard_before_current(); - if used_zebra_compat && self.show_warnings && !self.warned_zebra_compat { - log::warn!( - "recovered shortened Zebra BGP4MP records with missing envelope fields; substituting IPv4 zero addresses and interface index 0 (further occurrences for this parser will not be logged)" - ); - self.warned_zebra_compat = true; + if used_zebra_compat { + self.options.warn_zebra_compat_once(); } - if self.filters.is_empty() { - return Some(Ok(RecoveryEvent::Item(record))); - } - if let MrtMessage::TableDumpV2Message(TableDumpV2Message::PeerIndexTable(_)) = - &record.message - { - let _ = self.elementor.record_to_elems(record.clone()); - return Some(Ok(RecoveryEvent::Item(record))); - } - let elems = self.elementor.record_to_elems(record.clone()); - if elems.iter().any(|elem| elem.match_filters(&self.filters)) { + if record_matches_filters(&record, &self.filters, &mut self.elementor) { return Some(Ok(RecoveryEvent::Item(record))); } } @@ -204,11 +189,30 @@ impl Iterator for RecoveringRecordIterator { evidence, confirmed_records, }; - self.reader - .move_to(end_offset) - .expect("accepted recovery offset remains buffered"); - self.reader.discard_before_current(); - return Some(Ok(RecoveryEvent::Gap(gap))); + match self.reader.move_to(end_offset) { + Ok(true) => { + self.reader.discard_before_current(); + return Some(Ok(RecoveryEvent::Gap(gap))); + } + Ok(false) => { + self.finished = true; + return Some(Err(RecoveryError { + offset: record_start, + scanned_bytes: end_offset.saturating_sub(record_start), + error, + })); + } + Err(io_error) => { + self.finished = true; + return Some(Err(RecoveryError { + offset: record_start, + scanned_bytes: end_offset.saturating_sub(record_start), + error: ParserErrorWithBytes::from(ParserError::IoError( + io_error, + )), + })); + } + } } Ok(None) => { self.finished = true; @@ -286,7 +290,7 @@ fn validate_chain( let Some(candidate) = read_candidate(reader)? else { reader.move_to(record_start)?; if confirmed > 0 && reader.at_clean_eof()? { - return Ok(Some((evidence.expect("set with first record"), confirmed))); + return Ok(evidence.map(|evidence| (evidence, confirmed))); } return Ok(None); }; @@ -298,14 +302,11 @@ fn validate_chain( confirmed += 1; if confirmed < required && reader.at_clean_eof()? { - return Ok(Some((evidence.expect("set with first record"), confirmed))); + return Ok(evidence.map(|evidence| (evidence, confirmed))); } } - Ok(Some(( - evidence.expect("required is at least one"), - confirmed, - ))) + Ok(evidence.map(|evidence| (evidence, confirmed))) } fn read_candidate(reader: &mut ReplayReader) -> io::Result> { @@ -424,7 +425,7 @@ fn strict_bgp4mp_evidence(raw_record: &crate::RawMrtRecord) -> Option 8, - Bgp4MpType::StateChange | Bgp4MpType::StateChangeAs4 => unreachable!(), + Bgp4MpType::StateChange | Bgp4MpType::StateChangeAs4 => return None, }; let marker_offset = if uses_zebra_compat(raw_record.common_header.entry_subtype, body) { @@ -559,7 +560,7 @@ impl Read for ReplayReader { #[cfg(test)] mod tests { use super::*; - use crate::models::{Asn, Bgp4MpEnum, Bgp4MpMessage, BgpMessage, CommonHeader}; + use crate::models::{Asn, Bgp4MpEnum, Bgp4MpMessage, BgpMessage, CommonHeader, MrtMessage}; use bytes::{BufMut, BytesMut}; use std::io::Cursor; use std::net::{IpAddr, Ipv4Addr}; diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 07a411b..1fc8c4a 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -63,6 +63,17 @@ impl Default for ParserOptions { } } +impl ParserOptions { + pub(crate) fn warn_zebra_compat_once(&mut self) { + if self.show_warnings && !self.warned_zebra_compat { + warn!( + "recovered shortened Zebra BGP4MP records with missing envelope fields; substituting IPv4 zero addresses and interface index 0 (further occurrences for this parser will not be logged)" + ); + self.warned_zebra_compat = true; + } + } +} + #[cfg(feature = "oneio")] impl BgpkitParser> { /// Creating a new parser from a object that implements [Read] trait. @@ -301,12 +312,7 @@ impl BgpkitParser { } pub(crate) fn warn_zebra_compat_once(&mut self) { - if self.options.show_warnings && !self.options.warned_zebra_compat { - warn!( - "recovered shortened Zebra BGP4MP records with missing envelope fields; substituting IPv4 zero addresses and interface index 0 (further occurrences for this parser will not be logged)" - ); - self.options.warned_zebra_compat = true; - } + self.options.warn_zebra_compat_once(); } pub fn enable_core_dump(self) -> Self { From 5a86123855d4c7b7a5e87e1b5bba10ee907f8374 Mon Sep 17 00:00:00 2001 From: Ties de Kock Date: Wed, 12 Aug 2026 21:40:07 +0200 Subject: [PATCH 4/5] Github review comment --- src/parser/iters/recovery.rs | 42 +++--- tests/torn_mrt_records.rs | 270 +++++++++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+), 23 deletions(-) create mode 100644 tests/torn_mrt_records.rs diff --git a/src/parser/iters/recovery.rs b/src/parser/iters/recovery.rs index c9c8d39..d9437a8 100644 --- a/src/parser/iters/recovery.rs +++ b/src/parser/iters/recovery.rs @@ -88,8 +88,9 @@ pub struct RecoveryGap { } impl RecoveryGap { + /// Return the number of skipped bytes, or zero for an invalid inverted range. pub const fn skipped_bytes(&self) -> u64 { - self.end_offset - self.start_offset + self.end_offset.saturating_sub(self.start_offset) } } @@ -289,9 +290,6 @@ fn validate_chain( let record_start = reader.position(); let Some(candidate) = read_candidate(reader)? else { reader.move_to(record_start)?; - if confirmed > 0 && reader.at_clean_eof()? { - return Ok(evidence.map(|evidence| (evidence, confirmed))); - } return Ok(None); }; if family.is_some_and(|expected| expected != candidate.family) { @@ -300,10 +298,6 @@ fn validate_chain( family.get_or_insert(candidate.family); evidence.get_or_insert(candidate.evidence); confirmed += 1; - - if confirmed < required && reader.at_clean_eof()? { - return Ok(evidence.map(|evidence| (evidence, confirmed))); - } } Ok(evidence.map(|evidence| (evidence, confirmed))) @@ -528,14 +522,6 @@ impl ReplayReader { self.cursor = self.start + (offset - self.base_offset) as usize; Ok(true) } - - fn at_clean_eof(&mut self) -> io::Result { - let position = self.position(); - let mut byte = [0u8; 1]; - let read = self.read(&mut byte)?; - self.move_to(position)?; - Ok(read == 0) - } } impl Read for ReplayReader { @@ -647,20 +633,30 @@ mod tests { } #[test] - fn accepts_short_confirmation_chain_at_clean_eof() { + fn rejects_chain_shorter_than_configured_at_clean_eof() { let mut input = vec![0xff; 12]; input.extend_from_slice(&legacy_state(101)); input.extend_from_slice(&legacy_state(102)); let parser = BgpkitParser::from_reader(Cursor::new(input)); - let events = parser + let result = parser .into_recovering_record_iter(RecoveryConfig::default()) - .collect::, _>>() - .unwrap(); - let RecoveryEvent::Gap(gap) = &events[0] else { - panic!("expected gap") + .next() + .expect("one error"); + assert!(result.is_err()); + } + + #[test] + fn skipped_bytes_is_zero_for_inverted_range() { + let gap = RecoveryGap { + start_offset: 10, + end_offset: 5, + cause: String::new(), + evidence: RecoveryEvidence::LegacyMrtChain, + confirmed_records: 3, }; - assert_eq!(gap.confirmed_records, 2); + + assert_eq!(gap.skipped_bytes(), 0); } #[test] diff --git a/tests/torn_mrt_records.rs b/tests/torn_mrt_records.rs new file mode 100644 index 0000000..d474286 --- /dev/null +++ b/tests/torn_mrt_records.rs @@ -0,0 +1,270 @@ +//! Regression coverage for torn MRT writes in the early RIPE RIS archive. +//! +//! `rrc00/2000.03/updates.20000325.0345.gz` is physically corrupt: the historical +//! collector emitted two short writes into an otherwise well-formed stream. The +//! damage is confined to the first ~4.6 KiB of the 151,804-byte uncompressed body, +//! but the parser cannot resynchronize past either one, so it recovers 4 of the +//! file's 2,580 records. +//! +//! **Offset 188 — a 10-byte truncated record header:** +//! +//! ```text +//! 38 dc 36 68 00 05 00 01 00 00 | 38 dc 36 71 00 05 00 03 ... +//! ^timestamp ^BGP ^UPDATE ^only 2 of the 4 Length bytes +//! 953955944 (5) (1) ^next record starts here +//! ``` +//! +//! The writer stopped after 10 of the 12 header bytes. The parser therefore reads +//! `Length` as the four bytes `00 00 38 dc`, whose low half is stolen from the next +//! record's timestamp (`0x38dc3671`), giving 14,556. It consumes 14,556 bytes — +//! roughly 200 real records — and fails with [`ParserError::TruncatedMsg`], leaving +//! the reader at offset 14,756, which is not a record boundary. +//! +//! **Offset 4554 — 14 orphan body bytes:** +//! +//! ```text +//! cb 2d 13 | 40 03 04 cb 25 ff 7e | 18 d1 f7 aa +//! ^attr tail ^NEXT_HOP 203.37.255.126 ^NLRI 209.247.170.0/24 +//! ``` +//! +//! The tail of a BGP UPDATE body left over after a complete STATE_CHANGE record. +//! Timestamps run backwards across this point (953956818 before, 953955969 after), +//! consistent with interleaved buffer flushes. +//! +//! Deleting those 24 bytes makes the file parse cleanly — 2,580 records (2,239 +//! UPDATE, 307 STATE_CHANGE, 34 KEEPALIVE), 5,378 elements, zero errors — so there +//! is no body-level parsing defect here. What is missing is stream +//! resynchronization: on a framing error the reader has no pushback and cannot scan +//! forward for the next plausible common header, so every iterator either retries +//! at the same wrong offset or stops. +//! +//! The assertions below therefore pin **current, lossy** behaviour rather than +//! desired behaviour. Every count is a deterministic function of the fixture bytes. +//! If a change adds resynchronization, these numbers are expected to rise to the +//! recoverable totals quoted above. +//! +//! The same signature appears in every neighbouring rrc00 2000.03 file — exactly two +//! torn writes, one in the first ~200 bytes and one just past 4 KiB — so this is a +//! property of the historical writer, not of this one file. + +use bgpkit_parser::models::EntryType; +use bgpkit_parser::{BgpkitParser, DiagnosticEvent, MrtUpdate, ParserError}; +use std::collections::BTreeMap; + +const FIXTURE: &str = "tests/fixtures/ripe/rrc00/2000.03/updates.20000325.0345.gz"; + +/// Records ahead of the first torn write, all of which parse normally. The fixture +/// holds 2,580 recoverable records and 5,378 recoverable elements, so all but these +/// four are lost. +const RECORDS_BEFORE_FIRST_TORN_WRITE: usize = 4; +/// Errors produced while the parser hunts forward 12 bytes at a time after +/// desynchronizing. It never re-aligns, because the true record boundary is not a +/// multiple of 12 from where the oversized read left it. +const DESYNCHRONIZED_ERRORS: usize = 128; + +/// Header the parser manufactures from the 10-byte torn write at offset 188. +const TORN_HEADER_TIMESTAMP: u32 = 953_955_944; +const TORN_HEADER_LENGTH: u32 = 14_556; + +fn repo_fixture(path: &str) -> String { + format!("{}/{path}", env!("CARGO_MANIFEST_DIR")) +} + +#[test] +fn torn_writes_desynchronize_raw_record_framing() { + let source = repo_fixture(FIXTURE); + let mut raw_records = 0usize; + let mut headers = BTreeMap::<(u16, u16), usize>::new(); + + for raw_record in BgpkitParser::new(&source).unwrap().into_raw_record_iter() { + raw_records += 1; + *headers + .entry(( + raw_record.common_header.entry_type as u16, + raw_record.common_header.entry_subtype, + )) + .or_default() += 1; + } + + // Only 5 of the fixture's 2,580 records reach framing — 4 genuine ones plus the + // 14,556-byte torn "record" — after which the parser frames 3 more headers out of + // misaligned bytes. + assert_eq!(raw_records, 8); + assert_eq!( + headers, + BTreeMap::from([ + ( + (EntryType::NULL as u16, 256), + 3 // pure garbage, framed mid-record after desynchronizing + ), + ( + (EntryType::BGP as u16, 1), + 4 // 3 genuine UPDATEs plus the torn 14,556-byte "record" + ), + ((EntryType::BGP as u16, 7), 1), + ]) + ); +} + +#[test] +fn torn_writes_lose_all_but_four_records() { + let source = repo_fixture(FIXTURE); + + // Skipping iterators: the error arms can only retry at the same wrong offset or + // stop, so 2,576 of 2,580 records and 5,374 of 5,378 elements are lost. + assert_eq!( + BgpkitParser::new(&source) + .unwrap() + .into_record_iter() + .count(), + RECORDS_BEFORE_FIRST_TORN_WRITE + ); + assert_eq!( + BgpkitParser::new(&source).unwrap().into_elem_iter().count(), + RECORDS_BEFORE_FIRST_TORN_WRITE + ); + assert_eq!( + BgpkitParser::new(&source) + .unwrap() + .into_route_iter() + .count(), + RECORDS_BEFORE_FIRST_TORN_WRITE + ); + // One of the four surviving records is a KEEPALIVE, so the update iterator sees + // one fewer. + assert_eq!( + BgpkitParser::new(&source) + .unwrap() + .into_update_iter() + .count(), + RECORDS_BEFORE_FIRST_TORN_WRITE - 1 + ); + + // Fallible iterators surface the same loss as errors instead of silence. + let mut records = 0usize; + let mut record_errors = 0usize; + for result in BgpkitParser::new(&source) + .unwrap() + .into_fallible_record_iter() + { + match result { + Ok(_) => records += 1, + Err(_) => record_errors += 1, + } + } + assert_eq!(records, RECORDS_BEFORE_FIRST_TORN_WRITE); + assert_eq!(record_errors, DESYNCHRONIZED_ERRORS); + + let mut elements = 0usize; + let mut element_errors = 0usize; + for result in BgpkitParser::new(&source) + .unwrap() + .into_fallible_elem_iter() + { + match result { + Ok(_) => elements += 1, + Err(_) => element_errors += 1, + } + } + assert_eq!(elements, RECORDS_BEFORE_FIRST_TORN_WRITE); + assert_eq!(element_errors, DESYNCHRONIZED_ERRORS); + + let mut updates = 0usize; + let mut update_errors = 0usize; + for result in BgpkitParser::new(&source) + .unwrap() + .into_fallible_update_iter() + { + match result { + Ok(MrtUpdate::LegacyBgpUpdate(_)) => updates += 1, + Ok(update) => panic!("unexpected MRT update: {update:?}"), + Err(_) => update_errors += 1, + } + } + assert_eq!(updates, RECORDS_BEFORE_FIRST_TORN_WRITE - 1); + assert_eq!(update_errors, DESYNCHRONIZED_ERRORS); + + let mut routes = 0usize; + let mut route_errors = 0usize; + for result in BgpkitParser::new(&source) + .unwrap() + .into_fallible_route_iter() + { + match result { + Ok(_) => routes += 1, + Err(_) => route_errors += 1, + } + } + assert_eq!(routes, RECORDS_BEFORE_FIRST_TORN_WRITE); + assert_eq!(route_errors, DESYNCHRONIZED_ERRORS); +} + +#[test] +fn diagnostic_iterator_reports_both_torn_writes() { + let source = repo_fixture(FIXTURE); + let events: Vec = BgpkitParser::new(&source) + .unwrap() + .into_diagnostic_iter() + .collect(); + + // Four clean records, then one body error, then one framing error which + // terminates the iterator by design. + assert_eq!(events.len(), RECORDS_BEFORE_FIRST_TORN_WRITE + 2); + for event in &events[..RECORDS_BEFORE_FIRST_TORN_WRITE] { + let DiagnosticEvent::Record(record) = event else { + panic!("expected a clean record, got {event:?}"); + }; + assert_eq!(record.common_header.entry_type, EntryType::BGP); + } + + // The 14,556-byte body read fails, but the manufactured header itself framed, so + // the iterator reports it and continues. + let DiagnosticEvent::ParseError { + error, + common_header, + raw_bytes, + } = &events[RECORDS_BEFORE_FIRST_TORN_WRITE] + else { + panic!( + "expected a parse error for the torn header, got {:?}", + events[RECORDS_BEFORE_FIRST_TORN_WRITE] + ); + }; + let ParserError::TruncatedMsg(message) = error else { + panic!("expected TruncatedMsg, got {error:?}"); + }; + assert_eq!( + message, + "not enough bytes to read. remaining: 14542, required: 33274" + ); + let header = common_header.expect("the torn header parsed, so it is retained"); + assert_eq!(header.timestamp, TORN_HEADER_TIMESTAMP); + assert_eq!(header.entry_type, EntryType::BGP); + assert_eq!(header.entry_subtype, 1); + assert_eq!(header.length, TORN_HEADER_LENGTH); + assert_eq!( + raw_bytes.as_deref().map(<[u8]>::len), + Some(12 + TORN_HEADER_LENGTH as usize) + ); + + // The oversized read left the stream mid-record, so the next 12 bytes are not a + // header. There is no header to report, and the framing error terminates + // iteration rather than reinterpreting the remaining bytes. + let DiagnosticEvent::ParseError { + error, + common_header, + raw_bytes, + } = &events[RECORDS_BEFORE_FIRST_TORN_WRITE + 1] + else { + panic!( + "expected a framing error after desynchronizing, got {:?}", + events[RECORDS_BEFORE_FIRST_TORN_WRITE + 1] + ); + }; + let ParserError::ParseError(message) = error else { + panic!("expected ParseError, got {error:?}"); + }; + assert_eq!(message, "cannot parse entry type: 32305"); + assert!(common_header.is_none()); + assert_eq!(raw_bytes.as_deref().map(<[u8]>::len), Some(12)); +} From e2b94bfbf6926164cb2ac32350ea9eb90e517b04 Mon Sep 17 00:00:00 2001 From: Ties de Kock Date: Wed, 12 Aug 2026 22:51:37 +0200 Subject: [PATCH 5/5] Incorporate more feedback * Damage up to EOF is now a gap, not a hard error * Text-dumps cause Unsupported error (instead of zero events) * Removes copies from clean path and replaces reader * Handle framed-but-unparseable records differently from broken framing (incorrect length) --- CHANGELOG.md | 3 +- Cargo.toml | 1 + README.md | 5 +- src/bin/main.rs | 205 ++++++---- src/parser/iters/mod.rs | 17 +- src/parser/iters/recovery.rs | 706 +++++++++++++++++++++++++++++------ src/parser/mod.rs | 5 - tests/recovery.rs | 69 +++- 8 files changed, 793 insertions(+), 218 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f159ff..d8e97c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,7 +49,7 @@ All notable changes to this project will be documented in this file. #### Test data * **Historical RIPE regression fixtures**: Added original RRC00 update and bview gzip files from 1999 and January 2000 as repository-only, offline integration fixtures. -* **Opt-in MRT framing recovery**: Added `into_recovering_record_iter` and CLI `--recover` support for salvaging records after damaged legacy Type-5 or BGP4MP framing. Recovery validates a three-record chain, uses exact embedded BGP headers as BGP4MP anchors, and reports every skipped decompressed byte range as a typed gap event. +* **Opt-in MRT framing recovery**: Added `into_recovering_record_iter`, `into_recovering_elem_iter`, and CLI `--recover` support for salvaging records after damaged MRT framing. Recovery validates a three-record chain, uses exact embedded BGP headers as BGP4MP anchors, and reports every skipped decompressed byte range as a typed gap event. A record of any MRT type whose framing is intact but whose body fails to parse is skipped exactly (`AlignedRecordChain` evidence) when intact records follow its declared boundary, and damage extending to the end of the stream — e.g. a truncated final record — is reported as a terminal gap (`EndOfStream` evidence) instead of an error. Text-dump parsers, which have no MRT record representation, yield an explicit `Unsupported` error. #### Examples @@ -67,6 +67,7 @@ All notable changes to this project will be documented in this file. * **BGP OPEN parameter type 255 rejected**: RFC 9072 reserves type 255 as the extended-length marker; encoding it as a real parameter produced output that round-tripped to a structurally different message. * **BGP OPEN optional-parameter encoding**: Encode the Optional Parameters Length as the total byte length required by RFC 4271 instead of the number of parameters. OPEN messages now also use the extended length format from RFC 9072 when requested or required. * **Historical Quagga state changes**: Parse BGP4MP state-change records containing Quagga's `Clearing` (7) and `Deleted` (8) FSM states instead of logging an error and dropping the MRT record. +* **CLI broken-pipe exit status**: the CLI now exits 0 when its stdout consumer closes the pipe early (e.g. `bgpkit-parser updates.gz | head`), matching Unix convention; previous releases exited 1. Other write errors still exit 1. ### Contributors diff --git a/Cargo.toml b/Cargo.toml index 9208107..72b818e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ exclude = [ "tests/legacy_ripe_2000.rs", "tests/quagga_bgp_states.rs", "tests/recovery.rs", + "tests/torn_mrt_records.rs", ] [[bin]] diff --git a/README.md b/README.md index 070c63c..728fe1e 100644 --- a/README.md +++ b/README.md @@ -249,7 +249,10 @@ match process_mrt_file("http://example.com/updates.bz2") { **Recovering After Damaged MRT Framing** Recovery is opt-in and never reconstructs a damaged record. It reports the skipped decompressed -byte range before resuming at a conservatively validated record chain. +byte range before resuming at a conservatively validated record chain. A correctly framed record +whose body fails to parse is skipped exactly, and damage extending to the end of the stream +(e.g. a truncated final record) is reported as a terminal gap rather than an error. Use +`into_recovering_elem_iter` for the same events at the `BgpElem` level. ```rust use bgpkit_parser::{BgpkitParser, RecoveryConfig, RecoveryEvent}; diff --git a/src/bin/main.rs b/src/bin/main.rs index 6635854..e5b597f 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -4,9 +4,7 @@ use std::io::Write; use std::net::IpAddr; use std::path::PathBuf; -use bgpkit_parser::{ - BgpElem, BgpkitParser, Elementor, Filter, Filterable, RecoveryConfig, RecoveryEvent, -}; +use bgpkit_parser::{BgpElem, BgpkitParser, Elementor, RecoveryConfig, RecoveryEvent, RecoveryGap}; use clap::{Parser, ValueEnum}; use ipnet::IpNet; @@ -254,31 +252,48 @@ fn main() { opts.format }; - let filters = parser.filters().to_vec(); - let result = if opts.recover { - run_records( + let recovery_config = RecoveryConfig::default(); + // Element-level runs (element output or counting only elements) use the elem + // iterators, which apply filters per element; everything else stays at the record + // level. Counting both (-e -r) iterates records and converts once per record. + let use_elem_stream = (opts.elems_count && !opts.records_count) + || (!opts.elems_count && !opts.records_count && matches!(opts.level, OutputLevel::Elems)); + + let result = match (opts.recover, use_elem_stream) { + (true, true) => run_elems( + parser + .into_recovering_elem_iter(recovery_config) + .map(|event| event.map_err(|error| error.to_string())), + output_format, + opts.elems_count, + true, + ), + (false, true) => run_elems( + parser + .into_elem_iter() + .map(|elem| Ok(RecoveryEvent::Item(elem))), + output_format, + opts.elems_count, + false, + ), + (true, false) => run_records( parser - .into_recovering_record_iter(RecoveryConfig::default()) + .into_recovering_record_iter(recovery_config) .map(|event| event.map_err(|error| error.to_string())), - &filters, output_format, - opts.level, opts.elems_count, opts.records_count, true, - ) - } else { - run_records( + ), + (false, false) => run_records( parser .into_record_iter() .map(|record| Ok(RecoveryEvent::Item(record))), - &filters, output_format, - opts.level, opts.elems_count, opts.records_count, false, - ) + ), }; if let Err(error) = result { eprintln!("{error}"); @@ -286,11 +301,92 @@ fn main() { } } +/// Per-gap reporting and end-of-run summary for `--recover`. +struct RecoveryStats { + enabled: bool, + gap_count: usize, + skipped_bytes: u64, +} + +impl RecoveryStats { + fn new(enabled: bool) -> Self { + Self { + enabled, + gap_count: 0, + skipped_bytes: 0, + } + } + + fn observe(&mut self, gap: &RecoveryGap) { + self.gap_count += 1; + self.skipped_bytes += gap.skipped_bytes(); + eprintln!( + "recovered MRT framing: skipped bytes {}..{} ({} bytes, {:?}, {} confirming records): {}", + gap.start_offset, + gap.end_offset, + gap.skipped_bytes(), + gap.evidence, + gap.confirmed_records, + gap.cause + ); + } + + fn print_summary(&self) { + if self.enabled { + eprintln!( + "recovery summary: {} gaps, {} bytes skipped", + self.gap_count, self.skipped_bytes + ); + } + } +} + +fn run_elems( + events: I, + output_format: OutputFormat, + count_requested: bool, + report_recovery: bool, +) -> Result<(), String> +where + I: IntoIterator, String>>, +{ + let mut stdout = std::io::stdout(); + let mut elems_count = 0usize; + let mut elem_index = 0usize; + let mut stats = RecoveryStats::new(report_recovery); + let mut terminal_error = None; + + for event in events { + match event { + Err(error) => { + terminal_error = Some(error); + break; + } + Ok(RecoveryEvent::Gap(gap)) => stats.observe(&gap), + Ok(RecoveryEvent::Item(elem)) => { + elems_count += 1; + if count_requested { + continue; + } + let output = format_elem(&elem, output_format, elem_index); + elem_index += 1; + if !write_output(&mut stdout, &output)? { + return Ok(()); + } + } + } + } + + if count_requested { + println!("total elems: {elems_count}"); + } + stats.print_summary(); + terminal_error.map_or(Ok(()), Err) +} + fn run_records( events: I, - filters: &[Filter], output_format: OutputFormat, - output_level: OutputLevel, elems_count_requested: bool, records_count_requested: bool, report_recovery: bool, @@ -302,63 +398,30 @@ where let mut elementor = Elementor::new(); let mut records_count = 0usize; let mut elems_count = 0usize; - let mut elem_index = 0usize; - let mut gap_count = 0usize; - let mut skipped_bytes = 0u64; + let mut stats = RecoveryStats::new(report_recovery); + let mut terminal_error = None; for event in events { - match event? { - RecoveryEvent::Gap(gap) => { - gap_count += 1; - skipped_bytes += gap.skipped_bytes(); - eprintln!( - "recovered MRT framing: skipped bytes {}..{} ({} bytes, {:?}, {} confirming records): {}", - gap.start_offset, - gap.end_offset, - gap.skipped_bytes(), - gap.evidence, - gap.confirmed_records, - gap.cause - ); + match event { + Err(error) => { + terminal_error = Some(error); + break; } - RecoveryEvent::Item(record) => { + Ok(RecoveryEvent::Gap(gap)) => stats.observe(&gap), + Ok(RecoveryEvent::Item(record)) => { records_count += 1; - let needs_elems = elems_count_requested - || matches!(output_level, OutputLevel::Elems) && !records_count_requested; - let elems = if needs_elems { - let elems = elementor.record_to_elems(record.clone()); - if records_count_requested { - elems - } else { - elems - .into_iter() - .filter(|elem| elem.match_filters(filters)) - .collect() - } - } else { - Vec::new() - }; - elems_count += elems.len(); - - if elems_count_requested || records_count_requested { + if elems_count_requested { + // Counting both (-e -r): count every element of records that passed + // record-level filtering, matching the historical CLI behavior. + elems_count += elementor.record_to_elems(record).len(); continue; } - match output_level { - OutputLevel::Elems => { - for elem in elems { - let output = format_elem(&elem, output_format, elem_index); - elem_index += 1; - if !write_output(&mut stdout, &output)? { - return Ok(()); - } - } - } - OutputLevel::Records => { - let output = format_record(&record, output_format); - if !write_output(&mut stdout, &output)? { - return Ok(()); - } - } + if records_count_requested { + continue; + } + let output = format_record(&record, output_format); + if !write_output(&mut stdout, &output)? { + return Ok(()); } } } @@ -373,10 +436,8 @@ where (true, false) => println!("total elems: {elems_count}"), (false, false) => {} } - if report_recovery { - eprintln!("recovery summary: {gap_count} gaps, {skipped_bytes} bytes skipped"); - } - Ok(()) + stats.print_summary(); + terminal_error.map_or(Ok(()), Err) } fn write_output(stdout: &mut std::io::Stdout, output: &str) -> Result { diff --git a/src/parser/iters/mod.rs b/src/parser/iters/mod.rs index 10a86df..f58afad 100644 --- a/src/parser/iters/mod.rs +++ b/src/parser/iters/mod.rs @@ -24,8 +24,8 @@ pub use diagnostic::{DiagnosticEvent, DiagnosticIterator}; pub use fallible::{FallibleElemIterator, FallibleRecordIterator}; pub use raw::RawRecordIterator; pub use recovery::{ - RecoveringRecordIterator, RecoveryConfig, RecoveryError, RecoveryEvent, RecoveryEvidence, - RecoveryGap, + RecoveringElemIterator, RecoveringRecordIterator, RecoveryConfig, RecoveryError, RecoveryEvent, + RecoveryEvidence, RecoveryGap, }; pub use route::{FallibleRouteIterator, RouteIterator}; pub use update::{ @@ -106,7 +106,8 @@ impl BgpkitParser { /// /// Recovery never reconstructs a damaged record. It scans for a structurally valid boundary, /// confirms a chain of records, emits [`RecoveryEvent::Gap`], and then resumes normal parsing. - /// Offsets in recovery events refer to the decompressed MRT byte stream. + /// Damage extending to the end of the stream is reported as a terminal gap. Offsets in + /// recovery events refer to the decompressed MRT byte stream. pub fn into_recovering_record_iter( self, config: RecoveryConfig, @@ -114,6 +115,16 @@ impl BgpkitParser { RecoveringRecordIterator::new(self, config) } + /// Creates an opt-in iterator over BGP elements that reports skipped byte ranges while + /// recovering MRT framing. + /// + /// Behaves like [`into_recovering_record_iter`](Self::into_recovering_record_iter) but + /// converts each recovered record to [`BgpElem`]s, applying the parser's filters per + /// element. + pub fn into_recovering_elem_iter(self, config: RecoveryConfig) -> RecoveringElemIterator { + RecoveringElemIterator::new(self, config) + } + /// Creates an iterator over BGP announcements from MRT data. /// /// This iterator yields `MrtUpdate` items from both UPDATES files (BGP4MP messages) diff --git a/src/parser/iters/recovery.rs b/src/parser/iters/recovery.rs index d9437a8..c839217 100644 --- a/src/parser/iters/recovery.rs +++ b/src/parser/iters/recovery.rs @@ -3,13 +3,30 @@ //! Recovery is deliberately separate from the default iterators. It never attempts to //! reconstruct a damaged record: bytes are skipped until a conservatively validated chain of //! records is found, and the skipped range is reported as a [`RecoveryEvent::Gap`]. +//! +//! Damage is classified before scanning. When a record frames correctly — its header and +//! declared length were consumed exactly — but its body fails to parse, and intact records +//! (or a clean end of stream) follow at the declared boundary, exactly that record is +//! skipped without scanning. Damage that extends to the end of the stream is reported as a +//! terminal gap rather than an error, so trailing truncation — the most common real-world +//! corruption — still yields every intact record plus an explicit account of the discarded +//! tail. A [`RecoveryError`] is reserved for I/O failures, unsupported input, and scan +//! windows exhausted without finding a boundary mid-stream. +//! +//! The undamaged fast path reads straight from the underlying reader; bytes are only +//! buffered while a recovery scan is in progress. -use crate::models::{Bgp4MpType, EntryType, MrtRecord}; -use crate::parser::iters::record_matches_filters; +use crate::models::{Bgp4MpType, BgpElem, EntryType, MrtRecord}; +use crate::parser::iters::{record_matches_filters, write_mrt_core_dump}; use crate::parser::mrt::messages::bgp4mp::uses_zebra_compat; use crate::parser::mrt::mrt_header::parse_common_header_with_bytes; -use crate::parser::mrt::mrt_record::parse_mrt_record_with_zebra_compat; -use crate::parser::{BgpkitParser, Elementor, ParserError, ParserErrorWithBytes, ParserOptions}; +use crate::parser::mrt::mrt_record::{ + chunk_mrt_record, parse_mrt_record_with_zebra_compat, raw_record_uses_zebra_compat, +}; +use crate::parser::{ + BgpkitParser, Elementor, Filter, ParserError, ParserErrorWithBytes, ParserOptions, +}; +use crate::Filterable; use bytes::Bytes; use std::fmt::{Display, Formatter}; use std::io::{self, Read}; @@ -17,6 +34,7 @@ use std::io::{self, Read}; const DEFAULT_MAX_SCAN_BYTES: usize = 1024 * 1024; const DEFAULT_CONFIRMATION_RECORDS: u8 = 3; const MAX_RECOVERY_RECORD_LEN: u32 = 65_599; +const SCAN_FILL_CHUNK: usize = 8_192; /// Settings for opt-in MRT framing recovery. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -69,6 +87,13 @@ pub enum RecoveryEvidence { BgpMarkerChain, /// A BGP4MP state-change record, which has no embedded BGP message header. Bgp4MpStateChangeChain, + /// The damaged record's framing was intact: complete MRT records of any type (or a + /// clean end of stream) followed at its declared end offset, so exactly that record + /// was skipped without scanning. + AlignedRecordChain, + /// No boundary was validated before the stream ended; the gap extends to the end of + /// the input. + EndOfStream, } /// A byte range discarded while restoring MRT record framing. @@ -83,7 +108,8 @@ pub struct RecoveryGap { pub cause: String, /// Structural evidence used to accept `end_offset` as a new boundary. pub evidence: RecoveryEvidence, - /// Number of consecutive records validated at the recovered boundary. + /// Number of consecutive records validated at the recovered boundary. Zero when the + /// gap ends at the end of the stream. pub confirmed_records: u8, } @@ -101,7 +127,11 @@ pub enum RecoveryEvent { Gap(RecoveryGap), } -/// A framing error for which no sufficiently strong recovery boundary was found. +/// A framing error for which no recovery boundary was found within the scan window, an +/// I/O failure, or unsupported input. +/// +/// Damage that extends to the end of the stream is reported as a terminal +/// [`RecoveryEvent::Gap`] instead of this error. #[derive(Debug)] pub struct RecoveryError { pub offset: u64, @@ -127,22 +157,30 @@ impl std::error::Error for RecoveryError { /// Iterator over parsed MRT records and explicit recovery gaps. pub struct RecoveringRecordIterator { - reader: ReplayReader, + reader: CarryoverReader, config: RecoveryConfig, - filters: Vec, + filters: Vec, elementor: Elementor, options: ParserOptions, + core_dump: bool, + unsupported_input: Option, finished: bool, } impl RecoveringRecordIterator { pub(crate) fn new(parser: BgpkitParser, config: RecoveryConfig) -> Self { + let unsupported_input = parser.text_dump_iter.is_some().then(|| { + "text-dump parsers have no MRT record representation; iterate elements instead" + .to_string() + }); Self { - reader: ReplayReader::new(parser.reader), + reader: CarryoverReader::new(parser.reader), config, filters: parser.filters, elementor: Elementor::new(), options: parser.options, + core_dump: parser.core_dump, + unsupported_input, finished: false, } } @@ -155,20 +193,19 @@ impl Iterator for RecoveringRecordIterator { if self.finished { return None; } + if let Some(message) = self.unsupported_input.take() { + self.finished = true; + return Some(Err(RecoveryError { + offset: 0, + scanned_bytes: 0, + error: ParserErrorWithBytes::from(ParserError::Unsupported(message)), + })); + } loop { let record_start = self.reader.position(); - match parse_mrt_record_with_zebra_compat(&mut self.reader) { - Ok((record, used_zebra_compat)) => { - self.reader.discard_before_current(); - if used_zebra_compat { - self.options.warn_zebra_compat_once(); - } - - if record_matches_filters(&record, &self.filters, &mut self.elementor) { - return Some(Ok(RecoveryEvent::Item(record))); - } - } + let raw_record = match chunk_mrt_record(&mut self.reader) { + Ok(raw_record) => raw_record, Err(error) if matches!(error.error, ParserError::EofExpected) => { self.finished = true; return None; @@ -181,85 +218,163 @@ impl Iterator for RecoveringRecordIterator { error, })); } - Err(error) => match self.find_recovery(record_start, &error) { - Ok(Some((end_offset, evidence, confirmed_records))) => { - let gap = RecoveryGap { - start_offset: record_start, - end_offset, - cause: error.to_string(), - evidence, - confirmed_records, - }; - match self.reader.move_to(end_offset) { - Ok(true) => { - self.reader.discard_before_current(); - return Some(Ok(RecoveryEvent::Gap(gap))); - } - Ok(false) => { - self.finished = true; - return Some(Err(RecoveryError { - offset: record_start, - scanned_bytes: end_offset.saturating_sub(record_start), - error, - })); - } - Err(io_error) => { - self.finished = true; - return Some(Err(RecoveryError { - offset: record_start, - scanned_bytes: end_offset.saturating_sub(record_start), - error: ParserErrorWithBytes::from(ParserError::IoError( - io_error, - )), - })); - } - } - } - Ok(None) => { - self.finished = true; - return Some(Err(RecoveryError { - offset: record_start, - scanned_bytes: self - .reader - .buffered_end() - .saturating_sub(record_start) - .min(self.config.max_scan_bytes as u64), - error, - })); + Err(error) => return self.recover(record_start, None, error), + }; + + let used_zebra_compat = raw_record_uses_zebra_compat(&raw_record); + match raw_record.clone().parse() { + Ok(record) => { + if used_zebra_compat { + self.options.warn_zebra_compat_once(); } - Err(io_error) => { - self.finished = true; - return Some(Err(RecoveryError { - offset: record_start, - scanned_bytes: self.reader.position().saturating_sub(record_start), - error: ParserErrorWithBytes::from(ParserError::IoError(io_error)), - })); + if record_matches_filters(&record, &self.filters, &mut self.elementor) { + return Some(Ok(RecoveryEvent::Item(record))); } - }, + } + Err(error) => { + // The header and declared length were consumed exactly, so the + // stream may still be aligned even though the body is unparsable. + let framed_end = self.reader.position(); + let error = ParserErrorWithBytes { + error, + bytes: Some(raw_record.raw_bytes().to_vec()), + }; + return self.recover(record_start, Some(framed_end), error); + } } } } } impl RecoveringRecordIterator { - fn find_recovery( + fn recover( &mut self, - failed_start: u64, - _error: &ParserErrorWithBytes, - ) -> io::Result> { + record_start: u64, + framed_end: Option, + error: ParserErrorWithBytes, + ) -> Option, RecoveryError>> { + write_mrt_core_dump(self.core_dump, error.bytes.clone()); + let consumed = error.bytes.clone().unwrap_or_default(); + debug_assert_eq!(record_start + consumed.len() as u64, self.reader.position()); let confirmations = self.config.confirmation_records.max(1); - for distance in 1..=self.config.max_scan_bytes { - let candidate_offset = failed_start + distance as u64; - if !self.reader.move_to(candidate_offset)? { - return Ok(None); + let max_scan_bytes = self.config.max_scan_bytes; + let mut window = ReplayReader::seeded(&mut self.reader, consumed, record_start); + + if let Some(framed_end) = framed_end { + match confirm_aligned_boundary(&mut window, framed_end, confirmations) { + Ok(Some(confirmed_records)) => { + let leftover = window.into_leftover(framed_end); + self.reader.resume_with(leftover, framed_end); + return Some(Ok(RecoveryEvent::Gap(RecoveryGap { + start_offset: record_start, + end_offset: framed_end, + cause: error.to_string(), + evidence: RecoveryEvidence::AlignedRecordChain, + confirmed_records, + }))); + } + Ok(None) => {} + Err(io_error) => { + self.finished = true; + return Some(Err(RecoveryError { + offset: record_start, + scanned_bytes: 0, + error: ParserErrorWithBytes::from(ParserError::IoError(io_error)), + })); + } } - if let Some((evidence, confirmed)) = - validate_chain(&mut self.reader, candidate_offset, confirmations)? - { - return Ok(Some((candidate_offset, evidence, confirmed))); + } + + match find_recovery(&mut window, record_start, max_scan_bytes, confirmations) { + Ok(ScanOutcome::Found { + offset, + evidence, + confirmed_records, + }) => { + let leftover = window.into_leftover(offset); + self.reader.resume_with(leftover, offset); + Some(Ok(RecoveryEvent::Gap(RecoveryGap { + start_offset: record_start, + end_offset: offset, + cause: error.to_string(), + evidence, + confirmed_records, + }))) + } + Ok(ScanOutcome::EndOfStream { end_offset }) => { + let leftover = window.into_leftover(end_offset); + self.reader.resume_with(leftover, end_offset); + Some(Ok(RecoveryEvent::Gap(RecoveryGap { + start_offset: record_start, + end_offset, + cause: error.to_string(), + evidence: RecoveryEvidence::EndOfStream, + confirmed_records: 0, + }))) + } + Ok(ScanOutcome::WindowExhausted) => { + self.finished = true; + Some(Err(RecoveryError { + offset: record_start, + scanned_bytes: max_scan_bytes as u64, + error, + })) + } + Err(io_error) => { + self.finished = true; + Some(Err(RecoveryError { + offset: record_start, + scanned_bytes: window.position().saturating_sub(record_start), + error: ParserErrorWithBytes::from(ParserError::IoError(io_error)), + })) + } + } + } +} + +/// Iterator over BGP elements and explicit recovery gaps. +/// +/// Filters are applied per element; each record is converted to elements exactly once. +pub struct RecoveringElemIterator { + inner: RecoveringRecordIterator, + elementor: Elementor, + filters: Vec, + cache_elems: Vec, +} + +impl RecoveringElemIterator { + pub(crate) fn new(mut parser: BgpkitParser, config: RecoveryConfig) -> Self { + // Elements are filtered here; strip the parser filters so the inner record + // iterator does not also convert every record for record-level matching. + let filters = std::mem::take(&mut parser.filters); + Self { + inner: RecoveringRecordIterator::new(parser, config), + elementor: Elementor::new(), + filters, + cache_elems: Vec::new(), + } + } +} + +impl Iterator for RecoveringElemIterator { + type Item = Result, RecoveryError>; + + fn next(&mut self) -> Option { + loop { + if let Some(elem) = self.cache_elems.pop() { + return Some(Ok(RecoveryEvent::Item(elem))); + } + match self.inner.next()? { + Ok(RecoveryEvent::Item(record)) => { + let mut elems = self.elementor.record_to_elems(record); + elems.retain(|elem| elem.match_filters(&self.filters)); + elems.reverse(); + self.cache_elems = elems; + } + Ok(RecoveryEvent::Gap(gap)) => return Some(Ok(RecoveryEvent::Gap(gap))), + Err(error) => return Some(Err(error)), } } - Ok(None) } } @@ -274,6 +389,87 @@ struct Candidate { evidence: RecoveryEvidence, } +enum ScanOutcome { + Found { + offset: u64, + evidence: RecoveryEvidence, + confirmed_records: u8, + }, + EndOfStream { + end_offset: u64, + }, + WindowExhausted, +} + +/// Confirm that intact records (of any MRT type) parse at the failed record's declared +/// end offset, distinguishing an unparsable-but-correctly-framed record from framing +/// damage. A clean end of stream on the boundary is consistent with intact framing. +fn confirm_aligned_boundary( + window: &mut ReplayReader, + boundary: u64, + required: u8, +) -> io::Result> { + if !window.move_to(boundary)? { + return Ok(None); + } + let mut confirmed = 0u8; + while confirmed < required { + match parse_mrt_record_with_zebra_compat(window) { + Ok(_) => confirmed += 1, + Err(error) => { + return match error.error { + ParserError::EofExpected => Ok(Some(confirmed)), + ParserError::IoError(inner) | ParserError::EofError(inner) + if inner.kind() != io::ErrorKind::UnexpectedEof => + { + Err(inner) + } + _ => Ok(None), + }; + } + } + } + Ok(Some(confirmed)) +} + +fn is_anchor_entry_type(bytes: [u8; 2]) -> bool { + let value = u16::from_be_bytes(bytes); + value == EntryType::BGP as u16 + || value == EntryType::BGP4MP as u16 + || value == EntryType::BGP4MP_ET as u16 +} + +fn find_recovery( + window: &mut ReplayReader, + failed_start: u64, + max_scan_bytes: usize, + confirmations: u8, +) -> io::Result { + for distance in 1..=max_scan_bytes { + let candidate = failed_start + distance as u64; + // Cheap anchor pre-filter: only offsets whose entry-type field matches a + // recoverable stream family warrant header parsing and chain validation. + let Some(entry_type) = window.peek_two_at(candidate + 4)? else { + return Ok(ScanOutcome::EndOfStream { + end_offset: window.buffered_end(), + }); + }; + if !is_anchor_entry_type(entry_type) { + continue; + } + if let Some((evidence, confirmed_records)) = + validate_chain(window, candidate, confirmations)? + { + return Ok(ScanOutcome::Found { + offset: candidate, + evidence, + confirmed_records, + }); + } + } + Ok(ScanOutcome::WindowExhausted) +} + fn validate_chain( reader: &mut ReplayReader, offset: u64, @@ -330,13 +526,14 @@ fn read_candidate(reader: &mut ReplayReader) -> io::Result bool { ) } +/// Reader adapter that tracks the absolute decompressed-stream offset and can be handed +/// back unconsumed bytes after a recovery scan read past the resume boundary. +struct CarryoverReader { + inner: R, + carryover: Vec, + carry_pos: usize, + offset: u64, +} + +impl CarryoverReader { + fn new(inner: R) -> Self { + Self { + inner, + carryover: Vec::new(), + carry_pos: 0, + offset: 0, + } + } + + fn position(&self) -> u64 { + self.offset + } + + /// Resume reading at `offset`, serving `bytes` (followed by any bytes already held + /// but not yet served) before the underlying reader. + fn resume_with(&mut self, mut bytes: Vec, offset: u64) { + bytes.extend_from_slice(&self.carryover[self.carry_pos..]); + self.carryover = bytes; + self.carry_pos = 0; + self.offset = offset; + } +} + +impl Read for CarryoverReader { + fn read(&mut self, output: &mut [u8]) -> io::Result { + if output.is_empty() { + return Ok(0); + } + if self.carry_pos < self.carryover.len() { + let count = output.len().min(self.carryover.len() - self.carry_pos); + output[..count] + .copy_from_slice(&self.carryover[self.carry_pos..self.carry_pos + count]); + self.carry_pos += count; + self.offset += count as u64; + if self.carry_pos == self.carryover.len() { + self.carryover.clear(); + self.carry_pos = 0; + } + return Ok(count); + } + let read = self.inner.read(output)?; + self.offset += read as u64; + Ok(read) + } +} + +/// Bounded lookahead buffer used only while scanning for a recovery boundary. +/// +/// It is seeded with the bytes already consumed by the failed record and buffers further +/// bytes on demand so candidate boundaries can be revisited. It exists only for the +/// duration of one recovery attempt; the undamaged fast path never copies through it. struct ReplayReader { inner: R, data: Vec, - start: usize, cursor: usize, base_offset: u64, } impl ReplayReader { - fn new(inner: R) -> Self { + fn seeded(inner: R, data: Vec, base_offset: u64) -> Self { Self { inner, - data: Vec::new(), - start: 0, + data, cursor: 0, - base_offset: 0, + base_offset, } } fn position(&self) -> u64 { - self.base_offset + (self.cursor - self.start) as u64 + self.base_offset + self.cursor as u64 } fn buffered_end(&self) -> u64 { - self.base_offset + (self.data.len() - self.start) as u64 + self.base_offset + self.data.len() as u64 } - fn discard_before_current(&mut self) { - self.base_offset = self.position(); - self.start = self.cursor; - if self.start >= 64 * 1024 && self.start * 2 >= self.data.len() { - self.data.drain(..self.start); - self.cursor -= self.start; - self.start = 0; - } + /// Consume the window, returning the buffered bytes at and beyond `offset`. + fn into_leftover(mut self, offset: u64) -> Vec { + let index = (offset.saturating_sub(self.base_offset) as usize).min(self.data.len()); + self.data.split_off(index) } } impl ReplayReader { + /// Buffer through `offset` and place the cursor there. Returns false when the stream + /// ends first or `offset` precedes the window. fn move_to(&mut self, offset: u64) -> io::Result { if offset < self.base_offset { return Ok(false); } - while offset > self.buffered_end() { - self.cursor = self.data.len(); - let remaining = (offset - self.buffered_end()).min(8_192) as usize; - let mut scratch = vec![0u8; remaining]; - let read = self.read(&mut scratch)?; + let target = (offset - self.base_offset) as usize; + while self.data.len() < target { + let filled = self.data.len(); + let chunk = (target - filled).min(SCAN_FILL_CHUNK); + self.data.resize(filled + chunk, 0); + let read = self.inner.read(&mut self.data[filled..])?; + self.data.truncate(filled + read); if read == 0 { + self.cursor = self.data.len(); return Ok(false); } } - self.cursor = self.start + (offset - self.base_offset) as usize; + self.cursor = target; Ok(true) } + + /// Read two bytes at `offset`, buffering as needed. Returns `None` when the stream + /// ends first. Callers re-position with [`Self::move_to`] before parsing. + fn peek_two_at(&mut self, offset: u64) -> io::Result> { + if !self.move_to(offset + 2)? { + return Ok(None); + } + let index = (offset - self.base_offset) as usize; + Ok(Some([self.data[index], self.data[index + 1]])) + } } impl Read for ReplayReader { @@ -588,6 +855,18 @@ mod tests { .to_vec() } + /// A record with a well-formed common header and declared length, whose body cannot + /// be parsed as a BGP4MP message. + fn framed_record_with_garbage_body(timestamp: u32, body: &[u8]) -> Vec { + let mut bytes = BytesMut::new(); + bytes.put_u32(timestamp); + bytes.put_u16(EntryType::BGP4MP as u16); + bytes.put_u16(Bgp4MpType::Message as u16); + bytes.put_u32(body.len() as u32); + bytes.put_slice(body); + bytes.to_vec() + } + #[test] fn recovers_at_three_record_legacy_chain() { let first = legacy_state(100); @@ -618,32 +897,211 @@ mod tests { } #[test] - fn rejects_chain_shorter_than_configured_before_non_eof_garbage() { + fn emits_terminal_gap_when_chain_too_short_before_non_eof_garbage() { let mut input = vec![0xff; 12]; input.extend_from_slice(&legacy_state(101)); input.extend_from_slice(&legacy_state(102)); input.extend_from_slice(&[1, 2, 3]); + let total = input.len() as u64; let parser = BgpkitParser::from_reader(Cursor::new(input)); - let result = parser + let events = parser .into_recovering_record_iter(RecoveryConfig::default()) - .next() - .expect("one error"); - assert!(result.is_err()); + .collect::, _>>() + .unwrap(); + + // A two-record chain is below the configured confirmation count, so nothing is + // recovered; the damage extends to the end of the stream and is reported as a + // terminal gap rather than an error. + assert_eq!(events.len(), 1); + let RecoveryEvent::Gap(gap) = &events[0] else { + panic!("expected terminal gap") + }; + assert_eq!(gap.start_offset, 0); + assert_eq!(gap.end_offset, total); + assert_eq!(gap.evidence, RecoveryEvidence::EndOfStream); + assert_eq!(gap.confirmed_records, 0); } #[test] - fn rejects_chain_shorter_than_configured_at_clean_eof() { + fn emits_terminal_gap_when_chain_too_short_at_clean_eof() { let mut input = vec![0xff; 12]; input.extend_from_slice(&legacy_state(101)); input.extend_from_slice(&legacy_state(102)); + let total = input.len() as u64; let parser = BgpkitParser::from_reader(Cursor::new(input)); - let result = parser + let events = parser .into_recovering_record_iter(RecoveryConfig::default()) - .next() - .expect("one error"); - assert!(result.is_err()); + .collect::, _>>() + .unwrap(); + + assert_eq!(events.len(), 1); + let RecoveryEvent::Gap(gap) = &events[0] else { + panic!("expected terminal gap") + }; + assert_eq!(gap.start_offset, 0); + assert_eq!(gap.end_offset, total); + assert_eq!(gap.evidence, RecoveryEvidence::EndOfStream); + assert_eq!(gap.confirmed_records, 0); + } + + #[test] + fn truncated_final_record_yields_terminal_gap() { + let mut input = bgp4mp_keepalive(100); + input.extend_from_slice(&bgp4mp_keepalive(101)); + let boundary = input.len() as u64; + let tail = bgp4mp_keepalive(102); + input.extend_from_slice(&tail[..10]); + let total = input.len() as u64; + + let parser = BgpkitParser::from_reader(Cursor::new(input)); + let events = parser + .into_recovering_record_iter(RecoveryConfig::default()) + .collect::, _>>() + .unwrap(); + + assert_eq!(events.len(), 3); + assert!(matches!(events[0], RecoveryEvent::Item(_))); + assert!(matches!(events[1], RecoveryEvent::Item(_))); + let RecoveryEvent::Gap(gap) = &events[2] else { + panic!("expected terminal gap") + }; + assert_eq!(gap.start_offset, boundary); + assert_eq!(gap.end_offset, total); + assert_eq!(gap.evidence, RecoveryEvidence::EndOfStream); + assert_eq!(gap.confirmed_records, 0); + } + + #[test] + fn skips_exactly_one_framed_record_with_unparsable_body() { + let first = bgp4mp_keepalive(100); + let bad = framed_record_with_garbage_body(101, &[0xde, 0xad, 0xbe]); + let mut input = first.clone(); + input.extend_from_slice(&bad); + input.extend_from_slice(&bgp4mp_keepalive(102)); + input.extend_from_slice(&bgp4mp_keepalive(103)); + input.extend_from_slice(&bgp4mp_keepalive(104)); + + let parser = BgpkitParser::from_reader(Cursor::new(input)); + let events = parser + .into_recovering_record_iter(RecoveryConfig::default()) + .collect::, _>>() + .unwrap(); + + // The bad record framed correctly, so exactly its bytes are skipped without a + // boundary scan and all following records survive. + assert_eq!(events.len(), 5); + assert!(matches!(events[0], RecoveryEvent::Item(_))); + let RecoveryEvent::Gap(gap) = &events[1] else { + panic!("expected aligned-boundary gap") + }; + assert_eq!(gap.start_offset, first.len() as u64); + assert_eq!(gap.end_offset, (first.len() + bad.len()) as u64); + assert_eq!(gap.evidence, RecoveryEvidence::AlignedRecordChain); + assert_eq!(gap.confirmed_records, 3); + assert!(events[2..] + .iter() + .all(|event| matches!(event, RecoveryEvent::Item(_)))); + } + + #[test] + fn skips_framed_record_with_unparsable_body_at_clean_eof() { + let first = bgp4mp_keepalive(100); + let bad = framed_record_with_garbage_body(101, &[0xde, 0xad, 0xbe]); + let mut input = first.clone(); + input.extend_from_slice(&bad); + + let parser = BgpkitParser::from_reader(Cursor::new(input)); + let events = parser + .into_recovering_record_iter(RecoveryConfig::default()) + .collect::, _>>() + .unwrap(); + + assert_eq!(events.len(), 2); + assert!(matches!(events[0], RecoveryEvent::Item(_))); + let RecoveryEvent::Gap(gap) = &events[1] else { + panic!("expected aligned-boundary gap") + }; + assert_eq!(gap.start_offset, first.len() as u64); + assert_eq!(gap.end_offset, (first.len() + bad.len()) as u64); + assert_eq!(gap.evidence, RecoveryEvidence::AlignedRecordChain); + assert_eq!(gap.confirmed_records, 0); + } + + #[test] + fn misframed_record_falls_back_to_boundary_scan() { + let first = bgp4mp_keepalive(100); + // A header whose declared length overlaps the next record: the declared + // boundary is misaligned, so aligned-boundary confirmation must fail and the + // byte scan must find the true boundary. + let mut bad = BytesMut::new(); + bad.put_u32(101); + bad.put_u16(EntryType::BGP4MP as u16); + bad.put_u16(Bgp4MpType::Message as u16); + bad.put_u32(5); + let mut input = first.clone(); + input.extend_from_slice(&bad); + input.extend_from_slice(&[0xde, 0xad, 0xbe]); + input.extend_from_slice(&bgp4mp_keepalive(102)); + input.extend_from_slice(&bgp4mp_keepalive(103)); + input.extend_from_slice(&bgp4mp_keepalive(104)); + + let parser = BgpkitParser::from_reader(Cursor::new(input)); + let events = parser + .into_recovering_record_iter(RecoveryConfig::default()) + .collect::, _>>() + .unwrap(); + + assert_eq!(events.len(), 5); + let RecoveryEvent::Gap(gap) = &events[1] else { + panic!("expected recovery gap") + }; + assert_eq!(gap.start_offset, first.len() as u64); + assert_eq!(gap.end_offset, (first.len() + bad.len() + 3) as u64); + assert_eq!(gap.evidence, RecoveryEvidence::BgpMarkerChain); + assert_eq!(gap.confirmed_records, 3); + assert!(events[2..] + .iter() + .all(|event| matches!(event, RecoveryEvent::Item(_)))); + } + + #[test] + fn text_dump_parser_yields_unsupported_error() { + let dump = "BGP table version is 1, local router ID is 1.2.3.4, vrf id 0\n\ +Default local pref 100, local AS 65001\n\n\ + Network Next Hop Metric LocPrf Weight Path\n\ + *> 1.0.0.0/24 10.0.0.1 0 0 13335 i\n"; + let parser = BgpkitParser::from_text_reader(dump.as_bytes()).unwrap(); + let mut iter = parser.into_recovering_record_iter(RecoveryConfig::default()); + + let error = iter.next().expect("one error").unwrap_err(); + assert!(matches!(error.error.error, ParserError::Unsupported(_))); + assert!(iter.next().is_none()); + } + + #[test] + fn recovering_elem_iter_passes_gaps_through() { + let first = legacy_state(100); + let mut input = first.clone(); + input.extend_from_slice(&[0xde, 0xad, 0xbe, 0xef, 0x01]); + input.extend_from_slice(&legacy_state(101)); + input.extend_from_slice(&legacy_state(102)); + input.extend_from_slice(&legacy_state(103)); + + let parser = BgpkitParser::from_reader(Cursor::new(input)); + let events = parser + .into_recovering_elem_iter(RecoveryConfig::default()) + .collect::, _>>() + .unwrap(); + + // State-change records yield no elements, so only the gap surfaces. + assert_eq!(events.len(), 1); + let RecoveryEvent::Gap(gap) = &events[0] else { + panic!("expected recovery gap") + }; + assert_eq!(gap.start_offset, first.len() as u64); + assert_eq!(gap.end_offset, first.len() as u64 + 5); } #[test] diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 1fc8c4a..bc8c44e 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -306,11 +306,6 @@ impl BgpkitParser> { } impl BgpkitParser { - /// Return the filters currently configured on this parser. - pub fn filters(&self) -> &[Filter] { - &self.filters - } - pub(crate) fn warn_zebra_compat_once(&mut self) { self.options.warn_zebra_compat_once(); } diff --git a/tests/recovery.rs b/tests/recovery.rs index 7017d1d..1d92e60 100644 --- a/tests/recovery.rs +++ b/tests/recovery.rs @@ -1,32 +1,29 @@ -use bgpkit_parser::{ - BgpkitParser, Elementor, RecoveryConfig, RecoveryEvent, RecoveryEvidence, RecoveryGap, -}; +use bgpkit_parser::{BgpkitParser, RecoveryConfig, RecoveryEvent, RecoveryEvidence, RecoveryGap}; +use flate2::read::GzDecoder; +use std::io::{Cursor, Read}; const FIXTURE: &str = "tests/fixtures/ripe/rrc00/2000.03/updates.20000325.0345.gz"; +fn fixture_path() -> String { + format!("{}/{}", env!("CARGO_MANIFEST_DIR"), FIXTURE) +} + #[test] fn recovers_damaged_ripe_type_5_fixture() { - let source = format!("{}/{}", env!("CARGO_MANIFEST_DIR"), FIXTURE); let mut records = 0usize; - let mut elements = 0usize; let mut gaps = Vec::::new(); - let mut elementor = Elementor::new(); - for event in BgpkitParser::new(&source) + for event in BgpkitParser::new(&fixture_path()) .unwrap() .into_recovering_record_iter(RecoveryConfig::default()) { match event.unwrap() { - RecoveryEvent::Item(record) => { - records += 1; - elements += elementor.record_to_elems(record).len(); - } + RecoveryEvent::Item(_) => records += 1, RecoveryEvent::Gap(gap) => gaps.push(gap), } } assert_eq!(records, 2_580); - assert_eq!(elements, 5_378); assert_eq!(gaps.len(), 2); assert_eq!(gaps.iter().map(RecoveryGap::skipped_bytes).sum::(), 24); assert_eq!( @@ -44,3 +41,51 @@ fn recovers_damaged_ripe_type_5_fixture() { ] ); } + +#[test] +fn recovering_elem_iter_matches_fixture_totals() { + let mut elements = 0usize; + let mut gaps = 0usize; + + for event in BgpkitParser::new(&fixture_path()) + .unwrap() + .into_recovering_elem_iter(RecoveryConfig::default()) + { + match event.unwrap() { + RecoveryEvent::Item(_) => elements += 1, + RecoveryEvent::Gap(_) => gaps += 1, + } + } + + assert_eq!(elements, 5_378); + assert_eq!(gaps, 2); +} + +/// A truncated final record — the most common real-world corruption — must yield every +/// intact record plus a terminal gap, not a hard error that discards the whole file. +#[test] +fn truncated_fixture_tail_yields_terminal_gap() { + let mut bytes = Vec::new(); + GzDecoder::new(std::fs::File::open(fixture_path()).unwrap()) + .read_to_end(&mut bytes) + .unwrap(); + bytes.truncate(bytes.len() - 20); + let total = bytes.len() as u64; + + let mut records = 0usize; + let mut gaps = Vec::::new(); + for event in BgpkitParser::from_reader(Cursor::new(bytes)) + .into_recovering_record_iter(RecoveryConfig::default()) + { + match event.unwrap() { + RecoveryEvent::Item(_) => records += 1, + RecoveryEvent::Gap(gap) => gaps.push(gap), + } + } + + assert_eq!(records, 2_579); + assert_eq!(gaps.len(), 3); + let terminal = gaps.last().unwrap(); + assert_eq!(terminal.evidence, RecoveryEvidence::EndOfStream); + assert_eq!(terminal.end_offset, total); +}