feat: add indexing timestamp

* Add a timestamp marking the time a document was indexed. This is generally useful to understand when a document was indexed both for debugging and for use by tests.
* Add unit test to verify the indexed timestamp.
* Add integration test to verify index is updated when updating a doc

bombastic/index/src/lib.rs

@@ -17,7 +17,7 @@ use tantivy::{
     store::ZstdCompressor,
     DocAddress, IndexSettings, Searcher, SnippetGenerator,
 };
-use time::format_description::well_known::Rfc3339;
+use time::{format_description::well_known::Rfc3339, OffsetDateTime};
 use trustification_api::search::SearchOptions;
 use trustification_index::{
     boost, create_boolean_query, create_date_query, create_string_query, create_text_query, field2str, field2strvec,
@@ -55,6 +55,7 @@ pub struct PackageFields {
 }
 
 struct Fields {
+    indexed_timestamp: Field,
     sbom_id: Field,
     sbom_created: Field,
     sbom_created_inverse: Field,
@@ -74,6 +75,7 @@ impl Index {
     pub fn new() -> Self {
         let mut schema = Schema::builder();
         let fields = Fields {
+            indexed_timestamp: schema.add_date_field("indexed_timestamp", STORED),
             sbom_id: schema.add_text_field("sbom_id", STRING | FAST | STORED),
             sbom_created: schema.add_date_field("sbom_created", INDEXED | FAST | STORED),
             sbom_created_inverse: schema.add_date_field("sbom_created_inverse", FAST),
@@ -127,6 +129,10 @@ impl Index {
 
         document.add_text(self.fields.sbom_id, id);
         document.add_text(self.fields.sbom_name, &bom.document_creation_information.document_name);
+        document.add_date(
+            self.fields.indexed_timestamp,
+            DateTime::from_utc(OffsetDateTime::now_utc()),
+        );
 
         for creators in &bom.document_creation_information.creation_info.creators {
             document.add_text(self.fields.sbom_creators, creators);
@@ -216,6 +222,10 @@ impl Index {
         let mut document = doc!();
 
         document.add_text(self.fields.sbom_id, id);
+        document.add_date(
+            self.fields.indexed_timestamp,
+            DateTime::from_utc(OffsetDateTime::now_utc()),
+        );
         if let Some(metadata) = &bom.metadata {
             if let Some(timestamp) = &metadata.timestamp {
                 let timestamp = timestamp.to_string();
@@ -619,6 +629,7 @@ impl trustification_index::Index for Index {
 
 #[cfg(test)]
 mod tests {
+    use time::format_description;
     use trustification_index::IndexStore;
 
     use super::*;
@@ -801,6 +812,32 @@ mod tests {
         });
     }
 
+    #[tokio::test]
+    async fn test_metadata() {
+        let now = OffsetDateTime::now_utc();
+        assert_search(|index| {
+            let result = index
+                .search(
+                    "",
+                    0,
+                    10000,
+                    SearchOptions {
+                        explain: false,
+                        metadata: true,
+                    },
+                )
+                .unwrap();
+            assert_eq!(result.0.len(), 3);
+            for result in result.0 {
+                assert!(result.metadata.is_some());
+                let indexed_date = result.metadata.as_ref().unwrap()["indexed_timestamp"].clone();
+                let value: &str = indexed_date["values"][0].as_str().unwrap();
+                let indexed_date = OffsetDateTime::parse(value, &format_description::well_known::Rfc3339).unwrap();
+                assert!(indexed_date >= now);
+            }
+        });
+    }
+
     #[tokio::test]
     async fn test_explain() {
         assert_search(|index| {

integration-tests/Cargo.toml

@@ -18,6 +18,7 @@ log = "0.4"
 test-context = "0.1"
 test-with = "0.9"
 ntest = "0.9"
+time = "0.3"
 
 bombastic-api = { path = "../bombastic/api" }
 bombastic-indexer = { path = "../bombastic/indexer" }

integration-tests/tests/bombastic.rs

@@ -7,6 +7,7 @@ use std::time::Duration;
 use test_context::test_context;
 use tokio::fs::{remove_file, File};
 use trustification_auth::client::TokenInjector;
+use trustification_index::tantivy::time::OffsetDateTime;
 use urlencoding::encode;
 
 #[test_context(BombasticContext)]
@@ -118,7 +119,7 @@ async fn test_delete_missing(context: &mut BombasticContext) {
 
 #[test_context(BombasticContext)]
 #[tokio::test]
-#[ntest::timeout(60_000)]
+#[ntest::timeout(90_000)]
 async fn bombastic_search(context: &mut BombasticContext) {
     let mut input: Value = serde_json::from_str(include_str!("../../bombastic/testdata/ubi9-sbom.json")).unwrap();
 
@@ -174,6 +175,46 @@ async fn bombastic_search(context: &mut BombasticContext) {
     })
     .await;
 
+    // Update and check reindex
+    upload_sbom(context.port, &key, &input, &context.provider).await;
+    let now = OffsetDateTime::now_utc();
+    assert_within_timeout(Duration::from_secs(30), async {
+        loop {
+            let url = format!(
+                "http://localhost:{port}/api/v1/sbom/search?q={query}&metadata=true",
+                port = context.port,
+                query = encode(&key)
+            );
+            let response = reqwest::Client::new()
+                .get(url)
+                .inject_token(&context.provider.provider_manager)
+                .await
+                .unwrap()
+                .send()
+                .await
+                .unwrap();
+            assert_eq!(response.status(), StatusCode::OK);
+            let payload: Value = response.json().await.unwrap();
+            if payload["total"].as_u64().unwrap() >= 1 {
+                let format = &time::format_description::well_known::Rfc3339;
+                let ts = OffsetDateTime::parse(
+                    payload["result"][0]["$metadata"]["indexed_timestamp"]["values"][0]
+                        .as_str()
+                        .unwrap(),
+                    format,
+                )
+                .unwrap();
+                if ts > now {
+                    break;
+                } else {
+                    println!("Not yet updated, waiting");
+                }
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    })
+    .await;
+
     delete_sbom(context.port, &key, &context.provider).await;
     assert_within_timeout(Duration::from_secs(30), async {
         loop {

vexination/index/src/lib.rs

@@ -39,6 +39,8 @@ pub struct Index {
 }
 
 struct Fields {
+    indexed_timestamp: Field,
+
     advisory_id: Field,
     advisory_status: Field,
     advisory_title: Field,
@@ -108,6 +110,11 @@ impl trustification_index::Index for Index {
             self.fields.advisory_title => csaf.document.title.clone(),
         );
 
+        document.add_date(
+            self.fields.indexed_timestamp,
+            DateTime::from_utc(OffsetDateTime::now_utc()),
+        );
+
         if let Some(notes) = &csaf.document.notes {
             for note in notes {
                 match &note.category {
@@ -493,6 +500,8 @@ impl Index {
     // TODO use CONST for field names
     pub fn new() -> Self {
         let mut schema = Schema::builder();
+        let indexed_timestamp = schema.add_date_field("indexed_timestamp", STORED);
+
         let advisory_id = schema.add_text_field("advisory_id", STRING | FAST | STORED);
         let advisory_status = schema.add_text_field("advisory_status", STRING);
         let advisory_title = schema.add_text_field("advisory_title", TEXT | STORED);
@@ -523,6 +532,8 @@ impl Index {
         Self {
             schema: schema.build(),
             fields: Fields {
+                indexed_timestamp,
+
                 advisory_id,
                 advisory_status,
                 advisory_title,
@@ -749,6 +760,7 @@ fn rewrite_cpe(value: &str) -> String {
 
 #[cfg(test)]
 mod tests {
+    use time::format_description;
     use trustification_index::IndexStore;
 
     use super::*;
@@ -988,4 +1000,30 @@ mod tests {
             assert!(result.0[0].document.advisory_date > result.0[1].document.advisory_date);
         });
     }
+
+    #[tokio::test]
+    async fn test_metadata() {
+        let now = OffsetDateTime::now_utc();
+        assert_search(|index| {
+            let result = index
+                .search(
+                    "",
+                    0,
+                    10000,
+                    SearchOptions {
+                        explain: false,
+                        metadata: true,
+                    },
+                )
+                .unwrap();
+            assert_eq!(result.0.len(), 3);
+            for result in result.0 {
+                assert!(result.metadata.is_some());
+                let indexed_date = result.metadata.as_ref().unwrap()["indexed_timestamp"].clone();
+                let value: &str = indexed_date["values"][0].as_str().unwrap();
+                let indexed_date = OffsetDateTime::parse(value, &format_description::well_known::Rfc3339).unwrap();
+                assert!(indexed_date >= now);
+            }
+        });
+    }
 }