Serve current graph-data tar through cincinnati

cincinnati/src/plugins/internal/graph_builder/dkrv2_openshift_secondary_metadata_scraper/plugin.rs

@@ -1,6 +1,7 @@
 use crate as cincinnati;
 use crate::plugins::internal::dkrv2_openshift_secondary_metadata_scraper::gpg;
 use crate::plugins::internal::release_scrape_dockerv2::registry;
+use commons::{GRAPH_DATA_DIR_PARAM_KEY, SECONDARY_METADATA_PARAM_KEY};
 use reqwest::{Client, ClientBuilder};
 use std::path::{Path, PathBuf};
 use std::time::Duration;
@@ -26,9 +27,6 @@ pub static DEFAULT_SIGNATURE_BASEURL: &str =
     "https://mirror.openshift.com/pub/openshift-v4/signatures/openshift/release/";
 pub static DEFAULT_SIGNATURE_FETCH_TIMEOUT_SECS: u64 = 30;
 
-// Defines the key for placing the data directory path in the IO parameters
-pub static GRAPH_DATA_DIR_PARAM_KEY: &str = "io.openshift.upgrades.secondary_metadata.directory";
-
 /// Plugin settings.
 #[derive(Debug, SmartDefault, Clone, Deserialize)]
 #[serde(default)]
@@ -276,8 +274,26 @@ impl InternalPlugin for DkrV2OpenshiftSecondaryMetadataScraperPlugin {
         })
         .await?;
 
+        let graph_data_dir = data_dir.path().to_path_buf();
         self.update_cache_state(layers, data_dir).await;
 
+        let graph_data_tar_path = self.settings.output_directory.join("graph-data.tar.gz");
+
+        commons::create_tar(
+            graph_data_tar_path.clone().into_boxed_path(),
+            graph_data_dir.into_boxed_path(),
+        )
+        .await
+        .context("creating graph-data tar")?;
+
+        io.parameters.insert(
+            SECONDARY_METADATA_PARAM_KEY.to_string(),
+            graph_data_tar_path
+                .to_str()
+                .ok_or_else(|| format_err!("secondary_metadata path cannot be converted to str"))?
+                .to_string(),
+        );
+
         Ok(io)
     }
 }

cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/plugin.rs

@@ -6,6 +6,7 @@ use crate as cincinnati;
 use self::cincinnati::plugins::prelude::*;
 use self::cincinnati::plugins::prelude_plugin_impl::*;
 
+use commons::{GRAPH_DATA_DIR_PARAM_KEY, SECONDARY_METADATA_PARAM_KEY};
 use tokio::sync::Mutex as FuturesMutex;
 
 pub static DEFAULT_OUTPUT_WHITELIST: &[&str] = &[
@@ -15,9 +16,6 @@ pub static DEFAULT_OUTPUT_WHITELIST: &[&str] = &[
     "raw/metadata.json",
 ];
 
-// Defines the key for placing the data directory path in the IO parameters
-pub static GRAPH_DATA_DIR_PARAM_KEY: &str = "io.openshift.upgrades.secondary_metadata.directory";
-
 lazy_static::lazy_static! {
     pub static ref DEFAULT_REFERENCE_BRANCH: Option<String> = Some(String::from("master"));
 }
@@ -333,7 +331,7 @@ impl GithubOpenshiftSecondaryMetadataScraperPlugin {
     }
 
     /// Extract a given blob to the output directory, adhering to the output allowlist, and finally update the completed commit state.
-    async fn extract(&self, commit: github_v3::Commit, bytes: Box<[u8]>) -> Fallible<()> {
+    async fn extract(&self, commit: github_v3::Commit, bytes: Box<[u8]>) -> Fallible<PathBuf> {
         // Use a tempdir as intermediary extraction target, and later rename to the destination
         let tmpdir = tempfile::tempdir_in(&self.settings.output_directory)?;
 
@@ -431,8 +429,8 @@ impl GithubOpenshiftSecondaryMetadataScraperPlugin {
             // Set commit_completed to the one we've extracted.
             state_guard.commit_completed = Some(commit);
         }
-
-        Ok(())
+        let data_dir_path = self.data_dir.path().to_path_buf();
+        Ok(data_dir_path)
     }
 }
 
@@ -467,9 +465,29 @@ impl InternalPlugin for GithubOpenshiftSecondaryMetadataScraperPlugin {
                 .download_wanted()
                 .await
                 .context("Downloading tarball")?;
-            self.extract(commit, blob)
+            let graph_data_dir = self
+                .extract(commit, blob)
                 .await
                 .context("Extracting tarball")?;
+
+            let graph_data_tar_path = self.settings.output_directory.join("graph-data.tar.gz");
+
+            commons::create_tar(
+                graph_data_tar_path.clone().into_boxed_path(),
+                graph_data_dir.into_boxed_path(),
+            )
+            .await
+            .context("creating graph-data tar")?;
+
+            io.parameters.insert(
+                SECONDARY_METADATA_PARAM_KEY.to_string(),
+                graph_data_tar_path
+                    .to_str()
+                    .ok_or_else(|| {
+                        format_err!("secondary_metadata path cannot be converted to str")
+                    })?
+                    .to_string(),
+            );
         };
 
         Ok(io)

cincinnati/src/plugins/internal/graph_builder/openshift_secondary_metadata_parser/plugin.rs

@@ -1,10 +1,10 @@
 use crate as cincinnati;
 
-use self::cincinnati::plugins::internal::graph_builder::github_openshift_secondary_metadata_scraper::plugin::GRAPH_DATA_DIR_PARAM_KEY;
 use self::cincinnati::plugins::prelude::*;
 use self::cincinnati::plugins::prelude_plugin_impl::*;
 
 use crate::conditional_edges::{ConditionalEdge, ConditionalUpdateEdge, ConditionalUpdateRisk};
+use commons::GRAPH_DATA_DIR_PARAM_KEY;
 use std::collections::HashSet;
 use std::path::Path;
 

commons/Cargo.toml

@@ -18,10 +18,12 @@ serde_derive = "^1.0.123"
 tokio = { version = "1.16", features = [ "rt-multi-thread" ] }
 url = "^2.3"
 futures = "^0.3"
+flate2 = "^1.0.25"
 opentelemetry = "0.14.0"
 opentelemetry-jaeger = "0.13.0"
 reqwest = "^0.11"
 thrift = "0.17"
+tar = "^0.4.38"
 actix-service = "^2.0.2"
 hamcrest2 = "0.3.0"
 

commons/src/errors.rs

@@ -77,6 +77,10 @@ pub enum GraphError {
     /// Failed to parse as Semantic Version
     #[error("failed to process version: {}", _0)]
     ArchVersionError(String),
+
+    /// Failed to open a file
+    #[error("failed to open file: {}", _0)]
+    FileOpenError(String),
 }
 
 impl actix_web::error::ResponseError for GraphError {
@@ -119,6 +123,7 @@ impl GraphError {
             GraphError::MissingParams(_) => http::StatusCode::BAD_REQUEST,
             GraphError::InvalidParams(_) => http::StatusCode::BAD_REQUEST,
             GraphError::ArchVersionError(_) => http::StatusCode::INTERNAL_SERVER_ERROR,
+            GraphError::FileOpenError(_) => http::StatusCode::INTERNAL_SERVER_ERROR,
         }
     }
 
@@ -134,6 +139,7 @@ impl GraphError {
             GraphError::MissingParams(_) => "missing_params",
             GraphError::InvalidParams(_) => "invalid_params",
             GraphError::ArchVersionError(_) => "arch_version_error",
+            GraphError::FileOpenError(_) => "file_open_err",
         };
         kind.to_string()
     }

commons/src/lib.rs

@@ -17,18 +17,27 @@ pub mod testing;
 pub mod tracing;
 
 mod errors;
-pub use errors::{register_metrics, Fallible, GraphError, MISSING_APPSTATE_PANIC_MSG};
+pub use errors::{register_metrics, Error, Fallible, GraphError, MISSING_APPSTATE_PANIC_MSG};
 
 /// Commonly used imports for error handling.
 pub mod prelude_errors {
     pub use crate::errors::prelude::*;
 }
 
 use actix_web::http::header::{HeaderMap, HeaderValue, ACCEPT};
+use flate2::write::GzEncoder;
+use flate2::Compression;
 use std::collections::HashMap;
 use std::collections::HashSet;
+use std::fs::File;
+use std::path::Path;
 use url::form_urlencoded;
 
+/// Defines the key for placing the data directory path in the IO parameters
+pub static GRAPH_DATA_DIR_PARAM_KEY: &str = "io.openshift.upgrades.secondary_metadata.directory";
+/// Defines the key for placing the graph_data tar path in the IO parameters
+pub static SECONDARY_METADATA_PARAM_KEY: &str = "io.openshift.upgrades.secondary_metadata.tar";
+
 lazy_static! {
     /// list of cincinnati versions
     pub static ref CINCINNATI_VERSION: HashMap<&'static str, i32> =
@@ -162,6 +171,19 @@ pub fn validate_content_type(
     }
 }
 
+/// creates the tar file in the output directory from data_path
+pub async fn create_tar(output_path: Box<Path>, data_path: Box<Path>) -> Result<(), Error> {
+    let tar_gz = File::create(output_path)?;
+    let enc = GzEncoder::new(tar_gz, Compression::default());
+    let mut tar = tar::Builder::new(enc);
+    tar.append_dir_all(".", data_path)?;
+    let tar_status = tar.finish();
+    if tar_status.is_err() {
+        return Err(prelude_errors::format_err!("{:?}", tar_status.unwrap_err()));
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

dist/openshift/cincinnati.yaml

@@ -62,6 +62,8 @@ objects:
               ports:
                 - name: graph-builder
                   containerPort: ${{GB_PORT}}
+                - name: graph-builder-public
+                  containerPort: ${{GB_PUBLIC_PORT}}
                 - name: status-gb
                   containerPort: ${{GB_STATUS_PORT}}
               livenessProbe:
@@ -191,6 +193,20 @@ objects:
           targetPort: ${{GB_STATUS_PORT}}
       selector:
         deploymentconfig: cincinnati
+  - apiVersion: v1
+    kind: Service
+    metadata:
+      name: cincinnati-graph-builder-public
+      labels:
+        app: cincinnati-graph-builder
+    spec:
+      ports:
+        - name: graph-builder-public
+          protocol: TCP
+          port: ${{GB_PUBLIC_PORT}}
+          targetPort: ${{GB_PUBLIC_PORT}}
+      selector:
+        deploymentconfig: cincinnati
   - apiVersion: v1
     kind: Service
     metadata:
@@ -251,6 +267,7 @@ objects:
         pause_secs = ${GB_PAUSE_SECS}
         address = "${GB_ADDRESS}"
         port = ${GB_PORT}
+        public_port = "${GB_PUBLIC_PORT}"
 
         [status]
         address = "${GB_STATUS_ADDRESS}"
@@ -307,6 +324,9 @@ parameters:
   - name: GB_PORT
     value: "8080"
     displayName: Graph builder port
+  - name: GB_PUBLIC_PORT
+    value: "8090"
+    displayName: Graph builder public port
   - name: GB_ADDRESS
     value: "0.0.0.0"
     displayName: Graph builder address

graph-builder/Cargo.toml

@@ -9,6 +9,7 @@ build = "src/build.rs"
 actix = "0.13.0"
 actix-web = "^4.0.0-rc.3"
 chrono = "^0.4.20"
+actix-files = "^0.6.2"
 cincinnati = { path = "../cincinnati" }
 commons = { path = "../commons" }
 env_logger = "^0.9"