Move push of meta data out of graphql handler

This will enable adding retries in a loop for this. Change: meta-push
josh-project · Jan 11, 2023 · 8a3baec · 8a3baec
1 parent 34ccdc3
commit 8a3baec
Show file tree

Hide file tree

Showing 5 changed files with 124 additions and 96 deletions.
diff --git a/josh-proxy/src/bin/josh-proxy.rs b/josh-proxy/src/bin/josh-proxy.rs
@@ -1566,24 +1566,27 @@ async fn serve_graphql(
             tracing::Span::current(),
         ));
 
-        for (reference, oid) in context.to_push.lock()?.iter() {
-            josh_proxy::push_head_url(
-                context.transaction.lock()?.repo(),
-                &serv
-                    .repo_path
-                    .join("mirror")
-                    .join("objects")
-                    .to_str()
-                    .unwrap(),
-                *oid,
-                &reference,
-                &remote_url,
-                &remote_auth,
-                &temp_ns.name(),
-                "META_PUSH",
-                false,
-            )?;
-        }
+        let (refname, oid) = josh_proxy::merge_meta(
+            &*context.transaction.lock()?,
+            &*context.transaction_mirror.lock()?,
+            &*context.meta_add.lock()?,
+        )?;
+        josh_proxy::push_head_url(
+            context.transaction.lock()?.repo(),
+            &serv
+                .repo_path
+                .join("mirror")
+                .join("objects")
+                .to_str()
+                .unwrap(),
+            oid,
+            &refname,
+            &remote_url,
+            &remote_auth,
+            &temp_ns.name(),
+            "META_PUSH",
+            false,
+        )?;
         Ok(())
     })
     .in_current_span()

diff --git a/josh-proxy/src/lib.rs b/josh-proxy/src/lib.rs
@@ -821,3 +821,66 @@ fn changes_to_refs(
         })
         .collect())
 }
+
+pub fn merge_meta(
+    transaction: &josh::cache::Transaction,
+    transaction_mirror: &josh::cache::Transaction,
+    meta_add: &std::collections::HashMap<std::path::PathBuf, Vec<String>>,
+) -> josh::JoshResult<(String, git2::Oid)> {
+    let rev = transaction_mirror.refname("refs/josh/meta");
+
+    let r = transaction_mirror.repo().revparse_single(&rev);
+    let (tree, parent) = if let Ok(r) = r {
+        let meta_commit = transaction.repo().find_commit(r.id())?;
+        let tree = meta_commit.tree()?;
+        (tree, Some(meta_commit))
+    } else {
+        (josh::filter::tree::empty(transaction.repo()), None)
+    };
+
+    let mut tree = tree;
+
+    for (path, add_lines) in meta_add.iter() {
+        let prev = if let Ok(e) = tree.get_path(path) {
+            let blob = transaction.repo().find_blob(e.id())?;
+            std::str::from_utf8(blob.content())?.to_owned()
+        } else {
+            "".to_owned()
+        };
+
+        let mut lines = prev
+            .split('\n')
+            .filter(|x| !(*x).is_empty())
+            .collect::<Vec<_>>();
+        for marker in add_lines {
+            lines.push(marker);
+        }
+        lines.sort_unstable();
+        lines.dedup();
+
+        let blob = transaction.repo().blob(lines.join("\n").as_bytes())?;
+
+        tree = josh::filter::tree::insert(transaction.repo(), &tree, path, blob, 0o0100644)?;
+    }
+
+    let signature = if let Ok(time) = std::env::var("JOSH_COMMIT_TIME") {
+        git2::Signature::new(
+            "josh",
+            "josh@josh-project.dev",
+            &git2::Time::new(time.parse()?, 0),
+        )
+    } else {
+        git2::Signature::now("josh", "josh@josh-project.dev")
+    }?;
+
+    let oid = transaction.repo().commit(
+        None,
+        &signature,
+        &signature,
+        "marker",
+        &tree,
+        &parent.as_ref().into_iter().collect::<Vec<_>>(),
+    )?;
+
+    Ok(("refs/josh/meta".to_string(), oid))
+}
diff --git a/src/graphql.rs b/src/graphql.rs
@@ -834,7 +834,9 @@ impl Reference {
 pub struct Context {
     pub transaction: std::sync::Arc<std::sync::Mutex<cache::Transaction>>,
     pub transaction_mirror: std::sync::Arc<std::sync::Mutex<cache::Transaction>>,
-    pub to_push: std::sync::Arc<std::sync::Mutex<Vec<(String, git2::Oid)>>>,
+    pub meta_add: std::sync::Arc<
+        std::sync::Mutex<std::collections::HashMap<std::path::PathBuf, Vec<String>>>,
+    >,
     pub allow_refs: std::sync::Mutex<bool>,
 }
 
@@ -862,12 +864,6 @@ struct MarkersInput {
     data: Vec<String>,
 }
 
-#[derive(juniper::GraphQLInputObject)]
-struct MarkerInput {
-    position: String,
-    text: String,
-}
-
 fn format_marker(input: &str) -> JoshResult<String> {
     let value = serde_json::from_str::<serde_json::Value>(input)?;
     let line = serde_json::to_string(&value)?;
@@ -891,82 +887,34 @@ impl RepositoryMut {
                 return Err(josh_error("ref query not allowed").into());
             };
         }
-        let transaction = context.transaction.lock()?;
         let transaction_mirror = context.transaction_mirror.lock()?;
 
-        let rev = transaction_mirror.refname("refs/josh/meta");
-
+        // Just check that the commit exists
         transaction_mirror
             .repo()
             .find_commit(git2::Oid::from_str(&commit)?)?;
 
-        let r = transaction_mirror.repo().revparse_single(&rev);
-        let (tree, parent) = if let Ok(r) = r {
-            let commit = transaction.repo().find_commit(r.id())?;
-            let tree = commit.tree()?;
-            (tree, Some(commit))
-        } else {
-            (filter::tree::empty(transaction.repo()), None)
-        };
-
-        let mut tree = tree;
-
-        for mm in add {
-            let path = mm.path;
-            let path = &marker_path(&commit, &topic).join(&path);
-            let prev = if let Ok(e) = tree.get_path(path) {
-                let blob = transaction.repo().find_blob(e.id())?;
-                std::str::from_utf8(blob.content())?.to_owned()
-            } else {
-                "".to_owned()
-            };
+        if let Ok(mut meta_add) = context.meta_add.lock() {
+            for mm in add {
+                let path = mm.path;
+                let path = &marker_path(&commit, &topic).join(&path);
+                let mut lines = meta_add.get(path).unwrap_or(&vec![]).clone();
+
+                let mm = mm
+                    .data
+                    .iter()
+                    .map(String::as_str)
+                    .map(format_marker)
+                    .collect::<JoshResult<Vec<_>>>()?;
+
+                for marker in mm.into_iter() {
+                    lines.push(marker);
+                }
 
-            let mm = mm
-                .data
-                .iter()
-                .map(String::as_str)
-                .map(format_marker)
-                .collect::<JoshResult<Vec<_>>>()?;
-
-            let mut lines = prev
-                .split('\n')
-                .filter(|x| !(*x).is_empty())
-                .collect::<Vec<_>>();
-            for marker in mm.iter() {
-                lines.push(marker);
+                meta_add.insert(path.clone(), lines);
             }
-            lines.sort_unstable();
-            lines.dedup();
-
-            let blob = transaction.repo().blob(lines.join("\n").as_bytes())?;
-
-            tree = filter::tree::insert(transaction.repo(), &tree, path, blob, 0o0100644)?;
         }
 
-        let signature = if let Ok(time) = std::env::var("JOSH_COMMIT_TIME") {
-            git2::Signature::new(
-                "josh",
-                "josh@josh-project.dev",
-                &git2::Time::new(time.parse()?, 0),
-            )
-        } else {
-            git2::Signature::now("josh", "josh@josh-project.dev")
-        }?;
-
-        let oid = transaction.repo().commit(
-            None,
-            &signature,
-            &signature,
-            "marker",
-            &tree,
-            &parent.as_ref().into_iter().collect::<Vec<_>>(),
-        )?;
-
-        context
-            .to_push
-            .lock()?
-            .push(("refs/josh/meta".to_string(), oid));
-
         Ok(true)
     }
 }
@@ -1048,7 +996,7 @@ pub fn context(transaction: cache::Transaction, transaction_mirror: cache::Trans
     Context {
         transaction_mirror: std::sync::Arc::new(std::sync::Mutex::new(transaction_mirror)),
         transaction: std::sync::Arc::new(std::sync::Mutex::new(transaction)),
-        to_push: std::sync::Arc::new(std::sync::Mutex::new(vec![])),
+        meta_add: std::sync::Arc::new(std::sync::Mutex::new(Default::default())),
         allow_refs: std::sync::Mutex::new(false),
     }
 }

diff --git a/tests/proxy/authentication.t b/tests/proxy/authentication.t
@@ -137,8 +137,12 @@
   |   |-- info
   |   |   `-- exclude
   |   |-- objects
+  |   |   |-- 23
+  |   |   |   `-- 212c7a0bc8290223561c459b535de9fee6adb3
   |   |   |-- 3d
   |   |   |   `-- 77ff51363c9825cc2a221fc0ba5a883a1a2c72
+  |   |   |-- 4b
+  |   |   |   `-- 825dc642cb6eb9a060e54bf8d69288fbee4904
   |   |   |-- a0
   |   |   |   `-- 24003ee1acc6bf70318a46e7b6df651b9dc246
   |   |   |-- bb
@@ -154,8 +158,10 @@
   |       |       `-- real_repo.git
   |       |           |-- HEAD
   |       |           `-- refs
-  |       |               `-- heads
-  |       |                   `-- master
+  |       |               |-- heads
+  |       |               |   `-- master
+  |       |               `-- josh
+  |       |                   `-- meta
   |       `-- tags
   `-- overlay
       |-- HEAD
@@ -164,6 +170,8 @@
       |-- info
       |   `-- exclude
       |-- objects
+      |   |-- 23
+      |   |   `-- 212c7a0bc8290223561c459b535de9fee6adb3
       |   |-- 91
       |   |   `-- 0a3d87d1a2d548fdb3d188ffb65bb9c6bd1679
       |   |-- f2
@@ -175,4 +183,4 @@
           |-- namespaces
           `-- tags
   
-  32 directories, 19 files
+  36 directories, 23 files
diff --git a/tests/proxy/markers.t b/tests/proxy/markers.t
@@ -420,6 +420,8 @@
   |   |   |   `-- 0fb9ea1e23fb68a3c589d9b356b6a52bbd3c6f
   |   |   |-- e7
   |   |   |   `-- e7b082cc60acffc7991c438f6a03833bad2641
+  |   |   |-- ef
+  |   |   |   `-- de48c4b797b8a2608cc075b9bdb2e517252dba
   |   |   |-- f8
   |   |   |   `-- 1b303f7a6f22739b9513836d934f622243c15a
   |   |   |-- info
@@ -532,13 +534,17 @@
       |   |   `-- e7b082cc60acffc7991c438f6a03833bad2641
       |   |-- ed
       |   |   `-- fdf0b26b57e156cb1f118a633af077d9ba128a
+      |   |-- ef
+      |   |   `-- de48c4b797b8a2608cc075b9bdb2e517252dba
       |   |-- f8
       |   |   `-- 1b303f7a6f22739b9513836d934f622243c15a
+      |   |-- fe
+      |   |   `-- d37493bc8cfbfe2c3154ca4ba8990bdf92c5bb
       |   |-- info
       |   `-- pack
       `-- refs
           |-- heads
           |-- namespaces
           `-- tags
   
-  107 directories, 106 files
+  110 directories, 109 files