josh-project · LMG · Sep 21, 2022 · Aug 31, 2022
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/josh-proxy/Cargo.toml b/josh-proxy/Cargo.toml
@@ -12,32 +12,33 @@ version = "22.4.15"
 
 
 [dependencies]
+base64 = "0.13.0"
 clap = "3.2.17"
 futures = "0.3.23"
-tokio = { version = "1.20.1", features = ["full"] }
+git2 = "0.15.0"
 hyper = { version = "0.14.20", features = ["stream"] }
-hyper_cgi = "22.4.15"
+hyper-reverse-proxy = "0.5.1"
 hyper-staticfile = "0.9.1"
 hyper-tls = "0.5.0"
-toml = "0.5.9"
-git2 = "0.15.0"
-tracing = { version = "0.1.36", features = ["max_level_trace", "release_max_level_trace"] }
-tracing-futures = "0.2.5"
-tracing-subscriber = { version = "0.3.15", features = ["env-filter"] }
-tracing-opentelemetry = "0.17.4"
+hyper_cgi = "22.4.15"
+indoc = "1.0.7"
+josh = {path = "../"}
+juniper = { version = "0.15.10", features = ["expose-test-schema"] }
+lazy_static = "1.4.0"
 opentelemetry = "0.17.0"
 opentelemetry-jaeger = "0.16.0"
-reqwest= { version = "0.11.11", default-features = false, features = ["blocking", "json"] }
-uuid = { version = "1.1.2", features = ["v4"] }
-base64 = "0.13.0"
+percent-encoding = "2.1.0"
 regex = "1.6.0"
-lazy_static = "1.4.0"
-josh = {path = "../"}
-serde_json= "1.0.85"
+reqwest= { version = "0.11.11", default-features = false, features = ["blocking", "json"] }
 serde= "1.0.144"
+serde_json= "1.0.85"
+serde_yaml = "0.9.10"
+tokio = { version = "1.20.1", features = ["full"] }
+toml = "0.5.9"
+tracing = { version = "0.1.36", features = ["max_level_trace", "release_max_level_trace"] }
+tracing-futures = "0.2.5"
+tracing-opentelemetry = "0.17.4"
+tracing-subscriber = { version = "0.3.15", features = ["env-filter"] }
 unindent = "0.1.10"
-juniper = { version = "0.15.10", features = ["expose-test-schema"] }
 url = "2.2.2"
-percent-encoding = "2.1.0"
-indoc = "1.0.7"
-hyper-reverse-proxy = "0.5.1"
+uuid = { version = "1.1.2", features = ["v4"] }
diff --git a/josh-proxy/src/auth.rs b/josh-proxy/src/auth.rs
@@ -52,6 +52,21 @@ impl Handle {
     }
 }
 
+pub fn add_auth(token: &str) -> josh::JoshResult<Handle> {
+    let header = hyper::header::HeaderValue::from_str(&format!("Basic {}", base64::encode(token)))?;
+    let hp = Handle {
+        hash: format!(
+            "{:?}",
+            git2::Oid::hash_object(git2::ObjectType::Blob, header.as_bytes())?
+        ),
+    };
+    let p = Header {
+        header: Some(header),
+    };
+    AUTH.lock()?.insert(hp.clone(), p);
+    return Ok(hp);
+}
+
 pub async fn check_auth(url: &str, auth: &Handle, required: bool) -> josh::JoshResult<bool> {
     if required && auth.hash.is_empty() {
         return Ok(false);

diff --git a/josh-proxy/src/bin/josh-proxy.rs b/josh-proxy/src/bin/josh-proxy.rs
@@ -393,6 +393,58 @@ async fn handle_ui_request(
     return Ok(response);
 }
 
+#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)]
+struct RepoConfig {
+    repo: String,
+}
+
+async fn query_meta_repo(
+    serv: Arc<JoshProxyService>,
+    meta_repo: &str,
+    upstream_repo: &str,
+    auth: &josh_proxy::auth::Handle,
+) -> josh::JoshResult<RepoConfig> {
+    let remote_url = [serv.upstream_url.as_str(), meta_repo].join("");
+    match fetch_upstream(
+        serv.clone(),
+        meta_repo.to_owned(),
+        &auth,
+        remote_url.to_owned(),
+        &"HEAD",
+        false,
+    )
+    .in_current_span()
+    .await
+    {
+        Ok(true) => {}
+        _ => return Err(josh::josh_error("meta fetch failed")),
+    }
+
+    let transaction = josh::cache::Transaction::open(
+        &serv.repo_path.join("mirror"),
+        Some(&format!("refs/josh/upstream/{}/", &josh::to_ns(&meta_repo),)),
+    )?;
+
+    let meta_tree = transaction
+        .repo()
+        .find_reference(&transaction.refname("HEAD"))?
+        .peel_to_tree()?;
+
+    let meta_blob = josh::filter::tree::get_blob(
+        transaction.repo(),
+        &meta_tree,
+        &std::path::Path::new(&upstream_repo.trim_start_matches("/")).join("repo.yml"),
+    );
+
+    if meta_blob == "" {
+        return Err(josh::josh_error(&"meta repo entry not found"));
+    }
+
+    let config: RepoConfig = serde_yaml::from_str(&meta_blob)?;
+
+    return Ok(config);
+}
+
 #[tracing::instrument]
 async fn call_service(
     serv: Arc<JoshProxyService>,
@@ -462,10 +514,22 @@ async fn call_service(
         }
     };
 
-    let upstream_repo = parsed_url.upstream_repo;
+    let mut config = RepoConfig::default();
     let filter = parsed_url.filter;
 
-    let remote_url = [serv.upstream_url.as_str(), upstream_repo.as_str()].join("");
+    if let Ok(meta_repo) = std::env::var("JOSH_META_REPO") {
+        let auth = if let Ok(token) = std::env::var("JOSH_META_AUTH_TOKEN") {
+            josh_proxy::auth::add_auth(&token)?
+        } else {
+            auth.clone()
+        };
+        config =
+            query_meta_repo(serv.clone(), &meta_repo, &parsed_url.upstream_repo, &auth).await?;
+    } else {
+        config.repo = parsed_url.upstream_repo;
+    }
+
+    let remote_url = [serv.upstream_url.as_str(), config.repo.as_str()].join("");
 
     if parsed_url.pathinfo.starts_with("/info/lfs") {
         return Ok(Response::builder()
@@ -498,7 +562,7 @@ async fn call_service(
     let block = block.split(";").collect::<Vec<_>>();
 
     for b in block {
-        if b == upstream_repo {
+        if b == config.repo {
             return Ok(make_response(
                 hyper::Body::from(formatdoc!(
                     r#"
@@ -511,11 +575,11 @@ async fn call_service(
     }
 
     if parsed_url.api == "/~/graphql" {
-        return serve_graphql(serv, req, upstream_repo.to_owned(), remote_url, auth).await;
+        return serve_graphql(serv, req, config.repo.to_owned(), remote_url, auth).await;
     }
 
     if parsed_url.api == "/~/graphiql" {
-        let addr = format!("/~/graphql{}", upstream_repo);
+        let addr = format!("/~/graphql{}", config.repo);
         return Ok(tokio::task::spawn_blocking(move || {
             josh_proxy::juniper_hyper::graphiql(&addr, None)
         })
@@ -525,7 +589,7 @@ async fn call_service(
 
     match fetch_upstream(
         serv.clone(),
-        upstream_repo.to_owned(),
+        config.repo.to_owned(),
         &auth,
         remote_url.to_owned(),
         &headref,
@@ -552,10 +616,10 @@ async fn call_service(
         req.uri().query().map(|x| x.to_string()),
         parsed_url.pathinfo.is_empty(),
     ) {
-        return serve_query(serv, q, upstream_repo, filter, headref).await;
+        return serve_query(serv, q, config.repo, filter, headref).await;
     }
 
-    let temp_ns = prepare_namespace(serv.clone(), &upstream_repo, &filter, &headref)
+    let temp_ns = prepare_namespace(serv.clone(), &config.repo, &filter, &headref)
         .in_current_span()
         .await?;
 
@@ -588,7 +652,7 @@ async fn call_service(
         auth,
         port: serv.port.clone(),
         filter_spec: filter.clone(),
-        base_ns: josh::to_ns(&upstream_repo),
+        base_ns: josh::to_ns(&config.repo),
         git_ns: temp_ns.name().to_string(),
         git_dir: repo_path.clone(),
         mirror_git_dir: mirror_repo_path.clone(),

diff --git a/tests/proxy/clone_with_meta.t b/tests/proxy/clone_with_meta.t
@@ -0,0 +1,170 @@
+  $ export JOSH_META_REPO=/meta_repo.git
+  $ . ${TESTDIR}/setup_test_env.sh
+
+  $ cd ${TESTTMP}
+
+  $ git clone -q http://localhost:8001/meta_repo.git
+  warning: You appear to have cloned an empty repository.
+
+  $ cd meta_repo
+  $ mkdir -p path/to/my_repo.git
+
+  $ cat > path/to/my_repo.git/repo.yml <<EOF
+  > repo: /real_repo.git
+  > EOF
+
+  $ git add .
+  $ git commit -m "add my_repo"
+  [master (root-commit) fc1c140] add my_repo
+   1 file changed, 1 insertion(+)
+   create mode 100644 path/to/my_repo.git/repo.yml
+  $ git push
+  To http://localhost:8001/meta_repo.git
+   * [new branch]      master -> master
+
+
+  $ cd ${TESTTMP}
+
+  $ git clone -q http://localhost:8001/real_repo.git backing_repo
+  warning: You appear to have cloned an empty repository.
+
+  $ cd backing_repo
+
+  $ mkdir sub1
+  $ echo contents1 > sub1/file1
+  $ git add sub1
+  $ git commit -m "add file1"
+  [master (root-commit) bb282e9] add file1
+   1 file changed, 1 insertion(+)
+   create mode 100644 sub1/file1
+
+  $ mkdir sub2
+  $ echo contents1 > sub2/file2
+  $ git add sub2
+  $ git commit -m "add file2"
+  [master ffe8d08] add file2
+   1 file changed, 1 insertion(+)
+   create mode 100644 sub2/file2
+
+  $ git log --graph --pretty=%s
+  * add file2
+  * add file1
+
+  $ git push
+  To http://localhost:8001/real_repo.git
+   * [new branch]      master -> master
+
+  $ cd ${TESTTMP}
+
+
+  $ git clone -q http://localhost:8002/real_repo.git
+  remote: meta repo entry not found
+  fatal: unable to access 'http://localhost:8002/real_repo.git/': The requested URL returned error: 500
+  [128]
+
+  $ git clone -q http://localhost:8002/path/to/my_repo.git
+  $ cd my_repo
+  $ git ls-remote --symref
+  From http://localhost:8002/path/to/my_repo.git
+  ref: refs/heads/master\tHEAD (esc)
+  ffe8d082c1034053534ea8068f4205ac72a1098e\tHEAD (esc)
+  ffe8d082c1034053534ea8068f4205ac72a1098e\trefs/heads/master (esc)
+  $ cat .git/refs/remotes/origin/HEAD
+  ref: refs/remotes/origin/master
+
+  $ tree
+  .
+  |-- sub1
+  |   `-- file1
+  `-- sub2
+      `-- file2
+
+  2 directories, 2 files
+
+  $ git log --graph --pretty=%s
+  * add file2
+  * add file1
+
+
+  $ bash ${TESTDIR}/destroy_test_env.sh
+  "meta_repo.git" = [
+      ':/path',
+      ':/path/to',
+  ]
+  "real_repo.git" = [
+      ':/sub1',
+      ':/sub2',
+  ]
+  .
+  |-- josh
+  |   `-- 12
+  |       `-- sled
+  |           |-- blobs
+  |           |-- conf
+  |           `-- db
+  |-- mirror
+  |   |-- FETCH_HEAD
+  |   |-- HEAD
+  |   |-- config
+  |   |-- description
+  |   |-- info
+  |   |   `-- exclude
+  |   |-- objects
+  |   |   |-- 0e
+  |   |   |   `-- 573a1bd81cc9aefaf932187b9e68a1052a4ff6
+  |   |   |-- 1d
+  |   |   |   `-- ffbbd63f1d894f194cf0bd16a3f19b82269b53
+  |   |   |-- 23
+  |   |   |   `-- 87c32648eefdee78386575672ac091da849b08
+  |   |   |-- 3d
+  |   |   |   `-- 77ff51363c9825cc2a221fc0ba5a883a1a2c72
+  |   |   |-- 85
+  |   |   |   `-- 837e6104d0a81b944c067e16ddc83c7a38739f
+  |   |   |-- a0
+  |   |   |   `-- 24003ee1acc6bf70318a46e7b6df651b9dc246
+  |   |   |-- bb
+  |   |   |   `-- 282e9cdc1b972fffd08fd21eead43bc0c83cb8
+  |   |   |-- c8
+  |   |   |   `-- 2fc150c43f13cc56c0e9caeba01b58ec612022
+  |   |   |-- c9
+  |   |   |   `-- e952f9beba7da8de8ae3b350f15e3774645a54
+  |   |   |-- de
+  |   |   |   `-- 47e9b40111cce1577ab928e7c1ac57b41ee9b7
+  |   |   |-- ec
+  |   |   |   `-- 6006d85ed823a63d900cc7a0ed534ce3b8b5c4
+  |   |   |-- fc
+  |   |   |   `-- 1c140115774c6181f8c8d336e1714590f53230
+  |   |   |-- ff
+  |   |   |   `-- e8d082c1034053534ea8068f4205ac72a1098e
+  |   |   |-- info
+  |   |   `-- pack
+  |   `-- refs
+  |       |-- heads
+  |       |-- josh
+  |       |   `-- upstream
+  |       |       |-- meta_repo.git
+  |       |       |   |-- HEAD
+  |       |       |   `-- refs
+  |       |       |       `-- heads
+  |       |       |           `-- master
+  |       |       `-- real_repo.git
+  |       |           |-- HEAD
+  |       |           `-- refs
+  |       |               `-- heads
+  |       |                   `-- master
+  |       `-- tags
+  `-- overlay
+      |-- HEAD
+      |-- config
+      |-- description
+      |-- info
+      |   `-- exclude
+      |-- objects
+      |   |-- info
+      |   `-- pack
+      `-- refs
+          |-- heads
+          |-- namespaces
+          `-- tags
+
+  42 directories, 28 files
diff --git a/tests/proxy/setup_test_env.sh b/tests/proxy/setup_test_env.sh
@@ -9,6 +9,8 @@ git init -q --bare "${TESTTMP}/remote/blocked_repo.git/" 1> /dev/null
 git config -f "${TESTTMP}/remote/blocked_repo.git/config" http.receivepack true
 git init -q --bare "${TESTTMP}/remote/real/repo2.git/" 1> /dev/null
 git config -f "${TESTTMP}/remote/real/repo2.git/config" http.receivepack true
+git init -q --bare "${TESTTMP}/remote/meta_repo.git/" 1> /dev/null
+git config -f "${TESTTMP}/remote/meta_repo.git/config" http.receivepack true
 export RUST_LOG=trace
 
 export GIT_CONFIG_NOSYSTEM=1