From db082845c3cedcfb3949292f9da68dbf26842b5b Mon Sep 17 00:00:00 2001 From: Vladislav Ivanov Date: Tue, 15 Nov 2022 23:40:56 +0700 Subject: [PATCH] SSH pull and push support commit-id:534b1577 --- josh-proxy/src/bin/josh-proxy.rs | 64 ++++++++------ josh-proxy/src/lib.rs | 144 +++++++++++++++++++++---------- 2 files changed, 136 insertions(+), 72 deletions(-) diff --git a/josh-proxy/src/bin/josh-proxy.rs b/josh-proxy/src/bin/josh-proxy.rs index fce2a4c2c..224da3538 100644 --- a/josh-proxy/src/bin/josh-proxy.rs +++ b/josh-proxy/src/bin/josh-proxy.rs @@ -2,7 +2,7 @@ #[macro_use] extern crate lazy_static; -use josh_proxy::{FetchError, MetaConfig, RepoConfig, RepoUpdate}; +use josh_proxy::{FetchError, MetaConfig, RemoteAuth, RepoConfig, RepoUpdate}; use opentelemetry::global; use opentelemetry::sdk::propagation::TraceContextPropagator; use tracing_opentelemetry::OpenTelemetrySpanExt; @@ -102,12 +102,11 @@ async fn fetch_upstream( service: Arc, upstream_protocol: UpstreamProtocol, upstream_repo: String, - auth: &josh_proxy::auth::Handle, + remote_auth: &RemoteAuth, remote_url: String, headref: &str, force: bool, ) -> Result<(), FetchError> { - let auth = auth.clone(); let key = remote_url.clone(); if upstream_protocol == UpstreamProtocol::Ssh { @@ -173,12 +172,12 @@ async fn fetch_upstream( let span = tracing::span!(tracing::Level::TRACE, "fetch worker"); let us = upstream_repo.clone(); - let a = auth.clone(); let ru = remote_url.clone(); let permit = service.fetch_permits.acquire().await; + let task_remote_auth = remote_auth.clone(); let fetch_result = tokio::task::spawn_blocking(move || { let _span_guard = span.enter(); - josh_proxy::fetch_refs_from_url(&br_path, &us, &ru, &refs_to_fetch, &a) + josh_proxy::fetch_refs_from_url(&br_path, &us, &ru, &refs_to_fetch, &task_remote_auth) }) .await?; @@ -186,10 +185,10 @@ async fn fetch_upstream( let s = tracing::span!(tracing::Level::TRACE, "get_head worker"); let br_path = service.repo_path.join("mirror"); let ru = remote_url.clone(); - let a = auth.clone(); + let task_remote_auth = remote_auth.clone(); let hres = tokio::task::spawn_blocking(move || { let _e = s.enter(); - josh_proxy::get_head(&br_path, &ru, &a) + josh_proxy::get_head(&br_path, &ru, &task_remote_auth) }) .await?; @@ -199,8 +198,8 @@ async fn fetch_upstream( std::mem::drop(permit); - match fetch_result { - Ok(_) => { + match (fetch_result, remote_auth) { + (Ok(_), RemoteAuth::Http { auth }) => { fetch_timers.write()?.insert(key, std::time::Instant::now()); let (auth_user, _) = auth.parse().map_err(FetchError::from_josh_error)?; @@ -209,12 +208,13 @@ async fn fetch_upstream( service .poll .lock()? - .insert((upstream_repo, auth, remote_url)); + .insert((upstream_repo, auth.clone(), remote_url)); } Ok(()) } - Err(_) => fetch_result, + (Ok(_), _) => Ok(()), + (Err(e), _) => Err(e), } } @@ -444,7 +444,7 @@ async fn query_meta_repo( meta_repo: &str, upstream_protocol: UpstreamProtocol, upstream_repo: &str, - auth: &josh_proxy::auth::Handle, + remote_auth: &RemoteAuth, ) -> josh::JoshResult { let upstream = serv .upstream @@ -455,7 +455,7 @@ async fn query_meta_repo( serv.clone(), upstream_protocol, meta_repo.to_owned(), - &auth, + &remote_auth, remote_url.to_owned(), &"HEAD", false, @@ -510,14 +510,14 @@ async fn query_meta_repo( async fn make_meta_config( serv: Arc, - auth: Option<&josh_proxy::auth::Handle>, + remote_auth: Option<&RemoteAuth>, upstream_protocol: UpstreamProtocol, parsed_url: &FilteredRepoUrl, ) -> josh::JoshResult { let meta_repo = std::env::var("JOSH_META_REPO"); let auth_token = std::env::var("JOSH_META_AUTH_TOKEN"); - match (auth, meta_repo) { + match (remote_auth, meta_repo) { (None, _) | (_, Err(_)) => Ok(MetaConfig { config: RepoConfig { repo: parsed_url.upstream_repo.clone(), @@ -525,11 +525,20 @@ async fn make_meta_config( }, ..Default::default() }), - (Some(auth), Ok(meta_repo)) => { - let auth = if let Ok(token) = auth_token { - josh_proxy::auth::add_auth(&token)? - } else { - auth.clone() + (Some(remote_auth), Ok(meta_repo)) => { + let auth = match remote_auth { + RemoteAuth::Ssh { auth_socket } => RemoteAuth::Ssh { + auth_socket: auth_socket.clone(), + }, + RemoteAuth::Http { auth } => { + let auth = if let Ok(token) = auth_token { + josh_proxy::auth::add_auth(&token)? + } else { + auth.clone() + }; + + RemoteAuth::Http { auth } + } }; query_meta_repo( @@ -847,9 +856,10 @@ async fn call_service( } }; + let remote_auth = RemoteAuth::Http { auth: auth.clone() }; let meta = make_meta_config( serv.clone(), - Some(&auth), + Some(&remote_auth), UpstreamProtocol::Http, &parsed_url, ) @@ -928,7 +938,7 @@ async fn call_service( serv.clone(), UpstreamProtocol::Http, meta.config.repo.to_owned(), - &auth, + &remote_auth, remote_url.to_owned(), &headref, false, @@ -989,7 +999,7 @@ async fn call_service( let repo_update = josh_proxy::RepoUpdate { refs: HashMap::new(), remote_url: remote_url.clone(), - auth, + remote_auth, port: serv.port.clone(), filter_spec: josh::filter::spec(filter), base_ns: josh::to_ns(&meta.config.repo), @@ -1236,11 +1246,12 @@ async fn run_polling(serv: Arc) -> josh::JoshResult<()> { let polls = serv.poll.lock()?.clone(); for (upstream_repo, auth, url) in polls { + let remote_auth = RemoteAuth::Http { auth }; let fetch_result = fetch_upstream( serv.clone(), UpstreamProtocol::Http, upstream_repo.clone(), - &auth, + &remote_auth, url.clone(), "", true, @@ -1374,6 +1385,7 @@ async fn serve_graphql( false, )); + let remote_auth = RemoteAuth::Http { auth }; let res = { // First attempt to serve GraphQL query. If we can serve it // that means all requested revisions were specified by SHA and we could find @@ -1389,7 +1401,7 @@ async fn serve_graphql( serv.clone(), UpstreamProtocol::Http, upstream_repo.to_owned(), - &auth, + &remote_auth, remote_url.to_owned(), &"HEAD", false, @@ -1452,7 +1464,7 @@ async fn serve_graphql( *oid, &reference, &remote_url, - &auth, + &remote_auth, &temp_ns.name(), "META_PUSH", false, diff --git a/josh-proxy/src/lib.rs b/josh-proxy/src/lib.rs index 40a15aa46..5d3391405 100644 --- a/josh-proxy/src/lib.rs +++ b/josh-proxy/src/lib.rs @@ -88,11 +88,17 @@ fn baseref_and_options(refname: &str) -> josh::JoshResult<(String, String, Vec, pub remote_url: String, - pub auth: auth::Handle, + pub remote_auth: RemoteAuth, pub port: String, pub filter_spec: String, pub base_ns: String, @@ -295,7 +301,7 @@ pub fn process_repo_update(repo_update: RepoUpdate) -> josh::JoshResult oid, &reference, &repo_update.remote_url, - &repo_update.auth, + &repo_update.remote_auth, &repo_update.git_ns, &display_name, push_mode != PushMode::Normal, @@ -418,7 +424,7 @@ pub fn push_head_url( oid: git2::Oid, refname: &str, url: &str, - auth: &auth::Handle, + remote_auth: &RemoteAuth, namespace: &str, display_name: &str, force: bool, @@ -427,12 +433,6 @@ pub fn push_head_url( let spec = format!("{}:{}", &rn, &refname); - let shell = josh::shell::Shell { - cwd: repo.path().to_owned(), - }; - - let (username, password) = auth.parse()?; - let mut cmd = vec!["git", "push"]; if force { cmd.push("--force") @@ -441,16 +441,10 @@ pub fn push_head_url( cmd.push(&spec); let mut fakehead = repo.reference(&rn, oid, true, "push_head_url")?; - let (stdout, stderr, status) = shell.command_env( - &cmd, - &[], - &[ - ("GIT_PASSWORD", &password), - ("GIT_USER", &username), - ("GIT_ALTERNATE_OBJECT_DIRECTORIES", &alternate), - ], - ); + let (stdout, stderr, status) = + run_git_with_auth(&repo.path(), &cmd, &remote_auth, Some(alternate.to_owned()))?; fakehead.delete()?; + tracing::debug!("{}", &stderr); tracing::debug!("{}", &stdout); @@ -545,25 +539,84 @@ pub fn create_repo(path: &std::path::Path) -> josh::JoshResult<()> { Ok(()) } -pub fn get_head( - path: &std::path::Path, - url: &str, - auth: &auth::Handle, -) -> josh::JoshResult { +fn make_ssh_command() -> String { + let ssh_options = [ + "IdentitiesOnly=yes", + "LogLevel=ERROR", + "UserKnownHostsFile=/dev/null", + "StrictHostKeyChecking=no", + "PreferredAuthentications=publickey", + "ForwardX11=no", + "ForwardAgent=no", + ]; + + let ssh_options = ssh_options.map(|o| format!("-o {}", o)); + format!("ssh {}", ssh_options.join(" ")) +} + +fn run_git_with_auth( + cwd: &std::path::Path, + cmd: &[&str], + remote_auth: &RemoteAuth, + alt_object_dir: Option, +) -> josh::JoshResult<(String, String, i32)> { let shell = josh::shell::Shell { - cwd: path.to_owned(), + cwd: cwd.to_owned(), }; - let (username, password) = auth.parse()?; + let maybe_object_dir = match &alt_object_dir { + Some(dir) => { + vec![("GIT_ALTERNATE_OBJECT_DIRECTORIES", dir.as_str())] + } + None => vec![], + }; + + match remote_auth { + RemoteAuth::Ssh { auth_socket } => { + let ssh_command = make_ssh_command(); + + let env = [("GIT_SSH_COMMAND", ssh_command.as_str())]; + let env_notrace = [ + [("SSH_AUTH_SOCK", auth_socket.as_str())].as_slice(), + maybe_object_dir.as_slice(), + ] + .concat(); + Ok(shell.command_env(&cmd, &env, &env_notrace)) + } + RemoteAuth::Http { auth } => { + let (username, password) = auth.parse()?; + let env_notrace = [ + [ + ("GIT_PASSWORD", password.as_str()), + ("GIT_USER", username.as_str()), + ] + .as_slice(), + maybe_object_dir.as_slice(), + ] + .concat(); + + Ok(shell.command_env(&cmd, &[], &env_notrace)) + } + } +} + +pub fn get_head( + path: &std::path::Path, + url: &str, + remote_auth: &RemoteAuth, +) -> josh::JoshResult { let cmd = &["git", "ls-remote", "--symref", &url, "HEAD"]; + tracing::info!("get_head {:?} {:?} {:?}", cmd, path, ""); + let (stdout, _, code) = run_git_with_auth(&path, cmd, &remote_auth, None)?; - let (stdout, _stderr, _) = shell.command_env( - cmd, - &[], - &[("GIT_PASSWORD", &password), ("GIT_USER", &username)], - ); + if code != 0 { + return Err(josh_error(&format!( + "git subprocess exited with code {}", + code + ))); + } let head = stdout .lines() @@ -602,7 +655,7 @@ pub fn fetch_refs_from_url( upstream_repo: &str, url: &str, refs_prefixes: &[String], - auth: &auth::Handle, + remote_auth: &RemoteAuth, ) -> Result<(), FetchError> { let specs: Vec<_> = refs_prefixes .iter() @@ -616,10 +669,6 @@ pub fn fetch_refs_from_url( }) .collect(); - let shell = josh::shell::Shell { - cwd: path.to_owned(), - }; - let cmd = ["git", "fetch", "--prune", "--no-tags", &url] .map(str::to_owned) .to_vec(); @@ -628,23 +677,26 @@ pub fn fetch_refs_from_url( tracing::info!("fetch_refs_from_url {:?} {:?} {:?}", cmd, path, ""); - let (username, password) = auth.parse().map_err(FetchError::from_josh_error)?; - let (_stdout, stderr, _) = shell.command_env( - &cmd, - &[], - &[("GIT_PASSWORD", &password), ("GIT_USER", &username)], - ); + let (_, stderr, code) = + run_git_with_auth(&path, &cmd, &remote_auth, None).map_err(|e| FetchError::Other(e))?; + tracing::debug!("fetch_refs_from_url done {:?} {:?} {:?}", cmd, path, stderr); - if stderr.contains("fatal: Authentication failed") { + + if stderr.contains("fatal: Authentication failed") + || stderr.contains("fatal: Could not read") + || stderr.contains(": Permission denied") + { return Err(FetchError::AuthRequired); } - if stderr.contains("fatal:") { + + if stderr.contains("fatal:") || code != 0 { tracing::error!("{:?}", stderr); - return Err(FetchError::Other(josh::josh_error(&format!( - "git error: {:?}", - stderr + return Err(FetchError::Other(josh_error(&format!( + "git process exited with code {}: {:?}", + code, stderr )))); } + if stderr.contains("error:") { tracing::error!("{:?}", stderr); return Err(FetchError::Other(