Skip to content

running mega as sidecar in tauri app and update config #500

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
genedna merged 1 commit into from
Aug 9, 2024
Merged

running mega as sidecar in tauri app and update config #500

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 10 additions & 11 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ members = [
"lunar/src-tauri",
"atlas",
]
default-members = ["mega", "libra","aries"]
default-members = ["mega", "libra", "aries"]
exclude = ["craft"]
resolver = "1"

Expand All @@ -35,23 +35,22 @@ saturn = { path = "saturn" }
taurus = { path = "taurus" }
mega = { path = "mega" }
anyhow = "1.0.86"
serde = {version = "1.0.203", features = ["derive"]}
serde_json = "1.0.117"
serde = "1.0.205"
serde_json = "1.0.122"
tracing = "0.1.40"
tracing-subscriber = "0.3.18"
tracing-appender = "0.2"
thiserror = "1.0.61"
thiserror = "1.0.63"
rand = "0.8.5"
smallvec = "1.13.2"
tokio = "1.38.0"
tokio = "1.39.2"
tokio-stream = "0.1.15"
tokio-test = "0.4.4"
rayon = "1.10.0"
clap = "4.5.4"
async-trait = "0.1.80"
clap = "4.5.14"
async-trait = "0.1.81"
async-stream = "0.3.5"
bytes = "1.6.0"
memchr = "2.7.2"
bytes = "1.7.1"
memchr = "2.7.4"
chrono = "0.4.38"
sha1 = "0.10.6"
sha256 = "1.5"
Expand All @@ -67,7 +66,7 @@ tower = "0.4.13"
hex = "0.4.3"
sea-orm = "1.0.0"
flate2 = "1.0.30"
bstr = "1.9.1"
bstr = "1.10.0"
colored = "2.1.0"
idgenerator = "2.0.0"
num_cpus = "1.16.0"
Expand Down
21 changes: 0 additions & 21 deletions ceres/src/model/create_file.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,5 @@
use std::cell::RefCell;

use serde::{Deserialize, Serialize};

use mercury::{hash::SHA1, internal::object::tree::TreeItemMode};

use venus::monorepo::mega_node::MegaNode;

#[derive(PartialEq, Eq, Debug, Clone, Default, Serialize, Deserialize)]
pub struct CreateFileInfo {
/// can be a file or directory
Expand All @@ -16,18 +10,3 @@ pub struct CreateFileInfo {
// pub import_dir: bool,
pub content: Option<String>,
}


impl From<CreateFileInfo> for MegaNode {
fn from(value: CreateFileInfo) -> Self {
MegaNode {
name: value.name,
path: value.path.parse().unwrap(),
is_directory: value.is_directory,
children: RefCell::new(vec![]),
id: SHA1::default(),
mode: TreeItemMode::Tree,
commit_id: SHA1::default(),
}
}
}
1 change: 1 addition & 0 deletions gateway/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,4 @@ reqwest = { workspace = true, features = ["json"] }
uuid = { workspace = true, features = ["v4"] }
regex = "1.10.4"
ed25519-dalek = { version = "2.1.1", features = ["pkcs8"] }
lazy_static ={ workspace = true }
75 changes: 37 additions & 38 deletions gateway/src/https_server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ use axum::routing::get;
use axum::Router;
use axum_server::tls_rustls::RustlsConfig;
use clap::Args;
use lazy_static::lazy_static;
use regex::Regex;
use tokio::sync::Mutex;
use tower::ServiceBuilder;
Expand Down Expand Up @@ -160,6 +161,7 @@ pub async fn app(config: Config, host: String, port: u16, common: CommonOptions)
sessions: Arc::new(Mutex::new(HashMap::new())),
}),
)
// Using Regular Expressions for Path Matching in Protocol
.route(
"/*path",
get(get_method_router)
Expand All @@ -177,32 +179,43 @@ pub async fn app(config: Config, host: String, port: u16, common: CommonOptions)
.with_state(state)
}

lazy_static! {
//GET
static ref OBJECTS_REGEX: Regex = Regex::new(r"/objects/[a-z0-9]+$").unwrap();
static ref LOCKS_REGEX: Regex = Regex::new(r"/locks$").unwrap();
static ref INFO_REFS_REGEX: Regex = Regex::new(r"/info/refs$").unwrap();
//POST
static ref REGEX_LOCKS_VERIFY: Regex = Regex::new(r"/locks/verify$").unwrap();
static ref REGEX_UNLOCK: Regex = Regex::new(r"/unlock$").unwrap();
static ref REGEX_OBJECTS_BATCH: Regex = Regex::new(r"/objects/batch$").unwrap();
static ref REGEX_OBJECTS_CHUNKIDS: Regex = Regex::new(r"objects/chunkids$").unwrap();
static ref REGEX_GIT_UPLOAD_PACK: Regex = Regex::new(r"/git-upload-pack$").unwrap();
static ref REGEX_GIT_RECEIVE_PACK: Regex = Regex::new(r"/git-receive-pack$").unwrap();
}

async fn get_method_router(
state: State<AppState>,
Query(params): Query<GetParams>,
uri: Uri,
) -> Result<Response<Body>, (StatusCode, String)> {
let lfs_config: LfsConfig = state.deref().to_owned().into();
// Routing LFS services.
if Regex::new(r"/objects/[a-z0-9]+$")
.unwrap()
.is_match(uri.path())
{
if OBJECTS_REGEX.is_match(uri.path()) {
lfs::lfs_download_object(&lfs_config, uri.path()).await
} else if Regex::new(r"/locks$").unwrap().is_match(uri.path()) {
return lfs::lfs_retrieve_lock(&lfs_config, params).await;
} else if Regex::new(r"/info/refs$").unwrap().is_match(uri.path()) {
} else if LOCKS_REGEX.is_match(uri.path()) {
lfs::lfs_retrieve_lock(&lfs_config, params).await
} else if INFO_REFS_REGEX.is_match(uri.path()) {
let pack_protocol = SmartProtocol::new(
remove_git_suffix(uri, "/info/refs"),
state.context.clone(),
TransportProtocol::Http,
);
return crate::git_protocol::http::git_info_refs(params, pack_protocol).await;
crate::git_protocol::http::git_info_refs(params, pack_protocol).await
} else {
return Err((
Err((
StatusCode::NOT_FOUND,
String::from("Operation not supported\n"),
));
))
}
}

Expand All @@ -213,38 +226,27 @@ async fn post_method_router(
) -> Result<Response, (StatusCode, String)> {
let lfs_config: LfsConfig = state.deref().to_owned().into();
// Routing LFS services.
if Regex::new(r"/locks/verify$").unwrap().is_match(uri.path()) {
if REGEX_LOCKS_VERIFY.is_match(uri.path()) {
lfs::lfs_verify_lock(state, &lfs_config, req).await
} else if Regex::new(r"/locks$").unwrap().is_match(uri.path()) {
return lfs::lfs_create_lock(state, &lfs_config, req).await;
} else if Regex::new(r"/unlock$").unwrap().is_match(uri.path()) {
return lfs::lfs_delete_lock(state, &lfs_config, uri.path(), req).await;
} else if Regex::new(r"/objects/batch$").unwrap().is_match(uri.path()) {
return lfs::lfs_process_batch(state, &lfs_config, req).await;
} else if Regex::new(r"objects/chunkids$")
.unwrap()
.is_match(uri.path())
{
return lfs::lfs_fetch_chunk_ids(state, &lfs_config, req).await;
}
// Routing git services.
else if Regex::new(r"/git-upload-pack$")
.unwrap()
.is_match(uri.path())
{
} else if LOCKS_REGEX.is_match(uri.path()) {
lfs::lfs_create_lock(state, &lfs_config, req).await
} else if REGEX_UNLOCK.is_match(uri.path()) {
lfs::lfs_delete_lock(state, &lfs_config, uri.path(), req).await
} else if REGEX_OBJECTS_BATCH.is_match(uri.path()) {
lfs::lfs_process_batch(state, &lfs_config, req).await
} else if REGEX_OBJECTS_CHUNKIDS.is_match(uri.path()) {
lfs::lfs_fetch_chunk_ids(state, &lfs_config, req).await
} else if REGEX_GIT_UPLOAD_PACK.is_match(uri.path()) {
let mut pack_protocol = SmartProtocol::new(
remove_git_suffix(uri, "/git-upload-pack"),
remove_git_suffix(uri.clone(), "/git-upload-pack"),
state.context.clone(),
TransportProtocol::Http,
);
pack_protocol.service_type = Some(ServiceType::UploadPack);
crate::git_protocol::http::git_upload_pack(req, pack_protocol).await
} else if Regex::new(r"/git-receive-pack$")
.unwrap()
.is_match(uri.path())
{
} else if REGEX_GIT_RECEIVE_PACK.is_match(uri.path()) {
let mut pack_protocol = SmartProtocol::new(
remove_git_suffix(uri, "/git-receive-pack"),
remove_git_suffix(uri.clone(), "/git-receive-pack"),
state.context.clone(),
TransportProtocol::Http,
);
Expand All @@ -264,10 +266,7 @@ async fn put_method_router(
req: Request<Body>,
) -> Result<Response<Body>, (StatusCode, String)> {
let lfs_config: LfsConfig = state.deref().to_owned().into();
if Regex::new(r"/objects/[a-z0-9]+$")
.unwrap()
.is_match(uri.path())
{
if OBJECTS_REGEX.is_match(uri.path()) {
lfs::lfs_upload_object(&lfs_config, uri.path(), req).await
} else {
Err((
Expand Down
1 change: 0 additions & 1 deletion jupiter/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ async-trait = { workspace = true }
futures = { workspace = true }
serde_json = { workspace = true }
idgenerator = { workspace = true }
rayon = { workspace = true }
handlebars = "6.0.0"

[dev-dependencies]
Expand Down
93 changes: 57 additions & 36 deletions jupiter/src/storage/git_db_storage.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
use std::sync::{Arc, Mutex};

use async_trait::async_trait;
use futures::Stream;
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
use futures::{stream, Stream, StreamExt};
use sea_orm::sea_query::Expr;
use sea_orm::{
ActiveModelTrait, ColumnTrait, DatabaseConnection, DbBackend, DbErr, EntityTrait,
Expand Down Expand Up @@ -33,6 +32,15 @@ pub struct GitDbStorage {
pub raw_obj_threshold: usize,
}

#[derive(Debug)]
struct GitObjects {
pub commits: Vec<git_commit::ActiveModel>,
trees: Vec<git_tree::ActiveModel>,
blobs: Vec<git_blob::ActiveModel>,
raw_blobs: Vec<raw_blob::ActiveModel>,
tags: Vec<git_tag::ActiveModel>,
}

#[async_trait]
impl GitStorageProvider for GitDbStorage {
async fn save_ref(&self, repo: &Repo, refs: &RefCommand) -> Result<(), MegaError> {
Expand Down Expand Up @@ -127,50 +135,63 @@ impl GitDbStorage {
}

pub async fn save_entry(&self, repo: &Repo, entry_list: Vec<Entry>) -> Result<(), MegaError> {
let (commits, trees, blobs, raw_blobs, tags) = (
Mutex::new(Vec::new()),
Mutex::new(Vec::new()),
Mutex::new(Vec::new()),
Mutex::new(Vec::new()),
Mutex::new(Vec::new()),
);
entry_list.par_iter().for_each(|entry| {
let raw_obj = entry.process_entry();
let model = raw_obj.convert_to_git_model();
match model {
GitObjectModel::Commit(mut commit) => {
commit.repo_id = repo.repo_id;
commits.lock().unwrap().push(commit.into_active_model())
}
GitObjectModel::Tree(mut tree) => {
tree.repo_id = repo.repo_id;
trees.lock().unwrap().push(tree.clone().into_active_model());
let git_objects = Arc::new(Mutex::new(GitObjects {
commits: Vec::new(),
trees: Vec::new(),
blobs: Vec::new(),
raw_blobs: Vec::new(),
tags: Vec::new(),
}));

stream::iter(entry_list)
.for_each_concurrent(None, |entry| {
let git_objects = git_objects.clone();

async move {
let raw_obj = entry.process_entry();
let model = raw_obj.convert_to_git_model();
let mut git_objects = git_objects.lock().unwrap();

match model {
GitObjectModel::Commit(mut commit) => {
commit.repo_id = repo.repo_id;
git_objects.commits.push(commit.into_active_model())
}
GitObjectModel::Tree(mut tree) => {
tree.repo_id = repo.repo_id;
git_objects.trees.push(tree.clone().into_active_model());
}
GitObjectModel::Blob(mut blob, raw) => {
blob.repo_id = repo.repo_id;
git_objects.blobs.push(blob.clone().into_active_model());
git_objects.raw_blobs.push(raw.into_active_model());
}
GitObjectModel::Tag(mut tag) => {
tag.repo_id = repo.repo_id;
git_objects.tags.push(tag.into_active_model())
}
}
}
GitObjectModel::Blob(mut blob, raw) => {
blob.repo_id = repo.repo_id;
blobs.lock().unwrap().push(blob.clone().into_active_model());
raw_blobs.lock().unwrap().push(raw.into_active_model());
}
GitObjectModel::Tag(mut tag) => {
tag.repo_id = repo.repo_id;
tags.lock().unwrap().push(tag.into_active_model())
}
}
});
})
.await;

batch_save_model(self.get_connection(), commits.into_inner().unwrap())
let git_objects = Arc::try_unwrap(git_objects)
.expect("Failed to unwrap Arc")
.into_inner()
.unwrap();
batch_save_model(self.get_connection(), git_objects.commits)
.await
.unwrap();
batch_save_model(self.get_connection(), trees.into_inner().unwrap())
batch_save_model(self.get_connection(), git_objects.trees)
.await
.unwrap();
batch_save_model(self.get_connection(), blobs.into_inner().unwrap())
batch_save_model(self.get_connection(), git_objects.blobs)
.await
.unwrap();
batch_save_model(self.get_connection(), raw_blobs.into_inner().unwrap())
batch_save_model(self.get_connection(), git_objects.raw_blobs)
.await
.unwrap();
batch_save_model(self.get_connection(), tags.into_inner().unwrap())
batch_save_model(self.get_connection(), git_objects.tags)
.await
.unwrap();
Ok(())
Expand Down
Loading