Skip to content

Commit

Permalink
fix: move mapping retrieve in CustomMappign struct
Browse files Browse the repository at this point in the history
  • Loading branch information
nichmor committed May 23, 2024
1 parent 9f6fc90 commit 4784870
Show file tree
Hide file tree
Showing 5 changed files with 91 additions and 14 deletions.
4 changes: 2 additions & 2 deletions src/project/manifest/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use crate::project::manifest::channel::PrioritizedChannel;
use crate::project::manifest::environment::TomlEnvironmentMapOrSeq;
use crate::project::manifest::pypi_options::PypiOptions;
use crate::project::manifest::python::PyPiPackageName;
use crate::pypi_mapping::{ChannelName, MappingLocation, MappingSource};
use crate::pypi_mapping::{ChannelName, CustomMapping, MappingLocation, MappingSource};
use crate::task::TaskName;
use crate::{consts, project::SpecType, task::Task, utils::spanned::PixiSpanned};
pub use activation::Activation;
Expand Down Expand Up @@ -452,7 +452,7 @@ impl Manifest {
})
.collect::<miette::Result<HashMap<ChannelName, MappingLocation>>>()?;

Ok(MappingSource::Custom { mapping })
Ok(MappingSource::Custom(CustomMapping::new(mapping)))
},
None => Ok(MappingSource::Prefix),
}
Expand Down
1 change: 1 addition & 0 deletions src/project/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ use crate::activation::{get_environment_variables, run_activation};
use crate::config::Config;
use crate::consts::{self, PROJECT_MANIFEST, PYPROJECT_MANIFEST};
use crate::project::grouped_environment::GroupedEnvironment;

use crate::pypi_mapping::MappingSource;
use crate::utils::reqwest::build_reqwest_clients;
use manifest::{EnvironmentName, Manifest};
Expand Down
10 changes: 5 additions & 5 deletions src/pypi_mapping/custom_pypi_mapping.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use crate::pypi_mapping::MappingLocation;

use super::{
build_pypi_purl_from_package_record, is_conda_forge_record, prefix_pypi_name_mapping,
MappingMap, Reporter,
CustomMapping, MappingMap, Reporter,
};

pub async fn fetch_mapping_from_url<T>(
Expand Down Expand Up @@ -105,18 +105,18 @@ pub async fn fetch_custom_mapping(
/// Amend the records with pypi purls if they are not present yet.
pub async fn amend_pypi_purls(
client: &ClientWithMiddleware,
mapping_url: &MappingMap,
mapping_url: &CustomMapping,
conda_packages: &mut [RepoDataRecord],
reporter: Option<Arc<dyn Reporter>>,
) -> miette::Result<()> {
trim_conda_packages_channel_url_suffix(conda_packages);
let packages_for_prefix_mapping: Vec<RepoDataRecord> = conda_packages
.iter()
.filter(|package| !mapping_url.contains_key(&package.channel))
.filter(|package| !mapping_url.mapping.contains_key(&package.channel))
.cloned()
.collect();

let custom_mapping = fetch_custom_mapping(client, mapping_url).await?;
let custom_mapping = mapping_url.fetch_custom_mapping(client).await?;

// When all requested channels are present in the custom_mapping, we don't have to request from the prefix_mapping.
// This will avoid fetching unwanted URLs, e.g. behind corporate firewalls
Expand All @@ -133,7 +133,7 @@ pub async fn amend_pypi_purls(
prefix_pypi_name_mapping::conda_pypi_name_compressed_mapping(client).await?;

for record in conda_packages.iter_mut() {
if !mapping_url.contains_key(&record.channel) {
if !mapping_url.mapping.contains_key(&record.channel) {
prefix_pypi_name_mapping::amend_pypi_purls_for_record(
record,
&prefix_mapping,
Expand Down
89 changes: 82 additions & 7 deletions src/pypi_mapping/mod.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
use std::{collections::HashMap, path::PathBuf, str::FromStr, sync::Arc};

use async_once_cell::OnceCell as AsyncCell;
use http_cache_reqwest::{CACacheManager, Cache, CacheMode, HttpCache, HttpCacheOptions};
use miette::{Context, IntoDiagnostic};
use rattler_conda_types::{PackageRecord, PackageUrl, RepoDataRecord};
use reqwest_middleware::ClientBuilder;
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware};
use url::Url;

use crate::config::get_cache_dir;
use crate::{config::get_cache_dir, pypi_mapping::custom_pypi_mapping::fetch_mapping_from_url};

pub mod custom_pypi_mapping;
pub mod prefix_pypi_name_mapping;
Expand All @@ -27,21 +29,94 @@ pub enum MappingLocation {
Url(Url),
}

#[derive(Debug, Clone)]
/// Struct with a mapping of channel names to their respective mapping locations
/// location could be a remote url or local file
pub struct CustomMapping {
pub mapping: MappingMap,
}

impl CustomMapping {
/// Create a new `CustomMapping` with the specified mapping.
pub fn new(mapping: MappingMap) -> Self {
Self { mapping }
}

/// Fetch the custom mapping from the server or load from the local
pub async fn fetch_custom_mapping(
&self,
client: &ClientWithMiddleware,
) -> miette::Result<&'static HashMap<String, HashMap<String, Option<String>>>> {
static MAPPING: AsyncCell<HashMap<String, HashMap<String, Option<String>>>> =
AsyncCell::new();
MAPPING
.get_or_try_init(async {
let mut mapping_url_to_name: HashMap<String, HashMap<String, Option<String>>> =
Default::default();

for (name, url) in self.mapping.iter() {
// Fetch the mapping from the server or from the local

match url {
MappingLocation::Url(url) => {
let response = client
.get(url.clone())
.send()
.await
.into_diagnostic()
.context(format!(
"failed to download pypi mapping from {} location",
url.as_str()
))?;

if !response.status().is_success() {
return Err(miette::miette!(
"Could not request mapping located at {:?}",
url.as_str()
));
}

let mapping_by_name = fetch_mapping_from_url(client, url).await?;

mapping_url_to_name.insert(name.to_string(), mapping_by_name);
}
MappingLocation::Path(path) => {
let contents = std::fs::read_to_string(path)
.into_diagnostic()
.context(format!("mapping on {path:?} could not be loaded"))?;
let data: HashMap<String, Option<String>> =
serde_json::from_str(&contents).into_diagnostic().context(
format!(
"Failed to parse JSON mapping located at {}",
path.display()
),
)?;

mapping_url_to_name.insert(name.to_string(), data);
}
}
}

Ok(mapping_url_to_name)
})
.await
}
}

/// This enum represents the source of mapping
/// it can be user-defined ( custom )
/// or from prefix.dev ( prefix )

pub enum MappingSource {
Custom { mapping: MappingMap },
Custom(CustomMapping),
Prefix,
}

impl MappingSource {
/// Return the custom `MappingMap`
/// for `MappingSource::Custom`
pub fn custom(&self) -> Option<MappingMap> {
pub fn custom(&self) -> Option<CustomMapping> {
match self {
MappingSource::Custom { mapping } => Some(mapping.clone()),
MappingSource::Custom(mapping) => Some(mapping.clone()),
_ => None,
}
}
Expand Down Expand Up @@ -72,7 +147,7 @@ pub async fn amend_pypi_purls(
.build();

match mapping_source {
MappingSource::Custom { mapping } => {
MappingSource::Custom(mapping) => {
custom_pypi_mapping::amend_pypi_purls(&client, mapping, conda_packages, reporter)
.await?;
}
Expand Down
1 change: 1 addition & 0 deletions tests/solve_group_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use crate::common::{
package_database::{Package, PackageDatabase},
LockFileExt, PixiControl,
};

use pixi::pypi_mapping::{self};
use rattler_conda_types::{PackageName, Platform, RepoDataRecord};
use rattler_lock::DEFAULT_ENVIRONMENT_NAME;
Expand Down

0 comments on commit 4784870

Please sign in to comment.