From c0f02113446f1ef80aefd4578a12600d0df8e7e1 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Thu, 9 Oct 2025 15:27:45 -0700 Subject: [PATCH 1/2] Handle qs. --- CHANGELOG.md | 6 +++ crates/schematic/src/config/source.rs | 42 +------------------ crates/schematic/src/format.rs | 12 +++--- crates/schematic/src/helpers.rs | 51 ++++++++++++++++++++++++ crates/schematic/src/lib.rs | 1 + crates/schematic/src/validate/extends.rs | 22 +++------- crates/schematic/src/validate/url.rs | 2 +- 7 files changed, 73 insertions(+), 63 deletions(-) create mode 100644 crates/schematic/src/helpers.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 22d8faa6..8004b623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +#### 🐞 Fixes + +- Fixed an issue where source URLs that contain a query string would not be parsed. + ## 0.18.13 #### 🐞 Fixes diff --git a/crates/schematic/src/config/source.rs b/crates/schematic/src/config/source.rs index 7a71a2e4..01563476 100644 --- a/crates/schematic/src/config/source.rs +++ b/crates/schematic/src/config/source.rs @@ -1,6 +1,7 @@ use super::cacher::BoxedCacher; use super::error::ConfigError; use crate::format::Format; +use crate::helpers::*; use serde::Deserialize; use serde::{Serialize, de::DeserializeOwned}; use std::fs; @@ -169,44 +170,3 @@ impl Source { } } } - -/// Returns true if the value ends in a supported file extension. -pub fn is_source_format(value: &str) -> bool { - value.ends_with(".json") - || value.ends_with(".pkl") - || value.ends_with(".toml") - || value.ends_with(".yaml") - || value.ends_with(".yml") -} - -/// Returns true if the value looks like a file, by checking for `file://`, -/// path separators, or supported file extensions. -pub fn is_file_like(value: &str) -> bool { - value.starts_with("file://") - || value.starts_with('/') - || value.starts_with('\\') - || value.starts_with('.') - || value.contains('/') - || value.contains('\\') - || value.contains('.') -} - -/// Returns true if the value looks like a URL, by checking for `http://`, `https://`, or `www`. -pub fn is_url_like(value: &str) -> bool { - value.starts_with("https://") || value.starts_with("http://") || value.starts_with("www") -} - -/// Returns true if the value is a secure URL, by checking for `https://`. This check can be -/// bypassed for localhost URLs. -pub fn is_secure_url(value: &str) -> bool { - if value.contains("127.0.0.1") || value.contains("//localhost") { - return true; - } - - value.starts_with("https://") -} - -/// Strip a leading BOM from the string. -pub fn strip_bom(content: &str) -> &str { - content.trim_start_matches("\u{feff}") -} diff --git a/crates/schematic/src/format.rs b/crates/schematic/src/format.rs index 7680fc59..c993f2b8 100644 --- a/crates/schematic/src/format.rs +++ b/crates/schematic/src/format.rs @@ -1,3 +1,4 @@ +use crate::helpers::extract_ext; use miette::Diagnostic; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -33,12 +34,13 @@ impl Format { /// checking for a supported file extension. pub fn detect(value: &str) -> Result { let mut available: Vec<&str> = vec![]; + let ext = extract_ext(value).unwrap_or_default(); #[cfg(feature = "json")] { available.push("JSON"); - if value.ends_with(".json") { + if ext == ".json" { return Ok(Format::Json); } } @@ -47,7 +49,7 @@ impl Format { { available.push("Pkl"); - if value.ends_with(".pkl") { + if ext == ".pkl" { return Ok(Format::Pkl); } } @@ -56,7 +58,7 @@ impl Format { { available.push("RON"); - if value.ends_with(".ron") { + if ext == ".ron" { return Ok(Format::Ron); } } @@ -65,7 +67,7 @@ impl Format { { available.push("TOML"); - if value.ends_with(".toml") { + if ext == ".toml" { return Ok(Format::Toml); } } @@ -74,7 +76,7 @@ impl Format { { available.push("YAML"); - if value.ends_with(".yaml") || value.ends_with(".yml") { + if ext == ".yaml" || ext == ".yml" { return Ok(Format::Yaml); } } diff --git a/crates/schematic/src/helpers.rs b/crates/schematic/src/helpers.rs new file mode 100644 index 00000000..efb82d8a --- /dev/null +++ b/crates/schematic/src/helpers.rs @@ -0,0 +1,51 @@ +/// Returns true if the value ends in a supported file extension. +pub fn is_source_format(value: &str) -> bool { + extract_ext(value).is_some_and(|ext| { + ext == ".json" || ext == ".pkl" || ext == ".toml" || ext == ".yaml" || ext == ".yml" + }) +} + +/// Returns true if the value looks like a file, by checking for `file://`, +/// path separators, or supported file extensions. +pub fn is_file_like(value: &str) -> bool { + value.starts_with("file://") + || value.starts_with('/') + || value.starts_with('\\') + || value.starts_with('.') + || value.contains('/') + || value.contains('\\') + || value.contains('.') +} + +/// Returns true if the value looks like a URL, by checking for `http://`, `https://`, or `www`. +pub fn is_url_like(value: &str) -> bool { + value.starts_with("https://") || value.starts_with("http://") || value.starts_with("www") +} + +/// Returns true if the value is a secure URL, by checking for `https://`. This check can be +/// bypassed for localhost URLs. +pub fn is_secure_url(value: &str) -> bool { + if value.contains("127.0.0.1") || value.contains("//localhost") { + return true; + } + + value.starts_with("https://") +} + +/// Strip a leading BOM from the string. +pub fn strip_bom(content: &str) -> &str { + content.trim_start_matches("\u{feff}") +} + +/// Extract a file extension from the provided file path or URL. +pub fn extract_ext(value: &str) -> Option<&str> { + let value = if is_url_like(value) + && let Some(index) = value.rfind('?') + { + &value[0..index] + } else { + value + }; + + value.rfind('.').map(|index| &value[index..]) +} diff --git a/crates/schematic/src/lib.rs b/crates/schematic/src/lib.rs index 8476c7f8..6b04283a 100644 --- a/crates/schematic/src/lib.rs +++ b/crates/schematic/src/lib.rs @@ -1,6 +1,7 @@ #![allow(clippy::result_large_err)] mod format; +pub mod helpers; #[cfg(feature = "config")] mod config; diff --git a/crates/schematic/src/validate/extends.rs b/crates/schematic/src/validate/extends.rs index 4b6945ae..fba7b65f 100644 --- a/crates/schematic/src/validate/extends.rs +++ b/crates/schematic/src/validate/extends.rs @@ -1,7 +1,5 @@ -use crate::config::{ - ExtendsFrom, Path, PathSegment, ValidateError, ValidateResult, is_file_like, is_secure_url, - is_source_format, is_url_like, -}; +use crate::config::{ExtendsFrom, Path, PathSegment, ValidateError, ValidateResult}; +use crate::helpers::*; /// Validate an `extend` value is either a file path or secure URL. pub fn extends_string( @@ -19,18 +17,10 @@ pub fn extends_string( )); } - if !value.is_empty() { - let value = if is_url && let Some(index) = value.rfind('?') { - &value[0..index] - } else { - value - }; - - if !is_source_format(value) { - return Err(ValidateError::new( - "invalid file format, try a supported extension", - )); - } + if !value.is_empty() && !is_source_format(value) { + return Err(ValidateError::new( + "invalid file format, try a supported extension", + )); } if is_url && !is_secure_url(value) { diff --git a/crates/schematic/src/validate/url.rs b/crates/schematic/src/validate/url.rs index 2654922d..3e6548ca 100644 --- a/crates/schematic/src/validate/url.rs +++ b/crates/schematic/src/validate/url.rs @@ -1,5 +1,5 @@ use super::{ValidateError, ValidateResult, map_err}; -use crate::config::is_secure_url; +use crate::helpers::is_secure_url; pub use garde::rules::url::Url; /// Validate a string matches a URL. From 0669207083598eb05f8b7153bcb0cb2014d2a957 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Thu, 9 Oct 2025 15:40:15 -0700 Subject: [PATCH 2/2] Add tests. --- crates/schematic/src/helpers.rs | 12 ++++++-- crates/schematic/tests/helpers_test.rs | 39 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 crates/schematic/tests/helpers_test.rs diff --git a/crates/schematic/src/helpers.rs b/crates/schematic/src/helpers.rs index efb82d8a..e84289b3 100644 --- a/crates/schematic/src/helpers.rs +++ b/crates/schematic/src/helpers.rs @@ -39,13 +39,19 @@ pub fn strip_bom(content: &str) -> &str { /// Extract a file extension from the provided file path or URL. pub fn extract_ext(value: &str) -> Option<&str> { - let value = if is_url_like(value) - && let Some(index) = value.rfind('?') - { + // Remove any query string + let value = if let Some(index) = value.rfind('?') { &value[0..index] } else { value }; + // And only check the last segment + let value = if let Some(index) = value.rfind('/') { + &value[index + 1..] + } else { + value + }; + value.rfind('.').map(|index| &value[index..]) } diff --git a/crates/schematic/tests/helpers_test.rs b/crates/schematic/tests/helpers_test.rs new file mode 100644 index 00000000..8be19adc --- /dev/null +++ b/crates/schematic/tests/helpers_test.rs @@ -0,0 +1,39 @@ +use schematic::helpers::extract_ext; + +mod ext { + use super::*; + + #[test] + fn works_on_files() { + assert!(extract_ext("file").is_none()); + assert_eq!(extract_ext("file.json").unwrap(), ".json"); + assert_eq!(extract_ext("dir/file.yaml").unwrap(), ".yaml"); + assert_eq!(extract_ext("../file.toml").unwrap(), ".toml"); + assert_eq!(extract_ext("/root/file.other.json").unwrap(), ".json"); + } + + #[test] + fn works_on_urls() { + assert!(extract_ext("https://domain.com/file").is_none()); + assert_eq!( + extract_ext("https://domain.com/file.json").unwrap(), + ".json" + ); + assert_eq!( + extract_ext("http://domain.com/dir/file.yaml").unwrap(), + ".yaml" + ); + assert_eq!( + extract_ext("https://domain.com/file.toml?query").unwrap(), + ".toml" + ); + assert_eq!( + extract_ext("http://domain.com/root/file.other.json").unwrap(), + ".json" + ); + assert_eq!( + extract_ext("https://domain.com/other.segment/file.toml?query").unwrap(), + ".toml" + ); + } +}