diff --git a/Cargo.lock b/Cargo.lock index 828eab0..93d745c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -315,9 +315,11 @@ dependencies = [ name = "code0-definition-reader" version = "0.0.0" dependencies = [ + "log", "serde", "serde_json", "tucana", + "walkdir", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 393f1a6..5b6aada 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,4 +18,6 @@ tokio = "1.47.0" futures = "0.3.31" zip = "6.0.0" bytes = "1.10.1" -prost = "0.14.1" \ No newline at end of file +prost = "0.14.1" +walkdir = "2.5.0" +log = "0.4.28" \ No newline at end of file diff --git a/crates/cli/src/analyser/flow_type.rs b/crates/cli/src/analyser/flow_type.rs index 1ecb51a..beb8feb 100644 --- a/crates/cli/src/analyser/flow_type.rs +++ b/crates/cli/src/analyser/flow_type.rs @@ -36,27 +36,27 @@ impl Analyser { )); } - if let Some(identifier) = &flow.input_type_identifier { - if !self.data_type_identifier_exists(identifier, None) { - self.reporter.add(Diagnose::new( - name.clone(), - original.clone(), - DiagnosticKind::UndefinedDataTypeIdentifier { - identifier: identifier.clone(), - }, - )); - } + if let Some(identifier) = &flow.input_type_identifier + && !self.data_type_identifier_exists(identifier, None) + { + self.reporter.add(Diagnose::new( + name.clone(), + original.clone(), + DiagnosticKind::UndefinedDataTypeIdentifier { + identifier: identifier.clone(), + }, + )); } - if let Some(identifier) = &flow.return_type_identifier { - if !self.data_type_identifier_exists(identifier, None) { - self.reporter.add(Diagnose::new( - name.clone(), - original.clone(), - DiagnosticKind::UndefinedDataTypeIdentifier { - identifier: identifier.clone(), - }, - )); - } + if let Some(identifier) = &flow.return_type_identifier + && !self.data_type_identifier_exists(identifier, None) + { + self.reporter.add(Diagnose::new( + name.clone(), + original.clone(), + DiagnosticKind::UndefinedDataTypeIdentifier { + identifier: identifier.clone(), + }, + )); } for setting in &flow.settings { diff --git a/crates/package/Cargo.toml b/crates/package/Cargo.toml index c598b3f..3bae6d1 100644 --- a/crates/package/Cargo.toml +++ b/crates/package/Cargo.toml @@ -9,6 +9,8 @@ license = "Apache-2.0" readme = "../../README.md" [dependencies] -serde = { workspace = true } -serde_json = { workspace = true } -tucana = { workspace = true } \ No newline at end of file +walkdir = {workspace = true} +serde_json = {workspace = true} +serde = {workspace = true} +tucana = {workspace = true} +log = {workspace = true} \ No newline at end of file diff --git a/crates/package/src/enum/mod.rs b/crates/package/src/enum/mod.rs new file mode 100644 index 0000000..e98e7c8 --- /dev/null +++ b/crates/package/src/enum/mod.rs @@ -0,0 +1 @@ +pub mod reader_error; diff --git a/crates/package/src/enum/reader_error.rs b/crates/package/src/enum/reader_error.rs new file mode 100644 index 0000000..9c62cd0 --- /dev/null +++ b/crates/package/src/enum/reader_error.rs @@ -0,0 +1,24 @@ +use serde_json; +use std::io; +use std::path::PathBuf; + +#[derive(Debug)] +pub enum ReaderError { + JsonError { + path: PathBuf, + error: serde_json::Error, + }, + ReadFeatureError { + path: String, + source: Box, + }, + ReadDirectoryError { + path: PathBuf, + error: io::Error, + }, + ReadFileError { + path: PathBuf, + error: io::Error, + }, + DirectoryEntryError(io::Error), +} diff --git a/crates/package/src/lib.rs b/crates/package/src/lib.rs index a84d348..c6546bd 100644 --- a/crates/package/src/lib.rs +++ b/crates/package/src/lib.rs @@ -1,300 +1,155 @@ -pub mod package { - use serde::Serialize; - use std::io::ErrorKind; - use std::{ - fs::{self, DirEntry}, - io::Error, - path::Path, - }; - use tucana::shared::{DefinitionDataType, FlowType, RuntimeFunctionDefinition}; +mod r#enum; +mod r#struct; + +use crate::r#enum::reader_error::ReaderError; +use crate::r#struct::feature::Feature; +use serde::de::DeserializeOwned; +use std::fs; +use std::path::Path; +use tucana::shared::{DefinitionDataType, FlowType, RuntimeFunctionDefinition, Version}; +use walkdir::WalkDir; + +pub struct Reader { + should_break: bool, + accepted_features: Vec, + accepted_versions: Option, + path: String, +} - #[derive(Serialize, Clone, Debug)] - pub struct DefinitionError { - pub definition: String, - pub definition_type: MetaType, - pub error: String, +impl Reader { + pub fn configure( + path: String, + should_break: bool, + accepted_features: Vec, + accepted_versions: Option, + ) -> Self { + Self { + should_break, + accepted_features, + accepted_versions, + path, + } } - #[derive(Debug)] - pub struct Parser { - pub features: Vec, - } + pub fn read_features(&self, path: &str) -> Result, ReaderError> { + let definitions = Path::new(path); - #[derive(Serialize, Clone, Debug)] - pub struct Feature { - pub name: String, - pub data_types: Vec, - pub flow_types: Vec, - pub runtime_functions: Vec, - pub errors: Vec, - } - - impl Feature { - fn new(name: String) -> Self { - Feature { - name, - data_types: Vec::new(), - flow_types: Vec::new(), - runtime_functions: Vec::new(), - errors: Vec::new(), + match self.read_feature_content(definitions) { + Ok(features) => { + log::info!("Loaded Successfully {} features", features.len()); + Ok(features) + } + Err(err) => { + log::error!("Failed to read features from {}", path); + Err(ReaderError::ReadFeatureError { + path: path.to_string(), + source: Box::new(err), + }) } } } - impl Parser { - pub fn from_path(path: &str) -> Option { - let reader = Reader::from_path(path)?; - - Some(Self::from_reader(reader)) - } - - pub fn from_reader(reader: Reader) -> Self { - let mut features: Vec = vec![]; - - for meta in &reader.meta { - let feature = features.iter_mut().find(|f| f.name == meta.name); - - if let Some(existing) = feature { - Parser::append_meta(existing, meta); - } else { - let mut new_feature = Feature::new(meta.name.clone()); - Parser::append_meta(&mut new_feature, meta); - features.push(new_feature); - } + fn read_feature_content(&self, dir: &Path) -> Result, ReaderError> { + let mut features: Vec = Vec::new(); + let readdir = match fs::read_dir(dir) { + Ok(readdir) => readdir, + Err(err) => { + log::error!("Failed to read directory {}: {}", dir.display(), err); + return Err(ReaderError::ReadDirectoryError { + path: dir.to_path_buf(), + error: err, + }); } + }; - Parser { features } - } - - pub fn extract_identifier(definition: &str, meta_type: MetaType) -> String { - let field_name = match meta_type { - MetaType::DataType | MetaType::FlowType => "identifier", - MetaType::RuntimeFunction => "runtime_name", + for entry_result in readdir { + let entry = match entry_result { + Ok(entry) => entry, + Err(err) => { + log::error!("Failed to read directory entry: {}", err); + return Err(ReaderError::DirectoryEntryError(err)); + } }; - // Look for the field pattern: "field_name": "value" or "field_name":"value" - if let Some(start) = definition.find(&format!("\"{field_name}\"")) { - // Find the colon after the field name - if let Some(colon_pos) = definition[start..].find(':') { - let after_colon = &definition[start + colon_pos + 1..]; + let path = entry.path(); - // Skip whitespace and find the opening quote - let trimmed = after_colon.trim_start(); - if let Some(stripped) = trimmed.strip_prefix('"') { - // Find the closing quote - if let Some(end_quote) = stripped.find('"') { - return trimmed[1..end_quote + 1].to_string(); - } - } - } - } - - // Fallback: return the whole definition if identifier can't be extracted - definition.to_string() - } + if path.is_dir() { + let feature_name = path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .to_string(); - fn append_meta(feature: &mut Feature, meta: &Meta) { - let definition = meta.definition_string.as_str(); - match meta.r#type { - MetaType::DataType => { - match serde_json::from_str::(definition) { - Ok(data_type) => feature.data_types.push(data_type), - Err(err) => feature.errors.push(DefinitionError { - definition: Parser::extract_identifier(definition, MetaType::DataType), - definition_type: MetaType::DataType, - error: err.to_string(), - }), - } - } - MetaType::FlowType => match serde_json::from_str::(definition) { - Ok(flow_type) => feature.flow_types.push(flow_type), - Err(err) => feature.errors.push(DefinitionError { - definition: Parser::extract_identifier(definition, MetaType::FlowType), - definition_type: MetaType::FlowType, - error: err.to_string(), - }), - }, - MetaType::RuntimeFunction => { - match serde_json::from_str::(definition) { - Ok(func) => feature.runtime_functions.push(func), - Err(err) => feature.errors.push(DefinitionError { - definition: Parser::extract_identifier( - definition, - MetaType::RuntimeFunction, - ), - definition_type: MetaType::RuntimeFunction, - error: err.to_string(), - }), - } + if !self.accepted_features.is_empty() + && !self.accepted_features.contains(&feature_name) + { + log::info!("Skipping feature: {}", feature_name); + continue; } - } - } - } - #[derive(Serialize, Debug, Clone, Copy)] - pub enum MetaType { - FlowType, - DataType, - RuntimeFunction, - } + let data_types_path = path.join("data_type"); + let data_types: Vec = + self.collect_definitions(&data_types_path)?; - impl std::fmt::Display for MetaType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - MetaType::FlowType => write!(f, "FlowType"), - MetaType::DataType => write!(f, "DataType"), - MetaType::RuntimeFunction => write!(f, "RuntimeFunction"), - } - } - } + let flow_types_path = path.join("flow_type"); + let flow_types: Vec = self.collect_definitions(&flow_types_path)?; - pub struct Reader { - pub meta: Vec, - } - - #[derive(Clone)] - pub struct Meta { - pub name: String, - pub r#type: MetaType, - pub definition_string: String, - pub path: String, - } + let functions_path = path.join("runtime_definition"); + let functions: Vec = + self.collect_definitions(&functions_path)?; - impl Meta { - pub fn read_from_file

( - name: String, - r#type: MetaType, - file_path: P, - ) -> Result - where - P: AsRef, - { - let path = match file_path.as_ref().to_str() { - Some(path) => path, - None => return Err(Error::new(ErrorKind::InvalidInput, "Invalid path")), - }; + let feature = Feature { + name: feature_name, + data_types, + flow_types, + functions, + }; - if !path.ends_with("json") { - return Err(Error::new( - ErrorKind::InvalidInput, - format!( - "File {} does not end with .json", - file_path.as_ref().display() - ), - )); + features.push(feature); } - - let content = match fs::read_to_string(&file_path) { - Ok(content) => content, - Err(err) => { - println!("Error reading file: {err}"); - return Err(err); - } - }; - - Ok(Meta { - name, - r#type, - definition_string: content, - path: path.to_string(), - }) } + + Ok(features) } - /// Reader - /// - /// Expecting the file system to look like: - /// - - /// - - /// - - /// - - /// - - /// - - /// - - /// - - /// - - impl Reader { - pub fn from_path(path: &str) -> Option { - let mut result: Vec = vec![]; + fn collect_definitions(&self, dir: &Path) -> Result, ReaderError> + where + T: DeserializeOwned, + { + let mut definitions = Vec::new(); - // Reading the path folder - for feature_path in fs::read_dir(path).unwrap() { - let feature_path_result = match feature_path { - Ok(path) => path, - Err(_) => continue, - }; + if !dir.exists() { + return Ok(definitions); + } - let feature_name = match get_file_name(&feature_path_result) { - Some(file_name) => file_name, - None => continue, + for entry in WalkDir::new(dir).into_iter().filter_map(Result::ok) { + let path = entry.path(); + + if path.is_file() && path.extension().is_some_and(|ext| ext == "json") { + let content = match fs::read_to_string(path) { + Ok(content) => content, + Err(err) => { + log::error!("Failed to read file {}: {}", path.display(), err); + return Err(ReaderError::ReadFileError { + path: path.to_path_buf(), + error: err, + }); + } }; - // Reading the feature folder - for type_path in fs::read_dir(feature_path_result.path()).unwrap() { - let type_path_result = match type_path { - Ok(path) => path, - Err(_) => continue, - }; - - let meta_type = match get_file_name(&type_path_result) { - Some(name) => match name.as_str() { - "flow_type" => MetaType::FlowType, - "data_type" => MetaType::DataType, - "runtime_definition" => MetaType::RuntimeFunction, - _ => continue, - }, - None => continue, - }; - - // Reading the type folder - for definition_path in fs::read_dir(type_path_result.path()).unwrap() { - let definition_path_result = match definition_path { - Ok(path) => path, - Err(_) => continue, - }; - - if definition_path_result.file_type().unwrap().is_file() { - let meta = Meta::read_from_file( - feature_name.clone(), - meta_type, - definition_path_result.path(), - ); - - if let Ok(meta_result) = meta { - result.push(meta_result); - } - } else { - for sub_definition_path in - fs::read_dir(definition_path_result.path()).unwrap() - { - let sub_definition_path_result = match sub_definition_path { - Ok(path) => path, - Err(_) => continue, - }; - - let meta = Meta::read_from_file( - feature_name.clone(), - meta_type, - sub_definition_path_result.path(), - ); - - if let Ok(meta_result) = meta { - result.push(meta_result); - } - } - } + match serde_json::from_str::(&content) { + Ok(def) => definitions.push(def), + Err(e) => { + log::error!("Failed to parse JSON in file {}: {}", path.display(), e); + return Err(ReaderError::JsonError { + path: path.to_path_buf(), + error: e, + }); } } } - - Some(Reader { meta: result }) } - } - fn get_file_name(entry: &DirEntry) -> Option { - entry - .file_name() - .to_str() - .map(|file_name| file_name.to_string()) + Ok(definitions) } } diff --git a/crates/package/src/struct/feature.rs b/crates/package/src/struct/feature.rs new file mode 100644 index 0000000..8f815cc --- /dev/null +++ b/crates/package/src/struct/feature.rs @@ -0,0 +1,10 @@ +use serde::Deserialize; +use tucana::shared::{DefinitionDataType, FlowType, RuntimeFunctionDefinition}; + +#[derive(Deserialize, Debug, Clone)] +pub struct Feature { + pub name: String, + pub data_types: Vec, + pub flow_types: Vec, + pub functions: Vec, +} diff --git a/crates/package/src/struct/mod.rs b/crates/package/src/struct/mod.rs new file mode 100644 index 0000000..c193906 --- /dev/null +++ b/crates/package/src/struct/mod.rs @@ -0,0 +1 @@ +pub mod feature;