diff --git a/gts/README.md b/gts/README.md index 32d684d..ba771f8 100644 --- a/gts/README.md +++ b/gts/README.md @@ -1,6 +1,48 @@ # GTS Python Library -A minimal, idiomatic Python library for working with **GTS** ([Global Type System](https://github.com/gts-spec/gts-spec)) identifiers and JSON/JSON Schema artifacts. +A minimal, idiomatic Python library for working with **GTS** ([Global Type System](https://github.com/gts-spec/gts-spec)) identifiers and type definitions. + +## File Format Support + +GTS Python supports multiple file formats for schemas and instances: + +### JSON (Native) +Standard JSON format with `.json`, `.jsonc`, and `.gts` extensions. + +### YAML +Full YAML support with `.yaml` and `.yml` extensions. YAML files are automatically parsed and treated identically to JSON. + +```python +from gts import GtsFileReader + +# Reads both JSON and YAML files +reader = GtsFileReader("path/to/schemas/") +for entity in reader: + print(f"{entity.gts_id.id}: {entity.file.name}") +``` + +### TypeSpec +TypeSpec (`.tsp`) schemas must be pre-compiled to JSON Schema before use with gts-python. + +**Setup:** +```bash +# Install TypeSpec compiler +npm install -g @typespec/compiler @typespec/json-schema + +# Compile TypeSpec to JSON Schema +tsp compile --emit @typespec/json-schema your-schemas/ +``` + +**Usage:** +```python +from gts import GtsFileReader + +# Point to the generated JSON Schema output directory +reader = GtsFileReader("tsp-output/@typespec/json-schema/") +entities = list(reader) +``` + +See [gts-spec TypeSpec examples](https://github.com/globaltypesystem/gts-spec/tree/main/examples/typespec) for sample TypeSpec definitions. ## Featureset @@ -23,10 +65,10 @@ print(is_valid) # True or False ```python import json -from gts import JsonEntity, DEFAULT_GTS_CONFIG +from gts import GtsEntity, DEFAULT_GTS_CONFIG content = json.load(open("path/to/file.json")) -entity = JsonEntity(content=content, cfg=DEFAULT_GTS_CONFIG) +entity = GtsEntity(content=content, cfg=DEFAULT_GTS_CONFIG) if entity.gts_id: print(entity.gts_id.id) ``` diff --git a/gts/pyproject.toml b/gts/pyproject.toml index 561797a..cb55041 100644 --- a/gts/pyproject.toml +++ b/gts/pyproject.toml @@ -13,7 +13,8 @@ requires-python = ">=3.9" dependencies = [ "jsonschema>=4.18,<5", "fastapi>=0.110,<1", - "uvicorn>=0.23,<1" + "uvicorn>=0.23,<1", + "pyyaml>=6.0,<7" ] [project.urls] diff --git a/gts/src/gts/__init__.py b/gts/src/gts/__init__.py index b8669be..25b4501 100644 --- a/gts/src/gts/__init__.py +++ b/gts/src/gts/__init__.py @@ -6,12 +6,12 @@ from .entities import ( ValidationError, ValidationResult, - JsonFile, - JsonEntity, + GtsFile, + GtsEntity, GtsConfig, DEFAULT_GTS_CONFIG, ) -from .path_resolver import JsonPathResolver +from .path_resolver import GtsPathResolver from .store import ( GtsReader, GtsStore, @@ -26,12 +26,21 @@ "GtsWildcard", "ValidationError", "ValidationResult", - "JsonFile", - "JsonEntity", - "JsonPathResolver", + "GtsFile", + "GtsEntity", + "GtsPathResolver", "GtsConfig", "DEFAULT_GTS_CONFIG", "GtsReader", "GtsStore", "GtsFileReader", + # Backward compatibility aliases + "JsonFile", + "JsonEntity", + "JsonPathResolver", ] + +# Backward compatibility aliases +JsonFile = GtsFile +JsonEntity = GtsEntity +JsonPathResolver = GtsPathResolver diff --git a/gts/src/gts/entities.py b/gts/src/gts/entities.py index 5581b09..271b3ed 100644 --- a/gts/src/gts/entities.py +++ b/gts/src/gts/entities.py @@ -4,8 +4,8 @@ from typing import Any, Dict, List, Optional, Tuple from .gts import GtsID -from .path_resolver import JsonPathResolver -from .schema_cast import JsonEntityCastResult, SchemaCastError +from .path_resolver import GtsPathResolver +from .schema_cast import GtsEntityCastResult, SchemaCastError @dataclass @@ -24,7 +24,7 @@ class ValidationResult: @dataclass -class JsonFile: +class GtsFile: path: str name: str content: Any @@ -74,10 +74,10 @@ class GtsConfig: @dataclass -class JsonEntity: +class GtsEntity: gts_id: Optional[GtsID] = None is_schema: bool = False - file: Optional[JsonFile] = None + file: Optional[GtsFile] = None list_sequence: Optional[int] = None label: str = "" content: Any = None @@ -92,7 +92,7 @@ class JsonEntity: def __init__( self, *, - file: Optional[JsonFile] = None, + file: Optional[GtsFile] = None, list_sequence: Optional[int] = None, content: Any = None, cfg: Optional[GtsConfig] = None, @@ -168,11 +168,11 @@ def _is_json_schema_entity(self) -> bool: return True return False - def resolve_path(self, path: str) -> JsonPathResolver: - resolver = JsonPathResolver(self.gts_id.id if self.gts_id else '', self.content) + def resolve_path(self, path: str) -> GtsPathResolver: + resolver = GtsPathResolver(self.gts_id.id if self.gts_id else '', self.content) return resolver.resolve(path) - def cast(self, to_schema: JsonEntity, from_schema: JsonEntity, resolver: Optional[Any] = None) -> JsonEntityCastResult: + def cast(self, to_schema: GtsEntity, from_schema: GtsEntity, resolver: Optional[Any] = None) -> GtsEntityCastResult: if self.is_schema: # When casting a schema, from_schema might be a standard JSON Schema (no gts_id) # In that case, skip the sanity check @@ -182,7 +182,7 @@ def cast(self, to_schema: JsonEntity, from_schema: JsonEntity, resolver: Optiona raise SchemaCastError("Target must be a schema") if not from_schema.is_schema: raise SchemaCastError("Source schema must be a schema") - return JsonEntityCastResult.cast( + return GtsEntityCastResult.cast( self.gts_id.id, to_schema.gts_id.id, self.content, diff --git a/gts/src/gts/files_reader.py b/gts/src/gts/files_reader.py index cd852fc..1e60e58 100644 --- a/gts/src/gts/files_reader.py +++ b/gts/src/gts/files_reader.py @@ -1,12 +1,13 @@ from __future__ import annotations import json +import yaml from pathlib import Path import os from typing import Iterator, List, Optional, Any from .store import GtsReader -from .entities import JsonEntity, JsonFile, DEFAULT_GTS_CONFIG, GtsConfig +from .entities import GtsEntity, GtsFile, DEFAULT_GTS_CONFIG, GtsConfig import logging @@ -15,7 +16,7 @@ class GtsFileReader(GtsReader): - """Reads JSON entities from files and directories specified by path.""" + """Reads GTS entities from JSON and YAML files in directories specified by path.""" def __init__(self, path: str | List[str], cfg: Optional[GtsConfig] = None) -> None: """ @@ -34,13 +35,13 @@ def __init__(self, path: str | List[str], cfg: Optional[GtsConfig] = None) -> No self.cfg = cfg or DEFAULT_GTS_CONFIG self._files: List[Path] = [] self._current_index = 0 - self._current_file_entities: List[JsonEntity] = [] + self._current_file_entities: List[GtsEntity] = [] self._current_entity_index = 0 self._initialized = False def _collect_files(self) -> None: - """Collect all JSON files from the specified paths, following symlinks.""" - valid_extensions = {'.json', '.jsonc', '.gts'} + """Collect all JSON and YAML files from the specified paths, following symlinks.""" + valid_extensions = {'.json', '.jsonc', '.gts', '.yaml', '.yml'} seen: set[str] = set() collected: List[Path] = [] @@ -73,18 +74,21 @@ def _collect_files(self) -> None: self._files = collected - def _load_json_file(self, file_path: Path) -> Any: - """Load JSON content from a file.""" + def _load_file(self, file_path: Path) -> Any: + """Load content from JSON or YAML file.""" with file_path.open("r", encoding="utf-8") as f: - return json.load(f) + if file_path.suffix.lower() in {'.yaml', '.yml'}: + return yaml.safe_load(f) + else: + return json.load(f) - def _process_file(self, file_path: Path) -> List[JsonEntity]: - """Process a single JSON file and return list of JsonEntity objects.""" - entities: List[JsonEntity] = [] + def _process_file(self, file_path: Path) -> List[GtsEntity]: + """Process a single JSON or YAML file and return list of GtsEntity objects.""" + entities: List[GtsEntity] = [] try: - content = self._load_json_file(file_path) - json_file = JsonFile( + content = self._load_file(file_path) + json_file = GtsFile( path=str(file_path), name=file_path.name, content=content @@ -93,7 +97,7 @@ def _process_file(self, file_path: Path) -> List[JsonEntity]: # Handle both single objects and arrays if isinstance(content, list): for idx, item in enumerate(content): - entity = JsonEntity( + entity = GtsEntity( file=json_file, list_sequence=idx, content=item, @@ -103,7 +107,7 @@ def _process_file(self, file_path: Path) -> List[JsonEntity]: logging.debug(f"- discovered entity: {entity.gts_id.id}") entities.append(entity) else: - entity = JsonEntity( + entity = GtsEntity( file=json_file, list_sequence=None, content=content, @@ -118,8 +122,8 @@ def _process_file(self, file_path: Path) -> List[JsonEntity]: return entities - def __iter__(self) -> Iterator[JsonEntity]: - """Iterate over all JsonEntity objects from all files.""" + def __iter__(self) -> Iterator[GtsEntity]: + """Iterate over all GtsEntity objects from all files.""" if not self._initialized: self._collect_files() self._initialized = True @@ -130,9 +134,9 @@ def __iter__(self) -> Iterator[JsonEntity]: for entity in entities: yield entity - def read_by_id(self, entity_id: str) -> Optional[JsonEntity]: + def read_by_id(self, entity_id: str) -> Optional[GtsEntity]: """ - Read a JsonEntity by its ID. + Read a GtsEntity by its ID. For FileReader, this returns None as we don't support random access by ID. """ return None diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index 3dc29d5..d6db717 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -7,11 +7,11 @@ from pathlib import Path as SysPath from .gts import GtsID, GtsWildcard -from .entities import DEFAULT_GTS_CONFIG, GtsConfig, JsonEntity +from .entities import DEFAULT_GTS_CONFIG, GtsConfig, GtsEntity from .files_reader import GtsFileReader -from .path_resolver import JsonPathResolver +from .path_resolver import GtsPathResolver from .store import GtsStore, GtsStoreQueryResult -from .schema_cast import JsonEntityCastResult +from .schema_cast import GtsEntityCastResult # Interface helpers diff --git a/gts/src/gts/path_resolver.py b/gts/src/gts/path_resolver.py index aec5d94..9017025 100644 --- a/gts/src/gts/path_resolver.py +++ b/gts/src/gts/path_resolver.py @@ -5,7 +5,7 @@ @dataclass -class JsonPathResolver: +class GtsPathResolver: gts_id: str content: Any path: str = "" @@ -70,7 +70,7 @@ def _collect_from(self, node: Any) -> List[str]: self._list_available(node, '', acc) return acc - def resolve(self, path: str) -> JsonPathResolver: + def resolve(self, path: str) -> GtsPathResolver: self.path = path self.value = None self.resolved = False @@ -119,7 +119,7 @@ def resolve(self, path: str) -> JsonPathResolver: self.resolved = True return self - def failure(self, path: str, error: str) -> JsonPathResolver: + def failure(self, path: str, error: str) -> GtsPathResolver: self.path = path self.value = None self.resolved = False diff --git a/gts/src/gts/schema_cast.py b/gts/src/gts/schema_cast.py index 839e57d..228639d 100644 --- a/gts/src/gts/schema_cast.py +++ b/gts/src/gts/schema_cast.py @@ -15,7 +15,7 @@ class SchemaCastError(Exception): @dataclass -class JsonEntityCastResult: +class GtsEntityCastResult: from_id: str = "" to_id: str = "" direction: str = "unknown" @@ -77,7 +77,7 @@ def cast( from_schema_content: dict, to_schema_content: dict, resolver: Optional[Any] = None, - ) -> JsonEntityCastResult: + ) -> GtsEntityCastResult: # Flatten target schema to merge allOf and get all properties including const values target_schema = cls._flatten_schema(to_schema_content) @@ -272,8 +272,8 @@ def _cast_instance_to_schema( continue p_type = p_schema.get("type") if p_type == "object" and isinstance(val, dict): - nested_schema = JsonEntityCastResult._effective_object_schema(p_schema) - new_obj, add_sub, rem_sub, new_incompatibility_reasons = JsonEntityCastResult._cast_instance_to_schema( + nested_schema = GtsEntityCastResult._effective_object_schema(p_schema) + new_obj, add_sub, rem_sub, new_incompatibility_reasons = GtsEntityCastResult._cast_instance_to_schema( val, nested_schema, base_path=(f"{base_path}.{prop}" if base_path else prop), incompatibility_reasons=incompatibility_reasons ) result[prop] = new_obj @@ -283,11 +283,11 @@ def _cast_instance_to_schema( elif p_type == "array" and isinstance(val, list): items_schema = p_schema.get("items") if isinstance(items_schema, dict) and items_schema.get("type") == "object": - nested_schema = JsonEntityCastResult._effective_object_schema(items_schema) + nested_schema = GtsEntityCastResult._effective_object_schema(items_schema) new_list: List[Any] = [] for idx, item in enumerate(val): if isinstance(item, dict): - new_item, add_sub, rem_sub, new_incompatibility_reasons = JsonEntityCastResult._cast_instance_to_schema( + new_item, add_sub, rem_sub, new_incompatibility_reasons = GtsEntityCastResult._cast_instance_to_schema( item, nested_schema, base_path=(f"{base_path}.{prop}[{idx}]" if base_path else f"{prop}[{idx}]"), @@ -311,7 +311,7 @@ def _validate_with_gts_id_tolerance( ) -> None: """Validate instance against schema, but allow const values to differ if both are GTS IDs.""" # Create a modified schema that removes const constraints for GTS IDs - modified_schema = JsonEntityCastResult._remove_gts_const_constraints(schema) + modified_schema = GtsEntityCastResult._remove_gts_const_constraints(schema) if resolver is not None: js_validate(instance=instance, schema=modified_schema, resolver=resolver) @@ -333,10 +333,10 @@ def _remove_gts_const_constraints(schema: Any) -> Any: result["type"] = "string" continue elif isinstance(value, dict): - result[key] = JsonEntityCastResult._remove_gts_const_constraints(value) + result[key] = GtsEntityCastResult._remove_gts_const_constraints(value) elif isinstance(value, list): result[key] = [ - JsonEntityCastResult._remove_gts_const_constraints(item) + GtsEntityCastResult._remove_gts_const_constraints(item) if isinstance(item, dict) else item for item in value @@ -354,7 +354,7 @@ def _flatten_schema(schema: Dict[str, Any]) -> Dict[str, Any]: # Merge allOf schemas if "allOf" in schema: for sub_schema in schema["allOf"]: - flattened = JsonEntityCastResult._flatten_schema(sub_schema) + flattened = GtsEntityCastResult._flatten_schema(sub_schema) result["properties"].update(flattened.get("properties", {})) result["required"].extend(flattened.get("required", [])) # Preserve additionalProperties from sub-schemas (last one wins) @@ -450,7 +450,7 @@ def _check_constraint_compatibility( # Numeric constraints (for number/integer types) if prop_type in ("number", "integer"): errors.extend( - JsonEntityCastResult._check_min_max_constraint( + GtsEntityCastResult._check_min_max_constraint( prop, old_prop_schema, new_prop_schema, "minimum", "maximum", check_tightening ) ) @@ -458,7 +458,7 @@ def _check_constraint_compatibility( # String constraints if prop_type == "string": errors.extend( - JsonEntityCastResult._check_min_max_constraint( + GtsEntityCastResult._check_min_max_constraint( prop, old_prop_schema, new_prop_schema, "minLength", "maxLength", check_tightening ) ) @@ -466,7 +466,7 @@ def _check_constraint_compatibility( # Array constraints if prop_type == "array": errors.extend( - JsonEntityCastResult._check_min_max_constraint( + GtsEntityCastResult._check_min_max_constraint( prop, old_prop_schema, new_prop_schema, "minItems", "maxItems", check_tightening ) ) @@ -493,8 +493,8 @@ def _check_schema_compatibility( errors: List[str] = [] # Flatten schemas to handle allOf - old_flat = JsonEntityCastResult._flatten_schema(old_schema) - new_flat = JsonEntityCastResult._flatten_schema(new_schema) + old_flat = GtsEntityCastResult._flatten_schema(old_schema) + new_flat = GtsEntityCastResult._flatten_schema(new_schema) old_props = old_flat.get("properties", {}) new_props = new_flat.get("properties", {}) @@ -543,14 +543,14 @@ def _check_schema_compatibility( errors.append(f"Property '{prop}' removed enum values: {removed_enum_values}") # Check constraint compatibility - constraint_errors = JsonEntityCastResult._check_constraint_compatibility( + constraint_errors = GtsEntityCastResult._check_constraint_compatibility( prop, old_prop_schema, new_prop_schema, check_tightening=check_backward ) errors.extend(constraint_errors) # Recursively check nested object properties if old_type == "object" and new_type == "object": - nested_compat, nested_errors = JsonEntityCastResult._check_schema_compatibility( + nested_compat, nested_errors = GtsEntityCastResult._check_schema_compatibility( old_prop_schema, new_prop_schema, check_backward ) if not nested_compat: @@ -577,7 +577,7 @@ def _check_backward_compatibility( - Cannot add enum values - Cannot tighten constraints (decrease max, increase min, etc.) """ - return JsonEntityCastResult._check_schema_compatibility(old_schema, new_schema, check_backward=True) + return GtsEntityCastResult._check_schema_compatibility(old_schema, new_schema, check_backward=True) @staticmethod def _check_forward_compatibility( @@ -596,7 +596,7 @@ def _check_forward_compatibility( - Cannot remove enum values - Cannot relax constraints (increase max, decrease min, etc.) """ - return JsonEntityCastResult._check_schema_compatibility(old_schema, new_schema, check_backward=False) + return GtsEntityCastResult._check_schema_compatibility(old_schema, new_schema, check_backward=False) @staticmethod def _diff_objects( @@ -630,7 +630,7 @@ def _diff_objects( bf = bv.get("format") if af != bf: changed.append({"path": p, "change": f"format: {af} -> {bf}"}) - JsonEntityCastResult._diff_objects(av, bv, p, added, removed, changed) + GtsEntityCastResult._diff_objects(av, bv, p, added, removed, changed) a_req = set(obj_a.get("required", [])) if isinstance(obj_a, dict) else set() b_req = set(obj_b.get("required", [])) if isinstance(obj_b, dict) else set() @@ -659,12 +659,12 @@ def _only_optional_add_remove( ) -> bool: if not isinstance(a, dict) or not isinstance(b, dict): if a != b: - reasons.append(f"{JsonEntityCastResult._path_label(path)}: value changed") + reasons.append(f"{GtsEntityCastResult._path_label(path)}: value changed") return False return True - fa = JsonEntityCastResult._filtered(a) - fb = JsonEntityCastResult._filtered(b) + fa = GtsEntityCastResult._filtered(a) + fb = GtsEntityCastResult._filtered(b) if fa != fb: keys = set(fa.keys()) | set(fb.keys()) for k in sorted(keys): @@ -672,7 +672,7 @@ def _only_optional_add_remove( vb = fb.get(k, "") if va != vb: reasons.append( - f"{JsonEntityCastResult._path_label(path)}: keyword '{k}' changed" + f"{GtsEntityCastResult._path_label(path)}: keyword '{k}' changed" ) return False @@ -683,12 +683,12 @@ def _only_optional_add_remove( removed_req = sorted(list(a_req - b_req)) if added_req: reasons.append( - f"{JsonEntityCastResult._path_label(path)}: required added -> " + f"{GtsEntityCastResult._path_label(path)}: required added -> " f"{', '.join(added_req)}" ) if removed_req: reasons.append( - f"{JsonEntityCastResult._path_label(path)}: required removed -> " + f"{GtsEntityCastResult._path_label(path)}: required removed -> " f"{', '.join(removed_req)}" ) return False @@ -698,7 +698,7 @@ def _only_optional_add_remove( common = set(a_props.keys()) & set(b_props.keys()) for k in common: next_path = f"{path}.properties.{k}" if path else f"properties.{k}" - if not JsonEntityCastResult._only_optional_add_remove( + if not GtsEntityCastResult._only_optional_add_remove( a_props[k], b_props[k], next_path, reasons ): return False diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index d44d8b8..425c57a 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -7,8 +7,8 @@ from jsonschema import RefResolver from .gts import GtsID, GtsWildcard -from .entities import JsonEntity -from .schema_cast import JsonEntityCastResult +from .entities import GtsEntity +from .schema_cast import GtsEntityCastResult from .x_gts_ref import XGtsRefValidator import logging @@ -56,12 +56,12 @@ class GtsReader(ABC): """Abstract base class for reading JSON entities from various sources.""" @abstractmethod - def __iter__(self) -> Iterator[JsonEntity]: + def __iter__(self) -> Iterator[GtsEntity]: """Return an iterator that yields JsonEntity objects.""" pass @abstractmethod - def read_by_id(self, entity_id: str) -> Optional[JsonEntity]: + def read_by_id(self, entity_id: str) -> Optional[GtsEntity]: """ Read a JsonEntity by its ID. Returns None if the entity is not found. @@ -121,7 +121,7 @@ def _populate_from_reader(self) -> None: if entity.gts_id and entity.gts_id.id: self._by_id[entity.gts_id.id] = entity - def register(self, entity: JsonEntity) -> None: + def register(self, entity: GtsEntity) -> None: """Register a JsonEntity in the store.""" if not entity.gts_id or not entity.gts_id.id: raise ValueError("Entity must have a valid gts_id") @@ -143,7 +143,7 @@ def register_schema(self, type_id: str, schema: Dict[str, Any]) -> None: ) self._by_id[type_id] = entity - def get(self, entity_id: str) -> Optional[JsonEntity]: + def get(self, entity_id: str) -> Optional[GtsEntity]: """ Get a JsonEntity by its ID. If not found in cache, try to fetch from reader. @@ -331,7 +331,7 @@ def cast( self, from_id: str, target_schema_id: str, - ) -> JsonEntityCastResult: + ) -> GtsEntityCastResult: from_entity = self.get(from_id) if not from_entity: raise StoreGtsEntityNotFound(from_id) @@ -364,7 +364,7 @@ def is_minor_compatible( self, old_schema_id: str, new_schema_id: str, - ) -> JsonEntityCastResult: + ) -> GtsEntityCastResult: """ Check compatibility between two schemas.