From 50db755cc5b521ac560f4677167754b1a9f79fbb Mon Sep 17 00:00:00 2001 From: Laren-AWS Date: Mon, 11 Nov 2024 16:36:12 -0800 Subject: [PATCH 1/9] First stab at data class. --- aws_doc_sdk_examples_tools/categories.py | 60 +++++++++++++++++++ .../config/categories.yaml | 9 +++ 2 files changed, 69 insertions(+) create mode 100644 aws_doc_sdk_examples_tools/categories.py create mode 100644 aws_doc_sdk_examples_tools/config/categories.yaml diff --git a/aws_doc_sdk_examples_tools/categories.py b/aws_doc_sdk_examples_tools/categories.py new file mode 100644 index 0000000..fd5b93c --- /dev/null +++ b/aws_doc_sdk_examples_tools/categories.py @@ -0,0 +1,60 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, List, Optional +from dataclasses import dataclass, field + +from aws_doc_sdk_examples_tools import metadata_errors +from .metadata_errors import ( + MetadataErrors, + MetadataParseError, + check_mapping, +) + + +@dataclass +class Category: + key: str + title: str + title_abbrev: str + display: str + description: str + + @classmethod + def from_yaml(cls, key: str, yaml: Dict[str, Any]) -> tuple[Category, MetadataErrors]: + errors = MetadataErrors() + display = yaml.get("display", "") + description = yaml.get("description", "") + title = yaml.get("title", "") + title_abbrev = yaml.get("title_abbrev", "") + + return cls(key=key, display=display, description=description, title=title, title_abbrev=title_abbrev), errors + + +def parse(file: Path, yaml: Dict[str, Any]) -> tuple[Dict[str, Category], MetadataErrors]: + categories: Dict[str, Category] = {} + errors = MetadataErrors() + + for key in yaml: + category, errs = Category.from_yaml(key, yaml[key]) + categories[key] = category + for error in errs: + error.file = file + error.id = key + errors.extend(errs) + + return categories, errors + + +if __name__ == "__main__": + from pprint import pp + import yaml + + path = Path(__file__).parent / "config" / "categories.yaml" + with open(path) as file: + meta = yaml.safe_load(file) + categories, errors = parse(path, meta) + pp(categories) diff --git a/aws_doc_sdk_examples_tools/config/categories.yaml b/aws_doc_sdk_examples_tools/config/categories.yaml new file mode 100644 index 0000000..b8f98e0 --- /dev/null +++ b/aws_doc_sdk_examples_tools/config/categories.yaml @@ -0,0 +1,9 @@ +Basics: + title: "Learn the basics of {{.Services.Short}} with an &AWS; SDK" + title_abbrev: "Learn the basics" + display: "Basics" + description: "are code examples that show you how to perform the essential operations within a service." +TributaryLite: + display: "&AWS; community contributions" + description: "are examples that were created and are maintained by multiple teams across &AWS;. + To provide feedback, use the mechanism provided in the linked repositories." From fab90c21ef9e00f55005d3f867bc61bc2203fe7c Mon Sep 17 00:00:00 2001 From: Laren-AWS Date: Wed, 13 Nov 2024 14:24:27 -0800 Subject: [PATCH 2/9] Update basics template. --- aws_doc_sdk_examples_tools/config/categories.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_doc_sdk_examples_tools/config/categories.yaml b/aws_doc_sdk_examples_tools/config/categories.yaml index b8f98e0..9b74ca6 100644 --- a/aws_doc_sdk_examples_tools/config/categories.yaml +++ b/aws_doc_sdk_examples_tools/config/categories.yaml @@ -1,5 +1,5 @@ Basics: - title: "Learn the basics of {{.Services.Short}} with an &AWS; SDK" + title: "Learn the basics of {{.Short}} with an &AWS; SDK" title_abbrev: "Learn the basics" display: "Basics" description: "are code examples that show you how to perform the essential operations within a service." From 5415c30956d8a38c07a3f66cda87f186fc2bdf83 Mon Sep 17 00:00:00 2001 From: Laren-AWS Date: Fri, 15 Nov 2024 16:12:22 -0800 Subject: [PATCH 3/9] Add good draft of complex categories.yaml. --- .../config/categories.yaml | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/aws_doc_sdk_examples_tools/config/categories.yaml b/aws_doc_sdk_examples_tools/config/categories.yaml index 9b74ca6..355bd9b 100644 --- a/aws_doc_sdk_examples_tools/config/categories.yaml +++ b/aws_doc_sdk_examples_tools/config/categories.yaml @@ -1,9 +1,34 @@ -Basics: - title: "Learn the basics of {{.Short}} with an &AWS; SDK" - title_abbrev: "Learn the basics" - display: "Basics" - description: "are code examples that show you how to perform the essential operations within a service." -TributaryLite: - display: "&AWS; community contributions" - description: "are examples that were created and are maintained by multiple teams across &AWS;. - To provide feedback, use the mechanism provided in the linked repositories." +standard_categories: {"Hello", "Actions", "Basics", "Scenarios"} +categories: + Hello: + display: "Hello" + overrides: + title: "Hello {{.ServiceEntity.Short}};" + title_abbrev: "Hello {{.ServiceEntity.Short}};" + synopsis: "get started using {{.ServiceEntity.Short}}." + Actions: + display: "Actions" + overrides: + title: "Use {{.Action}}" + title_suffixes: + cli: " with a CLI" + sdk: " with an &AWS; SDK" + sdk_cli: " with an &AWS; SDK or CLI" + title_abbrev: "{{.Action}}" + synopsis: "use {{.Action}}." + description: "are code excerpts from larger programs and must be run in context. While actions + show you how to call individual service functions, you can see actions in context in their related scenarios." + Basics: + display: "Basics" + defaults: + title: "Learn the basics of {{.ServiceEntity.Short}} with an &AWS; SDK" + title_abbrev: "Learn the basics" + description: "are code examples that show you how to perform the essential operations within a service." + Scenarios: + display: "Scenarios" + description: "are code examples that show you how to accomplish specific tasks by + calling multiple functions within a service or combined with other &AWS-services;." + TributaryLite: + display: "&AWS; community contributions" + description: "are examples that were created and are maintained by multiple teams across &AWS;. + To provide feedback, use the mechanism provided in the linked repositories." From 33eef83e7b1984b256153913e83de2c00e0898e6 Mon Sep 17 00:00:00 2001 From: Laren-AWS Date: Fri, 6 Dec 2024 14:16:19 -0800 Subject: [PATCH 4/9] Update categories.yaml. --- aws_doc_sdk_examples_tools/config/categories.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_doc_sdk_examples_tools/config/categories.yaml b/aws_doc_sdk_examples_tools/config/categories.yaml index 355bd9b..8ee710b 100644 --- a/aws_doc_sdk_examples_tools/config/categories.yaml +++ b/aws_doc_sdk_examples_tools/config/categories.yaml @@ -1,4 +1,4 @@ -standard_categories: {"Hello", "Actions", "Basics", "Scenarios"} +standard_categories: ["Hello", "Basics", "Actions", "Scenarios"] categories: Hello: display: "Hello" From 832d40ab2c2de54277d12410cd3afe19b67a84d4 Mon Sep 17 00:00:00 2001 From: Laren-AWS Date: Wed, 11 Dec 2024 14:27:08 -0800 Subject: [PATCH 5/9] Load and process complex categories. --- aws_doc_sdk_examples_tools/categories.py | 75 +++++++++++++------ aws_doc_sdk_examples_tools/categories_test.py | 72 ++++++++++++++++++ .../config/categories.yaml | 4 +- aws_doc_sdk_examples_tools/doc_gen.py | 31 ++++++-- aws_doc_sdk_examples_tools/metadata_errors.py | 5 ++ aws_doc_sdk_examples_tools/metadata_test.py | 2 +- .../test_resources/categories.yaml | 21 ++++++ .../test_resources/empty_categories.yaml | 2 + aws_doc_sdk_examples_tools/yaml_mapper.py | 2 +- 9 files changed, 183 insertions(+), 31 deletions(-) create mode 100644 aws_doc_sdk_examples_tools/categories_test.py create mode 100644 aws_doc_sdk_examples_tools/test_resources/categories.yaml create mode 100644 aws_doc_sdk_examples_tools/test_resources/empty_categories.yaml diff --git a/aws_doc_sdk_examples_tools/categories.py b/aws_doc_sdk_examples_tools/categories.py index fd5b93c..8b894b4 100644 --- a/aws_doc_sdk_examples_tools/categories.py +++ b/aws_doc_sdk_examples_tools/categories.py @@ -10,43 +10,75 @@ from aws_doc_sdk_examples_tools import metadata_errors from .metadata_errors import ( MetadataErrors, - MetadataParseError, - check_mapping, ) @dataclass -class Category: - key: str +class TitleInfo: title: str title_abbrev: str + synopsis: Optional[str] = field(default=None) + title_suffixes: Dict[str, str] = field(default_factory=dict) + + @classmethod + def from_yaml(cls, yaml: Dict[str, str]) -> [TitleInfo, None]: + if yaml is None: + return None + + title = yaml.get("title") + title_suffixes = yaml.get("title_suffixes", {}) + title_abbrev = yaml.get("title_abbrev") + synopsis = yaml.get("synopsis") + + return cls(title=title, title_suffixes=title_suffixes, title_abbrev=title_abbrev, synopsis=synopsis) + + +@dataclass +class CategoryWithNoDisplayError(metadata_errors.MetadataError): + def message(self): + return "Category has no display value" + + +@dataclass +class Category: + key: str display: str - description: str + defaults: Optional[TitleInfo] = field(default=None) + overrides: Optional[TitleInfo] = field(default=None) + description: Optional[str] = field(default=None) + + def validate(self, errors: MetadataErrors): + if not self.display: + errors.append(CategoryWithNoDisplayError(id=self.key)) @classmethod def from_yaml(cls, key: str, yaml: Dict[str, Any]) -> tuple[Category, MetadataErrors]: errors = MetadataErrors() - display = yaml.get("display", "") - description = yaml.get("description", "") - title = yaml.get("title", "") - title_abbrev = yaml.get("title_abbrev", "") + display = yaml.get("display") + defaults = TitleInfo.from_yaml(yaml.get("defaults")) + overrides = TitleInfo.from_yaml(yaml.get("overrides")) + description = yaml.get("description") - return cls(key=key, display=display, description=description, title=title, title_abbrev=title_abbrev), errors + return cls(key=key, display=display, defaults=defaults, overrides=overrides, description=description), errors -def parse(file: Path, yaml: Dict[str, Any]) -> tuple[Dict[str, Category], MetadataErrors]: +def parse(file: Path, yaml: Dict[str, Any]) -> tuple[List[str], Dict[str, Category], MetadataErrors]: categories: Dict[str, Category] = {} errors = MetadataErrors() - for key in yaml: - category, errs = Category.from_yaml(key, yaml[key]) - categories[key] = category - for error in errs: - error.file = file - error.id = key - errors.extend(errs) + standard_cats = yaml.get("standard_categories", []) + for key, yaml_cat in yaml.get("categories", {}).items(): + if yaml_cat is None: + errors.append(metadata_errors.MissingCategoryBody(id=key, file=file)) + else: + category, cat_errs = Category.from_yaml(key, yaml_cat) + categories[key] = category + for error in cat_errs: + error.file = file + error.id = key + errors.extend(cat_errs) - return categories, errors + return standard_cats, categories, errors if __name__ == "__main__": @@ -56,5 +88,6 @@ def parse(file: Path, yaml: Dict[str, Any]) -> tuple[Dict[str, Category], Metada path = Path(__file__).parent / "config" / "categories.yaml" with open(path) as file: meta = yaml.safe_load(file) - categories, errors = parse(path, meta) - pp(categories) + standard_cats, cats, errs = parse(path, meta) + pp(standard_cats) + pp(cats) diff --git a/aws_doc_sdk_examples_tools/categories_test.py b/aws_doc_sdk_examples_tools/categories_test.py new file mode 100644 index 0000000..46ad227 --- /dev/null +++ b/aws_doc_sdk_examples_tools/categories_test.py @@ -0,0 +1,72 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path +from typing import Dict, Tuple +import pytest +import yaml + +from aws_doc_sdk_examples_tools import metadata_errors +from .categories import ( + parse, + Category, + CategoryWithNoDisplayError, + TitleInfo, +) + + +def load(path: str) -> Tuple[Dict[str, Category], metadata_errors.MetadataErrors]: + root = Path(__file__).parent + filename = root / "test_resources" / path + with open(filename) as file: + meta = yaml.safe_load(file) + return parse(filename, meta) + + +def test_empty_categories(): + _, errs = load("empty_categories.yaml") + assert [*errs] == [ + metadata_errors.MissingCategoryBody( + file=Path(__file__).parent / "test_resources/empty_categories.yaml", + id="EmptyCat", + ) + ] + + +def test_categories(): + categories, _ = load("categories.yaml") + assert categories == { + "Actions": Category( + key="Actions", + display="Actions test", + overrides=TitleInfo( + title="Title override", + title_suffixes={ + "cli": " with a CLI", + "sdk": " with an &AWS; SDK", + "sdk_cli": " with an &AWS; SDK or CLI", + }, + title_abbrev="Title abbrev override", + synopsis="synopsis test.", + ), + description="test description.", + ), + "Basics": Category( + key="Basics", + display="Basics", + defaults=TitleInfo( + title="Title default", + title_abbrev="Title abbrev default", + ), + description="default description.", + ), + "TributaryLite": Category( + key="TributaryLite", + display="Tea light", + description="light your way.", + ) + } + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/aws_doc_sdk_examples_tools/config/categories.yaml b/aws_doc_sdk_examples_tools/config/categories.yaml index 8ee710b..8a9fd4d 100644 --- a/aws_doc_sdk_examples_tools/config/categories.yaml +++ b/aws_doc_sdk_examples_tools/config/categories.yaml @@ -3,8 +3,8 @@ categories: Hello: display: "Hello" overrides: - title: "Hello {{.ServiceEntity.Short}};" - title_abbrev: "Hello {{.ServiceEntity.Short}};" + title: "Hello {{.ServiceEntity.Short}}" + title_abbrev: "Hello {{.ServiceEntity.Short}}" synopsis: "get started using {{.ServiceEntity.Short}}." Actions: display: "Actions" diff --git a/aws_doc_sdk_examples_tools/doc_gen.py b/aws_doc_sdk_examples_tools/doc_gen.py index 0ce6a41..567526c 100644 --- a/aws_doc_sdk_examples_tools/doc_gen.py +++ b/aws_doc_sdk_examples_tools/doc_gen.py @@ -12,6 +12,7 @@ # from os import glob +from .categories import Category, parse as parse_categories from .metadata import ( Example, DocFilenames, @@ -55,6 +56,8 @@ class DocGen: validation: ValidationConfig = field(default_factory=ValidationConfig) sdks: Dict[str, Sdk] = field(default_factory=dict) services: Dict[str, Service] = field(default_factory=dict) + standard_categories: List[str] = field(default_factory=list) + categories: Dict[str, Category] = field(default_factory=dict) snippets: Dict[str, Snippet] = field(default_factory=dict) snippet_files: Set[str] = field(default_factory=set) examples: Dict[str, Example] = field(default_factory=dict) @@ -201,6 +204,17 @@ def for_root( except Exception: pass + try: + categories_path = config / "categories.yaml" + with categories_path.open(encoding="utf-8") as file: + meta = yaml.safe_load(file) + standard_categories, categories, errs = parse_categories(categories_path, meta) + self.standard_categories = standard_categories + self.categories = categories + self.errors.extend(errs) + except Exception: + pass + try: entities_config_path = config / "entities.yaml" with entities_config_path.open(encoding="utf-8") as file: @@ -236,6 +250,7 @@ def process_metadata(self, path: Path) -> "DocGen": yaml.safe_load(file), self.sdks, self.services, + self.standard_categories, self.cross_blocks, self.validation, self.root, @@ -268,6 +283,8 @@ def validate(self): sdk.validate(self.errors) for service in self.services.values(): service.validate(self.errors) + for category in self.categories.values(): + category.validate(self.errors) for example in self.examples.values(): example.validate(self.errors, self.root) validate_metadata(self.root, self.validation.strict_titles, self.errors) @@ -339,6 +356,7 @@ def parse_examples( yaml: Dict[str, Any], sdks: Dict[str, Sdk], services: Dict[str, Service], + standard_categories: List[str], blocks: Set[str], validation: Optional[ValidationConfig], root: Optional[Path] = None, @@ -350,12 +368,13 @@ def parse_examples( example, example_errors = example_from_yaml( yaml[id], sdks, services, blocks, validation, root or file.parent ) - check_id_format( - id, - example.services, - validation.strict_titles and example.category == "Api", - example_errors, - ) + if example.category in standard_categories: + check_id_format( + id, + example.services, + validation.strict_titles and example.category == "Api", + example_errors, + ) for error in example_errors: error.file = file error.id = id diff --git a/aws_doc_sdk_examples_tools/metadata_errors.py b/aws_doc_sdk_examples_tools/metadata_errors.py index 0859a2f..fa1a2c2 100644 --- a/aws_doc_sdk_examples_tools/metadata_errors.py +++ b/aws_doc_sdk_examples_tools/metadata_errors.py @@ -369,6 +369,11 @@ class URLMissingTitle(SdkVersionError): def message(self): return f"URL {self.url} is missing a title" +@dataclass +class MissingCategoryBody(MetadataParseError): + def message(self): + return "category definition missing body" + @dataclass class ExampleMergeMismatchedId(MetadataError): diff --git a/aws_doc_sdk_examples_tools/metadata_test.py b/aws_doc_sdk_examples_tools/metadata_test.py index 6d7ceb4..8d0c7c9 100644 --- a/aws_doc_sdk_examples_tools/metadata_test.py +++ b/aws_doc_sdk_examples_tools/metadata_test.py @@ -140,7 +140,7 @@ def test_parse(): example = Example( file=Path("test_cpp.yaml"), id="medical-imaging_CreateDatastore", - category="Cross", + category="Scenarios", services={ "medical-imaging": set(["Operation1", "Operation2"]), "api-gateway": set(["Operation1", "Operation2"]), diff --git a/aws_doc_sdk_examples_tools/test_resources/categories.yaml b/aws_doc_sdk_examples_tools/test_resources/categories.yaml new file mode 100644 index 0000000..c71fbe0 --- /dev/null +++ b/aws_doc_sdk_examples_tools/test_resources/categories.yaml @@ -0,0 +1,21 @@ +categories: + Actions: + display: "Actions test" + overrides: + title: "Title override" + title_suffixes: + cli: " with a CLI" + sdk: " with an &AWS; SDK" + sdk_cli: " with an &AWS; SDK or CLI" + title_abbrev: "Title abbrev override" + synopsis: "synopsis test." + description: "test description." + Basics: + display: "Basics" + defaults: + title: "Title default" + title_abbrev: "Title abbrev default" + description: "default description." + TributaryLite: + display: "Tea light" + description: "light your way." diff --git a/aws_doc_sdk_examples_tools/test_resources/empty_categories.yaml b/aws_doc_sdk_examples_tools/test_resources/empty_categories.yaml new file mode 100644 index 0000000..d171f0d --- /dev/null +++ b/aws_doc_sdk_examples_tools/test_resources/empty_categories.yaml @@ -0,0 +1,2 @@ +categories: + EmptyCat: diff --git a/aws_doc_sdk_examples_tools/yaml_mapper.py b/aws_doc_sdk_examples_tools/yaml_mapper.py index 5e205dc..5c08351 100644 --- a/aws_doc_sdk_examples_tools/yaml_mapper.py +++ b/aws_doc_sdk_examples_tools/yaml_mapper.py @@ -42,7 +42,7 @@ def example_from_yaml( parsed_services = parse_services(yaml.get("services", {}), errors, services) category = yaml.get("category", "") if category == "": - category = "Api" if len(parsed_services) == 1 else "Cross" + category = "Api" if len(parsed_services) == 1 else "Scenarios" is_action = category == "Api" is_basics = category == "Basics" From ecbed04e4926e680332362ee3463cad0e247da11 Mon Sep 17 00:00:00 2001 From: Laren-AWS Date: Wed, 11 Dec 2024 16:55:43 -0800 Subject: [PATCH 6/9] Work around Actions/Api discrepancy for categories and fix unit tests. --- aws_doc_sdk_examples_tools/categories.py | 4 ++++ aws_doc_sdk_examples_tools/categories_test.py | 9 ++++----- aws_doc_sdk_examples_tools/doc_gen.py | 2 +- aws_doc_sdk_examples_tools/metadata_test.py | 17 ++++++++--------- .../test_resources/formaterror_metadata.yaml | 1 - 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/aws_doc_sdk_examples_tools/categories.py b/aws_doc_sdk_examples_tools/categories.py index 8b894b4..dd06c94 100644 --- a/aws_doc_sdk_examples_tools/categories.py +++ b/aws_doc_sdk_examples_tools/categories.py @@ -67,6 +67,10 @@ def parse(file: Path, yaml: Dict[str, Any]) -> tuple[List[str], Dict[str, Catego errors = MetadataErrors() standard_cats = yaml.get("standard_categories", []) + # Work around inconsistency where some tools use 'Actions' and DocGen uses 'Api' to refer to single-action examples. + for i in range(len(standard_cats)): + if standard_cats[i] == "Actions": + standard_cats[i] = "Api" for key, yaml_cat in yaml.get("categories", {}).items(): if yaml_cat is None: errors.append(metadata_errors.MissingCategoryBody(id=key, file=file)) diff --git a/aws_doc_sdk_examples_tools/categories_test.py b/aws_doc_sdk_examples_tools/categories_test.py index 46ad227..25ee37a 100644 --- a/aws_doc_sdk_examples_tools/categories_test.py +++ b/aws_doc_sdk_examples_tools/categories_test.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from pathlib import Path -from typing import Dict, Tuple +from typing import Dict, List, Tuple import pytest import yaml @@ -10,12 +10,11 @@ from .categories import ( parse, Category, - CategoryWithNoDisplayError, TitleInfo, ) -def load(path: str) -> Tuple[Dict[str, Category], metadata_errors.MetadataErrors]: +def load(path: str) -> Tuple[List[str], Dict[str, Category], metadata_errors.MetadataErrors]: root = Path(__file__).parent filename = root / "test_resources" / path with open(filename) as file: @@ -24,7 +23,7 @@ def load(path: str) -> Tuple[Dict[str, Category], metadata_errors.MetadataErrors def test_empty_categories(): - _, errs = load("empty_categories.yaml") + _, _, errs = load("empty_categories.yaml") assert [*errs] == [ metadata_errors.MissingCategoryBody( file=Path(__file__).parent / "test_resources/empty_categories.yaml", @@ -34,7 +33,7 @@ def test_empty_categories(): def test_categories(): - categories, _ = load("categories.yaml") + _, categories, _ = load("categories.yaml") assert categories == { "Actions": Category( key="Actions", diff --git a/aws_doc_sdk_examples_tools/doc_gen.py b/aws_doc_sdk_examples_tools/doc_gen.py index 567526c..6936b30 100644 --- a/aws_doc_sdk_examples_tools/doc_gen.py +++ b/aws_doc_sdk_examples_tools/doc_gen.py @@ -430,7 +430,7 @@ def get_doc_filenames(example_id: str, example: Example) -> Optional[DocFilename ) ) else: - anchor = "actions" if example.category == "Actions" else "scenarios" + anchor = "actions" if example.category == "Api" else "scenarios" sdk_pages[language.property][version.sdk_version] = SDKPageVersion( actions_scenarios={ service_id: f"{base_url}/{language.property}_{version.sdk_version}_{service_id}_code_examples.html#{anchor}" diff --git a/aws_doc_sdk_examples_tools/metadata_test.py b/aws_doc_sdk_examples_tools/metadata_test.py index 8d0c7c9..8a05258 100644 --- a/aws_doc_sdk_examples_tools/metadata_test.py +++ b/aws_doc_sdk_examples_tools/metadata_test.py @@ -33,7 +33,7 @@ def load( with path.open() as file: meta = yaml.safe_load(file) return parse_examples( - path, meta, doc_gen.sdks, doc_gen.services, blocks, doc_gen.validation + path, meta, doc_gen.sdks, doc_gen.services, doc_gen.standard_categories, blocks, doc_gen.validation ) @@ -88,12 +88,14 @@ def load( "JavaScript": Sdk(name="JavaScript", versions=[], guide="", property="javascript"), "PHP": Sdk(name="PHP", versions=[], guide="", property="php"), } +STANDARD_CATS = ["Api"] DOC_GEN = DocGen( root=Path(), errors=metadata_errors.MetadataErrors(), validation=ValidationConfig(), services=SERVICES, sdks=SDKS, + standard_categories=STANDARD_CATS, ) GOOD_SINGLE_CPP = """ @@ -118,7 +120,7 @@ def load( def test_parse(): meta = yaml.safe_load(GOOD_SINGLE_CPP) parsed, errors = parse_examples( - Path("test_cpp.yaml"), meta, SDKS, SERVICES, set(), DOC_GEN.validation + Path("test_cpp.yaml"), meta, SDKS, SERVICES, STANDARD_CATS, set(), DOC_GEN.validation ) assert len(errors) == 0 assert len(parsed) == 1 @@ -220,6 +222,7 @@ def test_parse_strict_titles(): meta, SDKS, SERVICES, + STANDARD_CATS, set(), ValidationConfig(strict_titles=True), ) @@ -259,7 +262,7 @@ def test_parse_strict_titles(): actions_scenarios={ "medical-imaging": make_doc_link( stub="cpp_1_medical-imaging_code_examples", - anchor="scenarios", + anchor="actions", ), } ) @@ -354,6 +357,7 @@ def test_parse_strict_title_errors(): meta, SDKS, SERVICES, + STANDARD_CATS, set(), ValidationConfig(strict_titles=True), ) @@ -401,6 +405,7 @@ def test_parse_cross(): meta, SDKS, SERVICES, + STANDARD_CATS, set(["cross_DeleteTopic_block.xml"]), DOC_GEN.validation, ) @@ -600,12 +605,6 @@ def test_verify_load_successful(): file=EMPTY_METADATA_PATH, id="medical-imaging_EmptyExample", ), - metadata_errors.ServiceNameFormat( - file=EMPTY_METADATA_PATH, - id="medical-imaging_EmptyExample", - svc="medical-imaging", - svcs=[], - ), ], [], ), diff --git a/aws_doc_sdk_examples_tools/test_resources/formaterror_metadata.yaml b/aws_doc_sdk_examples_tools/test_resources/formaterror_metadata.yaml index 9f6d95c..569daa2 100644 --- a/aws_doc_sdk_examples_tools/test_resources/formaterror_metadata.yaml +++ b/aws_doc_sdk_examples_tools/test_resources/formaterror_metadata.yaml @@ -2,7 +2,6 @@ WrongNameFormat: title: Test title title_abbrev: Test title abbrev synopsis: Test synopsis - category: Test languages: Java: versions: From 52ea08dc859adcfd54311c97498c58103e202baf Mon Sep 17 00:00:00 2001 From: Laren-AWS Date: Wed, 11 Dec 2024 17:02:26 -0800 Subject: [PATCH 7/9] Placate typing. --- aws_doc_sdk_examples_tools/categories.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/aws_doc_sdk_examples_tools/categories.py b/aws_doc_sdk_examples_tools/categories.py index dd06c94..99591c4 100644 --- a/aws_doc_sdk_examples_tools/categories.py +++ b/aws_doc_sdk_examples_tools/categories.py @@ -15,18 +15,18 @@ @dataclass class TitleInfo: - title: str - title_abbrev: str + title: Optional[str] = field(default=None) + title_abbrev: Optional[str] = field(default=None) synopsis: Optional[str] = field(default=None) title_suffixes: Dict[str, str] = field(default_factory=dict) @classmethod - def from_yaml(cls, yaml: Dict[str, str]) -> [TitleInfo, None]: + def from_yaml(cls, yaml: Dict[str, str]) -> Optional[TitleInfo]: if yaml is None: return None title = yaml.get("title") - title_suffixes = yaml.get("title_suffixes", {}) + title_suffixes: Dict[str, str] = yaml.get("title_suffixes", {}) title_abbrev = yaml.get("title_abbrev") synopsis = yaml.get("synopsis") @@ -54,7 +54,7 @@ def validate(self, errors: MetadataErrors): @classmethod def from_yaml(cls, key: str, yaml: Dict[str, Any]) -> tuple[Category, MetadataErrors]: errors = MetadataErrors() - display = yaml.get("display") + display = str(yaml.get("display")) defaults = TitleInfo.from_yaml(yaml.get("defaults")) overrides = TitleInfo.from_yaml(yaml.get("overrides")) description = yaml.get("description") From 7a5f11996c90c24254b72b82c1e31a0e6a3c1069 Mon Sep 17 00:00:00 2001 From: Laren-AWS Date: Wed, 11 Dec 2024 17:21:17 -0800 Subject: [PATCH 8/9] More placating of the type checker. --- aws_doc_sdk_examples_tools/categories.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws_doc_sdk_examples_tools/categories.py b/aws_doc_sdk_examples_tools/categories.py index 99591c4..942d1cd 100644 --- a/aws_doc_sdk_examples_tools/categories.py +++ b/aws_doc_sdk_examples_tools/categories.py @@ -18,15 +18,15 @@ class TitleInfo: title: Optional[str] = field(default=None) title_abbrev: Optional[str] = field(default=None) synopsis: Optional[str] = field(default=None) - title_suffixes: Dict[str, str] = field(default_factory=dict) + title_suffixes: str | Dict[str, str] = field(default_factory=dict) @classmethod - def from_yaml(cls, yaml: Dict[str, str]) -> Optional[TitleInfo]: + def from_yaml(cls, yaml: Dict[str, str] | None) -> Optional[TitleInfo]: if yaml is None: return None title = yaml.get("title") - title_suffixes: Dict[str, str] = yaml.get("title_suffixes", {}) + title_suffixes: str | Dict[str, str] = yaml.get("title_suffixes", {}) title_abbrev = yaml.get("title_abbrev") synopsis = yaml.get("synopsis") From 3c621d8534e9b6e8e0408c8bcab592084200fade Mon Sep 17 00:00:00 2001 From: Laren-AWS Date: Wed, 11 Dec 2024 17:22:41 -0800 Subject: [PATCH 9/9] Run black. --- aws_doc_sdk_examples_tools/categories.py | 30 +++++++++++++++---- aws_doc_sdk_examples_tools/categories_test.py | 6 ++-- aws_doc_sdk_examples_tools/doc_gen.py | 4 ++- aws_doc_sdk_examples_tools/metadata_errors.py | 1 + aws_doc_sdk_examples_tools/metadata_test.py | 16 ++++++++-- 5 files changed, 46 insertions(+), 11 deletions(-) diff --git a/aws_doc_sdk_examples_tools/categories.py b/aws_doc_sdk_examples_tools/categories.py index 942d1cd..b292389 100644 --- a/aws_doc_sdk_examples_tools/categories.py +++ b/aws_doc_sdk_examples_tools/categories.py @@ -30,7 +30,12 @@ def from_yaml(cls, yaml: Dict[str, str] | None) -> Optional[TitleInfo]: title_abbrev = yaml.get("title_abbrev") synopsis = yaml.get("synopsis") - return cls(title=title, title_suffixes=title_suffixes, title_abbrev=title_abbrev, synopsis=synopsis) + return cls( + title=title, + title_suffixes=title_suffixes, + title_abbrev=title_abbrev, + synopsis=synopsis, + ) @dataclass @@ -52,17 +57,30 @@ def validate(self, errors: MetadataErrors): errors.append(CategoryWithNoDisplayError(id=self.key)) @classmethod - def from_yaml(cls, key: str, yaml: Dict[str, Any]) -> tuple[Category, MetadataErrors]: + def from_yaml( + cls, key: str, yaml: Dict[str, Any] + ) -> tuple[Category, MetadataErrors]: errors = MetadataErrors() display = str(yaml.get("display")) defaults = TitleInfo.from_yaml(yaml.get("defaults")) overrides = TitleInfo.from_yaml(yaml.get("overrides")) description = yaml.get("description") - return cls(key=key, display=display, defaults=defaults, overrides=overrides, description=description), errors - - -def parse(file: Path, yaml: Dict[str, Any]) -> tuple[List[str], Dict[str, Category], MetadataErrors]: + return ( + cls( + key=key, + display=display, + defaults=defaults, + overrides=overrides, + description=description, + ), + errors, + ) + + +def parse( + file: Path, yaml: Dict[str, Any] +) -> tuple[List[str], Dict[str, Category], MetadataErrors]: categories: Dict[str, Category] = {} errors = MetadataErrors() diff --git a/aws_doc_sdk_examples_tools/categories_test.py b/aws_doc_sdk_examples_tools/categories_test.py index 25ee37a..603b222 100644 --- a/aws_doc_sdk_examples_tools/categories_test.py +++ b/aws_doc_sdk_examples_tools/categories_test.py @@ -14,7 +14,9 @@ ) -def load(path: str) -> Tuple[List[str], Dict[str, Category], metadata_errors.MetadataErrors]: +def load( + path: str, +) -> Tuple[List[str], Dict[str, Category], metadata_errors.MetadataErrors]: root = Path(__file__).parent filename = root / "test_resources" / path with open(filename) as file: @@ -63,7 +65,7 @@ def test_categories(): key="TributaryLite", display="Tea light", description="light your way.", - ) + ), } diff --git a/aws_doc_sdk_examples_tools/doc_gen.py b/aws_doc_sdk_examples_tools/doc_gen.py index 6936b30..52ee8eb 100644 --- a/aws_doc_sdk_examples_tools/doc_gen.py +++ b/aws_doc_sdk_examples_tools/doc_gen.py @@ -208,7 +208,9 @@ def for_root( categories_path = config / "categories.yaml" with categories_path.open(encoding="utf-8") as file: meta = yaml.safe_load(file) - standard_categories, categories, errs = parse_categories(categories_path, meta) + standard_categories, categories, errs = parse_categories( + categories_path, meta + ) self.standard_categories = standard_categories self.categories = categories self.errors.extend(errs) diff --git a/aws_doc_sdk_examples_tools/metadata_errors.py b/aws_doc_sdk_examples_tools/metadata_errors.py index fa1a2c2..08dc4cb 100644 --- a/aws_doc_sdk_examples_tools/metadata_errors.py +++ b/aws_doc_sdk_examples_tools/metadata_errors.py @@ -369,6 +369,7 @@ class URLMissingTitle(SdkVersionError): def message(self): return f"URL {self.url} is missing a title" + @dataclass class MissingCategoryBody(MetadataParseError): def message(self): diff --git a/aws_doc_sdk_examples_tools/metadata_test.py b/aws_doc_sdk_examples_tools/metadata_test.py index 8a05258..c7f705b 100644 --- a/aws_doc_sdk_examples_tools/metadata_test.py +++ b/aws_doc_sdk_examples_tools/metadata_test.py @@ -33,7 +33,13 @@ def load( with path.open() as file: meta = yaml.safe_load(file) return parse_examples( - path, meta, doc_gen.sdks, doc_gen.services, doc_gen.standard_categories, blocks, doc_gen.validation + path, + meta, + doc_gen.sdks, + doc_gen.services, + doc_gen.standard_categories, + blocks, + doc_gen.validation, ) @@ -120,7 +126,13 @@ def load( def test_parse(): meta = yaml.safe_load(GOOD_SINGLE_CPP) parsed, errors = parse_examples( - Path("test_cpp.yaml"), meta, SDKS, SERVICES, STANDARD_CATS, set(), DOC_GEN.validation + Path("test_cpp.yaml"), + meta, + SDKS, + SERVICES, + STANDARD_CATS, + set(), + DOC_GEN.validation, ) assert len(errors) == 0 assert len(parsed) == 1