diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e15dbda --- /dev/null +++ b/.gitignore @@ -0,0 +1,32 @@ +# Claude Code configuration directory +.claude/ + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg-info/ +dist/ +build/ + +# Virtual environments +.venv/ +venv/ +ENV/ +env/ + +# IDEs +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Logs +*.log diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..f4d3036 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,163 @@ +# Markdownlint configuration for OpenShift Virtualization test documentation +# See https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md for rule details + +# Enable all rules by default, then override specific ones below +default: true + +# MD001 - Heading levels should only increment by one level at a time +MD001: true + +# MD003 - Heading style consistency (atx, atx_closed, setext) +MD003: true + +# MD004 - Unordered list style consistency +MD004: true + +# MD005 - Consistent indentation for list items at the same level +MD005: true + +# MD007 - Unordered list indentation (how many spaces for nested items) +MD007: true + +# MD009 - Trailing spaces (allows 2 spaces for hard line breaks in markdown) +MD009: true + +# MD010 - Hard tabs - DISABLED: Code examples may use hard tabs +MD010: false + +# MD012 - Multiple consecutive blank lines +MD012: true + +# MD013 - Line length - DISABLED: No line length restrictions for flexibility +MD013: false + +# MD014 - Dollar signs used before commands without showing output +MD014: true + +# MD018 - No space after hash on atx style heading +MD018: true + +# MD019 - Multiple spaces after hash on atx style heading +MD019: true + +# MD020 - No space inside hashes on closed atx style heading +MD020: true + +# MD021 - Multiple spaces inside hashes on closed atx style heading +MD021: true + +# MD022 - Headings should be surrounded by blank lines - DISABLED: Flexible spacing +MD022: false + +# MD023 - Headings must start at the beginning of the line +MD023: true + +# MD024 - Multiple headings with the same content +MD024: false + +# MD025 - Multiple top-level headings - DISABLED: Allows multiple H1 headings in the same document +MD025: false + +# MD026 - Trailing punctuation in heading +MD026: true + +# MD027 - Multiple spaces after blockquote symbol +MD027: true + +# MD028 - Blank line inside blockquote +MD028: true + +# MD029 - Ordered list item prefix (ordered, one, zero) +MD029: true + +# MD030 - Spaces after list markers +MD030: true + +# MD031 - Fenced code blocks surrounded by blank lines - DISABLED: Flexible spacing +MD031: true + +# MD032 - Lists surrounded by blank lines - DISABLED: Flexible spacing +MD032: false + +# MD033 - Inline HTML - DISABLED: Allows HTML for complex tables and formatting +MD033: false + +# MD034 - Bare URL used - DISABLED: Allows URLs without angle brackets or link text +MD034: false + +# MD035 - Horizontal rule style +MD035: true + +# MD036 - Emphasis used instead of a heading - DISABLED: Allows emphasized text that looks like headings +MD036: false + +# MD037 - Spaces inside emphasis markers +MD037: true + +# MD038 - Spaces inside code span elements +MD038: true + +# MD039 - Spaces inside link text +MD039: true + +# MD040 - Fenced code blocks should have a language specified +MD040: true + +# MD041 - First line in file should be a top level heading - DISABLED: Flexible document structure +MD041: false + +# MD042 - No empty links +MD042: true + +# MD043 - Required heading structure +MD043: true + +# MD044 - Proper names should have the correct capitalization +MD044: true + +# MD045 - Images should have alternate text (alt text) +MD045: true + +# MD046 - Code block style (consistent, fenced, indented) +MD046: true + +# MD047 - Files should end with a single newline character - DISABLED: Flexible file endings +MD047: false + +# MD048 - Code fence style (consistent, backtick, tilde) +MD048: true + +# MD049 - Emphasis style - DISABLED: Allows both * and _ for emphasis +MD049: false + +# MD050 - Strong style - DISABLED: Allows both ** and __ for bold +MD050: false + +# MD051 - Link fragments should be valid +MD051: true + +# MD052 - Reference links and images should use a label that is defined +MD052: true + +# MD053 - Link and image reference definitions should be needed +MD053: true + +# Additional named rules (aliases for MD rules above) +line-length: false # Alias for MD013 +no-hard-tabs: false # Alias for MD010 - allows tabs in code examples +no-bare-urls: false # Alias for MD034 - allows bare URLs in documentation +single-title: false # Alias for MD025 - allows multiple H1 headings +no-emphasis-as-heading: false # Alias for MD036 - allows emphasized text as section markers +no-inline-html: false # Alias for MD033 - allows HTML for complex formatting +ul-style: false # Alias for MD004 - flexible unordered list markers (*, -, +) +ol-prefix: false # Alias for MD029 - flexible ordered list numbering +blanks-around-fences: false # Alias for MD031 - flexible spacing around code blocks +blanks-around-headings: false # Alias for MD022 - flexible spacing around headings +blanks-around-lists: false # Alias for MD032 - flexible spacing around lists +single-trailing-newline: false # Alias for MD047 - flexible file endings +no-multiple-blanks: false # Alias for MD012 - allows multiple blank lines for readability +no-trailing-spaces: false # Alias for MD009 - allows trailing spaces +emphasis-style: false # Alias for MD049 - flexible emphasis style +strong-style: false # Alias for MD050 - flexible strong/bold style +no-space-in-emphasis: false # Alias for MD037 - flexible spacing in emphasis +commands-show-output: false # Shell command examples don't need to show output diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..28507ce --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,28 @@ +# Pre-commit hooks for OpenShift Virtualization test documentation +# See https://pre-commit.com for more information + +repos: + # Markdownlint for Markdown file linting + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: v0.39.0 + hooks: + - id: markdownlint + name: Lint Markdown files + files: \.md$ + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v6.0.0" + hooks: + - id: check-merge-conflict + stages: [pre-commit] + - id: trailing-whitespace + stages: [pre-commit] + - id: end-of-file-fixer + stages: [pre-commit] + - id: detect-private-key + - id: mixed-line-ending + + - repo: https://github.com/gitleaks/gitleaks + rev: "v8.28.0" + hooks: + - id: gitleaks diff --git a/README.md b/README.md index b35067c..2f172ac 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # openshift-virtualizatio-tests-design-docs + This repository is used to manager openshift-virtualization-tests strategy & design docs, such as test plans (STPs) allowing enhanced SIG improvement and collaboration # Openshift Virtualization Quality Engineering Artifacts and Software Test Planning (STP) Repository @@ -14,7 +15,7 @@ This process is mandatory for formal QE sign-off and feature acceptance, as **au ## Glossary | Term | Definition | -|:--------|:----------------------------------------------------------------------------------------------------------------------| +| :------ | :-------------------------------------------------------------------------------------------------------------------- | | **STP** | **Software Test Plan**: Provides the overall roadmap for testing, detailing scope, approach, resources, and schedule. | | **STD** | **Software Test Description**: Provides the specific instructions for executing individual tests. | | **VEP** | **Virtualization Enhancement Proposal**. | @@ -23,25 +24,26 @@ This process is mandatory for formal QE sign-off and feature acceptance, as **au | **FF** | **Feature Freeze** - New features test cases ready & reviewed | | **BO** | **Blockers Only** - New features tested & regression tests run | | **CF** | **Code Freeze** - Regression+new tests pass & bug validation is done | -| **GA* | **General Availability** - Feature is available for general use | +| \**GA* | **General Availability** - Feature is available for general use | ## Process Flow: VEP to QE Sign-off The QE process is initiated upon feature definition, usually correlating with an approved VEP. -1. **Feature Review (Pre-STP)**: The QE owner reviews VEPs (Openshift Virtualization / OCP) and requirements to **understand the value** for RH customers. This stage confirms that requirements are **testable and unambiguous** and that acceptance criteria are clearly defined. Non-functional requirements (NFRs) like Downtime, Connectivity, Performance, Security, Monitoring (metrics/alerts), Portability, and Documentation must be covered. -2. **STP Creation**: The QE owner writes the STP, detailing the scope of testing (functional and non-functional requirements), the overall Test Strategy (including types like Functional, Regression, Upgrade, and Compatibility testing), Test Environment requirements, and Risk Management. The STP must include a clear plan to address risks, including calling out an explicit **list of functionalities that cannot be tested**. -3. **STD Creation**: Once the developer begins actual development, the QE owner writes the STD. STDs must create test cases **with the customer's perspective in mind**. Each step in the test case must be a test and **verify one thing**, and each step must include the expected results. -4. **Tiering and Automation**: QE works with the Development team to define coverage for **Tier 1 and Tier 2 tests**. Automation is crucial, and tests must be running as part of one of the **release checklist jobs**. -5. **Sign-off**: Upon meeting all **Exit Criteria** (high-priority defects resolved, test coverage achieved, test automation merged), the QE owner reviews documentation and formally signs off the feature. Jira tasks are used to **block the epic** until sign-off is complete. +1. **Feature Review (Pre-STP)**: The QE owner reviews VEPs (Openshift Virtualization / OCP) and requirements to **understand the value** for RH customers. This stage confirms that requirements are **testable and unambiguous** and that acceptance criteria are clearly defined. Non-functional requirements (NFRs) like Downtime, Connectivity, Performance, Security, Monitoring (metrics/alerts), Portability, and Documentation must be covered. +2. **STP Creation**: The QE owner writes the STP, detailing the scope of testing (functional and non-functional requirements), the overall Test Strategy (including types like Functional, Regression, Upgrade, and Compatibility testing), Test Environment requirements, and Risk Management. The STP must include a clear plan to address risks, including calling out an explicit **list of functionalities that cannot be tested**. +3. **STD Creation**: Once the developer begins actual development, the QE owner writes the STD. STDs must create test cases **with the customer's perspective in mind**. Each step in the test case must be a test and **verify one thing**, and each step must include the expected results. +4. **Tiering and Automation**: QE works with the Development team to define coverage for **Tier 1 and Tier 2 tests**. Automation is crucial, and tests must be running as part of one of the **release checklist jobs**. +5. **Sign-off**: Upon meeting all **Exit Criteria** (high-priority defects resolved, test coverage achieved, test automation merged), the QE owner reviews documentation and formally signs off the feature. Jira tasks are used to **block the epic** until sign-off is complete. ## Artifacts and Templates This repository hosts the templates for the core QE deliverable: **Software Test Plan (STP) Template**: Outlines the scope, approach, resources, and schedule for all testing activities. - * **Entry Criteria**: Requirements/design approved, environment set up, test cases reviewed. - * **Exit Criteria**: All high-priority defects resolved, automation merged, acceptance criteria met. + +- **Entry Criteria**: Requirements/design approved, environment set up, test cases reviewed. +- **Exit Criteria**: All high-priority defects resolved, automation merged, acceptance criteria met. ## Responsibilities @@ -49,20 +51,20 @@ This repository hosts the templates for the core QE deliverable: The QE Owner is responsible for the overall quality assurance process for a feature, which includes: -* Writing the STP and STD. -* Reviewing the design and technology to identify potential testing challenges. -* Ensuring the plan addresses non-functional requirements (NFRs). -* Managing risk, including adding an explicit list of untestable aspects to the STP. -* Making sure tests are running as part of one of the **release checklist jobs**. -* Performing the final **Sign off the feature as QE**. +- Writing the STP and STD. +- Reviewing the design and technology to identify potential testing challenges. +- Ensuring the plan addresses non-functional requirements (NFRs). +- Managing risk, including adding an explicit list of untestable aspects to the STP. +- Making sure tests are running as part of one of the **release checklist jobs**. +- Performing the final **Sign off the feature as QE**. ### Stakeholders (SIGs/Approvers) All relevant stakeholders, including SIG representatives, must be aware of and agree to the test plan. -* Stakeholders must **approve** the STP. -* Stakeholders must be aware of and **agree to take the risk** associated with any explicit list of functionalities that **cannot be tested**. -* SIGs are responsible for coordinating reviews and approvals of VEPs and STPs. +- Stakeholders must **approve** the STP. +- Stakeholders must be aware of and **agree to take the risk** associated with any explicit list of functionalities that **cannot be tested**. +- SIGs are responsible for coordinating reviews and approvals of VEPs and STPs. ## Deadlines and Check-ins @@ -70,28 +72,90 @@ Deadlines are governed by the Openshift Virtualization release cycle, focusing o The QE process is governed by Openshift Virtualization and KubeVirt release milestones, focusing on test planning completion and automation delivery. -1. **VEP Planning / Feature Review:** - * At the **beginning of every release cycle**, each SIG prioritizes VEPs and decides which ones are being tracked for the upcoming release. This centralized prioritization focuses community efforts on associated pull requests. - * QE involvement starts immediately with the **feature review**, ensuring the requirements are reviewed. The review includes verifying that requirements are **testable and unambiguous** and that the value of the feature for RH customers is understood. +1. **VEP Planning / Feature Review:** + + - At the **beginning of every release cycle**, each SIG prioritizes VEPs and decides which ones are being tracked for the upcoming release. This centralized prioritization focuses community efforts on associated pull requests. + - QE involvement starts immediately with the **feature review**, ensuring the requirements are reviewed. The review includes verifying that requirements are **testable and unambiguous** and that the value of the feature for RH customers is understood. + +2. **Enhancement Freeze (EF):** + + - The Software Test Plan (STP) must be written, reviewed, and approved by stakeholders. Testing cannot begin until the requirements and design documents are approved and stable, and test cases are reviewed. + +3. **Code Freeze (CF):** + + - QE sign-off requires that **test automation must be merged for GA**. The Exit Criteria for testing requires that test automation is merged. + - Features that do not land in the release branch prior to CF will need to file for an exception. + +> \[!NOTE\] QE sign-off is contingent on automation being merged. Features not landing in the release branch prior to CF will need to file for an exception. + +## Development Setup + +### Markdown Linting + +This repository uses automated linting to maintain consistent Markdown formatting across all documentation. + +#### Prerequisites + +Install the required tools: + +```bash +# Install markdownlint-cli +npm install -g markdownlint-cli + +# Install pre-commit (for automated checks) +pip install pre-commit + +# Install pre-commit hooks +pre-commit install +``` + +#### Running Linters + +**Manual linting:** + +```bash +# Lint all Markdown files +markdownlint '**/*.md' + +# Lint specific file +markdownlint stps/stp-template/stp.md + +# Auto-fix issues where possible +markdownlint '**/*.md' --fix +``` + +**Automated linting (via pre-commit):** + +Once pre-commit hooks are installed, linting runs automatically on every commit: + +```bash +# Run all pre-commit hooks manually +pre-commit run --all-files + +# Run only markdownlint +pre-commit run markdownlint --all-files +``` -2. **Enhancement Freeze (EF):** - * The Software Test Plan (STP) must be written, reviewed, and approved by stakeholders. Testing cannot begin until the requirements and design documents are approved and stable, and test cases are reviewed. +#### Linting Rules -3. **Code Freeze (CF):** - * QE sign-off requires that **test automation must be merged for GA**. The Exit Criteria for testing requires that test automation is merged. - * Features that do not land in the release branch prior to CF will need to file for an exception. +The repository uses `.markdownlint.yaml` with the following key settings: -> [!NOTE] -> QE sign-off is contingent on automation being merged. Features not landing in the release branch prior to CF will need to file for an exception. +- **Line length**: Disabled (no line length restrictions) +- **Bare URLs**: Allowed +- **Hard tabs**: Allowed (for code examples) +- **Inline HTML**: Allowed +- **Trailing whitespace**: Allowed +- **Emphasis style**: Flexible +See `.markdownlint.yaml` for complete configuration with detailed comments for each rule. ## **Common Questions** This section addresses frequently asked questions regarding VEP ownership, approvals, and managing delays, which impact the QE process and feature readiness. -| Question | Answer | -|:------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Do PRs need to be approved by STP approvers?** | No, the approval of Pull Requests (PRs) is the **whole SIG's responsibility** for their code. The approver should be aware of the STP and approve based on its content, and a defined process exists to ensure this occurs. | -| **What to do in case all PRs didn't make it on time?** | The author of the STP needs to **file for an exception**. The outcome will be determined individually by maintainers based on the context of the delay. | -| **How to raise attention for my STP?** | The QE has bi-weekly **recurring meetings**. The STP owner is encouraged to join the meeting and introduce the STP to the community to raise awareness. | -| **What if the feature is relates to more than one SIG?** | While features can relate to **multiple SIGs**, a single SIG must always own it. STP owners should reach out to other SIGs that are relevant for the feature and make sure that they also review the STP. | +| Question | Answer | +| :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Do PRs need to be approved by STP approvers?** | No, the approval of Pull Requests (PRs) is the **whole SIG's responsibility** for their code. The approver should be aware of the STP and approve based on its content, and a defined process exists to ensure this occurs. | +| **What to do in case all PRs didn't make it on time?** | The author of the STP needs to **file for an exception**. The outcome will be determined individually by maintainers based on the context of the delay. | +| **How to raise attention for my STP?** | The QE has bi-weekly **recurring meetings**. The STP owner is encouraged to join the meeting and introduce the STP to the community to raise awareness. | +| **What if the feature is relates to more than one SIG?** | While features can relate to **multiple SIGs**, a single SIG must always own it. STP owners should reach out to other SIGs that are relevant for the feature and make sure that they also review the STP. |