docs: add changelog (#257)

.pre-commit-config.yaml

@@ -4,6 +4,7 @@ repos:
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
+        exclude: ^(scripts/docs_changelog_generator/.*\.tpl)$
       - id: trailing-whitespace
       - id: check-merge-conflict
       - id: check-case-conflict

documentation/source/conf.py

@@ -50,6 +50,7 @@ def setup_paths():
     "sphinx.ext.todo",
     "sphinx.ext.viewcode",
     "sphinxcontrib.autodoc_pydantic",
+    "sphinx_toolbox.collapse",
 ]
 
 napoleon_use_admonition_for_examples = True

documentation/source/index.rst

@@ -12,6 +12,7 @@ You can start a trial version or request a demo via https://www.ibm.com/products
     V2 Migration Guide <v2_migration_guide>
     Examples <rst_source/examples>
     FAQ <faq>
+    Changelog <changelog>
 
 
 .. admonition:: Migration to V2

pyproject.toml

@@ -98,6 +98,13 @@ poethepoet = "^0.24.4"
 pandas = "^2.0.0"
 pyyaml = "^6.0.0"
 tqdm = "^4.66.1"
+sphinx-toolbox = "3.5.0"
+git-changelog = "2.4.0"
+gitchangelog = "3.0.4"
+pystache = "0.6.5"
+tabulate = "0.9.0"
+m2r = "^0.3.1"
+
 
 [tool.poetry.extras]
 langchain = ["langchain", "pyyaml"]
@@ -204,6 +211,26 @@ args = ["type"]
     help = "Remove build"
     sequence = [{ "script" = "shutil:rmtree('build/', ignore_errors=1)" }, { "script" = "shutil:rmtree('source/rst_source', ignore_errors=1)" }, { cmd = "make clean" }]
 
+[tool.poe.tasks.changelog]
+help = "Changelog related commands"
+control.expr = "type"
+args = [
+    { name = "type" },
+    { name = "version", positional = true },
+]
+
+    [[tool.poe.tasks.changelog.switch]]
+    case = "update"
+    help = "Update unreleased changelog section"
+    env = { "PYTHONPATH" = "scripts" }
+    cmd = "python scripts/docs_changelog_generator/generate_changelog.py --debug"
+
+    [[tool.poe.tasks.changelog.switch]]
+    case = "release"
+    help = "Remove build"
+    env = { "PYTHONPATH" = "scripts" }
+    cmd = "python scripts/docs_changelog_generator/release_changelog.py $version"
+
 [tool.poe.tasks.test]
 args = ["type"]
 control.expr = "type"

scripts/docs_changelog_generator/authors_by_name.yaml

@@ -0,0 +1,3 @@
+Tomas Dvorak: Tomas2D
+Radek Jezek: jezekra1
+David Kristek: David-Kristek

scripts/docs_changelog_generator/config.py

@@ -0,0 +1,22 @@
+from contextvars import ContextVar
+from pathlib import Path
+
+import dotenv
+from pydantic import BaseModel
+
+dirname = Path(__file__).parent.absolute()
+dotenv.load_dotenv()
+
+
+class ChangelogConfig(BaseModel):
+    output_file_path: Path = Path(dirname, "../../documentation/source/changelog.rst")
+    author_names_mapping_path: Path = dirname / Path("authors_by_name.yaml")
+    mustache_template_path: Path = dirname / Path("mustache_rst.tpl")
+    gitchangelog_config_path: Path = dirname / Path("gitchangelog_config.py")
+    repo_url: str = "https://github.com/IBM/ibm-generative-ai"
+    repo_api_url: str = "https://api.github.com/repos/IBM/ibm-generative-ai"
+    unreleased_version_label: str = "(unreleased)"  # Changing this requires manual update in existing changelog
+
+
+DefaultChangelogConfig = ChangelogConfig()
+config_context_var = ContextVar[ChangelogConfig]("config", default=DefaultChangelogConfig)

scripts/docs_changelog_generator/generate_changelog.py

@@ -0,0 +1,109 @@
+import os
+import re
+import textwrap
+
+import yaml
+from _common.logger import get_logger
+from dotenv import load_dotenv
+from tabulate import tabulate
+
+from docs_changelog_generator.config import ChangelogConfig, DefaultChangelogConfig, config_context_var
+from docs_changelog_generator.utils import (
+    GithubRSTLinks,
+    get_git_tags,
+    output_engine_pipeline,
+    shave_marks,
+    use_context,
+)
+from genai._generated.endpoints import ApiEndpoint
+
+load_dotenv()
+
+logger = get_logger(__name__)
+
+
+@output_engine_pipeline
+def inject_versions(data, opts, _config):
+    rows = [[endpoint.method, endpoint.path, endpoint.version] for endpoint in ApiEndpoint.__subclasses__()]
+    versions_table = tabulate(rows, headers=["Method", "Path", "Version (YYYY-MM-DD)"], tablefmt="rst")
+
+    data["api_versions_table"] = "\n".join(
+        [
+            "🔗 API Endpoint Versions",
+            "^^^^^^^^^^^^^^^^^^^^^^^^",
+            "",
+            ".. collapse:: API Endpoint Versions",
+            "",
+            textwrap.indent(versions_table, " " * 4),
+        ]
+    )
+    return data, opts
+
+
+@output_engine_pipeline
+def inject_author_usernames(data, opts, config: ChangelogConfig):
+    github_links = GithubRSTLinks(config.repo_url)
+
+    with open(config.author_names_mapping_path) as f:
+        author_names_map = yaml.safe_load(f)
+
+    if not isinstance(data["versions"], list):
+        data["versions"] = list(data["versions"])
+    for version in data["versions"]:
+        commits = [commit for section in version["sections"] for commit in section["commits"]]
+        for commit in commits:
+            resolved_author_names = []
+            for author in commit["authors"]:
+                author_shaved = shave_marks(author)
+                if author_shaved not in author_names_map:
+                    logger.warn(f'Author "{author}" not found in {config.author_names_mapping_path}')
+                    resolved_author_names.append(author)
+                else:
+                    resolved_author_names.append(github_links.link_to_user(author_names_map[author_shaved]))
+            commit["author_names_resolved"] = ", ".join(resolved_author_names)
+    return data, opts
+
+
+@output_engine_pipeline
+def inject_compare_link(data, opts, config):
+    github_links = GithubRSTLinks(config.repo_url)
+
+    if not isinstance(data["versions"], list):
+        data["versions"] = list(data["versions"])
+
+    tags = get_git_tags()
+
+    for version in data["versions"]:
+        if version["tag"] is None:
+            link = github_links.link_to_compare(tags[-1].identifier, "HEAD")  # latest ... HEAD
+        else:
+            version_index = tags.index(version["tag"])
+            link = github_links.link_to_compare(tags[version_index].identifier, tags[version_index - 1].identifier)
+        version["full_changelog_link"] = f"**Full Changelog**: {link}"
+    return data, opts
+
+
+def subject_process(text):
+    config = config_context_var.get()
+    github_links = GithubRSTLinks(config.repo_url)
+
+    pr_pattern = re.compile(r"\(#([0-9]+)\)$")
+    [pr_id] = pr_pattern.findall(text) or [None]
+    if pr_id:
+        return pr_pattern.sub(github_links.link_to_pr(pr_id), text)
+    return text
+
+
+def generate_changelog(config: ChangelogConfig):
+    from gitchangelog.gitchangelog import main as gitchangelog
+
+    os.environ.setdefault("GITCHANGELOG_CONFIG_FILENAME", str(config.gitchangelog_config_path))
+
+    with use_context(config_context_var, config):
+        gitchangelog()  # parses cmd arguments
+
+
+if __name__ == "__main__":
+    logger.info("Generating changelog from GIT")
+    generate_changelog(DefaultChangelogConfig)
+    logger.info("Done!")

scripts/docs_changelog_generator/gitchangelog_config.py

@@ -0,0 +1,56 @@
+from gitchangelog.gitchangelog import Caret, FileFirstRegexMatch, FileRegexSubst, mustache
+
+from docs_changelog_generator.config import config_context_var
+from docs_changelog_generator.generate_changelog import (
+    inject_author_usernames,
+    inject_compare_link,
+    inject_versions,
+    subject_process,
+)
+
+config = config_context_var.get()
+
+INSERT_POINT_REGEX = r"""(?isxu)
+^
+(
+  \s*Changelog\s*(\n|\r\n|\r)        ## ``Changelog`` line
+  ==+\s*(\n|\r\n|\r){2}              ## ``=========`` rest underline
+)
+
+(                     ## Match all between changelog and release rev
+    (
+      (?!
+         (?<=(\n|\r))                ## look back for newline
+         %(rev)s                     ## revision
+         \s+
+         \([0-9]+-[0-9]{2}-[0-9]{2}\)(\n|\r\n|\r)   ## date
+           --+(\n|\r\n|\r)                          ## ``---`` underline
+      )
+      .
+    )*
+)
+
+(?P<rev>%(rev)s)
+""" % {"rev": r"v[0-9]+\.[0-9]+(\.[0-9]+)?"}
+
+section_regexps = [
+    ("🌟 New", [r"^[nN]ew\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$"]),
+    ("🗒️ Changes", [r"^[cC]hg\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$"]),
+    ("🐛 Bug Fixes", [r"^[fF]ix\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$"]),
+    ("🚀 Features / Enhancements", [r"^[fF]eat\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$"]),
+    ("📖 Docs", [r"^[dD]ocs?\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$"]),
+    ("⚙️ Other", None),  ## Match all lines
+]
+
+
+subject_process = subject_process
+unreleased_version_label = config.unreleased_version_label
+
+revs = [Caret(FileFirstRegexMatch(config.output_file_path, INSERT_POINT_REGEX)), "HEAD"]
+
+output_engine = mustache(str(config.mustache_template_path))
+output_engine = inject_author_usernames(output_engine)
+output_engine = inject_compare_link(output_engine)
+output_engine = inject_versions(output_engine)
+
+publish = FileRegexSubst(config.output_file_path, INSERT_POINT_REGEX, r"\1\o\g<rev>")

scripts/docs_changelog_generator/mustache_rst.tpl

@@ -0,0 +1,29 @@
+{{#general_title}}
+{{{title}}}
+{{#title_chars}}={{/title_chars}}
+
+
+{{/general_title}}
+{{#versions}}
+{{{label}}}
+{{#label_chars}}-{{/label_chars}}
+{{#sections}}
+{{#display_label}}
+
+{{{label}}}
+{{#label_chars}}^{{/label_chars}}
+{{/display_label}}
+{{#commits}}
+- {{{subject}}} [{{{author_names_resolved}}}]
+{{#body}}
+{{{body_indented}}}
+{{/body}}
+{{/commits}}
+{{/sections}}
+
+{{{full_changelog_link}}}
+
+{{/versions}}
+
+{{{api_versions_table}}}
+

scripts/docs_changelog_generator/parse_releases.py

@@ -0,0 +1,64 @@
+import re
+from datetime import datetime
+
+import m2r
+import requests
+
+from docs_changelog_generator.config import ChangelogConfig, DefaultChangelogConfig
+from docs_changelog_generator.utils import GithubRSTLinks
+
+
+def parse_github_releases(config: ChangelogConfig) -> list[str]:
+    """Function to parse github release notes, if that's ever needed again."""
+    releases = requests.get(f"{config.repo_api_url}/releases").json()
+    github_links = GithubRSTLinks(config.repo_url)
+
+    result = ["Changelog", "=========", "\n"]
+    for release in releases:
+        name = (
+            release["name"].replace("Release ", "")
+            + f" ({datetime.fromisoformat(release['published_at']).strftime('%Y-%m-%d')})"
+        )
+        result += [name]
+        result += ["-" * len(name)]
+
+        class MyRenderer(m2r.RestRenderer):
+            hmarks = {1: "^", 2: "^", 3: "^", 4: "^", 5: "^", 6: "^"}  # unify all headings to the same level
+
+        body_pre_convert = release["body"]
+        # Remove "What's changed" header
+        body_pre_convert = re.sub(r"#+\s*What'?s? [Cc]hanged\s*\n", "", body_pre_convert)
+
+        # Convert MD to RST
+        body_converted = m2r.convert(body_pre_convert, renderer=MyRenderer())
+
+        # Convert compare links
+        body_converted = re.sub(
+            rf"{config.repo_url}/compare/(v[0-9]\.[0-9]\.[0-9])...(v[0-9]\.[0-9]\.[0-9])",
+            github_links.link_to_compare("\\1", "\\2"),
+            body_converted,
+        )
+        # Convert PR links
+        body_converted = re.sub(rf"{config.repo_url}/pull/([0-9]*)", github_links.link_to_pr("\\1"), body_converted)
+        # Convert User links
+        body_converted = re.sub(r"@(\S+)", github_links.link_to_user("\\1"), body_converted)
+
+        # Convert emojis
+        body_converted = (
+            body_converted.replace(r"\ :", ":")
+            .replace(":bug:", "🐛")
+            .replace(":hammer:", "🔨")
+            .replace(":sparkles:", "✨")
+            .replace(":wrench:", "🔧")
+            .replace(":art:", "🎨")
+            .replace(":rocket:", "🚀")
+        )
+        result += [body_converted, ""]
+    return [f"{line}\n" for line in result]
+
+
+if __name__ == "__main__":
+    config = DefaultChangelogConfig
+    changelog = parse_github_releases(config)
+    with open(config.output_file_path, "w") as f:
+        f.writelines(changelog)