Fixed a few bugs and added minor improvements to `scripts/new-release…

….py`

scripts/new-release.py

@@ -4,97 +4,142 @@
 Small script to generate the changelog for a new release. It uses information from
 both git and Github to create teh changelog in Markdown, which can be simply copy/pasted
 to the Github release page.
+
+The script requires a Github token to be set in the environment variable `GITHUB_REPO_TOKEN`.
+As the name implies, only scope `repo` is required for the token (which can be generated
+from https://github.com/settings/tokens/new).
 """
 
+import argparse
 import datetime
 import requests
 import subprocess
 import os
+import pathlib
+import tempfile
+
+__author__    =   "@_hugsy_"
+__version__   =   0.1
+__licence__   =   "MIT"
+__file__      =   "new-release.py"
+__desc__      =   "Generate a new release for a Github project."
+__usage__     =   f"""{__file__} v{__version__}\nby {__author__} under {__licence__}\nsyntax: {__file__} [options] args"""
 
-REPOSITORY  = "hugsy/gef"
-TOKEN       = os.getenv("GITHUB_REPO_TOKEN")
-DEBUG       = False
-OUTPUT_FILE = "/tmp/CHANGELOG.md"
+
+REPOSITORY     = "hugsy/gef"
+GITHUB_TOKEN   = os.getenv("GITHUB_REPO_TOKEN")
+DEBUG          = False
+OUTPUT_FILE    = pathlib.Path( tempfile.gettempdir() ) / "CHANGELOG.md"
 
 
 def dbg(x: str):
     if DEBUG:
         print(x)
+    return
+
 
-def shell(x: str):
+def shell(x: str) -> str:
     dbg(f"   executing: {x}")
     return subprocess.check_output(x, shell=True).strip().decode("utf8")
 
 
-version = datetime.date.today().strftime("%Y.%m")
-codename = shell("random-word").title()
-latest_tag = shell("git describe --abbrev=0")
+def generate_changelog(args: argparse.Namespace) -> bool:
+    """Generate the changelog for the new release."""
+    latest_tag = shell("git describe --abbrev=0")
 
-with open(OUTPUT_FILE, "w") as f:
-    print(f"Creating changelog for {version} in {OUTPUT_FILE}")
-    f.write(f"# Changelog: {version} - {codename}{os.linesep}{os.linesep}")
+    print(f"Creating changelog for {args.version} in {args.output_file.name}")
+    args.output_file.write(f"# Changelog: {args.version} - {args.codename}{os.linesep}{os.linesep}")
 
-    dbg(f"Adding commit summary...")
-    f.write(f"## Highlights of `{codename}`{os.linesep}{os.linesep}")
-    f.write(f"{os.linesep}{os.linesep}")
+    dbg("Adding commit summary...")
+    args.output_file.write(f"## Highlights of `{args.codename}`{os.linesep}{os.linesep}")
+    args.output_file.write(f"{os.linesep}{os.linesep}")
 
-    dbg(f"Adding contributor summary...")
-    f.write(f"## Contributors{os.linesep}{os.linesep}")
-    contributors = shell(f"git log {latest_tag}... --pretty=format:'%aN' | sort -u").splitlines()
-    total_commits = 0
-    f.write(f"| Name | Number of commits | {os.linesep}")
-    f.write(f"|--|--| {os.linesep}")
-    for contrib in contributors:
-        commits = shell(f'git log {latest_tag}...  --pretty=format:"%h" --author="{contrib}"').splitlines()
-        nb_commits = len(commits)
-        f.write(f"| {contrib} | {nb_commits} |{os.linesep}")
-        total_commits += int(nb_commits)
-    f.write("{os.linesep}{os.linesep}")
+    dbg("Adding contributor summary...")
+    args.output_file.write(f"## Contributors{os.linesep}{os.linesep}")
+    contributor_names = shell(f"git log {latest_tag}... --pretty=format:'%aN' | sort -u").splitlines()
+    commits = {}
+    for author in contributor_names:
+        author_commits = shell(f'git log {latest_tag}...  --pretty=format:"%h" --author="{author}"').splitlines()
+        commits[ author ] = len(author_commits)
+    total_commits = sum(commits.values())
 
-    dbg("Adding Github info...")
-    h = requests.get(f"https://api.github.com/repos/${REPOSITORY}/issues?state=closed&milestone.title=Release+{version}",
-                     headers={"Authorization": f"token {TOKEN}"})
+    args.output_file.write(f"| Author | Number of commits | {os.linesep}")
+    args.output_file.write(f"|:--|--:| {os.linesep}")
+    commits_sorted = dict(sorted(commits.items(), key=lambda item: -item[1]))
+    for author in commits_sorted:
+        args.output_file.write(f"| {author} | {commits[author]}|{os.linesep}")
+    args.output_file.write(f"{os.linesep}{os.linesep}")
 
-    js = h.json()
+    dbg("Adding Github info...")
+    url = f"https://api.github.com/repos/{args.repository}/issues?state=closed&milestone.title=Release%3a%20{args.version}"
+    js = requests.get(url, headers={"Authorization": f"token {args.token}"}).json()
     prs = { x['number']: x['html_url'] for x in js if "pull" in x['html_url'] }
     issues = { x['number']: x['html_url'] for x in js if "issues" in x['html_url'] }
+    closed_prs_item = " • ".join([f" [{nb}]({url}) " for nb, url in prs.items()])
+    closed_issues_item = " • ".join([f" [{nb}]({url}) " for nb, url in issues.items()])
+    args.output_file.write(f"""
+## Closed Issues
+
+  * {len(issues)} issues closed ({closed_issues_item})
+
+
+## Closed Pull Requests
+
+ * {len(prs)} PRs closed ({closed_prs_item})
+
+""")
+
+    dbg("Adding commit summary...")
+    log = shell(f"""git log "{latest_tag}"...HEAD  --pretty=format:' * %cs [%h](https://github.com/{args.repository}/commit/%H) • *%aN* • %s ' --reverse""")
+    diff = shell(f"""git diff --no-color --stat {latest_tag} HEAD""")
+    args.output_file.write(f"""
+## Commit details
+
+<details>
+<summary>
+{total_commits} commits since <b>{latest_tag}</b>
+</summary>
+
+### Commit log
 
-    f.write(f"## Closed Issues{os.linesep}{os.linesep}")
-    f.write(f" * {len(issues)} issues closed (")
-    for nb in issues:
-        url = issues[nb]
-        f.write(f" [{nb}]({url}) &bull; ")
-    f.write(f"){os.linesep}")
+{log}
 
-    f.write(f"{os.linesep}{os.linesep}")
+### File diff
 
-    f.write(f"## Closed Pull Requests{os.linesep}{os.linesep}")
-    f.write(f" * {len(prs)} PRs closed (")
-    for nb in prs:
-        url = prs[nb]
-        f.write(f" [{nb}]({url}) &bull; ")
-    f.write("){os.linesep}")
+```diff
+{diff}
+```
 
-    f.write("{os.linesep}{os.linesep}")
+</details>
 
-    dbg(f"Adding commit summary...")
-    f.write(f"## Commit details{os.linesep}{os.linesep}")
-    f.write(f"<details><summary>")
-    f.write(f"{total_commits} commits since <b>{latest_tag}</b>")
-    f.write(f"</summary>{os.linesep}{os.linesep}")
-    f.write( shell(f"""git log "{latest_tag}"...HEAD  --pretty=format:' * %cs [%h](http://github.com/hugsy/gef/commit/%H) &bull; *%aN* &bull; %s ' --reverse""") )
-    f.write(f"{os.linesep}")
-    f.write( shell(f"""git diff --no-color --stat {latest_tag} HEAD""") )
-    f.write(f"{os.linesep}")
-    f.write(f"</details>")
+""")
+    print(f"Done, the changelog file was written to `{args.output_file.name}`")
 
-    f.write(f"{os.linesep}{os.linesep}")
+    if args.push_release:
+        shell(f"""git tag --annotate "{args.version}" --message "Release {args.version} - {args.codename}" --sign""")
+        shell(f"""git push origin "{args.version}" """)
 
-    print(f"Done, the changelog file was written to `{OUTPUT_FILE}`")
+    return True
 
 
-print(f"Push new release {version} ({codename}) live? [y/N] ")
-if input().lower().startswith("y"):
-    shell(f"""git tag --annotate "{version}" --message "Release {version} - {codename}" --sign""")
-    shell(f"""git push origin "{version}" """)
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(usage = __usage__, description = __desc__, prog = __file__)
+    parser.add_argument("--debug", action="store_true", default=DEBUG,
+                        help="Enable debug output")
+    parser.add_argument("-r", "--repository", type=str, default=REPOSITORY,
+                        help="Specify the repository (default: '%(default)s')")
+    parser.add_argument("-t", "--token", dest="token", type=str, metavar="TOKEN",
+                        default=GITHUB_TOKEN, help="Specify the Github token to use (requires `repo` scope)")
+    parser.add_argument("-o", "--output-file", type=argparse.FileType('w', encoding='UTF-8'), default=open(str(OUTPUT_FILE.absolute()), 'w'),
+                        metavar="/path/to/output_file.md", help=f"Specify the output file (default: '{OUTPUT_FILE}')")
+    parser.add_argument("--version", type=str, default=datetime.date.today().strftime("%Y.%m"),
+                        help="Specify the version number (default: '%(default)s')")
+    parser.add_argument("--codename", type=str, default=shell("random-word").title(),
+                        help="Specify the version codename (default: '%(default)s')")
+    parser.add_argument("--push-release", action="store_true", default=False,
+                        help="Create the new tag and publish the release on Github")
 
+    x = parser.parse_args()
+    DEBUG = x.debug
+    generate_changelog(x)
+    exit(0)

tests/binaries/utils.h

@@ -6,31 +6,39 @@
  * optimal, as it adds an extra frame to the stack.
  */
 
-/* Intel x64 */
-#if defined(__x86_64__)
+/* Intel x64 (x86_64) */
+#if defined(__x86_64__) || defined(__amd64__)
 #define DebugBreak() __asm__("int $3")
 
-/* Intel x32 */
+/* Intel x32 (i686) */
 #elif defined(__i386) || defined(i386) || defined(__i386__)
 #define DebugBreak() __asm__("int $3")
 
-/* AARCH64 */
+/* AARCH64 (aarch64) */
 #elif defined(__aarch64__)
 #define DebugBreak() { raise( SIGINT ) ; }
 
-/* ARM */
+/* ARM (armv7le*/
 #elif defined(__arm__) || defined(__arm)
 #define DebugBreak() { raise( SIGINT ) ; }
 
 /* MIPS */
+/* MIPS64 (mips64el) */
 #elif defined(mips) || defined(__mips__) || defined(__mips)
-#define DebugBreak() __builtin_trap()
+#define DebugBreak() { raise( SIGINT ) ; }
 
 /* PowerPC */
+/* PowerPC64 (ppc64le) */
 #elif defined(__powerpc) || defined(__powerpc__) || defined(__powerpc64__) || defined(__POWERPC__) || defined(__ppc__) || defined(__PPC__) || defined(_ARCH_PPC)
-#define DebugBreak() __builtin_trap()
+#define DebugBreak() { raise( SIGINT ) ; }
+
+/* SPARC */
+/* SPARC64 */
+// #elif defined(__sparc) || defined(__sparc64__) || defined(__sparc__)
+// #define DebugBreak() { raise( SIGINT ) ; }
 
 /* the rest */
 #else
 #error "Unsupported architecture"
+// #define DebugBreak() __builtin_trap()
 #endif

tests/helpers.py

@@ -13,13 +13,15 @@
 STRIP_ANSI_DEFAULT = True
 DEFAULT_CONTEXT = "-code -stack"
 ARCH = (os.getenv("GEF_CI_ARCH") or platform.machine()).lower()
-CI_VALID_ARCHITECTURES = ("x86_64", "i686", "aarch64", "armv7l")
+CI_VALID_ARCHITECTURES_32B = ("i686", "armv7l")
+CI_VALID_ARCHITECTURES_64B = ("x86_64", "aarch64", "mips64el", "ppc64le")
+CI_VALID_ARCHITECTURES = CI_VALID_ARCHITECTURES_64B + CI_VALID_ARCHITECTURES_32B
 
 CommandType = NewType("CommandType", Union[str, Iterable[str]])
 
 
 def is_64b() -> bool:
-    return ARCH in ("x86_64", "aarch64")
+    return ARCH in CI_VALID_ARCHITECTURES_64B
 
 
 def ansi_clean(s: str) -> str:
@@ -28,7 +30,7 @@ def ansi_clean(s: str) -> str:
 
 
 def _add_command(commands: CommandType) -> List[str]:
-    if type(commands) == str:
+    if isinstance(commands, str):
         commands = [commands]
     return [_str for cmd in commands for _str in ["-ex", cmd]]