From 69fb0d4eeee8e97179aa75fdfa1bbb8d1bb230f7 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Fri, 24 Feb 2023 22:47:59 -0500 Subject: [PATCH] Add Projects with most patches in progress + allow ports to not publish pax files --- cicd/publish.groovy | 2 +- tools/getbinaries.py | 207 ++++++++++++++++++++++++------------------- 2 files changed, 119 insertions(+), 90 deletions(-) diff --git a/cicd/publish.groovy b/cicd/publish.groovy index d13d438b5..f3ce7bc9a 100755 --- a/cicd/publish.groovy +++ b/cicd/publish.groovy @@ -26,7 +26,7 @@ VERSION=`find . -name ".version" | xargs cat` if [ ! -f "$PAX" ]; then echo "Port pax file does not exist"; - exit 1; + exit 0; fi if [ -z "$DEPENDENCIES" ]; then diff --git a/tools/getbinaries.py b/tools/getbinaries.py index 4fffe9e22..398796ec1 100644 --- a/tools/getbinaries.py +++ b/tools/getbinaries.py @@ -18,15 +18,17 @@ todaysDate = datetime.now().strftime('%Y-%m-%d %H:%M:%S') progressPerStatus = { - "Green": 0, - "Blue": 0, - "Yellow": 0, - "Skipped": 0, - "Not built": 0 + "Green": 0, + "Blue": 0, + "Yellow": 0, + "Skipped": 0, + "Not built": 0 } statusPerPort = {} dependentOn = {} +patchesPerPort = {} +totalPatchLinesPerPort = {} # using an access token g = Github("access_token") @@ -39,65 +41,82 @@ g = Github(os.getenv('GITHUB_OAUTH_TOKEN')) with open('docs/Latest.md', 'w') as f: - sys.stdout = f # Change the standard output to the file we created. - print("# z/OS Open Tools - Packages\n") - print("Note: to download the latest packages, use the `zopen install` script from the [meta repo](https://github.com/ZOSOpenTools/meta)\n") - print("| Package | Status | Test Success Rate | Latest Release | Description |") - print("|---|---|---|---|---|") - - for r in g.get_user("ZOSOpenTools").get_repos(): - if not re.search("port$", r.name): - continue - print("| [" + r.name + "](" + r.html_url + ")", end='') - dependentOn[r.name] = [] - releases = r.get_releases() - if (releases.totalCount): - latestRelease = r.get_latest_release() - status = latestRelease.body - m = re.search(".*Test Status:[ ]+([^ ]+)[ ]+\(([^)]*)", status) - if m: - progressPerStatus[m.group(1)] += 1; - successRate = re.search("(\d+\.?\d+?)% success rate", m.group(2)); - print("|" + m.group(1), end='') - if successRate: - print("|" + successRate.group(1) + "%", end='') - statusPerPort[r.name] = int(successRate.group(1).split(".", 1)[0]); - else: - print("| N/A", end=''); - statusPerPort[r.name] = -1; - else: - progressPerStatus["Skipped"] += 1; - statusPerPort[r.name] = -1; - print("| No status", end=''); - print("| N/A", end=''); - else: - progressPerStatus["Not built"] += 1; - statusPerPort[r.name] = -2; - print("| No status", end=''); - print("| N/A", end=''); - #print("| [Releases](" + r.html_url + "/releases)", end='') - - if (releases.totalCount): - latestRelease = r.get_latest_release() - print("| [" + latestRelease.tag_name + "](" + latestRelease.html_url + ")", end='') - print("| " + latestRelease.body) - else: - print("| None | |") - - print("Last updated: ", todaysDate); + sys.stdout = f # Change the standard output to the file we created. + print("# z/OS Open Tools - Packages\n") + print("Note: to download the latest packages, use the `zopen install` script from the [meta repo](https://github.com/ZOSOpenTools/meta)\n") + print("| Package | Status | Test Success Rate | Latest Release | Description |") + print("|---|---|---|---|---|") + + for r in g.get_user("ZOSOpenTools").get_repos(): + if not re.search("port$", r.name): + continue + print("| [" + r.name + "](" + r.html_url + ")", end='') + dependentOn[r.name] = [] + releases = r.get_releases() + if (releases.totalCount): + latestRelease = r.get_latest_release() + status = latestRelease.body + m = re.search(".*Test Status:[ ]+([^ ]+)[ ]+\(([^)]*)", status) + if m: + progressPerStatus[m.group(1)] += 1; + successRate = re.search("(\d+\.?\d+?)% success rate", m.group(2)); + print("|" + m.group(1), end='') + if successRate: + print("|" + successRate.group(1) + "%", end='') + statusPerPort[r.name] = int(successRate.group(1).split(".", 1)[0]); + else: + print("| N/A", end=''); + statusPerPort[r.name] = -1; + else: + progressPerStatus["Skipped"] += 1; + statusPerPort[r.name] = -1; + print("| No status", end=''); + print("| N/A", end=''); + else: + progressPerStatus["Not built"] += 1; + statusPerPort[r.name] = -2; + print("| No status", end=''); + print("| N/A", end=''); + #print("| [Releases](" + r.html_url + "/releases)", end='') + + if (releases.totalCount): + latestRelease = r.get_latest_release() + print("| [" + latestRelease.tag_name + "](" + latestRelease.html_url + ")", end='') + print("| " + latestRelease.body) + else: + print("| None | |") + + # Clone the repo + subprocess.run(['git', 'clone', r.clone_url]) + + # Count the number of .patch files and total lines in patches directory + num_patches = 0 + total_lines = 0 + if os.path.isdir(r.name): + patch_dir = os.path.join(r.name, 'patches') + if os.path.isdir(patch_dir): + for filename in os.listdir(patch_dir): + if filename.endswith('.patch'): + num_patches += 1 + with open(os.path.join(patch_dir, filename), 'r') as patch_file: + total_lines += len(patch_file.readlines()) + patchesPerPort[r.name] = num_patches + totalPatchLinesPerPort[r.name] = total_lines + + print("Last updated: ", todaysDate); for rname, y in dependentOn.items(): - response = requests.get("https://raw.githubusercontent.com/ZOSOpenTools/" + rname + "/main/buildenv") - if response.status_code == 200: - name=re.sub('port$', '', rname) - matches = re.findall('export\s+ZOPEN.*DEPS\s*=\s*"([^"]*)"', response.text) - dependencies = [] - for match in matches: - dependencies += match.split(); - dependencies = list(set(dependencies)) - for x in dependencies: - if x + "port" in dependentOn: - dependentOn[x + "port"] += [name] + response = requests.get("https://raw.githubusercontent.com/ZOSOpenTools/" + rname + "/main/buildenv") + if response.status_code == 200: + name=re.sub('port$', '', rname) + matches = re.findall('export\s+ZOPEN.*DEPS\s*=\s*"([^"]*)"', response.text) + dependencies = [] + for match in matches: + dependencies += match.split(); + dependencies = list(set(dependencies)) + for x in dependencies: + if x + "port" in dependentOn: + dependentOn[x + "port"] += [name] # Data to plot labels = [] sizes = [] @@ -119,9 +138,9 @@ labels = [] sizes = [] for x, y in sorted(statusPerPort.items(), key=lambda x: x[1]): - if y >= 0: - labels.append(x) - sizes.append(y) + if y >= 0: + labels.append(x) + sizes.append(y) fig = plt.figure() fig.set_size_inches(20, 12) ax = fig.add_axes([0,0,1,1]) @@ -145,8 +164,8 @@ plt.savefig('docs/images/quality.png', bbox_inches="tight") with open('docs/Progress.md', 'w') as f: - sys.stdout = f # Change the standard output to the file we created. - print(""" + sys.stdout = f # Change the standard output to the file we created. + print(""" ## Overall Status * Green: All tests passing * Blue: Most tests passing @@ -160,30 +179,40 @@ ![image info](./images/quality.png) ## Projects with skipped or no tests (grey) - """); - for x, y in sorted(statusPerPort.items(), key=lambda x: x[1]): - if y == -1: - print("* [" + x + "](https://github.com/ZOSOpenTools/" + x + ")"); + """); + for x, y in sorted(statusPerPort.items(), key=lambda x: x[1]): + if y == -1: + print("* [" + x + "](https://github.com/ZOSOpenTools/" + x + ")"); - print("## Projects that do not have builds\n"); - for x, y in sorted(statusPerPort.items(), key=lambda x: x[1]): - if y == -2: - print("* [" + x + "](https://github.com/ZOSOpenTools/" + x + ")"); + print("## Projects that do not have builds\n"); + for x, y in sorted(statusPerPort.items(), key=lambda x: x[1]): + if y == -2: + print("* [" + x + "](https://github.com/ZOSOpenTools/" + x + ")"); - print(""" + print(""" ## Projects with the most dependencies """); - print("| Package | # of Dependent Projects | Test Success Rate | Dependent projects"); - print("|---|---|---|--|"); - for x,y in sorted(dependentOn.items(), reverse=True, key=lambda x: len(x[1])): - status = statusPerPort[x] - if status == -1: - status = "Skipped" - elif status == -2: - status = "No builds" - else: - status = str(status) + "%" - print("| [" + x + "](https://github.com/ZOSOpenTools/" + x + ") | " + str(len(y)) + " | " + status + " |" + ", ".join(str(e) for e in y)); - - print("\nLast updated: ", todaysDate); + print("| Package | # of Dependent Projects | Test Success Rate | Dependent projects"); + print("|---|---|---|--|"); + for x,y in sorted(dependentOn.items(), reverse=True, key=lambda x: len(x[1])): + status = statusPerPort[x] + if status == -1: + status = "Skipped" + elif status == -2: + status = "No builds" + else: + status = str(status) + "%" + print("| [" + x + "](https://github.com/ZOSOpenTools/" + x + ") | " + str(len(y)) + " | " + status + " |" + ", ".join(str(e) for e in y)); + + print(""" +## Projects with the most patches +"""); + print("| Package | # of Patches Lines | # of Patches"); + print("|---|---|--|"); + for x,y in sorted(totalPatchLinesPerPort.items(), reverse=True, key=lambda x: x[1]): + patches = patchesPerPort[x] + patchLines = totalPatchLinesPerPort[x] + print("| [" + x + "](https://github.com/ZOSOpenTools/" + x + ") | " + str(patchLines) + " | " + str(patches)); + + print("\nLast updated: ", todaysDate);