Skip to content

Progress page: add Projects with most patches + allow ports to skip publishing if no pax found #207

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
IgorTodorovskiIBM merged 1 commit into from
Mar 6, 2023
Merged

Progress page: add Projects with most patches + allow ports to skip publishing if no pax found #207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cicd/publish.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ VERSION=`find . -name ".version" | xargs cat`

if [ ! -f "$PAX" ]; then
echo "Port pax file does not exist";
exit 1;
exit 0;
fi

if [ -z "$DEPENDENCIES" ]; then
Expand Down
207 changes: 118 additions & 89 deletions tools/getbinaries.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,17 @@
todaysDate = datetime.now().strftime('%Y-%m-%d %H:%M:%S')

progressPerStatus = {
"Green": 0,
"Blue": 0,
"Yellow": 0,
"Skipped": 0,
"Not built": 0
"Green": 0,
"Blue": 0,
"Yellow": 0,
"Skipped": 0,
"Not built": 0
}

statusPerPort = {}
dependentOn = {}
patchesPerPort = {}
totalPatchLinesPerPort = {}

# using an access token
g = Github("access_token")
Expand All @@ -39,65 +41,82 @@
g = Github(os.getenv('GITHUB_OAUTH_TOKEN'))

with open('docs/Latest.md', 'w') as f:
sys.stdout = f # Change the standard output to the file we created.
print("# z/OS Open Tools - Packages\n")
print("Note: to download the latest packages, use the `zopen install` script from the [meta repo](https://github.com/ZOSOpenTools/meta)\n")
print("| Package | Status | Test Success Rate | Latest Release | Description |")
print("|---|---|---|---|---|")

for r in g.get_user("ZOSOpenTools").get_repos():
if not re.search("port$", r.name):
continue
print("| [" + r.name + "](" + r.html_url + ")", end='')
dependentOn[r.name] = []
releases = r.get_releases()
if (releases.totalCount):
latestRelease = r.get_latest_release()
status = latestRelease.body
m = re.search(".*Test Status:</b>[ ]+([^ ]+)[ ]+\(([^)]*)", status)
if m:
progressPerStatus[m.group(1)] += 1;
successRate = re.search("(\d+\.?\d+?)% success rate", m.group(2));
print("|" + m.group(1), end='')
if successRate:
print("|" + successRate.group(1) + "%", end='')
statusPerPort[r.name] = int(successRate.group(1).split(".", 1)[0]);
else:
print("| N/A", end='');
statusPerPort[r.name] = -1;
else:
progressPerStatus["Skipped"] += 1;
statusPerPort[r.name] = -1;
print("| No status", end='');
print("| N/A", end='');
else:
progressPerStatus["Not built"] += 1;
statusPerPort[r.name] = -2;
print("| No status", end='');
print("| N/A", end='');
#print("| [Releases](" + r.html_url + "/releases)", end='')

if (releases.totalCount):
latestRelease = r.get_latest_release()
print("| [" + latestRelease.tag_name + "](" + latestRelease.html_url + ")", end='')
print("| " + latestRelease.body)
else:
print("| None | |")

print("Last updated: ", todaysDate);
sys.stdout = f # Change the standard output to the file we created.
print("# z/OS Open Tools - Packages\n")
print("Note: to download the latest packages, use the `zopen install` script from the [meta repo](https://github.com/ZOSOpenTools/meta)\n")
print("| Package | Status | Test Success Rate | Latest Release | Description |")
print("|---|---|---|---|---|")

for r in g.get_user("ZOSOpenTools").get_repos():
if not re.search("port$", r.name):
continue
print("| [" + r.name + "](" + r.html_url + ")", end='')
dependentOn[r.name] = []
releases = r.get_releases()
if (releases.totalCount):
latestRelease = r.get_latest_release()
status = latestRelease.body
m = re.search(".*Test Status:</b>[ ]+([^ ]+)[ ]+\(([^)]*)", status)
if m:
progressPerStatus[m.group(1)] += 1;
successRate = re.search("(\d+\.?\d+?)% success rate", m.group(2));
print("|" + m.group(1), end='')
if successRate:
print("|" + successRate.group(1) + "%", end='')
statusPerPort[r.name] = int(successRate.group(1).split(".", 1)[0]);
else:
print("| N/A", end='');
statusPerPort[r.name] = -1;
else:
progressPerStatus["Skipped"] += 1;
statusPerPort[r.name] = -1;
print("| No status", end='');
print("| N/A", end='');
else:
progressPerStatus["Not built"] += 1;
statusPerPort[r.name] = -2;
print("| No status", end='');
print("| N/A", end='');
#print("| [Releases](" + r.html_url + "/releases)", end='')

if (releases.totalCount):
latestRelease = r.get_latest_release()
print("| [" + latestRelease.tag_name + "](" + latestRelease.html_url + ")", end='')
print("| " + latestRelease.body)
else:
print("| None | |")

# Clone the repo
subprocess.run(['git', 'clone', r.clone_url])

# Count the number of .patch files and total lines in patches directory
num_patches = 0
total_lines = 0
if os.path.isdir(r.name):
patch_dir = os.path.join(r.name, 'patches')
if os.path.isdir(patch_dir):
for filename in os.listdir(patch_dir):
if filename.endswith('.patch'):
num_patches += 1
with open(os.path.join(patch_dir, filename), 'r') as patch_file:
total_lines += len(patch_file.readlines())
patchesPerPort[r.name] = num_patches
totalPatchLinesPerPort[r.name] = total_lines

print("Last updated: ", todaysDate);

for rname, y in dependentOn.items():
response = requests.get("https://raw.githubusercontent.com/ZOSOpenTools/" + rname + "/main/buildenv")
if response.status_code == 200:
name=re.sub('port$', '', rname)
matches = re.findall('export\s+ZOPEN.*DEPS\s*=\s*"([^"]*)"', response.text)
dependencies = []
for match in matches:
dependencies += match.split();
dependencies = list(set(dependencies))
for x in dependencies:
if x + "port" in dependentOn:
dependentOn[x + "port"] += [name]
response = requests.get("https://raw.githubusercontent.com/ZOSOpenTools/" + rname + "/main/buildenv")
if response.status_code == 200:
name=re.sub('port$', '', rname)
matches = re.findall('export\s+ZOPEN.*DEPS\s*=\s*"([^"]*)"', response.text)
dependencies = []
for match in matches:
dependencies += match.split();
dependencies = list(set(dependencies))
for x in dependencies:
if x + "port" in dependentOn:
dependentOn[x + "port"] += [name]
# Data to plot
labels = []
sizes = []
Expand All @@ -119,9 +138,9 @@
labels = []
sizes = []
for x, y in sorted(statusPerPort.items(), key=lambda x: x[1]):
if y >= 0:
labels.append(x)
sizes.append(y)
if y >= 0:
labels.append(x)
sizes.append(y)
fig = plt.figure()
fig.set_size_inches(20, 12)
ax = fig.add_axes([0,0,1,1])
Expand All @@ -145,8 +164,8 @@
plt.savefig('docs/images/quality.png', bbox_inches="tight")

with open('docs/Progress.md', 'w') as f:
sys.stdout = f # Change the standard output to the file we created.
print("""
sys.stdout = f # Change the standard output to the file we created.
print("""
## Overall Status
* <span style="color:green">Green</a>: All tests passing
* <span style="color:blue">Blue</a>: Most tests passing
Expand All @@ -160,30 +179,40 @@
![image info](./images/quality.png)

## Projects with skipped or no tests (grey)
""");
for x, y in sorted(statusPerPort.items(), key=lambda x: x[1]):
if y == -1:
print("* [" + x + "](https://github.com/ZOSOpenTools/" + x + ")");
""");
for x, y in sorted(statusPerPort.items(), key=lambda x: x[1]):
if y == -1:
print("* [" + x + "](https://github.com/ZOSOpenTools/" + x + ")");

print("## Projects that do not have builds\n");
for x, y in sorted(statusPerPort.items(), key=lambda x: x[1]):
if y == -2:
print("* [" + x + "](https://github.com/ZOSOpenTools/" + x + ")");
print("## Projects that do not have builds\n");
for x, y in sorted(statusPerPort.items(), key=lambda x: x[1]):
if y == -2:
print("* [" + x + "](https://github.com/ZOSOpenTools/" + x + ")");

print("""
print("""
## Projects with the most dependencies
""");
print("| Package | # of Dependent Projects | Test Success Rate | Dependent projects");
print("|---|---|---|--|");
for x,y in sorted(dependentOn.items(), reverse=True, key=lambda x: len(x[1])):
status = statusPerPort[x]
if status == -1:
status = "Skipped"
elif status == -2:
status = "No builds"
else:
status = str(status) + "%"
print("| [" + x + "](https://github.com/ZOSOpenTools/" + x + ") | " + str(len(y)) + " | " + status + " |" + ", ".join(str(e) for e in y));

print("\nLast updated: ", todaysDate);
print("| Package | # of Dependent Projects | Test Success Rate | Dependent projects");
print("|---|---|---|--|");
for x,y in sorted(dependentOn.items(), reverse=True, key=lambda x: len(x[1])):
status = statusPerPort[x]
if status == -1:
status = "Skipped"
elif status == -2:
status = "No builds"
else:
status = str(status) + "%"
print("| [" + x + "](https://github.com/ZOSOpenTools/" + x + ") | " + str(len(y)) + " | " + status + " |" + ", ".join(str(e) for e in y));

print("""
## Projects with the most patches
""");
print("| Package | # of Patches Lines | # of Patches");
print("|---|---|--|");
for x,y in sorted(totalPatchLinesPerPort.items(), reverse=True, key=lambda x: x[1]):
patches = patchesPerPort[x]
patchLines = totalPatchLinesPerPort[x]
print("| [" + x + "](https://github.com/ZOSOpenTools/" + x + ") | " + str(patchLines) + " | " + str(patches));

print("\nLast updated: ", todaysDate);