Skip to content

Add new columns for daily report #171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
+103 −74
Open

Add new columns for daily report #171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
af8b460
Add new columns for daily report
naved001 Nov 24, 2025
f76b2d2
Restructure some things for a project invoice
naved001 Nov 24, 2025
271f32c
Fix tests
naved001 Nov 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 20 additions & 15 deletions openshift_metrics/invoice.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,19 +192,21 @@ class Rates:
gpu_h100: Decimal


@dataclass()
class ReportMetadata:
report_month: str
cluster_name: str
report_start_time: datetime.datetime
report_end_time: datetime.datetime
generated_at: datetime.datetime


@dataclass
class ProjectInvoce:
"""Represents the invoicing data for a project."""

invoice_month: str
project: str
project_id: str
pi: str
cluster_name: str
invoice_email: str
invoice_address: str
intitution: str
institution_specific_code: str
rates: Rates
su_definitions: dict
ignore_hours: Optional[List[Tuple[datetime.datetime, datetime.datetime]]] = None
Expand Down Expand Up @@ -240,27 +242,30 @@ def get_rate(self, su_type) -> Decimal:
return self.rates.gpu_h100
return Decimal(0)

def generate_invoice_rows(self, report_month) -> List[str]:
def generate_invoice_rows(self, metadata: ReportMetadata) -> List[str]:
rows = []
for su_type, hours in self.su_hours.items():
if hours > 0:
hours = math.ceil(hours)
rate = self.get_rate(su_type)
cost = (rate * hours).quantize(Decimal(".01"), rounding=ROUND_HALF_UP)
row = [
report_month,
metadata.report_month,
metadata.report_start_time.strftime("%Y-%m-%d%H:%M:%SZ"),
metadata.report_end_time.strftime("%Y-%m-%d%H:%M:%SZ"),
self.project,
self.project_id,
self.pi,
self.cluster_name,
self.invoice_email,
self.invoice_address,
self.intitution,
self.institution_specific_code,
"", # pi
metadata.cluster_name,
"", # invoice_email
"", # invoice_address
"", # institution
"", # institution_specific_code
hours,
su_type,
rate,
cost,
metadata.generated_at.strftime("%Y-%m-%d%H:%M:%SZ"),
]
rows.append(row)
return rows
33 changes: 23 additions & 10 deletions openshift_metrics/merge.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import sys
import logging
import argparse
from datetime import datetime, UTC
from datetime import datetime, UTC, timedelta
import json
from typing import Tuple
from decimal import Decimal
Expand Down Expand Up @@ -159,9 +159,7 @@ def main():
if cluster_name is None:
cluster_name = "Unknown Cluster"

logger.info(
f"Generating report from {report_start_date} to {report_end_date} for {cluster_name}"
)
logger.info(f"Total metric files read: {len(files)}")

report_month = datetime.strftime(
datetime.strptime(report_start_date, "%Y-%m-%d"), "%Y-%m"
Expand Down Expand Up @@ -212,8 +210,16 @@ def main():
else:
pod_report_file = f"Pod NERC OpenShift {report_month}.csv"

report_start_date = datetime.strptime(report_start_date, "%Y-%m-%d")
report_end_date = datetime.strptime(report_end_date, "%Y-%m-%d")
report_start_date = datetime.strptime(report_start_date, "%Y-%m-%d").replace(
tzinfo=UTC
)
report_end_date = datetime.strptime(report_end_date, "%Y-%m-%d").replace(
tzinfo=UTC
) + timedelta(days=1)

logger.info(
f"Generating report from {report_start_date} to {report_end_date} for {cluster_name}"
)

if report_start_date.month != report_end_date.month:
logger.warning("The report spans multiple months")
Expand All @@ -224,22 +230,29 @@ def main():
)

su_definitions = get_su_definitions(report_month)

report_metadata = invoice.ReportMetadata(
report_month=report_month,
cluster_name=cluster_name,
report_start_time=report_start_date,
report_end_time=report_end_date,
generated_at=datetime.now(UTC),
)

utils.write_metrics_by_namespace(
condensed_metrics_dict=condensed_metrics_dict,
file_name=invoice_file,
report_month=report_month,
report_metadata=report_metadata,
rates=invoice_rates,
su_definitions=su_definitions,
cluster_name=cluster_name,
ignore_hours=ignore_hours,
)
utils.write_metrics_by_classes(
condensed_metrics_dict=condensed_metrics_dict,
file_name=class_invoice_file,
report_month=report_month,
report_metadata=report_metadata,
rates=invoice_rates,
su_definitions=su_definitions,
cluster_name=cluster_name,
namespaces_with_classes=["rhods-notebooks"],
ignore_hours=ignore_hours,
)
Expand Down
79 changes: 50 additions & 29 deletions openshift_metrics/tests/test_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,15 @@ def test_write_metrics_log(self):


class TestWriteMetricsByNamespace(TestCase):
def setUp(self) -> None:
self.report_metadata = invoice.ReportMetadata(
report_month="2023-01",
cluster_name="test-cluster",
report_start_time=datetime(2023, 1, 1, tzinfo=UTC),
report_end_time=datetime(2023, 1, 3, tzinfo=UTC),
generated_at=datetime(2023, 1, 5, tzinfo=UTC),
)

def test_write_metrics_log(self):
test_metrics_dict = {
"namespace1": {
Expand Down Expand Up @@ -180,21 +189,20 @@ def test_write_metrics_log(self):
}

expected_output = (
"Invoice Month,Project - Allocation,Project - Allocation ID,Manager (PI),Cluster Name,Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost\n"
"2023-01,namespace1,namespace1,,test-cluster,,,,,1128,OpenShift CPU,0.013,14.66\n"
"2023-01,namespace2,namespace2,,test-cluster,,,,,96,OpenShift CPU,0.013,1.25\n"
"2023-01,namespace2,namespace2,,test-cluster,,,,,48,OpenShift GPUA100,1.803,86.54\n"
"2023-01,namespace2,namespace2,,test-cluster,,,,,48,OpenShift GPUA100SXM4,2.078,99.74\n"
"Invoice Month,Report Start Time,Report End Time,Project - Allocation,Project - Allocation ID,Manager (PI),Cluster Name,Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost,Generated At\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace1,namespace1,,test-cluster,,,,,1128,OpenShift CPU,0.013,14.66,2023-01-0500:00:00Z\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace2,namespace2,,test-cluster,,,,,96,OpenShift CPU,0.013,1.25,2023-01-0500:00:00Z\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace2,namespace2,,test-cluster,,,,,48,OpenShift GPUA100,1.803,86.54,2023-01-0500:00:00Z\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace2,namespace2,,test-cluster,,,,,48,OpenShift GPUA100SXM4,2.078,99.74,2023-01-0500:00:00Z\n"
)

with tempfile.NamedTemporaryFile(mode="w+") as tmp:
utils.write_metrics_by_namespace(
condensed_metrics_dict=test_metrics_dict,
file_name=tmp.name,
report_month="2023-01",
report_metadata=self.report_metadata,
rates=RATES,
su_definitions=SU_DEFINITIONS,
cluster_name="test-cluster",
)
self.assertEqual(tmp.read(), expected_output)

Expand Down Expand Up @@ -229,24 +237,32 @@ def test_write_metrics_for_vms(self):
}

expected_output = (
"Invoice Month,Project - Allocation,Project - Allocation ID,Manager (PI),Cluster Name,Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost\n"
"2023-01,namespace1,namespace1,,test-cluster,,,,,24,OpenShift GPUA100SXM4,2.078,49.87\n"
"2023-01,namespace1,namespace1,,test-cluster,,,,,24,OpenShift GPUH100,6.04,144.96\n"
"Invoice Month,Report Start Time,Report End Time,Project - Allocation,Project - Allocation ID,Manager (PI),Cluster Name,Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost,Generated At\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace1,namespace1,,test-cluster,,,,,24,OpenShift GPUA100SXM4,2.078,49.87,2023-01-0500:00:00Z\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace1,namespace1,,test-cluster,,,,,24,OpenShift GPUH100,6.04,144.96,2023-01-0500:00:00Z\n"
)

with tempfile.NamedTemporaryFile(mode="w+") as tmp:
utils.write_metrics_by_namespace(
condensed_metrics_dict=test_metrics_dict,
file_name=tmp.name,
report_month="2023-01",
report_metadata=self.report_metadata,
rates=RATES,
su_definitions=SU_DEFINITIONS,
cluster_name="test-cluster",
)
self.assertEqual(tmp.read(), expected_output)


class TestWriteMetricsByClasses(TestCase):
def setUp(self) -> None:
self.report_metadata = invoice.ReportMetadata(
report_month="2023-01",
cluster_name="test-cluster",
report_start_time=datetime(2023, 1, 1, tzinfo=UTC),
report_end_time=datetime(2023, 1, 3, tzinfo=UTC),
generated_at=datetime(2023, 1, 5, tzinfo=UTC),
)

def test_write_metrics_log(self):
test_metrics_dict = {
"namespace1": { # namespace is ignored entirely from the report
Expand Down Expand Up @@ -321,22 +337,21 @@ def test_write_metrics_log(self):
}

expected_output = (
"Invoice Month,Project - Allocation,Project - Allocation ID,Manager (PI),Cluster Name,Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost\n"
"2023-01,namespace2:noclass,namespace2:noclass,,test-cluster,,,,,96,OpenShift CPU,0.013,1.25\n"
"2023-01,namespace2:math-201,namespace2:math-201,,test-cluster,,,,,96,OpenShift CPU,0.013,1.25\n"
"2023-01,namespace2:math-201,namespace2:math-201,,test-cluster,,,,,24,OpenShift GPUA100,1.803,43.27\n"
"2023-01,namespace2:cs-101,namespace2:cs-101,,test-cluster,,,,,48,OpenShift GPUA100SXM4,2.078,99.74\n"
"Invoice Month,Report Start Time,Report End Time,Project - Allocation,Project - Allocation ID,Manager (PI),Cluster Name,Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost,Generated At\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace2:noclass,namespace2:noclass,,test-cluster,,,,,96,OpenShift CPU,0.013,1.25,2023-01-0500:00:00Z\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace2:math-201,namespace2:math-201,,test-cluster,,,,,96,OpenShift CPU,0.013,1.25,2023-01-0500:00:00Z\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace2:math-201,namespace2:math-201,,test-cluster,,,,,24,OpenShift GPUA100,1.803,43.27,2023-01-0500:00:00Z\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace2:cs-101,namespace2:cs-101,,test-cluster,,,,,48,OpenShift GPUA100SXM4,2.078,99.74,2023-01-0500:00:00Z\n"
)

with tempfile.NamedTemporaryFile(mode="w+") as tmp:
utils.write_metrics_by_classes(
condensed_metrics_dict=test_metrics_dict,
file_name=tmp.name,
report_month="2023-01",
report_metadata=self.report_metadata,
rates=RATES,
su_definitions=SU_DEFINITIONS,
namespaces_with_classes=["namespace2"],
cluster_name="test-cluster",
)
self.assertEqual(tmp.read(), expected_output)

Expand Down Expand Up @@ -370,18 +385,17 @@ def test_write_metrics_by_namespace_decimal(self):
self.assertEqual(cost, 0.45)

expected_output = (
"Invoice Month,Project - Allocation,Project - Allocation ID,Manager (PI),Cluster Name,Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost\n"
"2023-01,namespace1,namespace1,,test-cluster,,,,,35,OpenShift CPU,0.013,0.46\n"
"Invoice Month,Report Start Time,Report End Time,Project - Allocation,Project - Allocation ID,Manager (PI),Cluster Name,Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost,Generated At\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace1,namespace1,,test-cluster,,,,,35,OpenShift CPU,0.013,0.46,2023-01-0500:00:00Z\n"
)

with tempfile.NamedTemporaryFile(mode="w+") as tmp:
utils.write_metrics_by_namespace(
condensed_metrics_dict=test_metrics_dict,
file_name=tmp.name,
report_month="2023-01",
report_metadata=self.report_metadata,
su_definitions=SU_DEFINITIONS,
rates=RATES,
cluster_name="test-cluster",
)
self.assertEqual(tmp.read(), expected_output)

Expand Down Expand Up @@ -447,22 +461,29 @@ def setUp(self):
},
}

self.report_metadata = invoice.ReportMetadata(
report_month="2023-01",
cluster_name="test-cluster",
report_start_time=datetime(2023, 1, 1, tzinfo=UTC),
report_end_time=datetime(2023, 1, 3, tzinfo=UTC),
generated_at=datetime(2023, 1, 5, tzinfo=UTC),
)

def test_write_metrics_by_namespace_with_ignore_hours(self):
expected_output = (
"Invoice Month,Project - Allocation,Project - Allocation ID,Manager (PI),Cluster Name,Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost\n"
"2023-01,namespace1,namespace1,,test-cluster,,,,,12,OpenShift CPU,0.013,0.16\n"
"2023-01,namespace2,namespace2,,test-cluster,,,,,170,OpenShift CPU,0.013,2.21\n"
"2023-01,namespace2,namespace2,,test-cluster,,,,,37,OpenShift GPUA100SXM4,2.078,76.89\n"
"Invoice Month,Report Start Time,Report End Time,Project - Allocation,Project - Allocation ID,Manager (PI),Cluster Name,Invoice Email,Invoice Address,Institution,Institution - Specific Code,SU Hours (GBhr or SUhr),SU Type,Rate,Cost,Generated At\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace1,namespace1,,test-cluster,,,,,12,OpenShift CPU,0.013,0.16,2023-01-0500:00:00Z\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace2,namespace2,,test-cluster,,,,,170,OpenShift CPU,0.013,2.21,2023-01-0500:00:00Z\n"
"2023-01,2023-01-0100:00:00Z,2023-01-0300:00:00Z,namespace2,namespace2,,test-cluster,,,,,37,OpenShift GPUA100SXM4,2.078,76.89,2023-01-0500:00:00Z\n"
)

with tempfile.NamedTemporaryFile(mode="w+") as tmp:
utils.write_metrics_by_namespace(
condensed_metrics_dict=self.test_metrics_dict,
file_name=tmp.name,
report_month="2023-01",
report_metadata=self.report_metadata,
rates=RATES,
su_definitions=SU_DEFINITIONS,
cluster_name="test-cluster",
ignore_hours=self.ignore_times,
)
self.assertEqual(tmp.read(), expected_output)
Expand Down
Loading