Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove all "labels"-related code #782

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
+68 −7,733
Draft

Remove all "labels"-related code #782

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fc29e8f
Remove `label_index` feature flag
Swatinem Oct 14, 2024
c7b8873
Remove most of the "labels" concept
Swatinem Oct 14, 2024
a862064
rip out more of labels
Swatinem Oct 14, 2024
f2b8e07
unregister removed tasks
Swatinem Oct 14, 2024
78671e3
fix tests and more code removal related to partial sessions
Swatinem Oct 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Remove most of the "labels" concept
  • Loading branch information
@Swatinem
Swatinem committed Oct 14, 2024
commit c7b887304025fcfe71c27a129b2ca1e056400797
72 changes: 3 additions & 69 deletions services/report/languages/pycoverage.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import sentry_sdk

from services.report.languages.base import BaseLanguageProcessor
from services.report.report_builder import ReportBuilderSession, SpecialLabelsEnum
from services.report.report_builder import ReportBuilderSession

COVERAGE_HIT = 1
COVERAGE_MISS = 0
@@ -16,8 +16,6 @@ def matches_content(self, content: dict, first_line: str, name: str) -> bool:
def process(
self, content: dict, report_builder_session: ReportBuilderSession
) -> None:
labels_table = LabelsTable(report_builder_session, content)

for filename, file_coverage in content["files"].items():
_file = report_builder_session.create_coverage_file(filename)
if _file is None:
@@ -28,70 +26,6 @@ def process(
] + [(COVERAGE_MISS, ln) for ln in file_coverage["missing_lines"]]
for cov, ln in lines_and_coverage:
if ln > 0:
# label_list_of_lists: list[list[str]] | list[list[int]] = []
label_list_of_lists = [
[labels_table._normalize_label(testname)]
for testname in file_coverage.get("contexts", {}).get(
str(ln), []
)
]
_file.append(
ln,
report_builder_session.create_coverage_line(
cov,
labels_list_of_lists=label_list_of_lists,
),
)
_line = report_builder_session.create_coverage_line(cov)
_file.append(ln, _line)
report_builder_session.append(_file)


class LabelsTable:
def __init__(
self, report_builder_session: ReportBuilderSession, content: dict
) -> None:
self.labels_table: dict[str, str] = {}
self.reverse_table: dict[str, int] = {}
self.are_labels_already_encoded = False

# Compressed pycoverage files will include a labels_table
if "labels_table" in content:
self.labels_table = content["labels_table"]
# We can pre-populate some of the indexes that will be used
for idx, testname in self.labels_table.items():
clean_label = self._normalize_label(testname)
report_builder_session.label_index[int(idx)] = clean_label
self.are_labels_already_encoded = True

def _normalize_label(self, testname: int | float | str) -> str:
if isinstance(testname, int) or isinstance(testname, float):
# This is from a compressed report.
# Pull label from the labels_table
# But the labels_table keys are strings, because of JSON format
testname = self.labels_table[str(testname)]
if testname == "":
return SpecialLabelsEnum.CODECOV_ALL_LABELS_PLACEHOLDER.corresponding_label
return testname.split("|", 1)[0]

def _get_list_of_label_ids(
self,
current_label_idx: dict[int, str],
line_contexts: list[str | int],
) -> list[int]:
if self.are_labels_already_encoded:
# The line contexts already include indexes in the table.
# We can re-use the table and don't have to do anything with contexts.
return sorted(map(int, line_contexts))

# In this case we do need to fix the labels
label_ids_for_line = set()
for testname in line_contexts:
clean_label = self._normalize_label(testname)
if clean_label in self.reverse_table:
label_ids_for_line.add(self.reverse_table[clean_label])
else:
label_id = max([*current_label_idx.keys(), 0]) + 1
current_label_idx[label_id] = clean_label
self.reverse_table[clean_label] = label_id
label_ids_for_line.add(label_id)

return sorted(label_ids_for_line)
124 changes: 1 addition & 123 deletions services/report/report_builder.py
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
import dataclasses
import logging
from enum import Enum
from typing import Any, List, Sequence
from typing import Any, Sequence

from shared.reports.resources import LineSession, Report, ReportFile, ReportLine
from shared.reports.types import CoverageDatapoint
from shared.yaml.user_yaml import UserYaml

from helpers.labels import SpecialLabelsEnum
from services.path_fixer import PathFixer
from services.yaml.reader import read_yaml_field

@@ -36,8 +33,6 @@ def __init__(
self.filepath = report_filepath
self._report_builder = report_builder
self._report = Report()
self.label_index = {}
self._present_labels = set()

@property
def path_fixer(self):
@@ -53,109 +48,11 @@ def get_file(self, filename: str) -> ReportFile | None:
return self._report.get(filename)

def append(self, file: ReportFile):
if file is not None:
for line_number, line in file.lines:
if line.datapoints:
for datapoint in line.datapoints:
if datapoint.label_ids:
for label in datapoint.label_ids:
self._present_labels.add(label)
return self._report.append(file)

def output_report(self) -> Report:
"""
Outputs a Report.
This function applies all the needed modifications before a report
can be output

Returns:
Report: The legacy report desired
"""
if self._present_labels:
if self._present_labels and self._present_labels == {
SpecialLabelsEnum.CODECOV_ALL_LABELS_PLACEHOLDER
}:
log.warning(
"Report only has SpecialLabels. Might indicate it was not generated with contexts"
)
for file in self._report:
for line_number, line in file.lines:
self._possibly_modify_line_to_account_for_special_labels(
file, line_number, line
)
self._report._totals = None
return self._report

def _possibly_modify_line_to_account_for_special_labels(
self, file: ReportFile, line_number: int, line: ReportLine
) -> None:
"""Possibly modify the report line in the file
to account for any label in the SpecialLabelsEnum

Args:
file (ReportFile): The file we want to modify
line_number (int): The line number in case we
need to set the new line back into the files
line (ReportLine): The original line
"""
if not line.datapoints:
return

new_datapoints = [
item
for datapoint in line.datapoints
for item in self._possibly_convert_datapoints(datapoint)
]
if new_datapoints and new_datapoints != line.datapoints:
# A check to avoid unnecessary replacement
file[line_number] = dataclasses.replace(
line,
datapoints=sorted(
new_datapoints,
key=lambda x: (
x.sessionid,
x.coverage,
x.coverage_type,
),
),
)
file._totals = None

# TODO: This can be removed after label indexing is rolled out for all customers
def _possibly_convert_datapoints(
self, datapoint: CoverageDatapoint
) -> List[CoverageDatapoint]:
"""Possibly convert datapoints
The datapoint that might need to be converted

Args:
datapoint (CoverageDatapoint): The datapoint to convert
"""
if datapoint.label_ids and any(
label == SpecialLabelsEnum.CODECOV_ALL_LABELS_PLACEHOLDER
for label in datapoint.label_ids
):
new_label = (
SpecialLabelsEnum.CODECOV_ALL_LABELS_PLACEHOLDER.corresponding_label
)
return [
dataclasses.replace(
datapoint,
label_ids=sorted(
set(
[
label
for label in datapoint.label_ids
if label
!= SpecialLabelsEnum.CODECOV_ALL_LABELS_PLACEHOLDER
]
+ [new_label]
)
),
)
]
return [datapoint]

def create_coverage_file(
self, path: str, do_fix_path: bool = True
) -> ReportFile | None:
@@ -171,30 +68,12 @@ def create_coverage_line(
self,
coverage: int | str,
coverage_type: CoverageType | None = None,
labels_list_of_lists: list[list[str | SpecialLabelsEnum]]
| list[list[int]]
| None = None,
partials=None,
missing_branches=None,
complexity=None,
) -> ReportLine:
sessionid = self._report_builder.sessionid
coverage_type_str = coverage_type.map_to_string() if coverage_type else None
datapoints = (
[
CoverageDatapoint(
sessionid=sessionid,
coverage=coverage,
coverage_type=coverage_type_str,
label_ids=label_ids,
)
# Avoid creating datapoints that don't contain any labels
for label_ids in (labels_list_of_lists or [])
if label_ids
]
if self._report_builder.supports_labels()
else None
)
return ReportLine.create(
coverage=coverage,
type=coverage_type_str,
@@ -209,7 +88,6 @@ def create_coverage_line(
)
)
],
datapoints=datapoints,
complexity=complexity,
)

Loading
Oops, something went wrong.