Skip to content

Commit

Permalink
service: dev: Remove pindeps
Browse files Browse the repository at this point in the history 
Fixes: #1121
  • Loading branch information
@programmer290399
programmer290399 committed Jun 9, 2021
1 parent 51e21dc commit 0cdcb48
Showing 1 changed file with 0 additions and 292 deletions.
292 changes: 0 additions & 292 deletions dffml/service/dev.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -756,303 +756,11 @@ async def run(self):
self.logger.debug("Updated version file %s", version_file)


@configdataclass
class PinDepsConfig:
logs: pathlib.Path = field("Path to CI log zip file")
update: bool = field(
"Update all requirements.txt files with pinned dependencies",
default=False,
)


@dataclasses.dataclass
class PinDepsPlugin:
path: str
setup_cfg_path: pathlib.Path
setup_cfg_pre_deps: str
setup_cfg_post_deps: str
setup_cfg_pinned: str


class PinDeps(CMD):
"""
Parse a GitHub Actions log archive to produce pinned dependencies for each
requirements.txt
"""

CONFIG = PinDepsConfig
# Lowest supported version of Python by all plugins
LOWEST_PYTHON_VERSION = (3, 7)

@classmethod
def requirement_package_name(cls, line):
"""
Parse a line from requirements.txt and return the package name
"Per PEP 508, valid project names must: Consist only of ASCII letters,
digits, underscores (_), hyphens (-), and/or periods (.), and. Start & end
with an ASCII letter or digit."
"""
if not line:
raise ValueError("line is blank")

i = 0
name = ""
while i < len(line) and line[i] in (
"_",
"-",
".",
*string.ascii_letters,
*string.digits,
):
name += line[i]
i += 1

# Replace _ with - since that's what PyPi uses
return name.replace("_", "-")

@classmethod
def remove_ansi_escape(cls, contents):
"""
Found on Stack Overflow from Martijn Pieters, only modified input
variable name.
- https://stackoverflow.com/a/14693789
- https://creativecommons.org/licenses/by-sa/4.0/
"""
# 7-bit and 8-bit C1 ANSI sequences
ansi_escape_8bit = re.compile(
br"(?:\x1B[@-Z\\-_]|[\x80-\x9A\x9C-\x9F]|(?:\x1B\[|\x9B)[0-?]*[ -/]*[@-~])"
)
return ansi_escape_8bit.sub(b"", contents)

@classmethod
def parse_github_actions_log_file(cls, contents):
lines = (
cls.remove_ansi_escape(contents)
.replace(b"\r", b"")
.decode(errors="ignore")
.split("\n")
)

# Map packages to versions that were installed in CI
ci_installed = {}

for line in lines:
if "Requirement already satisfied:" in line:
# Check for packages that may have been installed previously that would show
# up as file:// URLs
# Example:
# Requirement already satisfied: vowpalwabbit>=8.8.1 in /usr/share/miniconda/lib/python3.7/site-packages (from dffml-model-vowpalWabbit==0.0.1) (8.8.1)
line = line.split()
# Get package name
package_name = cls.requirement_package_name(
line[line.index("satisfied:") + 1]
)
if not package_name:
continue
# Check if we don't have a version for this package yet
if package_name not in ci_installed:
# Get package version, strip ()
ci_installed[package_name] = line[-1][1:-1]
elif "==" in line:
# Skip any lines that are not in the format of:
# 2020-11-23T05:54:36.4861610Z absl-py==0.11.0
if line.count("Z ") != 1:
continue
line = line.split("Z ")[-1].split("==")
if not line[0] or not line[1]:
continue
# Get package name
package_name = cls.requirement_package_name(line[0])
if not package_name:
continue
# with contextlib.suppress(Exception):
ci_installed[package_name] = line[1]

# Add all the core plugins to the list of installed packages
for (
plugin_name,
plugin_directory,
) in PACKAGE_NAMES_TO_DIRECTORY.items():
ci_installed[plugin_name] = parse_version(
str(
REPO_ROOT.joinpath(
*plugin_directory,
plugin_name.replace("-", "_"),
"version.py",
)
)
)
print(
parse_version(
str(
pathlib.Path(
*plugin_directory,
plugin_name.replace("-", "_"),
"version.py",
)
)
)
)

return ci_installed

FIND_CI_INSTALLED_LINUX_PLUGIN_VERSION_REGEX = re.compile(
r"\(([a-z./]+), ([0-9.]+)\)"
)
FIND_CI_INSTALLED_NON_LINUX_VERSION_REGEX = re.compile(r"\(([0-9.]+)\)")

@classmethod
def find_ci_installed(cls, logs_dir: pathlib.Path):
ci_installed = {}

for log_file_path in logs_dir.rglob("*.txt"):
if "windows" in log_file_path.stem:
platform_system = "Windows"
elif "macos" in log_file_path.stem:
platform_system = "Darwin"
else:
platform_system = "Linux"

match = cls.FIND_CI_INSTALLED_LINUX_PLUGIN_VERSION_REGEX.search(
log_file_path.stem
)
if not match:
continue

plugin, python_version = match.groups()
# Only look for the pip freeze on the main plugin
if plugin != ".":
continue

contents = log_file_path.read_bytes()
if not b"pip freeze" in contents:
continue

if platform_system in ("Windows", "Darwin"):
python_version = cls.FIND_CI_INSTALLED_NON_LINUX_VERSION_REGEX.search(
log_file_path.stem
).groups()[
0
]

ci_installed[
(platform_system, python_version)
] = cls.parse_github_actions_log_file(contents)

return ci_installed

async def pin_deps(self, logs_dir: pathlib.Path):
root = pathlib.Path(__file__).parents[2]

ci_installed = self.find_ci_installed(logs_dir)

# for setup_cfg_path in root.rglob("setup.cfg"):
for package_directory in PACKAGE_DIRECTORY_TO_NAME.keys():
setup_cfg_path = root.joinpath(*package_directory, "setup.cfg")
setup_cfg_contents = setup_cfg_path.read_text()
# Pull the dependencies out of the install_requires list
setup_cfg_lines = setup_cfg_contents.split("\n")
setup_cfg_lines = [line.rstrip() for line in setup_cfg_lines]
dependencies = setup_cfg_lines[
setup_cfg_lines.index("install_requires =") + 1 :
]
# Find the last depedency in the list by looking for when the
# indentation level goes back to normal, meaning no indent.
for i, line in enumerate(dependencies):
current_indent = len(line) - len(line.lstrip())
if current_indent == 0:
break
# The contents of the file up until the point where the
# install_requires list starts
setup_cfg_pre_deps = setup_cfg_lines[
: setup_cfg_lines.index("install_requires =") + 1
]
# The contents of the file immediatly after the install_requires
# list. We are going to replace what's between these two.
setup_cfg_post_deps = setup_cfg_lines[
setup_cfg_lines.index("install_requires =") + 1 + i :
]
# The dependency list ends when the indent level changes
dependencies = dependencies[:i]
dependencies = [line.strip() for line in dependencies]
# Remove comments
# TODO Remove?: dependencies = [line for line in dependencies if not line.startswith("#")]
# Add all packages in setup.cfg files to set of required packages
for i, line in enumerate(dependencies):
# Skip comments
if line.startswith("#") or not line:
continue
package_name = self.requirement_package_name(line)
if not package_name:
continue
# Ensure package was installed in CI
for supported_os, supported_python in [
("Linux", "3.7"),
("Linux", "3.8"),
]:
if (supported_os, supported_python) not in ci_installed:
raise ValueError(
f"Supported OS/Python version {supported_os} {supported_python}: not in {ci_installed.keys()}"
)
if (
package_name
not in ci_installed[(supported_os, supported_python)]
):
raise ValueError(
f"Plugin {setup_cfg_path.parent}: {package_name!r} not in {ci_installed[(supported_os, supported_python)]}"
)
# Modify the line to be a pinned package
line_contents = []
for (
(platform_system, python_version),
packages_installed,
) in ci_installed.items():
if package_name not in packages_installed:
continue
package_version_installed = packages_installed[
package_name
]
line_contents.append(
f'{package_name}=={package_version_installed}; platform_system == "{platform_system}" and python_version == "{python_version}"',
)
dependencies[i] = "\t".join(line_contents)

self.logger.debug(f"{line:<40} | {dependencies[i]}")

yield PinDepsPlugin(
path=str(setup_cfg_path.relative_to(root)),
setup_cfg_path=setup_cfg_path,
setup_cfg_pre_deps=setup_cfg_pre_deps,
setup_cfg_post_deps=setup_cfg_post_deps,
setup_cfg_pinned=[
(" " * 4) + line.replace("\t", "\n" + (" " * 4))
for line in dependencies
],
)

async def run(self):
async for plugin in self.pin_deps(pathlib.Path(self.logs)):
yield plugin
if self.update:
plugin.setup_cfg_path.write_text(
"\n".join(
itertools.chain(
plugin.setup_cfg_pre_deps,
plugin.setup_cfg_pinned,
plugin.setup_cfg_post_deps,
)
)
)


class CI(CMD):
"""
CI related commands
"""

pindeps = PinDeps


class Bump(CMD):
"""
Expand Down

0 comments on commit 0cdcb48

Please sign in to comment.