From ace32d67d56f66ca8db2ab5854153431e3f65328 Mon Sep 17 00:00:00 2001 From: Jack McCluskey <34928439+jrmccluskey@users.noreply.github.com> Date: Tue, 6 Jun 2023 13:22:09 -0400 Subject: [PATCH] Remove Beam Dependency Report code (#26954) * Remove Beam Dependency Report code * Remove reference in README * Remove other references to the dependency job * More references --- .test-infra/jenkins/README.md | 1 - .../jenkins/dependency_check/__init__.py | 16 - .../dependency_check/bigquery_client_utils.py | 149 -------- .../dependency_check_report_generator.py | 350 ------------------ .../dependency_check_report_generator_test.py | 134 ------- .../dependency_check/generate_report.sh | 64 ---- .../report_generator_config.py | 81 ---- .../dependency_check/version_comparer.py | 51 --- .../dependency_check/version_comparer_test.py | 54 --- .../jenkins/job_Dependency_Check.groovy | 73 ---- build.gradle.kts | 10 - ownership/JAVA_DEPENDENCY_OWNERS.yaml | 5 - sdks/python/build.gradle | 5 - sdks/python/scripts/run_dependency_check.sh | 34 -- .../test-suites/tox/pycommon/build.gradle | 4 +- sdks/python/tox.ini | 10 - 16 files changed, 1 insertion(+), 1040 deletions(-) delete mode 100644 .test-infra/jenkins/dependency_check/__init__.py delete mode 100644 .test-infra/jenkins/dependency_check/bigquery_client_utils.py delete mode 100644 .test-infra/jenkins/dependency_check/dependency_check_report_generator.py delete mode 100644 .test-infra/jenkins/dependency_check/dependency_check_report_generator_test.py delete mode 100755 .test-infra/jenkins/dependency_check/generate_report.sh delete mode 100644 .test-infra/jenkins/dependency_check/report_generator_config.py delete mode 100644 .test-infra/jenkins/dependency_check/version_comparer.py delete mode 100644 .test-infra/jenkins/dependency_check/version_comparer_test.py delete mode 100644 .test-infra/jenkins/job_Dependency_Check.groovy delete mode 100755 sdks/python/scripts/run_dependency_check.sh diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index bdd3f2577ae1c..5f9b55d4e2366 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -285,7 +285,6 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | Name | Link | PR Trigger Phrase | Cron Status | |------|------|-------------------|-------------| -| beam_Dependency_Check | [cron](https://ci-beam.apache.org/job/beam_Dependency_Check/) | `Run Dependency Check` | [![Build Status](https://ci-beam.apache.org/job/beam_Dependency_Check/badge/icon)](https://ci-beam.apache.org/job/beam_Dependency_Check) | | beam_Metrics_Report | [cron](https://ci-beam.apache.org/job/beam_Metrics_Report/) | `Run Metrics Report` | [![Build Status](https://ci-beam.apache.org/job/beam_Metrics_Report/badge/icon)](https://ci-beam.apache.org/job/beam_Metrics_Report) | | beam_Release_NightlySnapshot | [cron](https://ci-beam.apache.org/job/beam_Release_NightlySnapshot/) | `Run Gradle Publish` | [![Build Status](https://ci-beam.apache.org/job/beam_Release_NightlySnapshot/badge/icon)](https://ci-beam.apache.org/job/beam_Release_NightlySnapshot) | | beam_Release_Python_NightlySnapshot | [cron](https://ci-beam.apache.org/job/beam_Release_Python_NightlySnapshot/) | `Run Python Publish` | [![Build Status](https://ci-beam.apache.org/job/beam_Release_Python_NightlySnapshot/badge/icon)](https://ci-beam.apache.org/job/beam_Release_Python_NightlySnapshot) | diff --git a/.test-infra/jenkins/dependency_check/__init__.py b/.test-infra/jenkins/dependency_check/__init__.py deleted file mode 100644 index cce3acad34a49..0000000000000 --- a/.test-infra/jenkins/dependency_check/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/.test-infra/jenkins/dependency_check/bigquery_client_utils.py b/.test-infra/jenkins/dependency_check/bigquery_client_utils.py deleted file mode 100644 index f7cd0fe24b9e4..0000000000000 --- a/.test-infra/jenkins/dependency_check/bigquery_client_utils.py +++ /dev/null @@ -1,149 +0,0 @@ -from __future__ import print_function -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import datetime -import logging -from google.cloud import bigquery - -logging.getLogger().setLevel(logging.INFO) - -class BigQueryClientUtils: - - def __init__(self, project_id, dataset_id, table_id): - self.project_id = project_id - self.dataset_id = dataset_id - self.table_id = table_id - self.bigquery_client = bigquery.Client(project_id) - self.table_ref = self.bigquery_client.dataset(dataset_id).table(table_id) - self.table = self.bigquery_client.get_table(self.table_ref) - - - def query_dep_info_by_version(self, dep, version): - """ - Query for dependency information of a specific version - Args: - dep, version - Return: - release_date, is_currently_used - """ - query = """SELECT release_date, is_currently_used - FROM `{0}.{1}.{2}` - WHERE package_name=\'{3}\' AND version=\'{4}\'""".format(self.project_id, - self.dataset_id, - self.table_id, - dep.strip(), - version.strip()) - - query_job = self.bigquery_client.query(query) - rows = list(query_job) - if len(rows) == 0: - logging.info("Did not find record of dependency {0} with version {1}.".format(dep, version)) - return None, False - assert len(rows) == 1 - logging.info("""Found record of dependency {0} with version {1}: - release date: {2}; is_currently_used: {3}.""".format(dep, version, rows[0]['release_date'], rows[0]['is_currently_used'])) - return rows[0]['release_date'], rows[0]['is_currently_used'] - - - def query_currently_used_dep_info_in_db(self, dep): - """ - Query for the info of the currently used version of a specific dependency - Args: - dep - Return: - version, release_date - """ - query = """SELECT version, release_date - FROM `{0}.{1}.{2}` - WHERE package_name=\'{3}\' AND is_currently_used=True""".format(self.project_id, - self.dataset_id, - self.table_id, - dep.strip()) - - query_job = self.bigquery_client.query(query) - rows = list(query_job) - if len(rows) == 0: - return None, None - assert len(rows) == 1 - return rows[0]['version'], rows[0]['release_date'] - - - def insert_dep_to_table(self, dep, version, release_date, is_currently_used=False): - """ - Add a dependency with version and release date into bigquery table - Args: - dep, version, is_currently_used (default False) - """ - query = """INSERT - `{0}.{1}.{2}` (package_name, version, release_date, is_currently_used) - VALUES (\'{3}\', \'{4}\', \'{5}\', {6})""".format(self.project_id, - self.dataset_id, - self.table_id, - dep.strip(), - version.strip(), - release_date, - is_currently_used) - logging.info("Inserting dep to table: \n {0}".format(query)) - try: - query_job = self.bigquery_client.query(query) - if not query_job.done(): - print(query_job.result()) - except: - raise - - - def delete_dep_from_table(self, dep, version): - """ - Remove a dependency record from the table. - Args: - dep, version - """ - query = """DELETE - FROM `{0}.{1}.{2}` - WHERE package_name=\'{3}\' AND version=\'{4}\'""".format(self.project_id, - self.dataset_id, - self.table_id, - dep.strip(), - version.strip()) - logging.info("Deleting dep from table: \n {0}".format(query)) - try: - query_job = self.bigquery_client.query(query) - if not query_job.done(): - print(query_job.result()) - except: - raise - - - def clean_stale_records_from_table(self): - """ - Remove stale records from the table. A record is stale if it is not currently used and the release date is behind 3 - years. - """ - query = """DELETE - FROM `{0}.{1}.{2}` - WHERE release_date < '{3}'""".format(self.project_id, - self.dataset_id, - self.table_id, - datetime.datetime.today().date() - datetime.timedelta(3*365)) - logging.info("Clean Up Starts") - try: - query_job = self.bigquery_client.query(query) - if not query_job.done(): - logging.error(query_job.result()) - except: - raise diff --git a/.test-infra/jenkins/dependency_check/dependency_check_report_generator.py b/.test-infra/jenkins/dependency_check/dependency_check_report_generator.py deleted file mode 100644 index a7f6035e4e2bc..0000000000000 --- a/.test-infra/jenkins/dependency_check/dependency_check_report_generator.py +++ /dev/null @@ -1,350 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import dependency_check.version_comparer as version_comparer -import logging -import os.path -import re -import requests -import sys -import traceback - -from datetime import datetime -from dependency_check.bigquery_client_utils import BigQueryClientUtils -from dependency_check.report_generator_config import ReportGeneratorConfig -from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry - - -logging.getLogger().setLevel(logging.INFO) - -class InvalidFormatError(Exception): - def __init__(self, message): - super(InvalidFormatError, self).__init__(message) - - -def extract_results(file_path): - """ - Extract the Java/Python dependency reports and return a collection of out-of-date dependencies. - Args: - file_path: the path of the raw reports - Return: - outdated_deps: a collection of dependencies that has updates - """ - outdated_deps = [] - try: - with open(file_path) as raw_report: - see_oudated_deps = False - for line in raw_report: - if see_oudated_deps: - outdated_deps.append(line) - if line.startswith('The following dependencies have later '): - see_oudated_deps = True - raw_report.close() - return outdated_deps - except: - raise - - -def extract_single_dep(dep): - """ - Extract a single dependency check record from Java and Python reports. - Args: - dep: e.g " - org.assertj:assertj-core [2.5.0 -> 3.10.0]". - Return: - dependency name, current version, latest version. - """ - pattern = " - ([\s\S]*)\[([\s\S]*) -> ([\s\S]*)\]" - match = re.match(pattern, dep) - if match is None: - raise InvalidFormatError("Failed to extract the dependency information: {}".format(dep)) - return match.group(1).strip(), match.group(2).strip(), match.group(3).strip() - - -def prioritize_dependencies(deps, sdk_type): - """ - Extracts and analyze dependency versions and release dates. - Returns a collection of dependencies which is "high priority" in html format: - 1. dependency has major release. e.g org.assertj:assertj-core [2.5.0 -> 3.10.0] - 2. dependency is 3 sub-versions behind the newest one. e.g org.tukaani:xz [1.5 -> 1.8] - 3. dependency has not been updated for more than 6 months. - - Args: - deps: A collection of outdated dependencies. - Return: - high_priority_deps: A collection of dependencies which need to be taken care of before next release. - """ - - project_id = ReportGeneratorConfig.GCLOUD_PROJECT_ID - dataset_id = ReportGeneratorConfig.DATASET_ID - table_id = ReportGeneratorConfig.get_bigquery_table_id(sdk_type) - high_priority_deps = [] - bigquery_client = BigQueryClientUtils(project_id, dataset_id, table_id) - - for dep in deps: - try: - if re.match(r'https?://', dep.lstrip()): - # Gradle-version-plugin's output contains URLs of the libraries - continue - logging.info("\n\nStart processing: " + dep) - dep_name, curr_ver, latest_ver = extract_single_dep(dep) - curr_release_date = None - latest_release_date = None - group_id = None - - if sdk_type == 'Java': - # extract the groupid and artifactid - group_id, artifact_id = dep_name.split(":") - dep_details_url = "{0}/{1}/{2}".format(ReportGeneratorConfig.MAVEN_CENTRAL_URL, group_id, artifact_id) - curr_release_date = find_release_time_from_maven_central(group_id, artifact_id, curr_ver) - latest_release_date = find_release_time_from_maven_central(group_id, artifact_id, latest_ver) - else: - dep_details_url = ReportGeneratorConfig.PYPI_URL + dep_name - curr_release_date = find_release_time_from_python_compatibility_checking_service(dep_name, curr_ver) - latest_release_date = find_release_time_from_python_compatibility_checking_service(dep_name, latest_ver) - - if not curr_release_date or not latest_release_date: - curr_release_date, latest_release_date = query_dependency_release_dates_from_bigquery(bigquery_client, - dep_name, - curr_ver, - latest_ver) - dep_info = """ - {1} - {2} - {3} - {4} - {5}""".format(dep_details_url, - dep_name, - curr_ver, - latest_ver, - curr_release_date, - latest_release_date) - if (version_comparer.compare_dependency_versions(curr_ver, latest_ver) or - compare_dependency_release_dates(curr_release_date, latest_release_date)): - high_priority_deps.append(dep_info) - - except: - traceback.print_exc() - continue - - bigquery_client.clean_stale_records_from_table() - return high_priority_deps - - -def find_release_time_from_maven_central(group_id, artifact_id, version): - """ - Find release dates from Maven Central REST API. - Args: - group_id: - artifact_id: - version: - Return: - release date - """ - url = "http://search.maven.org/solrsearch/select?q=g:{0}+AND+a:{1}+AND+v:{2}".format( - group_id, - artifact_id, - version - ) - logging.info('Finding release date of {0}:{1} {2} from the Maven Central'.format( - group_id, - artifact_id, - version - )) - try: - response = request_session_with_retries().get(url) - if not response.ok: - logging.error("""Failed finding the release date of {0}:{1} {2}. - The response status code is not ok: {3}""".format(group_id, - artifact_id, - version, - str(response.status_code))) - return None - response_data = response.json() - release_timestamp_mills = response_data['response']['docs'][0]['timestamp'] - release_date = datetime.fromtimestamp(release_timestamp_mills/1000).date() - return release_date - except Exception as e: - logging.error("Errors while extracting the release date: " + str(e)) - return None - - -def find_release_time_from_python_compatibility_checking_service(dep_name, version): - """ - Query release dates by using Python compatibility checking service. - Args: - dep_name: - version: - Return: - release date - """ - url = 'http://104.197.8.72/?package={0}=={1}&python-version=2'.format( - dep_name, - version - ) - logging.info('Finding release time of {0} {1} from the python compatibility checking service.'.format( - dep_name, - version - )) - try: - response = request_session_with_retries().get(url) - if not response.ok: - logging.error("""Failed finding the release date of {0} {1}. - The response status code is not ok: {2}""".format(dep_name, - version, - str(response.status_code))) - return None - response_data = response.json() - release_datetime = response_data['dependency_info'][dep_name]['installed_version_time'] - release_date = datetime.strptime(release_datetime, '%Y-%m-%dT%H:%M:%S').date() - return release_date - except Exception as e: - logging.error("Errors while extracting the release date: " + str(e)) - return None - - -def request_session_with_retries(): - """ - Create an http session with retries - """ - session = requests.Session() - retries = Retry(total=3) - session.mount('http://', HTTPAdapter(max_retries=retries)) - session.mount('https://', HTTPAdapter(max_retries=retries)) - return session - - -def query_dependency_release_dates_from_bigquery(bigquery_client, dep_name, curr_ver_in_beam, latest_ver): - """ - Query release dates of current version and the latest version from BQ tables. - Args: - bigquery_client: a bq client object that bundle configurations for API requests - dep_name: dependency name - curr_ver_in_beam: the current version used in beam - latest_ver: the later version - Return: - A tuple that contains `curr_release_date` and `latest_release_date`. - """ - try: - curr_release_date, is_currently_used_bool = bigquery_client.query_dep_info_by_version(dep_name, curr_ver_in_beam) - latest_release_date, _ = bigquery_client.query_dep_info_by_version(dep_name, latest_ver) - date_today = datetime.today().date() - - # sync to the bigquery table on the dependency status of the currently used version. - if not is_currently_used_bool: - currently_used_version_in_db, currently_used_release_date_in_db = bigquery_client.query_currently_used_dep_info_in_db(dep_name) - if currently_used_version_in_db is not None: - bigquery_client.delete_dep_from_table(dep_name, currently_used_version_in_db) - bigquery_client.insert_dep_to_table(dep_name, currently_used_version_in_db, currently_used_release_date_in_db, is_currently_used=False) - if curr_release_date is None: - bigquery_client.insert_dep_to_table(dep_name, curr_ver_in_beam, date_today, is_currently_used=True) - else: - bigquery_client.delete_dep_from_table(dep_name, curr_ver_in_beam) - bigquery_client.insert_dep_to_table(dep_name, curr_ver_in_beam, curr_release_date, is_currently_used=True) - # sync to the bigquery table on the dependency status of the latest version. - if latest_release_date is None: - bigquery_client.insert_dep_to_table(dep_name, latest_ver, date_today, is_currently_used=False) - latest_release_date = date_today - except Exception: - raise - return curr_release_date, latest_release_date - - -def compare_dependency_release_dates(curr_release_date, latest_release_date): - """ - Compare release dates of current using version and the latest version. - Return true if the current version is behind over 60 days. - Args: - curr_release_date - latest_release_date - Return: - boolean - """ - if not curr_release_date or not latest_release_date: - return False - else: - if (latest_release_date - curr_release_date).days >= ReportGeneratorConfig.MAX_STALE_DAYS: - return True - return False - - -def generate_report(sdk_type): - """ - Write SDK dependency check results into a html report. - Args: - sdk_type: String [Java, Python, TODO: Go] - """ - report_name = ReportGeneratorConfig.FINAL_REPORT - raw_report = ReportGeneratorConfig.get_raw_report(sdk_type) - - if os.path.exists(report_name): - append_write = 'a' - else: - append_write = 'w' - - try: - # Extract dependency check results from build/dependencyUpdate - report = open(report_name, append_write) - if os.path.isfile(raw_report): - outdated_deps = extract_results(raw_report) - else: - report.write("Did not find the raw report of dependency check: {}".format(raw_report)) - report.close() - return - - # Prioritize dependencies by comparing versions and release dates. - high_priority_deps = prioritize_dependencies(outdated_deps, sdk_type) - - # Write results to a report - subtitle = "

High Priority Dependency Updates Of Beam {} SDK:

\n".format(sdk_type) - table_fields = """ - {0} - {1} - {2} - {3} - {4} - """.format("Dependency Name", - "Current Version", - "Latest Version", - "Release Date Of the Current Used Version", - "Release Date Of The Latest Release") - report.write(subtitle) - report.write("\n") - report.write(table_fields) - for dep in high_priority_deps: - report.write("%s" % dep) - report.write("
\n") - except Exception as e: - traceback.print_exc() - logging.error("Failed generate the dependency report. " + str(e)) - report.write('

{0}

'.format(str(e))) - - report.close() - logging.info("Dependency check on {0} SDK complete. The report is created.".format(sdk_type)) - -def main(args): - """ - Main method. - Args: - args[0]: type of the check [Java, Python] - """ - generate_report(args[0]) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/.test-infra/jenkins/dependency_check/dependency_check_report_generator_test.py b/.test-infra/jenkins/dependency_check/dependency_check_report_generator_test.py deleted file mode 100644 index b7b2ec092a4c2..0000000000000 --- a/.test-infra/jenkins/dependency_check/dependency_check_report_generator_test.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -# -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This script performs testing of scenarios from verify_performance_test_results.py -# - -from __future__ import print_function -import unittest -from mock import patch, mock_open -from datetime import datetime -from .dependency_check_report_generator import prioritize_dependencies - - -_PROJECT_ID = 'mock-apache-beam-testing' -_DATASET_ID = 'mock-beam_dependency_states' -_TABLE_ID = 'mock-java_dependency_states' -_SDK_TYPE = 'Java' - -# initialize current/latest version release dates for low-priority (LP) and high-priority (HP) dependencies -_LP_CURR_VERSION_DATE = datetime.strptime('2000-01-01', '%Y-%m-%d') -_LATEST_VERSION_DATE = datetime.strptime('2000-01-02', '%Y-%m-%d') -_HP_CURR_VERSION_DATE = datetime.strptime('1999-01-01', '%Y-%m-%d') -_MOCKED_OWNERS_FILE = "deps: " - - -@patch('google.cloud.bigquery.Client') -@patch('dependency_check.bigquery_client_utils.BigQueryClientUtils.clean_stale_records_from_table') -class DependencyCheckReportGeneratorTest(unittest.TestCase): - """Tests for `dependency_check_report_generator.py`.""" - - def setUp(self): - print("\n\nTest : " + self._testMethodName) - - - @patch('dependency_check.bigquery_client_utils.BigQueryClientUtils') - def test_empty_dep_input(self, *args): - """ - Test on empty outdated dependencies. - Expect: empty report - """ - with patch('builtins.open', mock_open(read_data=_MOCKED_OWNERS_FILE)): - report = prioritize_dependencies([], _SDK_TYPE) - self.assertEqual(len(report), 0) - - - @patch('dependency_check.dependency_check_report_generator.find_release_time_from_maven_central', - side_effect = [_LP_CURR_VERSION_DATE, _LATEST_VERSION_DATE, - _LP_CURR_VERSION_DATE, _LATEST_VERSION_DATE, - _HP_CURR_VERSION_DATE, _LATEST_VERSION_DATE, - _LP_CURR_VERSION_DATE, _LATEST_VERSION_DATE,]) - def test_normal_dep_input(self, *args): - """ - Test on a normal outdated dependencies set. - Expect: group1:artifact1, group2:artifact2, and group3:artifact3 - """ - deps = [ - " - group1:artifact1 [1.0.0 -> 3.0.0]", - " - group2:artifact2 [1.0.0 -> 1.3.0]", - " - group3:artifact3 [1.0.0 -> 1.1.0]", - " - group4:artifact4 [1.0.0 -> 1.1.0]" - ] - with patch('builtins.open', mock_open(read_data=_MOCKED_OWNERS_FILE)): - report = prioritize_dependencies(deps, _SDK_TYPE) - self.assertEqual(len(report), 3) - self.assertIn('group1:artifact1', report[0]) - self.assertIn('group2:artifact2', report[1]) - self.assertIn('group3:artifact3', report[2]) - - - @patch('dependency_check.dependency_check_report_generator.find_release_time_from_maven_central', - side_effect = [_LP_CURR_VERSION_DATE, - _LATEST_VERSION_DATE,]) - def test_dep_with_nondigit_major_versions(self, *args): - """ - Test on a outdated dependency with non-digit major number. - Expect: group1:artifact1 - """ - deps = [" - group1:artifact1 [Release1-123 -> Release2-456]"] - with patch('builtins.open', mock_open(read_data=_MOCKED_OWNERS_FILE)): - report = prioritize_dependencies(deps, _SDK_TYPE) - self.assertEqual(len(report), 1) - self.assertIn('group1:artifact1', report[0]) - - - @patch('dependency_check.dependency_check_report_generator.find_release_time_from_maven_central', - side_effect = [_LP_CURR_VERSION_DATE, - _LATEST_VERSION_DATE,]) - def test_dep_with_nondigit_minor_versions(self, *args): - """ - Test on a outdated dependency with non-digit minor number. - Expect: group1:artifact1 - """ - deps = [" - group1:artifact1 [0.rc1.0 -> 0.rc2.0]"] - with patch('builtins.open', mock_open(read_data=_MOCKED_OWNERS_FILE)): - report = prioritize_dependencies(deps, _SDK_TYPE) - self.assertEqual(len(report), 1) - self.assertIn('group1:artifact1', report[0]) - - - @patch('dependency_check.dependency_check_report_generator.find_release_time_from_maven_central', - side_effect = [_HP_CURR_VERSION_DATE,_LATEST_VERSION_DATE,]) - def test_invalid_dep_input(self, *args): - """ - Test on a invalid outdated dependencies format. - Expect: Exception through out. And group2:artifact2 is picked. - """ - deps = [ - "- group1:artifact1 (1.0.0, 2.0.0)", - " - group2:artifact2 [1.0.0 -> 2.0.0]" - ] - with patch('builtins.open', mock_open(read_data=_MOCKED_OWNERS_FILE)): - report = prioritize_dependencies(deps, _SDK_TYPE) - self.assertEqual(len(report), 1) - self.assertIn('group2:artifact2', report[0]) - - -if __name__ == '__main__': - unittest.main() - diff --git a/.test-infra/jenkins/dependency_check/generate_report.sh b/.test-infra/jenkins/dependency_check/generate_report.sh deleted file mode 100755 index cfb66ec951995..0000000000000 --- a/.test-infra/jenkins/dependency_check/generate_report.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This script will be run by Jenkins as a Python dependency test. - -set -e -set -v - -# Get currently used Python version from args or assume a default. -PYTHON=${1:-python3} - -REPORT_DESCRIPTION=" -

A dependency update is high priority if it satisfies one of following criteria:

- - - -

In Beam, we make a best-effort attempt at keeping all dependencies up-to-date. - In the future, issues will be filed and tracked for these automatically, - but in the meantime you can search for existing issues or open a new one. -

-

For more information: Beam Dependency Guide

" - - -# Virtualenv for the rest of the script to run setup -$PYTHON -m venv dependency/check - -. ./dependency/check/bin/activate -pip install --upgrade pip setuptools wheel -pip install --upgrade google-cloud-bigquery google-cloud-bigtable google-cloud-core -rm -f build/dependencyUpdates/beam-dependency-check-report.txt - -# Install packages and run the unit tests of the report generator -pip install mock pyyaml -cd $WORKSPACE/src/.test-infra/jenkins -$PYTHON -m dependency_check.dependency_check_report_generator_test -$PYTHON -m dependency_check.version_comparer_test - -echo "" > $WORKSPACE/src/build/dependencyUpdates/beam-dependency-check-report.html - -$PYTHON -m dependency_check.dependency_check_report_generator Python - -$PYTHON -m dependency_check.dependency_check_report_generator Java - -echo "$REPORT_DESCRIPTION " >> $WORKSPACE/src/build/dependencyUpdates/beam-dependency-check-report.html diff --git a/.test-infra/jenkins/dependency_check/report_generator_config.py b/.test-infra/jenkins/dependency_check/report_generator_config.py deleted file mode 100644 index 18396ee43f8fd..0000000000000 --- a/.test-infra/jenkins/dependency_check/report_generator_config.py +++ /dev/null @@ -1,81 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Defines constants and helper methods used by the dependency_report_generator - -import os - -class ReportGeneratorConfig: - - # Jenkins Working Space - WORKING_SPACE = os.environ['WORKSPACE'] - - # Constants for dependency prioritization - GCLOUD_PROJECT_ID = 'apache-beam-testing' - DATASET_ID = 'beam_dependency_states' - PYTHON_DEP_TABLE_ID = 'python_dependency_states' - JAVA_DEP_TABLE_ID = 'java_dependency_states' - PYTHON_DEP_RAW_REPORT = WORKING_SPACE + '/src/build/dependencyUpdates/python_dependency_report.txt' - JAVA_DEP_RAW_REPORT = WORKING_SPACE + '/src/build/dependencyUpdates/report.txt' - FINAL_REPORT = WORKING_SPACE + '/src/build/dependencyUpdates/beam-dependency-check-report.html' - MAX_STALE_DAYS = 360 - MAX_MINOR_VERSION_DIFF = 3 - PYPI_URL = "https://pypi.org/project/" - MAVEN_CENTRAL_URL = "https://mvnrepository.com/artifact" - - # Dependency Owners - JAVA_DEP_OWNERS = WORKING_SPACE + '/src/ownership/JAVA_DEPENDENCY_OWNERS.yaml' - PYTHON_DEP_OWNERS = WORKING_SPACE + '/src/ownership/PYTHON_DEPENDENCY_OWNERS.yaml' - - - @classmethod - def get_bigquery_table_id(cls, sdk_type): - if sdk_type.lower() == 'java': - return cls.JAVA_DEP_TABLE_ID - elif sdk_type.lower() == 'python': - return cls.PYTHON_DEP_TABLE_ID - else: - raise UndefinedSDKTypeException("""Undefined SDK Type: {0}. - Could not find the BigQuery table for {1} dependencies.""".format(sdk_type, sdk_type)) - - - @classmethod - def get_raw_report(cls, sdk_type): - if sdk_type.lower() == 'java': - return cls.JAVA_DEP_RAW_REPORT - elif sdk_type.lower() == 'python': - return cls.PYTHON_DEP_RAW_REPORT - else: - raise UndefinedSDKTypeException("""Undefined SDK Type: {0}. - Could not find the dependency reports for the {1} SDK.""".format(sdk_type, sdk_type)) - - - @classmethod - def get_owners_file(cls, sdk_type): - if sdk_type.lower() == 'java': - return cls.JAVA_DEP_OWNERS - elif sdk_type.lower() == 'python': - return cls.PYTHON_DEP_OWNERS - else: - raise UndefinedSDKTypeException("""Undefined SDK Type: {0}. - Could not find the Owners file for the {1} SDK.""".format(sdk_type, sdk_type)) - - -class UndefinedSDKTypeException(Exception): - """Indicates an error has occurred in while reading constants.""" - - def __init__(self, msg): - super(UndefinedSDKTypeException, self).__init__(msg) diff --git a/.test-infra/jenkins/dependency_check/version_comparer.py b/.test-infra/jenkins/dependency_check/version_comparer.py deleted file mode 100644 index d8134ecf11d4f..0000000000000 --- a/.test-infra/jenkins/dependency_check/version_comparer.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from dependency_check.report_generator_config import ReportGeneratorConfig - -def compare_dependency_versions(curr_ver, latest_ver): - """ - Compare the current using version and the latest version. - Return true if a major version change was found, or 3 minor versions that the current version is behind. - Args: - curr_ver - latest_ver - Return: - boolean - """ - if curr_ver is None or latest_ver is None: - return True - else: - curr_ver_splitted = curr_ver.split('.') - latest_ver_splitted = latest_ver.split('.') - curr_major_ver = curr_ver_splitted[0] - latest_major_ver = latest_ver_splitted[0] - # compare major versions - if curr_major_ver != latest_major_ver: - return True - # compare minor versions - else: - curr_minor_ver = curr_ver_splitted[1] if len(curr_ver_splitted) > 1 else None - latest_minor_ver = latest_ver_splitted[1] if len(latest_ver_splitted) > 1 else None - if curr_minor_ver is not None and latest_minor_ver is not None: - if (not curr_minor_ver.isdigit() or not latest_minor_ver.isdigit()) and curr_minor_ver != latest_minor_ver: - return True - elif int(curr_minor_ver) + ReportGeneratorConfig.MAX_MINOR_VERSION_DIFF <= int(latest_minor_ver): - return True - # TODO: Comparing patch versions if needed. - return False diff --git a/.test-infra/jenkins/dependency_check/version_comparer_test.py b/.test-infra/jenkins/dependency_check/version_comparer_test.py deleted file mode 100644 index b37fb5f607a7d..0000000000000 --- a/.test-infra/jenkins/dependency_check/version_comparer_test.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from . import version_comparer -import unittest - -class VersionComparerTest(unittest.TestCase): - """Tests for `version_comparer.py`.""" - - def setUp(self): - print("\n\nTest : " + self._testMethodName) - - def test_compare_major_verison_true(self): - curr_ver = '1.0.0' - latest_ver = '2.0.0' - self.assertTrue(version_comparer.compare_dependency_versions(curr_ver, latest_ver)) - - def test_compare_minor_version_true(self): - curr_ver = '1.0.0' - latest_ver = '1.3.0' - self.assertTrue(version_comparer.compare_dependency_versions(curr_ver, latest_ver)) - - def test_compare_non_semantic_version_true(self): - curr_ver = '1.rc1' - latest_ver = '1.rc2' - self.assertTrue(version_comparer.compare_dependency_versions(curr_ver, latest_ver)) - - def test_compare_minor_version_false(self): - curr_ver = '1.0.0' - latest_ver = '1.2.0' - self.assertFalse(version_comparer.compare_dependency_versions(curr_ver, latest_ver)) - - def test_compare_same_version_false(self): - curr_ver = '1.0.0' - latest_ver = '1.0.0' - self.assertFalse(version_comparer.compare_dependency_versions(curr_ver, latest_ver)) - -if __name__ == '__main__': - unittest.main() diff --git a/.test-infra/jenkins/job_Dependency_Check.groovy b/.test-infra/jenkins/job_Dependency_Check.groovy deleted file mode 100644 index b80f2610314db..0000000000000 --- a/.test-infra/jenkins/job_Dependency_Check.groovy +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties - -job('beam_Dependency_Check') { - description('Runs Beam dependency check.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties( - delegate, 'master', 100, true, 'beam', false) - - // Allows triggering this build against pull requests. - commonJobProperties.enablePhraseTriggeringFromPullRequest( - delegate, - 'Beam Dependency Check', - 'Run Dependency Check', - false - ) - - commonJobProperties.setAutoJob( - delegate, - '@weekly') - - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks('runBeamDependencyCheck') - commonJobProperties.setGradleSwitches(delegate) - switches('-Drevision=release') - } - - shell('cd ' + commonJobProperties.checkoutDir + - ' && bash .test-infra/jenkins/dependency_check/generate_report.sh ' + - commonJobProperties.PYTHON) - } - - def date = new Date().format('yyyy-MM-dd') - publishers { - extendedEmail { - triggers { - always { - recipientList('dev@beam.apache.org') - contentType('text/html') - subject("Beam Dependency Check Report (${date})") - content('''${FILE, path="src/build/dependencyUpdates/beam-dependency-check-report.html"}''') - } - } - } - archiveArtifacts { - pattern('src/build/dependencyUpdates/beam-dependency-check-report.html') - onlyIfSuccessful() - } - wsCleanup { - excludePattern('src/build/dependencyUpdates/beam-dependency-check-report.html') - } - } -} diff --git a/build.gradle.kts b/build.gradle.kts index b89270a9537fe..77520c3ce7233 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -18,11 +18,6 @@ plugins { base - // This plugin provides a task to determine which dependencies have updates. - // Additionally, the plugin checks for updates to Gradle itself. - // - // See https://github.com/ben-manes/gradle-versions-plugin for further details. - id("com.github.ben-manes.versions") version "0.33.0" // Apply one top level rat plugin to perform any required license enforcement analysis id("org.nosphere.apache.rat") version "0.8.0" // Enable gradle-based release management @@ -580,11 +575,6 @@ tasks.register("javaExamplesDataflowPrecommit") { dependsOn(":runners:google-cloud-dataflow-java:examples-streaming:preCommit") } -tasks.register("runBeamDependencyCheck") { - dependsOn(":dependencyUpdates") - dependsOn(":sdks:python:dependencyUpdates") -} - tasks.register("whitespacePreCommit") { // TODO(https://github.com/apache/beam/issues/20209): Find a better way to specify the tasks without hardcoding py version. dependsOn(":sdks:python:test-suites:tox:py38:archiveFilesToLint") diff --git a/ownership/JAVA_DEPENDENCY_OWNERS.yaml b/ownership/JAVA_DEPENDENCY_OWNERS.yaml index 063b2bcc145fd..eb43b8a49fd68 100644 --- a/ownership/JAVA_DEPENDENCY_OWNERS.yaml +++ b/ownership/JAVA_DEPENDENCY_OWNERS.yaml @@ -144,11 +144,6 @@ deps: artifact: jackson-module-scala_2.11 owners: - com.github.ben-manes:gradle-versions-plugin: - group: com.github.ben-manes - artifact: gradle-versions-plugin - owners: - com.github.jengelman.gradle.plugins:shadow: group: com.github.jengelman.gradle.plugins artifact: shadow diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index 7dac7c2ec0ff6..0c2ed72dfba18 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -134,11 +134,6 @@ def depSnapshot = tasks.register("depSnapshot") { } } -tasks.register("dependencyUpdates") { - dependsOn ':dependencyUpdates' - dependsOn ':sdks:python:test-suites:tox:pycommon:dependency-check' -} - tasks.register("buildSnapshot") { dependsOn sdist dependsOn depSnapshot diff --git a/sdks/python/scripts/run_dependency_check.sh b/sdks/python/scripts/run_dependency_check.sh deleted file mode 100755 index d60d6016fd53b..0000000000000 --- a/sdks/python/scripts/run_dependency_check.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This script will be run by Jenkins as a Python dependency test. - -set -euv - -mkdir -p $WORKSPACE/src/build/dependencyUpdates -rm -f $WORKSPACE/src/build/dependencyUpdates/python_dependency_report.txt - -# List all outdated dependencies and write results in pythonDependencyReport -echo "The following dependencies have later release versions:" > $WORKSPACE/src/build/dependencyUpdates/python_dependency_report.txt -pip list --outdated | sed -n '1,2!p' | while IFS= read -r line -do - echo $line | while IFS=' ' read dep curr_ver new_ver type - do - echo $line - echo " - $dep [$curr_ver -> $new_ver]" >> $WORKSPACE/src/build/dependencyUpdates/python_dependency_report.txt - done -done diff --git a/sdks/python/test-suites/tox/pycommon/build.gradle b/sdks/python/test-suites/tox/pycommon/build.gradle index 7e697ec367e15..7a44c9eb92f01 100644 --- a/sdks/python/test-suites/tox/pycommon/build.gradle +++ b/sdks/python/test-suites/tox/pycommon/build.gradle @@ -28,6 +28,4 @@ toxTask "docs", "py38-docs" assemble.dependsOn docs task preCommitPyCommon() { -} - -toxTask "dependency-check", "py3-dependency-check" +} \ No newline at end of file diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 50a883e26a7a6..9088c1483906f 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -244,16 +244,6 @@ commands = yapf --version time yapf --diff --parallel --recursive apache_beam -[testenv:py3-dependency-check] -# TODO(https://github.com/apache/beam/issues/20337): botocore, a part of [aws], wants docutils<0.16, but Sphinx -# pulls in the latest docutils. Uncomment this line once botocore does not -# conflict with Sphinx: -# extras = docs,test,gcp,aws,interactive,interactive_test -extras = test,gcp,aws,dataframe,interactive,interactive_test -passenv = WORKSPACE -commands = - time {toxinidir}/scripts/run_dependency_check.sh - [testenv:jest] setenv = deps =