diff --git a/.github/workflows/bundle_cron.yml b/.github/workflows/bundle_cron.yml index 1377872..1631b66 100644 --- a/.github/workflows/bundle_cron.yml +++ b/.github/workflows/bundle_cron.yml @@ -33,10 +33,10 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" - - name: Set up Python 3.6 + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.9 - name: Versions run: | python3 --version diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..e7031e0 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,30 @@ +name: Run pre-commit + +on: [pull_request, push] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + - name: Set up Python 3.9 + uses: actions/setup-python@v1 + with: + python-version: 3.9 + - name: Versions + run: | + python3 --version + - name: Checkout Current Repo + uses: actions/checkout@v1 + - name: Pip install requirements + run: | + pip install --force-reinstall -r requirements.txt + - name: Pip install pre-commit + run: | + pip install pre-commit + - name: Run pre-commit hooks + run: | + pre-commit run --all-files diff --git a/.github/workflows/reports_cron.yml b/.github/workflows/reports_cron.yml index fe4e09f..80e5a3a 100644 --- a/.github/workflows/reports_cron.yml +++ b/.github/workflows/reports_cron.yml @@ -30,10 +30,10 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" - - name: Set up Python 3.6 + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.9 - name: Versions run: | python3 --version diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 64d5dee..430cead 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,10 +31,10 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - name: Versions run: | python3 --version @@ -51,4 +51,4 @@ jobs: ADABOT_GITHUB_ACCESS_TOKEN: ${{ secrets.ADABOT_GITHUB_ACCESS_TOKEN }} REDIS_PORT: ${{ job.services.redis.ports[6379] }} run: | - python3 -u -m pytest \ No newline at end of file + python3 -u -m pytest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..26889a2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +exclude: patches +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/psf/black + rev: 21.6b0 + hooks: + - id: black +- repo: https://github.com/pycqa/pylint + rev: v2.9.3 + hooks: + - id: pylint + name: pylint + types: [python] diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..fda603a --- /dev/null +++ b/.pylintrc @@ -0,0 +1,445 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick, + long-suffix,old-ne-operator,old-octal-literal,import-star-module-level, + raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored, + suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin, + buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin, + raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin, + coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import, + old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment, + indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method, + cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin, + map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating, + filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method, + rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import, + deprecated-string-function,deprecated-str-translate-call,import-error,bad-continuation, + subprocess-run-check,too-many-lines + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable= + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio).You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +# notes=FIXME,XXX,TODO +notes=FIXME,XXX + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members=sh.ErrorReturnCode_1,sh.ErrorReturnCode_128 + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +# expected-line-ending-format= +expected-line-ending-format=LF + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=yes + +# Minimum lines number of a similarity. +min-similarity-lines=50 + + +[BASIC] + +# Naming hint for argument names +argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct argument names +argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Naming hint for attribute names +attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct attribute names +attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class names +# class-name-hint=[A-Z_][a-zA-Z0-9]+$ +class-name-hint=[A-Z_][a-zA-Z0-9_]+$ + +# Regular expression matching correct class names +# class-rgx=[A-Z_][a-zA-Z0-9]+$ +class-rgx=[A-Z_][a-zA-Z0-9_]+$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming hint for function names +function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct function names +function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Good variable names which should always be accepted, separated by a comma +# good-names=i,j,k,ex,Run,_ +good-names=r,g,b,w,i,j,k,n,x,y,z,ex,ok,Run,_ + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for method names +method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct method names +method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Naming hint for variable names +variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct variable names +variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of attributes for a class (see R0902). +# max-attributes=7 +max-attributes=11 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=1 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/adabot-cron.sh b/adabot-cron.sh index 0eeaf07..5601b89 100755 --- a/adabot-cron.sh +++ b/adabot-cron.sh @@ -6,4 +6,3 @@ source .env/bin/activate source env.sh python -m adabot.circuitpython_bundle - diff --git a/adabot.screenrc b/adabot.screenrc index 3d9acd8..a5516be 100644 --- a/adabot.screenrc +++ b/adabot.screenrc @@ -14,4 +14,3 @@ screen -t celery_high 2 bash -c "source .env/bin/activate; source env.sh; celery screen -t celery_low 3 bash -c "source .env/bin/activate; source env.sh; celery -A rosie-ci.celery worker -n low -Q low || [ $? -eq 1 ] || sleep 1000" detach - diff --git a/adabot/arduino_libraries.py b/adabot/arduino_libraries.py index 8bd23c4..b9c23b6 100644 --- a/adabot/arduino_libraries.py +++ b/adabot/arduino_libraries.py @@ -20,8 +20,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +"""Adabot utility for Arduino Libraries.""" + import argparse -import datetime import logging import sys import traceback @@ -32,36 +33,59 @@ logger = logging.getLogger(__name__) ch = logging.StreamHandler(stream=sys.stdout) -logging.basicConfig( - level=logging.INFO, - format='%(message)s', - handlers=[ch] -) +logging.basicConfig(level=logging.INFO, format="%(message)s", handlers=[ch]) # Setup ArgumentParser -cmd_line_parser = argparse.ArgumentParser(description="Adabot utility for Arduino Libraries.", - prog="Adabot Arduino Libraries Utility") -cmd_line_parser.add_argument("-o", "--output_file", help="Output log to the filename provided.", - metavar="", dest="output_file") -cmd_line_parser.add_argument("-v", "--verbose", help="Set the level of verbosity printed to the command prompt." - " Zero is off; One is on (default).", type=int, default=1, dest="verbose", choices=[0,1]) +cmd_line_parser = argparse.ArgumentParser( + description="Adabot utility for Arduino Libraries.", + prog="Adabot Arduino Libraries Utility", +) +cmd_line_parser.add_argument( + "-o", + "--output_file", + help="Output log to the filename provided.", + metavar="", + dest="output_file", +) +cmd_line_parser.add_argument( + "-v", + "--verbose", + help="Set the level of verbosity printed to the command prompt." + " Zero is off; One is on (default).", + type=int, + default=1, + dest="verbose", + choices=[0, 1], +) all_libraries = [] adafruit_library_index = [] + def list_repos(): - """ Return a list of all Adafruit repositories with 'Arduino' in either the - name, description, or readme. Each list item is a dictionary of GitHub API - repository state. + """Return a list of all Adafruit repositories with 'Arduino' in either the + name, description, or readme. Each list item is a dictionary of GitHub API + repository state. """ repos = [] - result = github.get("/search/repositories", - params={"q":"Arduino in:name in:description in:readme fork:true user:adafruit archived:false OR Library in:name in:description in:readme fork:true user:adafruit archived:false OR Adafruit_ in:name fork:true user:adafruit archived:false AND NOT PCB in:name AND NOT Python in:name", - "per_page": 100, - "sort": "updated", - "order": "asc"}) + result = github.get( + "/search/repositories", + params={ + "q": ( + "Arduino in:name in:description in:readme fork:true user:adafruit archived:false" + " OR Library in:name in:description in:readme fork:true user:adafruit" + " archived:false OR Adafruit_ in:name fork:true user:adafruit archived:false AND" + " NOT PCB in:name AND NOT Python in:name" + ), + "per_page": 100, + "sort": "updated", + "order": "asc", + }, + ) while result.ok: - repos.extend(result.json()["items"]) # uncomment and comment below, to include all forks + repos.extend( + result.json()["items"] + ) # uncomment and comment below, to include all forks if result.links.get("next"): result = github.get(result.links["next"]["url"]) @@ -70,43 +94,57 @@ def list_repos(): return repos + def is_arduino_library(repo): - """ Returns if the repo is an Arduino library, as determined by the existence of - the 'library.properties' file. + """Returns if the repo is an Arduino library, as determined by the existence of + the 'library.properties' file. """ - lib_prop_file = requests.get("https://raw.githubusercontent.com/adafruit/" + repo["name"] + "/master/library.properties") + lib_prop_file = requests.get( + "https://raw.githubusercontent.com/adafruit/" + + repo["name"] + + "/master/library.properties" + ) return lib_prop_file.ok + def print_list_output(title, coll): - "" + """Helper function to format output.""" logger.info("") - logger.info(title.format(len(coll)-2)) - long_col = [(max([len(str(row[i])) for row in coll]) + 3) - for i in range(len(coll[0]))] + logger.info(title.format(len(coll) - 2)) + long_col = [ + (max([len(str(row[i])) for row in coll]) + 3) for i in range(len(coll[0])) + ] row_format = "".join(["{:<" + str(this_col) + "}" for this_col in long_col]) for lib in coll: - logger.info(row_format.format(*lib)) + logger.info("%s", row_format.format(*lib)) + def validate_library_properties(repo): - """ Checks if the latest GitHub Release Tag and version in the library_properties - file match. Will also check if the library_properties is there, but no release - has been made. + """Checks if the latest GitHub Release Tag and version in the library_properties + file match. Will also check if the library_properties is there, but no release + has been made. """ lib_prop_file = None lib_version = None release_tag = None - lib_prop_file = requests.get("https://raw.githubusercontent.com/adafruit/" + repo["name"] + "/master/library.properties") + lib_prop_file = requests.get( + "https://raw.githubusercontent.com/adafruit/" + + repo["name"] + + "/master/library.properties" + ) if not lib_prop_file.ok: - #print("{} skipped".format(repo["name"])) - return None # no library properties file! + # print("{} skipped".format(repo["name"])) + return None # no library properties file! lines = lib_prop_file.text.split("\n") for line in lines: if "version" in line: - lib_version = line[len("version="):] + lib_version = line[len("version=") :] break - get_latest_release = github.get("/repos/adafruit/" + repo["name"] + "/releases/latest") + get_latest_release = github.get( + "/repos/adafruit/" + repo["name"] + "/releases/latest" + ) if get_latest_release.ok: response = get_latest_release.json() if "tag_name" in response: @@ -118,61 +156,75 @@ def validate_library_properties(repo): release_tag = "Unknown" if lib_version and release_tag: - return [release_tag, lib_version] + return [release_tag, lib_version] return None + def validate_release_state(repo): """Validate if a repo 1) has a release, and 2) if there have been commits since the last release. Returns a list of string error messages for the repository. """ if not is_arduino_library(repo): - return + return None - compare_tags = github.get("/repos/" + repo["full_name"] + "/compare/master..." + repo['tag_name']) + compare_tags = github.get( + "/repos/" + repo["full_name"] + "/compare/master..." + repo["tag_name"] + ) if not compare_tags.ok: - logger.error("Error: failed to compare {0} 'master' to tag '{1}'".format(repo["name"], repo['tag_name'])) - return + logger.error( + "Error: failed to compare %s 'master' to tag '%s'", + repo["name"], + repo["tag_name"], + ) + return None compare_tags_json = compare_tags.json() if "status" in compare_tags_json: if compare_tags.json()["status"] != "identical": - #print("Compare {4} status: {0} \n Ahead: {1} \t Behind: {2} \t Commits: {3}".format( - # compare_tags_json["status"], compare_tags_json["ahead_by"], - # compare_tags_json["behind_by"], compare_tags_json["total_commits"], repo["full_name"])) - return [repo['tag_name'], compare_tags_json["behind_by"]] + return [repo["tag_name"], compare_tags_json["behind_by"]] elif "errors" in compare_tags_json: - logger.error("Error: comparing latest release to 'master' failed on '{0}'. Error Message: {1}".format( - repo["name"], compare_tags_json["message"])) + logger.error( + "Error: comparing latest release to 'master' failed on '%s'. Error Message: %s", + repo["name"], + compare_tags_json["message"], + ) - return + return None -def validate_actions(repo): - """Validate if a repo has workflows/githubci.yml - """ - repo_has_actions = requests.get("https://raw.githubusercontent.com/adafruit/" + repo["name"] + "/master/.github/workflows/githubci.yml") - return repo_has_actions.ok def validate_actions(repo): - """Validate if a repo has actions githubci.yml - """ - repo_has_actions = requests.get("https://raw.githubusercontent.com/adafruit/" + repo["name"] + "/master/.github/workflows/githubci.yml") + """Validate if a repo has workflows/githubci.yml""" + repo_has_actions = requests.get( + "https://raw.githubusercontent.com/adafruit/" + + repo["name"] + + "/master/.github/workflows/githubci.yml" + ) return repo_has_actions.ok + def validate_example(repo): - """Validate if a repo has any files in examples directory - """ + """Validate if a repo has any files in examples directory""" repo_has_ino = github.get("/repos/adafruit/" + repo["name"] + "/contents/examples") return repo_has_ino.ok and len(repo_has_ino.json()) + +# pylint: disable=too-many-branches def run_arduino_lib_checks(): + """Run necessary functions and outout the results.""" logger.info("Running Arduino Library Checks") logger.info("Getting list of libraries to check...") repo_list = list_repos() - logger.info("Found {} Arduino libraries to check\n".format(len(repo_list))) - failed_lib_prop = [[" Repo", "Release Tag", "library.properties Version"], [" ----", "-----------", "--------------------------"]] - needs_release_list = [[" Repo", "Latest Release", "Commits Behind"], [" ----", "--------------", "--------------"]] + logger.info("Found %s Arduino libraries to check\n", len(repo_list)) + failed_lib_prop = [ + [" Repo", "Release Tag", "library.properties Version"], + [" ----", "-----------", "--------------------------"], + ] + needs_release_list = [ + [" Repo", "Latest Release", "Commits Behind"], + [" ----", "--------------", "--------------"], + ] needs_registration_list = [[" Repo"], [" ----"]] missing_actions_list = [[" Repo"], [" ----"]] missing_library_properties_list = [[" Repo"], [" ----"]] @@ -183,35 +235,39 @@ def run_arduino_lib_checks(): # not a library continue - entry = {'name': repo["name"]} + entry = {"name": repo["name"]} lib_check = validate_library_properties(repo) if not lib_check: missing_library_properties_list.append([" " + str(repo["name"])]) continue - #print(repo['clone_url']) + # print(repo['clone_url']) needs_registration = False for lib in adafruit_library_index: - if (repo['clone_url'] == lib['repository']) or (repo['html_url'] == lib['website']): - entry['arduino_version'] = lib['version'] # found it! + if (repo["clone_url"] == lib["repository"]) or ( + repo["html_url"] == lib["website"] + ): + entry["arduino_version"] = lib["version"] # found it! break else: needs_registration = True if needs_registration: needs_registration_list.append([" " + str(repo["name"])]) - entry['release'] = lib_check[0] - entry['version'] = lib_check[1] - repo['tag_name'] = lib_check[0] + entry["release"] = lib_check[0] + entry["version"] = lib_check[1] + repo["tag_name"] = lib_check[0] needs_release = validate_release_state(repo) - entry['needs_release'] = needs_release + entry["needs_release"] = needs_release if needs_release: - needs_release_list.append([" " + str(repo["name"]), needs_release[0], needs_release[1]]) + needs_release_list.append( + [" " + str(repo["name"]), needs_release[0], needs_release[1]] + ) missing_actions = not validate_actions(repo) - entry['needs_actions'] = missing_actions + entry["needs_actions"] = missing_actions if missing_actions: missing_actions_list.append([" " + str(repo["name"])]) @@ -221,53 +277,67 @@ def run_arduino_lib_checks(): logging.info(entry) if len(failed_lib_prop) > 2: - print_list_output("Libraries Have Mismatched Release Tag and library.properties Version: ({})", failed_lib_prop) + print_list_output( + "Libraries Have Mismatched Release Tag and library.properties Version: ({})", + failed_lib_prop, + ) if len(needs_registration_list) > 2: - print_list_output("Libraries that are not registered with Arduino: ({})", needs_registration_list) + print_list_output( + "Libraries that are not registered with Arduino: ({})", + needs_registration_list, + ) if len(needs_release_list) > 2: - print_list_output("Libraries have commits since last release: ({})", needs_release_list); + print_list_output( + "Libraries have commits since last release: ({})", needs_release_list + ) if len(missing_actions_list) > 2: - print_list_output("Libraries that is not configured with Actions: ({})", missing_actions_list) + print_list_output( + "Libraries that is not configured with Actions: ({})", missing_actions_list + ) if len(missing_library_properties_list) > 2: - print_list_output("Libraries that is missing library.properties file: ({})", missing_library_properties_list) + print_list_output( + "Libraries that is missing library.properties file: ({})", + missing_library_properties_list, + ) -def main(verbosity=1, output_file=None): - +def main(verbosity=1, output_file=None): # pylint: disable=missing-function-docstring if output_file: - fh = logging.FileHandler(output_file) - logger.addHandler(fh) + file_handler = logging.FileHandler(output_file) + logger.addHandler(file_handler) + + if verbosity == 0: + logger.setLevel("CRITICAL") try: reply = requests.get("http://downloads.arduino.cc/libraries/library_index.json") if not reply.ok: - logging.error("Could not fetch http://downloads.arduino.cc/libraries/library_index.json") - exit() + logging.error( + "Could not fetch http://downloads.arduino.cc/libraries/library_index.json" + ) + sys.exit() arduino_library_index = reply.json() - for lib in arduino_library_index['libraries']: - if 'adafruit' in lib['url']: + for lib in arduino_library_index["libraries"]: + if "adafruit" in lib["url"]: adafruit_library_index.append(lib) run_arduino_lib_checks() except: - exc_type, exc_val, exc_tb = sys.exc_info() + _, exc_val, exc_tb = sys.exc_info() logger.error("Exception Occurred!", quiet=True) - logger.error(("-"*60), quiet=True) + logger.error(("-" * 60), quiet=True) logger.error("Traceback (most recent call last):") - tb = traceback.format_tb(exc_tb) - for line in tb: + trace = traceback.format_tb(exc_tb) + for line in trace: logger.error(line) logger.error(exc_val) raise + if __name__ == "__main__": cmd_line_args = cmd_line_parser.parse_args() - main( - verbosity=cmd_line_args.verbose, - output_file=cmd_line_args.output_file - ) - + main(verbosity=cmd_line_args.verbose, output_file=cmd_line_args.output_file) diff --git a/adabot/circuitpython_bundle.py b/adabot/circuitpython_bundle.py index 0c5ab4e..07274f5 100644 --- a/adabot/circuitpython_bundle.py +++ b/adabot/circuitpython_bundle.py @@ -20,35 +20,52 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -from adabot import github_requests as github -from adabot.lib import common_funcs +""" Checks each library in the CircuitPython Bundles for updates. + If updates are found the bundle is updated, updates are pushed to the + remote, and a new release is made. +""" + +from datetime import date +from io import StringIO import os -import subprocess import shlex -from io import StringIO -from datetime import date +import subprocess + +import redis as redis_py import sh from sh.contrib import git -import redis as redis_py +from adabot import github_requests as github +from adabot.lib import common_funcs -redis = None +REDIS = None if "GITHUB_WORKSPACE" in os.environ: - redis = redis_py.StrictRedis(port=os.environ["REDIS_PORT"]) + REDIS = redis_py.StrictRedis(port=os.environ["REDIS_PORT"]) else: - redis = redis_py.StrictRedis() + REDIS = redis_py.StrictRedis() + +BUNDLES = ["Adafruit_CircuitPython_Bundle", "CircuitPython_Community_Bundle"] -bundles = ["Adafruit_CircuitPython_Bundle", "CircuitPython_Community_Bundle"] def fetch_bundle(bundle, bundle_path): + """Clones `bundle` to `bundle_path`""" if not os.path.isdir(bundle_path): os.makedirs(bundle_path, exist_ok=True) if "GITHUB_WORKSPACE" in os.environ: - git_url = "https://" + os.environ["ADABOT_GITHUB_ACCESS_TOKEN"] + "@github.com/adafruit/" + git_url = ( + "https://" + + os.environ["ADABOT_GITHUB_ACCESS_TOKEN"] + + "@github.com/adafruit/" + ) git.clone("-o", "adafruit", git_url + bundle + ".git", bundle_path) else: - git.clone("-o", "adafruit", "https://github.com/adafruit/" + bundle + ".git", bundle_path) + git.clone( + "-o", + "adafruit", + "https://github.com/adafruit/" + bundle + ".git", + bundle_path, + ) working_directory = os.getcwd() os.chdir(bundle_path) git.pull() @@ -56,34 +73,46 @@ def fetch_bundle(bundle, bundle_path): git.submodule("update") os.chdir(working_directory) + +# pylint: disable=too-many-locals def check_lib_links_md(bundle_path): + """Checks and updates the `circuitpython_library_list` Markdown document + located in the Adafruit CircuitPython Bundle. + """ if not "Adafruit_CircuitPython_Bundle" in bundle_path: return [] - submodules_list = sorted(common_funcs.get_bundle_submodules(), - key=lambda module: module[1]["path"]) + submodules_list = sorted( + common_funcs.get_bundle_submodules(), key=lambda module: module[1]["path"] + ) lib_count = len(submodules_list) # used to generate commit message by comparing new libs to current list try: - with open(os.path.join(bundle_path, "circuitpython_library_list.md"), 'r') as f: - read_lines = f.read().splitlines() - except: + with open( + os.path.join(bundle_path, "circuitpython_library_list.md"), "r" + ) as lib_list: + read_lines = lib_list.read().splitlines() + except OSError: read_lines = [] - pass write_drivers = [] write_helpers = [] updates_made = [] for submodule in submodules_list: url = submodule[1]["url"] - url_name = url[url.rfind("/") + 1:(url.rfind(".") if url.rfind(".") > url.rfind("/") else len(url))] + url_name = url[ + url.rfind("/") + + 1 : (url.rfind(".") if url.rfind(".") > url.rfind("/") else len(url)) + ] pypi_name = "" - if common_funcs.repo_is_on_pypi({"name" : url_name}): - pypi_name = " ([PyPi](https://pypi.org/project/{}))".format(url_name.replace("_", "-").lower()) + if common_funcs.repo_is_on_pypi({"name": url_name}): + pypi_name = " ([PyPi](https://pypi.org/project/{}))".format( + url_name.replace("_", "-").lower() + ) docs_name = "" docs_link = common_funcs.get_docs_link(bundle_path, submodule) if docs_link: - docs_name = f" \([Docs]({docs_link}))" + docs_name = f" \([Docs]({docs_link}))" # pylint: disable=anomalous-backslash-in-string title = url_name.replace("_", " ") list_line = "* [{0}]({1}){2}{3}".format(title, url, pypi_name, docs_name) if list_line not in read_lines: @@ -93,25 +122,38 @@ def check_lib_links_md(bundle_path): elif "helpers" in submodule[1]["path"]: write_helpers.append(list_line) - with open(os.path.join(bundle_path, "circuitpython_library_list.md"), 'w') as f: - f.write("# Adafruit CircuitPython Libraries\n") - f.write("![Blinka Reading](https://raw.githubusercontent.com/adafruit/circuitpython-weekly-newsletter/gh-pages/assets/archives/22_1023blinka.png)\n\n") - f.write("Here is a listing of current Adafruit CircuitPython Libraries. There are {} libraries available.\n\n".format(lib_count)) - f.write("## Drivers:\n") + lib_list_header = [ + "# Adafruit CircuitPython Libraries", + ( + "![Blinka Reading](https://raw.githubusercontent.com/adafruit/circuitpython-weekly-" + "newsletter/gh-pages/assets/archives/22_1023blinka.png)" + ), + "Here is a listing of current Adafruit CircuitPython Libraries.", + f"There are {lib_count} libraries available.\n", + "## Drivers:\n", + ] + + with open( + os.path.join(bundle_path, "circuitpython_library_list.md"), "w" + ) as md_file: + md_file.write("\n".join(lib_list_header)) for line in sorted(write_drivers): - f.write(line + "\n") - f.write("\n## Helpers:\n") + md_file.write(line + "\n") + md_file.write("\n## Helpers:\n") for line in sorted(write_helpers): - f.write(line + "\n") + md_file.write(line + "\n") return updates_made + class Submodule: + """Context managing class to use with git submodules.""" + def __init__(self, directory): self.directory = directory + self.original_directory = os.path.abspath(os.getcwd()) def __enter__(self): - self.original_directory = os.path.abspath(os.getcwd()) os.chdir(self.directory) def __exit__(self, exc_type, exc_value, traceback): @@ -119,6 +161,7 @@ def __exit__(self, exc_type, exc_value, traceback): def commit_to_tag(repo_path, commit): + """Fetch the tag for `commit`.""" with Submodule(repo_path): try: output = StringIO() @@ -128,7 +171,9 @@ def commit_to_tag(repo_path, commit): pass return commit + def repo_version(): + """The version as defined by the tag.""" version = StringIO() try: git.describe("--tags", "--exact-match", _out=version) @@ -139,18 +184,22 @@ def repo_version(): def repo_sha(): + """The SHA of the repo.""" version = StringIO() git.log(pretty="format:%H", n=1, _out=version) return version.getvalue().strip() def repo_remote_url(repo_path): + """The URL for the remote branch.""" with Submodule(repo_path): output = StringIO() git.remote("get-url", "origin", _out=output) return output.getvalue().strip() + def update_bundle(bundle_path): + """Process all libraries in the bundle, and update their version if necessary.""" working_directory = os.path.abspath(os.getcwd()) os.chdir(bundle_path) git.submodule("foreach", "git", "fetch") @@ -158,9 +207,15 @@ def update_bundle(bundle_path): # They will contain a '-' in the tag, such as '3.0.0-beta.5'. # --exclude must be before --tags. # sh fails to find the subcommand so we use subprocess. - subprocess.run(shlex.split("git submodule foreach 'git checkout -q `git rev-list --exclude='*-*' --tags --max-count=1`'"), stdout=subprocess.DEVNULL) + subprocess.run( + shlex.split( + "git submodule foreach 'git checkout -q " + "`git rev-list --exclude='*-*' --tags --max-count=1`'" + ), + stdout=subprocess.DEVNULL, + ) status = StringIO() - result = git.status("--short", _out=status) + git.status("--short", _out=status) updates = [] status = status.getvalue().strip() if status: @@ -173,7 +228,7 @@ def update_bundle(bundle_path): # Compute the tag difference. diff = StringIO() - result = git.diff("--submodule=log", directory, _out=diff) + git.diff("--submodule=log", directory, _out=diff) diff_lines = diff.getvalue().split("\n") commit_range = diff_lines[0].split()[2] commit_range = commit_range.strip(":").split(".") @@ -185,61 +240,75 @@ def update_bundle(bundle_path): os.chdir(working_directory) lib_list_updates = check_lib_links_md(bundle_path) if lib_list_updates: - updates.append(("https://github.com/adafruit/Adafruit_CircuitPython_Bundle/circuitpython_library_list.md", - "NA", - "NA", - " > Added the following libraries: {}".format(", ".join(lib_list_updates)))) + updates.append( + ( + ( + "https://github.com/adafruit/Adafruit_CircuitPython_Bundle/" + "circuitpython_library_list.md" + ), + "NA", + "NA", + " > Added the following libraries: {}".format( + ", ".join(lib_list_updates) + ), + ) + ) return updates + def commit_updates(bundle_path, update_info): + """Commit changes to `bundle_path` using `update_info` for the commit message.""" working_directory = os.path.abspath(os.getcwd()) - message = ["Automated update by Adabot (adafruit/adabot@{})" - .format(repo_version())] + message = ["Automated update by Adabot (adafruit/adabot@{})".format(repo_version())] os.chdir(bundle_path) for url, old_commit, new_commit, summary in update_info: url_parts = url.split("/") user, repo = url_parts[-2:] summary = summary.replace("#", "{}/{}#".format(user, repo)) - message.append("Updating {} to {} from {}:\n{}".format(url, - new_commit, - old_commit, - summary)) + message.append( + "Updating {} to {} from {}:\n{}".format( + url, new_commit, old_commit, summary + ) + ) message = "\n\n".join(message) git.add(".") git.commit(message=message) os.chdir(working_directory) + def push_updates(bundle_path): + """Push bundle updates to the remote.""" working_directory = os.path.abspath(os.getcwd()) os.chdir(bundle_path) git.push() os.chdir(working_directory) + def get_contributors(repo, commit_range): + """Get contributors to `repo` for the `commit_range`.""" output = StringIO() try: git.log("--pretty=tformat:%H,%ae,%ce", commit_range, _out=output) except sh.ErrorReturnCode_128: print("Skipping contributors for:", repo) - pass output = output.getvalue().strip() contributors = {} if not output: return contributors for log_line in output.split("\n"): sha, author_email, committer_email = log_line.split(",") - author = redis.get("github_username:" + author_email) - committer = redis.get("github_username:" + committer_email) + author = REDIS.get("github_username:" + author_email) + committer = REDIS.get("github_username:" + committer_email) if not author or not committer: github_commit_info = github.get("/repos/" + repo + "/commits/" + sha) github_commit_info = github_commit_info.json() if github_commit_info["author"]: author = github_commit_info["author"]["login"] - redis.set("github_username:" + author_email, author) + REDIS.set("github_username:" + author_email, author) if github_commit_info["committer"]: committer = github_commit_info["committer"]["login"] - redis.set("github_username:" + committer_email, committer) + REDIS.set("github_username:" + committer_email, committer) else: author = author.decode("utf-8") committer = committer.decode("utf-8") @@ -256,25 +325,31 @@ def get_contributors(repo, commit_range): contributors[committer] += 1 return contributors + def repo_name(url): - # Strips off .git and splits on / + """Strips off .git and splits on /""" if url.endswith(".git"): url = url[:-4] url = url.split("/") return url[-2] + "/" + url[-1] + +# TODO: turn `master_list` into a set()? def add_contributors(master_list, additions): + """Adds contributors to `master_list` if not already in the list.""" for contributor in additions: if contributor not in master_list: master_list[contributor] = 0 master_list[contributor] += additions[contributor] + +# pylint: disable=too-many-locals,too-many-branches,too-many-statements def new_release(bundle, bundle_path): + """Creates a new release for `bundle`.""" working_directory = os.path.abspath(os.getcwd()) os.chdir(bundle_path) print(bundle) - current_release = github.get( - "/repos/adafruit/{}/releases/latest".format(bundle)) + current_release = github.get("/repos/adafruit/{}/releases/latest".format(bundle)) last_tag = current_release.json()["tag_name"] contributors = get_contributors("adafruit/" + bundle, last_tag + "..") added_submodules = [] @@ -289,9 +364,10 @@ def new_release(bundle, bundle_path): return current_submodule = None current_index = None + # pylint: disable=no-else-continue for line in output.split("\n"): if line.startswith("diff"): - current_submodule = line.split()[-1][len("b/"):] + current_submodule = line.split()[-1][len("b/") :] continue elif "index" in line: current_index = line @@ -317,8 +393,7 @@ def new_release(bundle, bundle_path): new_commit = commit_range.split(".")[-1] release_tag = commit_to_tag(directory, new_commit) with Submodule(directory): - submodule_contributors = get_contributors(repo_name(repo_url), - commit_range) + submodule_contributors = get_contributors(repo_name(repo_url), commit_range) add_contributors(contributors, submodule_contributors) repo_links[library_name] = repo_url[:-4] + "/releases/" + release_tag @@ -340,13 +415,26 @@ def new_release(bundle, bundle_path): contributors = sorted(contributors, key=contributors.__getitem__, reverse=True) contributors = ["@" + x for x in contributors] - release_description.append("As always, thank you to all of our contributors: " + ", ".join(contributors)) + release_description.append( + "As always, thank you to all of our contributors: " + ", ".join(contributors) + ) release_description.append("\n--------------------------\n") - release_description.append("The libraries in each release are compiled for all recent major versions of CircuitPython. Please download the one that matches the major version of your CircuitPython. For example, if you are running 6.0.0 you should download the `6.x` bundle.\n") - - release_description.append("To install, simply download the matching zip file, unzip it, and selectively copy the libraries you would like to install into the lib folder on your CIRCUITPY drive. This is especially important for non-express boards with limited flash, such as the [Trinket M0](https://www.adafruit.com/product/3500), [Gemma M0](https://www.adafruit.com/product/3501) and [Feather M0 Basic](https://www.adafruit.com/product/2772).") + release_description.append( + "The libraries in each release are compiled for all recent major versions of CircuitPython." + " Please download the one that matches the major version of your CircuitPython. For example" + ", if you are running 6.0.0 you should download the `6.x` bundle.\n" + ) + + release_description.append( + "To install, simply download the matching zip file, unzip it, and selectively copy the" + " libraries you would like to install into the lib folder on your CIRCUITPY drive. This is" + " especially important for non-express boards with limited flash, such as the" + " [Trinket M0](https://www.adafruit.com/product/3500)," + " [Gemma M0](https://www.adafruit.com/product/3501) and" + " [Feather M0 Basic](https://www.adafruit.com/product/2772)." + ) release = { "tag_name": "{0:%Y%m%d}".format(date.today()), @@ -354,7 +442,8 @@ def new_release(bundle, bundle_path): "name": "{0:%B} {0:%d}, {0:%Y} auto-release".format(date.today()), "body": "\n".join(release_description), "draft": False, - "prerelease": False} + "prerelease": False, + } print("Releasing {}".format(release["tag_name"])) print(release["body"]) @@ -367,20 +456,21 @@ def new_release(bundle, bundle_path): os.chdir(working_directory) + if __name__ == "__main__": - directory = os.path.abspath(".bundles") + bundles_dir = os.path.abspath(".bundles") if "GITHUB_WORKSPACE" in os.environ: git.config("--global", "user.name", "adabot") git.config("--global", "user.email", os.environ["ADABOT_EMAIL"]) - for bundle in bundles: - bundle_path = os.path.join(directory, bundle) + for cp_bundle in BUNDLES: + bundle_dir = os.path.join(bundles_dir, cp_bundle) try: - fetch_bundle(bundle, bundle_path) - update_info = update_bundle(bundle_path) - if update_info: - commit_updates(bundle_path, update_info) - push_updates(bundle_path) - new_release(bundle, bundle_path) + fetch_bundle(cp_bundle, bundle_dir) + updates_needed = update_bundle(bundle_dir) + if updates_needed: + commit_updates(bundle_dir, updates_needed) + push_updates(bundle_dir) + new_release(cp_bundle, bundle_dir) except RuntimeError as e: - print("Failed to update and release:", bundle) + print("Failed to update and release:", cp_bundle) print(e) diff --git a/adabot/circuitpython_libraries.py b/adabot/circuitpython_libraries.py index 4489141..255e3bf 100644 --- a/adabot/circuitpython_libraries.py +++ b/adabot/circuitpython_libraries.py @@ -21,8 +21,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +"""Adabot utility for CircuitPython Libraries.""" + import argparse -import copy import datetime import inspect import logging @@ -30,8 +31,6 @@ import sys import traceback -import requests - from adabot import github_requests as github from adabot import pypi_requests as pypi from adabot.lib import circuitpython_library_validators as cirpy_lib_vals @@ -42,59 +41,53 @@ logger = logging.getLogger(__name__) ch = logging.StreamHandler(stream=sys.stdout) -logging.basicConfig( - level=logging.INFO, - format='%(message)s', - handlers=[ch] -) +logging.basicConfig(level=logging.INFO, format="%(message)s", handlers=[ch]) # Setup ArgumentParser cmd_line_parser = argparse.ArgumentParser( description="Adabot utility for CircuitPython Libraries.", - prog="Adabot CircuitPython Libraries Utility" + prog="Adabot CircuitPython Libraries Utility", ) cmd_line_parser.add_argument( - "-o", "--output_file", + "-o", + "--output_file", help="Output log to the filename provided.", metavar="", - dest="output_file" + dest="output_file", ) cmd_line_parser.add_argument( - "-p", "--print", + "-p", + "--print", help="Set the level of verbosity printed to the command prompt." " Zero is off; One is on (default).", type=int, default=1, dest="verbose", - choices=[0,1] + choices=[0, 1], ) cmd_line_parser.add_argument( - "-e", "--error_depth", + "-e", + "--error_depth", help="Set the threshold for outputting an error list. Default is 5.", dest="error_depth", type=int, default=5, - metavar="n" + metavar="n", ) cmd_line_parser.add_argument( - "-v", "--validator", + "-v", + "--validator", help="Run validators with 'all', or only the validator(s) supplied in a string.", dest="validator", - metavar='all OR "validator1, validator2, ..."' + metavar='all OR "validator1, validator2, ..."', ) -# Define global state shared by the functions above: -# Submodules inside the bundle (result of get_bundle_submodules) -bundle_submodules = [] - -# Load the latest pylint version -latest_pylint = "2.0.1" - # Functions to run on repositories to validate their state. By convention these # return a list of string errors for the specified repository (a dictionary # of Github API repository object state). default_validators = [ - vals for vals in inspect.getmembers(cirpy_lib_vals.library_validator) + vals + for vals in inspect.getmembers(cirpy_lib_vals.LibraryValidator) if vals[0].startswith("validate") ] @@ -102,31 +95,36 @@ close_pr_sort_re = re.compile(r"(?<=\(Days\sopen:\s)(.+)(?=\))") blinka_repos = [ - 'Adafruit_Blinka', - 'Adafruit_Blinka_bleio', - 'Adafruit_Blinka_Displayio', - 'Adafruit_Python_PlatformDetect', - 'Adafruit_Python_PureIO', - 'Adafruit_Blinka_PyPortal', - 'Adafruit_Python_Extended_Bus' + "Adafruit_Blinka", + "Adafruit_Blinka_bleio", + "Adafruit_Blinka_Displayio", + "Adafruit_Python_PlatformDetect", + "Adafruit_Python_PureIO", + "Adafruit_Blinka_PyPortal", + "Adafruit_Python_Extended_Bus", ] -def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args, error_depth): +# pylint: disable=too-many-locals, too-many-branches, too-many-statements +def run_library_checks(validators, kw_args, error_depth): """runs the various library checking functions""" + + # Load the latest pylint version + latest_pylint = "2.0.1" pylint_info = pypi.get("/pypi/pylint/json") if pylint_info and pylint_info.ok: latest_pylint = pylint_info.json()["info"]["version"] - logger.info("Latest pylint is: {}".format(latest_pylint)) + logger.info("Latest pylint is: %s", latest_pylint) - repos = common_funcs.list_repos(include_repos=tuple(blinka_repos) + - ("CircuitPython_Community_Bundle", - "cookiecutter-adafruit-circuitpython")) + repos = common_funcs.list_repos( + include_repos=tuple(blinka_repos) + + ("CircuitPython_Community_Bundle", "cookiecutter-adafruit-circuitpython") + ) - logger.info("Found {} repos to check.".format(len(repos))) + logger.info("Found %s repos to check.", len(repos)) bundle_submodules = common_funcs.get_bundle_submodules() - logger.info("Found {} submodules in the bundle.".format(len(bundle_submodules))) + logger.info("Found %s submodules in the bundle.", len(bundle_submodules)) github_user = common_funcs.whois_github_user() - logger.info("Running GitHub checks as " + github_user) + logger.info("Running GitHub checks as %s", github_user) need_work = 0 lib_insights = common_funcs.InsightData() @@ -140,9 +138,9 @@ def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args, er new_libs = {} updated_libs = {} - validator = cirpy_lib_vals.library_validator(validators, - bundle_submodules, - latest_pylint, **kw_args) + validator = cirpy_lib_vals.LibraryValidator( + validators, bundle_submodules, latest_pylint, **kw_args + ) for repo in repos: if len(validators) != 0: errors = validator.run_repo_validation(repo) @@ -156,7 +154,7 @@ def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args, er if not isinstance(error, tuple): # check for an error occurring in the valiator module if error == cirpy_lib_vals.ERROR_OUTPUT_HANDLER: - #print(errors, "repo output handler error:", validator.output_file_data) + # print(errors, "repo output handler error:", validator.output_file_data) logger.info(", ".join(validator.output_file_data)) validator.output_file_data.clear() if error not in repos_by_error: @@ -175,7 +173,9 @@ def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args, er elif repo["name"] == "circuitpython": insights = core_insights closed_metric = bool(insights == lib_insights) - errors = validator.gather_insights(repo, insights, since, show_closed_metric=closed_metric) + errors = validator.gather_insights( + repo, insights, since, show_closed_metric=closed_metric + ) if errors: print("insights error") for error in errors: @@ -201,76 +201,82 @@ def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args, er logger.info("") logger.info("### Core") print_pr_overview(core_insights) - logger.info("* {} open pull requests".format(len(core_insights["open_prs"]))) - sorted_prs = sorted(core_insights["open_prs"], - key=lambda days: int(pr_sort_re.search(days).group(1)), - reverse=True) - for pr in sorted_prs: - logger.info(" * {}".format(pr)) + logger.info("* %s open pull requests", len(core_insights["open_prs"])) + sorted_prs = sorted( + core_insights["open_prs"], + key=lambda days: int(pr_sort_re.search(days).group(1)), + reverse=True, + ) + for pull_request in sorted_prs: + logger.info(" * %s", pull_request) print_issue_overview(core_insights) - logger.info("* {} open issues".format(len(core_insights["open_issues"]))) + logger.info("* %s open issues", len(core_insights["open_issues"])) logger.info(" * https://github.com/adafruit/circuitpython/issues") - logger.info("* {} active milestones".format(len(core_insights["milestones"]))) + logger.info("* %s active milestones", len(core_insights["milestones"])) ms_count = 0 for milestone in sorted(core_insights["milestones"].keys()): ms_count += core_insights["milestones"][milestone] - logger.info(" * {0}: {1} open issues".format(milestone, - core_insights["milestones"][milestone])) - logger.info(" * {} issues not assigned a milestone".format(len(core_insights["open_issues"]) - ms_count)) + logger.info( + " * %s: %s open issues", milestone, core_insights["milestones"][milestone] + ) + logger.info( + " * %s issues not assigned a milestone", + len(core_insights["open_issues"]) - ms_count, + ) logger.info("") ## temporarily disabling core download stats: # - GitHub API has been broken, due to the number of release artifacts # - Release asset delivery is being moved to AWS CloudFront/S3 - #print_circuitpython_download_stats() - logger.info( - "* Core download stats available at https://circuitpython.org/stats" - ) + # print_circuitpython_dl_stats() + logger.info("* Core download stats available at https://circuitpython.org/stats") logger.info("") logger.info("### Libraries") print_pr_overview(lib_insights) logger.info(" * Merged pull requests:") - sorted_prs = sorted(lib_insights["merged_prs"], - key=lambda days: int(close_pr_sort_re.search(days).group(1)), - reverse=True) - for pr in sorted_prs: - logger.info(" * {}".format(pr)) + sorted_prs = sorted( + lib_insights["merged_prs"], + key=lambda days: int(close_pr_sort_re.search(days).group(1)), + reverse=True, + ) + for pull_request in sorted_prs: + logger.info(" * %s", pull_request) print_issue_overview(lib_insights) logger.info("* https://circuitpython.org/contributing") - logger.info(" * {} open issues".format(len(lib_insights["open_issues"]))) - logger.info(" * {} good first issues".format(lib_insights["good_first_issues"])) + logger.info(" * %s open issues", len(lib_insights["open_issues"])) + logger.info(" * %s good first issues", lib_insights["good_first_issues"]) open_pr_days = [ - int(pr_sort_re.search(pr).group(1)) for pr in lib_insights["open_prs"] - if pr_sort_re.search(pr) is not None + int(pr_sort_re.search(pull_request).group(1)) + for pull_request in lib_insights["open_prs"] + if pr_sort_re.search(pull_request) is not None ] if len(lib_insights["open_prs"]) != 0: logger.info( - " * {0} open pull requests (Oldest: {1}, Newest: {2})".format( - len(lib_insights["open_prs"]), - max(open_pr_days), - max((min(open_pr_days), 1)) # ensure the minumum is '1' - ) + " * %s open pull requests (Oldest: %s, Newest: %s)", + len(lib_insights["open_prs"]), + max(open_pr_days), + max((min(open_pr_days), 1)), # ensure the minumum is '1' ) logger.info("Library updates in the last seven days:") if len(new_libs) != 0: logger.info("**New Libraries**") - for new in new_libs: - logger.info(" * [{}]({})".format(new, new_libs[new])) + for title, link in new_libs.items(): + logger.info(" * [%s](%s)", title, link) if len(updated_libs) != 0: logger.info("**Updated Libraries**") - for updated in updated_libs: - logger.info(" * [{}]({})".format(updated, updated_libs[updated])) + for title, link in updated_libs.items(): + logger.info(" * [%s](%s)", title, link) if len(validators) != 0: lib_repos = [] for repo in repos: - if (repo["owner"]["login"] == "adafruit" and - repo["name"].startswith("Adafruit_CircuitPython")): - lib_repos.append(repo) + if repo["owner"]["login"] == "adafruit" and repo["name"].startswith( + "Adafruit_CircuitPython" + ): + lib_repos.append(repo) - logger.info("{} out of {} repos need work.".format(need_work, - len(lib_repos))) + logger.info("%s out of %s repos need work.", need_work, len(lib_repos)) list_repos_for_errors = [cirpy_lib_vals.ERROR_NOT_IN_BUNDLE] logger.info("") @@ -279,48 +285,48 @@ def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args, er continue logger.info("") error_count = len(repos_by_error[error]) - logger.info("{} - {}".format(error, error_count)) + logger.info("%s - %s", error, error_count) if error_count <= error_depth or error in list_repos_for_errors: - logger.info("\n".join([" * " + x for x in repos_by_error[error]])) + logger.info( + "%s", "\n".join([" * " + x for x in repos_by_error[error]]) + ) logger.info("") logger.info("### Blinka") print_pr_overview(blinka_insights) - logger.info("* {} open pull requests".format(len(blinka_insights["open_prs"]))) - sorted_prs = sorted(blinka_insights["open_prs"], - key=lambda days: int(pr_sort_re.search(days).group(1)), - reverse=True) - for pr in sorted_prs: - logger.info(" * {}".format(pr)) + logger.info("* %s open pull requests", len(blinka_insights["open_prs"])) + sorted_prs = sorted( + blinka_insights["open_prs"], + key=lambda days: int(pr_sort_re.search(days).group(1)), + reverse=True, + ) + for pull_request in sorted_prs: + logger.info(" * %s", pull_request) print_issue_overview(blinka_insights) - logger.info("* {} open issues".format(len(blinka_insights["open_issues"]))) + logger.info("* %s open issues", len(blinka_insights["open_issues"])) logger.info(" * https://github.com/adafruit/Adafruit_Blinka/issues") - blinka_dl = dl_stats.piwheels_stats().get('adafruit-blinka', {}).get("month", "N/A") - logger.info("* Piwheels Downloads in the last month: {}".format(blinka_dl)) - logger.info( - "Number of supported boards: {}".format(blinka_funcs.board_count()) - ) + blinka_dl = dl_stats.piwheels_stats().get("adafruit-blinka", {}).get("month", "N/A") + logger.info("* Piwheels Downloads in the last month: %s", blinka_dl) + logger.info("Number of supported boards: %s", blinka_funcs.board_count()) -def print_circuitpython_download_stats(): + +# pylint: disable=too-many-branches,too-many-statements +def print_circuitpython_dl_stats(): """Gather and report analytics on the main CircuitPython repository.""" # TODO: with the move of release assets to AWS CloudFront/S3, update # this to use AWS CloudWatch metrics to gather download stats. # AWS' Python SDK `boto3` has CloudWatch interfaces which should - # enable this. https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudwatch.html + # enable this. try: response = github.get("/repos/adafruit/circuitpython/releases") except (ValueError, RuntimeError): - logger.info( - "Core CircuitPython GitHub download statistics request failed." - ) + logger.info("Core CircuitPython GitHub download statistics request failed.") return if not response.ok: - logger.info( - "Core CircuitPython GitHub download statistics request failed." - ) + logger.info("Core CircuitPython GitHub download statistics request failed.") return releases = response.json() @@ -342,7 +348,7 @@ def print_circuitpython_download_stats(): (\d\.\d\.\d.*) # version \.(?=uf2|bin|hex) # file extension """, - re.I | re.X + re.I | re.X, ) for release in releases: @@ -382,95 +388,163 @@ def print_circuitpython_download_stats(): total[release["tag_name"]] = 0 total[release["tag_name"]] += count - logger.info("Number of supported boards: {}".format(len(by_board))) + logger.info("Number of supported boards: %s", len(by_board)) logger.info("") logger.info("Download stats by board:") logger.info("") - by_board_list = [["Board", "{}".format(stable_tag.strip(" ")), "{}".format(prerelease_tag.strip(" "))],] + by_board_list = [ + [ + "Board", + "{}".format(stable_tag.strip(" ")), + "{}".format(prerelease_tag.strip(" ")), + ], + ] for board in sorted(by_board.items()): - by_board_list.append([str(board[0]), - (str(board[1][stable_tag]) if stable_tag in board[1] else "-"), - (str(board[1][prerelease_tag]) if prerelease_tag in board[1] else "-")]) - - long_col = [(max([len(str(row[i])) for row in by_board_list]) + 3) - for i in range(len(by_board_list[0]))] - #row_format = "".join(["{:<" + str(this_col) + "}" for this_col in long_col]) - row_format = "".join(["| {:<" + str(long_col[0]) + "}", - "|{:^" + str(long_col[1]) + "}", - "|{:^" + str(long_col[2]) + "}|"]) - - by_board_list.insert(1, - ["{}".format("-"*(long_col[0])), - "{}".format("-"*(long_col[1])), - "{}".format("-"*(long_col[2]))]) - - by_board_list.extend((["{}".format("-"*(long_col[0])), - "{}".format("-"*(long_col[1])), - "{}".format("-"*(long_col[2]))], - ["{0}{1}".format(" "*(long_col[0] - 6), "Total"), - "{}".format(total[stable_tag]), - "{}".format(total[prerelease_tag])], - ["{}".format("-"*(long_col[0])), - "{}".format("-"*(long_col[1])), - "{}".format("-"*(long_col[2]))])) + by_board_list.append( + [ + str(board[0]), + (str(board[1][stable_tag]) if stable_tag in board[1] else "-"), + (str(board[1][prerelease_tag]) if prerelease_tag in board[1] else "-"), + ] + ) + + long_col = [ + (max([len(str(row[i])) for row in by_board_list]) + 3) + for i in range(len(by_board_list[0])) + ] + # row_format = "".join(["{:<" + str(this_col) + "}" for this_col in long_col]) + row_format = "".join( + [ + "| {:<" + str(long_col[0]) + "}", + "|{:^" + str(long_col[1]) + "}", + "|{:^" + str(long_col[2]) + "}|", + ] + ) + + by_board_list.insert( + 1, + [ + "{}".format("-" * (long_col[0])), + "{}".format("-" * (long_col[1])), + "{}".format("-" * (long_col[2])), + ], + ) + + by_board_list.extend( + ( + [ + "{}".format("-" * (long_col[0])), + "{}".format("-" * (long_col[1])), + "{}".format("-" * (long_col[2])), + ], + [ + "{0}{1}".format(" " * (long_col[0] - 6), "Total"), + "{}".format(total[stable_tag]), + "{}".format(total[prerelease_tag]), + ], + [ + "{}".format("-" * (long_col[0])), + "{}".format("-" * (long_col[1])), + "{}".format("-" * (long_col[2])), + ], + ) + ) for row in by_board_list: - logger.info(row_format.format(*row)) + logger.info("%s", row_format.format(*row)) logger.info("") logger.info("Download stats by language:") logger.info("") - by_lang_list = [["Board", "{}".format(stable_tag.strip(" ")), "{}".format(prerelease_tag.strip(" "))],] + by_lang_list = [ + [ + "Board", + "{}".format(stable_tag.strip(" ")), + "{}".format(prerelease_tag.strip(" ")), + ], + ] for board in sorted(by_language.items()): - by_lang_list.append([str(board[0]), - (str(board[1][stable_tag]) if stable_tag in board[1] else "-"), - (str(board[1][prerelease_tag]) if prerelease_tag in board[1] else "-")]) - - long_col = [(max([len(str(row[i])) for row in by_lang_list]) + 3) - for i in range(len(by_lang_list[0]))] - #row_format = "".join(["{:<" + str(this_col) + "}" for this_col in long_col]) - row_format = "".join(["| {:<" + str(long_col[0]) + "}", - "|{:^" + str(long_col[1]) + "}", - "|{:^" + str(long_col[2]) + "}|"]) - - by_lang_list.insert(1, - ["{}".format("-"*(long_col[0])), - "{}".format("-"*(long_col[1])), - "{}".format("-"*(long_col[2]))]) - - by_lang_list.extend((["{}".format("-"*(long_col[0])), - "{}".format("-"*(long_col[1])), - "{}".format("-"*(long_col[2]))], - ["{0}{1}".format(" "*(long_col[0] - 6), "Total"), - "{}".format(total[stable_tag]), - "{}".format(total[prerelease_tag])], - ["{}".format("-"*(long_col[0])), - "{}".format("-"*(long_col[1])), - "{}".format("-"*(long_col[2]))])) + by_lang_list.append( + [ + str(board[0]), + (str(board[1][stable_tag]) if stable_tag in board[1] else "-"), + (str(board[1][prerelease_tag]) if prerelease_tag in board[1] else "-"), + ] + ) + + long_col = [ + (max([len(str(row[i])) for row in by_lang_list]) + 3) + for i in range(len(by_lang_list[0])) + ] + # row_format = "".join(["{:<" + str(this_col) + "}" for this_col in long_col]) + row_format = "".join( + [ + "| {:<" + str(long_col[0]) + "}", + "|{:^" + str(long_col[1]) + "}", + "|{:^" + str(long_col[2]) + "}|", + ] + ) + + by_lang_list.insert( + 1, + [ + "{}".format("-" * (long_col[0])), + "{}".format("-" * (long_col[1])), + "{}".format("-" * (long_col[2])), + ], + ) + + by_lang_list.extend( + ( + [ + "{}".format("-" * (long_col[0])), + "{}".format("-" * (long_col[1])), + "{}".format("-" * (long_col[2])), + ], + [ + "{0}{1}".format(" " * (long_col[0] - 6), "Total"), + "{}".format(total[stable_tag]), + "{}".format(total[prerelease_tag]), + ], + [ + "{}".format("-" * (long_col[0])), + "{}".format("-" * (long_col[1])), + "{}".format("-" * (long_col[2])), + ], + ) + ) for row in by_lang_list: - logger.info(row_format.format(*row)) - #for language in by_language: - # logger.info("* {} - {}".format(language, by_language[language])) + logger.info("%s", row_format.format(*row)) + # for language in by_language: + # logger.info("* %s - %s", language, by_language[language]) logger.info("") + def print_pr_overview(*insights): + """Prints an overview of Pull Requests""" merged_prs = sum([len(x["merged_prs"]) for x in insights]) authors = set().union(*[x["pr_merged_authors"] for x in insights]) reviewers = set().union(*[x["pr_reviewers"] for x in insights]) - logger.info("* {} pull requests merged".format(merged_prs)) - logger.info(" * {} authors - {}".format(len(authors), ", ".join(authors))) - logger.info(" * {} reviewers - {}".format(len(reviewers), ", ".join(reviewers))) + logger.info("* %s pull requests merged", merged_prs) + logger.info(" * %s authors - %s", len(authors), ", ".join(authors)) + logger.info(" * %s reviewers - %s", len(reviewers), ", ".join(reviewers)) + def print_issue_overview(*insights): + """Prints an overview of Issues""" closed_issues = sum([x["closed_issues"] for x in insights]) issue_closers = set().union(*[x["issue_closers"] for x in insights]) new_issues = sum([x["new_issues"] for x in insights]) issue_authors = set().union(*[x["issue_authors"] for x in insights]) - logger.info("* {} closed issues by {} people, {} opened by {} people" - .format(closed_issues, len(issue_closers), - new_issues, len(issue_authors))) + logger.info( + "* %s closed issues by %s people, %s opened by %s people", + closed_issues, + len(issue_closers), + new_issues, + len(issue_authors), + ) # print Hacktoberfest labels changes if its Hacktober in_season, season_action = hacktober.is_hacktober_season() @@ -486,24 +560,32 @@ def print_issue_overview(*insights): ) logger.info(hacktober_changes) + +# pylint: disable=too-many-branches def main(verbose=1, output_file=None, validator=None, error_depth=5): + """Main""" validator_kwarg_list = {} startup_message = [ "Running CircuitPython Library checks...", - "Report Date: {}".format(datetime.datetime.now().strftime("%d %B %Y, %I:%M%p")) + "Report Date: {}".format(datetime.datetime.now().strftime("%d %B %Y, %I:%M%p")), ] - verbosity = verbose - + if verbose == 0: + logger.setLevel("CRITICAL") + if output_file: - fh = logging.FileHandler(output_file) - logger.addHandler(fh) - startup_message.append(" - Report output will be saved to: {}".format(output_file)) + file_handler = logging.FileHandler(output_file) + logger.addHandler(file_handler) + startup_message.append( + " - Report output will be saved to: {}".format(output_file) + ) validators = [] validator_names = [] if validator: - startup_message.append(" - Depth for listing libraries with errors: {}".format(error_depth)) + startup_message.append( + " - Depth for listing libraries with errors: {}".format(error_depth) + ) if validator != "all": validators = [] @@ -512,59 +594,78 @@ def main(verbose=1, output_file=None, validator=None, error_depth=5): try: if not func_name.startswith("validate"): raise KeyError - #print('{}'.format(func_name)) + # print('{}'.format(func_name)) if "contents" not in func_name: validators.append( - [val[1] for val in default_validators if func_name in val[0]][0] + [ + val[1] + for val in default_validators + if func_name in val[0] + ][0] ) else: validators.insert( 0, - [val[1] for val in default_validators if func_name in val[0]][0] + [ + val[1] + for val in default_validators + if func_name in val[0] + ][0], ) validator_names.append(func_name) except KeyError: - #print(default_validators) - logger.info("Error: '{0}' is not an available validator.\n" \ - "Available validators are: {1}".format(func.strip(), - ", ".join([val[0] for val in default_validators]))) + # print(default_validators) + logger.info( + "Error: '%s' is not an available validator.\nAvailable validators are: %s", + func.strip(), + ", ".join([val[0] for val in default_validators]), + ) sys.exit() else: validators = [val_funcs[1] for val_funcs in default_validators] validator_names = [val_names[0] for val_names in default_validators] - startup_message.append(" - These validators will run: {}".format(", ".join(validator_names))) + startup_message.append( + " - These validators will run: {}".format(", ".join(validator_names)) + ) if "validate_contents" not in validator_names: validator_kwarg_list["validate_contents_quiet"] = True validators.insert( - 0, [val[1] for val in default_validators if "validate_contents" in val[0]][0] + 0, + [val[1] for val in default_validators if "validate_contents" in val[0]][ + 0 + ], ) try: for message in startup_message: logger.info(message) logger.info("") - #print(validators) - run_library_checks(validators, bundle_submodules, latest_pylint, - validator_kwarg_list, error_depth) + # print(validators) + run_library_checks( + validators, + validator_kwarg_list, + error_depth, + ) except: - exc_type, exc_val, exc_tb = sys.exc_info() + _, exc_val, exc_tb = sys.exc_info() logger.error("Exception Occurred!") - logger.error(("-"*60)) + logger.error(("-" * 60)) logger.error("Traceback (most recent call last):") - tb = traceback.format_tb(exc_tb) - for line in tb: + trace = traceback.format_tb(exc_tb) + for line in trace: logger.error(line) logger.error(exc_val) raise + if __name__ == "__main__": cli_args = cmd_line_parser.parse_args() main( verbose=cli_args.verbose, output_file=cli_args.output_file, validator=cli_args.validator, - error_depth=cli_args.error_depth - ) \ No newline at end of file + error_depth=cli_args.error_depth, + ) diff --git a/adabot/circuitpython_library_download_stats.py b/adabot/circuitpython_library_download_stats.py index 24849ff..5c9554f 100644 --- a/adabot/circuitpython_library_download_stats.py +++ b/adabot/circuitpython_library_download_stats.py @@ -20,6 +20,10 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +""" Collects download stats for the Adafruit CircuitPython Library Bundles + and each library. +""" + import datetime import sys import argparse @@ -28,21 +32,35 @@ import requests from adabot import github_requests as github -from adabot import pypi_requests as pypi from adabot.lib import common_funcs # Setup ArgumentParser -cmd_line_parser = argparse.ArgumentParser(description="Adabot utility for CircuitPython Library download stats." \ - " Provides stats for the Adafruit CircuitPython Bundle, and PyPi if available.", - prog="Adabot CircuitPython Libraries Download Stats") -cmd_line_parser.add_argument("-o", "--output_file", help="Output log to the filename provided.", - metavar="", dest="output_file") -cmd_line_parser.add_argument("-v", "--verbose", help="Set the level of verbosity printed to the command prompt." - " Zero is off; One is on (default).", type=int, default=1, dest="verbose", choices=[0,1]) +cmd_line_parser = argparse.ArgumentParser( + description="Adabot utility for CircuitPython Library download stats." + " Provides stats for the Adafruit CircuitPython Bundle, and PyPi if available.", + prog="Adabot CircuitPython Libraries Download Stats", +) +cmd_line_parser.add_argument( + "-o", + "--output_file", + help="Output log to the filename provided.", + metavar="", + dest="output_file", +) +cmd_line_parser.add_argument( + "-v", + "--verbose", + help="Set the level of verbosity printed to the command prompt." + " Zero is off; One is on (default).", + type=int, + default=1, + dest="verbose", + choices=[0, 1], +) # Global variables -output_filename = None -verbosity = 1 +OUTPUT_FILENAME = None +VERBOSITY = 1 file_data = [] # List containing libraries on PyPi that are not returned by the 'list_repos()' function, @@ -54,44 +72,57 @@ def piwheels_stats(): + """Get data dump of piwheels download stats""" stats = {} response = requests.get(PIWHEELS_PACKAGES_URL) if response.ok: packages = response.json() stats = { pkg: {"total": dl_all, "month": dl_month} - for pkg, dl_month, dl_all, *_ in packages if pkg.startswith("adafruit") + for pkg, dl_month, dl_all, *_ in packages + if pkg.startswith("adafruit") } return stats + def get_pypi_stats(): + """Map piwheels download stats for each repo""" successful_stats = {} failed_stats = [] repos = common_funcs.list_repos() dl_stats = piwheels_stats() for repo in repos: - if (repo["owner"]["login"] == "adafruit" and repo["name"].startswith("Adafruit_CircuitPython")): + if repo["owner"]["login"] == "adafruit" and repo["name"].startswith( + "Adafruit_CircuitPython" + ): if common_funcs.repo_is_on_pypi(repo): pkg_name = repo["name"].replace("_", "-").lower() if pkg_name in dl_stats: - successful_stats[repo["name"]] = (dl_stats[pkg_name]["month"], dl_stats[pkg_name]["total"]) + successful_stats[repo["name"]] = ( + dl_stats[pkg_name]["month"], + dl_stats[pkg_name]["total"], + ) else: failed_stats.append(repo["name"]) for lib in PYPI_FORCE_NON_CIRCUITPYTHON: pkg_name = lib.lower() if pkg_name in dl_stats: - successful_stats[lib] = (dl_stats[pkg_name]["month"], dl_stats[pkg_name]["total"]) + successful_stats[lib] = ( + dl_stats[pkg_name]["month"], + dl_stats[pkg_name]["total"], + ) else: - failed_stats.append(repo["name"]) + failed_stats.append(lib) return successful_stats, failed_stats + def get_bundle_stats(bundle): - """ Returns the download stats for 'bundle'. Uses release tag names to compile download - stats for the last 7 days. This assumes an Adabot release within that time frame, and - that tag name(s) will be the date (YYYYMMDD). + """Returns the download stats for 'bundle'. Uses release tag names to compile download + stats for the last 7 days. This assumes an Adabot release within that time frame, and + that tag name(s) will be the date (YYYYMMDD). """ stats_dict = {} bundle_stats = github.get("/repos/adafruit/" + bundle + "/releases") @@ -101,54 +132,76 @@ def get_bundle_stats(bundle): for release in bundle_stats.json(): try: - release_date = datetime.date(int(release["tag_name"][:4]), - int(release["tag_name"][4:6]), - int(release["tag_name"][6:])) - except: - output_handler("Skipping release. Tag name invalid: {}".format(release["tag_name"])) + release_date = datetime.date( + int(release["tag_name"][:4]), + int(release["tag_name"][4:6]), + int(release["tag_name"][6:]), + ) + except ValueError: + output_handler( + "Skipping release. Tag name invalid: {}".format(release["tag_name"]) + ) continue if (start_date - release_date).days > 7: break for asset in release["assets"]: if asset["name"].startswith("adafruit"): - asset_name = asset["name"][:asset["name"].rfind("-")] + asset_name = asset["name"][: asset["name"].rfind("-")] if asset_name in stats_dict: - stats_dict[asset_name] = stats_dict[asset_name] + asset["download_count"] + stats_dict[asset_name] = ( + stats_dict[asset_name] + asset["download_count"] + ) else: stats_dict[asset_name] = asset["download_count"] return stats_dict + def output_handler(message="", quiet=False): """Handles message output to prompt/file for functions.""" - if output_filename is not None: + if OUTPUT_FILENAME is not None: file_data.append(message) - if verbosity and not quiet: + if VERBOSITY and not quiet: print(message) + def run_stat_check(): + """Run and report all download stats.""" output_handler("Adafruit CircuitPython Library Download Stats") - output_handler("Report Date: {}".format(datetime.datetime.now().strftime("%d %B %Y, %I:%M%p"))) + output_handler( + "Report Date: {}".format(datetime.datetime.now().strftime("%d %B %Y, %I:%M%p")) + ) output_handler() output_handler("Adafruit_CircuitPython_Bundle downloads for the past week:") - for stat in sorted(get_bundle_stats("Adafruit_CircuitPython_Bundle").items(), - key=operator.itemgetter(1), reverse=True): + for stat in sorted( + get_bundle_stats("Adafruit_CircuitPython_Bundle").items(), + key=operator.itemgetter(1), + reverse=True, + ): output_handler(" {0}: {1}".format(stat[0], stat[1])) output_handler() pypi_downloads = {} pypi_failures = [] - downloads_list = [["| Library", "| Last Month", "| Total |"], - ["|:-------", "|:--------:", "|:-----:|"]] + downloads_list = [ + ["| Library", "| Last Month", "| Total |"], + ["|:-------", "|:--------:", "|:-----:|"], + ] output_handler("Adafruit CircuitPython Library Piwheels downloads:") output_handler() pypi_downloads, pypi_failures = get_pypi_stats() - for stat in sorted(pypi_downloads.items(), key=operator.itemgetter(1,1), reverse=True): - downloads_list.append(["| " + str(stat[0]), "| " + str(stat[1][0]), "| " + str(stat[1][1]) +" |"]) - - long_col = [(max([len(str(row[i])) for row in downloads_list]) + 3) - for i in range(len(downloads_list[0]))] + for stat in sorted( + pypi_downloads.items(), key=operator.itemgetter(1, 1), reverse=True + ): + downloads_list.append( + ["| " + str(stat[0]), "| " + str(stat[1][0]), "| " + str(stat[1][1]) + " |"] + ) + + long_col = [ + (max([len(str(row[i])) for row in downloads_list]) + 3) + for i in range(len(downloads_list[0])) + ] row_format = "".join(["{:<" + str(this_col) + "}" for this_col in long_col]) for lib in downloads_list: output_handler(row_format.format(*lib)) @@ -159,18 +212,19 @@ def run_stat_check(): for fail in pypi_failures: output_handler(" * {}".format(fail)) + if __name__ == "__main__": cmd_line_args = cmd_line_parser.parse_args() - verbosity = cmd_line_args.verbose + VERBOSITY = cmd_line_args.verbose if cmd_line_args.output_file: - output_filename = cmd_line_args.output_file + OUTPUT_FILENAME = cmd_line_args.output_file try: run_stat_check() except: - if output_filename is not None: + if OUTPUT_FILENAME is not None: exc_type, exc_val, exc_tb = sys.exc_info() output_handler("Exception Occurred!", quiet=True) - output_handler(("-"*60), quiet=True) + output_handler(("-" * 60), quiet=True) output_handler("Traceback (most recent call last):", quiet=True) tb = traceback.format_tb(exc_tb) for line in tb: @@ -180,7 +234,7 @@ def run_stat_check(): raise finally: - if output_filename is not None: - with open(output_filename, 'w') as f: + if OUTPUT_FILENAME is not None: + with open(OUTPUT_FILENAME, "w") as f: for line in file_data: f.write(str(line) + "\n") diff --git a/adabot/circuitpython_library_patches.py b/adabot/circuitpython_library_patches.py index 591a4ff..f6c9776 100644 --- a/adabot/circuitpython_library_patches.py +++ b/adabot/circuitpython_library_patches.py @@ -1,286 +1,373 @@ -import json -import requests -import os -import sys -import argparse -import shutil -import sh -from sh.contrib import git -from adabot.lib import common_funcs - - -working_directory = os.path.abspath(os.getcwd()) -lib_directory = working_directory + "/.libraries/" -patch_directory = working_directory + "/patches/" -repos = [] -check_errors = [] -apply_errors = [] -stats = [] - -""" -Setup the command line argument parsing object. -""" -cli_parser = argparse.ArgumentParser(description="Apply patches to any common file(s) in" - " all Adafruit CircuitPython Libraries.") -cli_parser.add_argument("-l", "--list", help="Lists the available patches to run.", - action='store_true') -cli_parser.add_argument("-p", help="Runs only the single patch referenced.", - metavar="", dest="patch") -cli_parser.add_argument("-f", help="Adds the referenced FLAGS to the git.am call." - " Only available when using '-p'. Enclose flags in brackets '[]'." - " Multiple flags can be passed. NOTE: '--signoff' is already used " - " used by default, and will be ignored. EXAMPLE: -f [-C0] -f [-s]", - metavar="FLAGS", action="append", dest="flags", type=str) -cli_parser.add_argument("--use-apply", help="Forces use of 'git apply' instead of 'git am'." - " This is necessary when needing to use 'apply' flags not available" - " to 'am' (e.g. '--unidiff-zero'). Only available when using '-p'.", - action="store_true", dest="use_apply") -cli_parser.add_argument("--dry-run", help="Accomplishes a dry run of patches, without applying" - " them.", action="store_true", dest="dry_run") -cli_parser.add_argument("--local", help="Force use of local patches. This skips verification" - " of patch files in the adabot GitHub repository. MUST use '--dry-run'" - " with this argument; this guards against applying unapproved patches.", - action="store_true", dest="run_local") - -def get_repo_list(): - """ Uses adabot.circuitpython_libraries module to get a list of - CircuitPython repositories. Filters the list down to adafruit - owned/sponsored CircuitPython libraries. - """ - repo_list = [] - get_repos = common_funcs.list_repos() - for repo in get_repos: - if not (repo["owner"]["login"] == "adafruit" and - repo["name"].startswith("Adafruit_CircuitPython")): - continue - repo_list.append(dict(name=repo["name"], url=repo["clone_url"])) - - return repo_list - -def get_patches(run_local): - """ Returns the list of patch files located in the adabot/patches - directory. - """ - return_list = [] - if not run_local: - contents = requests.get("https://api.github.com/repos/adafruit/adabot/contents/patches") - if contents.ok: - for patch in contents.json(): - patch_name = patch["name"] - return_list.append(patch_name) - else: - contents = os.listdir(patch_directory) - for file in contents: - if file.endswith(".patch"): - return_list.append(file) - - return return_list - -def apply_patch(repo_directory, patch_filepath, repo, patch, flags, use_apply): - """ Apply the `patch` in `patch_filepath` to the `repo` in - `repo_directory` using git am or git apply. The commit - with the user running the script (adabot if credentials are set - for that). - - When `use_apply` is true, the `--apply` flag is automatically added - to ensure that any passed flags that turn off apply (e.g. `--check`) - are overridden. - """ - if not os.getcwd() == repo_directory: - os.chdir(repo_directory) - - if not use_apply: - try: - git.am(flags, patch_filepath) - except sh.ErrorReturnCode as Err: - apply_errors.append(dict(repo_name=repo, - patch_name=patch, error=Err.stderr)) - return False - else: - apply_flags = ["--apply"] - for flag in flags: - if not flag == "--signoff": - apply_flags.append(flag) - try: - git.apply(apply_flags, patch_filepath) - except sh.ErrorReturnCode as Err: - apply_errors.append(dict(repo_name=repo, - patch_name=patch, error=Err.stderr)) - return False - - with open(patch_filepath) as f: - for line in f: - if "[PATCH]" in line: - message = '"' + line[(line.find("]") + 2):] + '"' - break - try: - git.commit("-a", "-m", message) - except sh.ErrorReturnCode as Err: - apply_errors.append(dict(repo_name=repo, - patch_name=patch, error=Err.stderr)) - return False - - try: - git.push() - except sh.ErrorReturnCode as Err: - apply_errors.append(dict(repo_name=repo, - patch_name=patch, error=Err.stderr)) - return False - return True - -def check_patches(repo, patches, flags, use_apply, dry_run): - """ Gather a list of patches from the `adabot/patches` directory - on the adabot repo. Clone the `repo` and run git apply --check - to test wether it requires any of the gathered patches. - - When `use_apply` is true, any flags except `--apply` are passed - through to the check call. This ensures that the check call is - representative of the actual apply call. - """ - applied = 0 - skipped = 0 - failed = 0 - - repo_directory = lib_directory + repo["name"] - - for patch in patches: - try: - os.chdir(lib_directory) - except FileNotFoundError: - os.mkdir(lib_directory) - os.chdir(lib_directory) - - try: - git.clone(repo["url"]) - except sh.ErrorReturnCode_128 as Err: - if b"already exists" in Err.stderr: - pass - else: - raise RuntimeError(Err.stderr) - os.chdir(repo_directory) - - patch_filepath = patch_directory + patch - - try: - check_flags = ["--check"] - if use_apply: - for flag in flags: - if not flag in ("--apply", "--signoff"): - check_flags.append(flag) - git.apply(check_flags, patch_filepath) - run_apply = True - except sh.ErrorReturnCode_1 as Err: - run_apply = False - if (b"error" not in Err.stderr or - b"patch does not apply" in Err.stderr): - parse_err = Err.stderr.decode() - parse_err = parse_err[parse_err.rfind(":")+1:-1] - print( - " . Skipping {}:{}".format(repo["name"], parse_err) - ) - skipped += 1 - else: - failed += 1 - error_str = str(Err.stderr, encoding="utf-8").replace("\n", " ") - error_start = error_str.rfind("error:") + 7 - check_errors.append(dict(repo_name=repo["name"], - patch_name=patch, error=error_str[error_start:])) - - except sh.ErrorReturnCode as Err: - run_apply = False - failed += 1 - error_str = str(Err.stderr, encoding="utf-8").replace("\n", " ") - error_start = error_str.rfind("error:") + 7 - check_errors.append(dict(repo_name=repo["name"], - patch_name=patch, error=error_str[error_start:])) - - if run_apply and not dry_run: - result = apply_patch(repo_directory, patch_filepath, repo["name"], - patch, flags, use_apply) - if result: - applied += 1 - else: - failed += 1 - elif run_apply and dry_run: - applied += 1 - - return [applied, skipped, failed] - -if __name__ == "__main__": - cli_args = cli_parser.parse_args() - use_apply = cli_args.use_apply - dry_run = cli_args.dry_run - run_local = cli_args.run_local - if run_local: - if dry_run or cli_args.list: - pass - else: - raise RuntimeError("'--local' can only be used in conjunction with" - " '--dry-run' or '--list'.") - - run_patches = get_patches(run_local) - flags = ["--signoff"] - - if cli_args.list: - print("Available Patches:", run_patches) - sys.exit() - if cli_args.patch: - if not cli_args.patch in run_patches: - raise ValueError("'{}' is not an available patchfile.".format(cli_args.patch)) - run_patches = [cli_args.patch] - if not cli_args.flags == None: - if not cli_args.patch: - raise RuntimeError("Must be used with a single patch. See help (-h) for usage.") - if "[-i]" in cli_args.flags: - raise ValueError("Interactive Mode flag not allowed.") - for flag in cli_args.flags: - if not flag == "[--signoff]": - flags.append(flag.strip("[]")) - if cli_args.use_apply: - if not cli_args.patch: - raise RuntimeError("Must be used with a single patch. See help (-h) for usage.") - - print(".... Beginning Patch Updates ....") - print(".... Working directory:", working_directory) - print(".... Library directory:", lib_directory) - print(".... Patches directory:", patch_directory) - - check_errors = [] - apply_errors = [] - stats = [0, 0, 0] - - print(".... Deleting any previously cloned libraries") - try: - libs = os.listdir(path=lib_directory) - for lib in libs: - shutil.rmtree(lib_directory + lib) - except FileNotFoundError: - pass - - repos = get_repo_list() - print(".... Running Patch Checks On", len(repos), "Repos ....") - - for repo in repos: - results = check_patches(repo, run_patches, flags, use_apply, dry_run) - for k in range(3): - stats[k] += results[k] - - print(".... Patch Updates Completed ....") - print(".... Patches Applied:", stats[0]) - print(".... Patches Skipped:", stats[1]) - print(".... Patches Failed:", stats[2], "\n") - print(".... Patch Check Failure Report ....") - if len(check_errors) > 0: - for error in check_errors: - print(">> Repo: {0}\tPatch: {1}\n Error: {2}".format(error["repo_name"], - error["patch_name"], error["error"])) - else: - print("No Failures") - print("\n") - print(".... Patch Apply Failure Report ....") - if len(apply_errors) > 0: - for error in apply_errors: - print(">> Repo: {0}\tPatch: {1}\n Error: {2}".format(error["repo_name"], - error["patch_name"], error["error"])) - else: - print("No Failures") +# The MIT License (MIT) +# +# Copyright (c) 2019 Michael Schroeder +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +"""Adabot utility for applying patches to all CircuitPython Libraries.""" + +import argparse +import os +import shutil +import sys + +import requests +import sh +from sh.contrib import git + +from adabot.lib import common_funcs + + +working_directory = os.path.abspath(os.getcwd()) +lib_directory = working_directory + "/.libraries/" +patch_directory = working_directory + "/patches/" +repos = [] +check_errors = [] +apply_errors = [] +stats = [] + +""" +Setup the command line argument parsing object. +""" +cli_parser = argparse.ArgumentParser( + description="Apply patches to any common file(s) in" + " all Adafruit CircuitPython Libraries." +) +cli_parser.add_argument( + "-l", "--list", help="Lists the available patches to run.", action="store_true" +) +cli_parser.add_argument( + "-p", + help="Runs only the single patch referenced.", + metavar="", + dest="patch", +) +cli_parser.add_argument( + "-f", + help="Adds the referenced FLAGS to the git.am call." + " Only available when using '-p'. Enclose flags in brackets '[]'." + " Multiple flags can be passed. NOTE: '--signoff' is already used " + " used by default, and will be ignored. EXAMPLE: -f [-C0] -f [-s]", + metavar="FLAGS", + action="append", + dest="flags", + type=str, +) +cli_parser.add_argument( + "--use-apply", + help="Forces use of 'git apply' instead of 'git am'." + " This is necessary when needing to use 'apply' flags not available" + " to 'am' (e.g. '--unidiff-zero'). Only available when using '-p'.", + action="store_true", + dest="use_apply", +) +cli_parser.add_argument( + "--dry-run", + help="Accomplishes a dry run of patches, without applying" " them.", + action="store_true", + dest="dry_run", +) +cli_parser.add_argument( + "--local", + help="Force use of local patches. This skips verification" + " of patch files in the adabot GitHub repository. MUST use '--dry-run'" + " with this argument; this guards against applying unapproved patches.", + action="store_true", + dest="run_local", +) + + +def get_repo_list(): + """Uses adabot.circuitpython_libraries module to get a list of + CircuitPython repositories. Filters the list down to adafruit + owned/sponsored CircuitPython libraries. + """ + repo_list = [] + get_repos = common_funcs.list_repos() + for repo in get_repos: + if not ( + repo["owner"]["login"] == "adafruit" + and repo["name"].startswith("Adafruit_CircuitPython") + ): + continue + repo_list.append(dict(name=repo["name"], url=repo["clone_url"])) + + return repo_list + + +def get_patches(run_local): + """Returns the list of patch files located in the adabot/patches + directory. + """ + return_list = [] + if not run_local: + contents = requests.get( + "https://api.github.com/repos/adafruit/adabot/contents/patches" + ) + if contents.ok: + for patch in contents.json(): + patch_name = patch["name"] + return_list.append(patch_name) + else: + contents = os.listdir(patch_directory) + for file in contents: + if file.endswith(".patch"): + return_list.append(file) + + return return_list + +# pylint: disable=too-many-arguments +def apply_patch(repo_directory, patch_filepath, repo, patch, flags, use_apply): + """Apply the `patch` in `patch_filepath` to the `repo` in + `repo_directory` using git am or git apply. The commit + with the user running the script (adabot if credentials are set + for that). + + When `use_apply` is true, the `--apply` flag is automatically added + to ensure that any passed flags that turn off apply (e.g. `--check`) + are overridden. + """ + if not os.getcwd() == repo_directory: + os.chdir(repo_directory) + + if not use_apply: + try: + git.am(flags, patch_filepath) + except sh.ErrorReturnCode as err: + apply_errors.append( + dict(repo_name=repo, patch_name=patch, error=err.stderr) + ) + return False + else: + apply_flags = ["--apply"] + for flag in flags: + if not flag == "--signoff": + apply_flags.append(flag) + try: + git.apply(apply_flags, patch_filepath) + except sh.ErrorReturnCode as err: + apply_errors.append( + dict(repo_name=repo, patch_name=patch, error=err.stderr) + ) + return False + + with open(patch_filepath) as patchfile: + for line in patchfile: + if "[PATCH]" in line: + message = '"' + line[(line.find("]") + 2) :] + '"' + break + try: + git.commit("-a", "-m", message) + except sh.ErrorReturnCode as err: + apply_errors.append( + dict(repo_name=repo, patch_name=patch, error=err.stderr) + ) + return False + + try: + git.push() + except sh.ErrorReturnCode as err: + apply_errors.append(dict(repo_name=repo, patch_name=patch, error=err.stderr)) + return False + return True + + +# pylint: disable=too-many-locals,too-many-branches,too-many-statements +def check_patches(repo, patches, flags, use_apply, dry_run): + """Gather a list of patches from the `adabot/patches` directory + on the adabot repo. Clone the `repo` and run git apply --check + to test wether it requires any of the gathered patches. + + When `use_apply` is true, any flags except `--apply` are passed + through to the check call. This ensures that the check call is + representative of the actual apply call. + """ + applied = 0 + skipped = 0 + failed = 0 + + repo_directory = lib_directory + repo["name"] + + for patch in patches: + try: + os.chdir(lib_directory) + except FileNotFoundError: + os.mkdir(lib_directory) + os.chdir(lib_directory) + + try: + git.clone(repo["url"]) + except sh.ErrorReturnCode_128 as err: + if b"already exists" in err.stderr: + pass + else: + raise RuntimeError(err.stderr) from None + os.chdir(repo_directory) + + patch_filepath = patch_directory + patch + + try: + check_flags = ["--check"] + if use_apply: + for flag in flags: + if not flag in ("--apply", "--signoff"): + check_flags.append(flag) + git.apply(check_flags, patch_filepath) + run_apply = True + except sh.ErrorReturnCode_1 as err: + run_apply = False + if b"error" not in err.stderr or b"patch does not apply" in err.stderr: + parse_err = err.stderr.decode() + parse_err = parse_err[parse_err.rfind(":") + 1 : -1] + print(" . Skipping {}:{}".format(repo["name"], parse_err)) + skipped += 1 + else: + failed += 1 + error_str = str(err.stderr, encoding="utf-8").replace("\n", " ") + error_start = error_str.rfind("error:") + 7 + check_errors.append( + dict( + repo_name=repo["name"], + patch_name=patch, + error=error_str[error_start:], + ) + ) + + except sh.ErrorReturnCode as err: + run_apply = False + failed += 1 + error_str = str(err.stderr, encoding="utf-8").replace("\n", " ") + error_start = error_str.rfind("error:") + 7 + check_errors.append( + dict( + repo_name=repo["name"], + patch_name=patch, + error=error_str[error_start:], + ) + ) + + if run_apply and not dry_run: + result = apply_patch( + repo_directory, patch_filepath, repo["name"], patch, flags, use_apply + ) + if result: + applied += 1 + else: + failed += 1 + elif run_apply and dry_run: + applied += 1 + + return [applied, skipped, failed] + + +if __name__ == "__main__": + cli_args = cli_parser.parse_args() + if cli_args.run_local: + if cli_args.dry_run or cli_args.list: + pass + else: + raise RuntimeError( + "'--local' can only be used in conjunction with" + " '--dry-run' or '--list'." + ) + + run_patches = get_patches(cli_args.run_local) + cmd_flags = ["--signoff"] + + if cli_args.list: + print("Available Patches:", run_patches) + sys.exit() + if cli_args.patch: + if not cli_args.patch in run_patches: + raise ValueError( + "'{}' is not an available patchfile.".format(cli_args.patch) + ) + run_patches = [cli_args.patch] + if cli_args.flags is not None: + if not cli_args.patch: + raise RuntimeError( + "Must be used with a single patch. See help (-h) for usage." + ) + if "[-i]" in cli_args.flags: + raise ValueError("Interactive Mode flag not allowed.") + for flag_arg in cli_args.flags: + if not flag_arg == "[--signoff]": + cmd_flags.append(flag_arg.strip("[]")) + if cli_args.use_apply: + if not cli_args.patch: + raise RuntimeError( + "Must be used with a single patch. See help (-h) for usage." + ) + + print(".... Beginning Patch Updates ....") + print(".... Working directory:", working_directory) + print(".... Library directory:", lib_directory) + print(".... Patches directory:", patch_directory) + + check_errors = [] + apply_errors = [] + stats = [0, 0, 0] + + print(".... Deleting any previously cloned libraries") + try: + libs = os.listdir(path=lib_directory) + for lib in libs: + shutil.rmtree(lib_directory + lib) + except FileNotFoundError: + pass + + repos = get_repo_list() + print(".... Running Patch Checks On", len(repos), "Repos ....") + + for repository in repos: + results = check_patches( + repository, + run_patches, + cmd_flags, + cli_args.use_apply, + cli_args.dry_run + ) + for k in range(3): + stats[k] += results[k] + + print(".... Patch Updates Completed ....") + print(".... Patches Applied:", stats[0]) + print(".... Patches Skipped:", stats[1]) + print(".... Patches Failed:", stats[2], "\n") + print(".... Patch Check Failure Report ....") + if len(check_errors) > 0: + for error in check_errors: + print( + ">> Repo: {0}\tPatch: {1}\n Error: {2}".format( + error["repo_name"], error["patch_name"], error["error"] + ) + ) + else: + print("No Failures") + print("\n") + print(".... Patch Apply Failure Report ....") + if len(apply_errors) > 0: + for error in apply_errors: + print( + ">> Repo: {0}\tPatch: {1}\n Error: {2}".format( + error["repo_name"], error["patch_name"], error["error"] + ) + ) + else: + print("No Failures") diff --git a/adabot/github_requests.py b/adabot/github_requests.py index e13f267..9e0ad73 100644 --- a/adabot/github_requests.py +++ b/adabot/github_requests.py @@ -19,33 +19,31 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -""" -`adafruit_adabot` -==================================================== -TODO(description) - -* Author(s): Scott Shawcroft -""" +"""Wrapper for GitHub requests.""" +from base64 import b64encode import datetime import functools import logging import os -import requests import time import traceback -from base64 import b64encode - +import requests import requests_cache TIMEOUT = 60 def setup_cache(expire_after=7200): - requests_cache.install_cache(cache_name='github_cache', backend='sqlite', expire_after=expire_after, - allowable_codes=(200, 404)) + """Sets up a cache for requests.""" + requests_cache.install_cache( + cache_name="github_cache", + backend="sqlite", + expire_after=expire_after, + allowable_codes=(200, 404), + ) def _fix_url(url): @@ -55,7 +53,10 @@ def _fix_url(url): def _fix_kwargs(kwargs): - api_version = "application/vnd.github.scarlet-witch-preview+json;application/vnd.github.hellcat-preview+json" + api_version = ( + "application/vnd.github.scarlet-witch-preview+json;" + "application/vnd.github.hellcat-preview+json" + ) if "headers" in kwargs: if "Accept" in kwargs["headers"]: kwargs["headers"]["Accept"] += ";" + api_version @@ -75,27 +76,42 @@ def _fix_kwargs(kwargs): def request(method, url, **kwargs): + """Processes request for `url`.""" try: - response = getattr(requests, method)(_fix_url(url), timeout=TIMEOUT, **_fix_kwargs(kwargs)) - from_cache = getattr(response, 'from_cache', False) - remaining = int(response.headers.get('X-RateLimit-Remaining', 0)) + response = getattr(requests, method)( + _fix_url(url), timeout=TIMEOUT, **_fix_kwargs(kwargs) + ) + from_cache = getattr(response, "from_cache", False) + remaining = int(response.headers.get("X-RateLimit-Remaining", 0)) logging.debug( - f"GET {url} {'(cache)' if from_cache else '(%d remaining)' % remaining} status={response.status_code}") - except Exception as e: + "GET %s %s status=%s", + url, + f"{'(cache)' if from_cache else '(%d remaining)' % remaining}", + response.status_code, + ) + except requests.RequestException: exception_text = traceback.format_exc() if "ADABOT_GITHUB_ACCESS_TOKEN" in os.environ: - exception_text = exception_text.replace(os.environ["ADABOT_GITHUB_ACCESS_TOKEN"], "[secure]") - logging.critical(exception_text) - raise RuntimeError("See log for error text that has been sanitized for secrets") + exception_text = exception_text.replace( + os.environ["ADABOT_GITHUB_ACCESS_TOKEN"], "[secure]" + ) + logging.critical("%s", exception_text) + raise RuntimeError( + "See log for error text that has been sanitized for secrets" + ) from None if not from_cache and remaining <= 1: - rate_limit_reset = datetime.datetime.fromtimestamp(int(response.headers["X-RateLimit-Reset"])) - logging.warning("GitHub API Rate Limit reached. Pausing until Rate Limit reset.") + rate_limit_reset = datetime.datetime.fromtimestamp( + int(response.headers["X-RateLimit-Reset"]) + ) + logging.warning( + "GitHub API Rate Limit reached. Pausing until Rate Limit reset." + ) while datetime.datetime.now() < rate_limit_reset: - logging.warning("Rate Limit will reset at: {}".format(rate_limit_reset)) + logging.warning("Rate Limit will reset at: %s", rate_limit_reset) reset_diff = rate_limit_reset - datetime.datetime.now() - logging.info("Sleeping {} seconds".format(reset_diff.seconds)) + logging.info("Sleeping %s seconds", reset_diff.seconds) time.sleep(reset_diff.seconds + 1) if remaining % 100 == 0: @@ -104,9 +120,8 @@ def request(method, url, **kwargs): return response -get = functools.partial(request, 'get') -post = functools.partial(request, 'post') -put = functools.partial(request, 'put') -delete = functools.partial(request, 'delete') -patch = functools.partial(request, 'patch') - +get = functools.partial(request, "get") +post = functools.partial(request, "post") +put = functools.partial(request, "put") +delete = functools.partial(request, "delete") +patch = functools.partial(request, "patch") diff --git a/adabot/lib/assign_hacktober_label.py b/adabot/lib/assign_hacktober_label.py index 54dd7d2..117f2af 100644 --- a/adabot/lib/assign_hacktober_label.py +++ b/adabot/lib/assign_hacktober_label.py @@ -20,6 +20,11 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +"""An utility to automatically apply the 'hacktoberfest' label to open issues +marked as 'good first issue', during DigitalOcean's/GitHub's Hacktoberfest +event. +""" + import argparse import datetime import requests @@ -28,9 +33,19 @@ from adabot.lib import common_funcs cli_args = argparse.ArgumentParser(description="Hacktoberfest Label Assigner") -cli_args.add_argument("-r", "--remove-label", action="store_true", - help="Option to remove Hacktoberfest labels, instead of adding them.", - dest="remove_label") +cli_args.add_argument( + "-r", + "--remove-label", + action="store_true", + help="Option to remove Hacktoberfest labels, instead of adding them.", + dest="remove_labels", +) +cli_args.add_argument( + "--dry-run", + action="store_true", + help="Option to remove Hacktoberfest labels, instead of adding them.", + dest="dry_run", +) # Hacktoberfest Season @@ -39,69 +54,57 @@ _ADD_SEASON = [(9, 29), (10, 30)] _REMOVE_SEASON = [(11, 1), (11, 10)] + def is_hacktober_season(): - """ Checks if the current day falls within either the add range (_ADD_SEASON) - or the remove range (_REMOVE_SEASON). Returns boolean if within - Hacktoberfest season, and which action to take. + """Checks if the current day falls within either the add range (_ADD_SEASON) + or the remove range (_REMOVE_SEASON). Returns boolean if within + Hacktoberfest season, and which action to take. """ today = datetime.date.today() - add_range = [ - datetime.date(today.year, *month_day) for month_day in _ADD_SEASON - ] + add_range = [datetime.date(today.year, *month_day) for month_day in _ADD_SEASON] remove_range = [ datetime.date(today.year, *month_day) for month_day in _REMOVE_SEASON ] if add_range[0] <= today <= add_range[1]: return True, "add" - elif remove_range[0] <= today <= remove_range[1]: + if remove_range[0] <= today <= remove_range[1]: return True, "remove" return False, None def get_open_issues(repo): - """ Retrieve all open issues for given repo. - """ + """Retrieve all open issues for given repo.""" params = { "state": "open", } response = github.get("/repos/" + repo["full_name"] + "/issues", params=params) if not response.ok: - print("Failed to retrieve issues for '{}'".format(repo["name"])) + print(f"Failed to retrieve issues for '{repo['name']}'") return False issues = [] while response.ok: - issues.extend([issue for issue in response.json() if "pull_request" not in issue]) + issues.extend( + [issue for issue in response.json() if "pull_request" not in issue] + ) - try: - links = response.headers["Link"] - except KeyError: - break - next_url = None - for link in links.split(","): - link, rel = link.split(";") - link = link.strip(" <>") - rel = rel.strip() - if rel == "rel=\"next\"": - next_url = link - break - if not next_url: + if response.links.get("next"): + response = requests.get(response.links["next"]["url"]) + else: break - response = requests.get(link, timeout=30) - return issues -def ensure_hacktober_label_exists(repo): - """ Checks if the 'Hacktoberfest' label exists on the repo. - If not, creates the label. +def ensure_hacktober_label_exists(repo, dry_run=False): + """Checks if the 'Hacktoberfest' label exists on the repo. + If not, creates the label. """ - response = github.get("/repos/" + repo["full_name"] + "/labels") + response = github.get(f"/repos/{repo['full_name']}/labels") if not response.ok: - print("Failed to retrieve labels for '{}'".format(repo["name"])) + print(f"Failed to retrieve labels for '{repo['name']}'") return False repo_labels = [label["name"] for label in response.json()] @@ -111,18 +114,20 @@ def ensure_hacktober_label_exists(repo): params = { "name": "Hacktoberfest", "color": "f2b36f", - "description": "DigitalOcean's Hacktoberfest" + "description": "DigitalOcean's Hacktoberfest", } - result = github.post("/repos/" + repo["full_name"] + "/labels", json=params) - if not result.status_code == 201: - print("Failed to create new Hacktoberfest label for: {}".format(repo["name"])) - return False + if not dry_run: + result = github.post(f"/repos/{repo['full_name']}/labels", json=params) + if not result.status_code == 201: + print(f"Failed to create new Hacktoberfest label for: {repo['name']}") + return False return True -def assign_hacktoberfest(repo, issues=None, remove_labels=False): - """ Gathers open issues on a repo, and assigns the 'Hacktoberfest' label - to each issue if its not already assigned. + +def assign_hacktoberfest(repo, issues=None, remove_labels=False, dry_run=False): + """Gathers open issues on a repo, and assigns the 'Hacktoberfest' label + to each issue if its not already assigned. """ labels_changed = 0 @@ -138,58 +143,59 @@ def assign_hacktoberfest(repo, issues=None, remove_labels=False): if remove_labels: if has_hacktober: label_names = [ - label for label in label_names - if label not in has_hacktober + label for label in label_names if label not in has_hacktober ] update_issue = True else: if has_good_first and not has_hacktober: - label_exists = ensure_hacktober_label_exists(repo) + label_exists = ensure_hacktober_label_exists(repo, dry_run) if not label_exists: continue update_issue = True if update_issue: - params = { - "labels": label_names - } - result = github.patch("/repos/" - + repo["full_name"] - + "/issues/" - + str(issue["number"]), - json=params) - - if result.ok: - labels_changed += 1 + params = {"labels": label_names} + if not dry_run: + result = github.patch( + f"/repos/{repo['full_name']}/issues/{str(issue['number'])}", + json=params, + ) + + if result.ok: + labels_changed += 1 + else: + # sadly, GitHub will only silently ignore labels that are + # not added and return a 200. so this will most likely only + # trigger on endpoint/connection failures. + print(f"Failed to add Hacktoberfest label to: {issue['url']}") else: - # sadly, GitHub will only silently ignore labels that are - # not added and return a 200. so this will most likely only - # trigger on endpoint/connection failures. - print("Failed to add Hacktoberfest label to: {}".format(issue["url"])) + labels_changed += 1 return labels_changed -def process_hacktoberfest(repo, issues=None, remove_labels=False): - result = assign_hacktoberfest(repo, issues, remove_labels) + +def process_hacktoberfest(repo, issues=None, remove_labels=False, dry_run=False): + """Run hacktoberfest functions and return the result.""" + result = assign_hacktoberfest(repo, issues, remove_labels, dry_run) return result if __name__ == "__main__": - labels_assigned = 0 + LABELS_ASSIGNED = 0 args = cli_args.parse_args() - remove_labels = args.remove_label - - if not remove_labels: + if not args.remove_labels: print("Checking for open issues to assign the Hacktoberfest label to...") else: print("Checking for open issues to remove the Hacktoberfest label from...") repos = common_funcs.list_repos() - for repo in repos: - labels_assigned += process_hacktoberfest(repo, remove_labels) + for repository in repos: + LABELS_ASSIGNED += process_hacktoberfest( + repository, remove_labels=args.remove_labels, dry_run=args.dry_run + ) - if not remove_labels: - print("Added the Hacktoberfest label to {} issues.".format(labels_assigned)) + if not args.remove_labels: + print(f"Added the Hacktoberfest label to {LABELS_ASSIGNED} issues.") else: - print("Removed the Hacktoberfest label from {} issues.".format(labels_assigned)) + print(f"Removed the Hacktoberfest label from {LABELS_ASSIGNED} issues.") diff --git a/adabot/lib/blinka_funcs.py b/adabot/lib/blinka_funcs.py index bfd38a9..1ae1d01 100644 --- a/adabot/lib/blinka_funcs.py +++ b/adabot/lib/blinka_funcs.py @@ -20,17 +20,20 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +"""Common functions used with Adabot & Blinka interactions.""" + from adabot import github_requests as github + def board_count(): - """ Retrieve the number of boards currently supported by Adafruit_Blinka, - via the count of files in circuitpython-org/_blinka. + """Retrieve the number of boards currently supported by Adafruit_Blinka, + via the count of files in circuitpython-org/_blinka. """ - board_count = 0 - cirpy_org_url = '/repos/adafruit/circuitpython-org/contents/_blinka' + count = 0 + cirpy_org_url = "/repos/adafruit/circuitpython-org/contents/_blinka" response = github.get(cirpy_org_url) if response.ok: response_json = response.json() - board_count = len(response_json) + count = len(response_json) - return board_count + return count diff --git a/adabot/lib/circuitpython_library_validators.py b/adabot/lib/circuitpython_library_validators.py index fab0a9e..42accd1 100644 --- a/adabot/lib/circuitpython_library_validators.py +++ b/adabot/lib/circuitpython_library_validators.py @@ -1,3 +1,4 @@ +# pylint: disable=no-self-use # The MIT License (MIT) # # Copyright (c) 2017 Scott Shawcroft for Adafruit Industries @@ -19,40 +20,48 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. + +"""Collection of validator methods to maintain a standard, as well as detect +errors, across the entire CirtuitPython library ecosystem.""" + import datetime +from io import StringIO import json import logging import pathlib import re -from io import StringIO from tempfile import TemporaryDirectory -import requests +from packaging.version import parse as pkg_version_parse +from packaging.requirements import Requirement, InvalidRequirement -import sh from pylint import lint from pylint.reporters import JSONReporter + +import requests + +import sh from sh.contrib import git + import yaml from adabot import github_requests as github -from adabot import pypi_requests as pypi from adabot.lib import common_funcs from adabot.lib import assign_hacktober_label as hacktober -from packaging.version import parse as pkg_version_parse -from packaging.requirements import Requirement as pkg_Requirement - class CapturedJsonReporter(JSONReporter): + """Helper class to stringify PyLint JSON reports.""" def __init__(self): self._stringio = StringIO() super().__init__(self._stringio) def get_result(self): + """The current value.""" return self._stringio.getvalue() + # Define constants for error strings to make checking against them more robust: ERROR_README_DOWNLOAD_FAILED = "Failed to download README" ERROR_README_IMAGE_MISSING_ALT = "README image missing alt text" @@ -60,29 +69,43 @@ def get_result(self): ERROR_README_MISSING_DISCORD_BADGE = "README missing Discord badge" ERROR_README_MISSING_RTD_BADGE = "README missing ReadTheDocs badge" ERROR_README_MISSING_CI_BADGE = "README missing CI badge" -ERROR_README_MISSING_CI_ACTIONS_BADGE = "README CI badge needs to be changed" \ -" to GitHub Actions" +ERROR_README_MISSING_CI_ACTIONS_BADGE = ( + "README CI badge needs to be changed to GitHub Actions" +) ERROR_PYFILE_DOWNLOAD_FAILED = "Failed to download .py code file" -ERROR_PYFILE_MISSING_STRUCT = ".py file contains reference to import ustruct" \ -" without reference to import struct. See issue " \ -"https://github.com/adafruit/circuitpython/issues/782" -ERROR_PYFILE_MISSING_RE = ".py file contains reference to import ure" \ -" without reference to import re. See issue " \ -"https://github.com/adafruit/circuitpython/issues/1582" -ERROR_PYFILE_MISSING_JSON = ".py file contains reference to import ujson" \ -" without reference to import json. See issue " \ -"https://github.com/adafruit/circuitpython/issues/1582" -ERROR_PYFILE_MISSING_ERRNO = ".py file contains reference to import uerrno" \ -" without reference to import errno. See issue " \ -"https://github.com/adafruit/circuitpython/issues/1582" +ERROR_PYFILE_MISSING_STRUCT = ( + ".py file contains reference to import ustruct" + " without reference to import struct. See issue " + "https://github.com/adafruit/circuitpython/issues/782" +) +ERROR_PYFILE_MISSING_RE = ( + ".py file contains reference to import ure" + " without reference to import re. See issue " + "https://github.com/adafruit/circuitpython/issues/1582" +) +ERROR_PYFILE_MISSING_JSON = ( + ".py file contains reference to import ujson" + " without reference to import json. See issue " + "https://github.com/adafruit/circuitpython/issues/1582" +) +ERROR_PYFILE_MISSING_ERRNO = ( + ".py file contains reference to import uerrno" + " without reference to import errno. See issue " + "https://github.com/adafruit/circuitpython/issues/1582" +) ERROR_MISMATCHED_READTHEDOCS = "Mismatched readthedocs.yml" ERROR_MISSING_DESCRIPTION = "Missing repository description" ERROR_MISSING_EXAMPLE_FILES = "Missing .py files in examples folder" ERROR_MISSING_EXAMPLE_FOLDER = "Missing examples folder" ERROR_EXAMPLE_MISSING_SENSORNAME = "Example file(s) missing sensor/library name" ERROR_MISSING_EXAMPLE_SIMPLETEST = "Missing simpletest example." -ERROR_MISSING_STANDARD_LABELS = "Missing one or more standard issue labels (bug, documentation, enhancement, good first issue)." -ERROR_MISSING_LIBRARIANS = "CircuitPythonLibrarians team missing or does not have write access" +ERROR_MISSING_STANDARD_LABELS = ( + "Missing one or more standard issue labels" + " (bug, documentation, enhancement, good first issue)." +) +ERROR_MISSING_LIBRARIANS = ( + "CircuitPythonLibrarians team missing or does not have write access" +) ERROR_MISSING_LICENSE = "Missing license." ERROR_MISSING_LINT = "Missing lint config" ERROR_MISSING_CODE_OF_CONDUCT = "Missing CODE_OF_CONDUCT.md" @@ -90,7 +113,9 @@ def get_result(self): ERROR_MISSING_READTHEDOCS = "Missing readthedocs.yml" ERROR_MISSING_SETUP_PY = "For pypi compatibility, missing setup.py" ERROR_MISSING_REQUIREMENTS_TXT = "For pypi compatibility, missing requirements.txt" -ERROR_MISSING_BLINKA = "For pypi compatibility, missing Adafruit-Blinka in requirements.txt" +ERROR_MISSING_BLINKA = ( + "For pypi compatibility, missing Adafruit-Blinka in requirements.txt" +) ERROR_NOT_IN_BUNDLE = "Not in bundle." ERROR_INCORRECT_DEFAULT_BRANCH = "Default branch is not main" ERROR_UNABLE_PULL_REPO_CONTENTS = "Unable to pull repo contents" @@ -105,16 +130,29 @@ def get_result(self): ERROR_RTD_FAILED_TO_LOAD_BUILDS = "Unable to load builds webpage" ERROR_RTD_FAILED_TO_LOAD_BUILD_INFO = "Failed to load build info" ERROR_RTD_OUTPUT_HAS_WARNINGS = "ReadTheDocs latest build has warnings and/or errors" -ERROR_RTD_AUTODOC_FAILED = "Autodoc failed on ReadTheDocs. (Likely need to automock an import.)" +ERROR_RTD_AUTODOC_FAILED = ( + "Autodoc failed on ReadTheDocs. (Likely need to automock an import.)" +) ERROR_RTD_SPHINX_FAILED = "Sphinx missing files" ERROR_GITHUB_RELEASE_FAILED = "Failed to fetch latest release from GitHub" ERROR_GITHUB_NO_RELEASE = "Library repository has no releases" -ERROR_GITHUB_COMMITS_SINCE_LAST_RELEASE_GTM = "Library has new commits since last release over a month ago" -ERROR_GITHUB_COMMITS_SINCE_LAST_RELEASE_1M = "Library has new commits since last release within the last month" -ERROR_GITHUB_COMMITS_SINCE_LAST_RELEASE_1W = "Library has new commits since last release within the last week" +ERROR_GITHUB_COMMITS_SINCE_LAST_RELEASE_GTM = ( + "Library has new commits since last release over a month ago" +) +ERROR_GITHUB_COMMITS_SINCE_LAST_RELEASE_1M = ( + "Library has new commits since last release within the last month" +) +ERROR_GITHUB_COMMITS_SINCE_LAST_RELEASE_1W = ( + "Library has new commits since last release within the last week" +) ERROR_GITHUB_FAILING_ACTIONS = "The most recent GitHub Actions run has failed" -ERROR_RTD_MISSING_LATEST_RELEASE = "ReadTheDocs missing the latest release. (Ignore me! RTD doesn't update when a new version is released. Only on pushes.)" -ERROR_DRIVERS_PAGE_DOWNLOAD_FAILED = "Failed to download drivers page from CircuitPython docs" +ERROR_RTD_MISSING_LATEST_RELEASE = ( + "ReadTheDocs missing the latest release. (Ignore me! RTD doesn't update when a new version is" + " released. Only on pushes.)" +) +ERROR_DRIVERS_PAGE_DOWNLOAD_FAILED = ( + "Failed to download drivers page from CircuitPython docs" +) ERROR_DRIVERS_PAGE_DOWNLOAD_MISSING_DRIVER = "CircuitPython drivers page missing driver" ERROR_UNABLE_PULL_REPO_DIR = "Unable to pull repository directory" ERROR_UNABLE_PULL_REPO_EXAMPLES = "Unable to pull repository examples files" @@ -125,7 +163,9 @@ def get_result(self): ERROR_NEW_REPO_IN_WORK = "New repo(s) currently in work, and unreleased" # Temp category for GitHub Actions migration. -ERROR_NEEDS_ACTION_MIGRATION = "Repo(s) need to be migrated from TravisCI to GitHub Actions" +ERROR_NEEDS_ACTION_MIGRATION = ( + "Repo(s) need to be migrated from TravisCI to GitHub Actions" +) # Since this has been refactored into a separate module, the connection to 'output_handler()' # and associated 'file_data' list is broken. To keep from possibly having conflicted @@ -135,7 +175,10 @@ def get_result(self): ERROR_OUTPUT_HANDLER = "A programmatic error occurred" # These are warnings or errors that sphinx generate that we're ok ignoring. -RTD_IGNORE_NOTICES = ("WARNING: html_static_path entry", "WARNING: nonlocal image URI found:") +RTD_IGNORE_NOTICES = ( + "WARNING: html_static_path entry", + "WARNING: nonlocal image URI found:", +) # Constant for bundle repo name. BUNDLE_REPO_NAME = "Adafruit_CircuitPython_Bundle" @@ -159,33 +202,21 @@ def get_result(self): ] STD_REPO_LABELS = { - "bug": { - "color": "ee0701" - }, - "documentation": { - "color": "d4c5f9" - }, - "enhancement": { - "color": "84b6eb" - }, - "good first issue": { - "color": "7057ff" - } + "bug": {"color": "ee0701"}, + "documentation": {"color": "d4c5f9"}, + "enhancement": {"color": "84b6eb"}, + "good first issue": {"color": "7057ff"}, } -# Cache CircuitPython's subprojects on ReadTheDocs so its not fetched every repo check. -rtd_subprojects = None - -# Cache the CircuitPython driver page so we can make sure every driver is linked to. -core_driver_page = None - -class library_validator(): - """ Class to hold instance variables needed to traverse the calling - code, and the validator functions. +class LibraryValidator: + """Class to hold instance variables needed to traverse the calling + code, and the validator functions. """ - def __init__(self, validators, bundle_submodules, latest_pylint, keep_repos=False, **kw_args): + def __init__( + self, validators, bundle_submodules, latest_pylint, keep_repos=False, **kw_args + ): self.validators = validators self.bundle_submodules = bundle_submodules self.latest_pylint = pkg_version_parse(latest_pylint) @@ -194,12 +225,13 @@ def __init__(self, validators, bundle_submodules, latest_pylint, keep_repos=Fals self.validate_contents_quiet = kw_args.get("validate_contents_quiet", False) self.has_setup_py_disabled = set() self.keep_repos = keep_repos - + self.rtd_subprojects = None + self.core_driver_page = None @property def rtd_yml_base(self): - """ The parsed YAML from `.readthedocs.yml` in the cookiecutter-adafruit-circuitpython repo. - Used to verify that a library's `.readthedocs.yml` matches this version. + """The parsed YAML from `.readthedocs.yml` in the cookiecutter-adafruit-circuitpython repo. + Used to verify that a library's `.readthedocs.yml` matches this version. """ if self._rtd_yaml_base is None: rtd_yml_dl_url = ( @@ -217,16 +249,12 @@ def rtd_yml_base(self): try: self._rtd_yaml_base = yaml.safe_load(rtd_yml.text) except yaml.YAMLError: - print( - "Error parsing cookiecutter .readthedocs.yml." - ) + print("Error parsing cookiecutter .readthedocs.yml.") self._rtd_yaml_base = "" else: - print( - "Error retrieving cookiecutter .readthedocs.yml" - ) + print("Error retrieving cookiecutter .readthedocs.yml") self._rtd_yaml_base = "" - + return self._rtd_yaml_base def run_repo_validation(self, repo): @@ -243,8 +271,10 @@ def validate_repo_state(self, repo): a dictionary with a GitHub API repository state (like from the list_repos function). Returns a list of string error messages for the repository. """ - if not (repo["owner"]["login"] == "adafruit" and - repo["name"].startswith("Adafruit_CircuitPython")): + if not ( + repo["owner"]["login"] == "adafruit" + and repo["name"].startswith("Adafruit_CircuitPython") + ): return [] search_keys = { @@ -276,23 +306,24 @@ def validate_repo_state(self, repo): if repo_fields.get("has_wiki"): errors.append(ERROR_WIKI_DISABLED) - if (not repo_fields.get("license") and - not repo["name"] in BUNDLE_IGNORE_LIST): - errors.append(ERROR_MISSING_LICENSE) + if not repo_fields.get("license") and not repo["name"] in BUNDLE_IGNORE_LIST: + errors.append(ERROR_MISSING_LICENSE) if not repo_fields.get("permissions", {}).get("push"): errors.append(ERROR_MISSING_LIBRARIANS) - repo_in_bundle = common_funcs.is_repo_in_bundle(repo_fields["clone_url"], - self.bundle_submodules) + repo_in_bundle = common_funcs.is_repo_in_bundle( + repo_fields["clone_url"], self.bundle_submodules + ) if not repo_in_bundle and not repo["name"] in BUNDLE_IGNORE_LIST: - # Don't assume the bundle will bundle itself and possibly - # other repos. - errors.append(ERROR_NOT_IN_BUNDLE) - - if (repo_fields.get("allow_squash_merge") or - repo_fields.get("allow_rebase_merge")): - errors.append(ERROR_ONLY_ALLOW_MERGES) + # Don't assume the bundle will bundle itself and possibly + # other repos. + errors.append(ERROR_NOT_IN_BUNDLE) + + if repo_fields.get("allow_squash_merge") or repo_fields.get( + "allow_rebase_merge" + ): + errors.append(ERROR_ONLY_ALLOW_MERGES) return errors def validate_actions_state(self, repo): @@ -300,14 +331,16 @@ def validate_actions_state(self, repo): has passed. Just returns a message stating that the most recent run failed. """ - if not (repo["owner"]["login"] == "adafruit" and - repo["name"].startswith("Adafruit_CircuitPython")): + if not ( + repo["owner"]["login"] == "adafruit" + and repo["name"].startswith("Adafruit_CircuitPython") + ): return [] actions_params = {"branch": repo["default_branch"]} response = github.get( - "/repos/" + repo["full_name"] + "/actions/runs", - params=actions_params) + "/repos/" + repo["full_name"] + "/actions/runs", params=actions_params + ) if not response.ok: return [ERROR_UNABLE_PULL_REPO_DETAILS] @@ -317,7 +350,7 @@ def validate_actions_state(self, repo): return [ERROR_GITHUB_FAILING_ACTIONS] return [] - + # pylint: disable=too-many-locals,too-many-return-statements,too-many-branches def validate_release_state(self, repo): """Validate if a repo 1) has a release, and 2) if there have been commits since the last release. Only files that drive user-facing changes @@ -343,104 +376,98 @@ def _filter_file_diffs(filenames): "README.rst", "pyproject.toml", } - compare_files = [ - name for name in filenames if not name.startswith(".") - ] - non_ignored_files = list( - set(compare_files).difference(_ignored_files) - ) + compare_files = [name for name in filenames if not name.startswith(".")] - return non_ignored_files + return list(set(compare_files).difference(_ignored_files)) - if not (repo["owner"]["login"] == "adafruit" and - repo["name"].startswith("Adafruit_CircuitPython")): + if not ( + repo["owner"]["login"] == "adafruit" + and repo["name"].startswith("Adafruit_CircuitPython") + ): return [] if repo["name"] in BUNDLE_IGNORE_LIST: return [] - repo_last_release = github.get("/repos/" - + repo["full_name"] - + "/releases/latest") + repo_last_release = github.get( + "/repos/" + repo["full_name"] + "/releases/latest" + ) if not repo_last_release.ok: return [ERROR_GITHUB_NO_RELEASE] repo_release_json = repo_last_release.json() - if "tag_name" in repo_release_json: - tag_name = repo_release_json["tag_name"] - elif "message" in repo_release_json: + if "message" in repo_release_json: if repo_release_json["message"] == "Not Found": return [ERROR_GITHUB_NO_RELEASE] - else: - # replace 'output_handler' with ERROR_OUTPUT_HANDLER - err_msg = [ - "Error: retrieving latest release information failed on ", - "'{}'. ".format(repo["name"]), - "Information Received: ", - "{}".format(repo_release_json["message"]) - ] - self.output_file_data.append("".join(err_msg)) - return [ERROR_OUTPUT_HANDLER] - main_branch = repo['default_branch'] - compare_tags = github.get("/repos/" - + repo["full_name"] - + "/compare/" - + tag_name - + "..." - + main_branch) - if not compare_tags.ok: - # replace 'output_handler' with ERROR_OUTPUT_HANDLER err_msg = [ - "Error: failed to compare {} '{}' ".format(repo["name"], main_branch), - "to tag '{}'".format(tag_name) + f"Error: retrieving latest release information failed on '{repo['name']}'.", + f"Information Received: {repo_release_json['message']}", ] self.output_file_data.append("".join(err_msg)) return [ERROR_OUTPUT_HANDLER] + + tag_name = repo_release_json.get("tag_name", "") + main_branch = repo["default_branch"] + compare_tags = github.get( + f"/repos/{repo['full_name']}/compare/{tag_name}...{main_branch}" + ) + if not compare_tags.ok: + self.output_file_data.append( + f"Error: failed to compare {repo['name']} '{main_branch}' to tag '{tag_name}'" + ) + return [ERROR_OUTPUT_HANDLER] compare_tags_json = compare_tags.json() if "status" in compare_tags_json: if compare_tags_json["status"] != "identical": - comp_filenames = [ - file["filename"] for file in compare_tags_json.get("files") - ] - filtered_files = _filter_file_diffs(comp_filenames) + filtered_files = _filter_file_diffs( + [file["filename"] for file in compare_tags_json.get("files")] + ) if filtered_files: oldest_commit_date = datetime.datetime.today() for commit in compare_tags_json["commits"]: commit_date_val = commit["commit"]["committer"]["date"] - commit_date = datetime.datetime.strptime(commit_date_val, - "%Y-%m-%dT%H:%M:%SZ") + commit_date = datetime.datetime.strptime( + commit_date_val, "%Y-%m-%dT%H:%M:%SZ" + ) if commit_date < oldest_commit_date: oldest_commit_date = commit_date date_diff = datetime.datetime.today() - oldest_commit_date - #print("{0} Release State:\n Tag Name: {1}\tRelease Date: {2}\n Today: {3}\t Released {4} days ago.".format(repo["name"], tag_name, oldest_commit_date, datetime.datetime.today(), date_diff.days)) - #print("Compare {4} status: {0} \n Ahead: {1} \t Behind: {2} \t Commits: {3}".format( - # compare_tags_json["status"], compare_tags_json["ahead_by"], - # compare_tags_json["behind_by"], compare_tags_json["total_commits"], repo["full_name"])) if date_diff.days > datetime.date.today().max.day: - return [(ERROR_GITHUB_COMMITS_SINCE_LAST_RELEASE_GTM, - date_diff.days)] - elif date_diff.days <= datetime.date.today().max.day: + return [ + ( + ERROR_GITHUB_COMMITS_SINCE_LAST_RELEASE_GTM, + date_diff.days, + ) + ] + if date_diff.days <= datetime.date.today().max.day: if date_diff.days > 7: - return [(ERROR_GITHUB_COMMITS_SINCE_LAST_RELEASE_1M, - date_diff.days)] - else: - return [(ERROR_GITHUB_COMMITS_SINCE_LAST_RELEASE_1W, - date_diff.days)] + return [ + ( + ERROR_GITHUB_COMMITS_SINCE_LAST_RELEASE_1M, + date_diff.days, + ) + ] + + return [ + ( + ERROR_GITHUB_COMMITS_SINCE_LAST_RELEASE_1W, + date_diff.days, + ) + ] elif "errors" in compare_tags_json: - # replace 'output_handler' with ERROR_OUTPUT_HANDLER err_msg = [ - "Error: comparing latest release to '{}' failed on ", - "'{}'. ".format(main_branch, repo["name"]), - "Error Message: {}".format(compare_tags_json["message"]) + f"Error: comparing latest release to '{main_branch}' failed on '{repo['name']}'. ", + f"Error Message: {compare_tags_json['message']}", ] self.output_file_data.append("".join(err_msg)) return [ERROR_OUTPUT_HANDLER] return [] - def _validate_readme(self, repo, download_url): + # pylint: disable=too-many-branches + def _validate_readme(self, download_url): # We use requests because file contents are hosted by # githubusercontent.com, not the API domain. contents = requests.get(download_url, timeout=30) @@ -482,10 +509,10 @@ def _validate_readme(self, repo, download_url): return errors - def _validate_py_for_u_modules(self, repo, download_url): - """ For a .py file, look for usage of "import u___" and - look for "import ___". If the "import u___" is - used with NO "import ____" generate an error. + def _validate_py_for_u_modules(self, download_url): + """For a .py file, look for usage of "import u___" and + look for "import ___". If the "import u___" is + used with NO "import ____" generate an error. """ # We use requests because file contents are hosted by # githubusercontent.com, not the API domain. @@ -496,40 +523,34 @@ def _validate_py_for_u_modules(self, repo, download_url): errors = [] lines = contents.text.split("\n") - ustruct_lines = ([l for l in lines - if re.match(r"[\s]*import[\s][\s]*ustruct", l)]) - struct_lines = ([l for l in lines - if re.match(r"[\s]*import[\s][\s]*struct", l)]) + ustruct_lines = [ + l for l in lines if re.match(r"[\s]*import[\s][\s]*ustruct", l) + ] + struct_lines = [l for l in lines if re.match(r"[\s]*import[\s][\s]*struct", l)] if ustruct_lines and not struct_lines: errors.append(ERROR_PYFILE_MISSING_STRUCT) - ure_lines = ([l for l in lines - if re.match(r"[\s]*import[\s][\s]*ure", l)]) - re_lines = ([l for l in lines - if re.match(r"[\s]*import[\s][\s]*re", l)]) + ure_lines = [l for l in lines if re.match(r"[\s]*import[\s][\s]*ure", l)] + re_lines = [l for l in lines if re.match(r"[\s]*import[\s][\s]*re", l)] if ure_lines and not re_lines: errors.append(ERROR_PYFILE_MISSING_RE) - ujson_lines = ([l for l in lines - if re.match(r"[\s]*import[\s][\s]*ujson", l)]) - json_lines = ([l for l in lines - if re.match(r"[\s]*import[\s][\s]*json", l)]) + ujson_lines = [l for l in lines if re.match(r"[\s]*import[\s][\s]*ujson", l)] + json_lines = [l for l in lines if re.match(r"[\s]*import[\s][\s]*json", l)] if ujson_lines and not json_lines: errors.append(ERROR_PYFILE_MISSING_JSON) - uerrno_lines = ([l for l in lines - if re.match(r"[\s]*import[\s][\s]*uerrno", l)]) - errno_lines = ([l for l in lines - if re.match(r"[\s]*import[\s][\s]*errno", l)]) + uerrno_lines = [l for l in lines if re.match(r"[\s]*import[\s][\s]*uerrno", l)] + errno_lines = [l for l in lines if re.match(r"[\s]*import[\s][\s]*errno", l)] if uerrno_lines and not errno_lines: errors.append(ERROR_PYFILE_MISSING_ERRNO) return errors - def _validate_actions_build_yml(self, repo, actions_build_info): + def _validate_actions_build_yml(self, actions_build_info): """Check the following configurations in the GitHub Actions - build.yml file: - - Pylint version is the latest release + build.yml file: + - Pylint version is the latest release """ download_url = actions_build_info["download_url"] @@ -554,8 +575,8 @@ def _validate_actions_build_yml(self, repo, actions_build_info): return [ERROR_PYLINT_VERSION_NOT_FIXED] try: - pylint_version = pkg_Requirement(pylint_info.group("pylint")) - except Exception: + pylint_version = Requirement(pylint_info.group("pylint")) + except InvalidRequirement: pass if not pylint_version: @@ -565,9 +586,8 @@ def _validate_actions_build_yml(self, repo, actions_build_info): return errors - def _validate_setup_py(self, repo, file_info): - """Check setup.py for pypi compatibility - """ + def _validate_setup_py(self, file_info): + """Check setup.py for pypi compatibility""" download_url = file_info["download_url"] contents = requests.get(download_url, timeout=30) if not contents.ok: @@ -575,12 +595,10 @@ def _validate_setup_py(self, repo, file_info): errors = [] - return errors def _validate_requirements_txt(self, repo, file_info): - """Check requirements.txt for pypi compatibility - """ + """Check requirements.txt for pypi compatibility""" download_url = file_info["download_url"] contents = requests.get(download_url, timeout=30) if not contents.ok: @@ -588,13 +606,13 @@ def _validate_requirements_txt(self, repo, file_info): errors = [] lines = contents.text.split("\n") - blinka_lines = ([l for l in lines - if re.match(r"[\s]*Adafruit-Blinka[\s]*", l)]) + blinka_lines = [l for l in lines if re.match(r"[\s]*Adafruit-Blinka[\s]*", l)] if not blinka_lines and repo["name"] not in LIBRARIES_DONT_NEED_BLINKA: errors.append(ERROR_MISSING_BLINKA) return errors + # pylint: disable=too-many-locals,too-many-branches,too-many-statements,too-many-return-statements def validate_contents(self, repo): """Validate the contents of a repository meets current CircuitPython criteria (within reason, functionality checks are not possible). Expects @@ -602,8 +620,10 @@ def validate_contents(self, repo): function). Returns a list of string error messages for the repository. """ - if not (repo["owner"]["login"] == "adafruit" and - repo["name"].startswith("Adafruit_CircuitPython")): + if not ( + repo["owner"]["login"] == "adafruit" + and repo["name"].startswith("Adafruit_CircuitPython") + ): return [] if repo["name"] == BUNDLE_REPO_NAME: return [] @@ -656,12 +676,11 @@ def validate_contents(self, repo): errors.append(ERROR_MISSING_README_RST) else: readme_info = None - for f in content_list: - if f["name"] == "README.rst": - readme_info = f + for file in content_list: + if file["name"] == "README.rst": + readme_info = file break - errors.extend(self._validate_readme(repo, - readme_info["download_url"])) + errors.extend(self._validate_readme(readme_info["download_url"])) if ".travis.yml" in files: errors.append(ERROR_NEEDS_ACTION_MIGRATION) @@ -682,18 +701,16 @@ def validate_contents(self, repo): actions_build_info = response.json() if actions_build_info: - errors.extend( - self._validate_actions_build_yml(repo, actions_build_info) - ) + errors.extend(self._validate_actions_build_yml(actions_build_info)) else: errors.append(ERROR_UNABLE_PULL_REPO_CONTENTS) if "readthedocs.yml" in files or ".readthedocs.yml" in files: if self.rtd_yml_base != "": - fn = "readthedocs.yml" + filename = "readthedocs.yml" if ".readthedocs.yml" in files: - fn = ".readthedocs.yml" - file_info = content_list[files.index(fn)] + filename = ".readthedocs.yml" + file_info = content_list[files.index(filename)] rtd_contents = requests.get(file_info["download_url"]) if rtd_contents.ok: try: @@ -710,7 +727,7 @@ def validate_contents(self, repo): if "setup.py" in files: file_info = content_list[files.index("setup.py")] - errors.extend(self._validate_setup_py(repo, file_info)) + errors.extend(self._validate_setup_py(file_info)) elif "setup.py.disabled" not in files: errors.append(ERROR_MISSING_SETUP_PY) @@ -721,10 +738,10 @@ def validate_contents(self, repo): else: errors.append(ERROR_MISSING_REQUIREMENTS_TXT) - - #Check for an examples folder. + # Check for an examples folder. dirs = [ - x["url"] for x in content_list + x["url"] + for x in content_list if (x["type"] == "dir" and x["name"] == "examples") ] examples_list = [] @@ -743,25 +760,26 @@ def validate_contents(self, repo): if len(examples_list) < 1: errors.append(ERROR_MISSING_EXAMPLE_FILES) else: - def __check_lib_name(repo_name, file_name): - """ Nested function to test example file names. - Allows examples to either match the repo name, - or have additional underscores separating the repo name. + + def __check_lib_name( + repo_name, file_name + ): # pylint: disable=unused-private-member + """Nested function to test example file names. + Allows examples to either match the repo name, + or have additional underscores separating the repo name. """ file_names = set() file_names.add(file_name) name_split = file_name.split("_") - name_rebuilt = ''.join( + name_rebuilt = "".join( (part for part in name_split if ".py" not in part) ) - if name_rebuilt: # avoid adding things like 'simpletest.py' -> '' + if name_rebuilt: # avoid adding things like 'simpletest.py' -> '' file_names.add(name_rebuilt) - return any( - name.startswith(repo_name) for name in file_names - ) + return any(name.startswith(repo_name) for name in file_names) lib_name_start = repo["name"].rfind("CircuitPython_") + 14 lib_name = repo["name"][lib_name_start:].lower() @@ -771,11 +789,10 @@ def __check_lib_name(repo_name, file_name): for example in examples_list: if example["name"].endswith(".py"): check_lib_name = __check_lib_name( - lib_name, - example["name"].lower() + lib_name, example["name"].lower() ) if not check_lib_name: - all_have_name = False + all_have_name = False if "simpletest" in example["name"].lower(): simpletest_exists = True if not all_have_name: @@ -786,94 +803,113 @@ def __check_lib_name(repo_name, file_name): errors.append(ERROR_MISSING_EXAMPLE_FOLDER) # first location .py files whose names begin with "adafruit_" - re_str = re.compile(r'adafruit\_[\w]*\.py') - pyfiles = ([x["download_url"] for x in content_list - if re_str.fullmatch(x["name"])]) + re_str = re.compile(r"adafruit\_[\w]*\.py") + pyfiles = [ + x["download_url"] for x in content_list if re_str.fullmatch(x["name"]) + ] for pyfile in pyfiles: # adafruit_xxx.py file; check if for proper usage of u___ versions of modules - errors.extend(self._validate_py_for_u_modules(repo, pyfile)) + errors.extend(self._validate_py_for_u_modules(pyfile)) # now location any directories whose names begin with "adafruit_" - re_str = re.compile(r'adafruit\_[\w]*') + re_str = re.compile(r"adafruit\_[\w]*") for adir in dirs: if re_str.fullmatch(adir): # retrieve the files in that directory - dir_file_list = github.get("/repos/" - + repo["full_name"] - + "/contents/" - + adir) + dir_file_list = github.get( + "/repos/" + repo["full_name"] + "/contents/" + adir + ) if not dir_file_list.ok: errors.append(ERROR_UNABLE_PULL_REPO_DIR) dir_file_list = dir_file_list.json() # search for .py files in that directory - dir_files = ([x["download_url"] for x in dir_file_list - if x["type"] == "file" - and x["name"].endswith(".py")]) + dir_files = [ + x["download_url"] + for x in dir_file_list + if x["type"] == "file" and x["name"].endswith(".py") + ] for dir_file in dir_files: # .py files in subdirectory adafruit_xxx # check if for proper usage of u___ versions of modules - errors.extend(self._validate_py_for_u_modules(repo, dir_file)) + errors.extend(self._validate_py_for_u_modules(dir_file)) return errors def validate_readthedocs(self, repo): - if not (repo["owner"]["login"] == "adafruit" and - repo["name"].startswith("Adafruit_CircuitPython")): + """Method to check the health of `repo`'s ReadTheDocs.""" + if not ( + repo["owner"]["login"] == "adafruit" + and repo["name"].startswith("Adafruit_CircuitPython") + ): return [] if repo["name"] in BUNDLE_IGNORE_LIST: return [] - global rtd_subprojects - if not rtd_subprojects: - rtd_response = requests.get("https://readthedocs.org/api/v2/project/74557/subprojects/", - timeout=15) + if not self.rtd_subprojects: + rtd_response = requests.get( + "https://readthedocs.org/api/v2/project/74557/subprojects/", timeout=15 + ) if not rtd_response.ok: return [ERROR_RTD_SUBPROJECT_FAILED] - rtd_subprojects = {} + self.rtd_subprojects = {} for subproject in rtd_response.json()["subprojects"]: - rtd_subprojects[common_funcs.sanitize_url(subproject["repo"])] = subproject + self.rtd_subprojects[ + common_funcs.sanitize_url(subproject["repo"]) + ] = subproject repo_url = common_funcs.sanitize_url(repo["clone_url"]) - if repo_url not in rtd_subprojects: + if repo_url not in self.rtd_subprojects: return [ERROR_RTD_SUBPROJECT_MISSING] errors = [] - subproject = rtd_subprojects[repo_url] + subproject = self.rtd_subprojects[repo_url] if 105398 not in subproject["users"]: errors.append(ERROR_RTD_ADABOT_MISSING) valid_versions = requests.get( - "https://readthedocs.org/api/v2/project/{}/active_versions/".format(subproject["id"]), - timeout=15) + "https://readthedocs.org/api/v2/project/{}/active_versions/".format( + subproject["id"] + ), + timeout=15, + ) if not valid_versions.ok: errors.append(ERROR_RTD_VALID_VERSIONS_FAILED) else: valid_versions = valid_versions.json() - latest_release = github.get("/repos/{}/releases/latest".format(repo["full_name"])) + latest_release = github.get( + "/repos/{}/releases/latest".format(repo["full_name"]) + ) if not latest_release.ok: errors.append(ERROR_GITHUB_RELEASE_FAILED) # disabling this for now, since it is ignored and always fails - #else: - # if latest_release.json()["tag_name"] not in [tag["verbose_name"] for tag in valid_versions["versions"]]: + # else: + # if ( + # latest_release.json()["tag_name"] not in + # [tag["verbose_name"] for tag in valid_versions["versions"]] + # ): # errors.append(ERROR_RTD_MISSING_LATEST_RELEASE) # There is no API which gives access to a list of builds for a project so we parse the html # webpage. builds_webpage = requests.get( "https://readthedocs.org/projects/{}/builds/".format(subproject["slug"]), - timeout=15) + timeout=15, + ) + # pylint: disable=too-many-nested-blocks + # TODO: look into reducing the number of nested blocks. if not builds_webpage.ok: errors.append(ERROR_RTD_FAILED_TO_LOAD_BUILDS) else: for line in builds_webpage.text.split("\n"): - if "
") - rel = rel.strip() - if rel == "rel=\"next\"": - next_url = link - break - if not next_url: - break - response = github.get(link) return results - + # pylint: disable=too-many-nested-blocks def gather_insights(self, repo, insights, since, show_closed_metric=False): """Gather analytics about a repository like open and merged pull requests. This expects a dictionary with GitHub API repository state (like from the @@ -968,16 +1008,22 @@ def gather_insights(self, repo, insights, since, show_closed_metric=False): if repo["owner"]["login"] != "adafruit": return [] - params = {"sort": "updated", - "state": "all", - "per_page": 100, - "since": since.strftime("%Y-%m-%dT%H:%M:%SZ")} - issues = self.github_get_all_pages("/repos/" + repo["full_name"] + "/issues", params=params) + params = { + "sort": "updated", + "state": "all", + "per_page": 100, + "since": since.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + issues = self.github_get_all_pages( + "/repos/" + repo["full_name"] + "/issues", params=params + ) if issues == ERROR_OUTPUT_HANDLER: return [ERROR_OUTPUT_HANDLER] for issue in issues: - created = datetime.datetime.strptime(issue["created_at"], "%Y-%m-%dT%H:%M:%SZ") + created = datetime.datetime.strptime( + issue["created_at"], "%Y-%m-%dT%H:%M:%SZ" + ) if "pull_request" in issue: pr_info = github.get(issue["pull_request"]["url"]) pr_info = pr_info.json() @@ -988,35 +1034,29 @@ def gather_insights(self, repo, insights, since, show_closed_metric=False): insights["active_prs"] += 1 else: merged = datetime.datetime.strptime( - issue["closed_at"], - "%Y-%m-%dT%H:%M:%SZ" - ) + issue["closed_at"], "%Y-%m-%dT%H:%M:%SZ" + ) if pr_info["merged"] and merged > since: merged_info = "" if show_closed_metric: created = datetime.datetime.strptime( - issue["created_at"], - "%Y-%m-%dT%H:%M:%SZ" + issue["created_at"], "%Y-%m-%dT%H:%M:%SZ" ) merged = datetime.datetime.strptime( - issue["closed_at"], - "%Y-%m-%dT%H:%M:%SZ" + issue["closed_at"], "%Y-%m-%dT%H:%M:%SZ" ) days_open = merged - created - if days_open.days < 0: # opened earlier today + if days_open.days < 0: # opened earlier today days_open += datetime.timedelta( days=(days_open.days * -1) ) elif days_open.days == 0: - days_open += datetime.timedelta( - days=(1) - ) + days_open += datetime.timedelta(days=(1)) merged_info = " (Days open: {})".format(days_open.days) pr_link = "{0}{1}".format( - issue["pull_request"]["html_url"], - merged_info + issue["pull_request"]["html_url"], merged_info ) insights["merged_prs"].append(pr_link) @@ -1027,7 +1067,9 @@ def gather_insights(self, repo, insights, since, show_closed_metric=False): for commit in pr_commits.json(): author = commit.get("author") if author: - insights["pr_merged_authors"].add(author["login"]) + insights["pr_merged_authors"].add( + author["login"] + ) else: insights["pr_merged_authors"].add(pr_info["user"]["login"]) @@ -1036,7 +1078,9 @@ def gather_insights(self, repo, insights, since, show_closed_metric=False): if pr_reviews.ok: for review in pr_reviews.json(): if review["state"].lower() == "approved": - insights["pr_reviewers"].add(review["user"]["login"]) + insights["pr_reviewers"].add( + review["user"]["login"] + ) else: insights["closed_prs"] += 1 else: @@ -1053,26 +1097,32 @@ def gather_insights(self, repo, insights, since, show_closed_metric=False): insights["issue_closers"].add(issue_info["closed_by"]["login"]) params = {"state": "open", "per_page": 100} - issues = self.github_get_all_pages("/repos/" + repo["full_name"] + "/issues", params=params) + issues = self.github_get_all_pages( + "/repos/" + repo["full_name"] + "/issues", params=params + ) if issues == ERROR_OUTPUT_HANDLER: return [ERROR_OUTPUT_HANDLER] for issue in issues: - created = datetime.datetime.strptime(issue["created_at"], "%Y-%m-%dT%H:%M:%SZ") + created = datetime.datetime.strptime( + issue["created_at"], "%Y-%m-%dT%H:%M:%SZ" + ) days_open = datetime.datetime.today() - created - if days_open.days < 0: # opened earlier today + if days_open.days < 0: # opened earlier today days_open += datetime.timedelta(days=(days_open.days * -1)) if "pull_request" in issue: - pr_link = "{0} (Open {1} days)".format(issue["pull_request"]["html_url"], - days_open.days) + pr_link = "{0} (Open {1} days)".format( + issue["pull_request"]["html_url"], days_open.days + ) insights["open_prs"].append(pr_link) else: - issue_link = "{0} (Open {1} days)".format(issue["html_url"], - days_open.days) + issue_link = "{0} (Open {1} days)".format( + issue["html_url"], days_open.days + ) insights["open_issues"].append(issue_link) if "labels" in issue: for i in issue["labels"]: - if i["name"] == 'good first issue': + if i["name"] == "good first issue": insights["good_first_issues"] += 1 # process Hacktoberfest labels if it is Hacktoberfest season @@ -1082,39 +1132,40 @@ def gather_insights(self, repo, insights, since, show_closed_metric=False): issue for issue in issues if "pull_request" not in issue ] if season_action == "add": - insights["hacktober_assigned"] += ( - hacktober.assign_hacktoberfest(repo, - issues=hacktober_issues) + insights["hacktober_assigned"] += hacktober.assign_hacktoberfest( + repo, issues=hacktober_issues ) elif season_action == "remove": - insights["hacktober_removed"] += ( - hacktober.assign_hacktoberfest(repo, - issues=hacktober_issues, - remove_labels=True) + insights["hacktober_removed"] += hacktober.assign_hacktoberfest( + repo, issues=hacktober_issues, remove_labels=True ) # get milestones for core repo if repo["name"] == "circuitpython": params = {"state": "open"} - response = github.get("/repos/adafruit/circuitpython/milestones", params=params) + response = github.get( + "/repos/adafruit/circuitpython/milestones", params=params + ) if not response.ok: - # replace 'output_handler' with ERROR_OUTPUT_HANDLER self.output_file_data.append("Failed to get core milestone insights.") return [ERROR_OUTPUT_HANDLER] - else: - milestones = response.json() - for milestone in milestones: - #print(milestone) - insights["milestones"][milestone["title"]] = milestone["open_issues"] + + milestones = response.json() + for milestone in milestones: + insights["milestones"][milestone["title"]] = milestone["open_issues"] return [] def validate_in_pypi(self, repo): """prints a list of Adafruit_CircuitPython libraries that are in pypi""" - if (repo["name"] in BUNDLE_IGNORE_LIST or - repo["name"] in self.has_setup_py_disabled): - return [] - if not (repo["owner"]["login"] == "adafruit" and - repo["name"].startswith("Adafruit_CircuitPython")): + if ( + repo["name"] in BUNDLE_IGNORE_LIST + or repo["name"] in self.has_setup_py_disabled + ): + return [] + if not ( + repo["owner"]["login"] == "adafruit" + and repo["name"].startswith("Adafruit_CircuitPython") + ): return [] if not common_funcs.repo_is_on_pypi(repo): return [ERROR_NOT_ON_PYPI] @@ -1125,7 +1176,9 @@ def validate_labels(self, repo): response = github.get("/repos/" + repo["full_name"] + "/labels") if not response.ok: # replace 'output_handler' with ERROR_OUTPUT_HANDLER - self.output_file_data.append("Labels request failed: {}".format(repo["full_name"])) + self.output_file_data.append( + "Labels request failed: {}".format(repo["full_name"]) + ) return [ERROR_OUTPUT_HANDLER] errors = [] @@ -1137,13 +1190,14 @@ def validate_labels(self, repo): if not label in repo_labels: response = github.post( "/repos/" + repo["full_name"] + "/labels", - json={"name": label, "color": info["color"]} + json={"name": label, "color": info["color"]}, ) if not response.ok: has_all_labels = False self.output_file_data.append( - "Request to add '{}' label failed: {}".format(label, - repo["full_name"]) + "Request to add '{}' label failed: {}".format( + label, repo["full_name"] + ) ) if ERROR_OUTPUT_HANDLER not in errors: errors.append(ERROR_OUTPUT_HANDLER) @@ -1154,7 +1208,7 @@ def validate_labels(self, repo): return errors def validate_passes_linting(self, repo): - """ Clones the repo and runs pylint on the Python files""" + """Clones the repo and runs pylint on the Python files""" if not repo["name"].startswith("Adafruit_CircuitPython"): return [] @@ -1175,23 +1229,25 @@ def validate_passes_linting(self, repo): ) return [ERROR_OUTPUT_HANDLER] - if self.keep_repos and (repo_dir / '.pylint-ok').exists(): + if self.keep_repos and (repo_dir / ".pylint-ok").exists(): return [] for file in repo_dir.rglob("*.py"): - if file.name in ignored_py_files or str(file.parent).endswith("examples"): + if file.name in ignored_py_files or str(file.parent).endswith( + "examples" + ): continue pylint_args = [str(file)] - if (repo_dir / '.pylintrc').exists(): + if (repo_dir / ".pylintrc").exists(): pylint_args += [f"--rcfile={str(repo_dir / '.pylintrc')}"] reporter = CapturedJsonReporter() logging.debug("Running pylint on %s", file) - linted = lint.Run(pylint_args, reporter=reporter, exit=False) - pylint_stderr = '' + lint.Run(pylint_args, reporter=reporter, exit=False) + pylint_stderr = "" pylint_stdout = reporter.get_result() if pylint_stderr: @@ -1212,13 +1268,13 @@ def validate_passes_linting(self, repo): return [ERROR_PYLINT_FAILED_LINTING] if self.keep_repos: - with open(repo_dir / '.pylint-ok', 'w') as f: - f.write(''.join(pylint_result)) + with open(repo_dir / ".pylint-ok", "w") as pylint_ok: + pylint_ok.write("".join(pylint_result)) return [] def validate_default_branch(self, repo): - """ Makes sure that the default branch is main """ + """Makes sure that the default branch is main""" if not repo["name"].startswith("Adafruit_CircuitPython"): return [] diff --git a/adabot/lib/common_funcs.py b/adabot/lib/common_funcs.py index 1b844bd..abd5610 100644 --- a/adabot/lib/common_funcs.py +++ b/adabot/lib/common_funcs.py @@ -23,6 +23,8 @@ # GitHub API Serch has stopped returning the core repo for some reason. Tried several # different search params, and came up emtpy. Hardcoding it as a failsafe. +"""Common functions used throughout Adabot.""" + import collections import datetime import os @@ -31,9 +33,11 @@ from adabot import github_requests as github from adabot import pypi_requests as pypi -core_repo_url = "/repos/adafruit/circuitpython" +CORE_REPO_URL = "/repos/adafruit/circuitpython" + def parse_gitmodules(input_text): + # pylint: disable=anomalous-backslash-in-string """Parse a .gitmodules file and return a list of all the git submodules defined inside of it. Each list item is 2-tuple with: - submodule name (string) @@ -55,23 +59,27 @@ def parse_gitmodules(input_text): surprisingly complex and no mature parsing modules are available (outside the code in git itself). """ + # pylint: enable=anomalous-backslash-in-string + # Assume no results if invalid input. if input_text is None: return [] # Define a regular expression to match a basic submodule section line and # capture its subsection value. - submodule_section_re = '^\[submodule "(.+)"\]$' + submodule_section_re = r'^\[submodule "(.+)"\]$' # Define a regular expression to match a variable setting line and capture # the variable name and value. This does NOT handle multi-line or quote # escaping (far outside the abilities of a regular expression). - variable_re = '^\s*([a-zA-Z0-9\-]+) =\s+(.+?)\s*$' + variable_re = r"^\s*([a-zA-Z0-9\-]+) =\s+(.+?)\s*$" # Process all the lines to parsing submodule sections and the variables # within them as they're found. results = [] submodule_name = None submodule_variables = {} for line in input_text.splitlines(): - submodule_section_match = re.match(submodule_section_re, line, flags=re.IGNORECASE) + submodule_section_match = re.match( + submodule_section_re, line, flags=re.IGNORECASE + ) variable_match = re.match(variable_re, line) if submodule_section_match: # Found a new section. End the current one if it had data and add @@ -85,12 +93,15 @@ def parse_gitmodules(input_text): # Force the variable name to lower case as variable names are # case-insensitive in git config sections and this makes later # processing easier (can assume lower-case names to find values). - submodule_variables[variable_match.group(1).lower()] = variable_match.group(2) + submodule_variables[variable_match.group(1).lower()] = variable_match.group( + 2 + ) # Add the last parsed section if it exists. if submodule_name is not None: results.append((submodule_name, submodule_variables)) return results + def get_bundle_submodules(): """Query Adafruit_CircuitPython_Bundle repository for all the submodules (i.e. modules included inside) and return a list of the found submodules. @@ -101,13 +112,16 @@ def get_bundle_submodules(): # Assume the bundle repository is public and get the .gitmodules file # without any authentication or Github API usage. Also assumes the # master branch of the bundle is the canonical source of the bundle release. - result = requests.get('https://raw.githubusercontent.com/adafruit/Adafruit_CircuitPython_Bundle/main/.gitmodules', - timeout=15) + result = requests.get( + "https://raw.githubusercontent.com/adafruit/Adafruit_CircuitPython_Bundle/main/.gitmodules", + timeout=15, + ) if result.status_code != 200: - #output_handler("Failed to access bundle .gitmodules file from GitHub!", quiet=True) - raise RuntimeError('Failed to access bundle .gitmodules file from GitHub!') + # output_handler("Failed to access bundle .gitmodules file from GitHub!", quiet=True) + raise RuntimeError("Failed to access bundle .gitmodules file from GitHub!") return parse_gitmodules(result.text) + def sanitize_url(url): """Convert a Github repository URL into a format which can be compared for equality with simple string comparison. Will strip out any leading URL @@ -122,14 +136,15 @@ def sanitize_url(url): # Strip out any preceding http://, https:// or git:// from the URL to # make URL comparisons safe (probably better to explicitly parse using # a URL module in the future). - scheme_end = url.find('://') + scheme_end = url.find("://") if scheme_end >= 0: url = url[scheme_end:] # Strip out any .git suffix if it exists. - if url.endswith('.git'): + if url.endswith(".git"): url = url[:-4] return url + def is_repo_in_bundle(repo_clone_url, bundle_submodules): """Return a boolean indicating if the specified repository (the clone URL as a string) is in the bundle. Specify bundle_submodules as a dictionary @@ -141,8 +156,8 @@ def is_repo_in_bundle(repo_clone_url, bundle_submodules): # this clone URL. Not the most efficient search but it's a handful of # items in the bundle. for submodule in bundle_submodules: - name, variables = submodule - submodule_url = variables.get('url', '') + _, variables = submodule + submodule_url = variables.get("url", "") # Compare URLs and skip to the next submodule if it's not a match. # Right now this is a case sensitive compare, but perhaps it should # be insensitive in the future (unsure if Github repos are sensitive). @@ -151,13 +166,14 @@ def is_repo_in_bundle(repo_clone_url, bundle_submodules): # URLs matched so now check if the submodule is placed in the libraries # subfolder of the bundle. Just look at the path from the submodule # state. - if variables.get('path', '').startswith('libraries/'): + if variables.get("path", "").startswith("libraries/"): # Success! Found the repo as a submodule of the libraries folder # in the bundle. return True # Failed to find the repo as a submodule of the libraries folders. return False + def list_repos(*, include_repos=None): """Return a list of all Adafruit repositories that start with Adafruit_CircuitPython. Each list item is a dictionary of GitHub API @@ -167,17 +183,29 @@ def list_repos(*, include_repos=None): are included. """ repos = [] - result = github.get("/search/repositories", - params={"q":"Adafruit_CircuitPython user:adafruit archived:false fork:true", - "per_page": 100, - "sort": "updated", - "order": "asc"} - ) + result = github.get( + "/search/repositories", + params={ + "q": "Adafruit_CircuitPython user:adafruit archived:false fork:true", + "per_page": 100, + "sort": "updated", + "order": "asc", + }, + ) while result.ok: - #repos.extend(result.json()["items"]) # uncomment and comment below, to include all forks - repos.extend(repo for repo in result.json()["items"] if (repo["owner"]["login"] == "adafruit" and - (repo["name"].startswith("Adafruit_CircuitPython") or repo["name"] == "circuitpython"))) + # repos.extend(result.json()["items"]) # uncomment and comment below, to include all forks + repos.extend( + repo + for repo in result.json()["items"] + if ( + repo["owner"]["login"] == "adafruit" + and ( + repo["name"].startswith("Adafruit_CircuitPython") + or repo["name"] == "circuitpython" + ) + ) + ) if result.links.get("next"): result = github.get(result.links["next"]["url"]) @@ -187,7 +215,7 @@ def list_repos(*, include_repos=None): repo_names = [repo["name"] for repo in repos] if "circuitpython" not in repo_names: - core = github.get(core_repo_url) + core = github.get(CORE_REPO_URL) if core.ok: repos.append(core.json()) @@ -202,11 +230,13 @@ def list_repos(*, include_repos=None): return repos + def get_docs_link(bundle_path, submodule): + """The URL to the documentation from the README.""" + lines = None try: - f = open(f"{bundle_path}/{submodule[1]['path']}/README.rst", 'r') - lines = f.read().split("\n") - f.close() + with open(f"{bundle_path}/{submodule[1]['path']}/README.rst", "r") as readme: + lines = readme.read().split("\n") for i in range(10): if "target" in lines[i] and "readthedocs" in lines[i]: return lines[i].replace(" :target: ", "") @@ -215,19 +245,21 @@ def get_docs_link(bundle_path, submodule): # didn't find readme return None + def repo_is_on_pypi(repo): """returns True when the provided repository is in pypi""" is_on = False - the_page = pypi.get("/pypi/"+repo["name"]+"/json") + the_page = pypi.get("/pypi/" + repo["name"] + "/json") if the_page and the_page.status_code == 200: is_on = True return is_on + def is_new_or_updated(repo): - """ Check the repo for new release(s) within the last week. Then determine - if all releases are within the last week to decide if this is a newly - released library, or an updated library. + """Check the repo for new release(s) within the last week. Then determine + if all releases are within the last week to decide if this is a newly + released library, or an updated library. """ today_minus_seven = datetime.datetime.today() - datetime.timedelta(days=7) @@ -235,23 +267,22 @@ def is_new_or_updated(repo): # first, check the latest release to see if within the last 7 days result = github.get("/repos/adafruit/" + repo["name"] + "/releases/latest") if not result.ok: - return + return None release_info = result.json() if "published_at" not in release_info: - return - else: - release_date = datetime.datetime.strptime( - release_info["published_at"], - "%Y-%m-%dT%H:%M:%SZ" - ) - if release_date < today_minus_seven: - return + return None + + release_date = datetime.datetime.strptime( + release_info["published_at"], "%Y-%m-%dT%H:%M:%SZ" + ) + if release_date < today_minus_seven: + return None # we have a release within the last 7 days. now check if its a newly # released library within the last week, or if its just an update result = github.get("/repos/adafruit/" + repo["name"] + "/releases") if not result.ok: - return + return None new_releases = 0 releases = result.json() @@ -259,20 +290,20 @@ def is_new_or_updated(repo): if not release["published_at"]: continue release_date = datetime.datetime.strptime( - release["published_at"], - "%Y-%m-%dT%H:%M:%SZ" + release["published_at"], "%Y-%m-%dT%H:%M:%SZ" ) if not release_date < today_minus_seven: new_releases += 1 if new_releases == len(releases): return "new" - else: - return "updated" + + return "updated" + def whois_github_user(): - """ Find who the user is that is running the current instance of adabot. - 'GITHUB_ACTOR' is an environment variable available on GitHub Actions. + """Find who the user is that is running the current instance of adabot. + 'GITHUB_ACTOR' is an environment variable available on GitHub Actions. """ user = None if "GITHUB_ACTOR" in os.environ: @@ -282,10 +313,11 @@ def whois_github_user(): return user + class InsightData(collections.UserDict): - """ Container class for holding insight data (issues, PRs, etc). - """ + """Container class for holding insight data (issues, PRs, etc).""" + # pylint: disable=super-init-not-called def __init__(self): self.data = { "merged_prs": [], diff --git a/adabot/pypi_requests.py b/adabot/pypi_requests.py index f83e397..7b7b0bd 100644 --- a/adabot/pypi_requests.py +++ b/adabot/pypi_requests.py @@ -19,15 +19,10 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -""" -`adafruit_adabot` -==================================================== - -TODO(description) +""" Helper for requests to pypi.org * Author(s): Michael McWethy """ -import os import requests @@ -37,5 +32,7 @@ def _fix_url(url): url = "https://pypi.org" + url return url + def get(url, **kwargs): + """Process a GET request from pypi.org""" return requests.get(_fix_url(url), timeout=30, **kwargs) diff --git a/adabot/travis_requests.py b/adabot/travis_requests.py deleted file mode 100644 index 8f87326..0000000 --- a/adabot/travis_requests.py +++ /dev/null @@ -1,68 +0,0 @@ -# The MIT License (MIT) -# -# Copyright (c) 2017 Scott Shawcroft for Adafruit Industries -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -""" -`adafruit_adabot` -==================================================== - -TODO(description) - -* Author(s): Scott Shawcroft -""" -import os -import sys - -import requests - - -def _fix_url(url): - if url.startswith("/"): - url = "https://api.travis-ci.com" + url - return url - -def _auth_token(): - if not "ADABOT_TRAVIS_ACCESS_TOKEN" in os.environ: - print("Please configure the ADABOT_TRAVIS_ACCESS_TOKEN environment variable.") - return "token " - return "token {}".format(os.environ["ADABOT_TRAVIS_ACCESS_TOKEN"]) - -def _fix_kwargs(kwargs): - user_agent = "AdafruitAdabot" - if "headers" in kwargs: - kwargs["headers"]["Authorization"] = _auth_token() - kwargs["headers"]["User-Agent"] = user_agent - kwargs["headers"]["Travis-API-Version"] = "3" - else: - kwargs["headers"] = { - "Authorization": _auth_token(), - "User-Agent": user_agent, - "Travis-API-Version": "3" - } - return kwargs - -def get(url, **kwargs): - return requests.get(_fix_url(url), timeout=30, **_fix_kwargs(kwargs)) - -def post(url, **kwargs): - return requests.post(_fix_url(url), timeout=30, **_fix_kwargs(kwargs)) - -def put(url, **kwargs): - return requests.put(_fix_url(url), timeout=30, **_fix_kwargs(kwargs)) diff --git a/adabot/update_cp_org_libraries.py b/adabot/update_cp_org_libraries.py index e55ba65..cbab102 100644 --- a/adabot/update_cp_org_libraries.py +++ b/adabot/update_cp_org_libraries.py @@ -20,6 +20,10 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +"""Adabot utility for updating circuitpython.org libraries info.""" + +# pylint: disable=redefined-outer-name + import argparse import datetime import inspect @@ -33,55 +37,48 @@ from adabot import github_requests as github from adabot import pypi_requests as pypi -DO_NOT_VALIDATE = ['CircuitPython_Community_Bundle', 'cookiecutter-adafruit-circuitpython'] +DO_NOT_VALIDATE = [ + "CircuitPython_Community_Bundle", + "cookiecutter-adafruit-circuitpython", +] # Setup ArgumentParser cmd_line_parser = argparse.ArgumentParser( description="Adabot utility for updating circuitpython.org libraries info.", - prog="Adabot circuitpython.org/libraries Updater" + prog="Adabot circuitpython.org/libraries Updater", ) cmd_line_parser.add_argument( - "-o", "--output_file", + "-o", + "--output_file", help="Output JSON file to the filename provided.", metavar="", - dest="output_file" + dest="output_file", ) cmd_line_parser.add_argument( "--cache-http", help="Cache HTTP requests using requests_cache", - action='store_true', - default=False + action="store_true", + default=False, ) cmd_line_parser.add_argument( - "--cache-ttl", - help="HTTP cache TTL", - type=int, - default=7200 + "--cache-ttl", help="HTTP cache TTL", type=int, default=7200 ) cmd_line_parser.add_argument( - "--keep-repos", - help="Keep repos between runs", - action='store_true', - default=False + "--keep-repos", help="Keep repos between runs", action="store_true", default=False ) cmd_line_parser.add_argument( - "--loglevel", - help="Adjust the log level (default ERROR)", - type=str, - default='ERROR' + "--loglevel", help="Adjust the log level (default ERROR)", type=str, default="ERROR" ) -sort_re = re.compile(r'(?<=\(Open\s)(.+)(?=\sdays)') +sort_re = re.compile(r"(?<=\(Open\s)(.+)(?=\sdays)") def get_open_issues_and_prs(repo): - """ Retreive all of the open issues (minus pull requests) for the repo. - """ + """Retreive all of the open issues (minus pull requests) for the repo.""" open_issues = [] open_pull_requests = [] - params = {"state":"open"} - result = github.get("/repos/adafruit/" + repo["name"] + "/issues", - params=params) + params = {"state": "open"} + result = github.get("/repos/adafruit/" + repo["name"] + "/issues", params=params) if not result.ok: return [], [] @@ -89,12 +86,11 @@ def get_open_issues_and_prs(repo): for issue in issues: created = datetime.datetime.strptime(issue["created_at"], "%Y-%m-%dT%H:%M:%SZ") days_open = datetime.datetime.today() - created - if days_open.days < 0: # opened earlier today + if days_open.days < 0: # opened earlier today days_open += datetime.timedelta(days=(days_open.days * -1)) - issue_title = "{0} (Open {1} days)".format(issue["title"], - days_open.days) - if "pull_request" not in issue: # ignore pull requests + issue_title = "{0} (Open {1} days)".format(issue["title"], days_open.days) + if "pull_request" not in issue: # ignore pull requests issue_labels = ["None"] if len(issue["labels"]) != 0: issue_labels = [label["name"] for label in issue["labels"]] @@ -113,31 +109,32 @@ def get_open_issues_and_prs(repo): def get_contributors(repo): + """Gather contributor information.""" contributors = [] reviewers = [] merged_pr_count = 0 - params = {"state":"closed", "sort":"updated", "direction":"desc"} - result = github.get("/repos/adafruit/" + repo["name"] + "/pulls", - params=params) + params = {"state": "closed", "sort": "updated", "direction": "desc"} + result = github.get("/repos/adafruit/" + repo["name"] + "/pulls", params=params) if result.ok: today_minus_seven = datetime.datetime.today() - datetime.timedelta(days=7) - prs = result.json() - for pr in prs: + pull_requests = result.json() + for pull_request in pull_requests: merged_at = datetime.datetime.min - if "merged_at" in pr: - if pr["merged_at"] is None: + if "merged_at" in pull_request: + if pull_request["merged_at"] is None: continue - merged_at = datetime.datetime.strptime(pr["merged_at"], - "%Y-%m-%dT%H:%M:%SZ") + merged_at = datetime.datetime.strptime( + pull_request["merged_at"], "%Y-%m-%dT%H:%M:%SZ" + ) else: continue if merged_at < today_minus_seven: continue - contributors.append(pr["user"]["login"]) + contributors.append(pull_request["user"]["login"]) merged_pr_count += 1 # get reviewers (merged_by, and any others) - single_pr = github.get(pr["url"]) + single_pr = github.get(pull_request["url"]) if not single_pr.ok: continue pr_info = single_pr.json() @@ -152,12 +149,15 @@ def get_contributors(repo): return contributors, reviewers, merged_pr_count +# pylint: disable=invalid-name if __name__ == "__main__": cmd_line_args = cmd_line_parser.parse_args() - logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', - datefmt='%Y-%m-%d %T', - level=getattr(logging, cmd_line_args.loglevel)) + logging.basicConfig( + format="%(asctime)s %(levelname)s %(message)s", + datefmt="%Y-%m-%d %T", + level=getattr(logging, cmd_line_args.loglevel), + ) logging.info("Running circuitpython.org/libraries updater...") @@ -177,8 +177,12 @@ def get_contributors(repo): if cmd_line_args.cache_http: cpy_vals.github.setup_cache(cmd_line_args.cache_ttl) - repos = common_funcs.list_repos(include_repos=("CircuitPython_Community_Bundle", - 'cookiecutter-adafruit-circuitpython',)) + repos = common_funcs.list_repos( + include_repos=( + "CircuitPython_Community_Bundle", + "cookiecutter-adafruit-circuitpython", + ) + ) new_libs = {} updated_libs = {} @@ -190,7 +194,8 @@ def get_contributors(repo): repos_by_error = {} default_validators = [ - vals[1] for vals in inspect.getmembers(cpy_vals.library_validator) + vals[1] + for vals in inspect.getmembers(cpy_vals.LibraryValidator) if vals[0].startswith("validate") ] bundle_submodules = common_funcs.get_bundle_submodules() @@ -200,17 +205,19 @@ def get_contributors(repo): if pylint_info and pylint_info.ok: latest_pylint = pylint_info.json()["info"]["version"] - validator = cpy_vals.library_validator( + validator = cpy_vals.LibraryValidator( default_validators, bundle_submodules, latest_pylint, - keep_repos=cmd_line_args.keep_repos + keep_repos=cmd_line_args.keep_repos, ) for repo in repos: - if (repo["name"] in cpy_vals.BUNDLE_IGNORE_LIST - or repo["name"] == "circuitpython"): - continue + if ( + repo["name"] in cpy_vals.BUNDLE_IGNORE_LIST + or repo["name"] == "circuitpython" + ): + continue repo_name = repo["name"] # get a list of new & updated libraries for the last week @@ -242,14 +249,14 @@ def get_contributors(repo): errors = [] try: errors = validator.run_repo_validation(repo) - except Exception as e: + except Exception as e: # pylint: disable=broad-except logging.exception("Unhandled exception %s", str(e)) errors.extend([cpy_vals.ERROR_OUTPUT_HANDLER]) for error in errors: if not isinstance(error, tuple): # check for an error occurring in the validator module if error == cpy_vals.ERROR_OUTPUT_HANDLER: - #print(errors, "repo output handler error:", validator.output_file_data) + # print(errors, "repo output handler error:", validator.output_file_data) logging.error(", ".join(validator.output_file_data)) validator.output_file_data.clear() if error not in repos_by_error: @@ -276,8 +283,8 @@ def get_contributors(repo): sorted_issues_list[issue] = open_issues_by_repo[issue] sorted_prs_list = {} - for pr in sorted(open_prs_by_repo, key=str.lower): - sorted_prs_list[pr] = open_prs_by_repo[pr] + for pull_request in sorted(open_prs_by_repo, key=str.lower): + sorted_prs_list[pull_request] = open_prs_by_repo[pull_request] sorted_repos_by_error = {} for error in sorted(repos_by_error, key=str.lower): @@ -286,8 +293,8 @@ def get_contributors(repo): # assemble the JSON data build_json = { "updated_at": run_time.strftime("%Y-%m-%dT%H:%M:%SZ"), - "contributors": [contrib for contrib in set(contributors)], - "reviewers": [rev for rev in set(reviewers)], + "contributors": list(set(contributors)), + "reviewers": list(set(reviewers)), "merged_pr_count": str(merged_pr_count_total), "library_updates": {"new": sorted_new_list, "updated": sorted_updated_list}, "open_issues": sorted_issues_list, diff --git a/pytest.ini b/pytest.ini index 0349a80..322c52c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,3 @@ [pytest] addopts = -v --tb=short --show-capture=no -testpaths = tests/unit/ tests/integration/ \ No newline at end of file +testpaths = tests/unit/ tests/integration/ diff --git a/requirements.txt b/requirements.txt index 4cb1cc7..8c0bfec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +black==21.6b0 packaging==20.3 pylint pytest diff --git a/tests/integration/test_arduino_libraries.py b/tests/integration/test_arduino_libraries.py index a12f107..2708d59 100644 --- a/tests/integration/test_arduino_libraries.py +++ b/tests/integration/test_arduino_libraries.py @@ -1,9 +1,35 @@ -import pytest +# The MIT License (MIT) +# +# Copyright (c) 2021 Michael Schroeder +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +"""Integration tests for 'adabot/arduino_libraries.py'""" + +import pytest # pylint: disable=unused-import from adabot import arduino_libraries from adabot import github_requests + def test_adafruit_libraries(monkeypatch): + """Test main arduino_libraries function, without writing an output file.""" def get_list_repos(): repos = [] @@ -16,7 +42,10 @@ def get_list_repos(): arduino_libraries.main() + +# pylint: disable=invalid-name def test_adafruit_libraries_output_file(monkeypatch, tmp_path, capsys): + """Test main arduino_libraries funciton, with writing an output file.""" def get_list_repos(): repos = [] @@ -31,4 +60,4 @@ def get_list_repos(): captured = capsys.readouterr() - assert tmp_output_file.read_text() == captured.out \ No newline at end of file + assert tmp_output_file.read_text() == captured.out diff --git a/tests/integration/test_circuitpython_libraries.py b/tests/integration/test_circuitpython_libraries.py index daab0a5..034945e 100644 --- a/tests/integration/test_circuitpython_libraries.py +++ b/tests/integration/test_circuitpython_libraries.py @@ -1,27 +1,64 @@ -import pytest +# The MIT License (MIT) +# +# Copyright (c) 2021 Michael Schroeder +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +"""Integration tests for 'adabot/circuitpython_libraries.py'""" + +import pytest # pylint: disable=unused-import from adabot.lib import common_funcs from adabot import github_requests from adabot import circuitpython_libraries + def test_circuitpython_libraires(monkeypatch): - + """Test main function of 'circuitpyton_libraries.py', without writing an output file.""" + # pylint: disable=unused-argument def mock_list_repos(*args, **kwargs): repos = [] - repos.append(github_requests.get("/repos/adafruit/Adafruit_CircuitPython_TestRepo").json()) + repos.append( + github_requests.get( + "/repos/adafruit/Adafruit_CircuitPython_TestRepo" + ).json() + ) return repos - + monkeypatch.setattr(common_funcs, "list_repos", mock_list_repos) circuitpython_libraries.main(validator="all") + +# pylint: disable=invalid-name def test_circuitpython_libraires_output_file(monkeypatch, tmp_path, capsys): - + """Test main funciton of 'circuitpython_libraries.py', with writing an output file.""" + # pylint: disable=unused-argument def mock_list_repos(*args, **kwargs): repos = [] - repos.append(github_requests.get("/repos/adafruit/Adafruit_CircuitPython_TestRepo").json()) + repos.append( + github_requests.get( + "/repos/adafruit/Adafruit_CircuitPython_TestRepo" + ).json() + ) return repos - + monkeypatch.setattr(common_funcs, "list_repos", mock_list_repos) tmp_output_file = tmp_path / "output_test.txt" @@ -30,4 +67,4 @@ def mock_list_repos(*args, **kwargs): captured = capsys.readouterr() - assert tmp_output_file.read_text() == captured.out \ No newline at end of file + assert tmp_output_file.read_text() == captured.out diff --git a/tests/unit/test_blinka_funcs.py b/tests/unit/test_blinka_funcs.py index 4047792..77771b0 100644 --- a/tests/unit/test_blinka_funcs.py +++ b/tests/unit/test_blinka_funcs.py @@ -1,6 +1,32 @@ -import pytest +# The MIT License (MIT) +# +# Copyright (c) 2021 Michael Schroeder +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +"""Unit tests for 'adabot/lib/blinka_funcs.py'""" + +import pytest # pylint: disable=unused-import from adabot.lib import blinka_funcs + def test_board_count(): - assert blinka_funcs.board_count() >= 0 \ No newline at end of file + """Test that 'board_count' returns a number.""" + assert blinka_funcs.board_count() >= 0 diff --git a/tests/unit/test_common_funcs.py b/tests/unit/test_common_funcs.py index 1e61dcf..e5f2426 100644 --- a/tests/unit/test_common_funcs.py +++ b/tests/unit/test_common_funcs.py @@ -1,10 +1,38 @@ -import pytest +# The MIT License (MIT) +# +# Copyright (c) 2021 Michael Schroeder +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +"""Unit tests for 'adabot/lib/common_funcs.py'""" + +import pytest # pylint: disable=unused-import from adabot.lib import common_funcs + def test_list_repos(): + """Test that list_repos returns a list object.""" repos = common_funcs.list_repos() assert isinstance(repos, list) + def test_repo_is_on_pypi_true(): + """Test 'repo_is_on_pypi'""" assert common_funcs.repo_is_on_pypi({"name": "pytest"}) diff --git a/tests/unit/test_github_requests.py b/tests/unit/test_github_requests.py index 1fa962d..eb10d0f 100644 --- a/tests/unit/test_github_requests.py +++ b/tests/unit/test_github_requests.py @@ -1,13 +1,43 @@ -import pytest +# The MIT License (MIT) +# +# Copyright (c) 2021 Michael Schroeder +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +"""Unit tests for 'adabot/github_requests.py'""" + +# pylint: disable=protected-access + +import pytest # pylint: disable=unused-import from adabot import github_requests + def test_fix_url(): + """Test URL fixing function.""" url = github_requests._fix_url("/meta") - assert url == "https://api.github.com/meta" + assert url == "https://api.github.com/meta" + def test_fix_kwargs(): + """Test kwarg fixing function.""" dummy_kwargs = github_requests._fix_kwargs({}) assert "headers" in dummy_kwargs - assert "Accept" in dummy_kwargs["headers"] \ No newline at end of file + assert "Accept" in dummy_kwargs["headers"] diff --git a/tests/unit/test_pypi_requests.py b/tests/unit/test_pypi_requests.py index fd27e00..142f9f9 100644 --- a/tests/unit/test_pypi_requests.py +++ b/tests/unit/test_pypi_requests.py @@ -1,7 +1,33 @@ -import pytest +# The MIT License (MIT) +# +# Copyright (c) 2021 Michael Schroeder +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +"""Unit tests for 'adabot/pypi_requests.py'""" + +import pytest # pylint: disable=unused-import from adabot import pypi_requests + def test_fix_url(): - url = pypi_requests._fix_url("/test") - assert url == "https://pypi.org/test" \ No newline at end of file + """Test URL fixing function.""" + url = pypi_requests._fix_url("/test") # pylint: disable=protected-access + assert url == "https://pypi.org/test"