Skip to content

Commit

Permalink
Lint Readable (#238)
Browse files Browse the repository at this point in the history
### Changes:
1. Logging option -v -vv -vv (verbosity levels - logging levels verbose-critical- .. - debug)
2. Logging path option - Until debug level.
3. Docker-sdk, git-python usage for better exceptions handling and output,
4. overall optimizations and code readability.
5. JSON output including all collected data for analytics.
6. Adding the following flags:
* test-xml - export test results as xml.
* json-report - export json report on all results.
* log - save the log to file.
7. Git python pack filtering - using set operations (save 19 sec- - each time).
8. Facts gathering optimization - Pipfile collecting, modules from master, additional packages adding to each package 'test-requirements.txt` in the package.
9. Context manager for mandatory test modules (CommonServerPython etc).
10. Code readability.
11. Docker image builds and not commit (Not the recommended way), Now we are using only Single Docker file to the setup test image.
  • Loading branch information
GalRabin committed Apr 2, 2020
1 parent 5bf3f0b commit 127c740
Show file tree
Hide file tree
Showing 32 changed files with 3,068 additions and 1,241 deletions.
102 changes: 54 additions & 48 deletions demisto_sdk/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,8 @@ def validate(config, **kwargs):
name="create-content-artifacts",
hidden=True,
short_help='Create content artifacts. This will generate content_new.zip file which can be used to '
'upload to your server in order to upload a whole new content version to your Demisto '
'instance.',
'upload to your server in order to upload a whole new content version to your Demisto '
'instance.',
)
@click.help_option(
'-h', '--help'
Expand Down Expand Up @@ -268,52 +268,58 @@ def secrets(config, **kwargs):

# ====================== lint ====================== #
@main.command(name="lint",
short_help="Run lintings (flake8, mypy, pylint, bandit, vulture) and pytest. pylint and pytest will run "
"within the docker image of an integration/script. Meant to be used with integrations/scripts "
"that use the folder (package) structure. Will lookup up what docker image to use and will "
"setup the dev dependencies and file in the target folder. ")
@click.help_option(
'-h', '--help'
)
@click.option(
"-d", "--dir", help="Specify directory of integration/script")
@click.option(
"--no-pylint", is_flag=True, help="Do NOT run pylint linter")
@click.option(
"--no-mypy", is_flag=True, help="Do NOT run mypy static type checking")
@click.option(
"--no-flake8", is_flag=True, help="Do NOT run flake8 linter")
@click.option(
"--no-bandit", is_flag=True, help="Do NOT run bandit linter")
@click.option(
"--no-vulture", is_flag=True, help="Do NOT run vulture linter")
@click.option(
"--no-test", is_flag=True, help="Do NOT test (skip pytest)")
@click.option(
"-r", "--root", is_flag=True, help="Run pytest container with root user")
@click.option(
"-k", "--keep-container", is_flag=True, help="Keep the test container")
@click.option(
"-v", "--verbose", is_flag=True, help="Verbose output - mainly for debugging purposes")
@click.option(
"--cpu-num",
help="Number of CPUs to run pytest on (can set to `auto` for automatic detection of the number of CPUs)",
default=0)
@click.option(
"-p", "--parallel", is_flag=True, help="Run tests in parallel")
@click.option(
"-m", "--max-workers", type=int, help="How many threads to run in parallel")
@click.option(
"-g", "--git", is_flag=True, help="Will run only on changed packages")
@click.option(
"-a", "--run-all-tests", is_flag=True, help="Run lint on all directories in content repo")
@click.option(
"--outfile", help="Save failing packages to a file"
)
@pass_config
def lint(config, dir, **kwargs):
linter = LintManager(configuration=config.configuration, project_dir_list=dir, **kwargs)
return linter.run_dev_packages()
short_help="Lint command will perform:\n 1. Package in host checks - flake8, bandit, mypy, vulture.\n 2. "
"Package in docker image checks - pylint, pytest, powershell - test, powershell - analyze.\n "
"Meant to be used with integrations/scripts that use the folder (package) structure. Will lookup up what"
"docker image to use and will setup the dev dependencies and file in the target folder. ")
@click.help_option('-h', '--help')
@click.option("-i", "--input", help="Specify directory of integration/script", type=click.Path(exists=True,
resolve_path=True))
@click.option("-g", "--git", is_flag=True, help="Will run only on changed packages")
@click.option("-a", "--all-packs", is_flag=True, help="Run lint on all directories in content repo")
@click.option('-v', "--verbose", count=True, help="Verbosity level -v / -vv / .. / -vvv",
type=click.IntRange(0, 3, clamp=True), default=2, show_default=True)
@click.option('-q', "--quiet", is_flag=True, help="Quiet output, only output results in the end")
@click.option("-p", "--parallel", default=1, help="Run tests in parallel", type=click.IntRange(0, 15, clamp=True),
show_default=True)
@click.option("--no-flake8", is_flag=True, help="Do NOT run flake8 linter")
@click.option("--no-bandit", is_flag=True, help="Do NOT run bandit linter")
@click.option("--no-mypy", is_flag=True, help="Do NOT run mypy static type checking")
@click.option("--no-vulture", is_flag=True, help="Do NOT run vulture linter")
@click.option("--no-pylint", is_flag=True, help="Do NOT run pylint linter")
@click.option("--no-test", is_flag=True, help="Do NOT test (skip pytest)")
@click.option("--no-pwsh-analyze", is_flag=True, help="Do NOT run powershell analyze")
@click.option("--no-pwsh-test", is_flag=True, help="Do NOT run powershell test")
@click.option("-kc", "--keep-container", is_flag=True, help="Keep the test container")
@click.option("--test-xml", help="Path to store pytest xml results", type=click.Path(exists=True, resolve_path=True))
@click.option("--json-report", help="Path to store json results", type=click.Path(exists=True, resolve_path=True))
@click.option("-lp", "--log-path", help="Path to store all levels of logs", type=click.Path(exists=True, resolve_path=True))
def lint(input: str, git: bool, all_packs: bool, verbose: int, quiet: bool, parallel: int, no_flake8: bool,
no_bandit: bool, no_mypy: bool, no_vulture: bool, no_pylint: bool, no_test: bool, no_pwsh_analyze: bool,
no_pwsh_test: bool, keep_container: bool, test_xml: str, json_report: str, log_path: str):
"""Lint command will perform:\n
1. Package in host checks - flake8, bandit, mypy, vulture.\n
2. Package in docker image checks - pylint, pytest, powershell - test, powershell - analyze.\n
Meant to be used with integrations/scripts that use the folder (package) structure. Will lookup up what
docker image to use and will setup the dev dependencies and file in the target folder."""
lint_manager = LintManager(input=input,
git=git,
all_packs=all_packs,
verbose=verbose,
quiet=quiet,
log_path=log_path)
return lint_manager.run_dev_packages(parallel=parallel,
no_flake8=no_flake8,
no_bandit=no_bandit,
no_mypy=no_mypy,
no_vulture=no_vulture,
no_pylint=no_pylint,
no_test=no_test,
no_pwsh_analyze=no_pwsh_analyze,
no_pwsh_test=no_pwsh_test,
keep_container=keep_container,
test_xml=test_xml,
json_report=json_report)


# ====================== format ====================== #
Expand Down

This file was deleted.

This file was deleted.

60 changes: 0 additions & 60 deletions demisto_sdk/commands/common/dev_sh_scripts/run_dev_tasks.sh

This file was deleted.

34 changes: 0 additions & 34 deletions demisto_sdk/commands/common/dev_sh_scripts/run_dev_tasks_pwsh.sh

This file was deleted.

45 changes: 0 additions & 45 deletions demisto_sdk/commands/common/dev_sh_scripts/run_mypy.sh

This file was deleted.

85 changes: 85 additions & 0 deletions demisto_sdk/commands/common/logger.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
import logging
import os


def logging_setup(verbose: int, quiet: bool, log_path: str) -> logging.Logger:
""" Init logger object for logging in demisto-sdk
For more info - https://docs.python.org/3/library/logging.html
Args:
verbose(int) verosity level - 1-3
quiet(bool): Whether to output a quiet response.
log_path(str): Path to save log of all levels
Returns:
logging.Logger: logger object
"""
if quiet:
verbose = 0
logger: logging.Logger = logging.getLogger('demisto-sdk')
logger.setLevel(logging.DEBUG)
log_level = logging.getLevelName((6 - 2 * verbose) * 10)
fmt = logging.Formatter('%(message)s')

if verbose:
console_handler = logging.StreamHandler()
console_handler.setLevel(log_level)
console_handler.setFormatter(fmt)
logger.addHandler(console_handler)

# Setting debug log file if in circleci
if log_path:
file_handler = logging.FileHandler(filename=os.path.join(log_path, 'lint_debug_log.log'))
file_handler.setFormatter(fmt)
file_handler.setLevel(level=logging.DEBUG)
logger.addHandler(file_handler)

logger.propagate = False

return logger


# Python program to print
# colored text and background
class Colors:
"""Colors class:reset all colors with colors.reset; two
sub classes fg for foreground
and bg for background; use as colors.subclass.colorname.
i.e. colors.fg.red or colors.bg.greenalso, the generic bold, disable,
underline, reverse, strike through,
and invisible work with the main class i.e. colors.bold"""
reset = '\033[0m'
bold = '\033[01m'
disable = '\033[02m'
underline = '\033[04m'
reverse = '\033[07m'
strikethrough = '\033[09m'
invisible = '\033[08m'

class Fg:
"""Forgrownd"""
black = '\033[30m'
red = '\033[31m'
green = '\033[32m'
orange = '\033[33m'
blue = '\033[34m'
purple = '\033[35m'
cyan = '\033[36m'
lightgrey = '\033[37m'
darkgrey = '\033[90m'
lightred = '\033[91m'
lightgreen = '\033[92m'
yellow = '\033[93m'
lightblue = '\033[94m'
pink = '\033[95m'
lightcyan = '\033[96m'

class Bg:
"""Backgrownd"""
black = '\033[40m'
red = '\033[41m'
green = '\033[42m'
orange = '\033[43m'
blue = '\033[44m'
purple = '\033[45m'
cyan = '\033[46m'
Loading

0 comments on commit 127c740

Please sign in to comment.