Skip to content

Commit

Permalink
[Pre Commit] infrastructure enhancements (#3850)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ilaner
ilaner committed Dec 14, 2023
1 parent 1e65edc commit 0b17afe
Show file tree
Hide file tree
Showing 23 changed files with 816 additions and 618 deletions.
7 changes: 7 additions & 0 deletions .pre-commit-config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ repos:
- cachetools==5.3.0 ; python_version >= "3.8" and python_version < "3.11"
- certifi==2022.12.7 ; python_version >= "3.8" and python_version < "3.11"
- cffi==1.15.1 ; python_version >= "3.8" and python_version < "3.11"
- cfgv==3.3.1 ; python_version >= "3.8" and python_version < "3.11"
- chardet==5.1.0 ; python_version >= "3.8" and python_version < "3.11"
- charset-normalizer==3.1.0 ; python_version >= "3.8" and python_version < "3.11"
- click==8.1.3 ; python_version >= "3.8" and python_version < "3.11"
Expand All @@ -51,9 +52,11 @@ repos:
- demisto-py==3.2.10 ; python_version >= "3.8" and python_version < "3.11"
- dictdiffer==0.9.0 ; python_version >= "3.8" and python_version < "3.11"
- dictor==0.1.11 ; python_version >= "3.8" and python_version < "3.11"
- distlib==0.3.6 ; python_version >= "3.8" and python_version < "3.11"
- docker==5.0.3 ; python_version >= "3.8" and python_version < "3.11"
- docopt==0.6.2 ; python_version >= "3.8" and python_version < "3.11"
- exceptiongroup==1.1.1 ; python_version >= "3.8" and python_version < "3.11"
- filelock==3.12.0 ; python_version >= "3.8" and python_version < "3.11"
- flatten-dict==0.4.2 ; python_version >= "3.8" and python_version < "3.11"
- freezegun==1.2.2 ; python_version >= "3.8" and python_version < "3.11"
- future==0.18.3 ; python_version >= "3.8" and python_version < "3.11"
Expand All @@ -74,6 +77,7 @@ repos:
- grpcio-status==1.48.2 ; python_version >= "3.8" and python_version < "3.11"
- grpcio==1.59.2 ; python_version >= "3.8" and python_version < "3.11"
- humanfriendly==10.0 ; python_version >= "3.8" and python_version < "3.11"
- identify==2.5.24 ; python_version >= "3.8" and python_version < "3.11"
- idna==3.4 ; python_version >= "3.8" and python_version < "3.11"
- imagesize==1.4.1 ; python_version >= "3.8" and python_version < "3.11"
- importlib-resources==5.12.0 ; python_version >= "3.8" and python_version < "3.11"
Expand All @@ -93,6 +97,7 @@ repos:
- neo4j==5.14.1 ; python_version >= "3.8" and python_version < "3.11"
- networkx==2.8.8 ; python_version >= "3.8" and python_version < "3.11"
- nltk==3.8.1 ; python_version >= "3.8" and python_version < "3.11"
- nodeenv==1.7.0 ; python_version >= "3.8" and python_version < "3.11"
- ordered-set==4.1.0 ; python_version >= "3.8" and python_version < "3.11"
- orjson==3.8.11 ; python_version >= "3.8" and python_version < "3.11"
- packaging==23.1 ; python_version >= "3.8" and python_version < "3.11"
Expand All @@ -102,6 +107,7 @@ repos:
- pkgutil-resolve-name==1.3.10 ; python_version >= "3.8" and python_version < "3.9"
- platformdirs==3.5.0 ; python_version >= "3.8" and python_version < "3.11"
- pluggy==1.0.0 ; python_version >= "3.8" and python_version < "3.11"
- pre-commit==3.5.0 ; python_version >= "3.8" and python_version < "3.11"
- prettytable==3.7.0 ; python_version >= "3.8" and python_version < "3.11"
- proto-plus==1.22.3 ; python_version >= "3.8" and python_version < "3.11"
- protobuf==3.19.6 ; python_version >= "3.8" and python_version < "3.11"
Expand Down Expand Up @@ -176,6 +182,7 @@ repos:
- tzlocal==4.3 ; python_version >= "3.8" and python_version < "3.11"
- ujson==5.7.0 ; python_version >= "3.8" and python_version < "3.11"
- urllib3==1.26.15 ; python_version >= "3.8" and python_version < "3.11"
- virtualenv==20.23.0 ; python_version >= "3.8" and python_version < "3.11"
- vulture==2.7 ; python_version >= "3.8" and python_version < "3.11"
- wcmatch==8.4.1 ; python_version >= "3.8" and python_version < "3.11"
- wcwidth==0.2.6 ; python_version >= "3.8" and python_version < "3.11"
Expand Down
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@
* Added support for `skip` property in **pre-commit** hooks.
* **generate-unit-tests** command will require installation with `pip install demisto-sdk[generate-unit-tests]`.
* added the *IN150* and *IN161* errors to *allowed ignore errors* list.
* Added support for `env`, `copy_files` property in **pre-commit** docker hooks.
* Added support to run specific hooks in **pre-commit**. Use with `demisto-sdk pre-commit <hook>`.
* **Breaking change**: Removed the command **run-unit-tests**. Use `demisto-sdk pre-commit pytest-in-docker` instead.
* **Breaking change**: Removed the `--unit-test` argument in **pre-commit**. To skip unit tests, run with `--no-docker` or with `skip=pytest-in-docker`,

## 1.24.0
* Fixed an issue where the error was not clear when trying to retrieve the server version.
Expand Down
291 changes: 100 additions & 191 deletions demisto_sdk/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import logging
import os
from pathlib import Path
from typing import IO, Any, Dict, Iterable, Optional, Tuple, Union
from typing import IO, Any, Dict, List, Optional, Tuple, Union

import typer
from pkg_resources import DistributionNotFound, get_distribution
Expand Down Expand Up @@ -3452,192 +3452,6 @@ def update_content_graph(
)


@main.command(short_help="Runs pre-commit hooks on the files in the repository")
@click.help_option("-h", "--help")
@click.option(
"-i",
"--input",
help="The path to the input file to run the command on.",
type=PathsParamType(
exists=True, resolve_path=True
), # PathsParamType allows passing a list of paths
)
@click.option(
"-s",
"--staged-only",
help="Whether to run only on staged files",
is_flag=True,
default=False,
)
@click.option(
"--commited-only",
help="Whether to run on commited files only",
is_flag=True,
default=False,
)
@click.option(
"-g",
"--git-diff",
help="Whether to use git to determine which files to run on",
is_flag=True,
default=False,
)
@click.option(
"-a",
"--all-files",
help="Whether to run on all files",
is_flag=True,
default=False,
)
@click.option(
"--mode",
help="Special mode to run the pre-commit with",
)
@click.option(
"-ut/--no-ut",
"--unit-test/--no-unit-test",
help="Whether to run unit tests for content items",
default=False,
)
@click.option(
"--skip",
help="A comma separated list of precommit hooks to skip",
)
@click.option(
"--validate/--no-validate",
help="Whether to run demisto-sdk validate",
default=True,
)
@click.option(
"--format/--no-format",
help="Whether to run demisto-sdk format",
default=False,
)
@click.option(
"--secrets/--no-secrets",
help="Whether to run demisto-sdk secrets",
default=True,
)
@click.option(
"-v",
"--verbose",
help="Verbose output of pre-commit",
is_flag=True,
default=False,
)
@click.option(
"--show-diff-on-failure",
help="Show diff on failure",
is_flag=True,
default=False,
)
@click.option(
"--sdk-ref",
help="The demisto-sdk ref to use for the pre-commit hooks",
default="",
)
@click.option(
"--dry-run",
help="Whether to run the pre-commit hooks in dry-run mode, which will only create the config file",
is_flag=True,
default=False,
)
@click.argument(
"file_paths",
nargs=-1,
type=click.Path(exists=True, resolve_path=True, path_type=Path),
)
@click.option(
"--docker/--no-docker",
help="Whether to run docker based hooks or not.",
default=True,
is_flag=True,
)
@click.pass_context
@logging_setup_decorator
def pre_commit(
ctx,
input: str,
staged_only: bool,
commited_only: bool,
git_diff: bool,
all_files: bool,
mode: Optional[str],
unit_test: bool,
skip: str,
validate: bool,
format: bool,
secrets: bool,
verbose: bool,
show_diff_on_failure: bool,
sdk_ref: str,
file_paths: Iterable[Path],
dry_run: bool,
docker: bool,
**kwargs,
):
from demisto_sdk.commands.pre_commit.pre_commit_command import pre_commit_manager

if file_paths and input:
logger.info(
"Both `--input` parameter and `file_paths` arguments were provided. Will use the `--input` parameter."
)
input_files = []
if input:
input_files = [Path(i) for i in input.split(",") if i]
elif file_paths:
input_files = list(file_paths)
if skip:
skip = skip.split(",") # type: ignore[assignment]

sys.exit(
pre_commit_manager(
input_files,
staged_only,
commited_only,
git_diff,
all_files,
mode,
unit_test,
skip,
validate,
format,
secrets,
verbose,
show_diff_on_failure,
run_docker_hooks=docker,
sdk_ref=sdk_ref,
dry_run=dry_run,
)
)


@main.command(short_help="Run unit tests in a docker for integrations and scripts")
@click.help_option("-h", "--help")
@click.option(
"-i",
"--input",
type=PathsParamType(
exists=True, resolve_path=True
), # PathsParamType allows passing a list of paths
help="The path of the content pack/file to validate specifically.",
)
@click.option(
"-v", "--verbose", is_flag=True, default=False, help="Verbose output of unit tests"
)
@click.argument("file_paths", nargs=-1, type=click.Path(exists=True, resolve_path=True))
@click.pass_context
@logging_setup_decorator
def run_unit_tests(
ctx, input: str, file_paths: Tuple[str, ...], verbose: bool, **kwargs
):
if input:
file_paths = tuple(input.split(","))
from demisto_sdk.commands.run_unit_tests.unit_tests_runner import unit_test_runner

sys.exit(unit_test_runner(file_paths, verbose))


@main.command(short_help="Setup integration environments")
@click.option(
"-i",
Expand Down Expand Up @@ -3708,11 +3522,106 @@ def exit_from_program(result=0, **kwargs):
sys.exit(result)


# ====================== Pre-Commit ====================== #
pre_commit_app = typer.Typer(name="Pre-Commit")


@pre_commit_app.command()
def pre_commit(
input_files: Optional[List[Path]] = typer.Option(
None,
"-i",
"--input",
"--files",
exists=True,
dir_okay=True,
resolve_path=True,
show_default=False,
help=("The paths to run pre-commit on. May pass multiple paths."),
),
staged_only: bool = typer.Option(
False, "--staged-only", help="Whether to run only on staged files"
),
commited_only: bool = typer.Option(
False, "--commited-only", help="Whether to run on commited files only"
),
git_diff: bool = typer.Option(
False,
"--git-diff",
"-g",
help="Whether to use git to determine which files to run on",
),
all_files: bool = typer.Option(
False, "--all-files", "-a", help="Whether to run on all files"
),
mode: str = typer.Option(
"", "--mode", help="Special mode to run the pre-commit with"
),
skip: Optional[List[str]] = typer.Option(
None, "--skip", help="A list of precommit hooks to skip"
),
validate: bool = typer.Option(
True, "--validate/--no-validate", help="Whether to run demisto-sdk validate"
),
format: bool = typer.Option(
False, "--format/--no-format", help="Whether to run demisto-sdk format"
),
secrets: bool = typer.Option(
True, "--secrets/--no-secrets", help="Whether to run demisto-sdk secrets"
),
verbose: bool = typer.Option(
False, "-v", "--verbose", help="Verbose output of pre-commit"
),
show_diff_on_failure: bool = typer.Option(
False, "--show-diff-on-failure", help="Show diff on failure"
),
dry_run: bool = typer.Option(
False,
"--dry-run",
help="Whether to run the pre-commit hooks in dry-run mode, which will only create the config file",
),
docker: bool = typer.Option(
True, "--docker/--no-docker", help="Whether to run docker based hooks or not."
),
run_hook: Optional[str] = typer.Argument(None, help="A specific hook to run"),
):
from demisto_sdk.commands.pre_commit.pre_commit_command import pre_commit_manager

return_code = pre_commit_manager(
input_files,
staged_only,
commited_only,
git_diff,
all_files,
mode,
skip,
validate,
format,
secrets,
verbose,
show_diff_on_failure,
run_docker_hooks=docker,
dry_run=dry_run,
run_hook=run_hook,
)
if return_code:
raise typer.Exit(1)


main.add_command(typer.main.get_command(pre_commit_app), "pre-commit")


# ====================== modeling-rules command group ====================== #
app = typer.Typer(name="modeling-rules", hidden=True, no_args_is_help=True)
app.command("test", no_args_is_help=True)(test_modeling_rule.test_modeling_rule)
app.command("init-test-data", no_args_is_help=True)(init_test_data.init_test_data)
typer_click_object = typer.main.get_command(app)
modeling_rules_app = typer.Typer(
name="modeling-rules", hidden=True, no_args_is_help=True
)
modeling_rules_app.command("test", no_args_is_help=True)(
test_modeling_rule.test_modeling_rule
)
modeling_rules_app.command("init-test-data", no_args_is_help=True)(
init_test_data.init_test_data
)
typer_click_object = typer.main.get_command(modeling_rules_app)
main.add_command(typer_click_object, "modeling-rules")

app_generate_modeling_rules = typer.Typer(
Expand Down

0 comments on commit 0b17afe

Please sign in to comment.