Merge branch 'master' into sdk-release

.github/pull_request_template.md

@@ -1,7 +1,9 @@
 <!-- REMINDER: THIS IS A PUBLIC REPO DO NOT POST HERE SECRETS/SENSITIVE DATA -->
 
 ## Status
-Ready/In Progress/In Hold (Reason for hold)
+- [x] In Progress
+- [ ] Ready
+- [ ] In Hold - (Reason for hold)
 
 ## Related Issues
 fixes: link to the issue
@@ -11,3 +13,7 @@ A few sentences describing the overall goals of the pull request's commits.
 
 ## Screenshots
 Paste here any images that will help the reviewer
+
+## Must have
+- [ ] Tests
+- [ ] Documentation

.pre-commit-config.yaml

@@ -4,7 +4,9 @@ repos:
     rev: v2.4.0
     hooks:
     -   id: trailing-whitespace
+        exclude: demisto_sdk/tests/test_files/*
     -   id: end-of-file-fixer
+        exclude: demisto_sdk/tests/test_files/*
     -   id: check-docstring-first
         exclude: demisto_sdk/tests/test_files/*
     -   id: check-json
@@ -20,3 +22,7 @@ repos:
     rev: v1.5
     hooks:
     -   id: autopep8
+-   repo: https://github.com/pre-commit/mirrors-isort
+    rev: v4.3.21
+    hooks:
+    -   id: isort

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
-
+* Added *-c, --command* option in *generate-docs* to generate specific command from integration.
+* Fixed an issue in getting a README/CHANGELOG files from git and loading it.
+* Removed release notes validation for new content.
+* Fixed secretes validations for files with the same name in a different directory.
+* demisto-sdk lint - parralel working with specify the number of workers.
+* demisto-sdk lint - logging levels output, 3 levels.
+* demisto-sdk lint - json report, structured error reports in json format.
+* demisto-sdk lint - xml junit report for unit-tests.
+* demisto-sdk lint - New packages used in order to excellarate execution time.
+
+#### 0.5.0
 [PyPI History][1]
 
 [1]: https://pypi.org/project/demisto-sdk/#history
@@ -12,6 +22,8 @@
     * Added the *-fv --from-version*, *-nv --no-validation* arguments.
     * Removed the *-t yml_type* argument, the file type will be inferred.
     * Removed the *-g use_git* argument, running format without arguments will run automatically on git diff.
+* Fixed an issue in loading playbooks with '=' character.
+* Fixed an issue in *validate* failed on deleted README files.
 
 ### 0.4.8
 * Added the *max* field to the Playbook schema, allowing to define it in tasks loop.

CONTRIBUTION.md

@@ -33,7 +33,7 @@ git clone https://github.com/demisto/demisto-sdk.git
    pip3 uninstall demisto-sdk
    ```
 
-2. Inside root directory of `demisto-sdk` repository - Install PyPi package as [editable package](https://pip.pypa.io/en/stable/reference/pip_install/):
+2. Inside root directory of `demisto-sdk` repository - Install PyPi package as [editable package](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs):
 
    ```shell
    pip3 install -e .
@@ -51,7 +51,7 @@ git clone https://github.com/demisto/demisto-sdk.git
 4. Install dev-requirements -
 
    ```shell
-   pip3 install <repo>/resources/utils/requirements-dev.txt
+   pip3 install -r <repo>/requirements-dev.txt
    ```
 
 ---

README.md

@@ -284,6 +284,14 @@ Generate documentation file for integration, playbook or script from yaml file.
     Each Command should be in a separate line. **For script** - the script example surrounded by double quotes.
     When the argument is empty, the documentation will be generate without examples.
 
+* **-c, --command**
+    **For integration only** - A comma-separated value of commands to replace (e.g `panorama-get-address,panorama-create-address`).
+    It can be used only with an existing README.md file.
+    If any new command is presented - the new command will be added to the end of the docs file.
+    For an existing command, it will try to replace the existing command in the README.md file. If it will fail, it will
+    add it to the end of the file as well.
+
+
 * **-id, --id_set**
     Path of updated id_set.json file, used for generates script documentation.
      When the argument is empty, the documentation will be generate without `Used In` section.
@@ -301,6 +309,9 @@ This will generate documentation file to Tanium V2 integration in /Users/Documen
 `demisto-sdk generate-docs -o /Users/Documentations -i /demisto/content/Scripts/script-PrintErrorEntry.yml -id /demisto/content/Tests/id_set.json -e "!PrintErrorEntry message=Hi"`
 This will generate documentation file to PrintErrorEntry script in /Users/Documentations/README.md. id_set.json should be updated to gets all the integration that uses this script.
 
+`demisto-sdk generate-docs -i /demisto/content/Integrations/Tanium_v2/Tanium_v2.yml -c tn-get-package`
+This will generate the command section for `tn-get-package` and will replace it in the README.md file located in /demisto/content/Integrations/Tanium_v2/.
+
 ## create-id-set
 Create the content dependency tree by ids.
 

demisto_sdk/__main__.py

@@ -1,34 +1,42 @@
 # Site packages
 import os
-from pkg_resources import get_distribution
 import sys
 
+from pkg_resources import get_distribution
+
 # Third party packages
 import click
-
-# Import demisto-sdk commands
-from demisto_sdk.commands.run_cmd.runner import Runner
+from demisto_sdk.commands.common.configuration import Configuration
+# Common tools
+from demisto_sdk.commands.common.tools import (find_type,
+                                               get_last_remote_release_version,
+                                               print_error, print_warning)
+from demisto_sdk.commands.create_artifacts.content_creator import \
+    ContentCreator
+from demisto_sdk.commands.create_id_set.create_id_set import IDSetCreator
+from demisto_sdk.commands.find_dependencies.find_dependencies import \
+    PackDependencies
 from demisto_sdk.commands.format.format_module import format_manager
-from demisto_sdk.commands.unify.unifier import Unifier
-from demisto_sdk.commands.upload.uploader import Uploader
+from demisto_sdk.commands.generate_docs.generate_integration_doc import \
+    generate_integration_doc
+from demisto_sdk.commands.generate_docs.generate_playbook_doc import \
+    generate_playbook_doc
+from demisto_sdk.commands.generate_docs.generate_script_doc import \
+    generate_script_doc
+from demisto_sdk.commands.generate_test_playbook.test_playbook_generator import \
+    PlaybookTestsGenerator
 from demisto_sdk.commands.init.initiator import Initiator
-from demisto_sdk.commands.split_yml.extractor import Extractor
-from demisto_sdk.commands.common.configuration import Configuration
+from demisto_sdk.commands.json_to_outputs.json_to_outputs import \
+    json_to_outputs
 from demisto_sdk.commands.lint.lint_manager import LintManager
-from demisto_sdk.commands.secrets.secrets import SecretsValidator
+# Import demisto-sdk commands
+from demisto_sdk.commands.run_cmd.runner import Runner
 from demisto_sdk.commands.run_playbook.playbook_runner import PlaybookRunner
+from demisto_sdk.commands.secrets.secrets import SecretsValidator
+from demisto_sdk.commands.split_yml.extractor import Extractor
+from demisto_sdk.commands.unify.unifier import Unifier
+from demisto_sdk.commands.upload.uploader import Uploader
 from demisto_sdk.commands.validate.file_validator import FilesValidator
-from demisto_sdk.commands.create_artifacts.content_creator import ContentCreator
-from demisto_sdk.commands.json_to_outputs.json_to_outputs import json_to_outputs
-from demisto_sdk.commands.generate_test_playbook.test_playbook_generator import PlaybookTestsGenerator
-from demisto_sdk.commands.generate_docs.generate_integration_doc import generate_integration_doc
-from demisto_sdk.commands.generate_docs.generate_script_doc import generate_script_doc
-from demisto_sdk.commands.generate_docs.generate_playbook_doc import generate_playbook_doc
-from demisto_sdk.commands.create_id_set.create_id_set import IDSetCreator
-from demisto_sdk.commands.find_dependencies.find_dependencies import PackDependencies
-
-# Common tools
-from demisto_sdk.commands.common.tools import print_error, print_warning, get_last_remote_release_version, find_type
 
 
 class DemistoSDK:
@@ -61,7 +69,7 @@ def main(config, version):
                       f'You should consider upgrading via "pip3 install --upgrade demisto-sdk" command.')
     if version:
         version = get_distribution('demisto-sdk').version
-        print(version)
+        print(f'demisto-sdk {version}')
 
 
 # ====================== split-yml ====================== #
@@ -219,8 +227,8 @@ def validate(config, **kwargs):
     name="create-content-artifacts",
     hidden=True,
     short_help='Create content artifacts. This will generate content_new.zip file which can be used to '
-    'upload to your server in order to upload a whole new content version to your Demisto '
-    'instance.',
+               'upload to your server in order to upload a whole new content version to your Demisto '
+               'instance.',
 )
 @click.help_option(
     '-h', '--help'
@@ -268,52 +276,58 @@ def secrets(config, **kwargs):
 
 # ====================== lint ====================== #
 @main.command(name="lint",
-              short_help="Run lintings (flake8, mypy, pylint, bandit, vulture) and pytest. pylint and pytest will run "
-                         "within the docker image of an integration/script. Meant to be used with integrations/scripts "
-                         "that use the folder (package) structure. Will lookup up what docker image to use and will "
-                         "setup the dev dependencies and file in the target folder. ")
-@click.help_option(
-    '-h', '--help'
-)
-@click.option(
-    "-d", "--dir", help="Specify directory of integration/script")
-@click.option(
-    "--no-pylint", is_flag=True, help="Do NOT run pylint linter")
-@click.option(
-    "--no-mypy", is_flag=True, help="Do NOT run mypy static type checking")
-@click.option(
-    "--no-flake8", is_flag=True, help="Do NOT run flake8 linter")
-@click.option(
-    "--no-bandit", is_flag=True, help="Do NOT run bandit linter")
-@click.option(
-    "--no-vulture", is_flag=True, help="Do NOT run vulture linter")
-@click.option(
-    "--no-test", is_flag=True, help="Do NOT test (skip pytest)")
-@click.option(
-    "-r", "--root", is_flag=True, help="Run pytest container with root user")
-@click.option(
-    "-k", "--keep-container", is_flag=True, help="Keep the test container")
-@click.option(
-    "-v", "--verbose", is_flag=True, help="Verbose output - mainly for debugging purposes")
-@click.option(
-    "--cpu-num",
-    help="Number of CPUs to run pytest on (can set to `auto` for automatic detection of the number of CPUs)",
-    default=0)
-@click.option(
-    "-p", "--parallel", is_flag=True, help="Run tests in parallel")
-@click.option(
-    "-m", "--max-workers", type=int, help="How many threads to run in parallel")
-@click.option(
-    "-g", "--git", is_flag=True, help="Will run only on changed packages")
-@click.option(
-    "-a", "--run-all-tests", is_flag=True, help="Run lint on all directories in content repo")
-@click.option(
-    "--outfile", help="Save failing packages to a file"
-)
-@pass_config
-def lint(config, dir, **kwargs):
-    linter = LintManager(configuration=config.configuration, project_dir_list=dir, **kwargs)
-    return linter.run_dev_packages()
+              short_help="Lint command will perform:\n 1. Package in host checks - flake8, bandit, mypy, vulture.\n 2. "
+                         "Package in docker image checks -  pylint, pytest, powershell - test, powershell - analyze.\n "
+                         "Meant to be used with integrations/scripts that use the folder (package) structure. Will lookup up what"
+                         "docker image to use and will setup the dev dependencies and file in the target folder. ")
+@click.help_option('-h', '--help')
+@click.option("-i", "--input", help="Specify directory of integration/script", type=click.Path(exists=True,
+                                                                                               resolve_path=True))
+@click.option("-g", "--git", is_flag=True, help="Will run only on changed packages")
+@click.option("-a", "--all-packs", is_flag=True, help="Run lint on all directories in content repo")
+@click.option('-v', "--verbose", count=True, help="Verbosity level -v / -vv / .. / -vvv",
+              type=click.IntRange(0, 3, clamp=True), default=2, show_default=True)
+@click.option('-q', "--quiet", is_flag=True, help="Quiet output, only output results in the end")
+@click.option("-p", "--parallel", default=1, help="Run tests in parallel", type=click.IntRange(0, 15, clamp=True),
+              show_default=True)
+@click.option("--no-flake8", is_flag=True, help="Do NOT run flake8 linter")
+@click.option("--no-bandit", is_flag=True, help="Do NOT run bandit linter")
+@click.option("--no-mypy", is_flag=True, help="Do NOT run mypy static type checking")
+@click.option("--no-vulture", is_flag=True, help="Do NOT run vulture linter")
+@click.option("--no-pylint", is_flag=True, help="Do NOT run pylint linter")
+@click.option("--no-test", is_flag=True, help="Do NOT test (skip pytest)")
+@click.option("--no-pwsh-analyze", is_flag=True, help="Do NOT run powershell analyze")
+@click.option("--no-pwsh-test", is_flag=True, help="Do NOT run powershell test")
+@click.option("-kc", "--keep-container", is_flag=True, help="Keep the test container")
+@click.option("--test-xml", help="Path to store pytest xml results", type=click.Path(exists=True, resolve_path=True))
+@click.option("--json-report", help="Path to store json results", type=click.Path(exists=True, resolve_path=True))
+@click.option("-lp", "--log-path", help="Path to store all levels of logs", type=click.Path(exists=True, resolve_path=True))
+def lint(input: str, git: bool, all_packs: bool, verbose: int, quiet: bool, parallel: int, no_flake8: bool,
+         no_bandit: bool, no_mypy: bool, no_vulture: bool, no_pylint: bool, no_test: bool, no_pwsh_analyze: bool,
+         no_pwsh_test: bool, keep_container: bool, test_xml: str, json_report: str, log_path: str):
+    """Lint command will perform:\n
+        1. Package in host checks - flake8, bandit, mypy, vulture.\n
+        2. Package in docker image checks -  pylint, pytest, powershell - test, powershell - analyze.\n
+    Meant to be used with integrations/scripts that use the folder (package) structure. Will lookup up what
+    docker image to use and will setup the dev dependencies and file in the target folder."""
+    lint_manager = LintManager(input=input,
+                               git=git,
+                               all_packs=all_packs,
+                               verbose=verbose,
+                               quiet=quiet,
+                               log_path=log_path)
+    return lint_manager.run_dev_packages(parallel=parallel,
+                                         no_flake8=no_flake8,
+                                         no_bandit=no_bandit,
+                                         no_mypy=no_mypy,
+                                         no_vulture=no_vulture,
+                                         no_pylint=no_pylint,
+                                         no_test=no_test,
+                                         no_pwsh_analyze=no_pwsh_analyze,
+                                         no_pwsh_test=no_pwsh_test,
+                                         keep_container=keep_container,
+                                         test_xml=test_xml,
+                                         json_report=json_report)
 
 
 # ====================== format ====================== #
@@ -524,6 +538,11 @@ def init(**kwargs):
 @click.option(
     "-uc", "--use_cases", help="For integration - Top use-cases. Number the steps by '*' (i.e. '* foo. * bar.')",
     required=False)
+@click.option(
+    "-c", "--command", help="A comma-separated command names to generate doc for, will ignore the rest of the commands."
+                            "e.g (xdr-get-incidents,xdr-update-incident",
+    required=False
+)
 @click.option(
     "-e", "--examples", help="Path for file containing command or script examples."
                              " Each Command should be in a separate line."
@@ -545,6 +564,7 @@ def init(**kwargs):
 def generate_doc(**kwargs):
     input_path = kwargs.get('input')
     output_path = kwargs.get('output')
+    command = kwargs.get('command')
     examples = kwargs.get('examples')
     permissions = kwargs.get('permissions')
     limitations = kwargs.get('limitations')
@@ -564,9 +584,16 @@ def generate_doc(**kwargs):
         print_error(F'Output directory {output_path} was not found.')
         return 1
 
+    if command:
+        if output_path and (not os.path.isfile(os.path.join(output_path, "README.md")))\
+                or (not output_path)\
+                and (not os.path.isfile(os.path.join(os.path.dirname(os.path.realpath(input_path)), "README.md"))):
+            print_error("The `command` argument must be presented with existing `README.md` docs.")
+            return 1
+
     file_type = find_type(kwargs.get('input', ''))
     if file_type not in ["integration", "script", "playbook"]:
-        print_error(F'File is not an Integration, Script or a Playbook.')
+        print_error('File is not an Integration, Script or a Playbook.')
         return 1
 
     print(f'Start generating {file_type} documentation...')
@@ -576,7 +603,7 @@ def generate_doc(**kwargs):
         return generate_integration_doc(input=input_path, output=output_path, use_cases=use_cases,
                                         examples=examples, permissions=permissions,
                                         command_permissions=command_permissions, limitations=limitations,
-                                        insecure=insecure, verbose=verbose)
+                                        insecure=insecure, verbose=verbose, command=command)
     elif file_type == 'script':
         return generate_script_doc(input=input_path, output=output_path, examples=examples, permissions=permissions,
                                    limitations=limitations, insecure=insecure, verbose=verbose)

demisto_sdk/commands/common/configuration.py

@@ -1,5 +1,5 @@
-import os
 import logging
+from pathlib import Path
 
 
 class Configuration:
@@ -15,6 +15,7 @@ class Configuration:
     def __init__(self, log_verbose=False, logging_level=logging.INFO):
         logging.basicConfig(level=logging_level)
         self.log_verbose = log_verbose
-        self.sdk_env_dir = os.path.dirname(os.path.dirname(os.path.join(__file__)))
-        self.env_dir = os.getcwd()
-        self.envs_dirs_base = os.path.join(self.sdk_env_dir, 'lint', 'dev_envs', 'default_python')
+        # refers to "demisto_sdk/commands" dir
+        self.sdk_env_dir = str(Path(__file__).parent.parent)
+        self.env_dir = str(Path().cwd())
+        self.envs_dirs_base = str(Path(self.sdk_env_dir) / 'lint' / 'resources' / 'pipfile_python')

demisto_sdk/commands/common/constants.py

@@ -10,6 +10,11 @@ def feed_wrong_from_version(file_path, given_fromversion, needed_from_version="5
         return "{} is a feed and has wrong fromversion. got `{}` expected `{}`" \
             .format(file_path, given_fromversion, needed_from_version)
 
+    @staticmethod
+    def pwsh_wrong_version(file_path, given_fromversion, needed_from_version='5.5.0'):
+        return (f'{file_path}: detected type: powershell and fromversion less than {needed_from_version}.'
+                f' Found version: {given_fromversion}')
+
     @staticmethod
     def not_used_display_name(file_path, field_name):
         return "The display details for {} will not be used in the file {} due to the type of the parameter".format(
@@ -255,7 +260,7 @@ def found_hidden_param(parameter_name):
 INDICATOR_FIELDS_DIR = 'IndicatorFields'
 LAYOUTS_DIR = 'Layouts'
 CLASSIFIERS_DIR = 'Classifiers'
-MISC_DIR = 'Packs/Base/Misc'
+MISC_DIR = 'Misc'
 CONNECTIONS_DIR = 'Connections'
 BETA_INTEGRATIONS_DIR = 'Beta_Integrations'
 PACKS_DIR = 'Packs'
@@ -1054,3 +1059,5 @@ class PB_Status:
         'type': 8
     }
 ]
+
+DOCS_COMMAND_SECTION_REGEX = r'(?:###\s{}).+?(?:(?=(?:\n###\s))|(?=(?:\n##\s))|\Z)'