Circle build: use virtual env (#4753)

* move to use venv for circle build

* venv fixes

* mypy fixes

* fix for venv

* run pytest with python2

* add demisto-py 1.0.1 to build requirements

.circleci/build-requirements.txt

@@ -0,0 +1,10 @@
+awscli==1.16.264
+slackclient==1.3.2
+pyspellchecker==0.5.2
+ansible==2.8.6
+ansible-runner==1.4.2
+rsa==3.4.2
+paramiko==2.6.0
+boto3==1.10.0
+pyminizip===0.2.4
+git+https://github.com/demisto/demisto-py.git@v1.0.1 # disable-secrets-detection

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: devdemisto/content-build-2and3:2.0.0.2832  # disable-secrets-detection
+      - image: devdemisto/content-build:3.0.0.3368  # disable-secrets-detection
     resource_class: medium+
     environment:
       CONTENT_VERSION: "19.10.2"
@@ -11,22 +11,40 @@ jobs:
     steps:
       - checkout
       - setup_remote_docker
+      - restore_cache:
+          key: venv-{{ checksum "dev-requirements-py2.txt" }}-{{ checksum "dev-requirements-py3.txt" }}-{{ checksum ".circleci/build-requirements.txt" }}
       - run:
           name: Prepare Environment
           when: always
           command: |
               echo 'export CIRCLE_ARTIFACTS="/home/circleci/project/artifacts"' >> $BASH_ENV
-              echo 'export PATH="/home/circleci/.pyenv/shims:/home/circleci/.local/bin:/home/circleci/.pyenv/bin:${PATH}"' >> $BASH_ENV # disable-secrets-detection
+              echo 'export PATH="/home/circleci/.local/bin:${PATH}"' >> $BASH_ENV # disable-secrets-detection
               echo 'export PYTHONPATH="/home/circleci/project:${PYTHONPATH}"' >> $BASH_ENV
               echo "=== sourcing $BASH_ENV ==="
               source $BASH_ENV
               sudo mkdir -p -m 777 $CIRCLE_ARTIFACTS
               chmod +x ./Tests/scripts/*
               chmod +x ./Tests/lastest_server_build_scripts/*
-              pyenv versions
+              echo "Parameters: NIGHTLY: $NIGHTLY, NON_AMI_RUN: $NON_AMI_RUN, SERVER_BRANCH_NAME: $SERVER_BRANCH_NAME"
+              if [ ! -e "venv" ]; then
+                echo "installing venv"
+                NO_HOOKS=1 .hooks/bootstrap
+                source ./venv/bin/activate
+                pip install -r .circleci/build-requirements.txt
+                pip3 install -r .circleci/build-requirements.txt
+              else
+                echo "venv exists (from cache). activating"
+                source ./venv/bin/activate
+              fi
+              # store in bash env so we load our venv in each step
+              echo 'source /home/circleci/project/venv/bin/activate' >> $BASH_ENV
               python --version
               python3 --version
-              echo "Parameters: NIGHTLY: $NIGHTLY, NON_AMI_RUN: $NON_AMI_RUN, SERVER_BRANCH_NAME: $SERVER_BRANCH_NAME"
+      - save_cache:
+          paths:
+            - venv
+          key: venv-{{ checksum "dev-requirements-py2.txt" }}-{{ checksum "dev-requirements-py3.txt" }}-{{ checksum ".circleci/build-requirements.txt" }}
+
       - add_ssh_keys:
           fingerprints:
               - "02:df:a5:6a:53:9a:f5:5d:bd:a6:fc:b2:db:9b:c9:47" # disable-secrets-detection
@@ -40,16 +58,16 @@ jobs:
           name: Infrastucture testing
           when: always
           command: |
-            pytest ./Tests/scripts/hook_validations/tests/ -v
-            pytest ./Tests/scripts/infrastructure_tests/ -v
-            pytest ./Tests/scripts/test_configure_tests.py -v
+            python -m pytest ./Tests/scripts/hook_validations/tests/ -v
+            python -m pytest ./Tests/scripts/infrastructure_tests/ -v
+            python -m pytest ./Tests/scripts/test_configure_tests.py -v
       - run:
           name: Validate Files and Yaml
           when: always
           command: |
               # Run flake8 on all excluding Integraions and Scripts (they will be handled in linting)
               ./Tests/scripts/pyflake.sh *.py
-              find . -maxdepth 1 -type d -not \( -path . -o -path ./Integrations -o -path ./Scripts -o -path ./Beta_Integrations  \) | xargs ./Tests/scripts/pyflake.sh
+              find . -maxdepth 1 -type d -not \( -path . -o -path ./Integrations -o -path ./Scripts -o -path ./Beta_Integrations -o -path ./venv \) | xargs ./Tests/scripts/pyflake.sh
 
               [ -n "${BACKWARD_COMPATIBILITY}" ] && CHECK_BACKWARD=false || CHECK_BACKWARD=true
               python ./Tests/scripts/validate_files.py -c true -b $CHECK_BACKWARD

Integrations/Active_Directory_Query/Active_Directory_Query.py

@@ -1,6 +1,6 @@
 import demistomock as demisto
 from CommonServerPython import *
-from typing import *
+from typing import List, Dict, Optional
 from ldap3 import Server, Connection, NTLM, SUBTREE, ALL_ATTRIBUTES, Tls, Entry
 from ldap3.extend import microsoft
 import ssl
@@ -804,7 +804,7 @@ def add_member_to_group(default_base_dn):
 
     success = microsoft.addMembersToGroups.ad_add_members_to_groups(conn, [member_dn], [grp_dn])
     if not success:
-        raise Exception("Failed to add {} to group {]}".format(
+        raise Exception("Failed to add {} to group {}}".format(
             args.get('username') or args.get('computer-name'),
             args.get('group_name')
         ))

Integrations/Active_Directory_Query/CHANGELOG.md

@@ -1,5 +1,5 @@
 ## [Unreleased]
-
+-
 
 ## [19.10.1] - 2019-10-15
 Added support for **debug-mode**, which logs extended information when enabled.

Integrations/Exchange2016_Compliance/CHANGELOG.md

@@ -1,2 +1,5 @@
 ## [Unreleased]
+-
+
+## [19.9.0] - 2019-09-04
 Improved security inside the docker container.

Integrations/Exchange2016_Compliance/Exchange2016_Compliance.py

@@ -324,12 +324,12 @@ def get_compliance_search(search_name):
 
     if stderr:
         return_error(stderr)
-    stdout = stdout.split('\n', 1)
-    status = stdout[0].strip()
+    stdsplit = stdout.split('\n', 1)
+    status = stdsplit[0].strip()
     results = [get_cs_status(search_name, status)]
 
-    if status == 'Completed' and len(stdout[1].strip()) > 4:
-        res = list(r[:-1].split(', ') if r[-1] == ',' else r.split(', ') for r in stdout[1][2:-4].split(r'\r\n'))
+    if status == 'Completed' and len(stdsplit[1].strip()) > 4:
+        res = list(r[:-1].split(', ') if r[-1] == ',' else r.split(', ') for r in stdsplit[1][2:-4].split(r'\r\n'))
         res = map(lambda x: {k: v for k, v in (s.split(': ') for s in x)}, res)
         results.append(
             {
@@ -415,7 +415,7 @@ def test_module():
         encode_and_submit_results(check_purge_compliance_search(**args))
     elif demisto.command() == 'test-module':
         test_module()
-except Exception, e:
+except Exception as e:
     if isinstance(e, WindowsError):  # pylint: disable=undefined-variable
         return_error("Could not open powershell on the target engine.")
     else:

Integrations/Exchange2016_Compliance/Exchange2016_Compliance.yml

@@ -17,14 +17,15 @@ configuration:
   defaultvalue: ""
   type: 0
   required: true
-- display: Trust any certificate (unsecure)
+- display: Trust any certificate (not secure)
   name: insecure
   defaultvalue: ""
   type: 8
   required: false
 script:
   script: ''
   type: python
+  subtype: python2
   commands:
   - name: exchange2016-start-compliance-search
     arguments:

Integrations/Looker/CHANGELOG.md

@@ -0,0 +1,2 @@
+## [Unreleased]
+-

Integrations/Looker/Looker.py

@@ -222,7 +222,7 @@ def get_entries_for_search_results(contents, look_id=None, result_format='json',
 
         if contents:
             entries.append(  # type: ignore
-                'This command has dynamic output keys.\n'
+                'This command has dynamic output keys.\n'  # type: ignore
                 'To access them in the context, copy the key\'s path from the column header in the results table.'
             )
 

Integrations/Netcraft/CHANGELOG.md

@@ -1,5 +1,5 @@
 ## [Unreleased]
-
+-
 
 ## [19.8.0] - 2019-08-06
 -

Integrations/Netcraft/Netcraft.py

@@ -5,7 +5,6 @@
 ''' IMPORTS '''
 import requests
 from requests.auth import HTTPBasicAuth
-from typing import *
 
 
 # Disable insecure warnings

dev-requirements-py2.txt

@@ -4,5 +4,7 @@ PyYAML==5.1.2
 pykwalify==1.7.0
 demisto-py==2.0.5
 pytest==4.6.5
+pytest-mock==1.11.1
 beautifulsoup4==4.8.1
-pypdf2==1.26.0
+pypdf2==1.26.0
+git+https://github.com/coyo8/parinx.git@6493798ceba8089345d970f71be4a896eb6b081d

dev-requirements-py3.txt

@@ -4,6 +4,7 @@ PyYAML==5.1.2
 pykwalify==1.7.0
 demisto-py==2.0.5
 pytest==5.2.1
+pytest-mock==1.11.1
 ruamel.yaml==0.16.5
 pipenv==2018.11.26
 autopep8==1.4.4