Update graph build

.gitlab/ci/.gitlab-ci.yml

@@ -1,6 +1,6 @@
 
 default:
-  image: docker-io.art.code.pan.run/devdemisto/gitlab-content-ci:1.0.0.41253
+  image: docker-io.art.code.pan.run/devdemisto/gitlab-content-ci:1.0.0.41284
   artifacts:
     expire_in: 30 days
     paths:
@@ -19,6 +19,7 @@ stages:
 variables:
   DONT_CACHE_LAST_RESPONSE: "true"
   GCS_MARKET_BUCKET: "marketplace-dist"
+  GCS_MARKET_BUCKET_DEV: "marketplace-dist-dev"
   GCS_MARKET_V2_BUCKET: "marketplace-v2-dist"
   GCS_MARKET_XPANSE_BUCKET: "xpanse-dist"
   STORAGE_BASE_PATH: ""

.gitlab/ci/global.yml

@@ -143,6 +143,13 @@
   - npm link jsdoc-to-markdown@5.0.3 | tee --append $ARTIFACTS_FOLDER/logs/installations.log  # disable-secrets-detection
   - section_end "Installing node modules"
 
+.get_last_upload_commit: &get_last_upload_commit
+  - section_start "Getting last bucket upload commit"
+  - gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" > auth.out 2>&1
+  - gsutil cp "gs://$GCS_MARKET_BUCKET/content/packs/index.json" "$ARTIFACTS_FOLDER/previous_index.json"
+  - export LAST_UPLOAD_COMMIT=$(cat $ARTIFACTS_FOLDER/previous_index.json | jq -r ".\"commit\"")
+  - section_end "Getting last bucket upload commit"
+
 .default-before-script:
   before_script:
     - *setup-network-certs
@@ -155,9 +162,10 @@
     - *install_node_modules
     - *install_venv
     - *get_contribution_pack
+    - *get_last_upload_commit
     - *install_ssh_keys
     - section_start "Build Parameters"
-    - set | grep -E "^NIGHTLY=|^INSTANCE_TESTS=|^SERVER_BRANCH_NAME=|^ARTIFACT_BUILD_NUM=|^DEMISTO_SDK_NIGHTLY=|^TIME_TO_LIVE=|^CONTRIB_BRANCH=|^FORCE_PACK_UPLOAD=|^PACKS_TO_UPLOAD=|^BUCKET_UPLOAD=|^STORAGE_BASE_PATH=|^OVERRIDE_ALL_PACKS=|^GCS_MARKET_BUCKET=|^GCS_MARKET_V2_BUCKET=|^GCS_MARKET_XPANSE_BUCKET=|^SLACK_CHANNEL=|^NVM_DIR=|^NODE_VERSION=|^PATH=|^ARTIFACTS_FOLDER=|^ENV_RESULTS_PATH="
+    - set | grep -E "^NIGHTLY=|^INSTANCE_TESTS=|^SERVER_BRANCH_NAME=|^ARTIFACT_BUILD_NUM=|^DEMISTO_SDK_NIGHTLY=|^TIME_TO_LIVE=|^CONTRIB_BRANCH=|^FORCE_PACK_UPLOAD=|^PACKS_TO_UPLOAD=|^BUCKET_UPLOAD=|^STORAGE_BASE_PATH=|^OVERRIDE_ALL_PACKS=|^GCS_MARKET_BUCKET=|^GCS_MARKET_V2_BUCKET=|^GCS_MARKET_XPANSE_BUCKET=|^SLACK_CHANNEL=|^NVM_DIR=|^NODE_VERSION=|^PATH=|^ARTIFACTS_FOLDER=|^ENV_RESULTS_PATH=|^LAST_UPLOAD_COMMIT="
     - python --version
     - python2 --version
     - python3 --version
@@ -242,10 +250,7 @@
           demisto-sdk lint -vvv -p 10 -a --test-xml ./unit-tests --log-path $ARTIFACTS_FOLDER --failure-report $ARTIFACTS_FOLDER --coverage-report $ARTIFACTS_FOLDER/coverage_report -dt 120 --time-measurements-dir $ARTIFACTS_FOLDER
         else
           if [[ -n $BUCKET_UPLOAD ]]; then
-            gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" > auth.out 2>&1
-            gsutil cp "gs://$GCS_MARKET_BUCKET/content/packs/index.json" "$ARTIFACTS_FOLDER/previous_index.json"
-            export COMMIT_HASH_COMPARE_TO=$(cat $ARTIFACTS_FOLDER/previous_index.json | jq -r ".\"commit\"")
-            demisto-sdk lint -vvv -p 8 -g --no-mypy --prev-ver $COMMIT_HASH_COMPARE_TO -v --test-xml ./unit-tests --log-path $ARTIFACTS_FOLDER --failure-report $ARTIFACTS_FOLDER --coverage-report $ARTIFACTS_FOLDER/coverage_report -idp $ARTIFACTS_FOLDER/id_set.json -cdam
+            demisto-sdk lint -vvv -p 8 -g --no-mypy --prev-ver $LAST_UPLOAD_COMMIT -v --test-xml ./unit-tests --log-path $ARTIFACTS_FOLDER --failure-report $ARTIFACTS_FOLDER --coverage-report $ARTIFACTS_FOLDER/coverage_report -idp $ARTIFACTS_FOLDER/id_set.json -cdam
           else
             echo "Skipping, Should run on circleCi."
           fi

.gitlab/ci/staging.yml

@@ -19,10 +19,36 @@
 
     - !reference [.download-demisto-conf]
     - !reference [.create-release-notes-and-common-docs]
-
-    - section_start "Create content graph" --collapsed
-    - demisto-sdk create-content-graph -o $ARTIFACTS_FOLDER/content-graph.dump
-    - section_end "Create content graph"
+
+    - section_start "Create or update content graph" --collapsed
+
+    - |
+      echo "Trying to pull all CSV import files from bucket"
+      gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" > auth.out 2>&1
+      # gsutil -m cp -r "gs://$GCS_MARKET_BUCKET_DEV/content_graph/*" /var/lib/neo4j/import
+      # TODO: uncomment line above after finish updating upload flow with graph
+
+    - |
+      if [ -z "$(ls -A /var/lib/neo4j/import)" ]; then
+        echo "No CSV files to import, creating the content graph"
+        demisto-sdk create-content-graph
+        echo "Successfully created content graph"
+      else
+        echo "Getting the list of packs to update in content graph"
+        PACKS_TO_UPDATE=( $(git diff $LAST_UPLOAD_COMMIT --name-only -- Packs/* | awk -F/ '{print $2}' | sort | uniq) )
+        echo $PACKS_TO_UPDATE
+
+        echo "Updating the content graph"
+        demisto-sdk update-content-graph -p $PACKS_TO_UPDATE
+        echo "Successfully updated content graph"
+      fi
+
+    - |
+      echo "Moving exported CSV files to artifacts"
+      mkdir $ARTIFACTS_FOLDER/content_graph
+      cp -R /var/lib/neo4j/import $ARTIFACTS_FOLDER/content_graph
+
+    - section_end "Update content graph"
 
     - section_start "Create Content Artifacts and Update Conf" --collapsed
     - |

Packs/FeedIntel471/ReleaseNotes/2_0_13.md

@@ -0,0 +1,6 @@
+
+#### Indicator Fields
+- **Forum Total Count**
+- **Forum Post Total Count**
+- **Instant Message Total Count**
+- **Report Total Count**

Packs/FeedIntel471/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Intel471 Feed",
     "description": "This content pack fetches actor and malware related indicators from Intel 471. It also fetches watcher alerts.",
     "support": "partner",
-    "currentVersion": "2.0.12",
+    "currentVersion": "2.0.13",
     "author": "Intel 471",
     "url": "https://www.intel471.com",
     "email": "support@intel471.com",

Packs/Zendesk/ReleaseNotes/2_0_2.md

@@ -0,0 +1,3 @@
+
+#### Incident Types
+- **Zendesk Ticket**

Packs/Zendesk/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Zendesk",
     "description": "IT service management",
     "support": "xsoar",
-    "currentVersion": "2.0.1",
+    "currentVersion": "2.0.2",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",