From 0fc616dd0b77e2c5cbb8ce8f8ea9427452ffda90 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 23 May 2023 17:46:27 +0200
Subject: [PATCH 01/33] Update test-with-integrations pipeline

---
 .buildkite/hooks/pre-command                  |  10 ++
 .../pipeline.test-with-integrations-repo.yml  |   7 +
 .buildkite/scripts/install_deps.sh            |  25 ++++
 .buildkite/scripts/test-with-integrations.sh  | 129 ++++++++++++++++++
 4 files changed, 171 insertions(+)
 create mode 100755 .buildkite/scripts/test-with-integrations.sh

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 28de6bd904..a988ae7835 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -70,3 +70,13 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-package-storage-publish" &&
     # publishing job
     export PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH})
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-with-integrations" && "$BUILDKITE_STEP_KEY" == "pr-integrations" ]]; then
+    export GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+
+    export GITHUB_USERNAME_SECRET="user"
+    export GITHUB_EMAIL_SECRET="myemail"
+    export GITHUB_TOKEN=""
+fi
diff --git a/.buildkite/pipeline.test-with-integrations-repo.yml b/.buildkite/pipeline.test-with-integrations-repo.yml
index 0a6532301a..2c08fd94fe 100644
--- a/.buildkite/pipeline.test-with-integrations-repo.yml
+++ b/.buildkite/pipeline.test-with-integrations-repo.yml
@@ -1,5 +1,7 @@
 env:
   GOLANG_VERSION: "1.20.3"
+  GH_CLI_VERSION: "2.29.0"
+  JQ_VERSION: "1.6"
 
 steps:
   - label: ":go: Run check-static"
@@ -9,4 +11,9 @@ steps:
       image: "golang:${GOLANG_VERSION}"
       cpu: "8"
       memory: "4G"
+  - label: ":hammer: Create PR in integrations"
+    key: pr-integrations
+    command: ".buildkite/scripts/test-with-integrations.sh"
+    agents:
+      provider: "gcp"
 
diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh
index 33c408a4c1..6094559867 100755
--- a/.buildkite/scripts/install_deps.sh
+++ b/.buildkite/scripts/install_deps.sh
@@ -39,3 +39,28 @@ with_docker_compose() {
     chmod +x ${WORKSPACE}/bin/docker-compose
     docker-compose version
 }
+
+with_github_cli() {
+    mkdir -p ${WORKSPACE}/bin
+    mkdir -p ${WORKSPACE}/tmp
+
+    local gh_filename="gh_${GH_CLI_VERSION}_linux_amd64"
+    local gh_tar_file="${gh_filename}.tar.gz"
+
+    retry 5 curl -sL -o ${WORKSPACE}/tmp "https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/${gh_tar_file}"
+
+    tar -C ${WORKSPACE}/bin -xpf ${WORKSPACE}/tmp/${gh_tar_file} ${gh_filename}/bin/gh --strip-components=2
+
+    chmod +x ${WORKSPACE}/bin/gh
+    rm -rf ${WORKSPACE}/tmp
+
+    gh version
+}
+
+with_jq() {
+    mkdir -p ${WORKSPACE}/bin
+    retry 5 curl -sL -o ${WORKSPACE}/bin/jq "https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64"
+
+    chmod +x ${WORKSPACE}/bin/jq
+    jq --version
+}
diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
new file mode 100755
index 0000000000..8d31ebe967
--- /dev/null
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -0,0 +1,129 @@
+#!/bin/bash
+
+set -euo pipefail
+
+WORKSPACE="$(pwd)"
+
+TMP_FOLDER_TEMPLATE="tmp.repo"
+
+cleanup() {
+    echo "Deleting temporal files..."
+    cd ${WORKSPACE}
+    rm -rf "${TMP_FOLDER_TEMPLATE}.*"
+    echo "Done."
+}
+
+trap cleanup EXIT
+source .buildkite/scripts/install_deps.sh
+
+add_bin_path
+
+with_github_cli
+with_jq
+
+INTEGRATIONS_SOURCE_BRANCH=main
+INTEGRATIONS_REPO=github.com:elastic/integrations.git
+INTEGRATIONS_PR_BRANCH="test-elastic-package-pr-${BUILDKITE_PULL_REQUEST}"
+INTEGRATIONS_PR_TITLE="Test elastic-package - DO NOT MERGE"
+
+
+get_pr_number() {
+    local branch="$1"
+    gh pr list -H "${branch}" --json number | jq -r '.[]|.number'
+}
+
+get_integrations_pr_link() {
+    local pr_number=$1
+    echo "https://github.com/elastic/integrations/pull/${pr_number}"
+}
+
+get_elastic_package_pr_link() {
+    echo "https://github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}/pull/${BUILDKITE_PULL_REQUEST}"
+}
+
+set_git_config() {
+    git config user.name "${GITHUB_USERNAME_SECRET}"
+    git config user.email "${GITHUB_EMAIL_SECRET}"
+}
+
+git_push_with_auth() {
+    local branch=$1
+
+    retry 3 git push https://${GITHUB_USERNAME_SECRET}:${GITHUB_TOKEN}@github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}.git "${branch}"
+}
+
+clone_repository() {
+    local target="$1"
+    retry 5 git clone https://github.com/elastic/integrations ${target}
+}
+
+create_pull_request() {
+    echo "Creating Pull Request"
+    retry 3 \
+        gh pr create \
+        --title "${INTEGRATIONS_PR_TITLE}" \
+        --body "Update elastic-package reference to ${GITHUB_PR_HEAD_SHA}.\nAutomated by [Buildkite build](${BUILDKITE_BUILD_URL})\n\nRelates: $(get_elastic_package_pr_link)" \
+        --draft \
+        --base ${INTEGRATIONS_SOURCE_BRANCH} \
+        --head ${INTEGRATIONS_PR_BRANCH} \
+        --assignee ${BUILDKITE_PR_HEAD_USER}
+}
+
+update_dependency() {
+    go mod edit -replace github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}=github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}@${GITHUB_PR_HEAD_SHA}
+    go mod tidy
+
+    git add go.mod
+    git add go.sum
+
+    git commit -m "Test elastic-package from PR ${BUILDKITE_PULL_REQUEST} - ${GITHUB_PR_HEAD_SHA}"
+}
+
+
+create_or_update_pull_request() {
+    local temp_path=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
+    local repo_path="${temp_path}/elastic-integrations"
+    local checkout_options=""
+    local integrations_pr_number=""
+
+    clone_repository "${repo_path}"
+    pushd "${repo_path}" > /dev/null
+
+    set_git_config
+
+    integrations_pr_number=$(get_pr_number "${INTEGRATIONS_PR_BRANCH}")
+    if [ -z "${integrations_pr_number}" ]; then
+        checkout_options=" -b "
+    fi
+    git checkout ${checkout_options} ${INTEGRATIONS_PR_BRANCH}
+
+    update_dependency
+
+    return
+    git_push_with_auth
+
+    if [ -z "${integrations_pr_number}" ]; then
+        create_pull_request
+    fi
+
+    popd > /dev/null
+
+    rm -rf "${temp_path}"
+}
+
+
+add_pr_comment() {
+    local pr_number="$1"
+    retry 3 \
+        gh pr comment ${pr_number} \
+        --body "Created or updated PR in integrations repostiory to test this vesrion. Check ${get_integrations_pr_link}" \
+        --repo ${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}
+}
+
+
+# integrations repository
+create_or_update_pull_request
+
+exit 0
+# elastic-package
+add_pr_comment "${BUILDKITE_PULL_REQUEST}"

From 2211434de31011baa6c7d2274005a285e933028e Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 23 May 2023 18:11:00 +0200
Subject: [PATCH 02/33] Adding some echos

---
 .buildkite/hooks/pre-command                 | 6 +++---
 .buildkite/scripts/test-with-integrations.sh | 8 ++++++--
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index a988ae7835..8e1853ea20 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -76,7 +76,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-with-integrations" &&
     export GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
     export GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
 
-    export GITHUB_USERNAME_SECRET="user"
-    export GITHUB_EMAIL_SECRET="myemail"
-    export GITHUB_TOKEN=""
+    export GITHUB_USERNAME_SECRET="useruser"
+    export GITHUB_EMAIL_SECRET="myemailemail"
+    export GITHUB_TOKEN="tokentoken"
 fi
diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 8d31ebe967..057e7b0b0a 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -18,7 +18,10 @@ source .buildkite/scripts/install_deps.sh
 
 add_bin_path
 
+echo "--- install gh cli"
 with_github_cli
+
+echo "--- install jq"
 with_jq
 
 INTEGRATIONS_SOURCE_BRANCH=main
@@ -121,9 +124,10 @@ add_pr_comment() {
 }
 
 
-# integrations repository
+echo "--- creating or updating integrations pull request"
 create_or_update_pull_request
 
 exit 0
-# elastic-package
+
+echo "--- adding comment into elastic-package pull request"
 add_pr_comment "${BUILDKITE_PULL_REQUEST}"

From bc78dcf307ef5ccbbc4097a7c0a529696ced5b0c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 23 May 2023 18:20:25 +0200
Subject: [PATCH 03/33] fix output parameter in curl

---
 .buildkite/scripts/install_deps.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh
index 6094559867..0cf6c68660 100755
--- a/.buildkite/scripts/install_deps.sh
+++ b/.buildkite/scripts/install_deps.sh
@@ -46,10 +46,12 @@ with_github_cli() {
 
     local gh_filename="gh_${GH_CLI_VERSION}_linux_amd64"
     local gh_tar_file="${gh_filename}.tar.gz"
+    local gh_tar_full_path="${WORKSPACE}/tmp/${gh_tar_file}"
 
-    retry 5 curl -sL -o ${WORKSPACE}/tmp "https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/${gh_tar_file}"
+    retry 5 curl -sL -o ${gh_tar_full_path} "https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/${gh_tar_file}"
 
-    tar -C ${WORKSPACE}/bin -xpf ${WORKSPACE}/tmp/${gh_tar_file} ${gh_filename}/bin/gh --strip-components=2
+    # just extract the binary file from the tar.gz
+    tar -C ${WORKSPACE}/bin -xpf ${gh_tar_full_path} ${gh_filename}/bin/gh --strip-components=2
 
     chmod +x ${WORKSPACE}/bin/gh
     rm -rf ${WORKSPACE}/tmp

From 4d3504bc848e6d6afb1300161800e0c21fc830a9 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 23 May 2023 18:48:32 +0200
Subject: [PATCH 04/33] fix template for mktemp

---
 .buildkite/scripts/test-with-integrations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 057e7b0b0a..45bb9c0a57 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -4,7 +4,7 @@ set -euo pipefail
 
 WORKSPACE="$(pwd)"
 
-TMP_FOLDER_TEMPLATE="tmp.repo"
+TMP_FOLDER_TEMPLATE="tmp.${GITHUB_PR_BASE_REPO}.XXXXXXXXX"
 
 cleanup() {
     echo "Deleting temporal files..."

From 0a5566b5f63013d8fc18dcba540a544c2422c31f Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 23 May 2023 19:00:44 +0200
Subject: [PATCH 05/33] Add depth in clone and some emojis

---
 .buildkite/scripts/test-with-integrations.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 45bb9c0a57..2980693596 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -57,7 +57,7 @@ git_push_with_auth() {
 
 clone_repository() {
     local target="$1"
-    retry 5 git clone https://github.com/elastic/integrations ${target}
+    retry 5 git clone --depth 1 https://github.com/elastic/integrations ${target}
 }
 
 create_pull_request() {
@@ -89,6 +89,7 @@ create_or_update_pull_request() {
     local checkout_options=""
     local integrations_pr_number=""
 
+    echo "Cloning repository"
     clone_repository "${repo_path}"
     pushd "${repo_path}" > /dev/null
 
@@ -96,16 +97,19 @@ create_or_update_pull_request() {
 
     integrations_pr_number=$(get_pr_number "${INTEGRATIONS_PR_BRANCH}")
     if [ -z "${integrations_pr_number}" ]; then
+        echo "Exists PR in integrations repository: ${integrations_pr_number}"
         checkout_options=" -b "
     fi
     git checkout ${checkout_options} ${INTEGRATIONS_PR_BRANCH}
 
+    echo "Updating dependency :pushpin:"
     update_dependency
 
     return
     git_push_with_auth
 
     if [ -z "${integrations_pr_number}" ]; then
+        echo "Creating pull request :github:"
         create_pull_request
     fi
 
@@ -129,5 +133,5 @@ create_or_update_pull_request
 
 exit 0
 
-echo "--- adding comment into elastic-package pull request"
+echo "--- adding comment into elastic-package pull request :memo:"
 add_pr_comment "${BUILDKITE_PULL_REQUEST}"

From d2718adc4036cb465c71c9dfc40f10d6096cccee Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 23 May 2023 19:05:12 +0200
Subject: [PATCH 06/33] Use github token

---
 .buildkite/hooks/pre-command | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 8e1853ea20..5ec9ce5ac1 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -78,5 +78,4 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-with-integrations" &&
 
     export GITHUB_USERNAME_SECRET="useruser"
     export GITHUB_EMAIL_SECRET="myemailemail"
-    export GITHUB_TOKEN="tokentoken"
 fi

From dd782f88d8365c3ba5fd42f1fc49942e2f77f810 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 23 May 2023 19:07:41 +0200
Subject: [PATCH 07/33] add comments

---
 .buildkite/scripts/test-with-integrations.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 2980693596..f866d04091 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -31,6 +31,7 @@ INTEGRATIONS_PR_TITLE="Test elastic-package - DO NOT MERGE"
 
 
 get_pr_number() {
+    # requires GITHUB_TOKEN
     local branch="$1"
     gh pr list -H "${branch}" --json number | jq -r '.[]|.number'
 }
@@ -61,6 +62,7 @@ clone_repository() {
 }
 
 create_pull_request() {
+    # requires GITHUB_TOKEN
     echo "Creating Pull Request"
     retry 3 \
         gh pr create \

From 0f34aba05e269b1afbd990bcadba4242d7f60ecd Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 23 May 2023 19:13:59 +0200
Subject: [PATCH 08/33] Add go instalation

---
 .buildkite/pipeline.test-with-integrations-repo.yml | 1 +
 .buildkite/scripts/test-with-integrations.sh        | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/.buildkite/pipeline.test-with-integrations-repo.yml b/.buildkite/pipeline.test-with-integrations-repo.yml
index 2c08fd94fe..d102e84edd 100644
--- a/.buildkite/pipeline.test-with-integrations-repo.yml
+++ b/.buildkite/pipeline.test-with-integrations-repo.yml
@@ -1,4 +1,5 @@
 env:
+  SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
   GOLANG_VERSION: "1.20.3"
   GH_CLI_VERSION: "2.29.0"
   JQ_VERSION: "1.6"
diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index f866d04091..28f4f36982 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -24,6 +24,7 @@ with_github_cli
 echo "--- install jq"
 with_jq
 
+
 INTEGRATIONS_SOURCE_BRANCH=main
 INTEGRATIONS_REPO=github.com:elastic/integrations.git
 INTEGRATIONS_PR_BRANCH="test-elastic-package-pr-${BUILDKITE_PULL_REQUEST}"
@@ -75,6 +76,10 @@ create_pull_request() {
 }
 
 update_dependency() {
+    # it needs to set the Golang version from the integrations repository (.go-version file)
+    echo "--- install go"
+    with_go
+
     go mod edit -replace github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}=github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}@${GITHUB_PR_HEAD_SHA}
     go mod tidy
 

From ed4cd88b94d4813505256ac10704c1ac17d647fc Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 12:18:24 +0200
Subject: [PATCH 09/33] Add git show

---
 .buildkite/hooks/pre-command                 |  4 ++--
 .buildkite/scripts/test-with-integrations.sh | 14 ++++++++++----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 5ec9ce5ac1..2e0e8573b8 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -72,8 +72,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-package-storage-publish" &&
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-with-integrations" && "$BUILDKITE_STEP_KEY" == "pr-integrations" ]]; then
-    export GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
-    export GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
+    # export GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
+    # export GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
     export GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
 
     export GITHUB_USERNAME_SECRET="useruser"
diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 28f4f36982..bee0aabcff 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -4,12 +4,13 @@ set -euo pipefail
 
 WORKSPACE="$(pwd)"
 
-TMP_FOLDER_TEMPLATE="tmp.${GITHUB_PR_BASE_REPO}.XXXXXXXXX"
+TMP_FOLDER_TEMPLATE_BASE="tmp.${GITHUB_PR_BASE_REPO}"
+TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 
 cleanup() {
     echo "Deleting temporal files..."
     cd ${WORKSPACE}
-    rm -rf "${TMP_FOLDER_TEMPLATE}.*"
+    rm -rf "${TMP_FOLDER_TEMPLATE_BASE}.*"
     echo "Done."
 }
 
@@ -77,7 +78,7 @@ create_pull_request() {
 
 update_dependency() {
     # it needs to set the Golang version from the integrations repository (.go-version file)
-    echo "--- install go"
+    echo "--- install go for integrations repository :go:"
     with_go
 
     go mod edit -replace github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}=github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}@${GITHUB_PR_HEAD_SHA}
@@ -87,6 +88,10 @@ update_dependency() {
     git add go.sum
 
     git commit -m "Test elastic-package from PR ${BUILDKITE_PULL_REQUEST} - ${GITHUB_PR_HEAD_SHA}"
+
+    echo ""
+    git show -p HEAD
+    echo ""
 }
 
 
@@ -112,7 +117,6 @@ create_or_update_pull_request() {
     echo "Updating dependency :pushpin:"
     update_dependency
 
-    return
     git_push_with_auth
 
     if [ -z "${integrations_pr_number}" ]; then
@@ -135,6 +139,8 @@ add_pr_comment() {
 }
 
 
+echo "> check ${GITHUB_USERNAME_SECRET}"
+echo "> check ${GITHUB_EMAIL_SECRET}"
 echo "--- creating or updating integrations pull request"
 create_or_update_pull_request
 

From bdbc9bdc99556cfb4f9c1c1fecc872251dc5db8c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 12:34:33 +0200
Subject: [PATCH 10/33] test

---
 .buildkite/hooks/pre-command | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 2e0e8573b8..4137ed7510 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -78,4 +78,5 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-with-integrations" &&
 
     export GITHUB_USERNAME_SECRET="useruser"
     export GITHUB_EMAIL_SECRET="myemailemail"
+    echo "In hook test ${GITHUB_USERNAME_SECRET} - ${GITHUB_EMAIL_SECRET}"
 fi

From 424a6efd483cc0be1b9034c5ddc9b9b3362e59b8 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 12:44:01 +0200
Subject: [PATCH 11/33] Add more messages

---
 .buildkite/scripts/test-with-integrations.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index bee0aabcff..7c21282855 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -117,6 +117,7 @@ create_or_update_pull_request() {
     echo "Updating dependency :pushpin:"
     update_dependency
 
+    echo "Pushing branch ${INTEGRATIONS_PR_BRANCH} to integrations repository..."
     git_push_with_auth
 
     if [ -z "${integrations_pr_number}" ]; then

From 99b9ed7492e16591fcb1478bac87e058ceb8528e Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 12:58:48 +0200
Subject: [PATCH 12/33] Add no pager

---
 .buildkite/scripts/test-with-integrations.sh | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 7c21282855..b52bb3a0d3 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -81,6 +81,7 @@ update_dependency() {
     echo "--- install go for integrations repository :go:"
     with_go
 
+    echo "--- Updating go.mod and go.sum :hammer_and_wrench:"
     go mod edit -replace github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}=github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}@${GITHUB_PR_HEAD_SHA}
     go mod tidy
 
@@ -90,7 +91,7 @@ update_dependency() {
     git commit -m "Test elastic-package from PR ${BUILDKITE_PULL_REQUEST} - ${GITHUB_PR_HEAD_SHA}"
 
     echo ""
-    git show -p HEAD
+    git --no-pager show -p HEAD
     echo ""
 }
 
@@ -109,19 +110,22 @@ create_or_update_pull_request() {
 
     integrations_pr_number=$(get_pr_number "${INTEGRATIONS_PR_BRANCH}")
     if [ -z "${integrations_pr_number}" ]; then
-        echo "Exists PR in integrations repository: ${integrations_pr_number}"
+        echo "No PR created, creating a new branch..."
         checkout_options=" -b "
+    else
+        echo "Exists PR in integrations repository: ${integrations_pr_number}"
     fi
+
     git checkout ${checkout_options} ${INTEGRATIONS_PR_BRANCH}
 
-    echo "Updating dependency :pushpin:"
+    echo "--- Updating dependency :pushpin:"
     update_dependency
 
-    echo "Pushing branch ${INTEGRATIONS_PR_BRANCH} to integrations repository..."
+    echo "--- Pushing branch ${INTEGRATIONS_PR_BRANCH} to integrations repository..."
     git_push_with_auth
 
     if [ -z "${integrations_pr_number}" ]; then
-        echo "Creating pull request :github:"
+        echo "--- Creating pull request :github:"
         create_pull_request
     fi
 

From d4c99fbf047a3d273dd1ec47a54a1a5e644d3a25 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 13:10:48 +0200
Subject: [PATCH 13/33] Test creation PR

---
 .buildkite/hooks/pre-command                 | 8 ++------
 .buildkite/scripts/test-with-integrations.sh | 8 ++------
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 4137ed7510..894e7e77cf 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -72,11 +72,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-package-storage-publish" &&
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-with-integrations" && "$BUILDKITE_STEP_KEY" == "pr-integrations" ]]; then
-    # export GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
-    # export GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
     export GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
-
-    export GITHUB_USERNAME_SECRET="useruser"
-    export GITHUB_EMAIL_SECRET="myemailemail"
-    echo "In hook test ${GITHUB_USERNAME_SECRET} - ${GITHUB_EMAIL_SECRET}"
 fi
diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index b52bb3a0d3..c21327bdc2 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -53,7 +53,7 @@ set_git_config() {
 }
 
 git_push_with_auth() {
-    local branch=$1
+    local branch="$1"
 
     retry 3 git push https://${GITHUB_USERNAME_SECRET}:${GITHUB_TOKEN}@github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}.git "${branch}"
 }
@@ -122,7 +122,7 @@ create_or_update_pull_request() {
     update_dependency
 
     echo "--- Pushing branch ${INTEGRATIONS_PR_BRANCH} to integrations repository..."
-    git_push_with_auth
+    git_push_with_auth ${INTEGRATIONS_PR_BRANCH}
 
     if [ -z "${integrations_pr_number}" ]; then
         echo "--- Creating pull request :github:"
@@ -144,12 +144,8 @@ add_pr_comment() {
 }
 
 
-echo "> check ${GITHUB_USERNAME_SECRET}"
-echo "> check ${GITHUB_EMAIL_SECRET}"
 echo "--- creating or updating integrations pull request"
 create_or_update_pull_request
 
-exit 0
-
 echo "--- adding comment into elastic-package pull request :memo:"
 add_pr_comment "${BUILDKITE_PULL_REQUEST}"

From 71198fb226d6c819fed397aff34bd6c0e45b1bf6 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 13:19:08 +0200
Subject: [PATCH 14/33] Parametrize git_push function

---
 .buildkite/scripts/test-with-integrations.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index c21327bdc2..dcc70b9e78 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -27,7 +27,8 @@ with_jq
 
 
 INTEGRATIONS_SOURCE_BRANCH=main
-INTEGRATIONS_REPO=github.com:elastic/integrations.git
+INTEGRATIONS_GITHUB_OWNER=elastic
+INTEGRATIONS_GITHUB_REPO_NAME=integrations.git
 INTEGRATIONS_PR_BRANCH="test-elastic-package-pr-${BUILDKITE_PULL_REQUEST}"
 INTEGRATIONS_PR_TITLE="Test elastic-package - DO NOT MERGE"
 
@@ -53,9 +54,11 @@ set_git_config() {
 }
 
 git_push_with_auth() {
-    local branch="$1"
+    local owner="$1"
+    local repository="$2"
+    local branch="$4"
 
-    retry 3 git push https://${GITHUB_USERNAME_SECRET}:${GITHUB_TOKEN}@github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}.git "${branch}"
+    retry 3 git push https://${GITHUB_USERNAME_SECRET}:${GITHUB_TOKEN}@github.com/${owner}/${repository}.git "${branch}"
 }
 
 clone_repository() {
@@ -122,7 +125,7 @@ create_or_update_pull_request() {
     update_dependency
 
     echo "--- Pushing branch ${INTEGRATIONS_PR_BRANCH} to integrations repository..."
-    git_push_with_auth ${INTEGRATIONS_PR_BRANCH}
+    git_push_with_auth ${INTEGRATIONS_GITHUB_OWNER} ${INTEGRATIONS_GITHUB_REPO_NAME} ${INTEGRATIONS_PR_BRANCH}
 
     if [ -z "${integrations_pr_number}" ]; then
         echo "--- Creating pull request :github:"

From 602a5eaeed796be2de4c12f1fe46c403f7ab9597 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 13:24:20 +0200
Subject: [PATCH 15/33] fix unbound parameter

---
 .buildkite/scripts/test-with-integrations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index dcc70b9e78..61744cff6c 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -56,7 +56,7 @@ set_git_config() {
 git_push_with_auth() {
     local owner="$1"
     local repository="$2"
-    local branch="$4"
+    local branch="$3"
 
     retry 3 git push https://${GITHUB_USERNAME_SECRET}:${GITHUB_TOKEN}@github.com/${owner}/${repository}.git "${branch}"
 }

From 7eff8fbcb928cf7144fe7edf41aba9b719efae9a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 13:29:05 +0200
Subject: [PATCH 16/33] fix integrations repo name

---
 .buildkite/scripts/test-with-integrations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 61744cff6c..e65ce195dc 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -28,7 +28,7 @@ with_jq
 
 INTEGRATIONS_SOURCE_BRANCH=main
 INTEGRATIONS_GITHUB_OWNER=elastic
-INTEGRATIONS_GITHUB_REPO_NAME=integrations.git
+INTEGRATIONS_GITHUB_REPO_NAME=integrations
 INTEGRATIONS_PR_BRANCH="test-elastic-package-pr-${BUILDKITE_PULL_REQUEST}"
 INTEGRATIONS_PR_TITLE="Test elastic-package - DO NOT MERGE"
 

From d86cddaa1676e570fcafd6929b39df62e49ccf51 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 13:49:49 +0200
Subject: [PATCH 17/33] Allow no changes commited

---
 .buildkite/scripts/test-with-integrations.sh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index e65ce195dc..0230cf3c49 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -66,7 +66,7 @@ clone_repository() {
     retry 5 git clone --depth 1 https://github.com/elastic/integrations ${target}
 }
 
-create_pull_request() {
+create_integrations_pull_request() {
     # requires GITHUB_TOKEN
     echo "Creating Pull Request"
     retry 3 \
@@ -76,7 +76,7 @@ create_pull_request() {
         --draft \
         --base ${INTEGRATIONS_SOURCE_BRANCH} \
         --head ${INTEGRATIONS_PR_BRANCH} \
-        --assignee ${BUILDKITE_PR_HEAD_USER}
+        --assignee ${GITHUB_PR_HEAD_USER}
 }
 
 update_dependency() {
@@ -91,7 +91,11 @@ update_dependency() {
     git add go.mod
     git add go.sum
 
-    git commit -m "Test elastic-package from PR ${BUILDKITE_PULL_REQUEST} - ${GITHUB_PR_HEAD_SHA}"
+    # allow not to commit if there are no changes
+    # previous execution could fail and just pushed the branch but PR is not created
+    if git diff-index --quiet HEAD ; then
+        git commit -m "Test elastic-package from PR ${BUILDKITE_PULL_REQUEST} - ${GITHUB_PR_HEAD_SHA}"
+    fi
 
     echo ""
     git --no-pager show -p HEAD
@@ -129,7 +133,7 @@ create_or_update_pull_request() {
 
     if [ -z "${integrations_pr_number}" ]; then
         echo "--- Creating pull request :github:"
-        create_pull_request
+        create_integrations_pull_request
     fi
 
     popd > /dev/null

From 5b28ed5b2c7ae4bcbe7a01a271c308f733be4118 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 16:54:32 +0200
Subject: [PATCH 18/33] Use full clone

---
 .buildkite/scripts/test-with-integrations.sh | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 0230cf3c49..de72c8a86a 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -63,7 +63,7 @@ git_push_with_auth() {
 
 clone_repository() {
     local target="$1"
-    retry 5 git clone --depth 1 https://github.com/elastic/integrations ${target}
+    retry 5 git clone https://github.com/elastic/integrations ${target}
 }
 
 create_integrations_pull_request() {
@@ -103,6 +103,14 @@ update_dependency() {
 }
 
 
+exists_branch() {
+    local owner="$1"
+    local repository="$2"
+    local branch="$1"
+
+    git ls-remote --exit-code --heads https://github.com/${owner}/${repository}.git ${branch} > /dev/null
+}
+
 create_or_update_pull_request() {
     local temp_path=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
     local repo_path="${temp_path}/elastic-integrations"
@@ -111,15 +119,18 @@ create_or_update_pull_request() {
 
     echo "Cloning repository"
     clone_repository "${repo_path}"
+
     pushd "${repo_path}" > /dev/null
 
     set_git_config
 
+    if ! exists_branch ${INTEGRATIONS_GITHUB_OWNER} ${INTEGRATIONS_GITHUB_REPO_NAME} ${INTEGRATIONS_PR_BRANCH} ; then
+        checkout_options=" -b "
+        echo "Creating a new branch..."
+    fi
+
     integrations_pr_number=$(get_pr_number "${INTEGRATIONS_PR_BRANCH}")
     if [ -z "${integrations_pr_number}" ]; then
-        echo "No PR created, creating a new branch..."
-        checkout_options=" -b "
-    else
         echo "Exists PR in integrations repository: ${integrations_pr_number}"
     fi
 

From b6f0088bd8c4090c36aa01b993f7450d24fc4d8e Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 17:10:30 +0200
Subject: [PATCH 19/33] More logs

---
 .buildkite/scripts/test-with-integrations.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index de72c8a86a..bed8850bcb 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -124,9 +124,12 @@ create_or_update_pull_request() {
 
     set_git_config
 
+    echo "Checking branch ${INTEGRATIONS_PR_BRANCH} in remote"
     if ! exists_branch ${INTEGRATIONS_GITHUB_OWNER} ${INTEGRATIONS_GITHUB_REPO_NAME} ${INTEGRATIONS_PR_BRANCH} ; then
         checkout_options=" -b "
         echo "Creating a new branch..."
+    else
+        echo "Already existed"
     fi
 
     integrations_pr_number=$(get_pr_number "${INTEGRATIONS_PR_BRANCH}")

From bb6a33a37c563d9d2b735a14ce1ed9657ad95218 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 17:26:24 +0200
Subject: [PATCH 20/33] Show output ls-remote

---
 .buildkite/scripts/test-with-integrations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index bed8850bcb..b0a8b0ae73 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -108,7 +108,7 @@ exists_branch() {
     local repository="$2"
     local branch="$1"
 
-    git ls-remote --exit-code --heads https://github.com/${owner}/${repository}.git ${branch} > /dev/null
+    git ls-remote --exit-code --heads https://github.com/${owner}/${repository}.git ${branch}
 }
 
 create_or_update_pull_request() {

From 857dcbc7c0d4b058d8a9ecfb50b7207e677bc0dd Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 17:35:25 +0200
Subject: [PATCH 21/33] More debug

---
 .buildkite/scripts/test-with-integrations.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index b0a8b0ae73..1df16e7588 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -124,7 +124,8 @@ create_or_update_pull_request() {
 
     set_git_config
 
-    echo "Checking branch ${INTEGRATIONS_PR_BRANCH} in remote"
+    echo "Checking branch ${INTEGRATIONS_PR_BRANCH} in remote ${INTEGRATIONS_GITHUB_OWNER}/${INTEGRATIONS_GITHUB_REPO_NAME}"
+    git branch -r |grep test-elastic-package
     if ! exists_branch ${INTEGRATIONS_GITHUB_OWNER} ${INTEGRATIONS_GITHUB_REPO_NAME} ${INTEGRATIONS_PR_BRANCH} ; then
         checkout_options=" -b "
         echo "Creating a new branch..."

From bd7da22d325957bd97eb0be3e1ab486d8951120f Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 17:52:14 +0200
Subject: [PATCH 22/33] more debug

---
 .buildkite/scripts/test-with-integrations.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 1df16e7588..90c2a24249 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -125,6 +125,8 @@ create_or_update_pull_request() {
     set_git_config
 
     echo "Checking branch ${INTEGRATIONS_PR_BRANCH} in remote ${INTEGRATIONS_GITHUB_OWNER}/${INTEGRATIONS_GITHUB_REPO_NAME}"
+    set -x
+    git status
     git branch -r |grep test-elastic-package
     if ! exists_branch ${INTEGRATIONS_GITHUB_OWNER} ${INTEGRATIONS_GITHUB_REPO_NAME} ${INTEGRATIONS_PR_BRANCH} ; then
         checkout_options=" -b "
@@ -132,6 +134,7 @@ create_or_update_pull_request() {
     else
         echo "Already existed"
     fi
+    set +x
 
     integrations_pr_number=$(get_pr_number "${INTEGRATIONS_PR_BRANCH}")
     if [ -z "${integrations_pr_number}" ]; then

From a69c391b2261e29d68ea7ca899ad076389fb2876 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 17:58:52 +0200
Subject: [PATCH 23/33] Fix parameter

---
 .buildkite/scripts/test-with-integrations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 90c2a24249..c8792b6b70 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -106,7 +106,7 @@ update_dependency() {
 exists_branch() {
     local owner="$1"
     local repository="$2"
-    local branch="$1"
+    local branch="$3"
 
     git ls-remote --exit-code --heads https://github.com/${owner}/${repository}.git ${branch}
 }

From c3389d94e7e6da3357fa98f83ed761bb7834026a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 18:12:04 +0200
Subject: [PATCH 24/33] Fix add pr comment

---
 .buildkite/scripts/test-with-integrations.sh | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index c8792b6b70..1efa46cf1b 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -125,16 +125,12 @@ create_or_update_pull_request() {
     set_git_config
 
     echo "Checking branch ${INTEGRATIONS_PR_BRANCH} in remote ${INTEGRATIONS_GITHUB_OWNER}/${INTEGRATIONS_GITHUB_REPO_NAME}"
-    set -x
-    git status
-    git branch -r |grep test-elastic-package
     if ! exists_branch ${INTEGRATIONS_GITHUB_OWNER} ${INTEGRATIONS_GITHUB_REPO_NAME} ${INTEGRATIONS_PR_BRANCH} ; then
         checkout_options=" -b "
         echo "Creating a new branch..."
     else
         echo "Already existed"
     fi
-    set +x
 
     integrations_pr_number=$(get_pr_number "${INTEGRATIONS_PR_BRANCH}")
     if [ -z "${integrations_pr_number}" ]; then
@@ -152,25 +148,30 @@ create_or_update_pull_request() {
     if [ -z "${integrations_pr_number}" ]; then
         echo "--- Creating pull request :github:"
         create_integrations_pull_request
+
+        sleep 10
+
+        integrations_pr_number=$(get_pr_number "${INTEGRATIONS_PR_BRANCH}")
     fi
 
     popd > /dev/null
 
     rm -rf "${temp_path}"
+
+    echo "--- adding comment into elastic-package pull request :memo:"
+    add_pr_comment "${BUILDKITE_PULL_REQUEST}" "$(get_integrations_pr_link ${integrations_pr_number})"
 }
 
 
 add_pr_comment() {
-    local pr_number="$1"
+    local elastic_pr_number="$1"
+    local integrations_pr_link="$2"
     retry 3 \
         gh pr comment ${pr_number} \
-        --body "Created or updated PR in integrations repostiory to test this vesrion. Check ${get_integrations_pr_link}" \
+        --body "Created or updated PR in integrations repostiory to test this vesrion. Check ${integrations_pr_link}" \
         --repo ${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}
 }
 
 
 echo "--- creating or updating integrations pull request"
 create_or_update_pull_request
-
-echo "--- adding comment into elastic-package pull request :memo:"
-add_pr_comment "${BUILDKITE_PULL_REQUEST}"

From 6b0cc43e6fd23f8e23e3308e42526a95430f8f3c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 18:23:29 +0200
Subject: [PATCH 25/33] Fix parameter

---
 .buildkite/scripts/test-with-integrations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 1efa46cf1b..c51f0937fd 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -167,7 +167,7 @@ add_pr_comment() {
     local elastic_pr_number="$1"
     local integrations_pr_link="$2"
     retry 3 \
-        gh pr comment ${pr_number} \
+        gh pr comment ${elastic_pr_number} \
         --body "Created or updated PR in integrations repostiory to test this vesrion. Check ${integrations_pr_link}" \
         --repo ${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}
 }

From 83b57e76e9be63c041703f440974523e09c1e5ac Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 18:37:14 +0200
Subject: [PATCH 26/33] Fix condition for commit

---
 .buildkite/scripts/test-with-integrations.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index c51f0937fd..b5bec63daa 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -84,7 +84,7 @@ update_dependency() {
     echo "--- install go for integrations repository :go:"
     with_go
 
-    echo "--- Updating go.mod and go.sum :hammer_and_wrench:"
+    echo "--- Updating go.mod and go.sum with ${GITHUB_PR_HEAD_SHA} :hammer_and_wrench:"
     go mod edit -replace github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}=github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}@${GITHUB_PR_HEAD_SHA}
     go mod tidy
 
@@ -93,12 +93,12 @@ update_dependency() {
 
     # allow not to commit if there are no changes
     # previous execution could fail and just pushed the branch but PR is not created
-    if git diff-index --quiet HEAD ; then
+    if ! git diff-index --quiet HEAD ; then
         git commit -m "Test elastic-package from PR ${BUILDKITE_PULL_REQUEST} - ${GITHUB_PR_HEAD_SHA}"
     fi
 
     echo ""
-    git --no-pager show -p HEAD
+    git --no-pager show --format=online HEAD
     echo ""
 }
 

From db51632343428432d2e982753685a7d8941edd68 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 19:33:13 +0200
Subject: [PATCH 27/33] Fix format

---
 .buildkite/scripts/test-with-integrations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index b5bec63daa..2d4984d897 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -98,7 +98,7 @@ update_dependency() {
     fi
 
     echo ""
-    git --no-pager show --format=online HEAD
+    git --no-pager show --format=oneline HEAD
     echo ""
 }
 

From ff1ce9430c84d0af9838c40ddcb79102cff1355e Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 19:44:37 +0200
Subject: [PATCH 28/33] Support new lines

---
 .buildkite/scripts/test-with-integrations.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 2d4984d897..c16b971cac 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -68,11 +68,14 @@ clone_repository() {
 
 create_integrations_pull_request() {
     # requires GITHUB_TOKEN
+    local temp_path=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
     echo "Creating Pull Request"
+    message="Update elastic-package reference to ${GITHUB_PR_HEAD_SHA}.\nAutomated by [Buildkite build](${BUILDKITE_BUILD_URL})\n\nRelates: $(get_elastic_package_pr_link)"
+    echo -e $message > ${temp_path}/body-pr.txt
     retry 3 \
         gh pr create \
         --title "${INTEGRATIONS_PR_TITLE}" \
-        --body "Update elastic-package reference to ${GITHUB_PR_HEAD_SHA}.\nAutomated by [Buildkite build](${BUILDKITE_BUILD_URL})\n\nRelates: $(get_elastic_package_pr_link)" \
+        --body-file ${temp_path}/body-pr.txt \
         --draft \
         --base ${INTEGRATIONS_SOURCE_BRANCH} \
         --head ${INTEGRATIONS_PR_BRANCH} \
@@ -166,6 +169,7 @@ create_or_update_pull_request() {
 add_pr_comment() {
     local elastic_pr_number="$1"
     local integrations_pr_link="$2"
+
     retry 3 \
         gh pr comment ${elastic_pr_number} \
         --body "Created or updated PR in integrations repostiory to test this vesrion. Check ${integrations_pr_link}" \

From f491262dcd1a0486707307da7eb76deefbf7904d Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 24 May 2023 20:31:37 +0200
Subject: [PATCH 29/33] Add commit link

---
 .buildkite/scripts/test-with-integrations.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index c16b971cac..d70e664792 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -48,6 +48,10 @@ get_elastic_package_pr_link() {
     echo "https://github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}/pull/${BUILDKITE_PULL_REQUEST}"
 }
 
+get_elastic_package_commit_link() {
+    echo "https://github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}/commit/${GITHUB_PR_HEAD_SHA}"
+}
+
 set_git_config() {
     git config user.name "${GITHUB_USERNAME_SECRET}"
     git config user.email "${GITHUB_EMAIL_SECRET}"
@@ -70,7 +74,7 @@ create_integrations_pull_request() {
     # requires GITHUB_TOKEN
     local temp_path=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
     echo "Creating Pull Request"
-    message="Update elastic-package reference to ${GITHUB_PR_HEAD_SHA}.\nAutomated by [Buildkite build](${BUILDKITE_BUILD_URL})\n\nRelates: $(get_elastic_package_pr_link)"
+    message="Update elastic-package reference to $(get_elastic_package_commit_link).\nAutomated by [Buildkite build](${BUILDKITE_BUILD_URL})\n\nRelates: $(get_elastic_package_pr_link)"
     echo -e $message > ${temp_path}/body-pr.txt
     retry 3 \
         gh pr create \

From 71c1c3b7696f5712cde5f4a79f75318c16203296 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 25 May 2023 12:34:38 +0200
Subject: [PATCH 30/33] Add depends_on conditions

---
 .buildkite/pipeline.test-with-integrations-repo.yml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.buildkite/pipeline.test-with-integrations-repo.yml b/.buildkite/pipeline.test-with-integrations-repo.yml
index d102e84edd..c3ba0a62bd 100644
--- a/.buildkite/pipeline.test-with-integrations-repo.yml
+++ b/.buildkite/pipeline.test-with-integrations-repo.yml
@@ -12,9 +12,22 @@ steps:
       image: "golang:${GOLANG_VERSION}"
       cpu: "8"
       memory: "4G"
+  - label: ":go: :linux: Run unit tests"
+    key: unit-tests-linux
+    command: "make test-go-ci"
+    artifact_paths:
+      - "build/test-results/*.xml"
+      - "build/test-coverage/*.xml"
+    agents:
+      image: "golang:${GOLANG_VERSION}"
+      cpu: "8"
+      memory: "4G"
   - label: ":hammer: Create PR in integrations"
     key: pr-integrations
     command: ".buildkite/scripts/test-with-integrations.sh"
     agents:
       provider: "gcp"
+    depends_on:
+      - check-static
+      - unit-tests-linux
 

From 81f8a2f72992dde31a1df3f01b2f1a0e1dd85475 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 25 May 2023 12:42:41 +0200
Subject: [PATCH 31/33] Rename to source

---
 .buildkite/scripts/test-with-integrations.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index d70e664792..653b73bd63 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -29,8 +29,8 @@ with_jq
 INTEGRATIONS_SOURCE_BRANCH=main
 INTEGRATIONS_GITHUB_OWNER=elastic
 INTEGRATIONS_GITHUB_REPO_NAME=integrations
-INTEGRATIONS_PR_BRANCH="test-elastic-package-pr-${BUILDKITE_PULL_REQUEST}"
-INTEGRATIONS_PR_TITLE="Test elastic-package - DO NOT MERGE"
+INTEGRATIONS_PR_BRANCH="test-${GITHUB_PR_BASE_REPO}-pr-${BUILDKITE_PULL_REQUEST}"
+INTEGRATIONS_PR_TITLE="Test ${GITHUB_PR_BASE_REPO} - DO NOT MERGE"
 
 
 get_pr_number() {
@@ -44,11 +44,11 @@ get_integrations_pr_link() {
     echo "https://github.com/elastic/integrations/pull/${pr_number}"
 }
 
-get_elastic_package_pr_link() {
+get_source_pr_link() {
     echo "https://github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}/pull/${BUILDKITE_PULL_REQUEST}"
 }
 
-get_elastic_package_commit_link() {
+get_source_commit_link() {
     echo "https://github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}/commit/${GITHUB_PR_HEAD_SHA}"
 }
 
@@ -74,7 +74,7 @@ create_integrations_pull_request() {
     # requires GITHUB_TOKEN
     local temp_path=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
     echo "Creating Pull Request"
-    message="Update elastic-package reference to $(get_elastic_package_commit_link).\nAutomated by [Buildkite build](${BUILDKITE_BUILD_URL})\n\nRelates: $(get_elastic_package_pr_link)"
+    message="Update ${GITHUB_PR_BASE_REPO} reference to $(get_source_commit_link).\nAutomated by [Buildkite build](${BUILDKITE_BUILD_URL})\n\nRelates: $(get_source_pr_link)"
     echo -e $message > ${temp_path}/body-pr.txt
     retry 3 \
         gh pr create \
@@ -165,17 +165,17 @@ create_or_update_pull_request() {
 
     rm -rf "${temp_path}"
 
-    echo "--- adding comment into elastic-package pull request :memo:"
+    echo "--- adding comment into ${GITHUB_PR_BASE_REPO} pull request :memo:"
     add_pr_comment "${BUILDKITE_PULL_REQUEST}" "$(get_integrations_pr_link ${integrations_pr_number})"
 }
 
 
 add_pr_comment() {
-    local elastic_pr_number="$1"
+    local source_pr_number="$1"
     local integrations_pr_link="$2"
 
     retry 3 \
-        gh pr comment ${elastic_pr_number} \
+        gh pr comment ${source_pr_number} \
         --body "Created or updated PR in integrations repostiory to test this vesrion. Check ${integrations_pr_link}" \
         --repo ${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}
 }

From c478ac1e45db52f0bc53ca98b10e6fdb925b9fce Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 25 May 2023 12:47:19 +0200
Subject: [PATCH 32/33] extract vars

---
 .buildkite/scripts/test-with-integrations.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 653b73bd63..6c8f6092cd 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -31,7 +31,7 @@ INTEGRATIONS_GITHUB_OWNER=elastic
 INTEGRATIONS_GITHUB_REPO_NAME=integrations
 INTEGRATIONS_PR_BRANCH="test-${GITHUB_PR_BASE_REPO}-pr-${BUILDKITE_PULL_REQUEST}"
 INTEGRATIONS_PR_TITLE="Test ${GITHUB_PR_BASE_REPO} - DO NOT MERGE"
-
+VERSION_DEP=""
 
 get_pr_number() {
     # requires GITHUB_TOKEN
@@ -92,7 +92,10 @@ update_dependency() {
     with_go
 
     echo "--- Updating go.mod and go.sum with ${GITHUB_PR_HEAD_SHA} :hammer_and_wrench:"
-    go mod edit -replace github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}=github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}@${GITHUB_PR_HEAD_SHA}
+    local source_dep="github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}${VERSION_DEP}"
+    local target_dep="github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}${VERSION_DEP}@${GITHUB_PR_HEAD_SHA}"
+
+    go mod edit -replace ${source_dep}=${target_dep}
     go mod tidy
 
     git add go.mod

From d7929dcf7a29842471f1d3b690ac78bb38e4eabe Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 1 Jun 2023 15:58:48 +0200
Subject: [PATCH 33/33] Update title with PR number

---
 .buildkite/scripts/test-with-integrations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 6c8f6092cd..d7cb53c8b9 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -30,7 +30,7 @@ INTEGRATIONS_SOURCE_BRANCH=main
 INTEGRATIONS_GITHUB_OWNER=elastic
 INTEGRATIONS_GITHUB_REPO_NAME=integrations
 INTEGRATIONS_PR_BRANCH="test-${GITHUB_PR_BASE_REPO}-pr-${BUILDKITE_PULL_REQUEST}"
-INTEGRATIONS_PR_TITLE="Test ${GITHUB_PR_BASE_REPO} - DO NOT MERGE"
+INTEGRATIONS_PR_TITLE="Test ${GITHUB_PR_BASE_REPO}#${BUILDKITE_PULL_REQUEST} - DO NOT MERGE"
 VERSION_DEP=""
 
 get_pr_number() {