From 32c42e46ed41710b0f21fe110e8aa83992d7a947 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Thu, 30 Jun 2022 19:47:32 +0200
Subject: [PATCH 01/21] Py package version changes.

---
 colab/setup.py      | 4 ++--
 grr/client/setup.py | 4 ++--
 grr/server/setup.py | 4 ++--
 grr/test/setup.py   | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/colab/setup.py b/colab/setup.py
index d0661c7bc..b6205a067 100644
--- a/colab/setup.py
+++ b/colab/setup.py
@@ -62,6 +62,6 @@ def make_release_tree(self, base_dir, files):
         "grr_response_proto==%s" % VERSION.get("Version", "packagedepends"),
         "humanize==2.4.0",
         "ipython==%s" % ("5.0.0" if sys.version_info < (3, 0) else "7.15.0"),
-        "numpy==1.18.5",
-        "pandas==1.0.4",
+        "numpy==1.23.0",
+        "pandas==1.4.3",
     ])
diff --git a/grr/client/setup.py b/grr/client/setup.py
index 74bc7ad13..f1a3a42e1 100644
--- a/grr/client/setup.py
+++ b/grr/client/setup.py
@@ -132,7 +132,7 @@ def run(self):
     include_package_data=True,
     python_requires=">=3.6",
     install_requires=[
-        "absl-py==0.9.0",
+        "absl-py==1.1.0",
         "grr-response-core==%s" % VERSION.get("Version", "packagedepends"),
         "PyInstaller==3.6",
         PYTSK3,
@@ -144,7 +144,7 @@ def run(self):
         # The following requirements are needed in Windows.
         ':sys_platform=="win32"': [
             "WMI==1.5.1",
-            "pywin32==228",
+            "pywin32==304",
         ],
     },
 )
diff --git a/grr/server/setup.py b/grr/server/setup.py
index 965a7d418..35086f37a 100644
--- a/grr/server/setup.py
+++ b/grr/server/setup.py
@@ -40,7 +40,7 @@ def make_ui_files():
   subprocess.check_call(
       "npm ci", shell=True, cwd="grr_response_server/gui/static")
   subprocess.check_call(
-      "npm run gulp compile", shell=True, cwd="grr_response_server/gui/static")
+      "npx gulp compile", shell=True, cwd="grr_response_server/gui/static")
   # Compile UI v2.
   subprocess.check_call("npm ci", shell=True, cwd="grr_response_server/gui/ui")
   subprocess.check_call(
@@ -194,7 +194,7 @@ def make_release_tree(self, base_dir, files):
         "portpicker==1.3.1",
         "prometheus_client==0.8.0",
         "pyjwt==1.7.1",
-        "pyOpenSSL==19.1.0",  # https://github.com/google/grr/issues/704
+        "pyOpenSSL==20.0.0",  # https://github.com/google/grr/issues/704
         "python-crontab==2.5.1",
         "python-debian==0.1.37",
         "Werkzeug==1.0.1",
diff --git a/grr/test/setup.py b/grr/test/setup.py
index 017e7a498..fbea1887e 100644
--- a/grr/test/setup.py
+++ b/grr/test/setup.py
@@ -59,7 +59,7 @@ def make_release_tree(self, base_dir, files):
     license="Apache License, Version 2.0",
     url="https://github.com/google/grr",
     install_requires=[
-        "absl-py==0.9.0",
+        "absl-py==1.1.0",
         "flaky==3.6.1",
         "pytest==6.2.2",
         "responses==0.12.1",

From 6a2682d0669a3b865cfdc21d6613dac4ef2d3bb6 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Thu, 30 Jun 2022 20:23:47 +0200
Subject: [PATCH 02/21] Remove non windows workflows.

---
 .github/workflows/build.yml | 316 ------------------------------------
 1 file changed, 316 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b20c59d02..d2e235a46 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -6,146 +6,6 @@ env:
   GCS_LATEST_PATH: _latest_server_deb
   DOCKER_REPOSITORY: grrdocker/grr
 jobs:
-
-  test-ubuntu:
-    runs-on: ubuntu-18.04
-    env:
-      CHROME_DEB: google-chrome-stable_current_amd64.deb
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Install
-        run: |
-          free -hmw
-          lscpu
-          sudo apt install -y libmysqlclient-dev
-          if [[ -z "$(type google-chrome 2>/dev/null)" ]]; then
-            wget "https://dl.google.com/linux/direct/${CHROME_DEB}" && sudo apt install -y "./${CHROME_DEB}";
-          fi
-          python -m venv "${HOME}/INSTALL"
-          travis/install.sh
-      - name: Test
-        run: |
-          source "${HOME}/INSTALL/bin/activate"
-          pip install pytest-xdist==2.2.1 pytest==6.2.5
-          # We have 4 vCPUs available, but only use 3 here to avoid timeouts like
-          # https://ci.appveyor.com/project/grr/grr-ia94e/builds/20483467/messages ,
-          # which happen when tests stall.
-          pytest --verbose -n 3 grr/ --ignore grr/server/grr_response_server/gui/selenium_tests/ --ignore grr/client/grr_response_client/client_actions/windows/
-          # jsTree tests seem to fail on Chrome 71 headless due to https://github.com/GoogleChrome/puppeteer/issues/3463
-          if [ $(google-chrome --version | grep -Eo " [0-9]{1,3}") != "71" ]; then (cd grr/server/grr_response_server/gui/static/ && npm run gulp test); fi
-
-  build-openapi:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Install
-        run: |
-          sudo apt install -y libmysqlclient-dev
-          python -m venv "${HOME}/INSTALL"
-          travis/install.sh
-      - name: Build
-        run: |
-          source "${HOME}/INSTALL/bin/activate"
-          mkdir -p _openapi_artifacts/openapi_description
-          mkdir -p _openapi_artifacts/documentation
-          travis/build_api_documentation.sh "_openapi_artifacts/openapi_description/openapi_description.json" "_openapi_artifacts/documentation/openapi_documentation.html"
-          ls -la _openapi_artifacts/*
-      - name: Upload OpenAPI to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: openapi
-          path: _openapi_artifacts/
-          retention-days: 1
-
-  build-linux:
-    runs-on: ubuntu-18.04
-    env:
-      GCS_TAG: ubuntu_64bit
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.6'
-      - name: Set up
-        run: |
-          sudo apt install fakeroot debhelper libffi-dev libssl-dev
-          pip install virtualenv
-          virtualenv "${HOME}/INSTALL"
-      - name: Build
-        run: |
-          travis/install_client_builder.sh
-          travis/build_templates.sh
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: ubuntu-installers
-          path: gcs_upload_dir/
-          retention-days: 1
-
-  build-osx:
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.6'
-      - name: Set up
-        run: |
-          pip install --upgrade setuptools virtualenv
-          virtualenv "${HOME}/INSTALL"
-      - name: Build installers
-        run: |
-          travis/install_client_builder.sh
-          travis/build_templates.sh
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: osx-installers
-          path: gcs_upload_dir/
-          retention-days: 1
-
-  build-centos:
-    runs-on: ubuntu-18.04
-    env:
-      GCS_TAG: centos_64bit
-      DOCKER_IMG: grrdocker/centos7
-      DOCKER_CONTAINER: centos_64bit_container
-      DOCKER_USER: grrbot
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.6'
-      - name: Build installers
-        run: |
-          sudo docker run -dit \
-            --volume "${PWD}:/mnt/grr" \
-            --workdir /mnt/grr \
-            --env DOCKER_USER="${DOCKER_USER}" \
-            --env TRAVIS_OS_NAME="linux" \
-            --name "${DOCKER_CONTAINER}" \
-            "${DOCKER_IMG}"
-          sudo docker exec "${DOCKER_CONTAINER}" travis/set_up_test_user.sh
-          sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" /usr/local/bin/python3.6 -m venv "/home/${DOCKER_USER}/INSTALL"
-          sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" travis/install_client_builder.sh
-          sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" travis/build_templates.sh
-          sudo docker exec "${DOCKER_CONTAINER}" rpm -vih gcs_upload_dir/*.rpm
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: centos-installers
-          path: gcs_upload_dir/
-          retention-days: 1
-
   build-windows:
     runs-on: windows-2022
     steps:
@@ -170,179 +30,3 @@ jobs:
           name: windows-installers
           path: gcs_upload_dir/
           retention-days: 1
-
-  build-server-deb:
-    runs-on: ubuntu-18.04
-    env:
-      GCS_TAG: server_deb
-    needs:
-      - build-centos
-      - build-linux
-      - build-osx
-      - build-windows
-    steps:
-      - uses: actions/checkout@v2
-      - name: Download installers from GitHub artifacts
-        id: download
-        uses: actions/download-artifact@v2
-        with:
-          path: ~/_artifacts
-      - name: Set up
-        run: |
-          sudo apt-get install -y fakeroot debhelper libffi-dev libssl-dev python3-dev python3-pip python3-venv wget openjdk-8-jdk zip git devscripts dh-systemd libmysqlclient-dev dh-virtualenv dh-make libc6-i386 lib32z1
-          pip3 install --upgrade setuptools virtualenv
-          virtualenv "${HOME}/INSTALL"
-      - name: Build
-        run: |
-          travis/install.sh
-
-          mkdir -p grr/config/grr_response_templates/templates
-
-          mv -v ~/_artifacts/windows-installers/GRR_*_amd64.exe.zip grr/config/grr_response_templates/templates
-          mv -v ~/_artifacts/windows-installers/GRR_*_amd64.msi.zip grr/config/grr_response_templates/templates
-          mv -v ~/_artifacts/ubuntu-installers/grr_*_amd64.deb.zip grr/config/grr_response_templates/templates
-          mv -v ~/_artifacts/centos-installers/grr_*_amd64.rpm.zip grr/config/grr_response_templates/templates
-          mv -v ~/_artifacts/osx-installers/grr_*_amd64.xar.zip grr/config/grr_response_templates/templates
-
-          travis/build_local_pyindex.sh
-          travis/build_server_deb.sh
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: server-deb
-          path: gcs_upload_dir/
-          retention-days: 1
-
-  test-ubuntu-e2e:
-    continue-on-error: true  # Debug follow up step.
-    runs-on: ubuntu-18.04
-    env:
-      GRR_ADMIN_PASS: 'e2e_tests'
-      APPVEYOR_MYSQL_PASS: 'root' 
-    needs:
-      - build-server-deb
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.6'
-      - name: Set up MySQL
-        run: |
-          sudo /etc/init.d/mysql start
-      - name: Download installers from GitHub artifacts
-        id: download
-        uses: actions/download-artifact@v2
-        with:
-          name: server-deb
-          path: _artifacts
-      - name: Install
-        run: |
-          free -hmw
-          lscpu
-          sudo apt install -y libmysqlclient-dev
-          if [[ -z "$(type google-chrome 2>/dev/null)" ]]; then
-            wget "https://dl.google.com/linux/direct/${CHROME_DEB}" && sudo apt install -y "./${CHROME_DEB}";
-          fi
-          python -m venv "${HOME}/INSTALL"
-          travis/install.sh
-          sudo -EH ./appveyor/e2e_tests/install_mem_usage_cron.sh
-          sudo -EH ./appveyor/e2e_tests/install_latest_server_deb.sh
-      - name: Test
-        run: |
-          sudo -EH ./appveyor/e2e_tests/run_e2e_tests.sh
-          sudo -EH ./appveyor/e2e_tests/test_repack.sh
-      - name: Upload logs and configs to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: e2e-test
-          path: appveyor_e2e_artifacts/
-          retention-days: 1
-
-
-  build-push-docker:
-    runs-on: ubuntu-18.04
-    needs:
-      - build-server-deb
-      # - test-ubuntu-e2e  # TODO: Comment back in after debugging is finished.
-      - test-ubuntu
-      - build-openapi
-    steps:
-      - uses: actions/checkout@v2
-      - name: Download installers from GitHub artifacts
-        id: download
-        uses: actions/download-artifact@v2
-        with:
-          name: server-deb
-          path: _artifacts
-      - name: Build Docker image
-        run: |
-          export BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3)
-          ./appveyor/docker_build/build_docker_image.sh
-      - if: ${{ github.event_name == 'push' }} 
-        name: Log in to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - if: ${{ github.event_name == 'push' }}
-        name: Push to ${{ env.DOCKER_REPOSITORY }}
-        run: |
-          docker push -a ${{ env.DOCKER_REPOSITORY }}
-  upload:
-    if: ${{ github.event_name == 'push' }}
-    runs-on: ubuntu-18.04
-    needs:
-      - build-push-docker
-    steps:
-      - uses: actions/checkout@v2
-      - name: Download installers from GitHub artifacts
-        id: download
-        uses: actions/download-artifact@v2
-        with:
-          path: _artifacts
-      - run: |
-          ls -la _artifacts/*/
-
-          COMMIT_TIME=$(git show -s --date='format-local:%Y-%m-%dT%H:%M:%SZ' --format="%cd" $GITHUB_SHA)
-          OUTPUT_DIR=gcs_upload_dir/${COMMIT_TIME}_${GITHUB_SHA}/
-          echo "OUTPUT_DIR=$OUTPUT_DIR" >> $GITHUB_ENV
-
-          mkdir -p $OUTPUT_DIR/centos/
-          mv -v _artifacts/centos-installers/* $OUTPUT_DIR/centos
-
-          mkdir -p $OUTPUT_DIR/ubuntu/
-          mv -v _artifacts/ubuntu-installers/* $OUTPUT_DIR/ubuntu
-
-          mkdir -p $OUTPUT_DIR/osx/
-          mv -v _artifacts/osx-installers/* $OUTPUT_DIR/osx
-
-          mkdir -p $OUTPUT_DIR/windows/
-          mv -v _artifacts/windows-installers/* $OUTPUT_DIR/windows
-
-          mkdir -p $OUTPUT_DIR/server_deb/
-          mv -v _artifacts/server-deb/* $OUTPUT_DIR/server_deb
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v0.2.1
-        with:
-          project_id: ${{ secrets.GCP_PROJECT_ID }}
-          service_account_key: ${{ secrets.GCP_SA_KEY }}
-          export_default_credentials: true
-      - name: Upload installers to GCS
-        uses: google-github-actions/upload-cloud-storage@main
-        with:
-          path: gcs_upload_dir/
-          destination: ${{ env.GCS_BUCKET }}
-          # Omit `path` (e.g. /home/runner/deploy/) in final GCS path.
-          parent: false
-      - name: Replace ${{ env.GCS_LATEST_PATH }} folder in GCS
-        run: |
-          gsutil rm gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/** || true
-          gsutil cp -r $OUTPUT_DIR/server_deb/* gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/
-      - name: Upload OpenAPI to GCS
-        uses: google-github-actions/upload-cloud-storage@main
-        with:
-          path: _artifacts/openapi/
-          destination: ${{ env.GCS_BUCKET_OPENAPI }}
-          # Omit `path` (e.g. /home/runner/deploy/) in final GCS path.
-          parent: false

From 6d46c179b2b15510850de405c5a922a7fcaefd5c Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Thu, 30 Jun 2022 20:35:45 +0200
Subject: [PATCH 03/21] Try build on py38.

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d2e235a46..380197056 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.7'
+          python-version: '3.8'
       - name: Build installers
         shell: bash
         run: |

From e8af0fe00f6e260b7c286cffc3ce7b90d628a5ba Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Tue, 12 Jul 2022 18:50:15 +0200
Subject: [PATCH 04/21] Add terminating newline to post request body in elastic
 outputplugin.

---
 .../grr_response_server/output_plugins/elasticsearch_plugin.py  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py
index 42a31882a..a58fd9675 100644
--- a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py
+++ b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py
@@ -146,7 +146,7 @@ def _SendEvents(self, events: List[JsonDict]) -> None:
     data = "\n".join([
         "{}\n{}".format(index_command, json.Dump(event, indent=None))
         for event in events
-    ])
+    ]) + "\n"
 
     response = requests.post(
         url=self._url, verify=self._verify_https, data=data, headers=headers)

From 9cddd85e011ba9e9c4061a17c5cccddb9cbc4572 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Tue, 12 Jul 2022 18:51:11 +0200
Subject: [PATCH 05/21] Add json content type headers to _bulk post in elastic
 outputplugin.

---
 .../output_plugins/elasticsearch_plugin.py                    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py
index a58fd9675..1d0800eea 100644
--- a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py
+++ b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py
@@ -135,9 +135,9 @@ def _SendEvents(self, events: List[JsonDict]) -> None:
     # https://www.elastic.co/guide/en/elasticsearch/reference/7.1/docs-bulk.html
 
     if self._token:
-      headers = {"Authorization": "Basic {}".format(self._token)}
+      headers = {"Authorization": "Basic {}".format(self._token), "Content-Type": "application/json"}
     else:
-      headers = {}
+      headers = {"Content-Type": "application/json"}
 
     index_command = json.Dump({"index": {"_index": self._index}}, indent=None)
 

From 922ded34181b0e1f5a9c3b1feabcdba78bc465f7 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Tue, 12 Jul 2022 18:50:15 +0200
Subject: [PATCH 06/21] Add terminating newline to post request body in elastic
 outputplugin.

---
 .../grr_response_server/output_plugins/elasticsearch_plugin.py  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py
index 42a31882a..a58fd9675 100644
--- a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py
+++ b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py
@@ -146,7 +146,7 @@ def _SendEvents(self, events: List[JsonDict]) -> None:
     data = "\n".join([
         "{}\n{}".format(index_command, json.Dump(event, indent=None))
         for event in events
-    ])
+    ]) + "\n"
 
     response = requests.post(
         url=self._url, verify=self._verify_https, data=data, headers=headers)

From db1aee4b7c5efe4dd47f9926306b0223412153b7 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Tue, 12 Jul 2022 18:51:11 +0200
Subject: [PATCH 07/21] Add json content type headers to _bulk post in elastic
 outputplugin.

---
 .../output_plugins/elasticsearch_plugin.py                    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py
index a58fd9675..1d0800eea 100644
--- a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py
+++ b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py
@@ -135,9 +135,9 @@ def _SendEvents(self, events: List[JsonDict]) -> None:
     # https://www.elastic.co/guide/en/elasticsearch/reference/7.1/docs-bulk.html
 
     if self._token:
-      headers = {"Authorization": "Basic {}".format(self._token)}
+      headers = {"Authorization": "Basic {}".format(self._token), "Content-Type": "application/json"}
     else:
-      headers = {}
+      headers = {"Content-Type": "application/json"}
 
     index_command = json.Dump({"index": {"_index": self._index}}, indent=None)
 

From dfd2395aacd389382309b2c9c35ddde0ae3be2c1 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Wed, 13 Jul 2022 15:19:03 +0200
Subject: [PATCH 08/21] Add changes to changelog.

---
 CHANGELOG | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CHANGELOG b/CHANGELOG
index 80678432c..c6c8e7171 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -5,6 +5,9 @@
 * Introduced Server.grr_binaries_readonly configuration option (set to False
   by default). When set to True, binaries and python hacks can't be overriden
   or deleted.
+* Updates elasticsearch output plugin post request to _bulk in the
+  elasticsearch api. Adds a terminating \n and content type headers for
+  application/json.
 
 ## 3.4.3.1
 

From 6d4580180e7a46275e0761b2974f3a6c706132bd Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Thu, 14 Jul 2022 12:14:56 +0200
Subject: [PATCH 09/21] Switch from collections. to collections.abc ref.

---
 grr/core/grr_response_core/lib/rdfvalues/structs.py | 2 +-
 grr/core/grr_response_core/lib/util/precondition.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/grr/core/grr_response_core/lib/rdfvalues/structs.py b/grr/core/grr_response_core/lib/rdfvalues/structs.py
index a882a2a3e..dc46cb939 100644
--- a/grr/core/grr_response_core/lib/rdfvalues/structs.py
+++ b/grr/core/grr_response_core/lib/rdfvalues/structs.py
@@ -1082,7 +1082,7 @@ def ConvertToWireFormat(self, value):
     return (self.encoded_tag, VarintEncode(len(data)), data)
 
 
-class RepeatedFieldHelper(collections.Sequence, object):
+class RepeatedFieldHelper(collections.abc.Sequence, object):
   """A helper for the RDFProto to handle repeated fields.
 
   This helper is intended to only be constructed from the RDFProto class.
diff --git a/grr/core/grr_response_core/lib/util/precondition.py b/grr/core/grr_response_core/lib/util/precondition.py
index e448287a4..17c8a3652 100644
--- a/grr/core/grr_response_core/lib/util/precondition.py
+++ b/grr/core/grr_response_core/lib/util/precondition.py
@@ -57,7 +57,7 @@ def AssertIterableType(iterable, expected_item_type):
     message %= iterable
     raise TypeError(message)
 
-  AssertType(iterable, collections.Iterable)
+  AssertType(iterable, collections.abc.Iterable)
   for item in iterable:
     AssertType(item, expected_item_type)
 

From 47309662821a8a466961d023174b238a94dc9482 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Thu, 14 Jul 2022 17:07:04 +0200
Subject: [PATCH 10/21] Version changes.

---
 colab/setup.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/colab/setup.py b/colab/setup.py
index b6205a067..7e750e09e 100644
--- a/colab/setup.py
+++ b/colab/setup.py
@@ -60,8 +60,7 @@ def make_release_tree(self, base_dir, files):
     install_requires=[
         "grr_api_client==%s" % VERSION.get("Version", "packagedepends"),
         "grr_response_proto==%s" % VERSION.get("Version", "packagedepends"),
-        "humanize==2.4.0",
         "ipython==%s" % ("5.0.0" if sys.version_info < (3, 0) else "7.15.0"),
-        "numpy==1.23.0",
-        "pandas==1.4.3",
+        "numpy==1.20.3",
+        "pandas==1.0.4",
     ])

From 018dd1601c64d95cfff8dc637c0668a593006f87 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Thu, 14 Jul 2022 17:09:13 +0200
Subject: [PATCH 11/21] Questionable Fix: Adds several unknown processes to the
 allowlist. Required to run on windows.

---
 conftest.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/conftest.py b/conftest.py
index d551a5a39..bc5eb012e 100644
--- a/conftest.py
+++ b/conftest.py
@@ -153,6 +153,12 @@ def thread_leak_check(request):
       "ApiSslServerTest",
       "GRRHTTPServerTestThread",
       "SharedMemDBTestThread",
+
+      # These threads seem to need to be allowed to run tests. Sample
+      # %contentroot%/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py --no-header --no-summary -q in %contentroot%\grr\server\grr_response_server\output_plugins
+      'pydevd.Writer',
+      'pydevd.Reader',
+      'pydevd.CommandThread',
   ]
 
   # Remove up to one instance of each allowed thread name.

From bef32c89dc300568ad52d99544f89c988e8dd81d Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Thu, 14 Jul 2022 17:13:32 +0200
Subject: [PATCH 12/21] Fix: Filter-out jsonl empty lines from json parsing.

---
 .../output_plugins/elasticsearch_plugin_test.py  | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
index 7578f9b30..f22287fa0 100644
--- a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
+++ b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
@@ -66,12 +66,20 @@ def _CallPlugin(self, plugin_args=None, responses=None, patcher=None):
 
   def _ParseEvents(self, patched):
     request = patched.call_args[KWARGS]['data']
-
     # Elasticsearch bulk requests are line-deliminated pairs, where the first
     # line is the index command and the second is the actual document to index
-    split_requests = [json.Parse(line) for line in request.split('\n')]
-    update_pairs = [(split_requests[i], split_requests[i + 1])
-                    for i in range(0, len(split_requests), 2)]
+    split_requests = []
+    splitRequest = request.split('\n')
+    for line in splitRequest:
+        # Skip terminating newlines - which crashes json.Parse
+        if not line:
+            continue
+        split_requests.append(json.Parse(line))
+    #update_pairs = [(split_requests[i], split_requests[i + 1])
+    #                for i in range(0, len(split_requests), 2)]
+    update_pairs = []
+    for i in range(0, len(split_requests), 2):
+        update_pairs.append([split_requests[i], split_requests[i + 1]])
 
     return update_pairs
 

From e1ff0d6c3f4bbc0180dc8b857b43f4bb52be6775 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Thu, 14 Jul 2022 18:08:00 +0200
Subject: [PATCH 13/21] Add elastic output plugin test for _bulk post
 terminating newline.

---
 .../output_plugins/elasticsearch_plugin_test.py          | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
index f22287fa0..e2b6ce9e3 100644
--- a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
+++ b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
@@ -203,6 +203,15 @@ def testRaisesForHttpError(self):
             responses=[rdf_client.Process(pid=42)],
             patcher=mock.patch.object(requests, 'post', post))
 
+  def testPostDataTerminatingNewline(self):
+      with test_lib.ConfigOverrider({
+          'Elasticsearch.url': 'http://a',
+          'Elasticsearch.token': 'b',
+      }):
+          mock_post = self._CallPlugin(
+              plugin_args=elasticsearch_plugin.ElasticsearchOutputPluginArgs(),
+              responses=[rdf_client.Process(pid=42)])
+      self.assertTrue(mock_post.call_args[KWARGS]['data'].endswith('\n'))
 
 if __name__ == '__main__':
   app.run(test_lib.main)

From 06dabbd09c83c58ced22f3b25ffc15f46b031a51 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Thu, 14 Jul 2022 18:11:56 +0200
Subject: [PATCH 14/21] Add elastic output plugin test for _bulk post content
 type json headers.

---
 .../output_plugins/elasticsearch_plugin_test.py               | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
index e2b6ce9e3..20a39b650 100644
--- a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
+++ b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
@@ -161,6 +161,10 @@ def testReadsConfigurationValuesCorrectly(self):
     self.assertFalse(mock_post.call_args[KWARGS]['verify'])
     self.assertEqual(mock_post.call_args[KWARGS]['headers']['Authorization'],
                      'Basic b')
+    self.assertTrue(
+        mock_post.call_args[KWARGS]['headers']['Content-Type'] == 'application/json' or
+        mock_post.call_args[KWARGS]['headers']['Content-Type'] == 'application/x-ndjson'
+    )
 
     bulk_pairs = self._ParseEvents(mock_post)
     self.assertEqual(bulk_pairs[0][0]['index']['_index'], 'e')

From 8e34d57f92556422894a341baca8bf6c81bd5979 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Thu, 14 Jul 2022 18:12:54 +0200
Subject: [PATCH 15/21] Change build-pipe for running only tests.

---
 .github/workflows/build.yml | 318 +++++++++++++++++++++++++++++++++++-
 1 file changed, 317 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 380197056..b20c59d02 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -6,13 +6,153 @@ env:
   GCS_LATEST_PATH: _latest_server_deb
   DOCKER_REPOSITORY: grrdocker/grr
 jobs:
+
+  test-ubuntu:
+    runs-on: ubuntu-18.04
+    env:
+      CHROME_DEB: google-chrome-stable_current_amd64.deb
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+      - name: Install
+        run: |
+          free -hmw
+          lscpu
+          sudo apt install -y libmysqlclient-dev
+          if [[ -z "$(type google-chrome 2>/dev/null)" ]]; then
+            wget "https://dl.google.com/linux/direct/${CHROME_DEB}" && sudo apt install -y "./${CHROME_DEB}";
+          fi
+          python -m venv "${HOME}/INSTALL"
+          travis/install.sh
+      - name: Test
+        run: |
+          source "${HOME}/INSTALL/bin/activate"
+          pip install pytest-xdist==2.2.1 pytest==6.2.5
+          # We have 4 vCPUs available, but only use 3 here to avoid timeouts like
+          # https://ci.appveyor.com/project/grr/grr-ia94e/builds/20483467/messages ,
+          # which happen when tests stall.
+          pytest --verbose -n 3 grr/ --ignore grr/server/grr_response_server/gui/selenium_tests/ --ignore grr/client/grr_response_client/client_actions/windows/
+          # jsTree tests seem to fail on Chrome 71 headless due to https://github.com/GoogleChrome/puppeteer/issues/3463
+          if [ $(google-chrome --version | grep -Eo " [0-9]{1,3}") != "71" ]; then (cd grr/server/grr_response_server/gui/static/ && npm run gulp test); fi
+
+  build-openapi:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+      - name: Install
+        run: |
+          sudo apt install -y libmysqlclient-dev
+          python -m venv "${HOME}/INSTALL"
+          travis/install.sh
+      - name: Build
+        run: |
+          source "${HOME}/INSTALL/bin/activate"
+          mkdir -p _openapi_artifacts/openapi_description
+          mkdir -p _openapi_artifacts/documentation
+          travis/build_api_documentation.sh "_openapi_artifacts/openapi_description/openapi_description.json" "_openapi_artifacts/documentation/openapi_documentation.html"
+          ls -la _openapi_artifacts/*
+      - name: Upload OpenAPI to GitHub artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: openapi
+          path: _openapi_artifacts/
+          retention-days: 1
+
+  build-linux:
+    runs-on: ubuntu-18.04
+    env:
+      GCS_TAG: ubuntu_64bit
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Set up
+        run: |
+          sudo apt install fakeroot debhelper libffi-dev libssl-dev
+          pip install virtualenv
+          virtualenv "${HOME}/INSTALL"
+      - name: Build
+        run: |
+          travis/install_client_builder.sh
+          travis/build_templates.sh
+          ls -la gcs_upload_dir
+      - name: Upload installers to GitHub artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: ubuntu-installers
+          path: gcs_upload_dir/
+          retention-days: 1
+
+  build-osx:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Set up
+        run: |
+          pip install --upgrade setuptools virtualenv
+          virtualenv "${HOME}/INSTALL"
+      - name: Build installers
+        run: |
+          travis/install_client_builder.sh
+          travis/build_templates.sh
+          ls -la gcs_upload_dir
+      - name: Upload installers to GitHub artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: osx-installers
+          path: gcs_upload_dir/
+          retention-days: 1
+
+  build-centos:
+    runs-on: ubuntu-18.04
+    env:
+      GCS_TAG: centos_64bit
+      DOCKER_IMG: grrdocker/centos7
+      DOCKER_CONTAINER: centos_64bit_container
+      DOCKER_USER: grrbot
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Build installers
+        run: |
+          sudo docker run -dit \
+            --volume "${PWD}:/mnt/grr" \
+            --workdir /mnt/grr \
+            --env DOCKER_USER="${DOCKER_USER}" \
+            --env TRAVIS_OS_NAME="linux" \
+            --name "${DOCKER_CONTAINER}" \
+            "${DOCKER_IMG}"
+          sudo docker exec "${DOCKER_CONTAINER}" travis/set_up_test_user.sh
+          sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" /usr/local/bin/python3.6 -m venv "/home/${DOCKER_USER}/INSTALL"
+          sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" travis/install_client_builder.sh
+          sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" travis/build_templates.sh
+          sudo docker exec "${DOCKER_CONTAINER}" rpm -vih gcs_upload_dir/*.rpm
+          ls -la gcs_upload_dir
+      - name: Upload installers to GitHub artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: centos-installers
+          path: gcs_upload_dir/
+          retention-days: 1
+
   build-windows:
     runs-on: windows-2022
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.8'
+          python-version: '3.7'
       - name: Build installers
         shell: bash
         run: |
@@ -30,3 +170,179 @@ jobs:
           name: windows-installers
           path: gcs_upload_dir/
           retention-days: 1
+
+  build-server-deb:
+    runs-on: ubuntu-18.04
+    env:
+      GCS_TAG: server_deb
+    needs:
+      - build-centos
+      - build-linux
+      - build-osx
+      - build-windows
+    steps:
+      - uses: actions/checkout@v2
+      - name: Download installers from GitHub artifacts
+        id: download
+        uses: actions/download-artifact@v2
+        with:
+          path: ~/_artifacts
+      - name: Set up
+        run: |
+          sudo apt-get install -y fakeroot debhelper libffi-dev libssl-dev python3-dev python3-pip python3-venv wget openjdk-8-jdk zip git devscripts dh-systemd libmysqlclient-dev dh-virtualenv dh-make libc6-i386 lib32z1
+          pip3 install --upgrade setuptools virtualenv
+          virtualenv "${HOME}/INSTALL"
+      - name: Build
+        run: |
+          travis/install.sh
+
+          mkdir -p grr/config/grr_response_templates/templates
+
+          mv -v ~/_artifacts/windows-installers/GRR_*_amd64.exe.zip grr/config/grr_response_templates/templates
+          mv -v ~/_artifacts/windows-installers/GRR_*_amd64.msi.zip grr/config/grr_response_templates/templates
+          mv -v ~/_artifacts/ubuntu-installers/grr_*_amd64.deb.zip grr/config/grr_response_templates/templates
+          mv -v ~/_artifacts/centos-installers/grr_*_amd64.rpm.zip grr/config/grr_response_templates/templates
+          mv -v ~/_artifacts/osx-installers/grr_*_amd64.xar.zip grr/config/grr_response_templates/templates
+
+          travis/build_local_pyindex.sh
+          travis/build_server_deb.sh
+          ls -la gcs_upload_dir
+      - name: Upload installers to GitHub artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: server-deb
+          path: gcs_upload_dir/
+          retention-days: 1
+
+  test-ubuntu-e2e:
+    continue-on-error: true  # Debug follow up step.
+    runs-on: ubuntu-18.04
+    env:
+      GRR_ADMIN_PASS: 'e2e_tests'
+      APPVEYOR_MYSQL_PASS: 'root' 
+    needs:
+      - build-server-deb
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Set up MySQL
+        run: |
+          sudo /etc/init.d/mysql start
+      - name: Download installers from GitHub artifacts
+        id: download
+        uses: actions/download-artifact@v2
+        with:
+          name: server-deb
+          path: _artifacts
+      - name: Install
+        run: |
+          free -hmw
+          lscpu
+          sudo apt install -y libmysqlclient-dev
+          if [[ -z "$(type google-chrome 2>/dev/null)" ]]; then
+            wget "https://dl.google.com/linux/direct/${CHROME_DEB}" && sudo apt install -y "./${CHROME_DEB}";
+          fi
+          python -m venv "${HOME}/INSTALL"
+          travis/install.sh
+          sudo -EH ./appveyor/e2e_tests/install_mem_usage_cron.sh
+          sudo -EH ./appveyor/e2e_tests/install_latest_server_deb.sh
+      - name: Test
+        run: |
+          sudo -EH ./appveyor/e2e_tests/run_e2e_tests.sh
+          sudo -EH ./appveyor/e2e_tests/test_repack.sh
+      - name: Upload logs and configs to GitHub artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: e2e-test
+          path: appveyor_e2e_artifacts/
+          retention-days: 1
+
+
+  build-push-docker:
+    runs-on: ubuntu-18.04
+    needs:
+      - build-server-deb
+      # - test-ubuntu-e2e  # TODO: Comment back in after debugging is finished.
+      - test-ubuntu
+      - build-openapi
+    steps:
+      - uses: actions/checkout@v2
+      - name: Download installers from GitHub artifacts
+        id: download
+        uses: actions/download-artifact@v2
+        with:
+          name: server-deb
+          path: _artifacts
+      - name: Build Docker image
+        run: |
+          export BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3)
+          ./appveyor/docker_build/build_docker_image.sh
+      - if: ${{ github.event_name == 'push' }} 
+        name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - if: ${{ github.event_name == 'push' }}
+        name: Push to ${{ env.DOCKER_REPOSITORY }}
+        run: |
+          docker push -a ${{ env.DOCKER_REPOSITORY }}
+  upload:
+    if: ${{ github.event_name == 'push' }}
+    runs-on: ubuntu-18.04
+    needs:
+      - build-push-docker
+    steps:
+      - uses: actions/checkout@v2
+      - name: Download installers from GitHub artifacts
+        id: download
+        uses: actions/download-artifact@v2
+        with:
+          path: _artifacts
+      - run: |
+          ls -la _artifacts/*/
+
+          COMMIT_TIME=$(git show -s --date='format-local:%Y-%m-%dT%H:%M:%SZ' --format="%cd" $GITHUB_SHA)
+          OUTPUT_DIR=gcs_upload_dir/${COMMIT_TIME}_${GITHUB_SHA}/
+          echo "OUTPUT_DIR=$OUTPUT_DIR" >> $GITHUB_ENV
+
+          mkdir -p $OUTPUT_DIR/centos/
+          mv -v _artifacts/centos-installers/* $OUTPUT_DIR/centos
+
+          mkdir -p $OUTPUT_DIR/ubuntu/
+          mv -v _artifacts/ubuntu-installers/* $OUTPUT_DIR/ubuntu
+
+          mkdir -p $OUTPUT_DIR/osx/
+          mv -v _artifacts/osx-installers/* $OUTPUT_DIR/osx
+
+          mkdir -p $OUTPUT_DIR/windows/
+          mv -v _artifacts/windows-installers/* $OUTPUT_DIR/windows
+
+          mkdir -p $OUTPUT_DIR/server_deb/
+          mv -v _artifacts/server-deb/* $OUTPUT_DIR/server_deb
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v0.2.1
+        with:
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+          export_default_credentials: true
+      - name: Upload installers to GCS
+        uses: google-github-actions/upload-cloud-storage@main
+        with:
+          path: gcs_upload_dir/
+          destination: ${{ env.GCS_BUCKET }}
+          # Omit `path` (e.g. /home/runner/deploy/) in final GCS path.
+          parent: false
+      - name: Replace ${{ env.GCS_LATEST_PATH }} folder in GCS
+        run: |
+          gsutil rm gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/** || true
+          gsutil cp -r $OUTPUT_DIR/server_deb/* gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/
+      - name: Upload OpenAPI to GCS
+        uses: google-github-actions/upload-cloud-storage@main
+        with:
+          path: _artifacts/openapi/
+          destination: ${{ env.GCS_BUCKET_OPENAPI }}
+          # Omit `path` (e.g. /home/runner/deploy/) in final GCS path.
+          parent: false

From 0f1625a0c2449c1661e389a878944bd6e1de4682 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Thu, 14 Jul 2022 18:17:09 +0200
Subject: [PATCH 16/21] Change build-pipe for running only tests.

---
 .github/workflows/build.yml | 312 +-----------------------------------
 1 file changed, 1 insertion(+), 311 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b20c59d02..316ee530b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -35,314 +35,4 @@ jobs:
           # which happen when tests stall.
           pytest --verbose -n 3 grr/ --ignore grr/server/grr_response_server/gui/selenium_tests/ --ignore grr/client/grr_response_client/client_actions/windows/
           # jsTree tests seem to fail on Chrome 71 headless due to https://github.com/GoogleChrome/puppeteer/issues/3463
-          if [ $(google-chrome --version | grep -Eo " [0-9]{1,3}") != "71" ]; then (cd grr/server/grr_response_server/gui/static/ && npm run gulp test); fi
-
-  build-openapi:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Install
-        run: |
-          sudo apt install -y libmysqlclient-dev
-          python -m venv "${HOME}/INSTALL"
-          travis/install.sh
-      - name: Build
-        run: |
-          source "${HOME}/INSTALL/bin/activate"
-          mkdir -p _openapi_artifacts/openapi_description
-          mkdir -p _openapi_artifacts/documentation
-          travis/build_api_documentation.sh "_openapi_artifacts/openapi_description/openapi_description.json" "_openapi_artifacts/documentation/openapi_documentation.html"
-          ls -la _openapi_artifacts/*
-      - name: Upload OpenAPI to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: openapi
-          path: _openapi_artifacts/
-          retention-days: 1
-
-  build-linux:
-    runs-on: ubuntu-18.04
-    env:
-      GCS_TAG: ubuntu_64bit
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.6'
-      - name: Set up
-        run: |
-          sudo apt install fakeroot debhelper libffi-dev libssl-dev
-          pip install virtualenv
-          virtualenv "${HOME}/INSTALL"
-      - name: Build
-        run: |
-          travis/install_client_builder.sh
-          travis/build_templates.sh
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: ubuntu-installers
-          path: gcs_upload_dir/
-          retention-days: 1
-
-  build-osx:
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.6'
-      - name: Set up
-        run: |
-          pip install --upgrade setuptools virtualenv
-          virtualenv "${HOME}/INSTALL"
-      - name: Build installers
-        run: |
-          travis/install_client_builder.sh
-          travis/build_templates.sh
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: osx-installers
-          path: gcs_upload_dir/
-          retention-days: 1
-
-  build-centos:
-    runs-on: ubuntu-18.04
-    env:
-      GCS_TAG: centos_64bit
-      DOCKER_IMG: grrdocker/centos7
-      DOCKER_CONTAINER: centos_64bit_container
-      DOCKER_USER: grrbot
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.6'
-      - name: Build installers
-        run: |
-          sudo docker run -dit \
-            --volume "${PWD}:/mnt/grr" \
-            --workdir /mnt/grr \
-            --env DOCKER_USER="${DOCKER_USER}" \
-            --env TRAVIS_OS_NAME="linux" \
-            --name "${DOCKER_CONTAINER}" \
-            "${DOCKER_IMG}"
-          sudo docker exec "${DOCKER_CONTAINER}" travis/set_up_test_user.sh
-          sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" /usr/local/bin/python3.6 -m venv "/home/${DOCKER_USER}/INSTALL"
-          sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" travis/install_client_builder.sh
-          sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" travis/build_templates.sh
-          sudo docker exec "${DOCKER_CONTAINER}" rpm -vih gcs_upload_dir/*.rpm
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: centos-installers
-          path: gcs_upload_dir/
-          retention-days: 1
-
-  build-windows:
-    runs-on: windows-2022
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Build installers
-        shell: bash
-        run: |
-          set -ex
-          pip install virtualenv wheel
-          python -u appveyor/windows_templates/build_windows_templates.py --grr_src=$GITHUB_WORKSPACE --output_dir=$GITHUB_WORKSPACE/output --test_repack_install
-          python -u appveyor/windows_templates/build_windows_templates.py --grr_src=$GITHUB_WORKSPACE --output_dir=$GITHUB_WORKSPACE/output_msi --test_repack_install --build_msi
-
-          mkdir -p gcs_upload_dir
-          mv -v output*/* gcs_upload_dir
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: windows-installers
-          path: gcs_upload_dir/
-          retention-days: 1
-
-  build-server-deb:
-    runs-on: ubuntu-18.04
-    env:
-      GCS_TAG: server_deb
-    needs:
-      - build-centos
-      - build-linux
-      - build-osx
-      - build-windows
-    steps:
-      - uses: actions/checkout@v2
-      - name: Download installers from GitHub artifacts
-        id: download
-        uses: actions/download-artifact@v2
-        with:
-          path: ~/_artifacts
-      - name: Set up
-        run: |
-          sudo apt-get install -y fakeroot debhelper libffi-dev libssl-dev python3-dev python3-pip python3-venv wget openjdk-8-jdk zip git devscripts dh-systemd libmysqlclient-dev dh-virtualenv dh-make libc6-i386 lib32z1
-          pip3 install --upgrade setuptools virtualenv
-          virtualenv "${HOME}/INSTALL"
-      - name: Build
-        run: |
-          travis/install.sh
-
-          mkdir -p grr/config/grr_response_templates/templates
-
-          mv -v ~/_artifacts/windows-installers/GRR_*_amd64.exe.zip grr/config/grr_response_templates/templates
-          mv -v ~/_artifacts/windows-installers/GRR_*_amd64.msi.zip grr/config/grr_response_templates/templates
-          mv -v ~/_artifacts/ubuntu-installers/grr_*_amd64.deb.zip grr/config/grr_response_templates/templates
-          mv -v ~/_artifacts/centos-installers/grr_*_amd64.rpm.zip grr/config/grr_response_templates/templates
-          mv -v ~/_artifacts/osx-installers/grr_*_amd64.xar.zip grr/config/grr_response_templates/templates
-
-          travis/build_local_pyindex.sh
-          travis/build_server_deb.sh
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: server-deb
-          path: gcs_upload_dir/
-          retention-days: 1
-
-  test-ubuntu-e2e:
-    continue-on-error: true  # Debug follow up step.
-    runs-on: ubuntu-18.04
-    env:
-      GRR_ADMIN_PASS: 'e2e_tests'
-      APPVEYOR_MYSQL_PASS: 'root' 
-    needs:
-      - build-server-deb
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.6'
-      - name: Set up MySQL
-        run: |
-          sudo /etc/init.d/mysql start
-      - name: Download installers from GitHub artifacts
-        id: download
-        uses: actions/download-artifact@v2
-        with:
-          name: server-deb
-          path: _artifacts
-      - name: Install
-        run: |
-          free -hmw
-          lscpu
-          sudo apt install -y libmysqlclient-dev
-          if [[ -z "$(type google-chrome 2>/dev/null)" ]]; then
-            wget "https://dl.google.com/linux/direct/${CHROME_DEB}" && sudo apt install -y "./${CHROME_DEB}";
-          fi
-          python -m venv "${HOME}/INSTALL"
-          travis/install.sh
-          sudo -EH ./appveyor/e2e_tests/install_mem_usage_cron.sh
-          sudo -EH ./appveyor/e2e_tests/install_latest_server_deb.sh
-      - name: Test
-        run: |
-          sudo -EH ./appveyor/e2e_tests/run_e2e_tests.sh
-          sudo -EH ./appveyor/e2e_tests/test_repack.sh
-      - name: Upload logs and configs to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: e2e-test
-          path: appveyor_e2e_artifacts/
-          retention-days: 1
-
-
-  build-push-docker:
-    runs-on: ubuntu-18.04
-    needs:
-      - build-server-deb
-      # - test-ubuntu-e2e  # TODO: Comment back in after debugging is finished.
-      - test-ubuntu
-      - build-openapi
-    steps:
-      - uses: actions/checkout@v2
-      - name: Download installers from GitHub artifacts
-        id: download
-        uses: actions/download-artifact@v2
-        with:
-          name: server-deb
-          path: _artifacts
-      - name: Build Docker image
-        run: |
-          export BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3)
-          ./appveyor/docker_build/build_docker_image.sh
-      - if: ${{ github.event_name == 'push' }} 
-        name: Log in to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - if: ${{ github.event_name == 'push' }}
-        name: Push to ${{ env.DOCKER_REPOSITORY }}
-        run: |
-          docker push -a ${{ env.DOCKER_REPOSITORY }}
-  upload:
-    if: ${{ github.event_name == 'push' }}
-    runs-on: ubuntu-18.04
-    needs:
-      - build-push-docker
-    steps:
-      - uses: actions/checkout@v2
-      - name: Download installers from GitHub artifacts
-        id: download
-        uses: actions/download-artifact@v2
-        with:
-          path: _artifacts
-      - run: |
-          ls -la _artifacts/*/
-
-          COMMIT_TIME=$(git show -s --date='format-local:%Y-%m-%dT%H:%M:%SZ' --format="%cd" $GITHUB_SHA)
-          OUTPUT_DIR=gcs_upload_dir/${COMMIT_TIME}_${GITHUB_SHA}/
-          echo "OUTPUT_DIR=$OUTPUT_DIR" >> $GITHUB_ENV
-
-          mkdir -p $OUTPUT_DIR/centos/
-          mv -v _artifacts/centos-installers/* $OUTPUT_DIR/centos
-
-          mkdir -p $OUTPUT_DIR/ubuntu/
-          mv -v _artifacts/ubuntu-installers/* $OUTPUT_DIR/ubuntu
-
-          mkdir -p $OUTPUT_DIR/osx/
-          mv -v _artifacts/osx-installers/* $OUTPUT_DIR/osx
-
-          mkdir -p $OUTPUT_DIR/windows/
-          mv -v _artifacts/windows-installers/* $OUTPUT_DIR/windows
-
-          mkdir -p $OUTPUT_DIR/server_deb/
-          mv -v _artifacts/server-deb/* $OUTPUT_DIR/server_deb
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v0.2.1
-        with:
-          project_id: ${{ secrets.GCP_PROJECT_ID }}
-          service_account_key: ${{ secrets.GCP_SA_KEY }}
-          export_default_credentials: true
-      - name: Upload installers to GCS
-        uses: google-github-actions/upload-cloud-storage@main
-        with:
-          path: gcs_upload_dir/
-          destination: ${{ env.GCS_BUCKET }}
-          # Omit `path` (e.g. /home/runner/deploy/) in final GCS path.
-          parent: false
-      - name: Replace ${{ env.GCS_LATEST_PATH }} folder in GCS
-        run: |
-          gsutil rm gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/** || true
-          gsutil cp -r $OUTPUT_DIR/server_deb/* gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/
-      - name: Upload OpenAPI to GCS
-        uses: google-github-actions/upload-cloud-storage@main
-        with:
-          path: _artifacts/openapi/
-          destination: ${{ env.GCS_BUCKET_OPENAPI }}
-          # Omit `path` (e.g. /home/runner/deploy/) in final GCS path.
-          parent: false
+          if [ $(google-chrome --version | grep -Eo " [0-9]{1,3}") != "71" ]; then (cd grr/server/grr_response_server/gui/static/ && npm run gulp test); fi
\ No newline at end of file

From 3ed3225f37e45f24b7d327a6d469e656acf5dfaf Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Thu, 14 Jul 2022 18:24:07 +0200
Subject: [PATCH 17/21] Change to py3.8

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 316ee530b..5f5befca4 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.7'
+          python-version: '3.8'
       - name: Install
         run: |
           free -hmw

From 31d63837e308dd4bd3f926a9ce82f74b54538629 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Mon, 25 Jul 2022 19:30:48 +0200
Subject: [PATCH 18/21] Remove commented code.

---
 .../output_plugins/elasticsearch_plugin_test.py                 | 2 --
 1 file changed, 2 deletions(-)

diff --git a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
index 20a39b650..3e87d599f 100644
--- a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
+++ b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
@@ -75,8 +75,6 @@ def _ParseEvents(self, patched):
         if not line:
             continue
         split_requests.append(json.Parse(line))
-    #update_pairs = [(split_requests[i], split_requests[i + 1])
-    #                for i in range(0, len(split_requests), 2)]
     update_pairs = []
     for i in range(0, len(split_requests), 2):
         update_pairs.append([split_requests[i], split_requests[i + 1]])

From 1f673cf1a14227bade3f737fb0e1415a90704340 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Mon, 25 Jul 2022 19:36:50 +0200
Subject: [PATCH 19/21] Fix snake_case.

---
 .../output_plugins/elasticsearch_plugin_test.py               | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
index 3e87d599f..eed829b7f 100644
--- a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
+++ b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
@@ -69,8 +69,8 @@ def _ParseEvents(self, patched):
     # Elasticsearch bulk requests are line-deliminated pairs, where the first
     # line is the index command and the second is the actual document to index
     split_requests = []
-    splitRequest = request.split('\n')
-    for line in splitRequest:
+    split_request = request.split('\n')
+    for line in split_request:
         # Skip terminating newlines - which crashes json.Parse
         if not line:
             continue

From 5bb79bcffdeb694a52e4342584e5a2737c14eac0 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Mon, 25 Jul 2022 19:52:02 +0200
Subject: [PATCH 20/21] Change assertTrue to assertIn.

---
 .../output_plugins/elasticsearch_plugin_test.py             | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
index eed829b7f..ad1b9d88a 100644
--- a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
+++ b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
@@ -159,9 +159,9 @@ def testReadsConfigurationValuesCorrectly(self):
     self.assertFalse(mock_post.call_args[KWARGS]['verify'])
     self.assertEqual(mock_post.call_args[KWARGS]['headers']['Authorization'],
                      'Basic b')
-    self.assertTrue(
-        mock_post.call_args[KWARGS]['headers']['Content-Type'] == 'application/json' or
-        mock_post.call_args[KWARGS]['headers']['Content-Type'] == 'application/x-ndjson'
+    self.assertIn(
+        mock_post.call_args[KWARGS]['headers']['Content-Type'],
+        ['application/json', 'application/x-ndjson']
     )
 
     bulk_pairs = self._ParseEvents(mock_post)

From 2fffcccd611ec188d5e84ce26b075987a3c89381 Mon Sep 17 00:00:00 2001
From: Fredrik Dyrvold <fredrik@dyrvold.dev>
Date: Mon, 25 Jul 2022 20:01:27 +0200
Subject: [PATCH 21/21] Switch from list to tuple.

---
 .../output_plugins/elasticsearch_plugin_test.py                 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
index ad1b9d88a..2871495ea 100644
--- a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
+++ b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py
@@ -161,7 +161,7 @@ def testReadsConfigurationValuesCorrectly(self):
                      'Basic b')
     self.assertIn(
         mock_post.call_args[KWARGS]['headers']['Content-Type'],
-        ['application/json', 'application/x-ndjson']
+        ('application/json', 'application/x-ndjson')
     )
 
     bulk_pairs = self._ParseEvents(mock_post)