fredx30 · fredx30 · Jun 30, 2022 · Jun 30, 2022 · Jun 30, 2022 · Jul 12, 2022
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.7'
+          python-version: '3.8'
       - name: Install
         run: |
           free -hmw
@@ -35,314 +35,4 @@ jobs:
           # which happen when tests stall.
           pytest --verbose -n 3 grr/ --ignore grr/server/grr_response_server/gui/selenium_tests/ --ignore grr/client/grr_response_client/client_actions/windows/
           # jsTree tests seem to fail on Chrome 71 headless due to https://github.com/GoogleChrome/puppeteer/issues/3463
-          if [ $(google-chrome --version | grep -Eo " [0-9]{1,3}") != "71" ]; then (cd grr/server/grr_response_server/gui/static/ && npm run gulp test); fi
-
-  build-openapi:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Install
-        run: |
-          sudo apt install -y libmysqlclient-dev
-          python -m venv "${HOME}/INSTALL"
-          travis/install.sh
-      - name: Build
-        run: |
-          source "${HOME}/INSTALL/bin/activate"
-          mkdir -p _openapi_artifacts/openapi_description
-          mkdir -p _openapi_artifacts/documentation
-          travis/build_api_documentation.sh "_openapi_artifacts/openapi_description/openapi_description.json" "_openapi_artifacts/documentation/openapi_documentation.html"
-          ls -la _openapi_artifacts/*
-      - name: Upload OpenAPI to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: openapi
-          path: _openapi_artifacts/
-          retention-days: 1
-
-  build-linux:
-    runs-on: ubuntu-18.04
-    env:
-      GCS_TAG: ubuntu_64bit
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.6'
-      - name: Set up
-        run: |
-          sudo apt install fakeroot debhelper libffi-dev libssl-dev
-          pip install virtualenv
-          virtualenv "${HOME}/INSTALL"
-      - name: Build
-        run: |
-          travis/install_client_builder.sh
-          travis/build_templates.sh
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: ubuntu-installers
-          path: gcs_upload_dir/
-          retention-days: 1
-
-  build-osx:
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.6'
-      - name: Set up
-        run: |
-          pip install --upgrade setuptools virtualenv
-          virtualenv "${HOME}/INSTALL"
-      - name: Build installers
-        run: |
-          travis/install_client_builder.sh
-          travis/build_templates.sh
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: osx-installers
-          path: gcs_upload_dir/
-          retention-days: 1
-
-  build-centos:
-    runs-on: ubuntu-18.04
-    env:
-      GCS_TAG: centos_64bit
-      DOCKER_IMG: grrdocker/centos7
-      DOCKER_CONTAINER: centos_64bit_container
-      DOCKER_USER: grrbot
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.6'
-      - name: Build installers
-        run: |
-          sudo docker run -dit \
-            --volume "${PWD}:/mnt/grr" \
-            --workdir /mnt/grr \
-            --env DOCKER_USER="${DOCKER_USER}" \
-            --env TRAVIS_OS_NAME="linux" \
-            --name "${DOCKER_CONTAINER}" \
-            "${DOCKER_IMG}"
-          sudo docker exec "${DOCKER_CONTAINER}" travis/set_up_test_user.sh
-          sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" /usr/local/bin/python3.6 -m venv "/home/${DOCKER_USER}/INSTALL"
-          sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" travis/install_client_builder.sh
-          sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" travis/build_templates.sh
-          sudo docker exec "${DOCKER_CONTAINER}" rpm -vih gcs_upload_dir/*.rpm
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: centos-installers
-          path: gcs_upload_dir/
-          retention-days: 1
-
-  build-windows:
-    runs-on: windows-2022
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Build installers
-        shell: bash
-        run: |
-          set -ex
-          pip install virtualenv wheel
-          python -u appveyor/windows_templates/build_windows_templates.py --grr_src=$GITHUB_WORKSPACE --output_dir=$GITHUB_WORKSPACE/output --test_repack_install
-          python -u appveyor/windows_templates/build_windows_templates.py --grr_src=$GITHUB_WORKSPACE --output_dir=$GITHUB_WORKSPACE/output_msi --test_repack_install --build_msi
-
-          mkdir -p gcs_upload_dir
-          mv -v output*/* gcs_upload_dir
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: windows-installers
-          path: gcs_upload_dir/
-          retention-days: 1
-
-  build-server-deb:
-    runs-on: ubuntu-18.04
-    env:
-      GCS_TAG: server_deb
-    needs:
-      - build-centos
-      - build-linux
-      - build-osx
-      - build-windows
-    steps:
-      - uses: actions/checkout@v2
-      - name: Download installers from GitHub artifacts
-        id: download
-        uses: actions/download-artifact@v2
-        with:
-          path: ~/_artifacts
-      - name: Set up
-        run: |
-          sudo apt-get install -y fakeroot debhelper libffi-dev libssl-dev python3-dev python3-pip python3-venv wget openjdk-8-jdk zip git devscripts dh-systemd libmysqlclient-dev dh-virtualenv dh-make libc6-i386 lib32z1
-          pip3 install --upgrade setuptools virtualenv
-          virtualenv "${HOME}/INSTALL"
-      - name: Build
-        run: |
-          travis/install.sh
-
-          mkdir -p grr/config/grr_response_templates/templates
-
-          mv -v ~/_artifacts/windows-installers/GRR_*_amd64.exe.zip grr/config/grr_response_templates/templates
-          mv -v ~/_artifacts/windows-installers/GRR_*_amd64.msi.zip grr/config/grr_response_templates/templates
-          mv -v ~/_artifacts/ubuntu-installers/grr_*_amd64.deb.zip grr/config/grr_response_templates/templates
-          mv -v ~/_artifacts/centos-installers/grr_*_amd64.rpm.zip grr/config/grr_response_templates/templates
-          mv -v ~/_artifacts/osx-installers/grr_*_amd64.xar.zip grr/config/grr_response_templates/templates
-
-          travis/build_local_pyindex.sh
-          travis/build_server_deb.sh
-          ls -la gcs_upload_dir
-      - name: Upload installers to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: server-deb
-          path: gcs_upload_dir/
-          retention-days: 1
-
-  test-ubuntu-e2e:
-    continue-on-error: true  # Debug follow up step.
-    runs-on: ubuntu-18.04
-    env:
-      GRR_ADMIN_PASS: 'e2e_tests'
-      APPVEYOR_MYSQL_PASS: 'root' 
-    needs:
-      - build-server-deb
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.6'
-      - name: Set up MySQL
-        run: |
-          sudo /etc/init.d/mysql start
-      - name: Download installers from GitHub artifacts
-        id: download
-        uses: actions/download-artifact@v2
-        with:
-          name: server-deb
-          path: _artifacts
-      - name: Install
-        run: |
-          free -hmw
-          lscpu
-          sudo apt install -y libmysqlclient-dev
-          if [[ -z "$(type google-chrome 2>/dev/null)" ]]; then
-            wget "https://dl.google.com/linux/direct/${CHROME_DEB}" && sudo apt install -y "./${CHROME_DEB}";
-          fi
-          python -m venv "${HOME}/INSTALL"
-          travis/install.sh
-          sudo -EH ./appveyor/e2e_tests/install_mem_usage_cron.sh
-          sudo -EH ./appveyor/e2e_tests/install_latest_server_deb.sh
-      - name: Test
-        run: |
-          sudo -EH ./appveyor/e2e_tests/run_e2e_tests.sh
-          sudo -EH ./appveyor/e2e_tests/test_repack.sh
-      - name: Upload logs and configs to GitHub artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: e2e-test
-          path: appveyor_e2e_artifacts/
-          retention-days: 1
-
-
-  build-push-docker:
-    runs-on: ubuntu-18.04
-    needs:
-      - build-server-deb
-      # - test-ubuntu-e2e  # TODO: Comment back in after debugging is finished.
-      - test-ubuntu
-      - build-openapi
-    steps:
-      - uses: actions/checkout@v2
-      - name: Download installers from GitHub artifacts
-        id: download
-        uses: actions/download-artifact@v2
-        with:
-          name: server-deb
-          path: _artifacts
-      - name: Build Docker image
-        run: |
-          export BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3)
-          ./appveyor/docker_build/build_docker_image.sh
-      - if: ${{ github.event_name == 'push' }} 
-        name: Log in to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - if: ${{ github.event_name == 'push' }}
-        name: Push to ${{ env.DOCKER_REPOSITORY }}
-        run: |
-          docker push -a ${{ env.DOCKER_REPOSITORY }}
-  upload:
-    if: ${{ github.event_name == 'push' }}
-    runs-on: ubuntu-18.04
-    needs:
-      - build-push-docker
-    steps:
-      - uses: actions/checkout@v2
-      - name: Download installers from GitHub artifacts
-        id: download
-        uses: actions/download-artifact@v2
-        with:
-          path: _artifacts
-      - run: |
-          ls -la _artifacts/*/
-
-          COMMIT_TIME=$(git show -s --date='format-local:%Y-%m-%dT%H:%M:%SZ' --format="%cd" $GITHUB_SHA)
-          OUTPUT_DIR=gcs_upload_dir/${COMMIT_TIME}_${GITHUB_SHA}/
-          echo "OUTPUT_DIR=$OUTPUT_DIR" >> $GITHUB_ENV
-
-          mkdir -p $OUTPUT_DIR/centos/
-          mv -v _artifacts/centos-installers/* $OUTPUT_DIR/centos
-
-          mkdir -p $OUTPUT_DIR/ubuntu/
-          mv -v _artifacts/ubuntu-installers/* $OUTPUT_DIR/ubuntu
-
-          mkdir -p $OUTPUT_DIR/osx/
-          mv -v _artifacts/osx-installers/* $OUTPUT_DIR/osx
-
-          mkdir -p $OUTPUT_DIR/windows/
-          mv -v _artifacts/windows-installers/* $OUTPUT_DIR/windows
-
-          mkdir -p $OUTPUT_DIR/server_deb/
-          mv -v _artifacts/server-deb/* $OUTPUT_DIR/server_deb
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v0.2.1
-        with:
-          project_id: ${{ secrets.GCP_PROJECT_ID }}
-          service_account_key: ${{ secrets.GCP_SA_KEY }}
-          export_default_credentials: true
-      - name: Upload installers to GCS
-        uses: google-github-actions/upload-cloud-storage@main
-        with:
-          path: gcs_upload_dir/
-          destination: ${{ env.GCS_BUCKET }}
-          # Omit `path` (e.g. /home/runner/deploy/) in final GCS path.
-          parent: false
-      - name: Replace ${{ env.GCS_LATEST_PATH }} folder in GCS
-        run: |
-          gsutil rm gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/** || true
-          gsutil cp -r $OUTPUT_DIR/server_deb/* gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/
-      - name: Upload OpenAPI to GCS
-        uses: google-github-actions/upload-cloud-storage@main
-        with:
-          path: _artifacts/openapi/
-          destination: ${{ env.GCS_BUCKET_OPENAPI }}
-          # Omit `path` (e.g. /home/runner/deploy/) in final GCS path.
-          parent: false
+          if [ $(google-chrome --version | grep -Eo " [0-9]{1,3}") != "71" ]; then (cd grr/server/grr_response_server/gui/static/ && npm run gulp test); fi
diff --git a/CHANGELOG b/CHANGELOG
@@ -5,6 +5,9 @@
 * Introduced Server.grr_binaries_readonly configuration option (set to False
   by default). When set to True, binaries and python hacks can't be overriden
   or deleted.
+* Updates elasticsearch output plugin post request to _bulk in the
+  elasticsearch api. Adds a terminating \n and content type headers for
+  application/json.
 
 ## 3.4.3.1
 

diff --git a/colab/setup.py b/colab/setup.py
@@ -60,8 +60,7 @@ def make_release_tree(self, base_dir, files):
     install_requires=[
         "grr_api_client==%s" % VERSION.get("Version", "packagedepends"),
         "grr_response_proto==%s" % VERSION.get("Version", "packagedepends"),
-        "humanize==2.4.0",
         "ipython==%s" % ("5.0.0" if sys.version_info < (3, 0) else "7.15.0"),
-        "numpy==1.18.5",
+        "numpy==1.20.3",
         "pandas==1.0.4",
     ])
diff --git a/conftest.py b/conftest.py
@@ -153,6 +153,12 @@ def thread_leak_check(request):
       "ApiSslServerTest",
       "GRRHTTPServerTestThread",
       "SharedMemDBTestThread",
+
+      # These threads seem to need to be allowed to run tests. Sample
+      # %contentroot%/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py --no-header --no-summary -q in %contentroot%\grr\server\grr_response_server\output_plugins
+      'pydevd.Writer',
+      'pydevd.Reader',
+      'pydevd.CommandThread',
   ]
 
   # Remove up to one instance of each allowed thread name.

diff --git a/grr/client/setup.py b/grr/client/setup.py
@@ -132,7 +132,7 @@ def run(self):
     include_package_data=True,
     python_requires=">=3.6",
     install_requires=[
-        "absl-py==0.9.0",
+        "absl-py==1.1.0",
         "grr-response-core==%s" % VERSION.get("Version", "packagedepends"),
         "PyInstaller==3.6",
         PYTSK3,
@@ -144,7 +144,7 @@ def run(self):
         # The following requirements are needed in Windows.
         ':sys_platform=="win32"': [
             "WMI==1.5.1",
-            "pywin32==228",
+            "pywin32==304",
         ],
     },
 )

diff --git a/grr/core/grr_response_core/lib/rdfvalues/structs.py b/grr/core/grr_response_core/lib/rdfvalues/structs.py
@@ -1082,7 +1082,7 @@ def ConvertToWireFormat(self, value):
     return (self.encoded_tag, VarintEncode(len(data)), data)
 
 
-class RepeatedFieldHelper(collections.Sequence, object):
+class RepeatedFieldHelper(collections.abc.Sequence, object):
   """A helper for the RDFProto to handle repeated fields.
 
   This helper is intended to only be constructed from the RDFProto class.

diff --git a/grr/core/grr_response_core/lib/util/precondition.py b/grr/core/grr_response_core/lib/util/precondition.py
@@ -57,7 +57,7 @@ def AssertIterableType(iterable, expected_item_type):
     message %= iterable
     raise TypeError(message)
 
-  AssertType(iterable, collections.Iterable)
+  AssertType(iterable, collections.abc.Iterable)
   for item in iterable:
     AssertType(item, expected_item_type)