diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b20c59d02..5f5befca4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.8' - name: Install run: | free -hmw @@ -35,314 +35,4 @@ jobs: # which happen when tests stall. pytest --verbose -n 3 grr/ --ignore grr/server/grr_response_server/gui/selenium_tests/ --ignore grr/client/grr_response_client/client_actions/windows/ # jsTree tests seem to fail on Chrome 71 headless due to https://github.com/GoogleChrome/puppeteer/issues/3463 - if [ $(google-chrome --version | grep -Eo " [0-9]{1,3}") != "71" ]; then (cd grr/server/grr_response_server/gui/static/ && npm run gulp test); fi - - build-openapi: - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Install - run: | - sudo apt install -y libmysqlclient-dev - python -m venv "${HOME}/INSTALL" - travis/install.sh - - name: Build - run: | - source "${HOME}/INSTALL/bin/activate" - mkdir -p _openapi_artifacts/openapi_description - mkdir -p _openapi_artifacts/documentation - travis/build_api_documentation.sh "_openapi_artifacts/openapi_description/openapi_description.json" "_openapi_artifacts/documentation/openapi_documentation.html" - ls -la _openapi_artifacts/* - - name: Upload OpenAPI to GitHub artifacts - uses: actions/upload-artifact@v2 - with: - name: openapi - path: _openapi_artifacts/ - retention-days: 1 - - build-linux: - runs-on: ubuntu-18.04 - env: - GCS_TAG: ubuntu_64bit - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.6' - - name: Set up - run: | - sudo apt install fakeroot debhelper libffi-dev libssl-dev - pip install virtualenv - virtualenv "${HOME}/INSTALL" - - name: Build - run: | - travis/install_client_builder.sh - travis/build_templates.sh - ls -la gcs_upload_dir - - name: Upload installers to GitHub artifacts - uses: actions/upload-artifact@v2 - with: - name: ubuntu-installers - path: gcs_upload_dir/ - retention-days: 1 - - build-osx: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.6' - - name: Set up - run: | - pip install --upgrade setuptools virtualenv - virtualenv "${HOME}/INSTALL" - - name: Build installers - run: | - travis/install_client_builder.sh - travis/build_templates.sh - ls -la gcs_upload_dir - - name: Upload installers to GitHub artifacts - uses: actions/upload-artifact@v2 - with: - name: osx-installers - path: gcs_upload_dir/ - retention-days: 1 - - build-centos: - runs-on: ubuntu-18.04 - env: - GCS_TAG: centos_64bit - DOCKER_IMG: grrdocker/centos7 - DOCKER_CONTAINER: centos_64bit_container - DOCKER_USER: grrbot - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.6' - - name: Build installers - run: | - sudo docker run -dit \ - --volume "${PWD}:/mnt/grr" \ - --workdir /mnt/grr \ - --env DOCKER_USER="${DOCKER_USER}" \ - --env TRAVIS_OS_NAME="linux" \ - --name "${DOCKER_CONTAINER}" \ - "${DOCKER_IMG}" - sudo docker exec "${DOCKER_CONTAINER}" travis/set_up_test_user.sh - sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" /usr/local/bin/python3.6 -m venv "/home/${DOCKER_USER}/INSTALL" - sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" travis/install_client_builder.sh - sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" travis/build_templates.sh - sudo docker exec "${DOCKER_CONTAINER}" rpm -vih gcs_upload_dir/*.rpm - ls -la gcs_upload_dir - - name: Upload installers to GitHub artifacts - uses: actions/upload-artifact@v2 - with: - name: centos-installers - path: gcs_upload_dir/ - retention-days: 1 - - build-windows: - runs-on: windows-2022 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Build installers - shell: bash - run: | - set -ex - pip install virtualenv wheel - python -u appveyor/windows_templates/build_windows_templates.py --grr_src=$GITHUB_WORKSPACE --output_dir=$GITHUB_WORKSPACE/output --test_repack_install - python -u appveyor/windows_templates/build_windows_templates.py --grr_src=$GITHUB_WORKSPACE --output_dir=$GITHUB_WORKSPACE/output_msi --test_repack_install --build_msi - - mkdir -p gcs_upload_dir - mv -v output*/* gcs_upload_dir - ls -la gcs_upload_dir - - name: Upload installers to GitHub artifacts - uses: actions/upload-artifact@v2 - with: - name: windows-installers - path: gcs_upload_dir/ - retention-days: 1 - - build-server-deb: - runs-on: ubuntu-18.04 - env: - GCS_TAG: server_deb - needs: - - build-centos - - build-linux - - build-osx - - build-windows - steps: - - uses: actions/checkout@v2 - - name: Download installers from GitHub artifacts - id: download - uses: actions/download-artifact@v2 - with: - path: ~/_artifacts - - name: Set up - run: | - sudo apt-get install -y fakeroot debhelper libffi-dev libssl-dev python3-dev python3-pip python3-venv wget openjdk-8-jdk zip git devscripts dh-systemd libmysqlclient-dev dh-virtualenv dh-make libc6-i386 lib32z1 - pip3 install --upgrade setuptools virtualenv - virtualenv "${HOME}/INSTALL" - - name: Build - run: | - travis/install.sh - - mkdir -p grr/config/grr_response_templates/templates - - mv -v ~/_artifacts/windows-installers/GRR_*_amd64.exe.zip grr/config/grr_response_templates/templates - mv -v ~/_artifacts/windows-installers/GRR_*_amd64.msi.zip grr/config/grr_response_templates/templates - mv -v ~/_artifacts/ubuntu-installers/grr_*_amd64.deb.zip grr/config/grr_response_templates/templates - mv -v ~/_artifacts/centos-installers/grr_*_amd64.rpm.zip grr/config/grr_response_templates/templates - mv -v ~/_artifacts/osx-installers/grr_*_amd64.xar.zip grr/config/grr_response_templates/templates - - travis/build_local_pyindex.sh - travis/build_server_deb.sh - ls -la gcs_upload_dir - - name: Upload installers to GitHub artifacts - uses: actions/upload-artifact@v2 - with: - name: server-deb - path: gcs_upload_dir/ - retention-days: 1 - - test-ubuntu-e2e: - continue-on-error: true # Debug follow up step. - runs-on: ubuntu-18.04 - env: - GRR_ADMIN_PASS: 'e2e_tests' - APPVEYOR_MYSQL_PASS: 'root' - needs: - - build-server-deb - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.6' - - name: Set up MySQL - run: | - sudo /etc/init.d/mysql start - - name: Download installers from GitHub artifacts - id: download - uses: actions/download-artifact@v2 - with: - name: server-deb - path: _artifacts - - name: Install - run: | - free -hmw - lscpu - sudo apt install -y libmysqlclient-dev - if [[ -z "$(type google-chrome 2>/dev/null)" ]]; then - wget "https://dl.google.com/linux/direct/${CHROME_DEB}" && sudo apt install -y "./${CHROME_DEB}"; - fi - python -m venv "${HOME}/INSTALL" - travis/install.sh - sudo -EH ./appveyor/e2e_tests/install_mem_usage_cron.sh - sudo -EH ./appveyor/e2e_tests/install_latest_server_deb.sh - - name: Test - run: | - sudo -EH ./appveyor/e2e_tests/run_e2e_tests.sh - sudo -EH ./appveyor/e2e_tests/test_repack.sh - - name: Upload logs and configs to GitHub artifacts - uses: actions/upload-artifact@v2 - with: - name: e2e-test - path: appveyor_e2e_artifacts/ - retention-days: 1 - - - build-push-docker: - runs-on: ubuntu-18.04 - needs: - - build-server-deb - # - test-ubuntu-e2e # TODO: Comment back in after debugging is finished. - - test-ubuntu - - build-openapi - steps: - - uses: actions/checkout@v2 - - name: Download installers from GitHub artifacts - id: download - uses: actions/download-artifact@v2 - with: - name: server-deb - path: _artifacts - - name: Build Docker image - run: | - export BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3) - ./appveyor/docker_build/build_docker_image.sh - - if: ${{ github.event_name == 'push' }} - name: Log in to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - if: ${{ github.event_name == 'push' }} - name: Push to ${{ env.DOCKER_REPOSITORY }} - run: | - docker push -a ${{ env.DOCKER_REPOSITORY }} - upload: - if: ${{ github.event_name == 'push' }} - runs-on: ubuntu-18.04 - needs: - - build-push-docker - steps: - - uses: actions/checkout@v2 - - name: Download installers from GitHub artifacts - id: download - uses: actions/download-artifact@v2 - with: - path: _artifacts - - run: | - ls -la _artifacts/*/ - - COMMIT_TIME=$(git show -s --date='format-local:%Y-%m-%dT%H:%M:%SZ' --format="%cd" $GITHUB_SHA) - OUTPUT_DIR=gcs_upload_dir/${COMMIT_TIME}_${GITHUB_SHA}/ - echo "OUTPUT_DIR=$OUTPUT_DIR" >> $GITHUB_ENV - - mkdir -p $OUTPUT_DIR/centos/ - mv -v _artifacts/centos-installers/* $OUTPUT_DIR/centos - - mkdir -p $OUTPUT_DIR/ubuntu/ - mv -v _artifacts/ubuntu-installers/* $OUTPUT_DIR/ubuntu - - mkdir -p $OUTPUT_DIR/osx/ - mv -v _artifacts/osx-installers/* $OUTPUT_DIR/osx - - mkdir -p $OUTPUT_DIR/windows/ - mv -v _artifacts/windows-installers/* $OUTPUT_DIR/windows - - mkdir -p $OUTPUT_DIR/server_deb/ - mv -v _artifacts/server-deb/* $OUTPUT_DIR/server_deb - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0.2.1 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - - name: Upload installers to GCS - uses: google-github-actions/upload-cloud-storage@main - with: - path: gcs_upload_dir/ - destination: ${{ env.GCS_BUCKET }} - # Omit `path` (e.g. /home/runner/deploy/) in final GCS path. - parent: false - - name: Replace ${{ env.GCS_LATEST_PATH }} folder in GCS - run: | - gsutil rm gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/** || true - gsutil cp -r $OUTPUT_DIR/server_deb/* gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/ - - name: Upload OpenAPI to GCS - uses: google-github-actions/upload-cloud-storage@main - with: - path: _artifacts/openapi/ - destination: ${{ env.GCS_BUCKET_OPENAPI }} - # Omit `path` (e.g. /home/runner/deploy/) in final GCS path. - parent: false + if [ $(google-chrome --version | grep -Eo " [0-9]{1,3}") != "71" ]; then (cd grr/server/grr_response_server/gui/static/ && npm run gulp test); fi \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG index 80678432c..c6c8e7171 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,6 +5,9 @@ * Introduced Server.grr_binaries_readonly configuration option (set to False by default). When set to True, binaries and python hacks can't be overriden or deleted. +* Updates elasticsearch output plugin post request to _bulk in the + elasticsearch api. Adds a terminating \n and content type headers for + application/json. ## 3.4.3.1 diff --git a/colab/setup.py b/colab/setup.py index d0661c7bc..7e750e09e 100644 --- a/colab/setup.py +++ b/colab/setup.py @@ -60,8 +60,7 @@ def make_release_tree(self, base_dir, files): install_requires=[ "grr_api_client==%s" % VERSION.get("Version", "packagedepends"), "grr_response_proto==%s" % VERSION.get("Version", "packagedepends"), - "humanize==2.4.0", "ipython==%s" % ("5.0.0" if sys.version_info < (3, 0) else "7.15.0"), - "numpy==1.18.5", + "numpy==1.20.3", "pandas==1.0.4", ]) diff --git a/conftest.py b/conftest.py index d551a5a39..bc5eb012e 100644 --- a/conftest.py +++ b/conftest.py @@ -153,6 +153,12 @@ def thread_leak_check(request): "ApiSslServerTest", "GRRHTTPServerTestThread", "SharedMemDBTestThread", + + # These threads seem to need to be allowed to run tests. Sample + # %contentroot%/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py --no-header --no-summary -q in %contentroot%\grr\server\grr_response_server\output_plugins + 'pydevd.Writer', + 'pydevd.Reader', + 'pydevd.CommandThread', ] # Remove up to one instance of each allowed thread name. diff --git a/grr/client/setup.py b/grr/client/setup.py index 74bc7ad13..f1a3a42e1 100644 --- a/grr/client/setup.py +++ b/grr/client/setup.py @@ -132,7 +132,7 @@ def run(self): include_package_data=True, python_requires=">=3.6", install_requires=[ - "absl-py==0.9.0", + "absl-py==1.1.0", "grr-response-core==%s" % VERSION.get("Version", "packagedepends"), "PyInstaller==3.6", PYTSK3, @@ -144,7 +144,7 @@ def run(self): # The following requirements are needed in Windows. ':sys_platform=="win32"': [ "WMI==1.5.1", - "pywin32==228", + "pywin32==304", ], }, ) diff --git a/grr/core/grr_response_core/lib/rdfvalues/structs.py b/grr/core/grr_response_core/lib/rdfvalues/structs.py index a882a2a3e..dc46cb939 100644 --- a/grr/core/grr_response_core/lib/rdfvalues/structs.py +++ b/grr/core/grr_response_core/lib/rdfvalues/structs.py @@ -1082,7 +1082,7 @@ def ConvertToWireFormat(self, value): return (self.encoded_tag, VarintEncode(len(data)), data) -class RepeatedFieldHelper(collections.Sequence, object): +class RepeatedFieldHelper(collections.abc.Sequence, object): """A helper for the RDFProto to handle repeated fields. This helper is intended to only be constructed from the RDFProto class. diff --git a/grr/core/grr_response_core/lib/util/precondition.py b/grr/core/grr_response_core/lib/util/precondition.py index e448287a4..17c8a3652 100644 --- a/grr/core/grr_response_core/lib/util/precondition.py +++ b/grr/core/grr_response_core/lib/util/precondition.py @@ -57,7 +57,7 @@ def AssertIterableType(iterable, expected_item_type): message %= iterable raise TypeError(message) - AssertType(iterable, collections.Iterable) + AssertType(iterable, collections.abc.Iterable) for item in iterable: AssertType(item, expected_item_type) diff --git a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py index 42a31882a..1d0800eea 100644 --- a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py +++ b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin.py @@ -135,9 +135,9 @@ def _SendEvents(self, events: List[JsonDict]) -> None: # https://www.elastic.co/guide/en/elasticsearch/reference/7.1/docs-bulk.html if self._token: - headers = {"Authorization": "Basic {}".format(self._token)} + headers = {"Authorization": "Basic {}".format(self._token), "Content-Type": "application/json"} else: - headers = {} + headers = {"Content-Type": "application/json"} index_command = json.Dump({"index": {"_index": self._index}}, indent=None) @@ -146,7 +146,7 @@ def _SendEvents(self, events: List[JsonDict]) -> None: data = "\n".join([ "{}\n{}".format(index_command, json.Dump(event, indent=None)) for event in events - ]) + ]) + "\n" response = requests.post( url=self._url, verify=self._verify_https, data=data, headers=headers) diff --git a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py index 7578f9b30..2871495ea 100644 --- a/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py +++ b/grr/server/grr_response_server/output_plugins/elasticsearch_plugin_test.py @@ -66,12 +66,18 @@ def _CallPlugin(self, plugin_args=None, responses=None, patcher=None): def _ParseEvents(self, patched): request = patched.call_args[KWARGS]['data'] - # Elasticsearch bulk requests are line-deliminated pairs, where the first # line is the index command and the second is the actual document to index - split_requests = [json.Parse(line) for line in request.split('\n')] - update_pairs = [(split_requests[i], split_requests[i + 1]) - for i in range(0, len(split_requests), 2)] + split_requests = [] + split_request = request.split('\n') + for line in split_request: + # Skip terminating newlines - which crashes json.Parse + if not line: + continue + split_requests.append(json.Parse(line)) + update_pairs = [] + for i in range(0, len(split_requests), 2): + update_pairs.append([split_requests[i], split_requests[i + 1]]) return update_pairs @@ -153,6 +159,10 @@ def testReadsConfigurationValuesCorrectly(self): self.assertFalse(mock_post.call_args[KWARGS]['verify']) self.assertEqual(mock_post.call_args[KWARGS]['headers']['Authorization'], 'Basic b') + self.assertIn( + mock_post.call_args[KWARGS]['headers']['Content-Type'], + ('application/json', 'application/x-ndjson') + ) bulk_pairs = self._ParseEvents(mock_post) self.assertEqual(bulk_pairs[0][0]['index']['_index'], 'e') @@ -195,6 +205,15 @@ def testRaisesForHttpError(self): responses=[rdf_client.Process(pid=42)], patcher=mock.patch.object(requests, 'post', post)) + def testPostDataTerminatingNewline(self): + with test_lib.ConfigOverrider({ + 'Elasticsearch.url': 'http://a', + 'Elasticsearch.token': 'b', + }): + mock_post = self._CallPlugin( + plugin_args=elasticsearch_plugin.ElasticsearchOutputPluginArgs(), + responses=[rdf_client.Process(pid=42)]) + self.assertTrue(mock_post.call_args[KWARGS]['data'].endswith('\n')) if __name__ == '__main__': app.run(test_lib.main) diff --git a/grr/server/setup.py b/grr/server/setup.py index 965a7d418..35086f37a 100644 --- a/grr/server/setup.py +++ b/grr/server/setup.py @@ -40,7 +40,7 @@ def make_ui_files(): subprocess.check_call( "npm ci", shell=True, cwd="grr_response_server/gui/static") subprocess.check_call( - "npm run gulp compile", shell=True, cwd="grr_response_server/gui/static") + "npx gulp compile", shell=True, cwd="grr_response_server/gui/static") # Compile UI v2. subprocess.check_call("npm ci", shell=True, cwd="grr_response_server/gui/ui") subprocess.check_call( @@ -194,7 +194,7 @@ def make_release_tree(self, base_dir, files): "portpicker==1.3.1", "prometheus_client==0.8.0", "pyjwt==1.7.1", - "pyOpenSSL==19.1.0", # https://github.com/google/grr/issues/704 + "pyOpenSSL==20.0.0", # https://github.com/google/grr/issues/704 "python-crontab==2.5.1", "python-debian==0.1.37", "Werkzeug==1.0.1", diff --git a/grr/test/setup.py b/grr/test/setup.py index 017e7a498..fbea1887e 100644 --- a/grr/test/setup.py +++ b/grr/test/setup.py @@ -59,7 +59,7 @@ def make_release_tree(self, base_dir, files): license="Apache License, Version 2.0", url="https://github.com/google/grr", install_requires=[ - "absl-py==0.9.0", + "absl-py==1.1.0", "flaky==3.6.1", "pytest==6.2.2", "responses==0.12.1",