diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..87068ad --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# Spark History Server MCP Configuration + +# MCP Server Settings +MCP_PORT=18888 +MCP_DEBUG=false + +# Spark Authentication (Optional) +# SPARK_USERNAME=your_spark_username +# SPARK_PASSWORD=your_spark_password +# SPARK_TOKEN=your_spark_token + +# Example for production: +# SPARK_USERNAME=prod_user +# SPARK_PASSWORD=secure_password_here +# SPARK_TOKEN=jwt_token_here diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..11ad813 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,54 @@ +# ๐Ÿ”„ Pull Request + +## ๐Ÿ“ Description +Brief description of changes and motivation. + +## ๐ŸŽฏ Type of Change + +- [ ] ๐Ÿ› Bug fix (non-breaking change that fixes an issue) +- [ ] โœจ New feature (non-breaking change that adds functionality) +- [ ] ๐Ÿ’ฅ Breaking change (fix or feature that would cause existing functionality to change) +- [ ] ๐Ÿ“– Documentation update +- [ ] ๐Ÿงช Test improvement +- [ ] ๐Ÿ”ง Refactoring (no functional changes) + +## ๐Ÿงช Testing + +- [ ] โœ… All existing tests pass (`uv run pytest`) +- [ ] ๐Ÿ”ฌ Tested with MCP Inspector +- [ ] ๐Ÿ“Š Tested with sample Spark data +- [ ] ๐Ÿš€ Tested with real Spark History Server (if applicable) + +### ๐Ÿ”ฌ Test Commands Run +```bash +# Example: +# uv run pytest test_tools.py -v +# npx @modelcontextprotocol/inspector uv run main.py +``` + +## ๐Ÿ› ๏ธ New Tools Added (if applicable) + +- **Tool Name**: `new_tool_name` +- **Purpose**: What it does +- **Usage**: Example parameters + +## ๐Ÿ“ธ Screenshots (if applicable) + + +## โœ… Checklist +- [ ] ๐Ÿ” Code follows project style guidelines +- [ ] ๐Ÿงช Added tests for new functionality +- [ ] ๐Ÿ“– Updated documentation (README, TESTING.md, etc.) +- [ ] ๐Ÿ”ง Pre-commit hooks pass +- [ ] ๐Ÿ“ Added entry to CHANGELOG.md (if significant change) + +## ๐Ÿ“š Related Issues + +Fixes #(issue number) +Related to #(issue number) + +## ๐Ÿค” Additional Context + + +--- +**๐ŸŽ‰ Thank you for contributing!** Your effort helps make Spark monitoring more intelligent. diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8be62ab..29e902d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -21,4 +21,4 @@ jobs: enable-cache: true - name: Run pre-commit - run: uv run pre-commit run --all-files --show-diff-on-failure \ No newline at end of file + run: uv run pre-commit run --all-files --show-diff-on-failure diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..00b1207 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,139 @@ +name: CI + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main ] + +jobs: + build: + name: Code Quality Checks + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v2 + + - name: Set up Python + run: uv python install 3.12 + + - name: Install dependencies + run: uv sync --group dev + + - name: Install pre-commit + run: uv add --group dev pre-commit + + - name: Run pre-commit + run: uv run pre-commit run --all-files --show-diff-on-failure + + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.12"] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v2 + + - name: Set up Python ${{ matrix.python-version }} + run: uv python install ${{ matrix.python-version }} + + - name: Install dependencies + run: uv sync --group dev + + - name: Lint with ruff + run: uv run ruff check . + + # TODO: Re-enable mypy after fixing type annotations + # - name: Type check with mypy + # run: uv run mypy *.py --ignore-missing-imports + + - name: Test with pytest + run: uv run pytest --cov=. --cov-report=xml --cov-report=term-missing + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + if: success() + + integration: + runs-on: ubuntu-latest + needs: test + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v2 + + - name: Set up Python + run: uv python install 3.12 + + - name: Install dependencies + run: uv sync + + - name: Setup test configuration + run: | + # Ensure config.yaml exists and is properly configured for CI + if [ ! -f config.yaml ]; then + echo "Creating default config.yaml for CI" + cat > config.yaml << EOF + servers: + default: + default: true + url: "http://localhost:18080" + EOF + fi + + - name: Verify test data + run: | + echo "Verifying test data structure..." + ls -la examples/basic/ + ls -la examples/basic/events/ + cat examples/basic/history-server.conf + + - name: Start Spark History Server + run: | + echo "Starting Spark History Server with Docker..." + docker run -d \ + --name spark-history-server \ + -v $(pwd)/examples/basic:/mnt/data \ + -p 18080:18080 \ + docker.io/apache/spark:3.5.5 \ + /opt/java/openjdk/bin/java \ + -cp '/opt/spark/conf:/opt/spark/jars/*' \ + -Xmx1g \ + org.apache.spark.deploy.history.HistoryServer \ + --properties-file /mnt/data/history-server.conf + + - name: Wait for Spark History Server + run: | + timeout 60 bash -c 'until curl -f http://localhost:18080; do sleep 2; done' + + - name: Test MCP Server startup + run: | + # Test import structure + uv run python -c "import app; print('โœ“ App imports successfully')" + uv run python -c "import main; print('โœ“ Main imports successfully')" + + # Test MCP server can start (brief startup test) + timeout 10 uv run python main.py & + SERVER_PID=$! + sleep 5 + kill $SERVER_PID 2>/dev/null || true + echo "โœ“ MCP Server startup test completed" + + - name: Cleanup + if: always() + run: | + echo "Cleaning up Docker containers..." + docker stop spark-history-server 2>/dev/null || true + docker rm spark-history-server 2>/dev/null || true diff --git a/.gitignore b/.gitignore index 302ff1a..765cbd6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,272 @@ -# Python-generated files +# Python __pycache__/ -*.py[oc] +*.py[cod] +*$py.class + +# Security and code quality reports +bandit-report.json +.mypy_cache/ +*.so +.Python build/ +develop-eggs/ dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ wheels/ -*.egg-info +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST -# Virtual environments +# Virtual Environments +.env .venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +.virtualenv + +# PyCharm +.idea/ + +# VS Code +.vscode/ +*.code-workspace + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +Pipfile.lock + +# poetry +poetry.lock + +# PEP 582 +__pypackages__/ + +# Celery +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.env.local +.env.development.local +.env.test.local +.env.production.local + +# mypy +.mypy_cache/ +.dmypy.json +dmyj.son + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# UV +.uv/ + +# macOS +.DS_Store +.AppleDouble +.LSOverride +Icon? +._* +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# Windows +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db +*.tmp +*.temp +Desktop.ini +$RECYCLE.BIN/ +*.cab +*.msi +*.msix +*.msm +*.msp +*.lnk + +# Linux +*~ +.fuse_hidden* +.directory +.Trash-* +.nfs* + +# Docker +.dockerignore +Dockerfile.dockerignore + +# Logs +logs/ +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Coverage directory used by tools like istanbul +coverage/ +*.lcov +.nyc_output + +# node_modules (if using Node.js tools) +node_modules/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env.development +.env.test +.env.production + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +public + +# Storybook build outputs +.out +.storybook-out +storybook-static + +# Temporary folders +tmp/ +temp/ + +# Editor backups +*~ +*.swp +*.swo +*# +.#* + +# OS generated files +.DS_Store* +ehthumbs.db +Thumbs.db + +# Test artifacts +.pytest_cache/ +.coverage +htmlcov/ +.tox/ +.nox/ +.cache + +# Spark specific +*.log +spark-warehouse/ +derby.log +metastore_db/ + +# MCP specific +mcp.log +*.mcp + +# Configuration files with secrets (keep template versions) +config.local.yaml +config.prod.yaml +secrets.yaml +.secrets + +# Backup files +*.bak +*.backup +*~ + +# Lock files (except essential ones) +package-lock.json +yarn.lock +# Keep uv.lock for reproducible builds + +# IDE files +*.sublime-project +*.sublime-workspace + +# Temporary test data +test_data/ +temp_events/ +*.tmp.json + +# Aider (AI coding assistant) .aider* diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..998322d --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,17 @@ +# Markdown linting configuration +# Disable problematic rules for our documentation style + +# MD013: Line length - disabled because code blocks can be long +MD013: false + +# MD033: HTML tags - disabled because we use HTML for centering, etc. +MD033: false + +# MD041: First line should be h1 - disabled because we have badges first +MD041: false + +# MD024: Multiple headers with same content - disabled for FAQ sections +MD024: false + +# MD026: Trailing punctuation in header - disabled for emoji headers +MD026: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 60f9a62..3f57155 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,51 @@ repos: -- repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. - rev: v0.12.0 - hooks: - # Run the linter. - - id: ruff-check - types_or: [ python, pyi ] - args: [ --fix ] - # Run the formatter. - - id: ruff-format - types_or: [ python, pyi ] + # Pre-commit hooks for general file hygiene + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + exclude: ^deploy/kubernetes/helm/.*\.yaml$ + - id: check-json + - id: check-toml + - id: check-merge-conflict + - id: check-added-large-files + args: ['--maxkb=1000'] + - id: mixed-line-ending + args: ['--fix=lf'] + + # Python linting and formatting with Ruff + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.0 + hooks: + # Run the linter + - id: ruff-check + types_or: [python, pyi] + args: [--fix] + # Run the formatter + - id: ruff-format + types_or: [python, pyi] + + # Type checking with mypy (temporarily disabled - TODO: fix type annotations) + # - repo: https://github.com/pre-commit/mirrors-mypy + # rev: v1.11.2 + # hooks: + # - id: mypy + # additional_dependencies: [types-PyYAML, types-requests] + # args: [--ignore-missing-imports] + + # Security scanning + - repo: https://github.com/PyCQA/bandit + rev: 1.7.10 + hooks: + - id: bandit + args: [--recursive, --format, json, --output, bandit-report.json] + exclude: ^tests/ + + # Documentation and markdown + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: v0.42.0 + hooks: + - id: markdownlint + args: [--disable, MD013, MD033, MD041] # Disable line length, HTML, first line rules diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d52d0d6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,237 @@ +# ๐Ÿค Contributing to Spark History Server MCP + +Thank you for your interest in contributing! This guide will help you get started with contributing to the Spark History Server MCP project. + +## ๐Ÿš€ Quick Start for Contributors + +### ๐Ÿ“‹ Prerequisites +- ๐Ÿ Python 3.12+ +- โšก [uv](https://docs.astral.sh/uv/getting-started/installation/) package manager +- ๐Ÿ”ฅ Docker (for local testing with Spark History Server) +- ๐Ÿ“ฆ Node.js (for MCP Inspector testing) + +### ๐Ÿ› ๏ธ Development Setup + +1. **๐Ÿด Fork and clone the repository** +```bash +git clone https://github.com/YOUR_USERNAME/spark-history-server-mcp.git +cd spark-history-server-mcp +``` + +2. **๐Ÿ“ฆ Install dependencies** +```bash +uv sync --group dev +``` + +3. **๐Ÿ”ง Install pre-commit hooks** +```bash +uv run pre-commit install +``` + +4. **๐Ÿงช Run tests to verify setup** +```bash +uv run pytest +``` + +## ๐Ÿงช Testing Your Changes + +### ๐Ÿ”ฌ Local Testing with MCP Inspector +```bash +# Terminal 1: Start Spark History Server with sample data +./start_local_spark_history.sh + +# Terminal 2: Test your changes +npx @modelcontextprotocol/inspector uv run main.py +# Opens browser at http://localhost:6274 for interactive testing +``` + +### โœ… Run Full Test Suite +```bash +# Run all tests +uv run pytest + +# Run tests with coverage +uv run pytest --cov=. --cov-report=html + +# Run specific test file +uv run pytest test_tools.py -v +``` + +### ๐Ÿ” Code Quality Checks +```bash +# Lint and format (runs automatically on commit) +uv run ruff check --fix +uv run ruff format + +# Type checking +uv run mypy . + +# Security scanning +uv run bandit -r . -f json -o bandit-report.json +``` + +## ๐Ÿ“ Contribution Guidelines + +### ๐ŸŽฏ Areas for Contribution + +#### ๐Ÿ”ง High Priority +- **New MCP Tools**: Additional Spark analysis tools +- **Performance Improvements**: Optimize API calls and data processing +- **Error Handling**: Better error messages and recovery +- **Documentation**: Examples, tutorials, and guides + +#### ๐Ÿ“Š Medium Priority +- **Testing**: More comprehensive test coverage +- **Monitoring**: Metrics and observability features +- **Configuration**: More flexible configuration options +- **CI/CD**: GitHub Actions improvements + +#### ๐Ÿ’ก Ideas Welcome +- **AI Agent Examples**: New integration patterns +- **Deployment**: Additional deployment methods +- **Analytics**: Advanced Spark job analysis tools + +### ๐Ÿ”€ Pull Request Process + +1. **๐ŸŒฟ Create a feature branch** +```bash +git checkout -b feature/your-new-feature +git checkout -b fix/bug-description +git checkout -b docs/improve-readme +``` + +2. **๐Ÿ’ป Make your changes** +- Follow existing code style and patterns +- Add tests for new functionality +- Update documentation as needed +- Ensure all pre-commit hooks pass + +3. **โœ… Test thoroughly** +```bash +# Run full test suite +uv run pytest + +# Test with MCP Inspector +npx @modelcontextprotocol/inspector uv run main.py + +# Test with real Spark data if possible +``` + +4. **๐Ÿ“ค Submit pull request** +- Use descriptive commit messages +- Reference any related issues +- Include screenshots for UI changes +- Update CHANGELOG.md if applicable + +### ๐Ÿ’ป Code Style + +We use **Ruff** for linting and formatting (automatically enforced by pre-commit): + +- **Line length**: 88 characters +- **Target**: Python 3.12+ +- **Import sorting**: Automatic with Ruff +- **Type hints**: Encouraged but not required for all functions + +### ๐Ÿงช Adding New MCP Tools + +When adding new tools, follow this pattern: + +```python +@mcp.tool() +def your_new_tool( + spark_id: str, + server: Optional[str] = None, + # other parameters +) -> YourReturnType: + """ + Brief description of what this tool does. + + Args: + spark_id: The Spark application ID + server: Optional server name to use + + Returns: + Description of return value + """ + ctx = mcp.get_context() + client = get_client_or_default(ctx, server) + + # Your implementation here + return client.your_method(spark_id) +``` + +**Don't forget to add tests:** + +```python +@patch("tools.get_client_or_default") +def test_your_new_tool(self, mock_get_client): + """Test your new tool functionality""" + # Setup mocks + mock_client = MagicMock() + mock_client.your_method.return_value = expected_result + mock_get_client.return_value = mock_client + + # Call the tool + result = your_new_tool("spark-app-123") + + # Verify results + self.assertEqual(result, expected_result) + mock_client.your_method.assert_called_once_with("spark-app-123") +``` + +## ๐Ÿ› Reporting Issues + +### ๐Ÿ” Bug Reports +Include: +- **Environment**: Python version, OS, uv version +- **Steps to reproduce**: Clear step-by-step instructions +- **Expected vs actual behavior**: What should happen vs what happens +- **Logs**: Relevant error messages or logs +- **Sample data**: Spark application IDs that reproduce the issue (if possible) + +### ๐Ÿ’ก Feature Requests +Include: +- **Use case**: Why is this feature needed? +- **Proposed solution**: How should it work? +- **Alternatives**: Other approaches considered +- **Examples**: Sample usage or screenshots + +## ๐Ÿ“– Documentation + +### ๐Ÿ“ Types of Documentation +- **README.md**: Main project overview and quick start +- **TESTING.md**: Comprehensive testing guide +- **examples/integrations/**: AI agent integration examples +- **Code comments**: Inline documentation for complex logic + +### ๐ŸŽจ Documentation Style +- Use emojis consistently for visual appeal +- Include code examples for all features +- Provide screenshots for UI elements +- Keep language clear and beginner-friendly + +## ๐ŸŒŸ Recognition + +Contributors are recognized in: +- **GitHub Contributors** section +- **Release notes** for significant contributions +- **Project documentation** for major features + +## ๐Ÿ“ž Getting Help + +- **๐Ÿ’ฌ Discussions**: [GitHub Discussions](https://github.com/DeepDiagnostix-AI/spark-history-server-mcp/discussions) +- **๐Ÿ› Issues**: [GitHub Issues](https://github.com/DeepDiagnostix-AI/spark-history-server-mcp/issues) +- **๐Ÿ“š Documentation**: Check existing docs first + +## ๐Ÿ“œ Code of Conduct + +- **๐Ÿค Be respectful**: Treat everyone with kindness and professionalism +- **๐ŸŽฏ Stay on topic**: Keep discussions relevant to the project +- **๐Ÿง  Be constructive**: Provide helpful feedback and suggestions +- **๐ŸŒ Be inclusive**: Welcome contributors of all backgrounds and skill levels + +--- + +**๐ŸŽ‰ Thank you for contributing to Spark History Server MCP!** + +Your contributions help make Apache Spark monitoring more intelligent and accessible to the community. diff --git a/Dockerfile b/Dockerfile index f22dd28..d5e9e23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,6 @@ RUN groupadd -r app && useradd -r -g app app COPY --from=builder --chown=app:app /app /app WORKDIR /app -WORKDIR /app ENV PATH="/app/.venv/bin:$PATH" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6a23991 --- /dev/null +++ b/LICENSE @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (which shall not include communication that is conspicuously + marked or otherwise designated in writing by the copyright owner + as "Not a Contribution"). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based upon (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and derivative works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control + systems, and issue tracking systems that are managed by, or on behalf + of, the Licensor for the purpose of discussing and improving the Work, + but excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution". + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to use, reproduce, modify, display, perform, + sublicense, and distribute the Work and any Derivative Works thereof. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (a) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (b) You must retain, in the Source form of any Derivative Works that You + distribute, all copyright, trademark, patent, attribution and other + notices from the Source form of the Work, excluding those notices + that do not pertain to any part of the Derivative Works; and + + (c) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained within + such NOTICE file, excluding those notices that do not pertain to + any part of the Derivative Works, in at least one of the following + places: within a NOTICE text file distributed as part of the + Derivative Works; within the Source form or documentation, if + provided along with the Derivative Works; or, within a display + generated by the Derivative Works, if and wherever such third-party + notices normally appear. The contents of the NOTICE file are for + informational purposes only and do not modify the License. You may + add Your own attribution notices within Derivative Works that You + distribute, alongside or as an addendum to the NOTICE text from the + Work, provided that such additional attribution notices cannot be + construed as modifying the License. + + You may add Your own copyright notice to Your modifications and may + provide additional or different license terms and conditions for use, + reproduction, or distribution of Your modifications, or for any such + Derivative Works as a whole, provided Your use, reproduction, and + distribution of the Work otherwise complies with the conditions stated + in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Acceptance and Termination. If, at any time, You expressly assented + to this License, that assent indicates Your acceptance of this License + and all its terms and conditions. If You distribute or communicate + copies of the Work or a Derivative Works to anyone, that action + constitutes Your acceptance of this License and all its terms and + conditions. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. Don't include + the brackets! The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same page as the copyright notice for easier identification within + third-party archives. + + Copyright 2024 Manabu McCloskey, Vara Bonthu + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a6b2a75 --- /dev/null +++ b/Makefile @@ -0,0 +1,65 @@ +.PHONY: install test lint format clean dev setup help + +# Default target +help: + @echo "Available commands:" + @echo " install - Install project dependencies" + @echo " dev - Install development dependencies" + @echo " test - Run tests" + @echo " test-cov - Run tests with coverage" + @echo " lint - Run linting (ruff + mypy)" + @echo " format - Format code (black + ruff fix)" + @echo " clean - Clean temporary files" + @echo " setup - Setup development environment" + @echo " start-spark - Start Spark History Server" + @echo " start-mcp - Start MCP Server" + @echo " start-inspector - Start MCP Inspector" + +install: + uv sync + +dev: install + uv sync --group dev + +test: + uv run pytest + +test-cov: + uv run pytest --cov=. --cov-report=html --cov-report=term-missing + +lint: + @echo "Running ruff..." + uv run ruff check . + @echo "Running mypy..." + uv run mypy *.py --ignore-missing-imports + +format: + @echo "Running black..." + uv run black . + @echo "Running ruff fix..." + uv run ruff check --fix . + +clean: + rm -rf .pytest_cache/ .coverage htmlcov/ dist/ build/ + find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true + find . -type f -name "*.pyc" -delete + +setup: dev + chmod +x *.sh + +# MCP Development shortcuts +start-spark: + @echo "Starting Spark History Server..." + ./start_local_spark_history.sh + +start-mcp: + @echo "Starting MCP Server..." + uv run main.py + +start-inspector: + @echo "Starting MCP Inspector..." + DANGEROUSLY_OMIT_AUTH=true npx @modelcontextprotocol/inspector + +# Validation +validate: lint test + @echo "โœ… All validations passed!" diff --git a/README.md b/README.md index 4a88032..3c41402 100644 --- a/README.md +++ b/README.md @@ -1,112 +1,231 @@ -# Spark History Server MCP Server +# ๐Ÿ”ฅ Spark History Server MCP -Welcome to the Spark History Server MCP Server! This tool bridges Apache Spark's history data with LLM-powered analysis through the Model Context Protocol (MCP). +[![CI](https://github.com/DeepDiagnostix-AI/spark-history-server-mcp/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/DeepDiagnostix-AI/spark-history-server-mcp/actions) +[![Python 3.12+](https://img.shields.io/badge/python-3.12+-blue.svg)](https://www.python.org/downloads/) +[![MCP](https://img.shields.io/badge/MCP-Compatible-green.svg)](https://modelcontextprotocol.io/) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -This MCP server exposes most Apache History Server REST APIs as tools, allowing you to analyze Spark job performance data through natural language interactions. Since Apache Spark History server supports read operations only, this tool is likewise focused on data retrieval and analysis rather than modifications. +> **๐Ÿค– Connect AI agents to Apache Spark History Server for intelligent job analysis and performance monitoring** -In addition to standard REST APIs, we've included enhanced analytics capabilities such as identifying the n slowest jobs, pinpointing bottleneck stages, and generating comprehensive executor metric summaries. +Transform your Spark infrastructure monitoring with AI! This Model Context Protocol (MCP) server enables AI agents to analyze job performance, identify bottlenecks, and provide intelligent insights from your Spark History Server data. -Some use cases for this include: -- Investigating job failure scenarios with natural language queries -- Identifying performance bottlenecks in complex Spark applications -- Optimizing jobs based on historical execution data -- Extracting insights from Spark metrics without writing custom queries +## ๐ŸŽฏ What is This? -This MCP server was tested with Qwen3 32B and 235B, but should work with other LLMs that support the Model Context Protocol. +**Spark History Server MCP** bridges AI agents with your existing Apache Spark infrastructure, enabling: -# Getting started +- ๐Ÿ” **Query job details** through natural language +- ๐Ÿ“Š **Analyze performance metrics** across applications +- ๐Ÿ”„ **Compare multiple jobs** to identify regressions +- ๐Ÿšจ **Investigate failures** with detailed error analysis +- ๐Ÿ“ˆ **Generate insights** from historical execution data -## Prerequisites +## ๐Ÿ—๏ธ Architecture -- Podman or Docker -- [uv](https://docs.astral.sh/uv/getting-started/installation/) - -Python version and dependencies are available in [`.python-version`](./.python-version) and [`pyproject.toml`](./pyproject.toml). +```mermaid +graph TB + A[๐Ÿค– AI Agent/LLM] --> B[๐Ÿ“ก MCP Client] + B --> C[โšก Spark History MCP Server] + C --> D[๐Ÿ”ฅ Your Spark History Server] + D --> E[๐Ÿ“„ Spark Event Logs] + F[๐Ÿ”ง LangChain Agent] --> B + G[๐Ÿ“ฑ Custom AI App] --> B + H[๐Ÿ”ฌ MCP Inspector] --> B +``` -## Usage -### Setting Up Your Environment +**๐Ÿ”— Components:** +- **๐Ÿ”ฅ Spark History Server**: Your existing infrastructure serving Spark event data +- **โšก MCP Server**: This project - provides MCP tools for querying Spark data +- **๐Ÿค– AI Agents**: LangChain, custom agents, or any MCP-compatible client -**Step 1: Prepare the Spark History Server** +## โšก Quick Start -This MCP server requires a running Spark History Server to connect to. For your convenience: -- We've included example Spark event data in the repo -- Location: `examples/basic/events` directory -- These sample events will help you test the setup +### ๐Ÿ“‹ Prerequisites +- ๐Ÿ”ฅ Existing Spark History Server (running and accessible) +- ๐Ÿ Python 3.12+ +- โšก [uv](https://docs.astral.sh/uv/getting-started/installation/) package manager -Run the following command to launch a Spark History Server with our sample data: +### ๐Ÿš€ Setup +```bash +git clone https://github.com/DeepDiagnostix-AI/spark-history-server-mcp.git +cd spark-history-server-mcp +uv sync +``` +### โš™๏ธ Configuration +Edit `config.yaml`: +```yaml +servers: + default: + url: "http://your-spark-history-server:18080" + auth: # optional + username: "user" + password: "pass" +``` +### ๐Ÿ”ฌ Testing with MCP Inspector ```bash -# This command: -# - Uses Docker to run Apache Spark 3.5.5 -# - Mounts your local examples directory -# - Exposes port 18080 for the History Server -# - Configures the server using our example configuration - -docker run -it \ - -v $(pwd)/examples/basic:/mnt/data \ - -p 18080:18080 \ - docker.io/apache/spark:3.5.5 \ - /opt/java/openjdk/bin/java \ - -cp '/opt/spark/conf:/opt/spark/jars/*' \ - -Xmx1g \ - org.apache.spark.deploy.history.HistoryServer \ - --properties-file /mnt/data/history-server.conf +# Start MCP server with Inspector (opens browser automatically) +npx @modelcontextprotocol/inspector uv run main.py ``` -Once running, the History Server will be available at http://localhost:18080 +**๐ŸŒ Test in Browser** - The MCP Inspector opens at http://localhost:6274 for interactive tool testing! + +## ๐Ÿ“ธ Screenshots + +### ๐Ÿ” Get Spark Application +![Get Application](screenshots/get-application.png) + +### โšก Job Performance Comparison +![Job Comparison](screenshots/job-compare.png) +*Compare performance metrics between different Spark jobs* +![alt text](job-compare.png) + + +## ๐Ÿ› ๏ธ Available Tools -**Step 2: Launch the MCP Server** +| ๐Ÿ”ง Tool | ๐Ÿ“ Description | +|---------|----------------| +| `list_applications` | ๐Ÿ“‹ List Spark applications with filtering | +| `get_application_details` | ๐Ÿ“Š Get comprehensive application info | +| `get_application_jobs` | ๐Ÿ”— List jobs within an application | +| `get_job_details` | ๐Ÿ” Get detailed job information | +| `get_stage_details` | โšก Analyze stage-level metrics | +| `get_task_details` | ๐ŸŽฏ Examine individual task performance | +| `get_executor_summary` | ๐Ÿ–ฅ๏ธ Review executor utilization | +| `compare_job_performance` | ๐Ÿ“ˆ Compare multiple jobs | +| `get_application_environment` | โš™๏ธ Review Spark configuration | +| `get_storage_info` | ๐Ÿ’พ Analyze RDD storage usage | +| `get_sql_execution_details` | ๐Ÿ”Ž Deep dive into SQL queries | -With the History Server running, start the MCP server in a new terminal: +## ๐Ÿš€ Production Deployment + +Deploy using Kubernetes with Helm: + +> โš ๏ธ **Work in Progress**: We are still testing and will soon publish the container image and Helm registry to GitHub for easy deployment. ```bash -# Make sure uv is installed before running this command -uv run main.py +# ๐Ÿ“ฆ Deploy with Helm +helm install spark-history-mcp ./deploy/kubernetes/helm/spark-history-mcp/ + +# ๐ŸŽฏ Production configuration +helm install spark-history-mcp ./deploy/kubernetes/helm/spark-history-mcp/ \ + --set replicaCount=3 \ + --set autoscaling.enabled=true \ + --set monitoring.enabled=true ``` - -The MCP server will start on port 18888 by default. -**Step 3: Run the MCP Inspector Tool** +๐Ÿ“š See [`deploy/kubernetes/helm/`](deploy/kubernetes/helm/) for complete deployment manifests and configuration options. -MCP Inspector is a debugging tool that helps you interact with MCP-enabled services: +## ๐Ÿงช Testing & Development +### ๐Ÿ”ฌ Local Development ```bash -# Note: Disabling auth is only recommended for local development -DANGEROUSLY_OMIT_AUTH=true npx @modelcontextprotocol/inspector +# ๐Ÿ”ฅ Start local Spark History Server with sample data +./start_local_spark_history.sh + +# โšก Start MCP server +uv run main.py + +# ๐ŸŒ Test with MCP Inspector +npx @modelcontextprotocol/inspector uv run main.py +``` + +### ๐Ÿ“Š Sample Data +The repository includes real Spark event logs for testing: +- `spark-bcec39f6201b42b9925124595baad260` - โœ… Successful ETL job +- `spark-110be3a8424d4a2789cb88134418217b` - ๐Ÿ”„ Data processing job +- `spark-cc4d115f011443d787f03a71a476a745` - ๐Ÿ“ˆ Multi-stage analytics job + +๐Ÿ“– **Complete testing guide**: **[TESTING.md](TESTING.md)** + +## โš™๏ธ Configuration + +### ๐ŸŒ Multi-server Setup +```yaml +servers: + production: + default: true + url: "http://prod-spark-history:18080" + auth: + username: "user" + password: "pass" + staging: + url: "http://staging-spark-history:18080" ``` -โš ๏ธ Security Note: The above command disables authentication for testing. Never use this setting in production environments. +### ๐Ÿ” Environment Variables +```bash +SPARK_USERNAME=your_username +SPARK_PASSWORD=your_password +SPARK_TOKEN=your_jwt_token +MCP_PORT=18888 +MCP_DEBUG=false +``` + +## ๐Ÿค– AI Agent Integration + +For production AI agent integration, see [`examples/integrations/`](examples/integrations/): + +- ๐Ÿฆ™ [LlamaIndex](examples/integrations/llamaindex.md) - Vector indexing and search +- ๐Ÿ”— [LangGraph](examples/integrations/langgraph.md) - Multi-agent workflows + +๐Ÿงช **For local testing and development, use [TESTING.md](TESTING.md) with MCP Inspector.** + +## ๐ŸŽฏ Example Use Cases + +### ๐Ÿ” Performance Investigation +``` +๐Ÿค– AI Query: "Why is my ETL job running slower than usual?" + +๐Ÿ“Š MCP Actions: +โœ… Analyze application metrics +โœ… Compare with historical performance +โœ… Identify bottleneck stages +โœ… Generate optimization recommendations +``` + +### ๐Ÿšจ Failure Analysis +``` +๐Ÿค– AI Query: "What caused job 42 to fail?" + +๐Ÿ” MCP Actions: +โœ… Examine failed tasks and error messages +โœ… Review executor logs and resource usage +โœ… Identify root cause and suggest fixes +``` + +### ๐Ÿ“ˆ Comparative Analysis +``` +๐Ÿค– AI Query: "Compare today's batch job with yesterday's run" + +๐Ÿ“Š MCP Actions: +โœ… Compare execution times and resource usage +โœ… Identify performance deltas +โœ… Highlight configuration differences +``` -**Step 4: Connect to the Inspector Interface** +## ๐Ÿค Contributing -Open your browser and navigate to: http://localhost:6274 - -Configure the connection: -1. In the Transport Type dropdown, select Streamable HTTP -2. Set the URL field to: http://localhost:18888/mcp -3. Click the Connect button +1. ๐Ÿด Fork the repository +2. ๐ŸŒฟ Create feature branch: `git checkout -b feature/new-tool` +3. ๐Ÿงช Add tests for new functionality +4. โœ… Run tests: `uv run pytest` +5. ๐Ÿ“ค Submit pull request +## ๐Ÿ“„ License -**Step 5: Explore Available Tools** +Apache License 2.0 - see [LICENSE](LICENSE) file for details. -Once connected: -1. Navigate to the Tools tab -2. Click the List Tools button to see available operations -3. You should see a list of tools for interacting with Spark History data - -**Step 6: Try a Sample Query** +--- -Let's test the setup by retrieving application details: +
-1. Select the get_application tool from the list -2. In the application_id field, enter: spark-bcec39f6201b42b9925124595baad260 -3. Click Run Tool +**๐Ÿ”ฅ Connect your Spark infrastructure to AI agents** -You should see a JSON response containing details about the Spark application. +[๐Ÿš€ Get Started](#-quick-start) | [๐Ÿ› ๏ธ View Tools](#%EF%B8%8F-available-tools) | [๐Ÿงช Test Now](TESTING.md) | [๐Ÿค Contribute](#-contributing) -### Configuration Options +*Built by the community, for the community* ๐Ÿ’™ -Configuration is done through a configuration file called [`config.yaml`](./config.yaml). +
diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..fb3d1ed --- /dev/null +++ b/TESTING.md @@ -0,0 +1,185 @@ +# Testing Guide: Spark History Server MCP + +## ๐Ÿงช Quick Test with MCP Inspector (5 minutes) + +### Prerequisites +- Docker must be running (for Spark History Server) +- Node.js installed (for MCP Inspector) +- Run commands from project root directory + +### Setup (2 terminals) + +```bash +# Terminal 1: Start Spark History Server with sample data +./start_local_spark_history.sh + +# Terminal 2: Start MCP server with Inspector +npx @modelcontextprotocol/inspector uv run main.py +# This will open http://localhost:6274 in your browser +``` + +### Alternative: Start MCP Server Separately +```bash +# Terminal 1: Start Spark History Server +./start_local_spark_history.sh + +# Terminal 2: Start MCP Server +uv run main.py + +# Terminal 3: Start MCP Inspector (connects to existing MCP server) +DANGEROUSLY_OMIT_AUTH=true npx @modelcontextprotocol/inspector +``` + +#### Expected Output from Terminal 1: +``` +๐Ÿ“Š Available Test Applications: +=============================== +๐Ÿ“‹ spark-110be3a8424d4a2789cb88134418217b (512K) +๐Ÿ“‹ spark-bcec39f6201b42b9925124595baad260 (104K) +๐Ÿ“‹ spark-cc4d115f011443d787f03a71a476a745 (704K) + +๐Ÿ“ Will be available at: http://localhost:18080 +``` + +### Test Applications Available +Your 3 real Spark applications (all successful): +- `spark-bcec39f6201b42b9925124595baad260` +- `spark-110be3a8424d4a2789cb88134418217b` +- `spark-cc4d115f011443d787f03a71a476a745` + +## ๐ŸŒ Using MCP Inspector + +Once the MCP Inspector opens in your browser (http://localhost:6274), you can: + +1. **View Available Tools** - See all MCP tools in the left sidebar +2. **Test Tools Interactively** - Click any tool to see its parameters +3. **Execute Tools** - Fill in parameters and run tools +4. **View Results** - See structured responses from your Spark History Server + +### Example Tool Tests: + +#### Get Application Details +- **Tool**: `get_application` +- **Parameter**: `spark_id` = `spark-cc4d115f011443d787f03a71a476a745` +- **Expected**: Application info including name, duration, status + +#### List All Applications +- **Tool**: `list_applications` +- **Parameters**: (none required) +- **Expected**: Array of 3 applications + +#### Compare Job Performance +- **Tool**: `compare_job_performance` +- **Parameters**: + - `spark_id1` = `spark-bcec39f6201b42b9925124595baad260` + - `spark_id2` = `spark-110be3a8424d4a2789cb88134418217b` +- **Expected**: Performance comparison metrics + +## ๐Ÿ”ฌ Detailed Test Cases + +### 1. **Basic Connectivity** +```json +Tool: list_applications +Expected: 3 applications returned +``` + +### 2. **Job Environment Comparison** +```json +Tool: compare_job_environments +Parameters: { + "spark_id1": "spark-bcec39f6201b42b9925124595baad260", + "spark_id2": "spark-110be3a8424d4a2789cb88134418217b" +} +Expected: Configuration differences including: +- Runtime comparison (Java/Scala versions) +- Spark property differences +- System property differences +``` + +### 3. **Performance Comparison** +```json +Tool: compare_job_performance +Parameters: { + "spark_id1": "spark-bcec39f6201b42b9925124595baad260", + "spark_id2": "spark-cc4d115f011443d787f03a71a476a745" +} +Expected: Performance metrics including: +- Resource allocation comparison +- Executor metrics comparison +- Job performance ratios +``` + +### 4. **Bottleneck Analysis** +```json +Tool: get_job_bottlenecks +Parameters: { + "spark_id": "spark-cc4d115f011443d787f03a71a476a745" +} +Expected: Performance analysis with: +- Slowest stages identification +- Resource bottlenecks +- Optimization recommendations +``` + +### 5. **Resource Timeline** +```json +Tool: get_resource_usage_timeline +Parameters: { + "spark_id": "spark-bcec39f6201b42b9925124595baad260" +} +Expected: Timeline showing: +- Executor addition/removal events +- Stage execution timeline +- Resource utilization over time +``` + +## โœ… Success Criteria + +- [ ] All 3 applications visible in list_applications +- [ ] Job comparison tools return detailed analysis +- [ ] Performance comparison shows meaningful differences +- [ ] Bottleneck analysis provides recommendations +- [ ] No errors in any tool execution +- [ ] Upstream stage optimizations work (no overwhelming data) + +## ๐Ÿ› ๏ธ Troubleshooting + +### Common Issues + +#### Script Issues +```bash +# If you get "Docker not running" error: +./start_local_spark_history.sh --dry-run # Check prerequisites + +# If you get "No containers to stop" warning: +# This is normal - just means no previous containers are running + +# To get help with script options: +./start_local_spark_history.sh --help +``` + +#### MCP Server Issues +```bash +# If MCP server fails to start: +# 1. Ensure Spark History Server is running (Terminal 1) +# 2. Check if port 18080 is accessible: curl http://localhost:18080 +# 3. Verify config.yaml exists and has correct server URL +``` + +#### Inspector Connection Issues +```bash +# If MCP Inspector can't connect: +# 1. Ensure MCP server is running (Terminal 2) +# 2. Try restarting the MCP server +# 3. Check for any error messages in Terminal 2 +``` + +## ๐Ÿš€ Ready for Production + +Once all tests pass, the enhanced MCP server with job comparison capabilities is ready for production use! + +### Quick Validation Commands +```bash +# Validate everything is working: +curl http://localhost:18080/api/v1/applications # Should return 3 applications +``` diff --git a/app.py b/app.py index cf6f71f..e322427 100644 --- a/app.py +++ b/app.py @@ -1,14 +1,16 @@ -from typing import Optional -from contextlib import asynccontextmanager +import json +import os from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from dataclasses import dataclass -import json from datetime import datetime +from typing import Optional -from spark_client import SparkRestClient -from config import Config from mcp.server.fastmcp import FastMCP +from config import Config +from spark_client import SparkRestClient + @dataclass class AppContext: @@ -41,8 +43,8 @@ async def app_lifespan(server: FastMCP) -> AsyncIterator[AppContext]: mcp = FastMCP("Spark Events", lifespan=app_lifespan) -mcp.settings.port = 18888 -mcp.settings.debug = True +mcp.settings.port = int(os.getenv("MCP_PORT", "18888")) +mcp.settings.debug = os.getenv("MCP_DEBUG", "false").lower() == "true" # Import tools to register them with MCP import tools # noqa: E402,F401 diff --git a/config.py b/config.py index 29bc164..18337e7 100644 --- a/config.py +++ b/config.py @@ -1,6 +1,7 @@ import os -import yaml from typing import Dict + +import yaml from pydantic import BaseModel, Field @@ -11,6 +12,16 @@ class AuthConfig(BaseModel): password: str = Field(None, alias="password") token: str = Field(None, alias="token") + def __init__(self, **data): + # Support environment variables for sensitive data + if not data.get("username"): + data["username"] = os.getenv("SPARK_USERNAME") + if not data.get("password"): + data["password"] = os.getenv("SPARK_PASSWORD") + if not data.get("token"): + data["token"] = os.getenv("SPARK_TOKEN") + super().__init__(**data) + class ServerConfig(BaseModel): """Server configuration for the Spark server.""" diff --git a/config.yaml b/config.yaml index 9f9f79b..8302956 100644 --- a/config.yaml +++ b/config.yaml @@ -1,10 +1,32 @@ servers: - my-server: + local: default: true url: "http://localhost:18080" - auth: - username: user - password: password - token: token + # Optional authentication (can also use environment variables) + # auth: + # username: "your_username" # or use SPARK_USERNAME env var + # password: "your_password" # or use SPARK_PASSWORD env var + # token: "your_token" # or use SPARK_TOKEN env var -# spark-988f126664e3473ebd004d8de4344137 \ No newline at end of file + # Production server example + # production: + # url: "https://spark-history.company.com:18080" + # auth: + # Use environment variables for production + # username: ${SPARK_USERNAME} + # password: ${SPARK_PASSWORD} + # token: ${SPARK_TOKEN} + + # Staging server example + # staging: + # url: "https://staging-spark-history.company.com:18080" + # auth: + # username: "staging_user" + # token: "staging_token" + +# Environment Variables: +# SPARK_USERNAME - Default username for authentication +# SPARK_PASSWORD - Default password for authentication +# SPARK_TOKEN - Default token for authentication +# MCP_PORT - Port for MCP server (default: 18888) +# MCP_DEBUG - Enable debug mode (default: false) diff --git a/deploy/kubernetes/helm/README.md b/deploy/kubernetes/helm/README.md new file mode 100644 index 0000000..f7156cd --- /dev/null +++ b/deploy/kubernetes/helm/README.md @@ -0,0 +1,441 @@ +# Helm Chart for Spark History Server MCP + +This Helm chart provides a production-ready deployment of the Spark History Server MCP on Kubernetes. + +## ๐Ÿš€ Quick Start + +### Prerequisites +- Kubernetes 1.20+ +- Helm 3.8+ +- Spark History Server running in cluster or accessible via network + +### Install from Local Chart + +```bash +# Install with default values +helm install spark-history-mcp ./deploy/kubernetes/helm/spark-history-mcp/ + +# Install with custom release name and namespace +helm install my-spark-mcp ./deploy/kubernetes/helm/spark-history-mcp/ \ + --namespace spark-history-mcp \ + --create-namespace +``` + +### Install from Repository (Future Release) + +```bash +# Add the helm chart repository (when published) +helm repo add spark-history-mcp https://deepdiagnostix-ai.github.io/spark-history-server-mcp + +# Install the chart +helm install my-spark-mcp spark-history-mcp/spark-history-mcp +``` + +### Install with Custom Values + +```bash +# Create custom values file +cat > my-values.yaml << EOF +replicaCount: 3 + +config: + servers: + production: + default: true + url: "http://spark-history.production:18080" + +auth: + enabled: true + secret: + create: true + username: "spark_user" + password: "secure_password" + +ingress: + enabled: true + hosts: + - host: spark-mcp.company.com + paths: + - path: / + pathType: Prefix + +monitoring: + enabled: true + serviceMonitor: + enabled: true + +autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 +EOF + +# Install with custom values +helm install my-spark-mcp ./deploy/kubernetes/helm/spark-history-mcp/ -f my-values.yaml +``` + +## โš™๏ธ Configuration + +### Common Configuration Examples + +#### 1. Multiple Spark History Servers +```yaml +config: + servers: + production: + default: true + url: "http://prod-spark-history:18080" + staging: + url: "http://staging-spark-history:18080" + development: + url: "http://dev-spark-history:18080" +``` + +#### 2. Authentication Setup +```yaml +auth: + enabled: true + secret: + create: true + username: "spark_admin" + password: "super_secure_password" + token: "jwt_token_here" +``` + +#### 3. High Availability Setup +```yaml +replicaCount: 3 + +autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 20 + targetCPUUtilizationPercentage: 70 + +podDisruptionBudget: + enabled: true + minAvailable: 2 + +affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - spark-history-mcp + topologyKey: kubernetes.io/hostname +``` + +#### 4. Ingress with TLS +```yaml +ingress: + enabled: true + className: "nginx" + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/rate-limit-rps: "100" + hosts: + - host: spark-mcp.company.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: spark-mcp-tls + hosts: + - spark-mcp.company.com +``` + +#### 5. Monitoring and Observability +```yaml +monitoring: + enabled: true + serviceMonitor: + enabled: true + namespace: "monitoring" + interval: 30s + labels: + release: prometheus + +podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "18888" + prometheus.io/path: "/metrics" +``` + +#### 6. Resource Management +```yaml +resources: + limits: + memory: 4Gi + cpu: 2000m + requests: + memory: 1Gi + cpu: 500m + +nodeSelector: + kubernetes.io/arch: amd64 + node-type: compute + +tolerations: + - key: "spark-workload" + operator: "Equal" + value: "true" + effect: "NoSchedule" +``` + +#### 7. Security Configuration +```yaml +podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + +networkPolicy: + enabled: true + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: ai-agents + ports: + - protocol: TCP + port: 18888 +``` + +### Environment-Specific Values + +#### Development Environment (`values-dev.yaml`) +```yaml +replicaCount: 1 + +config: + debug: true + servers: + local: + default: true + url: "http://spark-history-dev:18080" + +resources: + limits: + memory: 1Gi + cpu: 500m + requests: + memory: 256Mi + cpu: 100m + +ingress: + enabled: true + hosts: + - host: spark-mcp-dev.local + paths: + - path: / + pathType: Prefix +``` + +#### Production Environment (`values-prod.yaml`) +```yaml +replicaCount: 5 + +config: + debug: false + servers: + production: + default: true + url: "http://spark-history-prod:18080" + +auth: + enabled: true + secret: + create: false + name: "spark-mcp-prod-auth" + +autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 20 + +monitoring: + enabled: true + serviceMonitor: + enabled: true + namespace: "monitoring" + +networkPolicy: + enabled: true + +podDisruptionBudget: + enabled: true + minAvailable: 2 +``` + +## ๐Ÿ“Š Monitoring + +### Prometheus Integration + +When monitoring is enabled, the chart creates: +- ServiceMonitor for Prometheus scraping +- Grafana dashboard ConfigMaps +- Alert rules for common issues + +```bash +# Install with monitoring +helm install my-spark-mcp ./spark-history-mcp/ \ + --set monitoring.enabled=true \ + --set monitoring.serviceMonitor.enabled=true +``` + +### Custom Metrics + +Add custom environment variables for metrics: + +```yaml +env: + - name: ENABLE_CUSTOM_METRICS + value: "true" + - name: METRICS_INTERVAL + value: "30" +``` + +## ๐Ÿ”ง Troubleshooting + +### Common Issues + +#### 1. Pod Not Starting +```bash +# Check pod status +kubectl describe pod -l app.kubernetes.io/name=spark-history-mcp + +# Check logs +kubectl logs -l app.kubernetes.io/name=spark-history-mcp +``` + +#### 2. Configuration Issues +```bash +# Check rendered templates +helm template my-spark-mcp ./spark-history-mcp/ -f my-values.yaml + +# Verify ConfigMap +kubectl get configmap -l app.kubernetes.io/name=spark-history-mcp -o yaml +``` + +#### 3. Connectivity Issues +```bash +# Test service connectivity +kubectl run test-pod --rm -i --tty --image=curlimages/curl -- sh +curl http://spark-history-mcp:18888/health +``` + +### Debug Mode + +Enable debug mode for troubleshooting: + +```yaml +config: + debug: true + +# Add debug sidecar +sidecars: + - name: debug + image: busybox + command: ["sleep", "3600"] +``` + +## ๐Ÿ”„ Upgrades + +### Upgrade Chart + +```bash +# Upgrade to new version +helm upgrade my-spark-mcp ./spark-history-mcp/ -f my-values.yaml + +# Rollback if needed +helm rollback my-spark-mcp 1 +``` + +### Migration Guide + +When upgrading from v0.0.x to v0.1.x: + +1. **Backup configuration**: +```bash +kubectl get configmap -l app.kubernetes.io/name=spark-history-mcp -o yaml > backup-config.yaml +``` + +2. **Update values file** according to new schema +3. **Perform rolling upgrade**: +```bash +helm upgrade my-spark-mcp ./spark-history-mcp/ -f updated-values.yaml +``` + +## ๐Ÿงช Testing + +### Validate Installation + +```bash +# Run Helm tests +helm test my-spark-mcp + +# Manual validation +kubectl run test-mcp --rm -i --tty --image=curlimages/curl -- sh +curl -X POST http://spark-history-mcp:18888/tools \ + -H "Content-Type: application/json" \ + -d '{"tool": "list_applications", "parameters": {}}' +``` + +### Load Testing + +```yaml +# Add load testing job +apiVersion: batch/v1 +kind: Job +metadata: + name: spark-mcp-load-test +spec: + template: + spec: + containers: + - name: load-test + image: loadimpact/k6 + command: ["k6", "run", "/scripts/load-test.js"] + volumeMounts: + - name: scripts + mountPath: /scripts + volumes: + - name: scripts + configMap: + name: load-test-scripts +``` + +## ๐Ÿ“š Additional Resources + +- [Kubernetes Documentation](../README.md) - Detailed K8s deployment guide +- [Values Reference](values.yaml) - Complete values documentation +- [Templates](templates/) - Kubernetes manifest templates +- [Examples](examples/) - Real-world configuration examples + +## ๐Ÿค Contributing + +Contributions to the Helm chart are welcome: + +- Chart improvements and new features +- Documentation updates +- Testing enhancements +- Bug fixes + +See the main project [Contributing Guide](../../../README.md#-contributing) for details. diff --git a/deploy/kubernetes/helm/spark-history-mcp/Chart.yaml b/deploy/kubernetes/helm/spark-history-mcp/Chart.yaml new file mode 100644 index 0000000..bfd181e --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +name: spark-history-mcp +description: A Helm chart for Spark History Server MCP +type: application +version: 0.1.0 +appVersion: "0.1.0" +keywords: + - spark + - mcp + - analytics + - performance + - history-server +home: https://github.com/DeepDiagnostix-AI/spark-history-server-mcp +sources: + - https://github.com/DeepDiagnostix-AI/spark-history-server-mcp +maintainers: + - name: Manabu McCloskey + email: Manabu.McCloskey@gmail.com + - name: Vara Bonthu + email: vara.bonthu@gmail.com +annotations: + category: Analytics diff --git a/deploy/kubernetes/helm/spark-history-mcp/templates/NOTES.txt b/deploy/kubernetes/helm/spark-history-mcp/templates/NOTES.txt new file mode 100644 index 0000000..d3bf6bb --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/templates/NOTES.txt @@ -0,0 +1,47 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "spark-history-mcp.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "spark-history-mcp.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "spark-history-mcp.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "spark-history-mcp.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} + +2. Test the MCP server: + kubectl run test-mcp --rm -i --tty --image=curlimages/curl -- sh + # Inside the test pod: + curl -X POST http://{{ include "spark-history-mcp.fullname" . }}:{{ .Values.service.port }}/tools \ + -H "Content-Type: application/json" \ + -d '{"tool": "list_applications", "parameters": {}}' + +3. Check the status: + helm status {{ .Release.Name }} + kubectl get pods -l app.kubernetes.io/name={{ include "spark-history-mcp.name" . }} + +{{- if .Values.monitoring.enabled }} +4. Access monitoring: + - Prometheus metrics: http://{{ include "spark-history-mcp.fullname" . }}:{{ .Values.service.port }}/metrics +{{- end }} + +{{- if not .Values.config.servers }} +โš ๏ธ WARNING: No Spark History Servers configured. Please update your values.yaml to include: + config: + servers: + default: + default: true + url: "http://your-spark-history-server:18080" +{{- end }} diff --git a/deploy/kubernetes/helm/spark-history-mcp/templates/_helpers.tpl b/deploy/kubernetes/helm/spark-history-mcp/templates/_helpers.tpl new file mode 100644 index 0000000..9e442c7 --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/templates/_helpers.tpl @@ -0,0 +1,121 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "spark-history-mcp.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "spark-history-mcp.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "spark-history-mcp.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "spark-history-mcp.labels" -}} +helm.sh/chart: {{ include "spark-history-mcp.chart" . }} +{{ include "spark-history-mcp.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "spark-history-mcp.selectorLabels" -}} +app.kubernetes.io/name: {{ include "spark-history-mcp.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "spark-history-mcp.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "spark-history-mcp.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create the name of the config map +*/}} +{{- define "spark-history-mcp.configMapName" -}} +{{- printf "%s-config" (include "spark-history-mcp.fullname" .) }} +{{- end }} + +{{/* +Create the name of the secret +*/}} +{{- define "spark-history-mcp.secretName" -}} +{{- if .Values.auth.secret.create }} +{{- printf "%s-auth" (include "spark-history-mcp.fullname" .) }} +{{- else }} +{{- default (printf "%s-auth" (include "spark-history-mcp.fullname" .)) .Values.auth.secret.name }} +{{- end }} +{{- end }} + +{{/* +Create image name +*/}} +{{- define "spark-history-mcp.image" -}} +{{- printf "%s:%s" .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) }} +{{- end }} + +{{/* +Create environment variables +*/}} +{{- define "spark-history-mcp.env" -}} +- name: MCP_PORT + value: {{ .Values.config.port | quote }} +- name: MCP_DEBUG + value: {{ .Values.config.debug | quote }} +{{- if .Values.auth.enabled }} +- name: SPARK_USERNAME + valueFrom: + secretKeyRef: + name: {{ include "spark-history-mcp.secretName" . }} + key: username + optional: true +- name: SPARK_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "spark-history-mcp.secretName" . }} + key: password + optional: true +- name: SPARK_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "spark-history-mcp.secretName" . }} + key: token + optional: true +{{- end }} +{{- range .Values.env }} +- name: {{ .name }} + value: {{ .value | quote }} +{{- end }} +{{- end }} diff --git a/deploy/kubernetes/helm/spark-history-mcp/templates/configmap.yaml b/deploy/kubernetes/helm/spark-history-mcp/templates/configmap.yaml new file mode 100644 index 0000000..1e8bf67 --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/templates/configmap.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "spark-history-mcp.configMapName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-history-mcp.labels" . | nindent 4 }} +data: + config.yaml: | + servers: + {{- range $name, $server := .Values.config.servers }} + {{ $name }}: + {{- if $server.default }} + default: {{ $server.default }} + {{- end }} + url: {{ $server.url | quote }} + {{- if $server.auth }} + auth: + {{- toYaml $server.auth | nindent 10 }} + {{- end }} + {{- end }} diff --git a/deploy/kubernetes/helm/spark-history-mcp/templates/deployment.yaml b/deploy/kubernetes/helm/spark-history-mcp/templates/deployment.yaml new file mode 100644 index 0000000..f7db6a9 --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/templates/deployment.yaml @@ -0,0 +1,115 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "spark-history-mcp.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-history-mcp.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "spark-history-mcp.selectorLabels" . | nindent 6 }} + {{- with .Values.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.auth.enabled }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "spark-history-mcp.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "spark-history-mcp.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: {{ include "spark-history-mcp.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.config.port }} + protocol: TCP + env: + {{- include "spark-history-mcp.env" . | nindent 12 }} + {{- with .Values.envFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /app/config.yaml + subPath: config.yaml + readOnly: true + {{- if .Values.persistence.enabled }} + - name: data + mountPath: {{ .Values.persistence.mountPath }} + {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.startupProbe }} + startupProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ include "spark-history-mcp.configMapName" . }} + defaultMode: 0644 + {{- if .Values.persistence.enabled }} + - name: data + persistentVolumeClaim: + claimName: {{ include "spark-history-mcp.fullname" . }} + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/kubernetes/helm/spark-history-mcp/templates/hpa.yaml b/deploy/kubernetes/helm/spark-history-mcp/templates/hpa.yaml new file mode 100644 index 0000000..fd22de8 --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/templates/hpa.yaml @@ -0,0 +1,33 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "spark-history-mcp.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-history-mcp.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "spark-history-mcp.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/deploy/kubernetes/helm/spark-history-mcp/templates/ingress.yaml b/deploy/kubernetes/helm/spark-history-mcp/templates/ingress.yaml new file mode 100644 index 0000000..ecd71be --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/templates/ingress.yaml @@ -0,0 +1,60 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "spark-history-mcp.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class")) }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-history-mcp.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/kubernetes/helm/spark-history-mcp/templates/pdb.yaml b/deploy/kubernetes/helm/spark-history-mcp/templates/pdb.yaml new file mode 100644 index 0000000..3c8db08 --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/templates/pdb.yaml @@ -0,0 +1,19 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "spark-history-mcp.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-history-mcp.labels" . | nindent 4 }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "spark-history-mcp.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/kubernetes/helm/spark-history-mcp/templates/pvc.yaml b/deploy/kubernetes/helm/spark-history-mcp/templates/pvc.yaml new file mode 100644 index 0000000..73274d2 --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/templates/pvc.yaml @@ -0,0 +1,22 @@ +{{- if .Values.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "spark-history-mcp.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-history-mcp.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- if .Values.persistence.storageClass }} + {{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: {{ .Values.persistence.storageClass }} + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/kubernetes/helm/spark-history-mcp/templates/secret.yaml b/deploy/kubernetes/helm/spark-history-mcp/templates/secret.yaml new file mode 100644 index 0000000..1b2e5a9 --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/templates/secret.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.auth.enabled .Values.auth.secret.create }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "spark-history-mcp.secretName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-history-mcp.labels" . | nindent 4 }} +type: Opaque +stringData: + username: {{ .Values.auth.secret.username | default "" | quote }} + password: {{ .Values.auth.secret.password | default "" | quote }} + token: {{ .Values.auth.secret.token | default "" | quote }} +{{- end }} diff --git a/deploy/kubernetes/helm/spark-history-mcp/templates/service.yaml b/deploy/kubernetes/helm/spark-history-mcp/templates/service.yaml new file mode 100644 index 0000000..4f8a985 --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/templates/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "spark-history-mcp.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-history-mcp.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: http + selector: + {{- include "spark-history-mcp.selectorLabels" . | nindent 4 }} diff --git a/deploy/kubernetes/helm/spark-history-mcp/templates/serviceaccount.yaml b/deploy/kubernetes/helm/spark-history-mcp/templates/serviceaccount.yaml new file mode 100644 index 0000000..8264960 --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/templates/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "spark-history-mcp.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-history-mcp.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: true +{{- end }} diff --git a/deploy/kubernetes/helm/spark-history-mcp/templates/servicemonitor.yaml b/deploy/kubernetes/helm/spark-history-mcp/templates/servicemonitor.yaml new file mode 100644 index 0000000..0bc6df3 --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/templates/servicemonitor.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.monitoring.enabled .Values.monitoring.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "spark-history-mcp.fullname" . }} + namespace: {{ .Values.monitoring.serviceMonitor.namespace | default .Release.Namespace }} + labels: + {{- include "spark-history-mcp.labels" . | nindent 4 }} + {{- with .Values.monitoring.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.monitoring.serviceMonitor.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "spark-history-mcp.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + endpoints: + - port: http + interval: {{ .Values.monitoring.serviceMonitor.interval }} + path: {{ .Values.monitoring.serviceMonitor.path }} + scheme: http +{{- end }} diff --git a/deploy/kubernetes/helm/spark-history-mcp/values.yaml b/deploy/kubernetes/helm/spark-history-mcp/values.yaml new file mode 100644 index 0000000..076539c --- /dev/null +++ b/deploy/kubernetes/helm/spark-history-mcp/values.yaml @@ -0,0 +1,208 @@ +# Default values for spark-history-mcp +# This is a YAML-formatted file. + +# Image configuration +image: + repository: ghcr.io/deepdiagnostix-ai/spark-history-mcp + pullPolicy: IfNotPresent + tag: "v0.1.0" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +# Pod replica count (used when autoscaling is disabled) +replicaCount: 2 + +# Service account configuration +serviceAccount: + create: true + annotations: {} + name: "" + +# Pod security context +podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + +# Container security context +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + +# Service configuration +service: + type: ClusterIP + port: 18888 + targetPort: http + annotations: {} + +# Ingress configuration +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # cert-manager.io/cluster-issuer: "letsencrypt-prod" + hosts: + - host: spark-mcp.local + paths: + - path: / + pathType: Prefix + tls: [] + # - secretName: spark-mcp-tls + # hosts: + # - spark-mcp.local + +# Resource configuration +resources: + limits: + memory: 2Gi + cpu: 1000m + requests: + memory: 512Mi + cpu: 250m + +# Autoscaling configuration +autoscaling: + enabled: false + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + +# MCP server configuration +config: + port: 18888 + debug: false + + # Spark History Server configuration + servers: + default: + default: true + url: "http://spark-history-server:18080" + # Example: Additional servers + # staging: + # url: "http://staging-spark-history:18080" + # production: + # url: "http://prod-spark-history:18080" + +# Authentication configuration +auth: + enabled: false + secret: + create: false + name: "" + username: "" + password: "" + token: "" + +# Environment variables +env: [] + # - name: CUSTOM_VAR + # value: "custom_value" + +# Environment variables from secrets/configmaps +envFrom: [] + +# Health checks +livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + +startupProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 12 + +# Node selector +nodeSelector: {} + +# Tolerations +tolerations: [] + +# Affinity +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - spark-history-mcp + topologyKey: kubernetes.io/hostname + +# Pod annotations +podAnnotations: {} + +# Pod labels +podLabels: {} + +# Monitoring configuration +monitoring: + enabled: false + serviceMonitor: + enabled: false + namespace: "" + interval: 30s + path: /metrics + labels: {} + annotations: {} + +# Persistence (for logs, if needed) +persistence: + enabled: false + storageClass: "" + accessMode: ReadWriteOnce + size: 1Gi + mountPath: /app/logs + +# Pod disruption budget +podDisruptionBudget: + enabled: false + minAvailable: 1 + +# Extra volumes +extraVolumes: [] + +# Extra volume mounts +extraVolumeMounts: [] + +# Init containers +initContainers: [] + +# Sidecar containers +sidecars: [] + +# Deployment strategy +strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 1 diff --git a/examples/basic/README.md b/examples/basic/README.md index 7f58359..e3eac55 100644 --- a/examples/basic/README.md +++ b/examples/basic/README.md @@ -1 +1,53 @@ -podman run -it -v $(pwd)/examples/basic:/mnt/data -p 18080:18080 spark:3.5.5 /opt/java/openjdk/bin/java -cp '/opt/spark/conf:/opt/spark/jars/*' -Xmx1g org.apache.spark.deploy.history.HistoryServer --properties-file /mnt/data/history-server.conf \ No newline at end of file +# Spark History Server Test Setup + +## Real Spark Event Data + +This directory contains real Spark application event logs for testing the MCP server functionality. All applications completed successfully and are ideal for testing job comparison features. + +### Test Applications + +- **`eventlog_v2_spark-bcec39f6201b42b9925124595baad260/`** + - **Status**: โœ… Completed Successfully + - **Use Case**: Basic functionality testing and job comparison baseline + +- **`eventlog_v2_spark-110be3a8424d4a2789cb88134418217b/`** + - **Status**: โœ… Completed Successfully + - **Use Case**: Job comparison testing (compare with bcec39f application) + +- **`eventlog_v2_spark-cc4d115f011443d787f03a71a476a745/`** + - **Status**: โœ… Completed Successfully + - **Use Case**: Performance analysis and multi-application comparison + +### Starting the History Server + +Run the Spark History Server with the real event data: + +```bash +podman run -it -v $(pwd)/examples/basic:/mnt/data -p 18080:18080 spark:3.5.5 \ + /opt/java/openjdk/bin/java -cp '/opt/spark/conf:/opt/spark/jars/*' -Xmx1g \ + org.apache.spark.deploy.history.HistoryServer \ + --properties-file /mnt/data/history-server.conf +``` + +### Testing Scenarios + +1. **Basic Functionality**: Use any application for quick MCP tool validation + ```bash + # Test with: spark-bcec39f6201b42b9925124595baad260 + ``` + +2. **Job Comparison**: Compare configurations and performance between applications + ```bash + # Compare: spark-bcec39f6201b42b9925124595baad260 vs spark-110be3a8424d4a2789cb88134418217b + ``` + +3. **Multi-Application Analysis**: Test with all 3 applications for comprehensive analysis + ```bash + # All apps: bcec39f, 110be3a, cc4d115 + ``` + +### Accessing the History Server + +- Web UI: http://localhost:18080 +- REST API: http://localhost:18080/api/v1/applications +- MCP Server: Connect to your MCP server pointing to http://localhost:18080 diff --git a/examples/basic/events/eventlog_v2_spark-110be3a8424d4a2789cb88134418217b/appstatus_spark-110be3a8424d4a2789cb88134418217b b/examples/basic/events/eventlog_v2_spark-110be3a8424d4a2789cb88134418217b/appstatus_spark-110be3a8424d4a2789cb88134418217b new file mode 100644 index 0000000..e69de29 diff --git a/examples/basic/events/eventlog_v2_spark-110be3a8424d4a2789cb88134418217b/events_1_spark-110be3a8424d4a2789cb88134418217b b/examples/basic/events/eventlog_v2_spark-110be3a8424d4a2789cb88134418217b/events_1_spark-110be3a8424d4a2789cb88134418217b new file mode 100644 index 0000000..1b81567 --- /dev/null +++ b/examples/basic/events/eventlog_v2_spark-110be3a8424d4a2789cb88134418217b/events_1_spark-110be3a8424d4a2789cb88134418217b @@ -0,0 +1,132 @@ +{"Event":"SparkListenerLogStart","Spark Version":"3.5.3"} +{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"memoryOverhead":{"Resource Name":"memoryOverhead","Amount":4096,"Discovery Script":"","Vendor":""},"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":4096,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"taxi-trip-3a9e8397aec45f6f-driver-svc.spark-team-a.svc","Port":7079},"Maximum Memory":2388236697,"Timestamp":1750983887336,"Maximum Onheap Memory":2388236697,"Maximum Offheap Memory":0} +{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/opt/java/openjdk","Java Version":"17.0.12 (Eclipse Adoptium)","Scala Version":"version 2.12.18"},"Spark Properties":{"spark.speculation":"false","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.kubernetes.submission.waitAppCompletion":"false","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.host":"taxi-trip-3a9e8397aec45f6f-driver-svc.spark-team-a.svc","spark.serializer.objectStreamReset":"100","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.kubernetes.namespace":"spark-team-a","spark.eventLog.enabled":"true","spark.executor.memoryOverhead":"4g","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.122.12","spark.driver.port":"7078","spark.rdd.compress":"True","spark.kubernetes.executor.label.version":"3.5.3","spark.driver.blockManager.port":"7079","spark.hadoop.fs.s3a.path.style.access":"true","spark.app.name":"NewYorkTaxiData_2025_06_27_00_24_44","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.network.timeout":"2400","spark.hadoop.fs.s3a.connection.maximum":"200","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.scheduler.mode":"FIFO","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.memory":"4g","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.executor.instances":"4","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.submit.pyFiles":"","spark.app.submitTime":"1750983883971","spark.driver.memoryOverhead":"4g","spark.kubernetes.memoryOverheadFactor":"0.4","spark.driver.bindAddress":"100.64.122.12","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.kubernetes.resource.type":"python","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.app.startTime":"1750983884577","spark.executor.id":"driver","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.limit.cores":"3400m","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.driver.cores":"1","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.driver.limit.cores":"1200m","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.executor.label.app":"taxi-trip","spark.ui.prometheus.enabled":"true","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.driver.label.queue":"root.test","spark.executor.memory":"4g","spark.local.dir":"/data","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.cores":"1","spark.executor.processTreeMetrics.enabled":"true","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.hadoop.fs.s3a.readahead.range":"256K","spark.app.id":"spark-110be3a8424d4a2789cb88134418217b","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.rolling.enabled":"true","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"200","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.viewfs.overload.scheme.target.o3fs.impl":"org.apache.hadoop.fs.ozone.OzoneFileSystem","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","yarn.resourcemanager.application-tag-based-placement.enable":"false","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min":"3600","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","yarn.nodemanager.node-attributes.resync-interval-ms":"120000","yarn.nodemanager.container-log-monitor.interval-ms":"60000","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","fs.viewfs.overload.scheme.target.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","mapreduce.job.acl-view-job":" ","fs.s3a.retry.limit":"7","mapreduce.jobhistory.loadedjobs.cache.size":"5","mapreduce.outputcommitter.factory.scheme.abfs":"org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory","yarn.router.interceptor.user-thread-pool.allow-core-thread-time-out":"false","yarn.log-aggregation.enable-local-cleanup":"true","fs.viewfs.overload.scheme.target.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.shuffle.pathcache.expire-after-access-minutes":"5","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes":"runc","fs.viewfs.overload.scheme.target.http.impl":"org.apache.hadoop.fs.http.HttpFileSystem","yarn.nodemanager.least-load-policy-selector.fail-on-error":"true","yarn.resourcemanager.nodemanagers.heartbeat-interval-slowdown-factor":"1.0","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.router.submit.interval.time":"10ms","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","yarn.router.interceptor.user-thread-pool.keep-alive-time":"30s","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","fs.s3a.connection.ttl":"5m","yarn.router.asc-interceptor-max-size":"1MB","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","yarn.nodemanager.opportunistic-containers-queue-policy":"BY_QUEUE_LEN","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.federation.gpg.policy.generator.load-based.pending.maximum":"1000","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.nodemanager.aux-services.manifest.enabled":"false","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","fs.s3a.select.input.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","yarn.dispatcher.print-thread-pool.core-pool-size":"1","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.federation.gpg.policy.generator.interval-ms":"3600000","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch":"false","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.federation.state-store.sql.idle-time-out":"10m","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","io.compression.codec.zstd.level":"3","yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed":"false","hadoop.metrics.jvm.use-thread-mxbean":"false","ipc.[port_number].faircallqueue.multiplexer.weights":"8,4,2,1","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","ipc.[port_number].callqueue.overflow.trigger.failover":"false","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","mapreduce.outputcommitter.factory.scheme.gs":"org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory","ha.health-monitor.rpc.connect.max.retries":"1","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable":"false","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.container-log-monitor.dir-size-limit-bytes":"1000000000","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.http.idle_timeout.ms":"60000","hadoop.registry.zk.retry.ceiling.ms":"60000","ipc.client.connection.idle-scan-interval.ms":"10000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","fs.viewfs.overload.scheme.target.hdfs.impl":"org.apache.hadoop.hdfs.DistributedFileSystem","seq.io.sort.mb":"100","fs.iostatistics.logging.level":"debug","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","ipc.[port_number].decay-scheduler.decay-factor":"0.5","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.webapp.filter-invalid-xml-chars":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-interval-secs":"600","fs.s3a.select.input.csv.record.delimiter":"\\n","fs.s3a.change.detection.source":"etag","ipc.[port_number].backoff.enable":"false","yarn.nodemanager.distributed-scheduling.enabled":"false","yarn.federation.cache.class":"org.apache.hadoop.yarn.server.federation.cache.FederationJCache","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","yarn.webapp.enable-rest-app-submissions":"true","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","ipc.server.metrics.update.runner.interval":"5000","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.runtime.linux.runc.image-toplevel-dir":"/runc-root","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","io.compression.codec.lz4.use.lz4hc":"false","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","yarn.nodemanager.runtime.linux.runc.allowed-container-networks":"host,none,bridge","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","ipc.server.reuseaddr":"true","fs.ftp.timeout":"0","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","fs.AbstractFileSystem.o3fs.impl":"org.apache.hadoop.fs.ozone.OzFs","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.resourcemanager.enable-node-untracked-without-include-path":"false","yarn.nodemanager.webapp.cross-origin.enabled":"false","yarn.federation.gpg.subcluster.heartbeat.expiration-ms":"30m","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","fs.viewfs.overload.scheme.target.swebhdfs.impl":"org.apache.hadoop.hdfs.web.SWebHdfsFileSystem","yarn.client.failover-no-ha-proxy-provider":"org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider","fs.s3a.change.detection.mode":"server","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","yarn.resourcemanager.submission-preprocessor.enabled":"false","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","ipc.[port_number].scheduler.impl":"org.apache.hadoop.ipc.DefaultRpcScheduler","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.timeline-service.writer.async.queue.capacity":"100","yarn.router.webapp.appsinfo-cached-count":"100","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"true","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","yarn.resourcemanager.nodemanagers.heartbeat-interval-speedup-factor":"1.0","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.nodemanager.least-load-policy-selector.enabled":"false","yarn.nodemanager.least-load-policy-selector.pending-container.threshold":"10000","hadoop.caller.context.signature.max.size":"40","ipc.[port_number].decay-scheduler.backoff.responsetime.enable":"false","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","ipc.[port_number].decay-scheduler.metrics.top.user.count":"10","tfile.io.chunk.size":"1048576","yarn.dispatcher.print-events-info.threshold":"5000","yarn.nodemanager.log-container-debug-info-on-error.enabled":"false","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.dispatcher.metric.enable":"false","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.num-manifests-to-cache":"10","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","fs.s3a.select.enabled":"true","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.webapp.ui1.tools.enable":"true","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","hadoop.http.sni.host.check.enabled":"false","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","fs.getspaceused.jitterMillis":"60000","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","ipc.scheduler.impl":"org.apache.hadoop.ipc.DefaultRpcScheduler","yarn.resourcemanager.zk-client-ssl.enabled":"false","io.skip.checksum.errors":"false","yarn.nodemanager.log.trigger.delete.by-size.enabled":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"1200000","yarn.app.mapreduce.am.webapp.https.enabled":"false","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","yarn.resourcemanager.delegation-token.always-cancel":"*********(redacted)","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.timeline-service.flowname.max-size":"0","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.application-tag-based-placement.force-lowercase":"true","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds":"10s,20s,30s,40s","fs.s3a.buffer.dir":"${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a","hadoop.ssl.enabled.protocols":"TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","ipc.server.max.response.size":"1048576","fs.s3a.accesspoint.required":"false","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.resourcemanager.proxy.connection.timeout":"60000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.resourcemanager.activities-manager.app-activities.max-queue-length":"100","yarn.resourcemanager.application-https.policy":"NONE","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","yarn.federation.gpg.application.cleaner.contact.router.spec":"3,10,600000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","yarn.workflow-id.tag-prefix":"workflowid:","fs.azure.local.sas.key.mode":"false","yarn.federation.gpg.policy.generator.class":"org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.NoOpGlobalPolicy","ipc.maximum.data.length":"134217728","fs.s3a.endpoint":"s3.amazonaws.com","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","ipc.server.tcpnodelay":"true","yarn.resourcemanager.resource-tracker.nm.ip-hostname-check":"false","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","yarn.router.webapp.appsinfo-enabled":"false","fs.AbstractFileSystem.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","fs.s3a.downgrade.syncable.exceptions":"true","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.resourcemanager.submission-preprocessor.file-refresh-interval-ms":"60000","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.resourcemanager.activities-manager.cleanup-interval-ms":"5000","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","ipc.[port_number].identity-provider.impl":"org.apache.hadoop.ipc.UserIdentityProvider","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","yarn.nodemanager.container-log-monitor.total-size-limit-bytes":"10000000000","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.federation.amrmproxy.register.uam.retry-count":"3","yarn.nodemanager.health-checker.scripts":"script","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","yarn.federation.cache-entity.nums":"1000","hadoop.security.resolver.impl":"org.apache.hadoop.net.DNSDomainNameResolver","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"96","yarn.nodemanager.runtime.linux.docker.image-update":"false","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","fs.viewfs.overload.scheme.target.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","yarn.dispatcher.cpu-monitor.samples-per-min":"60","hadoop.security.token.service.use_ip":"*********(redacted)","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.nodemanager.containers-launcher.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","io.compression.codec.lzo.buffersize":"65536","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ipc.callqueue.impl":"java.util.concurrent.LinkedBlockingQueue","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.server.purge.interval":"15","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.router.interceptor.user-thread-pool.maximum-pool-size":"5","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","fs.s3a.committer.abort.pending.uploads":"true","yarn.webapp.filter-entity-list-by-user":"false","yarn.resourcemanager.activities-manager.app-activities.ttl-ms":"600000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","yarn.federation.gpg.policy.generator.load-based.edit.maximum":"3","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.apps.cache.size":"1000","yarn.app.mapreduce.am.webapp.https.client.auth":"false","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","ipc.[port_number].decay-scheduler.thresholds":"13,25,50","yarn.federation.gpg.webapp.https.address":"0.0.0.0:8070","ipc.server.read.threadpool.size":"1","fs.s3a.audit.enabled":"true","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","ipc.server.read.connection-queue.size":"100","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","fs.viewfs.overload.scheme.target.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","ipc.cost-provider.impl":"org.apache.hadoop.ipc.DefaultCostProvider","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.viewfs.overload.scheme.target.file.impl":"org.apache.hadoop.fs.LocalFileSystem","yarn.federation.gpg.application.cleaner.interval-ms":"-1s","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.nodemanager.least-load-policy-selector.use-active-core":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.apps.cache.expire":"30s","ipc.server.log.slow.rpc.threshold.ms":"0","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.connection.establish.timeout":"30s","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.apps.cache.enable":"false","yarn.federation.non-ha.enabled":"false","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","yarn.federation.state-store.heartbeat.initial-delay":"30s","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","hadoop.security.group.mapping.ldap.num.attempts":"3","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","ipc.client.connect.max.retries.on.sasl":"5","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.viewfs.overload.scheme.target.webhdfs.impl":"org.apache.hadoop.hdfs.web.WebHdfsFileSystem","yarn.router.webapp.proxy.enable":"true","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.http.metrics.enabled":"true","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","fs.s3a.ssl.channel.mode":"default_jsse","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.resourcemanager.delegation.token.remove-scan-interval":"*********(redacted)","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","ipc.identity-provider.impl":"org.apache.hadoop.ipc.UserIdentityProvider","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.federation.amrmproxy.register.uam.interval":"100ms","yarn.resourcemanager.node-labels.provider.update-newly-registered-nodes-interval-ms":"30000","fs.s3a.change.detection.version.required":"true","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.fs-store.file.replication":"0","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.resourcemanager.proxy.timeout.enabled":"true","yarn.resourcemanager.activities-manager.scheduler-activities.ttl-ms":"600000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","yarn.nodemanager.disk-health-checker.disk-free-space-threshold.enabled":"true","io.compression.codec.snappy.buffersize":"262144","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","io.erasurecode.codec.native.enabled":"true","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","fs.AbstractFileSystem.ofs.impl":"org.apache.hadoop.fs.ozone.RootedOzFs","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","yarn.nodemanager.container-executor.exit-code-file.timeout-ms":"2000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.router.webapp.cross-origin.enabled":"false","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"20","fs.trash.clean.trashroot.enable":"false","hadoop.http.authentication.type":"simple","fs.viewfs.overload.scheme.target.oss.impl":"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem","yarn.federation.gpg.policy.generator.interval":"1h","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","ipc.[port_number].weighted-cost.handler":"1","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.federation.state-store.sql.minimum-idle":"1","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","yarn.federation.gpg.subcluster.cleaner.interval-ms":"-1ms","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","fs.s3a.select.output.csv.field.delimiter":",","yarn.nodemanager.health-checker.timeout-ms":"1200000","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","fs.s3a.select.output.csv.record.delimiter":"\\n","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","fs.viewfs.overload.scheme.target.https.impl":"org.apache.hadoop.fs.http.HttpsFileSystem","yarn.federation.gpg.policy.generator.readonly":"false","yarn.router.deregister.subcluster.enabled":"true","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","fs.viewfs.overload.scheme.target.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.federation.gpg.scheduled.executor.threads":"10","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","yarn.nodemanager.aux-services.%s.classpath":"NONE","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.federation.gpg.webapp.connect-timeout":"30s","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","yarn.nodemanager.runtime.linux.runc.allowed-container-runtimes":"runc","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"100ms","yarn.router.subcluster.cleaner.interval.time":"60s","yarn.nodemanager.log.delete.threshold":"100g","seq.io.sort.factor":"100","fs.viewfs.overload.scheme.target.ofs.impl":"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.resourcemanager.nodestore-rootdir.retry-interval-ms":"1000","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","hadoop.security.group.mapping.ldap.num.attempts.before.failover":"3","mapreduce.task.profile":"false","yarn.federation.gpg.application.cleaner.class":"org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner.DefaultApplicationCleaner","hadoop.prometheus.endpoint.enabled":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","fs.creation.parallel.count":"64","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.federation.state-store.sql.max-life-time":"30m","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","mapreduce.job.dfs.storage.capacity.kill-limit-exceed":"false","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","yarn.resourcemanager.opportunistic.max.container-allocation.per.am.heartbeat":"-1","fs.automatic.close":"true","yarn.resourcemanager.delegation-token-renewer.thread-retry-interval":"*********(redacted)","yarn.resourcemanager.node-labels.am.allow-non-exclusive-allocation":"false","fs.s3a.select.input.csv.quote.character":"\"","yarn.nodemanager.hostname":"0.0.0.0","ipc.[port_number].cost-provider.impl":"org.apache.hadoop.ipc.DefaultCostProvider","yarn.nodemanager.runtime.linux.runc.manifest-to-resources-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.HdfsManifestToResourcesPlugin","yarn.nodemanager.remote-app-log-dir-include-older":"true","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-to-keep":"100","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.resourcemanager.nodestore-rootdir.num-retries":"1000","ipc.[port_number].weighted-cost.lockshared":"10","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60s","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","fs.s3a.select.output.csv.quote.escape.character":"\\\\","yarn.dispatcher.print-thread-pool.maximum-pool-size":"5","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","hadoop.kerberos.keytab.login.autorenewal.enabled":"false","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","hadoop.caller.context.separator":",","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms":"1000","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","hadoop.security.crypto.codec.classes.sm4.ctr.nopadding":"org.apache.hadoop.crypto.OpensslSm4CtrCryptoCodec, org.apache.hadoop.crypto.JceSm4CtrCryptoCodec","yarn.federation.gpg.webapp.read-timeout":"30s","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.nodemanager.disk-health-checker.disk-utilization-threshold.enabled":"true","fs.s3a.executor.capacity":"16","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"true","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.application.max-tags":"10","hadoop.domainname.resolver.impl":"org.apache.hadoop.net.DNSDomainNameResolver","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","fs.azure.buffer.dir":"${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size":"1000","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","hadoop.security.secure.random.impl":"org.apache.hadoop.crypto.random.OpensslSecureRandom","mapreduce.job.running.reduce.limit":"0","fs.s3a.select.errors.include.sql":"false","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","yarn.federation.gpg.policy.generator.load-based.weight.minimum":"0","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","yarn.log-aggregation.debug.filesize":"104857600","yarn.dispatcher.print-thread-pool.keep-alive-time":"10s","yarn.router.subcluster.heartbeat.expiration.time":"30m","fs.s3a.max.total.tasks":"32","fs.s3a.readahead.range":"256K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.fast.upload":"true","fs.s3a.attempts.maximum":"5","yarn.federation.amrmproxy.allocation.history.max.entry":"100","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.resourcemanager.delegation-token-renewer.thread-timeout":"*********(redacted)","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.nodemanager.aux-services.manifest.reload-ms":"0","yarn.nodemanager.emit-container-events":"true","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","ha.failover-controller.active-standby-elector.zk.op.retries":"3","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.task.spill.files.count.limit":"-1","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","fs.viewfs.overload.scheme.target.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","yarn.federation.gpg.webapp.address":"0.0.0.0:8069","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","fs.s3a.input.fadvise":"random","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.hdfs-hash-file":"/runc-root/image-tag-to-hash","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.federation.failover.random.order":"false","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin","io.bytes.per.checksum":"512","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"auto","yarn.timeline-service.client.internal-timers-ttl-secs":"420","io.compression.codec.zstd.buffersize":"0","fs.s3a.select.output.csv.quote.character":"\"","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","ipc.callqueue.overflow.trigger.failover":"false","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","fs.s3a.select.output.csv.quote.fields":"always","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"120000","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","yarn.resourcemanager.delegation-token-renewer.thread-retry-max-attempts":"*********(redacted)","ipc.client.low-latency":"false","yarn.scheduler.skip.node.multiplier":"2","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"900000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-timeout-interval-secs":"360","fs.s3a.socket.recv.buffer":"8192","ipc.backoff.enable":"false","rpc.metrics.timeunit":"MILLISECONDS","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.nodemanager.pluggable-device-framework.enabled":"false","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","ipc.[port_number].callqueue.impl":"java.util.concurrent.LinkedBlockingQueue","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","ipc.[port_number].weighted-cost.lockfree":"1","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","mapreduce.jvm.add-opens-as-default":"true","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","fs.s3a.aws.credentials.provider":"*********(redacted)","yarn.log-aggregation.retain-seconds":"-1","yarn.router.interceptor.allow-partial-result.enable":"false","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","mapreduce.task.ping-for-liveliness-check.enabled":"false","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","fs.azure.enable.readahead":"true","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.federation.gpg.webapp.cross-origin.enabled":"false","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","io.compression.codec.lzo.class":"org.apache.hadoop.io.compress.LzoCodec","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","yarn.router.interceptor.user-thread-pool.minimum-pool-size":"5","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","yarn.federation.state-store.clean-up-retry-sleep-time":"1s","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"64M","fs.s3a.select.input.csv.comment.marker":"#","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.iostatistics.thread.level.enabled":"true","ipc.[port_number].weighted-cost.response":"1","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","mapreduce.task.stuck.timeout-ms":"600000","yarn.resourcemanager.application.max-tag.length":"100","yarn.resourcemanager.ha.enabled":"false","dfs.client.ignore.namenode.default.kms.uri":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","yarn.federation.state-store.sql.conn-time-out":"10s","io.compression.codec.lz4.buffersize":"262144","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","ipc.client.async.calls.max":"100","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms":"1000","hadoop.http.jmx.nan-filter.enabled":"false","yarn.router.scheduled.executor.threads":"1","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","fs.s3a.select.input.csv.field.delimiter":",","yarn.nodemanager.least-load-policy-selector.multiplier":"50000","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","yarn.federation.state-store.sql.pool-name":"YARN-Federation-DataBasePool","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"append","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","yarn.nodemanager.amrmproxy.wait.uam-register.done":"false","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","yarn.federation.gpg.policy.generator.load-based.pending.minimum":"100","ipc.[port_number].weighted-cost.lockexclusive":"100","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","yarn.nodemanager.container-log-monitor.enable":"false","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"24h","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"256","ipc.[port_number].decay-scheduler.period-ms":"5000","yarn.nodemanager.container-localizer.java.opts.add-exports-as-default":"true","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.cache-refresh-interval-secs":"60","map.sort.class":"org.apache.hadoop.util.QuickSort","yarn.federation.state-store.max-applications":"1000","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed":"false","ipc.server.handler.queue.size":"100","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds":"60","yarn.federation.registry.base-dir":"yarnfederation/","yarn.nodemanager.health-checker.run-before-startup":"false","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.shuffle.pathcache.concurrency-level":"16","mapreduce.job.ubertask.maxreduces":"1","mapreduce.shuffle.pathcache.max-weight":"10485760","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","fs.s3a.select.input.compression":"none","yarn.client.nodemanager-connect.retry-interval-ms":"10000","ipc.[port_number].scheduler.priority.levels":"4","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.federation.state-store.clean-up-retry-count":"1","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","fs.s3a.select.input.csv.header":"none","yarn.federation.gpg.policy.generator.load-based.scaling":"LINEAR","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-size":"500","yarn.timeline-service.webapp.rest-csrf.enabled":"false","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb":"0"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"17","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Eclipse Adoptium","java.vm.specification.version":"17","user.home":"/home/spark","sun.arch.data.model":"64","sun.boot.library.path":"/opt/java/openjdk/lib","user.dir":"/opt/spark","java.library.path":"/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib","os.arch":"amd64","java.vm.version":"17.0.12+7","jetty.git.hash":"cef3fbd6d736a21e7d541a5db490381d95a2047d","java.runtime.version":"17.0.12+7","java.vm.info":"mixed mode, sharing","java.runtime.name":"OpenJDK Runtime Environment","java.version.date":"2024-07-16","file.separator":"/","java.class.version":"61.0","java.specification.name":"Java Platform API Specification","file.encoding":"UTF-8","jdk.reflect.useDirectMethodHandle":"false","user.timezone":"Etc/UTC","kubernetes.request.retry.backoffLimit":"3","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","java.vm.compressedOopsMode":"Zero based","os.version":"6.1.140-154.222.amzn2023.x86_64","native.encoding":"UTF-8","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.version":"Temurin-17.0.12+7","java.vendor.url":"https://adoptium.net/","os.name":"Linux","java.vm.vendor":"Eclipse Adoptium","jdk.debug":"release","java.vendor.url.bug":"https://github.com/adoptium/adoptium-support/issues","user.name":"spark","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --deploy-mode client --conf spark.driver.bindAddress=100.64.122.12 --conf spark.executorEnv.SPARK_DRIVER_POD_IP=100.64.122.12 --properties-file /opt/spark/conf/spark.properties --class org.apache.spark.deploy.PythonRunner s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/scripts/pyspark-taxi-trip.py s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input/ s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/","java.home":"/opt/java/openjdk","java.version":"17.0.12","sun.io.unicode.encoding":"UnicodeLittle"},"Metrics Properties":{"driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","*.sink.servlet.path":"/metrics/json","*.sink.servlet.class":"org.apache.spark.metrics.sink.MetricsServlet","applications.sink.servlet.path":"/metrics/applications/json","master.sink.servlet.path":"/metrics/master/json","executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet"},"Classpath Entries":{"/opt/spark/jars/hadoop-client-runtime-3.4.1.jar":"System Classpath","/opt/spark/jars/spark-network-common_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/netty-transport-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar":"System Classpath","/opt/spark/jars/kubernetes-model-common-6.7.2.jar":"System Classpath","/opt/spark/jars/logging-interceptor-3.12.12.jar":"System Classpath","/opt/spark/jars/paranamer-2.8.jar":"System Classpath","/opt/spark/jars/breeze_2.12-2.1.0.jar":"System Classpath","/opt/spark/jars/scala-compiler-2.12.18.jar":"System Classpath","/opt/spark/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/opt/spark/jars/spark-hive-thriftserver_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/netty-handler-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/jersey-hk2-2.40.jar":"System Classpath","/opt/spark/jars/netty-buffer-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/leveldbjni-all-1.8.jar":"System Classpath","/opt/spark/jars/kubernetes-model-gatewayapi-6.7.2.jar":"System Classpath","/opt/spark/jars/commons-compress-1.23.0.jar":"System Classpath","/opt/spark/jars/log4j-api-2.20.0.jar":"System Classpath","/opt/spark/jars/super-csv-2.2.0.jar":"System Classpath","/opt/spark/jars/javolution-5.5.1.jar":"System Classpath","/opt/spark/jars/hive-shims-scheduler-2.3.9.jar":"System Classpath","/opt/spark/jars/spark-kubernetes_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/spire-util_2.12-0.17.0.jar":"System Classpath","/opt/spark/jars/netty-transport-classes-epoll-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/gson-2.2.4.jar":"System Classpath","/opt/spark/jars/spark-common-utils_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/kubernetes-client-6.7.2.jar":"System Classpath","/opt/spark/jars/parquet-format-structures-1.13.1.jar":"System Classpath","/opt/spark/jars/parquet-column-1.13.1.jar":"System Classpath","/opt/spark/jars/tink-1.9.0.jar":"System Classpath","/opt/spark/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/opt/spark/jars/antlr-runtime-3.5.2.jar":"System Classpath","/opt/spark/jars/arrow-memory-core-12.0.1.jar":"System Classpath","/opt/spark/jars/commons-dbcp-1.4.jar":"System Classpath","/opt/spark/jars/chill_2.12-0.10.0.jar":"System Classpath","/opt/spark/jars/log4j-core-2.20.0.jar":"System Classpath","/opt/spark/jars/algebra_2.12-2.0.1.jar":"System Classpath","/opt/spark/jars/xz-1.9.jar":"System Classpath","/opt/spark/jars/spark-sql-api_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/kubernetes-model-metrics-6.7.2.jar":"System Classpath","/opt/spark/jars/antlr4-runtime-4.9.3.jar":"System Classpath","/opt/spark/jars/hive-service-rpc-3.1.3.jar":"System Classpath","/opt/spark/jars/activation-1.1.1.jar":"System Classpath","/opt/spark/jars/spark-repl_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/objenesis-3.3.jar":"System Classpath","/opt/spark/jars/joda-time-2.12.5.jar":"System Classpath","/opt/spark/jars/netty-codec-http-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar":"System Classpath","/opt/spark/jars/hive-storage-api-2.8.1.jar":"System Classpath","/opt/spark/jars/jackson-databind-2.15.2.jar":"System Classpath","/opt/spark/jars/kubernetes-model-policy-6.7.2.jar":"System Classpath","/opt/spark/jars/spark-mllib-local_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/cats-kernel_2.12-2.1.1.jar":"System Classpath","/opt/spark/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/opt/spark/jars/lapack-3.0.3.jar":"System Classpath","/opt/spark/jars/spark-sketch_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/xbean-asm9-shaded-4.23.jar":"System Classpath","/opt/spark/jars/JTransforms-3.1.jar":"System Classpath","/opt/spark/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/opt/spark/jars/jul-to-slf4j-2.0.7.jar":"System Classpath","/opt/spark/jars/commons-collections-3.2.2.jar":"System Classpath","/opt/spark/jars/oro-2.0.8.jar":"System Classpath","/opt/spark/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/opt/spark/jars/derby-10.14.2.0.jar":"System Classpath","/opt/spark/jars/kubernetes-model-coordination-6.7.2.jar":"System Classpath","/opt/spark/jars/javassist-3.29.2-GA.jar":"System Classpath","/opt/spark/jars/kubernetes-model-scheduling-6.7.2.jar":"System Classpath","/opt/spark/jars/netty-common-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/metrics-jvm-4.2.19.jar":"System Classpath","/opt/spark/jars/commons-pool-1.5.4.jar":"System Classpath","/opt/spark/jars/commons-codec-1.16.1.jar":"System Classpath","/opt/spark/jars/breeze-macros_2.12-2.1.0.jar":"System Classpath","/opt/spark/jars/kubernetes-model-networking-6.7.2.jar":"System Classpath","/opt/spark/jars/py4j-0.10.9.7.jar":"System Classpath","/opt/spark/jars/annotations-17.0.0.jar":"System Classpath","/opt/spark/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/opt/spark/jars/blas-3.0.3.jar":"System Classpath","/opt/spark/jars/json4s-ast_2.12-3.7.0-M11.jar":"System Classpath","/opt/spark/jars/scala-collection-compat_2.12-2.7.0.jar":"System Classpath","/opt/spark/jars/commons-lang-2.6.jar":"System Classpath","/opt/spark/jars/compress-lzf-1.1.2.jar":"System Classpath","/opt/spark/jars/zookeeper-jute-3.6.3.jar":"System Classpath","/opt/spark/jars/metrics-graphite-4.2.19.jar":"System Classpath","/opt/spark/jars/snakeyaml-engine-2.6.jar":"System Classpath","/opt/spark/jars/json4s-jackson_2.12-3.7.0-M11.jar":"System Classpath","/opt/spark/jars/kubernetes-model-discovery-6.7.2.jar":"System Classpath","/opt/spark/jars/jta-1.1.jar":"System Classpath","/opt/spark/jars/jersey-server-2.40.jar":"System Classpath","/opt/spark/jars/orc-core-1.9.4-shaded-protobuf.jar":"System Classpath","/opt/spark/jars/jersey-common-2.40.jar":"System Classpath","/opt/spark/jars/libthrift-0.12.0.jar":"System Classpath","/opt/spark/jars/spark-tags_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/metrics-json-4.2.19.jar":"System Classpath","/opt/spark/jars/hive-serde-2.3.9.jar":"System Classpath","/opt/spark/jars/RoaringBitmap-0.9.45.jar":"System Classpath","/opt/spark/jars/metrics-core-4.2.19.jar":"System Classpath","/opt/spark/jars/parquet-encoding-1.13.1.jar":"System Classpath","/opt/spark/jars/hive-common-2.3.9.jar":"System Classpath","/opt/spark/jars/hk2-api-2.6.1.jar":"System Classpath","/opt/spark/jars/kubernetes-model-admissionregistration-6.7.2.jar":"System Classpath","/opt/spark/jars/snakeyaml-2.0.jar":"System Classpath","/opt/spark/jars/guava-14.0.1.jar":"System Classpath","/opt/spark/jars/slf4j-api-2.0.7.jar":"System Classpath","/opt/spark/jars/hive-metastore-2.3.9.jar":"System Classpath","/opt/spark/jars/jcl-over-slf4j-2.0.7.jar":"System Classpath","/opt/spark/jars/json4s-core_2.12-3.7.0-M11.jar":"System Classpath","/opt/spark/jars/kubernetes-model-autoscaling-6.7.2.jar":"System Classpath","/opt/spark/jars/univocity-parsers-2.9.1.jar":"System Classpath","/opt/spark/jars/spark-unsafe_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/spark-yarn_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/spark-streaming_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/commons-text-1.10.0.jar":"System Classpath","/opt/spark/jars/kubernetes-model-apiextensions-6.7.2.jar":"System Classpath","/opt/spark/jars/netty-transport-classes-kqueue-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/orc-shims-1.9.4.jar":"System Classpath","/opt/spark/jars/jersey-client-2.40.jar":"System Classpath","/opt/spark/jars/netty-codec-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/curator-client-2.13.0.jar":"System Classpath","/opt/spark/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/opt/spark/jars/spark-hive_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/jsr305-3.0.0.jar":"System Classpath","/opt/spark/jars/spire_2.12-0.17.0.jar":"System Classpath","/opt/spark/jars/orc-mapreduce-1.9.4-shaded-protobuf.jar":"System Classpath","/opt/spark/jars/jdo-api-3.0.1.jar":"System Classpath","/opt/spark/jars/spark-core_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/hive-exec-2.3.9-core.jar":"System Classpath","/opt/spark/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/opt/spark/jars/scala-library-2.12.18.jar":"System Classpath","/opt/spark/jars/ivy-2.5.1.jar":"System Classpath","/opt/spark/jars/minlog-1.3.0.jar":"System Classpath","/opt/spark/jars/shims-0.9.45.jar":"System Classpath","/opt/spark/jars/netty-handler-proxy-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/jackson-annotations-2.15.2.jar":"System Classpath","/opt/spark/jars/kubernetes-model-core-6.7.2.jar":"System Classpath","/opt/spark/jars/kubernetes-model-certificates-6.7.2.jar":"System Classpath","/opt/spark/jars/mesos-1.4.3-shaded-protobuf.jar":"System Classpath","/opt/spark/jars/netty-all-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/scala-reflect-2.12.18.jar":"System Classpath","/opt/spark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-x86_64.jar":"System Classpath","/opt/spark/conf/":"System Classpath","/opt/spark/jars/curator-recipes-2.13.0.jar":"System Classpath","/opt/spark/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/opt/spark/jars/parquet-common-1.13.1.jar":"System Classpath","/opt/spark/jars/hadoop-aws-3.4.1.jar":"System Classpath","/opt/spark/jars/log4j-slf4j2-impl-2.20.0.jar":"System Classpath","/opt/spark/jars/kubernetes-model-batch-6.7.2.jar":"System Classpath","/opt/spark/jars/kubernetes-httpclient-okhttp-6.7.2.jar":"System Classpath","/opt/spark/jars/spire-macros_2.12-0.17.0.jar":"System Classpath","/opt/spark/jars/datasketches-java-3.3.0.jar":"System Classpath","/opt/spark/jars/metrics-jmx-4.2.19.jar":"System Classpath","/opt/spark/jars/hk2-locator-2.6.1.jar":"System Classpath","/opt/spark/jars/lz4-java-1.8.0.jar":"System Classpath","/opt/spark/jars/scala-xml_2.12-2.1.0.jar":"System Classpath","/opt/spark/jars/hive-shims-0.23-2.3.9.jar":"System Classpath","/opt/spark/jars/spire-platform_2.12-0.17.0.jar":"System Classpath","/opt/spark/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/opt/spark/jars/commons-lang3-3.12.0.jar":"System Classpath","/opt/spark/jars/JLargeArrays-1.5.jar":"System Classpath","/opt/spark/jars/kubernetes-model-rbac-6.7.2.jar":"System Classpath","/opt/spark/jars/jakarta.inject-2.6.1.jar":"System Classpath","/opt/spark/jars/hive-shims-2.3.9.jar":"System Classpath","/opt/spark/jars/spark-launcher_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/parquet-jackson-1.13.1.jar":"System Classpath","/opt/spark/jars/httpcore-4.4.16.jar":"System Classpath","/opt/spark/jars/jodd-core-3.5.2.jar":"System Classpath","/opt/spark/jars/avro-mapred-1.11.2.jar":"System Classpath","/opt/spark/jars/jackson-core-2.15.2.jar":"System Classpath","/opt/spark/jars/spark-catalyst_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-aarch_64.jar":"System Classpath","/opt/spark/jars/json-1.8.jar":"System Classpath","/opt/spark/jars/commons-crypto-1.1.0.jar":"System Classpath","/opt/spark/jars/arrow-vector-12.0.1.jar":"System Classpath","/opt/spark/jars/stax-api-1.0.1.jar":"System Classpath","/opt/spark/jars/hive-beeline-2.3.9.jar":"System Classpath","/opt/spark/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/opt/spark/jars/hadoop-common-3.4.1.jar":"System Classpath","/opt/spark/jars/httpclient-4.5.14.jar":"System Classpath","/opt/spark/jars/commons-io-2.16.1.jar":"System Classpath","/opt/spark/jars/okhttp-3.12.12.jar":"System Classpath","/opt/spark/jars/spark-sql_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/commons-logging-1.1.3.jar":"System Classpath","/opt/spark/jars/datanucleus-core-4.1.17.jar":"System Classpath","/opt/spark/jars/jline-2.14.6.jar":"System Classpath","/opt/spark/jars/kryo-shaded-4.0.2.jar":"System Classpath","/opt/spark/jars/commons-cli-1.5.0.jar":"System Classpath","/opt/spark/jars/jackson-module-scala_2.12-2.15.2.jar":"System Classpath","/opt/spark/jars/HikariCP-2.5.1.jar":"System Classpath","/opt/spark/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/opt/spark/jars/json4s-scalap_2.12-3.7.0-M11.jar":"System Classpath","/opt/spark/jars/chill-java-0.10.0.jar":"System Classpath","/opt/spark/jars/arpack_combined_all-0.1.jar":"System Classpath","/opt/spark/jars/scala-parser-combinators_2.12-2.3.0.jar":"System Classpath","/opt/spark/jars/zookeeper-3.6.3.jar":"System Classpath","/opt/spark/jars/hive-llap-common-2.3.9.jar":"System Classpath","/opt/spark/jars/spark-mesos_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/hive-jdbc-2.3.9.jar":"System Classpath","/opt/spark/jars/netty-resolver-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/jpam-1.1.jar":"System Classpath","/opt/spark/jars/avro-1.11.2.jar":"System Classpath","/opt/spark/jars/opencsv-2.3.jar":"System Classpath","/opt/spark/jars/netty-transport-native-unix-common-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/parquet-hadoop-1.13.1.jar":"System Classpath","/opt/spark/jars/curator-framework-2.13.0.jar":"System Classpath","/opt/spark/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/opt/spark/jars/jackson-dataformat-yaml-2.15.2.jar":"System Classpath","/opt/spark/jars/arrow-memory-netty-12.0.1.jar":"System Classpath","/opt/spark/jars/stream-2.9.6.jar":"System Classpath","/opt/spark/jars/hive-cli-2.3.9.jar":"System Classpath","/opt/spark/jars/kubernetes-model-events-6.7.2.jar":"System Classpath","/opt/spark/jars/spark-graphx_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/jackson-datatype-jsr310-2.15.2.jar":"System Classpath","/opt/spark/jars/avro-ipc-1.11.2.jar":"System Classpath","/opt/spark/jars/rocksdbjni-8.3.2.jar":"System Classpath","/opt/spark/jars/bundle-2.29.0.jar":"System Classpath","/opt/spark/jars/kubernetes-model-node-6.7.2.jar":"System Classpath","/opt/spark/jars/commons-compiler-3.1.9.jar":"System Classpath","/opt/spark/jars/ST4-4.0.4.jar":"System Classpath","/opt/spark/jars/spark-kvstore_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/log4j-1.2-api-2.20.0.jar":"System Classpath","/opt/spark/jars/hadoop-client-api-3.4.1.jar":"System Classpath","/opt/spark/jars/datasketches-memory-2.1.0.jar":"System Classpath","/opt/spark/jars/spark-mllib_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/kubernetes-client-api-6.7.2.jar":"System Classpath","/opt/spark/jars/arrow-format-12.0.1.jar":"System Classpath","/opt/spark/jars/arpack-3.0.3.jar":"System Classpath","/opt/spark/jars/kubernetes-model-resource-6.7.2.jar":"System Classpath","/opt/spark/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/opt/spark/jars/zstd-jni-1.5.5-4.jar":"System Classpath","/opt/spark/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/opt/spark/jars/hadoop-shaded-guava-1.1.1.jar":"System Classpath","/opt/spark/jars/okio-1.17.6.jar":"System Classpath","/opt/spark/jars/hadoop-yarn-server-web-proxy-3.4.1.jar":"System Classpath","/opt/spark/jars/hk2-utils-2.6.1.jar":"System Classpath","/opt/spark/jars/audience-annotations-0.5.0.jar":"System Classpath","/opt/spark/jars/flatbuffers-java-1.12.0.jar":"System Classpath","/opt/spark/jars/janino-3.1.9.jar":"System Classpath","/opt/spark/jars/aircompressor-0.27.jar":"System Classpath","/opt/spark/jars/commons-math3-3.6.1.jar":"System Classpath","/opt/spark/jars/transaction-api-1.1.jar":"System Classpath","/opt/spark/jars/kubernetes-model-storageclass-6.7.2.jar":"System Classpath","/opt/spark/jars/libfb303-0.9.3.jar":"System Classpath","/opt/spark/jars/snappy-java-1.1.10.5.jar":"System Classpath","/opt/spark/jars/hive-shims-common-2.3.9.jar":"System Classpath","/opt/spark/jars/jersey-container-servlet-core-2.40.jar":"System Classpath","/opt/spark/jars/netty-codec-http2-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/pickle-1.3.jar":"System Classpath","/opt/spark/jars/kubernetes-model-flowcontrol-6.7.2.jar":"System Classpath","/opt/spark/jars/threeten-extra-1.7.1.jar":"System Classpath","/opt/spark/jars/spark-network-shuffle_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/kubernetes-model-apps-6.7.2.jar":"System Classpath","/opt/spark/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/opt/spark/jars/kubernetes-model-extensions-6.7.2.jar":"System Classpath","/opt/spark/jars/netty-codec-socks-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/jersey-container-servlet-2.40.jar":"System Classpath","/opt/spark/jars/zjsonpatch-0.3.0.jar":"System Classpath","/opt/spark/jars/commons-collections4-4.4.jar":"System Classpath"}} +{"Event":"SparkListenerApplicationStart","App Name":"NewYorkTaxiData_2025_06_27_00_24_44","App ID":"spark-110be3a8424d4a2789cb88134418217b","Timestamp":1750983884577,"User":"spark"} +{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1750983918532,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[0],"Properties":{"spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750983918544,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}","resource.executor.cores":"1","spark.rdd.scope.noOverride":"true"}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1750983953852,"Executor ID":"3","Executor Info":{"Host":"100.64.216.237","Total Cores":1,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1750983953852}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1750983953853,"Executor ID":"4","Executor Info":{"Host":"100.64.138.5","Total Cores":1,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1750983953853}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1750983953853,"Executor ID":"1","Executor Info":{"Host":"100.64.221.138","Total Cores":1,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1750983953853}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1750983953860,"Executor ID":"2","Executor Info":{"Host":"100.64.162.9","Total Cores":1,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1750983953860}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"3","Host":"100.64.216.237","Port":35755},"Maximum Memory":2388236697,"Timestamp":1750983953909,"Maximum Onheap Memory":2388236697,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"100.64.221.138","Port":43395},"Maximum Memory":2388236697,"Timestamp":1750983953910,"Maximum Onheap Memory":2388236697,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"4","Host":"100.64.138.5","Port":34097},"Maximum Memory":2388236697,"Timestamp":1750983953910,"Maximum Onheap Memory":2388236697,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"2","Host":"100.64.162.9","Port":43483},"Maximum Memory":2388236697,"Timestamp":1750983953918,"Maximum Onheap Memory":2388236697,"Maximum Offheap Memory":0} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750983953947,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750983953947,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983956078,"Failed":false,"Killed":false,"Accumulables":[{"ID":1,"Name":"internal.metrics.executorDeserializeTime","Update":301,"Value":301,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorDeserializeCpuTime","Update":241759717,"Value":241759717,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorRunTime","Update":1773,"Value":1773,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.executorCpuTime","Update":92913313,"Value":92913313,"Internal":true,"Count Failed Values":true},{"ID":5,"Name":"internal.metrics.resultSize","Update":2470,"Value":2470,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.jvmGCTime","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.resultSerializationTime","Update":5,"Value":5,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":301,"Executor Deserialize CPU Time":241759717,"Executor Run Time":1773,"Executor CPU Time":92913313,"Peak Execution Memory":0,"Result Size":2470,"JVM GC Time":20,"Result Serialization Time":5,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750983918544,"Completion Time":1750983956084,"Accumulables":[{"ID":1,"Name":"internal.metrics.executorDeserializeTime","Value":301,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorDeserializeCpuTime","Value":241759717,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorRunTime","Value":1773,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.executorCpuTime","Value":92913313,"Internal":true,"Count Failed Values":true},{"ID":5,"Name":"internal.metrics.resultSize","Value":2470,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.jvmGCTime","Value":20,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.resultSerializationTime","Value":5,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1750983956086,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"rootExecutionId":0,"description":"showString at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nCollectLimit (4)\n+- * Project (3)\n +- * ColumnarToRow (2)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct\n\n(2) ColumnarToRow [codegen id : 1]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(3) Project [codegen id : 1]\nOutput [20]: [toprettystring(VendorID#0L, Some(Etc/UTC)) AS toprettystring(VendorID)#80, toprettystring(tpep_pickup_datetime#1, Some(Etc/UTC)) AS toprettystring(tpep_pickup_datetime)#81, toprettystring(tpep_dropoff_datetime#2, Some(Etc/UTC)) AS toprettystring(tpep_dropoff_datetime)#82, toprettystring(passenger_count#3, Some(Etc/UTC)) AS toprettystring(passenger_count)#83, toprettystring(trip_distance#4, Some(Etc/UTC)) AS toprettystring(trip_distance)#84, toprettystring(RatecodeID#5, Some(Etc/UTC)) AS toprettystring(RatecodeID)#85, toprettystring(store_and_fwd_flag#6, Some(Etc/UTC)) AS toprettystring(store_and_fwd_flag)#86, toprettystring(PULocationID#7L, Some(Etc/UTC)) AS toprettystring(PULocationID)#87, toprettystring(DOLocationID#8L, Some(Etc/UTC)) AS toprettystring(DOLocationID)#88, toprettystring(payment_type#9L, Some(Etc/UTC)) AS toprettystring(payment_type)#89, toprettystring(fare_amount#10, Some(Etc/UTC)) AS toprettystring(fare_amount)#90, toprettystring(extra#11, Some(Etc/UTC)) AS toprettystring(extra)#91, toprettystring(mta_tax#12, Some(Etc/UTC)) AS toprettystring(mta_tax)#92, toprettystring(tip_amount#13, Some(Etc/UTC)) AS toprettystring(tip_amount)#93, toprettystring(tolls_amount#14, Some(Etc/UTC)) AS toprettystring(tolls_amount)#94, toprettystring(improvement_surcharge#15, Some(Etc/UTC)) AS toprettystring(improvement_surcharge)#95, toprettystring(total_amount#16, Some(Etc/UTC)) AS toprettystring(total_amount)#96, toprettystring(congestion_surcharge#17, Some(Etc/UTC)) AS toprettystring(congestion_surcharge)#97, toprettystring(airport_fee#18, Some(Etc/UTC)) AS toprettystring(airport_fee)#98, 2025-06-27 00:25:56.642038 AS toprettystring(current_date)#99]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(4) CollectLimit\nInput [20]: [toprettystring(VendorID)#80, toprettystring(tpep_pickup_datetime)#81, toprettystring(tpep_dropoff_datetime)#82, toprettystring(passenger_count)#83, toprettystring(trip_distance)#84, toprettystring(RatecodeID)#85, toprettystring(store_and_fwd_flag)#86, toprettystring(PULocationID)#87, toprettystring(DOLocationID)#88, toprettystring(payment_type)#89, toprettystring(fare_amount)#90, toprettystring(extra)#91, toprettystring(mta_tax)#92, toprettystring(tip_amount)#93, toprettystring(tolls_amount)#94, toprettystring(improvement_surcharge)#95, toprettystring(total_amount)#96, toprettystring(congestion_surcharge)#97, toprettystring(airport_fee)#98, toprettystring(current_date)#99]\nArguments: 21\n\n","sparkPlanInfo":{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(VendorID#0L, Some(Etc/UTC)) AS toprettystring(VendorID)#80, toprettystring(tpep_pickup_datetime#1, Some(Etc/UTC)) AS toprettystring(tpep_pickup_datetime)#81, toprettystring(tpep_dropoff_datetime#2, Some(Etc/UTC)) AS toprettystring(tpep_dropoff_datetime)#82, toprettystring(passenger_count#3, Some(Etc/UTC)) AS toprettystring(passenger_count)#83, toprettystring(trip_distance#4, Some(Etc/UTC)) AS toprettystring(trip_distance)#84, toprettystring(RatecodeID#5, Some(Etc/UTC)) AS toprettystring(RatecodeID)#85, toprettystring(store_and_fwd_flag#6, Some(Etc/UTC)) AS toprettystring(store_and_fwd_flag)#86, toprettystring(PULocationID#7L, Some(Etc/UTC)) AS toprettystring(PULocationID)#87, toprettystring(DOLocationID#8L, Some(Etc/UTC)) AS toprettystring(DOLocationID)#88, toprettystring(payment_type#9L, Some(Etc/UTC)) AS toprettystring(payment_type)#89, toprettystring(fare_amount#10, Some(Etc/UTC)) AS toprettystring(fare_amount)#90, toprettystring(extra#11, Some(Etc/UTC)) AS toprettystring(extra)#91, toprettystring(mta_tax#12, Some(Etc/UTC)) AS toprettystring(mta_tax)#92, toprettystring(tip_amount#13, Some(Etc/UTC)) AS toprettystring(tip_amount)#93, toprettystring(tolls_amount#14, Some(Etc/UTC)) AS toprettystring(tolls_amount)#94, toprettystring(improvement_surcharge#15, Some(Etc/UTC)) AS toprettystring(improvement_surcharge)#95, toprettystring(total_amount#16, Some(Etc/UTC)) AS toprettystring(total_amount)#96, toprettystring(congestion_surcharge#17, Some(Etc/UTC)) AS toprettystring(congestion_surcharge)#97, toprettystring(airport_fee#18, Some(Etc/UTC)) AS toprettystring(airport_fee)#98, 2025-06-27 00:25:56.642038 AS toprettystring(current_date)#99]","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [VendorID#0L,tpep_pickup_datetime#1,tpep_dropoff_datetime#2,passenger_count#3,trip_distance#4,RatecodeID#5,store_and_fwd_flag#6,PULocationID#7L,DOLocationID#8L,payment_type#9L,fare_amount#10,extra#11,mta_tax#12,tip_amount#13,tolls_amount#14,improvement_surcharge#15,total_amount#16,congestion_surcharge#17,airport_fee#18] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":61,"metricType":"sum"},{"name":"scan time","accumulatorId":60,"metricType":"timing"},{"name":"metadata time","accumulatorId":62,"metricType":"timing"},{"name":"size of files read","accumulatorId":63,"metricType":"size"},{"name":"number of output rows","accumulatorId":59,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":57,"metricType":"sum"},{"name":"number of input batches","accumulatorId":58,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":56,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":54,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":48,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":55,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":49,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":46,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":43,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":52,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":45,"metricType":"sum"},{"name":"records read","accumulatorId":42,"metricType":"sum"},{"name":"local bytes read","accumulatorId":40,"metricType":"size"},{"name":"fetch wait time","accumulatorId":41,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":38,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":44,"metricType":"sum"},{"name":"local blocks read","accumulatorId":37,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":47,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":36,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":50,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":51,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":39,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":53,"metricType":"size"}]},"time":1750983957434,"modifiedConfigs":{},"jobTags":[]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[61,51],[62,4],[63,1945137399]]} +{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1750983958326,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"FileScanRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"5\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[1],"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750983884577","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_00_24_44","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.122.12","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-3a9e8397aec45f6f-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-110be3a8424d4a2789cb88134418217b","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"0","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.122.12","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750983883971","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"0","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"FileScanRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"5\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750983958329,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750983884577","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_00_24_44","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.122.12","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-3a9e8397aec45f6f-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-110be3a8424d4a2789cb88134418217b","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","resource.executor.cores":"1","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"0","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.122.12","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750983883971","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"0","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750983958350,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750983958350,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983961400,"Failed":false,"Killed":false,"Accumulables":[{"ID":57,"Name":"number of output rows","Update":"4096","Value":"4096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"number of input batches","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"number of output rows","Update":"4096","Value":"4096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"scan time","Update":"2380","Value":"2380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":64,"Name":"internal.metrics.executorDeserializeTime","Update":326,"Value":326,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeCpuTime","Update":292055128,"Value":292055128,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorRunTime","Update":2689,"Value":2689,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorCpuTime","Update":2292010070,"Value":2292010070,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.resultSize","Update":3678,"Value":3678,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.jvmGCTime","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":95,"Name":"internal.metrics.input.bytesRead","Update":38137744,"Value":38137744,"Internal":true,"Count Failed Values":true},{"ID":96,"Name":"internal.metrics.input.recordsRead","Update":4096,"Value":4096,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":326,"Executor Deserialize CPU Time":292055128,"Executor Run Time":2689,"Executor CPU Time":2292010070,"Peak Execution Memory":0,"Result Size":3678,"JVM GC Time":20,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":38137744,"Records Read":4096},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"FileScanRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"5\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750983958329,"Completion Time":1750983961401,"Accumulables":[{"ID":57,"Name":"number of output rows","Value":"4096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"number of input batches","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"number of output rows","Value":"4096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"scan time","Value":"2380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":64,"Name":"internal.metrics.executorDeserializeTime","Value":326,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeCpuTime","Value":292055128,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorRunTime","Value":2689,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorCpuTime","Value":2292010070,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.resultSize","Value":3678,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.jvmGCTime","Value":20,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.resultSerializationTime","Value":3,"Internal":true,"Count Failed Values":true},{"ID":95,"Name":"internal.metrics.input.bytesRead","Value":38137744,"Internal":true,"Count Failed Values":true},{"ID":96,"Name":"internal.metrics.input.recordsRead","Value":4096,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1750983961402,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1750983962737,"errorMessage":""} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"rootExecutionId":1,"description":"count at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (5)\n+- HashAggregate (4)\n +- Exchange (3)\n +- HashAggregate (2)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput: []\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct<>\n\n(2) HashAggregate\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#164L]\nResults [1]: [count#165L]\n\n(3) Exchange\nInput [1]: [count#165L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=27]\n\n(4) HashAggregate\nInput [1]: [count#165L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#161L]\nResults [1]: [count(1)#161L AS count#162L]\n\n(5) AdaptiveSparkPlan\nOutput [1]: [count#162L]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=27]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]","ReadSchema":"struct<>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":135,"metricType":"sum"},{"name":"scan time","accumulatorId":134,"metricType":"timing"},{"name":"metadata time","accumulatorId":136,"metricType":"timing"},{"name":"size of files read","accumulatorId":137,"metricType":"size"},{"name":"number of output rows","accumulatorId":133,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":129,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":130,"metricType":"timing"},{"name":"peak memory","accumulatorId":128,"metricType":"size"},{"name":"number of output rows","accumulatorId":127,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":132,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":131,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":125,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":119,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":126,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":120,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":117,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":114,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":123,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":116,"metricType":"sum"},{"name":"records read","accumulatorId":113,"metricType":"sum"},{"name":"local bytes read","accumulatorId":111,"metricType":"size"},{"name":"fetch wait time","accumulatorId":112,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":109,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":115,"metricType":"sum"},{"name":"local blocks read","accumulatorId":108,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":118,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":107,"metricType":"sum"},{"name":"data size","accumulatorId":105,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":121,"metricType":"size"},{"name":"number of partitions","accumulatorId":106,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":122,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":110,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":124,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":101,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":102,"metricType":"timing"},{"name":"peak memory","accumulatorId":100,"metricType":"size"},{"name":"number of output rows","accumulatorId":99,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":104,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":103,"metricType":"average"}]}],"metadata":{},"metrics":[]},"time":1750983962944,"modifiedConfigs":{},"jobTags":[]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (10)\n+- == Current Plan ==\n HashAggregate (6)\n +- ShuffleQueryStage (5)\n +- Exchange (4)\n +- * HashAggregate (3)\n +- * ColumnarToRow (2)\n +- Scan parquet (1)\n+- == Initial Plan ==\n HashAggregate (9)\n +- Exchange (8)\n +- HashAggregate (7)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput: []\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct<>\n\n(2) ColumnarToRow [codegen id : 1]\nInput: []\n\n(3) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#164L]\nResults [1]: [count#165L]\n\n(4) Exchange\nInput [1]: [count#165L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=39]\n\n(5) ShuffleQueryStage\nOutput [1]: [count#165L]\nArguments: 0\n\n(6) HashAggregate\nInput [1]: [count#165L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#161L]\nResults [1]: [count(1)#161L AS count#162L]\n\n(7) HashAggregate\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#164L]\nResults [1]: [count#165L]\n\n(8) Exchange\nInput [1]: [count#165L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=27]\n\n(9) HashAggregate\nInput [1]: [count#165L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#161L]\nResults [1]: [count(1)#161L AS count#162L]\n\n(10) AdaptiveSparkPlan\nOutput [1]: [count#162L]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=39]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]","ReadSchema":"struct<>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":135,"metricType":"sum"},{"name":"scan time","accumulatorId":134,"metricType":"timing"},{"name":"metadata time","accumulatorId":136,"metricType":"timing"},{"name":"size of files read","accumulatorId":137,"metricType":"size"},{"name":"number of output rows","accumulatorId":133,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":173,"metricType":"sum"},{"name":"number of input batches","accumulatorId":174,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":169,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":170,"metricType":"timing"},{"name":"peak memory","accumulatorId":168,"metricType":"size"},{"name":"number of output rows","accumulatorId":167,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":172,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":171,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":166,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":164,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":158,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":165,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":159,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":156,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":153,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":162,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":155,"metricType":"sum"},{"name":"records read","accumulatorId":152,"metricType":"sum"},{"name":"local bytes read","accumulatorId":150,"metricType":"size"},{"name":"fetch wait time","accumulatorId":151,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":148,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":154,"metricType":"sum"},{"name":"local blocks read","accumulatorId":147,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":157,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":146,"metricType":"sum"},{"name":"data size","accumulatorId":144,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":160,"metricType":"size"},{"name":"number of partitions","accumulatorId":145,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":161,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":149,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":163,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":140,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":141,"metricType":"timing"},{"name":"peak memory","accumulatorId":139,"metricType":"size"},{"name":"number of output rows","accumulatorId":138,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":143,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":142,"metricType":"average"}]}],"metadata":{},"metrics":[]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[135,51],[136,0],[137,1945137399]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[145,1]]} +{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1750983963140,"Stage Infos":[{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":17,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"FileScanRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[2],"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750983884577","spark.rdd.scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_00_24_44","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.122.12","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-3a9e8397aec45f6f-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-110be3a8424d4a2789cb88134418217b","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"1","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.122.12","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750983883971","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"1","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.rdd.scope.noOverride":"true","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":17,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"FileScanRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750983963144,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750983884577","spark.rdd.scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_00_24_44","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.122.12","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-3a9e8397aec45f6f-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-110be3a8424d4a2789cb88134418217b","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","resource.executor.cores":"1","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"1","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.122.12","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750983883971","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"1","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.rdd.scope.noOverride":"true","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750983963233,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1750983963235,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1750983963236,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1750983963236,"Executor ID":"2","Host":"100.64.162.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1750983963925,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750983963233,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983963927,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"7391793","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"526","Value":"526","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"6974263","Value":"6974263","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"571","Value":"571","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"554","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"7391793","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"1806","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":61,"Value":61,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":53914010,"Value":53914010,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":616,"Value":616,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":231036847,"Value":231036847,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2176,"Value":2176,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":6974263,"Value":6974263,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":31206,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":7391793,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":61,"Executor Deserialize CPU Time":53914010,"Executor Run Time":616,"Executor CPU Time":231036847,"Peak Execution Memory":0,"Result Size":2176,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":6974263,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1750983964489,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1750983963925,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983964490,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"14783586","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"539","Value":"1065","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"547201","Value":"7521464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"549","Value":"1120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"548","Value":"1102","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"14783586","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"3612","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":65,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4149520,"Value":58063530,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":551,"Value":1167,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":110954458,"Value":341991305,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":4309,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":118,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":547201,"Value":7521464,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":62412,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":14783586,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4149520,"Executor Run Time":551,"Executor CPU Time":110954458,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":547201,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1750983964658,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1750983963236,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983964659,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"22175379","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"969","Value":"2034","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"48","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"7342253","Value":"14863717","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"1019","Value":"2139","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"1005","Value":"2107","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"22175379","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"5418","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":182,"Value":247,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":126747158,"Value":184810688,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":1228,"Value":2395,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":500888602,"Value":842879907,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2176,"Value":6485,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":177,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":7342253,"Value":14863717,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":93618,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":22175379,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":182,"Executor Deserialize CPU Time":126747158,"Executor Run Time":1228,"Executor CPU Time":500888602,"Peak Execution Memory":0,"Result Size":2176,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":7342253,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1750983965146,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1750983964489,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983965147,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"29567172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"637","Value":"2671","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"64","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"369795","Value":"15233512","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"642","Value":"2781","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"642","Value":"2749","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"29567172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"7224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":250,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3436973,"Value":188247661,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":645,"Value":3040,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":91222942,"Value":934102849,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":8618,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":236,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":369795,"Value":15233512,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":124824,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":29567172,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3436973,"Executor Run Time":645,"Executor CPU Time":91222942,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":369795,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":8,"Attempt":0,"Partition ID":8,"Launch Time":1750983965411,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1750983964658,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983965412,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"36958965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"722","Value":"3393","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"80","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"295","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"500613","Value":"15734125","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"738","Value":"3519","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"737","Value":"3486","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"36958965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"9030","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":254,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4040794,"Value":192288455,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":740,"Value":3780,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":111906774,"Value":1046009623,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":10751,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":295,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":500613,"Value":15734125,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":156030,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":36958965,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":197860416,"JVMOffHeapMemory":101614056,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":502037,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":502037,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16842509,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9902387200,"ProcessTreeJVMRSSMemory":624971776,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":3,"MinorGCTime":27,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":27},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4040794,"Executor Run Time":740,"Executor CPU Time":111906774,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":500613,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":9,"Attempt":0,"Partition ID":9,"Launch Time":1750983965738,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1750983965146,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983965739,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"44350758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"571","Value":"3964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"96","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"354","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"527846","Value":"16261971","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"578","Value":"4097","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"578","Value":"4064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"44350758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"10836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":257,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3273782,"Value":195562237,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":581,"Value":4361,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":80237899,"Value":1126247522,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":12884,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":354,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":527846,"Value":16261971,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":187236,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":44350758,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3273782,"Executor Run Time":581,"Executor CPU Time":80237899,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":527846,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":10,"Attempt":0,"Partition ID":10,"Launch Time":1750983966067,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":8,"Attempt":0,"Partition ID":8,"Launch Time":1750983965411,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983966068,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"51742551","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"636","Value":"4600","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"514370","Value":"16776341","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"642","Value":"4739","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"641","Value":"4705","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"51742551","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"12642","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":260,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3882663,"Value":199444900,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":644,"Value":5005,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":87877720,"Value":1214125242,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":15017,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":413,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":514370,"Value":16776341,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":218442,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":51742551,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3882663,"Executor Run Time":644,"Executor CPU Time":87877720,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":514370,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":11,"Attempt":0,"Partition ID":11,"Launch Time":1750983966270,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1750983963235,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983966271,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"59134344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"2268","Value":"6868","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"7064232","Value":"23840573","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"2316","Value":"7055","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"2302","Value":"7007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"59134344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"14448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":376,"Value":636,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":316699220,"Value":516144120,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":2626,"Value":7631,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":1912124457,"Value":3126249699,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2217,"Value":17234,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.jvmGCTime","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":472,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":7064232,"Value":23840573,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":249648,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":59134344,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":376,"Executor Deserialize CPU Time":316699220,"Executor Run Time":2626,"Executor CPU Time":1912124457,"Peak Execution Memory":0,"Result Size":2217,"JVM GC Time":23,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":7064232,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":12,"Attempt":0,"Partition ID":12,"Launch Time":1750983966447,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":9,"Attempt":0,"Partition ID":9,"Launch Time":1750983965738,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983966448,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"66526137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"689","Value":"7557","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"531","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"497465","Value":"24338038","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"695","Value":"7750","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"695","Value":"7702","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"66526137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"16254","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":638,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2945074,"Value":519089194,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":697,"Value":8328,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":76684344,"Value":3202934043,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":19367,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":531,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":497465,"Value":24338038,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":280854,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":66526137,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2945074,"Executor Run Time":697,"Executor CPU Time":76684344,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":497465,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":13,"Attempt":0,"Partition ID":13,"Launch Time":1750983966572,"Executor ID":"2","Host":"100.64.162.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1750983963236,"Executor ID":"2","Host":"100.64.162.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983966573,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"73917930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"2570","Value":"10127","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"590","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"7230742","Value":"31568780","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"2620","Value":"10370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"2604","Value":"10306","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"73917930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"18060","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":372,"Value":1010,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":323595231,"Value":842684425,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":2930,"Value":11258,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":2127676305,"Value":5330610348,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2217,"Value":21584,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.jvmGCTime","Update":22,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":590,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":7230742,"Value":31568780,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":312060,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":73917930,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":372,"Executor Deserialize CPU Time":323595231,"Executor Run Time":2930,"Executor CPU Time":2127676305,"Peak Execution Memory":0,"Result Size":2217,"JVM GC Time":22,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":7230742,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":14,"Attempt":0,"Partition ID":14,"Launch Time":1750983966731,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":10,"Attempt":0,"Partition ID":10,"Launch Time":1750983966067,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983966732,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"81309723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"647","Value":"10774","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"649","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"364828","Value":"31933608","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"652","Value":"11022","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"651","Value":"10957","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"81309723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"19866","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1012,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2689022,"Value":845373447,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":654,"Value":11912,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":82704593,"Value":5413314941,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":23717,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":649,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":364828,"Value":31933608,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":343266,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":81309723,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2689022,"Executor Run Time":654,"Executor CPU Time":82704593,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":364828,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":15,"Attempt":0,"Partition ID":15,"Launch Time":1750983966962,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":11,"Attempt":0,"Partition ID":11,"Launch Time":1750983966270,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983966963,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"88701516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"660","Value":"11434","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"192","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"708","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"380039","Value":"32313647","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"669","Value":"11691","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"668","Value":"11625","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"88701516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"21672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":1016,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4331468,"Value":849704915,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":680,"Value":12592,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":106033948,"Value":5519348889,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2131,"Value":25848,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":708,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":380039,"Value":32313647,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":374472,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":88701516,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4331468,"Executor Run Time":680,"Executor CPU Time":106033948,"Peak Execution Memory":0,"Result Size":2131,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":380039,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":16,"Attempt":0,"Partition ID":16,"Launch Time":1750983967041,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":12,"Attempt":0,"Partition ID":12,"Launch Time":1750983966447,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983967042,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"96093309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"580","Value":"12014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"767","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"509656","Value":"32823303","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"583","Value":"12274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"583","Value":"12208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"96093309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"23478","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1018,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2517599,"Value":852222514,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":585,"Value":13177,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":80595595,"Value":5599944484,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":27981,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":767,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":509656,"Value":32823303,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":405678,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":96093309,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2517599,"Executor Run Time":585,"Executor CPU Time":80595595,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":509656,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":13,"Attempt":0,"Partition ID":13,"Launch Time":1750983966572,"Executor ID":"2","Host":"100.64.162.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983967270,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"103485102","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"659","Value":"12673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"826","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"553425","Value":"33376728","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"670","Value":"12944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"669","Value":"12877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"103485102","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"25284","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":1023,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5418291,"Value":857640805,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":683,"Value":13860,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":120451071,"Value":5720395555,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2131,"Value":30112,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":826,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":553425,"Value":33376728,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":436884,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":103485102,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":5418291,"Executor Run Time":683,"Executor CPU Time":120451071,"Peak Execution Memory":0,"Result Size":2131,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":553425,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":14,"Attempt":0,"Partition ID":14,"Launch Time":1750983966731,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983967438,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"110876895","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"689","Value":"13362","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"885","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"539937","Value":"33916665","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"694","Value":"13638","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"693","Value":"13570","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"110876895","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"27090","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1026,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3010044,"Value":860650849,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":696,"Value":14556,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":76920493,"Value":5797316048,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":32245,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":885,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":539937,"Value":33916665,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":468090,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":110876895,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3010044,"Executor Run Time":696,"Executor CPU Time":76920493,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":539937,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":15,"Attempt":0,"Partition ID":15,"Launch Time":1750983966962,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983967571,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"118268688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"591","Value":"13953","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"347438","Value":"34264103","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"595","Value":"14233","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"595","Value":"14165","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"118268688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"28896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1029,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3447158,"Value":864098007,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":597,"Value":15153,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":80583722,"Value":5877899770,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2131,"Value":34376,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":944,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":347438,"Value":34264103,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":499296,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":118268688,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3447158,"Executor Run Time":597,"Executor CPU Time":80583722,"Peak Execution Memory":0,"Result Size":2131,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":347438,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":16,"Attempt":0,"Partition ID":16,"Launch Time":1750983967041,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983967615,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"558","Value":"14511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"469018","Value":"34733121","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"561","Value":"14794","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"561","Value":"14726","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"30702","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1031,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2918370,"Value":867016377,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":563,"Value":15716,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":66946758,"Value":5944846528,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":36509,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1003,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":469018,"Value":34733121,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":530502,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":125660481,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2918370,"Executor Run Time":563,"Executor CPU Time":66946758,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":469018,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":17,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"FileScanRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750983963144,"Completion Time":1750983967616,"Accumulables":[{"ID":133,"Name":"number of output rows","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Value":"14511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Value":"272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Value":"1003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Value":"34733121","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Value":"14794","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Value":"14726","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Value":"30702","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Value":1031,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Value":867016377,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Value":15716,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Value":5944846528,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Value":36509,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.jvmGCTime","Value":45,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.resultSerializationTime","Value":6,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1003,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":17,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Value":34733121,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Value":530502,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Value":125660481,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1750983967620,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (10)\n+- == Final Plan ==\n * HashAggregate (6)\n +- ShuffleQueryStage (5), Statistics(sizeInBytes=272.0 B, rowCount=17)\n +- Exchange (4)\n +- * HashAggregate (3)\n +- * ColumnarToRow (2)\n +- Scan parquet (1)\n+- == Initial Plan ==\n HashAggregate (9)\n +- Exchange (8)\n +- HashAggregate (7)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput: []\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct<>\n\n(2) ColumnarToRow [codegen id : 1]\nInput: []\n\n(3) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#164L]\nResults [1]: [count#165L]\n\n(4) Exchange\nInput [1]: [count#165L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=39]\n\n(5) ShuffleQueryStage\nOutput [1]: [count#165L]\nArguments: 0\n\n(6) HashAggregate [codegen id : 2]\nInput [1]: [count#165L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#161L]\nResults [1]: [count(1)#161L AS count#162L]\n\n(7) HashAggregate\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#164L]\nResults [1]: [count#165L]\n\n(8) Exchange\nInput [1]: [count#165L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=27]\n\n(9) HashAggregate\nInput [1]: [count#165L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#161L]\nResults [1]: [count(1)#161L AS count#162L]\n\n(10) AdaptiveSparkPlan\nOutput [1]: [count#162L]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=39]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]","ReadSchema":"struct<>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":135,"metricType":"sum"},{"name":"scan time","accumulatorId":134,"metricType":"timing"},{"name":"metadata time","accumulatorId":136,"metricType":"timing"},{"name":"size of files read","accumulatorId":137,"metricType":"size"},{"name":"number of output rows","accumulatorId":133,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":173,"metricType":"sum"},{"name":"number of input batches","accumulatorId":174,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":169,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":170,"metricType":"timing"},{"name":"peak memory","accumulatorId":168,"metricType":"size"},{"name":"number of output rows","accumulatorId":167,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":172,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":171,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":166,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":164,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":158,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":165,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":159,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":156,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":153,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":162,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":155,"metricType":"sum"},{"name":"records read","accumulatorId":152,"metricType":"sum"},{"name":"local bytes read","accumulatorId":150,"metricType":"size"},{"name":"fetch wait time","accumulatorId":151,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":148,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":154,"metricType":"sum"},{"name":"local blocks read","accumulatorId":147,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":157,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":146,"metricType":"sum"},{"name":"data size","accumulatorId":144,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":160,"metricType":"size"},{"name":"number of partitions","accumulatorId":145,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":161,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":149,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":163,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":213,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":214,"metricType":"timing"},{"name":"peak memory","accumulatorId":212,"metricType":"size"},{"name":"number of output rows","accumulatorId":211,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":216,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":215,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":210,"metricType":"timing"}]}],"metadata":{},"metrics":[]}} +{"Event":"SparkListenerJobStart","Job ID":3,"Submission Time":1750983967658,"Stage Infos":[{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":17,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"FileScanRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"26\",\"name\":\"mapPartitionsInternal\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"25\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[3,4],"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750983884577","spark.rdd.scope":"{\"id\":\"27\",\"name\":\"collect\"}","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_00_24_44","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.122.12","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-3a9e8397aec45f6f-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-110be3a8424d4a2789cb88134418217b","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","__fetch_continuous_blocks_in_batch_enabled":"true","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"1","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.122.12","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750983883971","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"1","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.rdd.scope.noOverride":"true","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"26\",\"name\":\"mapPartitionsInternal\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"25\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750983967663,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750983884577","spark.rdd.scope":"{\"id\":\"27\",\"name\":\"collect\"}","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_00_24_44","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.122.12","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-3a9e8397aec45f6f-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-110be3a8424d4a2789cb88134418217b","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","resource.executor.cores":"1","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","__fetch_continuous_blocks_in_batch_enabled":"true","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"1","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.122.12","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750983883971","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"1","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.rdd.scope.noOverride":"true","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750983967734,"Executor ID":"3","Host":"100.64.216.237","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750983967734,"Executor ID":"3","Host":"100.64.216.237","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983967899,"Failed":false,"Killed":false,"Accumulables":[{"ID":146,"Name":"remote blocks read","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":147,"Name":"local blocks read","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":148,"Name":"remote bytes read","Update":"590","Value":"590","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":150,"Name":"local bytes read","Update":"413","Value":"413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":151,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":152,"Name":"records read","Update":"17","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":161,"Name":"remote reqs duration","Update":"37","Value":"37","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":210,"Name":"duration","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":211,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":214,"Name":"time in aggregation build","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":217,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":218,"Name":"internal.metrics.executorDeserializeCpuTime","Update":11921976,"Value":11921976,"Internal":true,"Count Failed Values":true},{"ID":219,"Name":"internal.metrics.executorRunTime","Update":137,"Value":137,"Internal":true,"Count Failed Values":true},{"ID":220,"Name":"internal.metrics.executorCpuTime","Update":71635215,"Value":71635215,"Internal":true,"Count Failed Values":true},{"ID":221,"Name":"internal.metrics.resultSize","Update":3995,"Value":3995,"Internal":true,"Count Failed Values":true},{"ID":228,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":10,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":229,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":230,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":590,"Value":590,"Internal":true,"Count Failed Values":true},{"ID":231,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":232,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":413,"Value":413,"Internal":true,"Count Failed Values":true},{"ID":233,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":234,"Name":"internal.metrics.shuffle.read.recordsRead","Update":17,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":235,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":236,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":237,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":238,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":239,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":240,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":241,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":242,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":243,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":37,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":244,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":11921976,"Executor Run Time":137,"Executor CPU Time":71635215,"Peak Execution Memory":0,"Result Size":3995,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":10,"Local Blocks Fetched":7,"Fetch Wait Time":0,"Remote Bytes Read":590,"Remote Bytes Read To Disk":0,"Local Bytes Read":413,"Total Records Read":17,"Remote Requests Duration":37,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"26\",\"name\":\"mapPartitionsInternal\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"25\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750983967663,"Completion Time":1750983967901,"Accumulables":[{"ID":146,"Name":"remote blocks read","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":147,"Name":"local blocks read","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":148,"Name":"remote bytes read","Value":"590","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":150,"Name":"local bytes read","Value":"413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":151,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":152,"Name":"records read","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":161,"Name":"remote reqs duration","Value":"37","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":210,"Name":"duration","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":211,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":214,"Name":"time in aggregation build","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":217,"Name":"internal.metrics.executorDeserializeTime","Value":18,"Internal":true,"Count Failed Values":true},{"ID":218,"Name":"internal.metrics.executorDeserializeCpuTime","Value":11921976,"Internal":true,"Count Failed Values":true},{"ID":219,"Name":"internal.metrics.executorRunTime","Value":137,"Internal":true,"Count Failed Values":true},{"ID":220,"Name":"internal.metrics.executorCpuTime","Value":71635215,"Internal":true,"Count Failed Values":true},{"ID":221,"Name":"internal.metrics.resultSize","Value":3995,"Internal":true,"Count Failed Values":true},{"ID":228,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":10,"Internal":true,"Count Failed Values":true},{"ID":229,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":7,"Internal":true,"Count Failed Values":true},{"ID":230,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":590,"Internal":true,"Count Failed Values":true},{"ID":231,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":232,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":413,"Internal":true,"Count Failed Values":true},{"ID":233,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":234,"Name":"internal.metrics.shuffle.read.recordsRead","Value":17,"Internal":true,"Count Failed Values":true},{"ID":235,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":236,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":237,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":238,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":239,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":240,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":241,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":242,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":243,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":37,"Internal":true,"Count Failed Values":true},{"ID":244,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":3,"Completion Time":1750983967901,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1750983967903,"errorMessage":""} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":2,"rootExecutionId":2,"description":"parquet at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (6)\n+- Execute InsertIntoHadoopFsRelationCommand (5)\n +- WriteFiles (4)\n +- Exchange (3)\n +- Project (2)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct\n\n(2) Project\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 00:25:56.642038 AS current_date#38]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(3) Exchange\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=65]\n\n(4) WriteFiles\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\n\n(5) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]\n\n(6) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"Exchange","simpleString":"Exchange RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=65]","children":[{"nodeName":"Project","simpleString":"Project [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 00:25:56.642038 AS current_date#38]","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [VendorID#0L,tpep_pickup_datetime#1,tpep_dropoff_datetime#2,passenger_count#3,trip_distance#4,RatecodeID#5,store_and_fwd_flag#6,PULocationID#7L,DOLocationID#8L,payment_type#9L,fare_amount#10,extra#11,mta_tax#12,tip_amount#13,tolls_amount#14,improvement_surcharge#15,total_amount#16,congestion_surcharge#17,airport_fee#18] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":282,"metricType":"sum"},{"name":"scan time","accumulatorId":281,"metricType":"timing"},{"name":"metadata time","accumulatorId":283,"metricType":"timing"},{"name":"size of files read","accumulatorId":284,"metricType":"size"},{"name":"number of output rows","accumulatorId":280,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":278,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":272,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":279,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":273,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":270,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":267,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":276,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":269,"metricType":"sum"},{"name":"records read","accumulatorId":266,"metricType":"sum"},{"name":"local bytes read","accumulatorId":264,"metricType":"size"},{"name":"fetch wait time","accumulatorId":265,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":262,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":268,"metricType":"sum"},{"name":"local blocks read","accumulatorId":261,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":271,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":260,"metricType":"sum"},{"name":"data size","accumulatorId":258,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":274,"metricType":"size"},{"name":"number of partitions","accumulatorId":259,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":275,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":263,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":277,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":256,"metricType":"timing"},{"name":"number of written files","accumulatorId":252,"metricType":"sum"},{"name":"job commit time","accumulatorId":257,"metricType":"timing"},{"name":"number of output rows","accumulatorId":254,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":255,"metricType":"sum"},{"name":"written output","accumulatorId":253,"metricType":"size"}]}],"metadata":{},"metrics":[]},"time":1750983967966,"modifiedConfigs":{},"jobTags":[]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (12)\n+- == Current Plan ==\n Execute InsertIntoHadoopFsRelationCommand (7)\n +- WriteFiles (6)\n +- ShuffleQueryStage (5)\n +- Exchange (4)\n +- * Project (3)\n +- * ColumnarToRow (2)\n +- Scan parquet (1)\n+- == Initial Plan ==\n Execute InsertIntoHadoopFsRelationCommand (11)\n +- WriteFiles (10)\n +- Exchange (9)\n +- Project (8)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct\n\n(2) ColumnarToRow [codegen id : 1]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(3) Project [codegen id : 1]\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 00:25:56.642038 AS current_date#38]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(4) Exchange\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=80]\n\n(5) ShuffleQueryStage\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: 0\n\n(6) WriteFiles\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\n\n(7) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]\n\n(8) Project\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 00:25:56.642038 AS current_date#38]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(9) Exchange\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=65]\n\n(10) WriteFiles\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\n\n(11) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]\n\n(12) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=80]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 00:25:56.642038 AS current_date#38]","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [VendorID#0L,tpep_pickup_datetime#1,tpep_dropoff_datetime#2,passenger_count#3,trip_distance#4,RatecodeID#5,store_and_fwd_flag#6,PULocationID#7L,DOLocationID#8L,payment_type#9L,fare_amount#10,extra#11,mta_tax#12,tip_amount#13,tolls_amount#14,improvement_surcharge#15,total_amount#16,congestion_surcharge#17,airport_fee#18] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":282,"metricType":"sum"},{"name":"scan time","accumulatorId":281,"metricType":"timing"},{"name":"metadata time","accumulatorId":283,"metricType":"timing"},{"name":"size of files read","accumulatorId":284,"metricType":"size"},{"name":"number of output rows","accumulatorId":280,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":308,"metricType":"sum"},{"name":"number of input batches","accumulatorId":309,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":307,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":305,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":299,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":306,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":300,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":297,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":294,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":303,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":296,"metricType":"sum"},{"name":"records read","accumulatorId":293,"metricType":"sum"},{"name":"local bytes read","accumulatorId":291,"metricType":"size"},{"name":"fetch wait time","accumulatorId":292,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":289,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":295,"metricType":"sum"},{"name":"local blocks read","accumulatorId":288,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":298,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":287,"metricType":"sum"},{"name":"data size","accumulatorId":285,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":301,"metricType":"size"},{"name":"number of partitions","accumulatorId":286,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":302,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":290,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":304,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":256,"metricType":"timing"},{"name":"number of written files","accumulatorId":252,"metricType":"sum"},{"name":"job commit time","accumulatorId":257,"metricType":"timing"},{"name":"number of output rows","accumulatorId":254,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":255,"metricType":"sum"},{"name":"written output","accumulatorId":253,"metricType":"size"}]}],"metadata":{},"metrics":[]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[282,51],[283,0],[284,1945137399]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[286,2]]} +{"Event":"SparkListenerJobStart","Job ID":4,"Submission Time":1750983968128,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":17,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"36\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"FileScanRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[5],"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750983884577","spark.rdd.scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_00_24_44","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.122.12","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-3a9e8397aec45f6f-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-110be3a8424d4a2789cb88134418217b","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","__fetch_continuous_blocks_in_batch_enabled":"true","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"2","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.122.12","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750983883971","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"2","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.rdd.scope.noOverride":"true","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":17,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"36\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"FileScanRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750983968131,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750983884577","spark.rdd.scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_00_24_44","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.122.12","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-3a9e8397aec45f6f-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-110be3a8424d4a2789cb88134418217b","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","resource.executor.cores":"1","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","__fetch_continuous_blocks_in_batch_enabled":"true","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"2","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.122.12","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750983883971","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"2","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.rdd.scope.noOverride":"true","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750983968139,"Executor ID":"2","Host":"100.64.162.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1750983968140,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1750983968140,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1750983968140,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":24,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1750983985001,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1750983968140,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983985002,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"7391793","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3159","Value":"3159","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"1299239496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"316901240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"7391793","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"300151938","Value":"300151938","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"7225","Value":"7225","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"7391793","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"1806","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9255877,"Value":9255877,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":16836,"Value":16836,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":14874484338,"Value":14874484338,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2079,"Value":2079,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":54,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":316901240,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":7391793,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":300151938,"Value":300151938,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":114444762,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":7391793,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2113392264,"JVMOffHeapMemory":119347528,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1490613,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1612103029,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12655787,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10242043904,"ProcessTreeJVMRSSMemory":3120824320,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":13,"MinorGCTime":79,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":79},"Task Metrics":{"Executor Deserialize Time":15,"Executor Deserialize CPU Time":9255877,"Executor Run Time":16836,"Executor CPU Time":14874484338,"Peak Execution Memory":0,"Result Size":2079,"JVM GC Time":54,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":300151938,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":25,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1750983985222,"Executor ID":"2","Host":"100.64.162.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750983968139,"Executor ID":"2","Host":"100.64.162.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983985223,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"14783586","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3172","Value":"6331","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"2598478992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"633802480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"14783586","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"304712736","Value":"604864674","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"7162","Value":"14387","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"14783586","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"3612","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":85,"Value":100,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":17874103,"Value":27129980,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":16989,"Value":33825,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":15536794336,"Value":30411278674,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2034,"Value":4113,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":49,"Value":103,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":633802480,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":14783586,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":304712736,"Value":604864674,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":228889524,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":14783586,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1999967792,"JVMOffHeapMemory":111721176,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":994380,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611606796,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10238988288,"ProcessTreeJVMRSSMemory":2777853952,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":11,"MinorGCTime":78,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":78},"Task Metrics":{"Executor Deserialize Time":85,"Executor Deserialize CPU Time":17874103,"Executor Run Time":16989,"Executor CPU Time":15536794336,"Peak Execution Memory":0,"Result Size":2034,"JVM GC Time":49,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":304712736,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":26,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1750983985725,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1750983968140,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983985726,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"22175379","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3740","Value":"10071","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"3897718488","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"950703720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"22175379","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"307133780","Value":"911998454","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"7679","Value":"22066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"22175379","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"5418","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":116,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10028892,"Value":37158872,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":17560,"Value":51385,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":15482399834,"Value":45893678508,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":6149,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":43,"Value":146,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":950703720,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":22175379,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":307133780,"Value":911998454,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":343334286,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":22175379,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2402648944,"JVMOffHeapMemory":115802296,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":994380,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611606796,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16848975,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10243186688,"ProcessTreeJVMRSSMemory":2942275584,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":10,"MinorGCTime":70,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":70},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":10028892,"Executor Run Time":17560,"Executor CPU Time":15482399834,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":43,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":307133780,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":27,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1750983985948,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1750983968140,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750983985949,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"29567172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"4102","Value":"14173","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"5196957984","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"1267604960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"29567172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"304614037","Value":"1216612491","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"7949","Value":"30015","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"29567172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"7224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":82,"Value":198,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":13453257,"Value":50612129,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":17717,"Value":69102,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":15329711362,"Value":61223389870,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2034,"Value":8183,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":45,"Value":191,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":1267604960,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":29567172,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":304614037,"Value":1216612491,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":457779048,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":29567172,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2085798272,"JVMOffHeapMemory":112242520,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":994380,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611606796,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10171875328,"ProcessTreeJVMRSSMemory":2642079744,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":10,"MinorGCTime":75,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":75},"Task Metrics":{"Executor Deserialize Time":82,"Executor Deserialize CPU Time":13453257,"Executor Run Time":17717,"Executor CPU Time":15329711362,"Peak Execution Memory":0,"Result Size":2034,"JVM GC Time":45,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":304614037,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":28,"Index":8,"Attempt":0,"Partition ID":8,"Launch Time":1750984000325,"Executor ID":"2","Host":"100.64.162.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":25,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1750983985222,"Executor ID":"2","Host":"100.64.162.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984000326,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"36958965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2619","Value":"16792","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"6496197480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"1584506200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"36958965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"308193923","Value":"1524806414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5614","Value":"35629","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"36958965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"9030","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":201,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3482510,"Value":54094639,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15090,"Value":84192,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13540205739,"Value":74763595609,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2077,"Value":10260,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":56,"Value":247,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":1584506200,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":36958965,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":308193923,"Value":1524806414,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":572223810,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":36958965,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2171052440,"JVMOffHeapMemory":114081112,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":994380,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611606796,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10238988288,"ProcessTreeJVMRSSMemory":3085496320,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":34,"MinorGCTime":132,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":132},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3482510,"Executor Run Time":15090,"Executor CPU Time":13540205739,"Peak Execution Memory":0,"Result Size":2077,"JVM GC Time":56,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":308193923,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":29,"Index":9,"Attempt":0,"Partition ID":9,"Launch Time":1750984000644,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":24,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1750983985001,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984000645,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"44350758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3317","Value":"20109","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"7795436976","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"1901407440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"44350758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"301290471","Value":"1826096885","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6169","Value":"41798","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"44350758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"10836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":203,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2544503,"Value":56639142,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15633,"Value":99825,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13310656811,"Value":88074252420,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2079,"Value":12339,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":50,"Value":297,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":1901407440,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":44350758,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":301290471,"Value":1826096885,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":686668572,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":44350758,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2112970368,"JVMOffHeapMemory":117475592,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1490613,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1612103029,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12655787,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10242043904,"ProcessTreeJVMRSSMemory":3164770304,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":36,"MinorGCTime":130,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":130},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2544503,"Executor Run Time":15633,"Executor CPU Time":13310656811,"Peak Execution Memory":0,"Result Size":2079,"JVM GC Time":50,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":301290471,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":30,"Index":10,"Attempt":0,"Partition ID":10,"Launch Time":1750984001623,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":27,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1750983985948,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984001624,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"51742551","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2739","Value":"22848","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"9094676472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"2218308680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"51742551","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"310630748","Value":"2136727633","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6153","Value":"47951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"51742551","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"12642","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":206,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3378610,"Value":60017752,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15665,"Value":115490,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":14065608393,"Value":102139860813,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2034,"Value":14373,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":10,"Value":307,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":2218308680,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":51742551,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":310630748,"Value":2136727633,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":801113334,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":51742551,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3483541984,"JVMOffHeapMemory":114642928,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":994380,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611606796,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10238984192,"ProcessTreeJVMRSSMemory":3885121536,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":13,"MinorGCTime":85,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":85},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3378610,"Executor Run Time":15665,"Executor CPU Time":14065608393,"Peak Execution Memory":0,"Result Size":2034,"JVM GC Time":10,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":310630748,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":31,"Index":11,"Attempt":0,"Partition ID":11,"Launch Time":1750984001909,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":26,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1750983985725,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984001910,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"59134344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3188","Value":"26036","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"10393915968","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"2535209920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"59134344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"316344291","Value":"2453071924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6539","Value":"54490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"59134344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"14448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":209,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3261660,"Value":63279412,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":16173,"Value":131663,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":14192907235,"Value":116332768048,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":16409,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":2,"Value":309,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":2535209920,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":59134344,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":316344291,"Value":2453071924,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":915558096,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":59134344,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4208296816,"JVMOffHeapMemory":117345872,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":994380,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611606796,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16848975,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10243186688,"ProcessTreeJVMRSSMemory":4461654016,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":10,"MinorGCTime":70,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":70},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3261660,"Executor Run Time":16173,"Executor CPU Time":14192907235,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":2,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":316344291,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":32,"Index":12,"Attempt":0,"Partition ID":12,"Launch Time":1750984015391,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":29,"Index":9,"Attempt":0,"Partition ID":9,"Launch Time":1750984000644,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984015392,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"66526137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2689","Value":"28725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"11693155464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"2852111160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"66526137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"297729827","Value":"2750801751","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5462","Value":"59952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"66526137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"16254","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":211,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2541019,"Value":65820431,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":14736,"Value":146399,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13025798192,"Value":129358566240,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2079,"Value":18488,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":51,"Value":360,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":2852111160,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":66526137,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":297729827,"Value":2750801751,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1030002858,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":66526137,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2143723296,"JVMOffHeapMemory":118471128,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1490613,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1612103029,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12655787,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10242043904,"ProcessTreeJVMRSSMemory":3173888000,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":59,"MinorGCTime":180,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":180},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2541019,"Executor Run Time":14736,"Executor CPU Time":13025798192,"Peak Execution Memory":0,"Result Size":2079,"JVM GC Time":51,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":297729827,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":33,"Index":13,"Attempt":0,"Partition ID":13,"Launch Time":1750984015909,"Executor ID":"2","Host":"100.64.162.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":28,"Index":8,"Attempt":0,"Partition ID":8,"Launch Time":1750984000325,"Executor ID":"2","Host":"100.64.162.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984015910,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"73917930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3400","Value":"32125","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"12992394960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"3169012400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"73917930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"300660967","Value":"3051462718","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6285","Value":"66237","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"73917930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"18060","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":214,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3115541,"Value":68935972,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15572,"Value":161971,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13183069114,"Value":142541635354,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2077,"Value":20565,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":59,"Value":419,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":3169012400,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":73917930,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":300660967,"Value":3051462718,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1144447620,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":73917930,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2077234928,"JVMOffHeapMemory":115214600,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":994380,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611606796,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10238988288,"ProcessTreeJVMRSSMemory":3245383680,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":61,"MinorGCTime":192,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":192},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3115541,"Executor Run Time":15572,"Executor CPU Time":13183069114,"Peak Execution Memory":0,"Result Size":2077,"JVM GC Time":59,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":300660967,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":34,"Index":14,"Attempt":0,"Partition ID":14,"Launch Time":1750984017059,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":31,"Index":11,"Attempt":0,"Partition ID":11,"Launch Time":1750984001909,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984017060,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"81309723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2941","Value":"35066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"14291634456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"3485913640","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"81309723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"313148923","Value":"3364611641","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5558","Value":"71795","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"81309723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"19866","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":216,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2577761,"Value":71513733,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15141,"Value":177112,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13242834777,"Value":155784470131,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":22601,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":5,"Value":424,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":3485913640,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":81309723,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":313148923,"Value":3364611641,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1258892382,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":81309723,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3668272480,"JVMOffHeapMemory":115251848,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":994380,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611606796,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16848975,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10243186688,"ProcessTreeJVMRSSMemory":4529758208,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":13,"MinorGCTime":77,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":77},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2577761,"Executor Run Time":15141,"Executor CPU Time":13242834777,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":5,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":313148923,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":35,"Index":15,"Attempt":0,"Partition ID":15,"Launch Time":1750984017642,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":30,"Index":10,"Attempt":0,"Partition ID":10,"Launch Time":1750984001623,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984017643,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"88701516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3573","Value":"38639","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"15590873952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"3802814880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"88701516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"326765098","Value":"3691376739","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6611","Value":"78406","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"88701516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"21672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":219,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3247257,"Value":74760990,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":16009,"Value":193121,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13596327482,"Value":169380797613,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2034,"Value":24635,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":10,"Value":434,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":3802814880,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":88701516,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":326765098,"Value":3691376739,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1373337144,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":88701516,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2777583120,"JVMOffHeapMemory":115656072,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":994380,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611606796,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10238984192,"ProcessTreeJVMRSSMemory":4602392576,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":16,"MinorGCTime":95,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":95},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3247257,"Executor Run Time":16009,"Executor CPU Time":13596327482,"Peak Execution Memory":0,"Result Size":2034,"JVM GC Time":10,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":326765098,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":36,"Index":16,"Attempt":0,"Partition ID":16,"Launch Time":1750984030478,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":32,"Index":12,"Attempt":0,"Partition ID":12,"Launch Time":1750984015391,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984030479,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"96093309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2988","Value":"41627","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"16890113448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"4119716120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"96093309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"303866776","Value":"3995243515","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5748","Value":"84154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"96093309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"23478","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":221,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2260274,"Value":77021264,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15075,"Value":208196,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13057237257,"Value":182438034870,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2079,"Value":26714,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":49,"Value":483,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":4119716120,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":96093309,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":303866776,"Value":3995243515,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1487781906,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":96093309,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2103793288,"JVMOffHeapMemory":119211544,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1490613,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1612103029,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12655787,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10242043904,"ProcessTreeJVMRSSMemory":3174260736,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":82,"MinorGCTime":230,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":230},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2260274,"Executor Run Time":15075,"Executor CPU Time":13057237257,"Peak Execution Memory":0,"Result Size":2079,"JVM GC Time":49,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":303866776,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":33,"Index":13,"Attempt":0,"Partition ID":13,"Launch Time":1750984015909,"Executor ID":"2","Host":"100.64.162.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984031175,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"103485102","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3123","Value":"44750","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"18189352944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"4436617360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"103485102","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"302530375","Value":"4297773890","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5896","Value":"90050","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"103485102","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"25284","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":224,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3141552,"Value":80162816,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15254,"Value":223450,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13132105345,"Value":195570140215,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2077,"Value":28791,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":52,"Value":535,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":4436617360,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":103485102,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":302530375,"Value":4297773890,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1602226668,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":103485102,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2146690800,"JVMOffHeapMemory":116176056,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":994380,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611606796,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10238988288,"ProcessTreeJVMRSSMemory":3249741824,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":86,"MinorGCTime":243,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":243},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3141552,"Executor Run Time":15254,"Executor CPU Time":13132105345,"Peak Execution Memory":0,"Result Size":2077,"JVM GC Time":52,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":302530375,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":34,"Index":14,"Attempt":0,"Partition ID":14,"Launch Time":1750984017059,"Executor ID":"1","Host":"100.64.221.138","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984033070,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"110876895","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3775","Value":"48525","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"19488592440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"4753518600","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"110876895","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"314774457","Value":"4612548347","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6413","Value":"96463","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"110876895","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"27090","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":226,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2604985,"Value":82767801,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":16002,"Value":239452,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13301782422,"Value":208871922637,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":30827,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":8,"Value":543,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":4753518600,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":110876895,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":314774457,"Value":4612548347,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1716671430,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":110876895,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2965690248,"JVMOffHeapMemory":116167072,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":994380,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611606796,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16848975,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10243186688,"ProcessTreeJVMRSSMemory":4566544384,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":16,"MinorGCTime":85,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":85},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2604985,"Executor Run Time":16002,"Executor CPU Time":13301782422,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":8,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":314774457,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":35,"Index":15,"Attempt":0,"Partition ID":15,"Launch Time":1750984017642,"Executor ID":"4","Host":"100.64.138.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984034475,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"118268688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3163","Value":"51688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"20787831936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"5070419840","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"118268688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"325902136","Value":"4938450483","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6141","Value":"102604","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"118268688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"28896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":228,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2912362,"Value":85680163,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":16823,"Value":256275,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":14778063455,"Value":223649986092,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2034,"Value":32861,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":7,"Value":550,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":5070419840,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":118268688,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":325902136,"Value":4938450483,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1831116192,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":118268688,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3423505936,"JVMOffHeapMemory":116099424,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":994380,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611606796,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10238984192,"ProcessTreeJVMRSSMemory":4638560256,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":19,"MinorGCTime":102,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":102},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2912362,"Executor Run Time":16823,"Executor CPU Time":14778063455,"Peak Execution Memory":0,"Result Size":2034,"JVM GC Time":7,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":325902136,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":36,"Index":16,"Attempt":0,"Partition ID":16,"Launch Time":1750984030478,"Executor ID":"3","Host":"100.64.216.237","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984045590,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3454","Value":"55142","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"22087071432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316901240","Value":"5387321080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"303168742","Value":"5241619225","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6183","Value":"108787","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"30702","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":230,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2525172,"Value":88205335,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15101,"Value":271376,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":12655052060,"Value":236305038152,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2079,"Value":34940,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":50,"Value":600,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316901240,"Value":5387321080,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":125660481,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":303168742,"Value":5241619225,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1945560954,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":125660481,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2133403784,"JVMOffHeapMemory":119905064,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1490613,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1612103029,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12655787,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10242043904,"ProcessTreeJVMRSSMemory":3191775232,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":106,"MinorGCTime":279,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":279},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2525172,"Executor Run Time":15101,"Executor CPU Time":12655052060,"Peak Execution Memory":0,"Result Size":2079,"JVM GC Time":50,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316901240,"Shuffle Write Time":303168742,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":17,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"36\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"FileScanRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750983968131,"Completion Time":1750984045591,"Accumulables":[{"ID":280,"Name":"number of output rows","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Value":"55142","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Value":"22087071432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Value":"5387321080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Value":"5241619225","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Value":"108787","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Value":"30702","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Value":230,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Value":88205335,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Value":271376,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Value":236305038152,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Value":34940,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Value":600,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Value":16,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":5387321080,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":125660481,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Value":5241619225,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Value":1945560954,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Value":125660481,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":4,"Completion Time":1750984045592,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (12)\n+- == Final Plan ==\n Execute InsertIntoHadoopFsRelationCommand (7)\n +- WriteFiles (6)\n +- ShuffleQueryStage (5), Statistics(sizeInBytes=20.6 GiB, rowCount=1.26E+8)\n +- Exchange (4)\n +- * Project (3)\n +- * ColumnarToRow (2)\n +- Scan parquet (1)\n+- == Initial Plan ==\n Execute InsertIntoHadoopFsRelationCommand (11)\n +- WriteFiles (10)\n +- Exchange (9)\n +- Project (8)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct\n\n(2) ColumnarToRow [codegen id : 1]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(3) Project [codegen id : 1]\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 00:25:56.642038 AS current_date#38]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(4) Exchange\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=80]\n\n(5) ShuffleQueryStage\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: 0\n\n(6) WriteFiles\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\n\n(7) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]\n\n(8) Project\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 00:25:56.642038 AS current_date#38]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(9) Exchange\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=65]\n\n(10) WriteFiles\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\n\n(11) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]\n\n(12) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=80]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 00:25:56.642038 AS current_date#38]","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [VendorID#0L,tpep_pickup_datetime#1,tpep_dropoff_datetime#2,passenger_count#3,trip_distance#4,RatecodeID#5,store_and_fwd_flag#6,PULocationID#7L,DOLocationID#8L,payment_type#9L,fare_amount#10,extra#11,mta_tax#12,tip_amount#13,tolls_amount#14,improvement_surcharge#15,total_amount#16,congestion_surcharge#17,airport_fee#18] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":282,"metricType":"sum"},{"name":"scan time","accumulatorId":281,"metricType":"timing"},{"name":"metadata time","accumulatorId":283,"metricType":"timing"},{"name":"size of files read","accumulatorId":284,"metricType":"size"},{"name":"number of output rows","accumulatorId":280,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":308,"metricType":"sum"},{"name":"number of input batches","accumulatorId":309,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":307,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":305,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":299,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":306,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":300,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":297,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":294,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":303,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":296,"metricType":"sum"},{"name":"records read","accumulatorId":293,"metricType":"sum"},{"name":"local bytes read","accumulatorId":291,"metricType":"size"},{"name":"fetch wait time","accumulatorId":292,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":289,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":295,"metricType":"sum"},{"name":"local blocks read","accumulatorId":288,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":298,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":287,"metricType":"sum"},{"name":"data size","accumulatorId":285,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":301,"metricType":"size"},{"name":"number of partitions","accumulatorId":286,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":302,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":290,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":304,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":349,"metricType":"timing"},{"name":"number of written files","accumulatorId":345,"metricType":"sum"},{"name":"job commit time","accumulatorId":350,"metricType":"timing"},{"name":"number of output rows","accumulatorId":347,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":348,"metricType":"sum"},{"name":"written output","accumulatorId":346,"metricType":"size"}]}],"metadata":{},"metrics":[]}} +{"Event":"SparkListenerJobStart","Job ID":5,"Submission Time":1750984046008,"Stage Infos":[{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":17,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"36\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"FileScanRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":17,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":2,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"41\",\"name\":\"WriteFiles\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"43\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[6,7],"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750983884577","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_00_24_44","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.122.12","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-3a9e8397aec45f6f-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-110be3a8424d4a2789cb88134418217b","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","__fetch_continuous_blocks_in_batch_enabled":"true","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"2","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.122.12","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750983883971","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"2","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":2,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"41\",\"name\":\"WriteFiles\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"43\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750984046009,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750983884577","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_00_24_44","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.122.12","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-3a9e8397aec45f6f-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-110be3a8424d4a2789cb88134418217b","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","resource.executor.cores":"1","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","__fetch_continuous_blocks_in_batch_enabled":"true","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"2","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.122.12","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750983883971","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"1ffa6acb-7695-4ec6-8790-c2f8a4f761f1","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"2","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerTaskStart","Stage ID":7,"Stage Attempt ID":0,"Task Info":{"Task ID":37,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750984046038,"Executor ID":"4","Host":"100.64.138.5","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":7,"Stage Attempt ID":0,"Task Info":{"Task ID":38,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1750984046039,"Executor ID":"3","Host":"100.64.216.237","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":7,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":38,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1750984046039,"Executor ID":"3","Host":"100.64.216.237","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984170092,"Failed":false,"Killed":false,"Accumulables":[{"ID":287,"Name":"remote blocks read","Update":"12","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":288,"Name":"local blocks read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":289,"Name":"remote bytes read","Update":"1901407860","Value":"1901407860","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":291,"Name":"local bytes read","Update":"792254150","Value":"792254150","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":292,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":293,"Name":"records read","Update":"62830237","Value":"62830237","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":302,"Name":"remote reqs duration","Update":"1682","Value":"1682","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":349,"Name":"task commit time","Update":"2214","Value":"2214","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":351,"Name":"internal.metrics.executorDeserializeTime","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.executorDeserializeCpuTime","Update":37593188,"Value":37593188,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.executorRunTime","Update":123995,"Value":123995,"Internal":true,"Count Failed Values":true},{"ID":354,"Name":"internal.metrics.executorCpuTime","Update":120789382453,"Value":120789382453,"Internal":true,"Count Failed Values":true},{"ID":355,"Name":"internal.metrics.resultSize","Update":4826,"Value":4826,"Internal":true,"Count Failed Values":true},{"ID":356,"Name":"internal.metrics.jvmGCTime","Update":157,"Value":157,"Internal":true,"Count Failed Values":true},{"ID":357,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":362,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":363,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":364,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":1901407860,"Value":1901407860,"Internal":true,"Count Failed Values":true},{"ID":365,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":366,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":792254150,"Value":792254150,"Internal":true,"Count Failed Values":true},{"ID":367,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":368,"Name":"internal.metrics.shuffle.read.recordsRead","Update":62830237,"Value":62830237,"Internal":true,"Count Failed Values":true},{"ID":369,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":370,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":371,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":372,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":373,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":374,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":375,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":376,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":377,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":1682,"Value":1682,"Internal":true,"Count Failed Values":true},{"ID":378,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":384,"Name":"internal.metrics.output.bytesWritten","Update":1351323194,"Value":1351323194,"Internal":true,"Count Failed Values":true},{"ID":385,"Name":"internal.metrics.output.recordsWritten","Update":62830237,"Value":62830237,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3603929104,"JVMOffHeapMemory":134197400,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1835751,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1835751,"OffHeapUnifiedMemory":0,"DirectPoolMemory":330459240,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10580037632,"ProcessTreeJVMRSSMemory":5095907328,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":125,"MinorGCTime":432,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":432},"Task Metrics":{"Executor Deserialize Time":45,"Executor Deserialize CPU Time":37593188,"Executor Run Time":123995,"Executor CPU Time":120789382453,"Peak Execution Memory":0,"Result Size":4826,"JVM GC Time":157,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":12,"Local Blocks Fetched":5,"Fetch Wait Time":0,"Remote Bytes Read":1901407860,"Remote Bytes Read To Disk":0,"Local Bytes Read":792254150,"Total Records Read":62830237,"Remote Requests Duration":1682,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":1351323194,"Records Written":62830237},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":7,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":37,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750984046038,"Executor ID":"4","Host":"100.64.138.5","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750984182934,"Failed":false,"Killed":false,"Accumulables":[{"ID":287,"Name":"remote blocks read","Update":"13","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":288,"Name":"local blocks read","Update":"4","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":289,"Name":"remote bytes read","Update":"2059856590","Value":"3961264450","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":291,"Name":"local bytes read","Update":"633802480","Value":"1426056630","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":292,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":293,"Name":"records read","Update":"62830244","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":302,"Name":"remote reqs duration","Update":"1629","Value":"3311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":349,"Name":"task commit time","Update":"2819","Value":"5033","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":351,"Name":"internal.metrics.executorDeserializeTime","Update":55,"Value":100,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.executorDeserializeCpuTime","Update":48630909,"Value":86224097,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.executorRunTime","Update":136830,"Value":260825,"Internal":true,"Count Failed Values":true},{"ID":354,"Name":"internal.metrics.executorCpuTime","Update":133145802304,"Value":253935184757,"Internal":true,"Count Failed Values":true},{"ID":355,"Name":"internal.metrics.resultSize","Update":4826,"Value":9652,"Internal":true,"Count Failed Values":true},{"ID":356,"Name":"internal.metrics.jvmGCTime","Update":96,"Value":253,"Internal":true,"Count Failed Values":true},{"ID":357,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":362,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":13,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":363,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":4,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":364,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":2059856590,"Value":3961264450,"Internal":true,"Count Failed Values":true},{"ID":365,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":366,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":633802480,"Value":1426056630,"Internal":true,"Count Failed Values":true},{"ID":367,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":368,"Name":"internal.metrics.shuffle.read.recordsRead","Update":62830244,"Value":125660481,"Internal":true,"Count Failed Values":true},{"ID":369,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":370,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":371,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":372,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":373,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":374,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":375,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":376,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":377,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":1629,"Value":3311,"Internal":true,"Count Failed Values":true},{"ID":378,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":384,"Name":"internal.metrics.output.bytesWritten","Update":1351572967,"Value":2702896161,"Internal":true,"Count Failed Values":true},{"ID":385,"Name":"internal.metrics.output.recordsWritten","Update":62830244,"Value":125660481,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2764891600,"JVMOffHeapMemory":130704632,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1339518,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1339518,"OffHeapUnifiedMemory":0,"DirectPoolMemory":330378149,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10580033536,"ProcessTreeJVMRSSMemory":5133750272,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":33,"MinorGCTime":198,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":198},"Task Metrics":{"Executor Deserialize Time":55,"Executor Deserialize CPU Time":48630909,"Executor Run Time":136830,"Executor CPU Time":133145802304,"Peak Execution Memory":0,"Result Size":4826,"JVM GC Time":96,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":13,"Local Blocks Fetched":4,"Fetch Wait Time":0,"Remote Bytes Read":2059856590,"Remote Bytes Read To Disk":0,"Local Bytes Read":633802480,"Total Records Read":62830244,"Remote Requests Duration":1629,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":1351572967,"Records Written":62830244},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":2,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"41\",\"name\":\"WriteFiles\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"43\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750984046009,"Completion Time":1750984182935,"Accumulables":[{"ID":287,"Name":"remote blocks read","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":288,"Name":"local blocks read","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":289,"Name":"remote bytes read","Value":"3961264450","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":291,"Name":"local bytes read","Value":"1426056630","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":292,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":293,"Name":"records read","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":302,"Name":"remote reqs duration","Value":"3311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":349,"Name":"task commit time","Value":"5033","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":351,"Name":"internal.metrics.executorDeserializeTime","Value":100,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.executorDeserializeCpuTime","Value":86224097,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.executorRunTime","Value":260825,"Internal":true,"Count Failed Values":true},{"ID":354,"Name":"internal.metrics.executorCpuTime","Value":253935184757,"Internal":true,"Count Failed Values":true},{"ID":355,"Name":"internal.metrics.resultSize","Value":9652,"Internal":true,"Count Failed Values":true},{"ID":356,"Name":"internal.metrics.jvmGCTime","Value":253,"Internal":true,"Count Failed Values":true},{"ID":357,"Name":"internal.metrics.resultSerializationTime","Value":4,"Internal":true,"Count Failed Values":true},{"ID":362,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":25,"Internal":true,"Count Failed Values":true},{"ID":363,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":9,"Internal":true,"Count Failed Values":true},{"ID":364,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":3961264450,"Internal":true,"Count Failed Values":true},{"ID":365,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":366,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1426056630,"Internal":true,"Count Failed Values":true},{"ID":367,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":368,"Name":"internal.metrics.shuffle.read.recordsRead","Value":125660481,"Internal":true,"Count Failed Values":true},{"ID":369,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":370,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":371,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":372,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":373,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":374,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":375,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":376,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":377,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":3311,"Internal":true,"Count Failed Values":true},{"ID":378,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":384,"Name":"internal.metrics.output.bytesWritten","Value":2702896161,"Internal":true,"Count Failed Values":true},{"ID":385,"Name":"internal.metrics.output.recordsWritten","Value":125660481,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":5,"Completion Time":1750984182935,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[345,2],[350,5548],[347,125660481],[348,0],[346,2702896161]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":2,"time":1750984188491,"errorMessage":""} +{"Event":"SparkListenerApplicationEnd","Timestamp":1750984188493} diff --git a/examples/basic/events/eventlog_v2_spark-cc4d115f011443d787f03a71a476a745/appstatus_spark-cc4d115f011443d787f03a71a476a745 b/examples/basic/events/eventlog_v2_spark-cc4d115f011443d787f03a71a476a745/appstatus_spark-cc4d115f011443d787f03a71a476a745 new file mode 100644 index 0000000..e69de29 diff --git a/examples/basic/events/eventlog_v2_spark-cc4d115f011443d787f03a71a476a745/events_1_spark-cc4d115f011443d787f03a71a476a745 b/examples/basic/events/eventlog_v2_spark-cc4d115f011443d787f03a71a476a745/events_1_spark-cc4d115f011443d787f03a71a476a745 new file mode 100644 index 0000000..eb333d4 --- /dev/null +++ b/examples/basic/events/eventlog_v2_spark-cc4d115f011443d787f03a71a476a745/events_1_spark-cc4d115f011443d787f03a71a476a745 @@ -0,0 +1,200 @@ +{"Event":"SparkListenerLogStart","Spark Version":"3.5.3"} +{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"memoryOverhead":{"Resource Name":"memoryOverhead","Amount":4096,"Discovery Script":"","Vendor":""},"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":4096,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"taxi-trip-77d59097af8544ae-driver-svc.spark-team-a.svc","Port":7079},"Maximum Memory":2388236697,"Timestamp":1750996614851,"Maximum Onheap Memory":2388236697,"Maximum Offheap Memory":0} +{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/opt/java/openjdk","Java Version":"17.0.12 (Eclipse Adoptium)","Scala Version":"version 2.12.18"},"Spark Properties":{"spark.speculation":"false","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.kubernetes.submission.waitAppCompletion":"false","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.host":"taxi-trip-77d59097af8544ae-driver-svc.spark-team-a.svc","spark.serializer.objectStreamReset":"100","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.kubernetes.namespace":"spark-team-a","spark.eventLog.enabled":"true","spark.executor.memoryOverhead":"4g","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.63.89","spark.driver.port":"7078","spark.rdd.compress":"True","spark.kubernetes.executor.label.version":"3.5.3","spark.driver.blockManager.port":"7079","spark.hadoop.fs.s3a.path.style.access":"true","spark.app.name":"NewYorkTaxiData_2025_06_27_03_56_52","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.network.timeout":"2400","spark.hadoop.fs.s3a.connection.maximum":"200","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.scheduler.mode":"FIFO","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.memory":"4g","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.executor.instances":"4","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.submit.pyFiles":"","spark.app.submitTime":"1750996611643","spark.driver.memoryOverhead":"4g","spark.kubernetes.memoryOverheadFactor":"0.4","spark.driver.bindAddress":"100.64.63.89","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.kubernetes.resource.type":"python","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.app.startTime":"1750996612238","spark.executor.id":"driver","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.limit.cores":"3400m","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.driver.cores":"1","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.driver.limit.cores":"1200m","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.executor.label.app":"taxi-trip","spark.ui.prometheus.enabled":"true","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.driver.label.queue":"root.test","spark.executor.memory":"4g","spark.local.dir":"/data","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.cores":"1","spark.executor.processTreeMetrics.enabled":"true","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.hadoop.fs.s3a.readahead.range":"256K","spark.app.id":"spark-cc4d115f011443d787f03a71a476a745","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.rolling.enabled":"true","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"200","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.viewfs.overload.scheme.target.o3fs.impl":"org.apache.hadoop.fs.ozone.OzoneFileSystem","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","yarn.resourcemanager.application-tag-based-placement.enable":"false","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min":"3600","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","yarn.nodemanager.node-attributes.resync-interval-ms":"120000","yarn.nodemanager.container-log-monitor.interval-ms":"60000","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","fs.viewfs.overload.scheme.target.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","mapreduce.job.acl-view-job":" ","fs.s3a.retry.limit":"7","mapreduce.jobhistory.loadedjobs.cache.size":"5","mapreduce.outputcommitter.factory.scheme.abfs":"org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory","yarn.router.interceptor.user-thread-pool.allow-core-thread-time-out":"false","yarn.log-aggregation.enable-local-cleanup":"true","fs.viewfs.overload.scheme.target.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.shuffle.pathcache.expire-after-access-minutes":"5","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes":"runc","fs.viewfs.overload.scheme.target.http.impl":"org.apache.hadoop.fs.http.HttpFileSystem","yarn.nodemanager.least-load-policy-selector.fail-on-error":"true","yarn.resourcemanager.nodemanagers.heartbeat-interval-slowdown-factor":"1.0","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.router.submit.interval.time":"10ms","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","yarn.router.interceptor.user-thread-pool.keep-alive-time":"30s","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","fs.s3a.connection.ttl":"5m","yarn.router.asc-interceptor-max-size":"1MB","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","yarn.nodemanager.opportunistic-containers-queue-policy":"BY_QUEUE_LEN","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.federation.gpg.policy.generator.load-based.pending.maximum":"1000","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.nodemanager.aux-services.manifest.enabled":"false","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","fs.s3a.select.input.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","yarn.dispatcher.print-thread-pool.core-pool-size":"1","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.federation.gpg.policy.generator.interval-ms":"3600000","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch":"false","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.federation.state-store.sql.idle-time-out":"10m","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","io.compression.codec.zstd.level":"3","yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed":"false","hadoop.metrics.jvm.use-thread-mxbean":"false","ipc.[port_number].faircallqueue.multiplexer.weights":"8,4,2,1","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","ipc.[port_number].callqueue.overflow.trigger.failover":"false","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","mapreduce.outputcommitter.factory.scheme.gs":"org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory","ha.health-monitor.rpc.connect.max.retries":"1","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable":"false","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.container-log-monitor.dir-size-limit-bytes":"1000000000","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.http.idle_timeout.ms":"60000","hadoop.registry.zk.retry.ceiling.ms":"60000","ipc.client.connection.idle-scan-interval.ms":"10000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","fs.viewfs.overload.scheme.target.hdfs.impl":"org.apache.hadoop.hdfs.DistributedFileSystem","seq.io.sort.mb":"100","fs.iostatistics.logging.level":"debug","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","ipc.[port_number].decay-scheduler.decay-factor":"0.5","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.webapp.filter-invalid-xml-chars":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-interval-secs":"600","fs.s3a.select.input.csv.record.delimiter":"\\n","fs.s3a.change.detection.source":"etag","ipc.[port_number].backoff.enable":"false","yarn.nodemanager.distributed-scheduling.enabled":"false","yarn.federation.cache.class":"org.apache.hadoop.yarn.server.federation.cache.FederationJCache","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","yarn.webapp.enable-rest-app-submissions":"true","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","ipc.server.metrics.update.runner.interval":"5000","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.runtime.linux.runc.image-toplevel-dir":"/runc-root","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","io.compression.codec.lz4.use.lz4hc":"false","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","yarn.nodemanager.runtime.linux.runc.allowed-container-networks":"host,none,bridge","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","ipc.server.reuseaddr":"true","fs.ftp.timeout":"0","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","fs.AbstractFileSystem.o3fs.impl":"org.apache.hadoop.fs.ozone.OzFs","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.resourcemanager.enable-node-untracked-without-include-path":"false","yarn.nodemanager.webapp.cross-origin.enabled":"false","yarn.federation.gpg.subcluster.heartbeat.expiration-ms":"30m","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","fs.viewfs.overload.scheme.target.swebhdfs.impl":"org.apache.hadoop.hdfs.web.SWebHdfsFileSystem","yarn.client.failover-no-ha-proxy-provider":"org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider","fs.s3a.change.detection.mode":"server","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","yarn.resourcemanager.submission-preprocessor.enabled":"false","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","ipc.[port_number].scheduler.impl":"org.apache.hadoop.ipc.DefaultRpcScheduler","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.timeline-service.writer.async.queue.capacity":"100","yarn.router.webapp.appsinfo-cached-count":"100","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"true","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","yarn.resourcemanager.nodemanagers.heartbeat-interval-speedup-factor":"1.0","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.nodemanager.least-load-policy-selector.enabled":"false","yarn.nodemanager.least-load-policy-selector.pending-container.threshold":"10000","hadoop.caller.context.signature.max.size":"40","ipc.[port_number].decay-scheduler.backoff.responsetime.enable":"false","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","ipc.[port_number].decay-scheduler.metrics.top.user.count":"10","tfile.io.chunk.size":"1048576","yarn.dispatcher.print-events-info.threshold":"5000","yarn.nodemanager.log-container-debug-info-on-error.enabled":"false","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.dispatcher.metric.enable":"false","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.num-manifests-to-cache":"10","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","fs.s3a.select.enabled":"true","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.webapp.ui1.tools.enable":"true","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","hadoop.http.sni.host.check.enabled":"false","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","fs.getspaceused.jitterMillis":"60000","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","ipc.scheduler.impl":"org.apache.hadoop.ipc.DefaultRpcScheduler","yarn.resourcemanager.zk-client-ssl.enabled":"false","io.skip.checksum.errors":"false","yarn.nodemanager.log.trigger.delete.by-size.enabled":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"1200000","yarn.app.mapreduce.am.webapp.https.enabled":"false","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","yarn.resourcemanager.delegation-token.always-cancel":"*********(redacted)","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.timeline-service.flowname.max-size":"0","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.application-tag-based-placement.force-lowercase":"true","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds":"10s,20s,30s,40s","fs.s3a.buffer.dir":"${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a","hadoop.ssl.enabled.protocols":"TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","ipc.server.max.response.size":"1048576","fs.s3a.accesspoint.required":"false","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.resourcemanager.proxy.connection.timeout":"60000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.resourcemanager.activities-manager.app-activities.max-queue-length":"100","yarn.resourcemanager.application-https.policy":"NONE","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","yarn.federation.gpg.application.cleaner.contact.router.spec":"3,10,600000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","yarn.workflow-id.tag-prefix":"workflowid:","fs.azure.local.sas.key.mode":"false","yarn.federation.gpg.policy.generator.class":"org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.NoOpGlobalPolicy","ipc.maximum.data.length":"134217728","fs.s3a.endpoint":"s3.amazonaws.com","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","ipc.server.tcpnodelay":"true","yarn.resourcemanager.resource-tracker.nm.ip-hostname-check":"false","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","yarn.router.webapp.appsinfo-enabled":"false","fs.AbstractFileSystem.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","fs.s3a.downgrade.syncable.exceptions":"true","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.resourcemanager.submission-preprocessor.file-refresh-interval-ms":"60000","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.resourcemanager.activities-manager.cleanup-interval-ms":"5000","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","ipc.[port_number].identity-provider.impl":"org.apache.hadoop.ipc.UserIdentityProvider","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","yarn.nodemanager.container-log-monitor.total-size-limit-bytes":"10000000000","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.federation.amrmproxy.register.uam.retry-count":"3","yarn.nodemanager.health-checker.scripts":"script","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","yarn.federation.cache-entity.nums":"1000","hadoop.security.resolver.impl":"org.apache.hadoop.net.DNSDomainNameResolver","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"96","yarn.nodemanager.runtime.linux.docker.image-update":"false","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","fs.viewfs.overload.scheme.target.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","yarn.dispatcher.cpu-monitor.samples-per-min":"60","hadoop.security.token.service.use_ip":"*********(redacted)","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.nodemanager.containers-launcher.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","io.compression.codec.lzo.buffersize":"65536","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ipc.callqueue.impl":"java.util.concurrent.LinkedBlockingQueue","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.server.purge.interval":"15","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.router.interceptor.user-thread-pool.maximum-pool-size":"5","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","fs.s3a.committer.abort.pending.uploads":"true","yarn.webapp.filter-entity-list-by-user":"false","yarn.resourcemanager.activities-manager.app-activities.ttl-ms":"600000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","yarn.federation.gpg.policy.generator.load-based.edit.maximum":"3","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.apps.cache.size":"1000","yarn.app.mapreduce.am.webapp.https.client.auth":"false","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","ipc.[port_number].decay-scheduler.thresholds":"13,25,50","yarn.federation.gpg.webapp.https.address":"0.0.0.0:8070","ipc.server.read.threadpool.size":"1","fs.s3a.audit.enabled":"true","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","ipc.server.read.connection-queue.size":"100","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","fs.viewfs.overload.scheme.target.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","ipc.cost-provider.impl":"org.apache.hadoop.ipc.DefaultCostProvider","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.viewfs.overload.scheme.target.file.impl":"org.apache.hadoop.fs.LocalFileSystem","yarn.federation.gpg.application.cleaner.interval-ms":"-1s","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.nodemanager.least-load-policy-selector.use-active-core":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.apps.cache.expire":"30s","ipc.server.log.slow.rpc.threshold.ms":"0","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.connection.establish.timeout":"30s","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.apps.cache.enable":"false","yarn.federation.non-ha.enabled":"false","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","yarn.federation.state-store.heartbeat.initial-delay":"30s","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","hadoop.security.group.mapping.ldap.num.attempts":"3","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","ipc.client.connect.max.retries.on.sasl":"5","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.viewfs.overload.scheme.target.webhdfs.impl":"org.apache.hadoop.hdfs.web.WebHdfsFileSystem","yarn.router.webapp.proxy.enable":"true","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.http.metrics.enabled":"true","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","fs.s3a.ssl.channel.mode":"default_jsse","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.resourcemanager.delegation.token.remove-scan-interval":"*********(redacted)","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","ipc.identity-provider.impl":"org.apache.hadoop.ipc.UserIdentityProvider","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.federation.amrmproxy.register.uam.interval":"100ms","yarn.resourcemanager.node-labels.provider.update-newly-registered-nodes-interval-ms":"30000","fs.s3a.change.detection.version.required":"true","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.fs-store.file.replication":"0","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.resourcemanager.proxy.timeout.enabled":"true","yarn.resourcemanager.activities-manager.scheduler-activities.ttl-ms":"600000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","yarn.nodemanager.disk-health-checker.disk-free-space-threshold.enabled":"true","io.compression.codec.snappy.buffersize":"262144","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","io.erasurecode.codec.native.enabled":"true","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","fs.AbstractFileSystem.ofs.impl":"org.apache.hadoop.fs.ozone.RootedOzFs","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","yarn.nodemanager.container-executor.exit-code-file.timeout-ms":"2000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.router.webapp.cross-origin.enabled":"false","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"20","fs.trash.clean.trashroot.enable":"false","hadoop.http.authentication.type":"simple","fs.viewfs.overload.scheme.target.oss.impl":"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem","yarn.federation.gpg.policy.generator.interval":"1h","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","ipc.[port_number].weighted-cost.handler":"1","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.federation.state-store.sql.minimum-idle":"1","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","yarn.federation.gpg.subcluster.cleaner.interval-ms":"-1ms","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","fs.s3a.select.output.csv.field.delimiter":",","yarn.nodemanager.health-checker.timeout-ms":"1200000","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","fs.s3a.select.output.csv.record.delimiter":"\\n","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","fs.viewfs.overload.scheme.target.https.impl":"org.apache.hadoop.fs.http.HttpsFileSystem","yarn.federation.gpg.policy.generator.readonly":"false","yarn.router.deregister.subcluster.enabled":"true","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","fs.viewfs.overload.scheme.target.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.federation.gpg.scheduled.executor.threads":"10","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","yarn.nodemanager.aux-services.%s.classpath":"NONE","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.federation.gpg.webapp.connect-timeout":"30s","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","yarn.nodemanager.runtime.linux.runc.allowed-container-runtimes":"runc","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"100ms","yarn.router.subcluster.cleaner.interval.time":"60s","yarn.nodemanager.log.delete.threshold":"100g","seq.io.sort.factor":"100","fs.viewfs.overload.scheme.target.ofs.impl":"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.resourcemanager.nodestore-rootdir.retry-interval-ms":"1000","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","hadoop.security.group.mapping.ldap.num.attempts.before.failover":"3","mapreduce.task.profile":"false","yarn.federation.gpg.application.cleaner.class":"org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner.DefaultApplicationCleaner","hadoop.prometheus.endpoint.enabled":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","fs.creation.parallel.count":"64","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.federation.state-store.sql.max-life-time":"30m","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","mapreduce.job.dfs.storage.capacity.kill-limit-exceed":"false","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","yarn.resourcemanager.opportunistic.max.container-allocation.per.am.heartbeat":"-1","fs.automatic.close":"true","yarn.resourcemanager.delegation-token-renewer.thread-retry-interval":"*********(redacted)","yarn.resourcemanager.node-labels.am.allow-non-exclusive-allocation":"false","fs.s3a.select.input.csv.quote.character":"\"","yarn.nodemanager.hostname":"0.0.0.0","ipc.[port_number].cost-provider.impl":"org.apache.hadoop.ipc.DefaultCostProvider","yarn.nodemanager.runtime.linux.runc.manifest-to-resources-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.HdfsManifestToResourcesPlugin","yarn.nodemanager.remote-app-log-dir-include-older":"true","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-to-keep":"100","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.resourcemanager.nodestore-rootdir.num-retries":"1000","ipc.[port_number].weighted-cost.lockshared":"10","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60s","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","fs.s3a.select.output.csv.quote.escape.character":"\\\\","yarn.dispatcher.print-thread-pool.maximum-pool-size":"5","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","hadoop.kerberos.keytab.login.autorenewal.enabled":"false","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","hadoop.caller.context.separator":",","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms":"1000","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","hadoop.security.crypto.codec.classes.sm4.ctr.nopadding":"org.apache.hadoop.crypto.OpensslSm4CtrCryptoCodec, org.apache.hadoop.crypto.JceSm4CtrCryptoCodec","yarn.federation.gpg.webapp.read-timeout":"30s","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.nodemanager.disk-health-checker.disk-utilization-threshold.enabled":"true","fs.s3a.executor.capacity":"16","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"true","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.application.max-tags":"10","hadoop.domainname.resolver.impl":"org.apache.hadoop.net.DNSDomainNameResolver","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","fs.azure.buffer.dir":"${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/abfs","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size":"1000","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","hadoop.security.secure.random.impl":"org.apache.hadoop.crypto.random.OpensslSecureRandom","mapreduce.job.running.reduce.limit":"0","fs.s3a.select.errors.include.sql":"false","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","yarn.federation.gpg.policy.generator.load-based.weight.minimum":"0","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","yarn.log-aggregation.debug.filesize":"104857600","yarn.dispatcher.print-thread-pool.keep-alive-time":"10s","yarn.router.subcluster.heartbeat.expiration.time":"30m","fs.s3a.max.total.tasks":"32","fs.s3a.readahead.range":"256K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.fast.upload":"true","fs.s3a.attempts.maximum":"5","yarn.federation.amrmproxy.allocation.history.max.entry":"100","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.resourcemanager.delegation-token-renewer.thread-timeout":"*********(redacted)","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.nodemanager.aux-services.manifest.reload-ms":"0","yarn.nodemanager.emit-container-events":"true","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","ha.failover-controller.active-standby-elector.zk.op.retries":"3","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.task.spill.files.count.limit":"-1","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","fs.viewfs.overload.scheme.target.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","yarn.federation.gpg.webapp.address":"0.0.0.0:8069","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","fs.s3a.input.fadvise":"random","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.hdfs-hash-file":"/runc-root/image-tag-to-hash","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.federation.failover.random.order":"false","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin","io.bytes.per.checksum":"512","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"auto","yarn.timeline-service.client.internal-timers-ttl-secs":"420","io.compression.codec.zstd.buffersize":"0","fs.s3a.select.output.csv.quote.character":"\"","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","ipc.callqueue.overflow.trigger.failover":"false","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","fs.s3a.select.output.csv.quote.fields":"always","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"120000","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","yarn.resourcemanager.delegation-token-renewer.thread-retry-max-attempts":"*********(redacted)","ipc.client.low-latency":"false","yarn.scheduler.skip.node.multiplier":"2","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"900000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-timeout-interval-secs":"360","fs.s3a.socket.recv.buffer":"8192","ipc.backoff.enable":"false","rpc.metrics.timeunit":"MILLISECONDS","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.nodemanager.pluggable-device-framework.enabled":"false","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","ipc.[port_number].callqueue.impl":"java.util.concurrent.LinkedBlockingQueue","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","ipc.[port_number].weighted-cost.lockfree":"1","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","mapreduce.jvm.add-opens-as-default":"true","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","fs.s3a.aws.credentials.provider":"*********(redacted)","yarn.log-aggregation.retain-seconds":"-1","yarn.router.interceptor.allow-partial-result.enable":"false","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","mapreduce.task.ping-for-liveliness-check.enabled":"false","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","fs.azure.enable.readahead":"true","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.federation.gpg.webapp.cross-origin.enabled":"false","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","io.compression.codec.lzo.class":"org.apache.hadoop.io.compress.LzoCodec","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","yarn.router.interceptor.user-thread-pool.minimum-pool-size":"5","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","yarn.federation.state-store.clean-up-retry-sleep-time":"1s","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"64M","fs.s3a.select.input.csv.comment.marker":"#","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.iostatistics.thread.level.enabled":"true","ipc.[port_number].weighted-cost.response":"1","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","mapreduce.task.stuck.timeout-ms":"600000","yarn.resourcemanager.application.max-tag.length":"100","yarn.resourcemanager.ha.enabled":"false","dfs.client.ignore.namenode.default.kms.uri":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","yarn.federation.state-store.sql.conn-time-out":"10s","io.compression.codec.lz4.buffersize":"262144","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","ipc.client.async.calls.max":"100","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms":"1000","hadoop.http.jmx.nan-filter.enabled":"false","yarn.router.scheduled.executor.threads":"1","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","fs.s3a.select.input.csv.field.delimiter":",","yarn.nodemanager.least-load-policy-selector.multiplier":"50000","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","yarn.federation.state-store.sql.pool-name":"YARN-Federation-DataBasePool","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"append","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","yarn.nodemanager.amrmproxy.wait.uam-register.done":"false","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","yarn.federation.gpg.policy.generator.load-based.pending.minimum":"100","ipc.[port_number].weighted-cost.lockexclusive":"100","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","yarn.nodemanager.container-log-monitor.enable":"false","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"24h","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"256","ipc.[port_number].decay-scheduler.period-ms":"5000","yarn.nodemanager.container-localizer.java.opts.add-exports-as-default":"true","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.cache-refresh-interval-secs":"60","map.sort.class":"org.apache.hadoop.util.QuickSort","yarn.federation.state-store.max-applications":"1000","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed":"false","ipc.server.handler.queue.size":"100","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds":"60","yarn.federation.registry.base-dir":"yarnfederation/","yarn.nodemanager.health-checker.run-before-startup":"false","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.shuffle.pathcache.concurrency-level":"16","mapreduce.job.ubertask.maxreduces":"1","mapreduce.shuffle.pathcache.max-weight":"10485760","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","fs.s3a.select.input.compression":"none","yarn.client.nodemanager-connect.retry-interval-ms":"10000","ipc.[port_number].scheduler.priority.levels":"4","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.federation.state-store.clean-up-retry-count":"1","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","fs.s3a.select.input.csv.header":"none","yarn.federation.gpg.policy.generator.load-based.scaling":"LINEAR","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-size":"500","yarn.timeline-service.webapp.rest-csrf.enabled":"false","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb":"0"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"17","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Eclipse Adoptium","java.vm.specification.version":"17","user.home":"/home/spark","sun.arch.data.model":"64","sun.boot.library.path":"/opt/java/openjdk/lib","user.dir":"/opt/spark","java.library.path":"/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib","os.arch":"amd64","java.vm.version":"17.0.12+7","jetty.git.hash":"cef3fbd6d736a21e7d541a5db490381d95a2047d","java.runtime.version":"17.0.12+7","java.vm.info":"mixed mode, sharing","java.runtime.name":"OpenJDK Runtime Environment","java.version.date":"2024-07-16","file.separator":"/","java.class.version":"61.0","java.specification.name":"Java Platform API Specification","file.encoding":"UTF-8","jdk.reflect.useDirectMethodHandle":"false","user.timezone":"Etc/UTC","kubernetes.request.retry.backoffLimit":"3","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","java.vm.compressedOopsMode":"Zero based","os.version":"6.1.140-154.222.amzn2023.x86_64","native.encoding":"UTF-8","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.version":"Temurin-17.0.12+7","java.vendor.url":"https://adoptium.net/","os.name":"Linux","java.vm.vendor":"Eclipse Adoptium","jdk.debug":"release","java.vendor.url.bug":"https://github.com/adoptium/adoptium-support/issues","user.name":"spark","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --deploy-mode client --conf spark.driver.bindAddress=100.64.63.89 --conf spark.executorEnv.SPARK_DRIVER_POD_IP=100.64.63.89 --properties-file /opt/spark/conf/spark.properties --class org.apache.spark.deploy.PythonRunner s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/scripts/pyspark-taxi-trip.py s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input/ s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/","java.home":"/opt/java/openjdk","java.version":"17.0.12","sun.io.unicode.encoding":"UnicodeLittle"},"Metrics Properties":{"driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","*.sink.servlet.path":"/metrics/json","*.sink.servlet.class":"org.apache.spark.metrics.sink.MetricsServlet","applications.sink.servlet.path":"/metrics/applications/json","master.sink.servlet.path":"/metrics/master/json","executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet"},"Classpath Entries":{"/opt/spark/jars/hadoop-client-runtime-3.4.1.jar":"System Classpath","/opt/spark/jars/spark-network-common_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/netty-transport-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar":"System Classpath","/opt/spark/jars/kubernetes-model-common-6.7.2.jar":"System Classpath","/opt/spark/jars/logging-interceptor-3.12.12.jar":"System Classpath","/opt/spark/jars/paranamer-2.8.jar":"System Classpath","/opt/spark/jars/breeze_2.12-2.1.0.jar":"System Classpath","/opt/spark/jars/scala-compiler-2.12.18.jar":"System Classpath","/opt/spark/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/opt/spark/jars/spark-hive-thriftserver_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/netty-handler-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/jersey-hk2-2.40.jar":"System Classpath","/opt/spark/jars/netty-buffer-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/leveldbjni-all-1.8.jar":"System Classpath","/opt/spark/jars/kubernetes-model-gatewayapi-6.7.2.jar":"System Classpath","/opt/spark/jars/commons-compress-1.23.0.jar":"System Classpath","/opt/spark/jars/log4j-api-2.20.0.jar":"System Classpath","/opt/spark/jars/super-csv-2.2.0.jar":"System Classpath","/opt/spark/jars/javolution-5.5.1.jar":"System Classpath","/opt/spark/jars/hive-shims-scheduler-2.3.9.jar":"System Classpath","/opt/spark/jars/spark-kubernetes_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/spire-util_2.12-0.17.0.jar":"System Classpath","/opt/spark/jars/netty-transport-classes-epoll-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/gson-2.2.4.jar":"System Classpath","/opt/spark/jars/spark-common-utils_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/kubernetes-client-6.7.2.jar":"System Classpath","/opt/spark/jars/parquet-format-structures-1.13.1.jar":"System Classpath","/opt/spark/jars/parquet-column-1.13.1.jar":"System Classpath","/opt/spark/jars/tink-1.9.0.jar":"System Classpath","/opt/spark/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/opt/spark/jars/antlr-runtime-3.5.2.jar":"System Classpath","/opt/spark/jars/arrow-memory-core-12.0.1.jar":"System Classpath","/opt/spark/jars/commons-dbcp-1.4.jar":"System Classpath","/opt/spark/jars/chill_2.12-0.10.0.jar":"System Classpath","/opt/spark/jars/log4j-core-2.20.0.jar":"System Classpath","/opt/spark/jars/algebra_2.12-2.0.1.jar":"System Classpath","/opt/spark/jars/xz-1.9.jar":"System Classpath","/opt/spark/jars/spark-sql-api_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/kubernetes-model-metrics-6.7.2.jar":"System Classpath","/opt/spark/jars/antlr4-runtime-4.9.3.jar":"System Classpath","/opt/spark/jars/hive-service-rpc-3.1.3.jar":"System Classpath","/opt/spark/jars/activation-1.1.1.jar":"System Classpath","/opt/spark/jars/spark-repl_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/objenesis-3.3.jar":"System Classpath","/opt/spark/jars/joda-time-2.12.5.jar":"System Classpath","/opt/spark/jars/netty-codec-http-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar":"System Classpath","/opt/spark/jars/hive-storage-api-2.8.1.jar":"System Classpath","/opt/spark/jars/jackson-databind-2.15.2.jar":"System Classpath","/opt/spark/jars/kubernetes-model-policy-6.7.2.jar":"System Classpath","/opt/spark/jars/spark-mllib-local_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/cats-kernel_2.12-2.1.1.jar":"System Classpath","/opt/spark/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/opt/spark/jars/lapack-3.0.3.jar":"System Classpath","/opt/spark/jars/spark-sketch_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/xbean-asm9-shaded-4.23.jar":"System Classpath","/opt/spark/jars/JTransforms-3.1.jar":"System Classpath","/opt/spark/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/opt/spark/jars/jul-to-slf4j-2.0.7.jar":"System Classpath","/opt/spark/jars/commons-collections-3.2.2.jar":"System Classpath","/opt/spark/jars/oro-2.0.8.jar":"System Classpath","/opt/spark/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/opt/spark/jars/derby-10.14.2.0.jar":"System Classpath","/opt/spark/jars/kubernetes-model-coordination-6.7.2.jar":"System Classpath","/opt/spark/jars/javassist-3.29.2-GA.jar":"System Classpath","/opt/spark/jars/kubernetes-model-scheduling-6.7.2.jar":"System Classpath","/opt/spark/jars/netty-common-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/metrics-jvm-4.2.19.jar":"System Classpath","/opt/spark/jars/commons-pool-1.5.4.jar":"System Classpath","/opt/spark/jars/commons-codec-1.16.1.jar":"System Classpath","/opt/spark/jars/breeze-macros_2.12-2.1.0.jar":"System Classpath","/opt/spark/jars/kubernetes-model-networking-6.7.2.jar":"System Classpath","/opt/spark/jars/py4j-0.10.9.7.jar":"System Classpath","/opt/spark/jars/annotations-17.0.0.jar":"System Classpath","/opt/spark/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/opt/spark/jars/blas-3.0.3.jar":"System Classpath","/opt/spark/jars/json4s-ast_2.12-3.7.0-M11.jar":"System Classpath","/opt/spark/jars/scala-collection-compat_2.12-2.7.0.jar":"System Classpath","/opt/spark/jars/commons-lang-2.6.jar":"System Classpath","/opt/spark/jars/compress-lzf-1.1.2.jar":"System Classpath","/opt/spark/jars/zookeeper-jute-3.6.3.jar":"System Classpath","/opt/spark/jars/metrics-graphite-4.2.19.jar":"System Classpath","/opt/spark/jars/snakeyaml-engine-2.6.jar":"System Classpath","/opt/spark/jars/json4s-jackson_2.12-3.7.0-M11.jar":"System Classpath","/opt/spark/jars/kubernetes-model-discovery-6.7.2.jar":"System Classpath","/opt/spark/jars/jta-1.1.jar":"System Classpath","/opt/spark/jars/jersey-server-2.40.jar":"System Classpath","/opt/spark/jars/orc-core-1.9.4-shaded-protobuf.jar":"System Classpath","/opt/spark/jars/jersey-common-2.40.jar":"System Classpath","/opt/spark/jars/libthrift-0.12.0.jar":"System Classpath","/opt/spark/jars/spark-tags_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/metrics-json-4.2.19.jar":"System Classpath","/opt/spark/jars/hive-serde-2.3.9.jar":"System Classpath","/opt/spark/jars/RoaringBitmap-0.9.45.jar":"System Classpath","/opt/spark/jars/metrics-core-4.2.19.jar":"System Classpath","/opt/spark/jars/parquet-encoding-1.13.1.jar":"System Classpath","/opt/spark/jars/hive-common-2.3.9.jar":"System Classpath","/opt/spark/jars/hk2-api-2.6.1.jar":"System Classpath","/opt/spark/jars/kubernetes-model-admissionregistration-6.7.2.jar":"System Classpath","/opt/spark/jars/snakeyaml-2.0.jar":"System Classpath","/opt/spark/jars/guava-14.0.1.jar":"System Classpath","/opt/spark/jars/slf4j-api-2.0.7.jar":"System Classpath","/opt/spark/jars/hive-metastore-2.3.9.jar":"System Classpath","/opt/spark/jars/jcl-over-slf4j-2.0.7.jar":"System Classpath","/opt/spark/jars/json4s-core_2.12-3.7.0-M11.jar":"System Classpath","/opt/spark/jars/kubernetes-model-autoscaling-6.7.2.jar":"System Classpath","/opt/spark/jars/univocity-parsers-2.9.1.jar":"System Classpath","/opt/spark/jars/spark-unsafe_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/spark-yarn_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/spark-streaming_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/commons-text-1.10.0.jar":"System Classpath","/opt/spark/jars/kubernetes-model-apiextensions-6.7.2.jar":"System Classpath","/opt/spark/jars/netty-transport-classes-kqueue-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/orc-shims-1.9.4.jar":"System Classpath","/opt/spark/jars/jersey-client-2.40.jar":"System Classpath","/opt/spark/jars/netty-codec-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/curator-client-2.13.0.jar":"System Classpath","/opt/spark/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/opt/spark/jars/spark-hive_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/jsr305-3.0.0.jar":"System Classpath","/opt/spark/jars/spire_2.12-0.17.0.jar":"System Classpath","/opt/spark/jars/orc-mapreduce-1.9.4-shaded-protobuf.jar":"System Classpath","/opt/spark/jars/jdo-api-3.0.1.jar":"System Classpath","/opt/spark/jars/spark-core_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/hive-exec-2.3.9-core.jar":"System Classpath","/opt/spark/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/opt/spark/jars/scala-library-2.12.18.jar":"System Classpath","/opt/spark/jars/ivy-2.5.1.jar":"System Classpath","/opt/spark/jars/minlog-1.3.0.jar":"System Classpath","/opt/spark/jars/shims-0.9.45.jar":"System Classpath","/opt/spark/jars/netty-handler-proxy-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/jackson-annotations-2.15.2.jar":"System Classpath","/opt/spark/jars/kubernetes-model-core-6.7.2.jar":"System Classpath","/opt/spark/jars/kubernetes-model-certificates-6.7.2.jar":"System Classpath","/opt/spark/jars/mesos-1.4.3-shaded-protobuf.jar":"System Classpath","/opt/spark/jars/netty-all-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/scala-reflect-2.12.18.jar":"System Classpath","/opt/spark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-x86_64.jar":"System Classpath","/opt/spark/conf/":"System Classpath","/opt/spark/jars/curator-recipes-2.13.0.jar":"System Classpath","/opt/spark/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/opt/spark/jars/parquet-common-1.13.1.jar":"System Classpath","/opt/spark/jars/hadoop-aws-3.4.1.jar":"System Classpath","/opt/spark/jars/log4j-slf4j2-impl-2.20.0.jar":"System Classpath","/opt/spark/jars/kubernetes-model-batch-6.7.2.jar":"System Classpath","/opt/spark/jars/kubernetes-httpclient-okhttp-6.7.2.jar":"System Classpath","/opt/spark/jars/spire-macros_2.12-0.17.0.jar":"System Classpath","/opt/spark/jars/datasketches-java-3.3.0.jar":"System Classpath","/opt/spark/jars/metrics-jmx-4.2.19.jar":"System Classpath","/opt/spark/jars/hk2-locator-2.6.1.jar":"System Classpath","/opt/spark/jars/lz4-java-1.8.0.jar":"System Classpath","/opt/spark/jars/scala-xml_2.12-2.1.0.jar":"System Classpath","/opt/spark/jars/hive-shims-0.23-2.3.9.jar":"System Classpath","/opt/spark/jars/spire-platform_2.12-0.17.0.jar":"System Classpath","/opt/spark/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/opt/spark/jars/commons-lang3-3.12.0.jar":"System Classpath","/opt/spark/jars/JLargeArrays-1.5.jar":"System Classpath","/opt/spark/jars/kubernetes-model-rbac-6.7.2.jar":"System Classpath","/opt/spark/jars/jakarta.inject-2.6.1.jar":"System Classpath","/opt/spark/jars/hive-shims-2.3.9.jar":"System Classpath","/opt/spark/jars/spark-launcher_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/parquet-jackson-1.13.1.jar":"System Classpath","/opt/spark/jars/httpcore-4.4.16.jar":"System Classpath","/opt/spark/jars/jodd-core-3.5.2.jar":"System Classpath","/opt/spark/jars/avro-mapred-1.11.2.jar":"System Classpath","/opt/spark/jars/jackson-core-2.15.2.jar":"System Classpath","/opt/spark/jars/spark-catalyst_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-aarch_64.jar":"System Classpath","/opt/spark/jars/json-1.8.jar":"System Classpath","/opt/spark/jars/commons-crypto-1.1.0.jar":"System Classpath","/opt/spark/jars/arrow-vector-12.0.1.jar":"System Classpath","/opt/spark/jars/stax-api-1.0.1.jar":"System Classpath","/opt/spark/jars/hive-beeline-2.3.9.jar":"System Classpath","/opt/spark/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/opt/spark/jars/hadoop-common-3.4.1.jar":"System Classpath","/opt/spark/jars/httpclient-4.5.14.jar":"System Classpath","/opt/spark/jars/commons-io-2.16.1.jar":"System Classpath","/opt/spark/jars/okhttp-3.12.12.jar":"System Classpath","/opt/spark/jars/spark-sql_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/commons-logging-1.1.3.jar":"System Classpath","/opt/spark/jars/datanucleus-core-4.1.17.jar":"System Classpath","/opt/spark/jars/jline-2.14.6.jar":"System Classpath","/opt/spark/jars/kryo-shaded-4.0.2.jar":"System Classpath","/opt/spark/jars/commons-cli-1.5.0.jar":"System Classpath","/opt/spark/jars/jackson-module-scala_2.12-2.15.2.jar":"System Classpath","/opt/spark/jars/HikariCP-2.5.1.jar":"System Classpath","/opt/spark/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/opt/spark/jars/json4s-scalap_2.12-3.7.0-M11.jar":"System Classpath","/opt/spark/jars/chill-java-0.10.0.jar":"System Classpath","/opt/spark/jars/arpack_combined_all-0.1.jar":"System Classpath","/opt/spark/jars/scala-parser-combinators_2.12-2.3.0.jar":"System Classpath","/opt/spark/jars/zookeeper-3.6.3.jar":"System Classpath","/opt/spark/jars/hive-llap-common-2.3.9.jar":"System Classpath","/opt/spark/jars/spark-mesos_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/hive-jdbc-2.3.9.jar":"System Classpath","/opt/spark/jars/netty-resolver-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/jpam-1.1.jar":"System Classpath","/opt/spark/jars/avro-1.11.2.jar":"System Classpath","/opt/spark/jars/opencsv-2.3.jar":"System Classpath","/opt/spark/jars/netty-transport-native-unix-common-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/parquet-hadoop-1.13.1.jar":"System Classpath","/opt/spark/jars/curator-framework-2.13.0.jar":"System Classpath","/opt/spark/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/opt/spark/jars/jackson-dataformat-yaml-2.15.2.jar":"System Classpath","/opt/spark/jars/arrow-memory-netty-12.0.1.jar":"System Classpath","/opt/spark/jars/stream-2.9.6.jar":"System Classpath","/opt/spark/jars/hive-cli-2.3.9.jar":"System Classpath","/opt/spark/jars/kubernetes-model-events-6.7.2.jar":"System Classpath","/opt/spark/jars/spark-graphx_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/jackson-datatype-jsr310-2.15.2.jar":"System Classpath","/opt/spark/jars/avro-ipc-1.11.2.jar":"System Classpath","/opt/spark/jars/rocksdbjni-8.3.2.jar":"System Classpath","/opt/spark/jars/bundle-2.29.0.jar":"System Classpath","/opt/spark/jars/kubernetes-model-node-6.7.2.jar":"System Classpath","/opt/spark/jars/commons-compiler-3.1.9.jar":"System Classpath","/opt/spark/jars/ST4-4.0.4.jar":"System Classpath","/opt/spark/jars/spark-kvstore_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/log4j-1.2-api-2.20.0.jar":"System Classpath","/opt/spark/jars/hadoop-client-api-3.4.1.jar":"System Classpath","/opt/spark/jars/datasketches-memory-2.1.0.jar":"System Classpath","/opt/spark/jars/spark-mllib_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/kubernetes-client-api-6.7.2.jar":"System Classpath","/opt/spark/jars/arrow-format-12.0.1.jar":"System Classpath","/opt/spark/jars/arpack-3.0.3.jar":"System Classpath","/opt/spark/jars/kubernetes-model-resource-6.7.2.jar":"System Classpath","/opt/spark/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/opt/spark/jars/zstd-jni-1.5.5-4.jar":"System Classpath","/opt/spark/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/opt/spark/jars/hadoop-shaded-guava-1.1.1.jar":"System Classpath","/opt/spark/jars/okio-1.17.6.jar":"System Classpath","/opt/spark/jars/hadoop-yarn-server-web-proxy-3.4.1.jar":"System Classpath","/opt/spark/jars/hk2-utils-2.6.1.jar":"System Classpath","/opt/spark/jars/audience-annotations-0.5.0.jar":"System Classpath","/opt/spark/jars/flatbuffers-java-1.12.0.jar":"System Classpath","/opt/spark/jars/janino-3.1.9.jar":"System Classpath","/opt/spark/jars/aircompressor-0.27.jar":"System Classpath","/opt/spark/jars/commons-math3-3.6.1.jar":"System Classpath","/opt/spark/jars/transaction-api-1.1.jar":"System Classpath","/opt/spark/jars/kubernetes-model-storageclass-6.7.2.jar":"System Classpath","/opt/spark/jars/libfb303-0.9.3.jar":"System Classpath","/opt/spark/jars/snappy-java-1.1.10.5.jar":"System Classpath","/opt/spark/jars/hive-shims-common-2.3.9.jar":"System Classpath","/opt/spark/jars/jersey-container-servlet-core-2.40.jar":"System Classpath","/opt/spark/jars/netty-codec-http2-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/pickle-1.3.jar":"System Classpath","/opt/spark/jars/kubernetes-model-flowcontrol-6.7.2.jar":"System Classpath","/opt/spark/jars/threeten-extra-1.7.1.jar":"System Classpath","/opt/spark/jars/spark-network-shuffle_2.12-3.5.3.jar":"System Classpath","/opt/spark/jars/kubernetes-model-apps-6.7.2.jar":"System Classpath","/opt/spark/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/opt/spark/jars/kubernetes-model-extensions-6.7.2.jar":"System Classpath","/opt/spark/jars/netty-codec-socks-4.1.96.Final.jar":"System Classpath","/opt/spark/jars/jersey-container-servlet-2.40.jar":"System Classpath","/opt/spark/jars/zjsonpatch-0.3.0.jar":"System Classpath","/opt/spark/jars/commons-collections4-4.4.jar":"System Classpath"}} +{"Event":"SparkListenerApplicationStart","App Name":"NewYorkTaxiData_2025_06_27_03_56_52","App ID":"spark-cc4d115f011443d787f03a71a476a745","Timestamp":1750996612238,"User":"spark"} +{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1750996646244,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[0],"Properties":{"spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750996646256,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}","resource.executor.cores":"1","spark.rdd.scope.noOverride":"true"}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1750996676192,"Executor ID":"2","Executor Info":{"Host":"100.64.240.100","Total Cores":1,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1750996676192}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"2","Host":"100.64.240.100","Port":40373},"Maximum Memory":2388236697,"Timestamp":1750996676246,"Maximum Onheap Memory":2388236697,"Maximum Offheap Memory":0} +{"Event":"SparkListenerExecutorAdded","Timestamp":1750996676258,"Executor ID":"3","Executor Info":{"Host":"100.64.131.144","Total Cores":1,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1750996676258}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1750996676258,"Executor ID":"1","Executor Info":{"Host":"100.64.190.213","Total Cores":1,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1750996676258}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"100.64.190.213","Port":33709},"Maximum Memory":2388236697,"Timestamp":1750996676314,"Maximum Onheap Memory":2388236697,"Maximum Offheap Memory":0} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750996676284,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"3","Host":"100.64.131.144","Port":36267},"Maximum Memory":2388236697,"Timestamp":1750996676315,"Maximum Onheap Memory":2388236697,"Maximum Offheap Memory":0} +{"Event":"SparkListenerExecutorAdded","Timestamp":1750996676321,"Executor ID":"4","Executor Info":{"Host":"100.64.247.4","Total Cores":1,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1750996676321}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"4","Host":"100.64.247.4","Port":43421},"Maximum Memory":2388236697,"Timestamp":1750996676374,"Maximum Onheap Memory":2388236697,"Maximum Offheap Memory":0} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750996676284,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996678444,"Failed":false,"Killed":false,"Accumulables":[{"ID":1,"Name":"internal.metrics.executorDeserializeTime","Update":303,"Value":303,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorDeserializeCpuTime","Update":246357317,"Value":246357317,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorRunTime","Update":1773,"Value":1773,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.executorCpuTime","Update":91991016,"Value":91991016,"Internal":true,"Count Failed Values":true},{"ID":5,"Name":"internal.metrics.resultSize","Update":2470,"Value":2470,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.jvmGCTime","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.resultSerializationTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":303,"Executor Deserialize CPU Time":246357317,"Executor Run Time":1773,"Executor CPU Time":91991016,"Peak Execution Memory":0,"Result Size":2470,"JVM GC Time":20,"Result Serialization Time":4,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750996646256,"Completion Time":1750996678453,"Accumulables":[{"ID":1,"Name":"internal.metrics.executorDeserializeTime","Value":303,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorDeserializeCpuTime","Value":246357317,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorRunTime","Value":1773,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.executorCpuTime","Value":91991016,"Internal":true,"Count Failed Values":true},{"ID":5,"Name":"internal.metrics.resultSize","Value":2470,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.jvmGCTime","Value":20,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.resultSerializationTime","Value":4,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1750996678458,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"rootExecutionId":0,"description":"showString at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nCollectLimit (4)\n+- * Project (3)\n +- * ColumnarToRow (2)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct\n\n(2) ColumnarToRow [codegen id : 1]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(3) Project [codegen id : 1]\nOutput [20]: [toprettystring(VendorID#0L, Some(Etc/UTC)) AS toprettystring(VendorID)#80, toprettystring(tpep_pickup_datetime#1, Some(Etc/UTC)) AS toprettystring(tpep_pickup_datetime)#81, toprettystring(tpep_dropoff_datetime#2, Some(Etc/UTC)) AS toprettystring(tpep_dropoff_datetime)#82, toprettystring(passenger_count#3, Some(Etc/UTC)) AS toprettystring(passenger_count)#83, toprettystring(trip_distance#4, Some(Etc/UTC)) AS toprettystring(trip_distance)#84, toprettystring(RatecodeID#5, Some(Etc/UTC)) AS toprettystring(RatecodeID)#85, toprettystring(store_and_fwd_flag#6, Some(Etc/UTC)) AS toprettystring(store_and_fwd_flag)#86, toprettystring(PULocationID#7L, Some(Etc/UTC)) AS toprettystring(PULocationID)#87, toprettystring(DOLocationID#8L, Some(Etc/UTC)) AS toprettystring(DOLocationID)#88, toprettystring(payment_type#9L, Some(Etc/UTC)) AS toprettystring(payment_type)#89, toprettystring(fare_amount#10, Some(Etc/UTC)) AS toprettystring(fare_amount)#90, toprettystring(extra#11, Some(Etc/UTC)) AS toprettystring(extra)#91, toprettystring(mta_tax#12, Some(Etc/UTC)) AS toprettystring(mta_tax)#92, toprettystring(tip_amount#13, Some(Etc/UTC)) AS toprettystring(tip_amount)#93, toprettystring(tolls_amount#14, Some(Etc/UTC)) AS toprettystring(tolls_amount)#94, toprettystring(improvement_surcharge#15, Some(Etc/UTC)) AS toprettystring(improvement_surcharge)#95, toprettystring(total_amount#16, Some(Etc/UTC)) AS toprettystring(total_amount)#96, toprettystring(congestion_surcharge#17, Some(Etc/UTC)) AS toprettystring(congestion_surcharge)#97, toprettystring(airport_fee#18, Some(Etc/UTC)) AS toprettystring(airport_fee)#98, 2025-06-27 03:57:59.158854 AS toprettystring(current_date)#99]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(4) CollectLimit\nInput [20]: [toprettystring(VendorID)#80, toprettystring(tpep_pickup_datetime)#81, toprettystring(tpep_dropoff_datetime)#82, toprettystring(passenger_count)#83, toprettystring(trip_distance)#84, toprettystring(RatecodeID)#85, toprettystring(store_and_fwd_flag)#86, toprettystring(PULocationID)#87, toprettystring(DOLocationID)#88, toprettystring(payment_type)#89, toprettystring(fare_amount)#90, toprettystring(extra)#91, toprettystring(mta_tax)#92, toprettystring(tip_amount)#93, toprettystring(tolls_amount)#94, toprettystring(improvement_surcharge)#95, toprettystring(total_amount)#96, toprettystring(congestion_surcharge)#97, toprettystring(airport_fee)#98, toprettystring(current_date)#99]\nArguments: 21\n\n","sparkPlanInfo":{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(VendorID#0L, Some(Etc/UTC)) AS toprettystring(VendorID)#80, toprettystring(tpep_pickup_datetime#1, Some(Etc/UTC)) AS toprettystring(tpep_pickup_datetime)#81, toprettystring(tpep_dropoff_datetime#2, Some(Etc/UTC)) AS toprettystring(tpep_dropoff_datetime)#82, toprettystring(passenger_count#3, Some(Etc/UTC)) AS toprettystring(passenger_count)#83, toprettystring(trip_distance#4, Some(Etc/UTC)) AS toprettystring(trip_distance)#84, toprettystring(RatecodeID#5, Some(Etc/UTC)) AS toprettystring(RatecodeID)#85, toprettystring(store_and_fwd_flag#6, Some(Etc/UTC)) AS toprettystring(store_and_fwd_flag)#86, toprettystring(PULocationID#7L, Some(Etc/UTC)) AS toprettystring(PULocationID)#87, toprettystring(DOLocationID#8L, Some(Etc/UTC)) AS toprettystring(DOLocationID)#88, toprettystring(payment_type#9L, Some(Etc/UTC)) AS toprettystring(payment_type)#89, toprettystring(fare_amount#10, Some(Etc/UTC)) AS toprettystring(fare_amount)#90, toprettystring(extra#11, Some(Etc/UTC)) AS toprettystring(extra)#91, toprettystring(mta_tax#12, Some(Etc/UTC)) AS toprettystring(mta_tax)#92, toprettystring(tip_amount#13, Some(Etc/UTC)) AS toprettystring(tip_amount)#93, toprettystring(tolls_amount#14, Some(Etc/UTC)) AS toprettystring(tolls_amount)#94, toprettystring(improvement_surcharge#15, Some(Etc/UTC)) AS toprettystring(improvement_surcharge)#95, toprettystring(total_amount#16, Some(Etc/UTC)) AS toprettystring(total_amount)#96, toprettystring(congestion_surcharge#17, Some(Etc/UTC)) AS toprettystring(congestion_surcharge)#97, toprettystring(airport_fee#18, Some(Etc/UTC)) AS toprettystring(airport_fee)#98, 2025-06-27 03:57:59.158854 AS toprettystring(current_date)#99]","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [VendorID#0L,tpep_pickup_datetime#1,tpep_dropoff_datetime#2,passenger_count#3,trip_distance#4,RatecodeID#5,store_and_fwd_flag#6,PULocationID#7L,DOLocationID#8L,payment_type#9L,fare_amount#10,extra#11,mta_tax#12,tip_amount#13,tolls_amount#14,improvement_surcharge#15,total_amount#16,congestion_surcharge#17,airport_fee#18] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":61,"metricType":"sum"},{"name":"scan time","accumulatorId":60,"metricType":"timing"},{"name":"metadata time","accumulatorId":62,"metricType":"timing"},{"name":"size of files read","accumulatorId":63,"metricType":"size"},{"name":"number of output rows","accumulatorId":59,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":57,"metricType":"sum"},{"name":"number of input batches","accumulatorId":58,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":56,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":54,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":48,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":55,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":49,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":46,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":43,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":52,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":45,"metricType":"sum"},{"name":"records read","accumulatorId":42,"metricType":"sum"},{"name":"local bytes read","accumulatorId":40,"metricType":"size"},{"name":"fetch wait time","accumulatorId":41,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":38,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":44,"metricType":"sum"},{"name":"local blocks read","accumulatorId":37,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":47,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":36,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":50,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":51,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":39,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":53,"metricType":"size"}]},"time":1750996680061,"modifiedConfigs":{},"jobTags":[]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[61,101],[62,6],[63,3852134849]]} +{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1750996680834,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"5\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"FileScanRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[1],"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750996612238","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_03_56_52","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.63.89","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-77d59097af8544ae-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-cc4d115f011443d787f03a71a476a745","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"0","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.63.89","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750996611643","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"0","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"5\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"FileScanRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750996680837,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750996612238","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_03_56_52","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.63.89","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-77d59097af8544ae-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-cc4d115f011443d787f03a71a476a745","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","resource.executor.cores":"1","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"0","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.63.89","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750996611643","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"0","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750996680919,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750996680919,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996682134,"Failed":false,"Killed":false,"Accumulables":[{"ID":57,"Name":"number of output rows","Update":"4096","Value":"4096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"number of input batches","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"number of output rows","Update":"4096","Value":"4096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"scan time","Update":"872","Value":"872","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":64,"Name":"internal.metrics.executorDeserializeTime","Update":104,"Value":104,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeCpuTime","Update":95944834,"Value":95944834,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorRunTime","Update":1094,"Value":1094,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorCpuTime","Update":955416790,"Value":955416790,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.resultSize","Update":3634,"Value":3634,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":95,"Name":"internal.metrics.input.bytesRead","Update":38137744,"Value":38137744,"Internal":true,"Count Failed Values":true},{"ID":96,"Name":"internal.metrics.input.recordsRead","Update":4096,"Value":4096,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":104,"Executor Deserialize CPU Time":95944834,"Executor Run Time":1094,"Executor CPU Time":955416790,"Peak Execution Memory":0,"Result Size":3634,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":38137744,"Records Read":4096},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"5\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"FileScanRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Scan parquet \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750996680837,"Completion Time":1750996682136,"Accumulables":[{"ID":57,"Name":"number of output rows","Value":"4096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"number of input batches","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"number of output rows","Value":"4096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"scan time","Value":"872","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":64,"Name":"internal.metrics.executorDeserializeTime","Value":104,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeCpuTime","Value":95944834,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorRunTime","Value":1094,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorCpuTime","Value":955416790,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.resultSize","Value":3634,"Internal":true,"Count Failed Values":true},{"ID":70,"Name":"internal.metrics.resultSerializationTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":95,"Name":"internal.metrics.input.bytesRead","Value":38137744,"Internal":true,"Count Failed Values":true},{"ID":96,"Name":"internal.metrics.input.recordsRead","Value":4096,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1750996682137,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1750996683529,"errorMessage":""} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"rootExecutionId":1,"description":"count at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (5)\n+- HashAggregate (4)\n +- Exchange (3)\n +- HashAggregate (2)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput: []\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct<>\n\n(2) HashAggregate\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#164L]\nResults [1]: [count#165L]\n\n(3) Exchange\nInput [1]: [count#165L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=27]\n\n(4) HashAggregate\nInput [1]: [count#165L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#161L]\nResults [1]: [count(1)#161L AS count#162L]\n\n(5) AdaptiveSparkPlan\nOutput [1]: [count#162L]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=27]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]","ReadSchema":"struct<>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":135,"metricType":"sum"},{"name":"scan time","accumulatorId":134,"metricType":"timing"},{"name":"metadata time","accumulatorId":136,"metricType":"timing"},{"name":"size of files read","accumulatorId":137,"metricType":"size"},{"name":"number of output rows","accumulatorId":133,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":129,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":130,"metricType":"timing"},{"name":"peak memory","accumulatorId":128,"metricType":"size"},{"name":"number of output rows","accumulatorId":127,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":132,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":131,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":125,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":119,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":126,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":120,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":117,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":114,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":123,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":116,"metricType":"sum"},{"name":"records read","accumulatorId":113,"metricType":"sum"},{"name":"local bytes read","accumulatorId":111,"metricType":"size"},{"name":"fetch wait time","accumulatorId":112,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":109,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":115,"metricType":"sum"},{"name":"local blocks read","accumulatorId":108,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":118,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":107,"metricType":"sum"},{"name":"data size","accumulatorId":105,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":121,"metricType":"size"},{"name":"number of partitions","accumulatorId":106,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":122,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":110,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":124,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":101,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":102,"metricType":"timing"},{"name":"peak memory","accumulatorId":100,"metricType":"size"},{"name":"number of output rows","accumulatorId":99,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":104,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":103,"metricType":"average"}]}],"metadata":{},"metrics":[]},"time":1750996683745,"modifiedConfigs":{},"jobTags":[]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (10)\n+- == Current Plan ==\n HashAggregate (6)\n +- ShuffleQueryStage (5)\n +- Exchange (4)\n +- * HashAggregate (3)\n +- * ColumnarToRow (2)\n +- Scan parquet (1)\n+- == Initial Plan ==\n HashAggregate (9)\n +- Exchange (8)\n +- HashAggregate (7)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput: []\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct<>\n\n(2) ColumnarToRow [codegen id : 1]\nInput: []\n\n(3) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#164L]\nResults [1]: [count#165L]\n\n(4) Exchange\nInput [1]: [count#165L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=39]\n\n(5) ShuffleQueryStage\nOutput [1]: [count#165L]\nArguments: 0\n\n(6) HashAggregate\nInput [1]: [count#165L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#161L]\nResults [1]: [count(1)#161L AS count#162L]\n\n(7) HashAggregate\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#164L]\nResults [1]: [count#165L]\n\n(8) Exchange\nInput [1]: [count#165L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=27]\n\n(9) HashAggregate\nInput [1]: [count#165L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#161L]\nResults [1]: [count(1)#161L AS count#162L]\n\n(10) AdaptiveSparkPlan\nOutput [1]: [count#162L]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=39]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]","ReadSchema":"struct<>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":135,"metricType":"sum"},{"name":"scan time","accumulatorId":134,"metricType":"timing"},{"name":"metadata time","accumulatorId":136,"metricType":"timing"},{"name":"size of files read","accumulatorId":137,"metricType":"size"},{"name":"number of output rows","accumulatorId":133,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":173,"metricType":"sum"},{"name":"number of input batches","accumulatorId":174,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":169,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":170,"metricType":"timing"},{"name":"peak memory","accumulatorId":168,"metricType":"size"},{"name":"number of output rows","accumulatorId":167,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":172,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":171,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":166,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":164,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":158,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":165,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":159,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":156,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":153,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":162,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":155,"metricType":"sum"},{"name":"records read","accumulatorId":152,"metricType":"sum"},{"name":"local bytes read","accumulatorId":150,"metricType":"size"},{"name":"fetch wait time","accumulatorId":151,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":148,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":154,"metricType":"sum"},{"name":"local blocks read","accumulatorId":147,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":157,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":146,"metricType":"sum"},{"name":"data size","accumulatorId":144,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":160,"metricType":"size"},{"name":"number of partitions","accumulatorId":145,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":161,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":149,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":163,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":140,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":141,"metricType":"timing"},{"name":"peak memory","accumulatorId":139,"metricType":"size"},{"name":"number of output rows","accumulatorId":138,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":143,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":142,"metricType":"average"}]}],"metadata":{},"metrics":[]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[135,101],[136,0],[137,3852134849]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[145,1]]} +{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1750996683937,"Stage Infos":[{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":34,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"FileScanRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[2],"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750996612238","spark.rdd.scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_03_56_52","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.63.89","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-77d59097af8544ae-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-cc4d115f011443d787f03a71a476a745","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"1","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.63.89","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750996611643","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"1","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.rdd.scope.noOverride":"true","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":34,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"FileScanRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750996683941,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750996612238","spark.rdd.scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_03_56_52","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.63.89","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-77d59097af8544ae-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-cc4d115f011443d787f03a71a476a745","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","resource.executor.cores":"1","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"1","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.63.89","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750996611643","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"1","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.rdd.scope.noOverride":"true","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750996683956,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1750996683957,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1750996683957,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1750996683957,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1750996684772,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1750996683957,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996684775,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"7391793","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"636","Value":"636","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"7065071","Value":"7065071","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"675","Value":"675","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"661","Value":"661","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"7391793","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"1806","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":96,"Value":96,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":41056841,"Value":41056841,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":709,"Value":709,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":184302154,"Value":184302154,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2176,"Value":2176,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":7065071,"Value":7065071,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":31206,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":7391793,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":96,"Executor Deserialize CPU Time":41056841,"Executor Run Time":709,"Executor CPU Time":184302154,"Peak Execution Memory":0,"Result Size":2176,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":7065071,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1750996685461,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1750996684772,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996685462,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"14783586","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"663","Value":"1299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"566872","Value":"7631943","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"673","Value":"1348","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"673","Value":"1334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"14783586","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"3612","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":99,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3952830,"Value":45009671,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":676,"Value":1385,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":101795962,"Value":286098116,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":4309,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":118,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":566872,"Value":7631943,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":62412,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":14783586,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3952830,"Executor Run Time":676,"Executor CPU Time":101795962,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":566872,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1750996686004,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1750996685461,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996686005,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"22175379","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"523","Value":"1822","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"48","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"509939","Value":"8141882","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"528","Value":"1876","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"528","Value":"1862","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"22175379","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"5418","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":102,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3618200,"Value":48627871,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":531,"Value":1916,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":87221878,"Value":373319994,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":6442,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":177,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":509939,"Value":8141882,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":93618,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":22175379,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3618200,"Executor Run Time":531,"Executor CPU Time":87221878,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":509939,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1750996686601,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1750996686004,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996686602,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"29567172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"578","Value":"2400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"64","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"487166","Value":"8629048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"584","Value":"2460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"584","Value":"2446","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"29567172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"7224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":104,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2776022,"Value":51403893,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":586,"Value":2502,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":80748405,"Value":454068399,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":8575,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":236,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":487166,"Value":8629048,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":124824,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":29567172,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2776022,"Executor Run Time":586,"Executor CPU Time":80748405,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":487166,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":8,"Attempt":0,"Partition ID":8,"Launch Time":1750996686985,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1750996683957,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996686987,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"36958965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"2288","Value":"4688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"80","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"295","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"6883834","Value":"15512882","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"2326","Value":"4786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"2312","Value":"4758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"36958965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"9030","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":364,"Value":468,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":320322641,"Value":371726534,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":2630,"Value":5132,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":1853126091,"Value":2307194490,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2219,"Value":10794,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.jvmGCTime","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":295,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":6883834,"Value":15512882,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":156030,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":36958965,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":364,"Executor Deserialize CPU Time":320322641,"Executor Run Time":2630,"Executor CPU Time":1853126091,"Peak Execution Memory":0,"Result Size":2219,"JVM GC Time":23,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":6883834,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":9,"Attempt":0,"Partition ID":9,"Launch Time":1750996687008,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1750996683957,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996687009,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"44350758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"2307","Value":"6995","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"96","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"354","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"6974124","Value":"22487006","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"2348","Value":"7134","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"2334","Value":"7092","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"44350758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"10836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":368,"Value":836,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":317441146,"Value":689167680,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":2651,"Value":7783,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":1872420839,"Value":4179615329,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2217,"Value":13011,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.jvmGCTime","Update":24,"Value":47,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":354,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":6974124,"Value":22487006,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":187236,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":44350758,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":368,"Executor Deserialize CPU Time":317441146,"Executor Run Time":2651,"Executor CPU Time":1872420839,"Peak Execution Memory":0,"Result Size":2217,"JVM GC Time":24,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":6974124,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":10,"Attempt":0,"Partition ID":10,"Launch Time":1750996687140,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750996683956,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996687141,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"51742551","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"2372","Value":"9367","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"7119988","Value":"29606994","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"2414","Value":"9548","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"2400","Value":"9492","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"51742551","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"12642","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":383,"Value":1219,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":340103751,"Value":1029271431,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":2767,"Value":10550,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":2087027887,"Value":6266643216,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2219,"Value":15230,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.jvmGCTime","Update":25,"Value":72,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":413,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":7119988,"Value":29606994,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":218442,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":51742551,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":383,"Executor Deserialize CPU Time":340103751,"Executor Run Time":2767,"Executor CPU Time":2087027887,"Peak Execution Memory":0,"Result Size":2219,"JVM GC Time":25,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":7119988,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":11,"Attempt":0,"Partition ID":11,"Launch Time":1750996687202,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1750996686601,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996687203,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"59134344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"584","Value":"9951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"496623","Value":"30103617","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"589","Value":"10137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"588","Value":"10080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"59134344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"14448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1221,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2462881,"Value":1031734312,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":591,"Value":11141,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":77439705,"Value":6344082921,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":17363,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":472,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":496623,"Value":30103617,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":249648,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":59134344,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2462881,"Executor Run Time":591,"Executor CPU Time":77439705,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":496623,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":12,"Attempt":0,"Partition ID":12,"Launch Time":1750996687605,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":8,"Attempt":0,"Partition ID":8,"Launch Time":1750996686985,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996687607,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"66526137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"587","Value":"10538","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"531","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"542814","Value":"30646431","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"596","Value":"10733","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"595","Value":"10675","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"66526137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"16254","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":1225,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4620012,"Value":1036354324,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":607,"Value":11748,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":109619638,"Value":6453702559,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":19496,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":531,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":542814,"Value":30646431,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":280854,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":66526137,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4620012,"Executor Run Time":607,"Executor CPU Time":109619638,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":542814,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":13,"Attempt":0,"Partition ID":13,"Launch Time":1750996687612,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":9,"Attempt":0,"Partition ID":9,"Launch Time":1750996687008,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996687613,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"73917930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"571","Value":"11109","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"590","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"374175","Value":"31020606","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"580","Value":"11313","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"580","Value":"11255","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"73917930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"18060","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":1229,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4537182,"Value":1040891506,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":591,"Value":12339,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":107580484,"Value":6561283043,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2131,"Value":21627,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":590,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":374175,"Value":31020606,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":312060,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":73917930,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4537182,"Executor Run Time":591,"Executor CPU Time":107580484,"Peak Execution Memory":0,"Result Size":2131,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":374175,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":14,"Attempt":0,"Partition ID":14,"Launch Time":1750996687702,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":10,"Attempt":0,"Partition ID":10,"Launch Time":1750996687140,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996687703,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"81309723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"524","Value":"11633","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"649","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"545238","Value":"31565844","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"536","Value":"11849","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"535","Value":"11790","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"81309723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"19866","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":1234,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5163168,"Value":1046054674,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":548,"Value":12887,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":120836143,"Value":6682119186,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":23760,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":649,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":545238,"Value":31565844,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":343266,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":81309723,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":5163168,"Executor Run Time":548,"Executor CPU Time":120836143,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":545238,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":15,"Attempt":0,"Partition ID":15,"Launch Time":1750996687827,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":11,"Attempt":0,"Partition ID":11,"Launch Time":1750996687202,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996687827,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"88701516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"609","Value":"12242","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"192","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"708","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"354422","Value":"31920266","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"613","Value":"12462","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"612","Value":"12402","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"88701516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"21672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1236,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2726747,"Value":1048781421,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":615,"Value":13502,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":80910672,"Value":6763029858,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":25893,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":708,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":354422,"Value":31920266,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":374472,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":88701516,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2726747,"Executor Run Time":615,"Executor CPU Time":80910672,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":354422,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":16,"Attempt":0,"Partition ID":16,"Launch Time":1750996688164,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":13,"Attempt":0,"Partition ID":13,"Launch Time":1750996687612,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996688165,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"96093309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"533","Value":"12775","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"767","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"539828","Value":"32460094","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"538","Value":"13000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"537","Value":"12939","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"96093309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"23478","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1239,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3522171,"Value":1052303592,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":540,"Value":14042,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":86695145,"Value":6849725003,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2131,"Value":28024,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":767,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":539828,"Value":32460094,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":405678,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":96093309,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3522171,"Executor Run Time":540,"Executor CPU Time":86695145,"Peak Execution Memory":0,"Result Size":2131,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":539828,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":17,"Attempt":0,"Partition ID":17,"Launch Time":1750996688217,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":12,"Attempt":0,"Partition ID":12,"Launch Time":1750996687605,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996688218,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"103485102","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"593","Value":"13368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"826","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"367259","Value":"32827353","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"598","Value":"13598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"597","Value":"13536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"103485102","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"25284","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1242,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3603033,"Value":1055906625,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":600,"Value":14642,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":86657465,"Value":6936382468,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":30157,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":826,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":367259,"Value":32827353,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":436884,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":103485102,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3603033,"Executor Run Time":600,"Executor CPU Time":86657465,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":367259,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":18,"Attempt":0,"Partition ID":18,"Launch Time":1750996688305,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":14,"Attempt":0,"Partition ID":14,"Launch Time":1750996687702,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996688305,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"110876895","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"581","Value":"13949","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"885","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"562518","Value":"33389871","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"586","Value":"14184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"585","Value":"14121","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"110876895","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"27090","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":1246,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3979941,"Value":1059886566,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":589,"Value":15231,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":89736232,"Value":7026118700,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":32290,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":885,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":562518,"Value":33389871,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":468090,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":110876895,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3979941,"Executor Run Time":589,"Executor CPU Time":89736232,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":562518,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":19,"Attempt":0,"Partition ID":19,"Launch Time":1750996688349,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":15,"Attempt":0,"Partition ID":15,"Launch Time":1750996687827,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996688350,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"118268688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"508","Value":"14457","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"476008","Value":"33865879","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"512","Value":"14696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"511","Value":"14632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"118268688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"28896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1248,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2501355,"Value":1062387921,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":514,"Value":15745,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":68737414,"Value":7094856114,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2176,"Value":34466,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.jvmGCTime","Update":13,"Value":85,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":944,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":476008,"Value":33865879,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":499296,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":118268688,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2501355,"Executor Run Time":514,"Executor CPU Time":68737414,"Peak Execution Memory":0,"Result Size":2176,"JVM GC Time":13,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":476008,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":20,"Attempt":0,"Partition ID":20,"Launch Time":1750996688757,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":17,"Attempt":0,"Partition ID":17,"Launch Time":1750996688217,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996688757,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"518","Value":"14975","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"531596","Value":"34397475","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"525","Value":"15221","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"525","Value":"15157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"30702","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1251,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3136325,"Value":1065524246,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":528,"Value":16273,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":81496701,"Value":7176352815,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":36599,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1003,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":531596,"Value":34397475,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":530502,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":125660481,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3136325,"Executor Run Time":528,"Executor CPU Time":81496701,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":531596,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":21,"Attempt":0,"Partition ID":21,"Launch Time":1750996688782,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":16,"Attempt":0,"Partition ID":16,"Launch Time":1750996688164,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996688783,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"133052274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"600","Value":"15575","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1062","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"502776","Value":"34900251","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"605","Value":"15826","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"605","Value":"15762","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"133052274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"32508","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1254,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3178528,"Value":1068702774,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":608,"Value":16881,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":85131706,"Value":7261484521,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2131,"Value":38730,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1062,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":502776,"Value":34900251,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":561708,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":133052274,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3178528,"Executor Run Time":608,"Executor CPU Time":85131706,"Peak Execution Memory":0,"Result Size":2131,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":502776,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":24,"Index":22,"Attempt":0,"Partition ID":22,"Launch Time":1750996688833,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":18,"Attempt":0,"Partition ID":18,"Launch Time":1750996688305,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996688834,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"140444067","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"506","Value":"16081","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"304","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1121","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"548313","Value":"35448564","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"514","Value":"16340","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"514","Value":"16276","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"140444067","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"34314","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1257,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3612137,"Value":1072314911,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":517,"Value":17398,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":88582926,"Value":7350067447,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":40863,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1121,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":548313,"Value":35448564,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":592914,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":140444067,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3612137,"Executor Run Time":517,"Executor CPU Time":88582926,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":548313,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":25,"Index":23,"Attempt":0,"Partition ID":23,"Launch Time":1750996688945,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":19,"Attempt":0,"Partition ID":19,"Launch Time":1750996688349,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996688945,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"147835860","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"584","Value":"16665","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"552546","Value":"36001110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"586","Value":"16926","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"586","Value":"16862","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"147835860","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"36120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1259,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2337791,"Value":1074652702,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":588,"Value":17986,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":62275718,"Value":7412343165,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":42996,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1180,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":552546,"Value":36001110,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":624120,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":147835860,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2337791,"Executor Run Time":588,"Executor CPU Time":62275718,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":552546,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":26,"Index":24,"Attempt":0,"Partition ID":24,"Launch Time":1750996689288,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":20,"Attempt":0,"Partition ID":20,"Launch Time":1750996688757,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996689289,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"155227653","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"513","Value":"17178","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"336","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1239","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"585977","Value":"36587087","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"518","Value":"17444","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"518","Value":"17380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"155227653","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"37926","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1262,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2985864,"Value":1077638566,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":520,"Value":18506,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":77714842,"Value":7490058007,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":45129,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1239,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":585977,"Value":36587087,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":655326,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":155227653,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2985864,"Executor Run Time":520,"Executor CPU Time":77714842,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":585977,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":27,"Index":25,"Attempt":0,"Partition ID":25,"Launch Time":1750996689306,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":21,"Attempt":0,"Partition ID":21,"Launch Time":1750996688782,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996689307,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"162619446","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"507","Value":"17685","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"529749","Value":"37116836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"512","Value":"17956","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"512","Value":"17892","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"162619446","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"39732","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1264,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2809929,"Value":1080448495,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":514,"Value":19020,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":78582338,"Value":7568640345,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2131,"Value":47260,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1298,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":529749,"Value":37116836,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":686532,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":162619446,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2809929,"Executor Run Time":514,"Executor CPU Time":78582338,"Peak Execution Memory":0,"Result Size":2131,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":529749,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":28,"Index":26,"Attempt":0,"Partition ID":26,"Launch Time":1750996689358,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":24,"Index":22,"Attempt":0,"Partition ID":22,"Launch Time":1750996688833,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996689359,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"170011239","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"507","Value":"18192","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1357","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"522272","Value":"37639108","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"512","Value":"18468","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"512","Value":"18404","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"170011239","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"41538","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1267,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3146367,"Value":1083594862,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":515,"Value":19535,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":83361375,"Value":7652001720,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":49393,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1357,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":522272,"Value":37639108,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":717738,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":170011239,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3146367,"Executor Run Time":515,"Executor CPU Time":83361375,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":522272,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":29,"Index":27,"Attempt":0,"Partition ID":27,"Launch Time":1750996689460,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":25,"Index":23,"Attempt":0,"Partition ID":23,"Launch Time":1750996688945,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996689461,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"177403032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"503","Value":"18695","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1416","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"387689","Value":"38026797","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"505","Value":"18973","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"505","Value":"18909","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"177403032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"43344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1269,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2594566,"Value":1086189428,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":507,"Value":20042,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":63222220,"Value":7715223940,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":51526,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1416,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":387689,"Value":38026797,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":748944,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":177403032,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2594566,"Executor Run Time":507,"Executor CPU Time":63222220,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":387689,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":30,"Index":28,"Attempt":0,"Partition ID":28,"Launch Time":1750996689852,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":26,"Index":24,"Attempt":0,"Partition ID":24,"Launch Time":1750996689288,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996689853,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"184794825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"548","Value":"19243","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1475","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"593853","Value":"38620650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"551","Value":"19524","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"551","Value":"19460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"184794825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"45150","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1272,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3107074,"Value":1089296502,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":553,"Value":20595,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":79239265,"Value":7794463205,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":53659,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1475,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":593853,"Value":38620650,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":780150,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":184794825,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3107074,"Executor Run Time":553,"Executor CPU Time":79239265,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":593853,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":31,"Index":29,"Attempt":0,"Partition ID":29,"Launch Time":1750996689972,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":27,"Index":25,"Attempt":0,"Partition ID":25,"Launch Time":1750996689306,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996689973,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"192186618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"650","Value":"19893","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"416","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1534","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"26","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"506361","Value":"39127011","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"653","Value":"20177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"26","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"652","Value":"20112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"192186618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"46956","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1275,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3162742,"Value":1092459244,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":655,"Value":21250,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":78185155,"Value":7872648360,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2131,"Value":55790,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1534,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":506361,"Value":39127011,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":811356,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":192186618,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3162742,"Executor Run Time":655,"Executor CPU Time":78185155,"Peak Execution Memory":0,"Result Size":2131,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":506361,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":32,"Index":30,"Attempt":0,"Partition ID":30,"Launch Time":1750996690008,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":28,"Index":26,"Attempt":0,"Partition ID":26,"Launch Time":1750996689358,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996690009,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"199578411","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"635","Value":"20528","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1593","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"558682","Value":"39685693","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"638","Value":"20815","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"637","Value":"20749","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"199578411","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"48762","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":1278,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3107509,"Value":1095566753,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":640,"Value":21890,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":88440878,"Value":7961089238,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":57923,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1593,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":558682,"Value":39685693,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":842562,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":199578411,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3107509,"Executor Run Time":640,"Executor CPU Time":88440878,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":558682,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":33,"Index":31,"Attempt":0,"Partition ID":31,"Launch Time":1750996690025,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":29,"Index":27,"Attempt":0,"Partition ID":27,"Launch Time":1750996689460,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996690026,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"206970204","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"553","Value":"21081","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1652","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"480011","Value":"40165704","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"556","Value":"21371","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"556","Value":"21305","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"206970204","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"50568","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1280,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2201367,"Value":1097768120,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":558,"Value":22448,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":58822217,"Value":8019911455,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":60056,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1652,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":480011,"Value":40165704,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":873768,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":206970204,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2201367,"Executor Run Time":558,"Executor CPU Time":58822217,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":480011,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":34,"Index":32,"Attempt":0,"Partition ID":32,"Launch Time":1750996690481,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":30,"Index":28,"Attempt":0,"Partition ID":28,"Launch Time":1750996689852,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996690482,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"214361997","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"615","Value":"21696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1711","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"498721","Value":"40664425","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"618","Value":"21989","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"617","Value":"21922","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"214361997","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"52374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1282,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2451510,"Value":1100219630,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":620,"Value":23068,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":67008285,"Value":8086919740,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":62189,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1711,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":498721,"Value":40664425,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":904974,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":214361997,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2451510,"Executor Run Time":620,"Executor CPU Time":67008285,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":498721,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":35,"Index":33,"Attempt":0,"Partition ID":33,"Launch Time":1750996690524,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":31,"Index":29,"Attempt":0,"Partition ID":29,"Launch Time":1750996689972,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996690525,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"221753790","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"538","Value":"22234","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"30","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"486685","Value":"41151110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"541","Value":"22530","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"30","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"541","Value":"22463","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"221753790","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"54180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1284,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2339181,"Value":1102558811,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":543,"Value":23611,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":70482390,"Value":8157402130,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2131,"Value":64320,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1770,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":486685,"Value":41151110,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":936180,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":221753790,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2339181,"Executor Run Time":543,"Executor CPU Time":70482390,"Peak Execution Memory":0,"Result Size":2131,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":486685,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":33,"Index":31,"Attempt":0,"Partition ID":31,"Launch Time":1750996690025,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996690586,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"229145583","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"549","Value":"22783","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1829","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"31","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"327013","Value":"41478123","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"551","Value":"23081","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"31","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"551","Value":"23014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"229145583","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"55986","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1286,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2132530,"Value":1104691341,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":553,"Value":24164,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":65673017,"Value":8223075147,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":66453,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1829,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":31,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":327013,"Value":41478123,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":967386,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":229145583,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2132530,"Executor Run Time":553,"Executor CPU Time":65673017,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":327013,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":32,"Index":30,"Attempt":0,"Partition ID":30,"Launch Time":1750996690008,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996690612,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"236537376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"589","Value":"23372","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"512","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"522343","Value":"42000466","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"592","Value":"23673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"591","Value":"23605","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"236537376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"57792","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1288,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2594291,"Value":1107285632,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":594,"Value":24758,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":77709899,"Value":8300785046,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":68586,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1888,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":522343,"Value":42000466,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":998592,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":236537376,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2594291,"Executor Run Time":594,"Executor CPU Time":77709899,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":522343,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":35,"Index":33,"Attempt":0,"Partition ID":33,"Launch Time":1750996690524,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996690941,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"4927862","Value":"241465238","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"403","Value":"23775","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"528","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"1947","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"370161","Value":"42370627","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"405","Value":"24078","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"405","Value":"24010","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"4927862","Value":"241465238","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1204","Value":"58996","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1290,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2532672,"Value":1109818304,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":407,"Value":25165,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":43734165,"Value":8344519211,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2131,"Value":70717,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":1947,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":370161,"Value":42370627,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":20804,"Value":1019396,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":4927862,"Value":241465238,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2532672,"Executor Run Time":407,"Executor CPU Time":43734165,"Peak Execution Memory":0,"Result Size":2131,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":370161,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":20804,"Records Read":4927862},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":34,"Index":32,"Attempt":0,"Partition ID":32,"Launch Time":1750996690481,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996691007,"Failed":false,"Killed":false,"Accumulables":[{"ID":133,"Name":"number of output rows","Update":"7391793","Value":"248857031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Update":"512","Value":"24287","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Update":"16","Value":"544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Update":"59","Value":"2006","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Update":"1","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Update":"497077","Value":"42867704","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"515","Value":"24593","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"514","Value":"24524","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Update":"7391793","Value":"248857031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Update":"1806","Value":"60802","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":1292,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2580103,"Value":1112398407,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Update":517,"Value":25682,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Update":61307555,"Value":8405826766,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Update":2133,"Value":72850,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":2006,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Update":497077,"Value":42867704,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Update":31206,"Value":1050602,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":248857031,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2580103,"Executor Run Time":517,"Executor CPU Time":61307555,"Peak Execution Memory":0,"Result Size":2133,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":497077,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":31206,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":34,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"FileScanRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750996683941,"Completion Time":1750996691009,"Accumulables":[{"ID":133,"Name":"number of output rows","Value":"248857031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"scan time","Value":"24287","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":144,"Name":"data size","Value":"544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"shuffle bytes written","Value":"2006","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"shuffle records written","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"shuffle write time","Value":"42867704","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Value":"24593","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Value":"24524","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"number of output rows","Value":"248857031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of input batches","Value":"60802","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":175,"Name":"internal.metrics.executorDeserializeTime","Value":1292,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.executorDeserializeCpuTime","Value":1112398407,"Internal":true,"Count Failed Values":true},{"ID":177,"Name":"internal.metrics.executorRunTime","Value":25682,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorCpuTime","Value":8405826766,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.resultSize","Value":72850,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.jvmGCTime","Value":85,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.resultSerializationTime","Value":7,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2006,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":34,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.shuffle.write.writeTime","Value":42867704,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.input.bytesRead","Value":1050602,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.input.recordsRead","Value":248857031,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1750996691013,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (10)\n+- == Final Plan ==\n * HashAggregate (6)\n +- ShuffleQueryStage (5), Statistics(sizeInBytes=544.0 B, rowCount=34)\n +- Exchange (4)\n +- * HashAggregate (3)\n +- * ColumnarToRow (2)\n +- Scan parquet (1)\n+- == Initial Plan ==\n HashAggregate (9)\n +- Exchange (8)\n +- HashAggregate (7)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput: []\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct<>\n\n(2) ColumnarToRow [codegen id : 1]\nInput: []\n\n(3) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#164L]\nResults [1]: [count#165L]\n\n(4) Exchange\nInput [1]: [count#165L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=39]\n\n(5) ShuffleQueryStage\nOutput [1]: [count#165L]\nArguments: 0\n\n(6) HashAggregate [codegen id : 2]\nInput [1]: [count#165L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#161L]\nResults [1]: [count(1)#161L AS count#162L]\n\n(7) HashAggregate\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#164L]\nResults [1]: [count#165L]\n\n(8) Exchange\nInput [1]: [count#165L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=27]\n\n(9) HashAggregate\nInput [1]: [count#165L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#161L]\nResults [1]: [count(1)#161L AS count#162L]\n\n(10) AdaptiveSparkPlan\nOutput [1]: [count#162L]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=39]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]","ReadSchema":"struct<>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":135,"metricType":"sum"},{"name":"scan time","accumulatorId":134,"metricType":"timing"},{"name":"metadata time","accumulatorId":136,"metricType":"timing"},{"name":"size of files read","accumulatorId":137,"metricType":"size"},{"name":"number of output rows","accumulatorId":133,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":173,"metricType":"sum"},{"name":"number of input batches","accumulatorId":174,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":169,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":170,"metricType":"timing"},{"name":"peak memory","accumulatorId":168,"metricType":"size"},{"name":"number of output rows","accumulatorId":167,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":172,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":171,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":166,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":164,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":158,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":165,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":159,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":156,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":153,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":162,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":155,"metricType":"sum"},{"name":"records read","accumulatorId":152,"metricType":"sum"},{"name":"local bytes read","accumulatorId":150,"metricType":"size"},{"name":"fetch wait time","accumulatorId":151,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":148,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":154,"metricType":"sum"},{"name":"local blocks read","accumulatorId":147,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":157,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":146,"metricType":"sum"},{"name":"data size","accumulatorId":144,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":160,"metricType":"size"},{"name":"number of partitions","accumulatorId":145,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":161,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":149,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":163,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":213,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":214,"metricType":"timing"},{"name":"peak memory","accumulatorId":212,"metricType":"size"},{"name":"number of output rows","accumulatorId":211,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":216,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":215,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":210,"metricType":"timing"}]}],"metadata":{},"metrics":[]}} +{"Event":"SparkListenerJobStart","Job ID":3,"Submission Time":1750996691059,"Stage Infos":[{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":34,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"FileScanRDD","Scope":"{\"id\":\"20\",\"name\":\"Scan parquet \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"26\",\"name\":\"mapPartitionsInternal\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"25\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[3,4],"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750996612238","spark.rdd.scope":"{\"id\":\"27\",\"name\":\"collect\"}","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_03_56_52","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.63.89","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-77d59097af8544ae-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-cc4d115f011443d787f03a71a476a745","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","__fetch_continuous_blocks_in_batch_enabled":"true","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"1","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.63.89","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750996611643","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"1","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.rdd.scope.noOverride":"true","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"26\",\"name\":\"mapPartitionsInternal\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"25\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750996691114,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750996612238","spark.rdd.scope":"{\"id\":\"27\",\"name\":\"collect\"}","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_03_56_52","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.63.89","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-77d59097af8544ae-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-cc4d115f011443d787f03a71a476a745","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","resource.executor.cores":"1","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","__fetch_continuous_blocks_in_batch_enabled":"true","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"1","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.63.89","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750996611643","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"1","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.rdd.scope.noOverride":"true","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":36,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750996691121,"Executor ID":"2","Host":"100.64.240.100","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":36,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750996691121,"Executor ID":"2","Host":"100.64.240.100","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996691241,"Failed":false,"Killed":false,"Accumulables":[{"ID":146,"Name":"remote blocks read","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":147,"Name":"local blocks read","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":148,"Name":"remote bytes read","Update":"1357","Value":"1357","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":150,"Name":"local bytes read","Update":"649","Value":"649","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":151,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":152,"Name":"records read","Update":"34","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":161,"Name":"remote reqs duration","Update":"49","Value":"49","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":210,"Name":"duration","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":211,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":214,"Name":"time in aggregation build","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":217,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":218,"Name":"internal.metrics.executorDeserializeCpuTime","Update":11885188,"Value":11885188,"Internal":true,"Count Failed Values":true},{"ID":219,"Name":"internal.metrics.executorRunTime","Update":93,"Value":93,"Internal":true,"Count Failed Values":true},{"ID":220,"Name":"internal.metrics.executorCpuTime","Update":73222345,"Value":73222345,"Internal":true,"Count Failed Values":true},{"ID":221,"Name":"internal.metrics.resultSize","Update":3995,"Value":3995,"Internal":true,"Count Failed Values":true},{"ID":228,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":229,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":230,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":1357,"Value":1357,"Internal":true,"Count Failed Values":true},{"ID":231,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":232,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":649,"Value":649,"Internal":true,"Count Failed Values":true},{"ID":233,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":234,"Name":"internal.metrics.shuffle.read.recordsRead","Update":34,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":235,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":236,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":237,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":238,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":239,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":240,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":241,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":242,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":243,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":49,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":244,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":11885188,"Executor Run Time":93,"Executor CPU Time":73222345,"Peak Execution Memory":0,"Result Size":3995,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":23,"Local Blocks Fetched":11,"Fetch Wait Time":0,"Remote Bytes Read":1357,"Remote Bytes Read To Disk":0,"Local Bytes Read":649,"Total Records Read":34,"Remote Requests Duration":49,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"26\",\"name\":\"mapPartitionsInternal\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"25\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3615)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750996691114,"Completion Time":1750996691242,"Accumulables":[{"ID":146,"Name":"remote blocks read","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":147,"Name":"local blocks read","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":148,"Name":"remote bytes read","Value":"1357","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":150,"Name":"local bytes read","Value":"649","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":151,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":152,"Name":"records read","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":161,"Name":"remote reqs duration","Value":"49","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":210,"Name":"duration","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":211,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":214,"Name":"time in aggregation build","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":217,"Name":"internal.metrics.executorDeserializeTime","Value":18,"Internal":true,"Count Failed Values":true},{"ID":218,"Name":"internal.metrics.executorDeserializeCpuTime","Value":11885188,"Internal":true,"Count Failed Values":true},{"ID":219,"Name":"internal.metrics.executorRunTime","Value":93,"Internal":true,"Count Failed Values":true},{"ID":220,"Name":"internal.metrics.executorCpuTime","Value":73222345,"Internal":true,"Count Failed Values":true},{"ID":221,"Name":"internal.metrics.resultSize","Value":3995,"Internal":true,"Count Failed Values":true},{"ID":228,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":23,"Internal":true,"Count Failed Values":true},{"ID":229,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":11,"Internal":true,"Count Failed Values":true},{"ID":230,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":1357,"Internal":true,"Count Failed Values":true},{"ID":231,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":232,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":649,"Internal":true,"Count Failed Values":true},{"ID":233,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":234,"Name":"internal.metrics.shuffle.read.recordsRead","Value":34,"Internal":true,"Count Failed Values":true},{"ID":235,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":236,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":237,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":238,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":239,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":240,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":241,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":242,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":243,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":49,"Internal":true,"Count Failed Values":true},{"ID":244,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":3,"Completion Time":1750996691242,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1750996691244,"errorMessage":""} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":2,"rootExecutionId":2,"description":"parquet at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (6)\n+- Execute InsertIntoHadoopFsRelationCommand (5)\n +- WriteFiles (4)\n +- Exchange (3)\n +- Project (2)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct\n\n(2) Project\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 03:57:59.158854 AS current_date#38]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(3) Exchange\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=65]\n\n(4) WriteFiles\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\n\n(5) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]\n\n(6) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"Exchange","simpleString":"Exchange RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=65]","children":[{"nodeName":"Project","simpleString":"Project [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 03:57:59.158854 AS current_date#38]","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [VendorID#0L,tpep_pickup_datetime#1,tpep_dropoff_datetime#2,passenger_count#3,trip_distance#4,RatecodeID#5,store_and_fwd_flag#6,PULocationID#7L,DOLocationID#8L,payment_type#9L,fare_amount#10,extra#11,mta_tax#12,tip_amount#13,tolls_amount#14,improvement_surcharge#15,total_amount#16,congestion_surcharge#17,airport_fee#18] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":282,"metricType":"sum"},{"name":"scan time","accumulatorId":281,"metricType":"timing"},{"name":"metadata time","accumulatorId":283,"metricType":"timing"},{"name":"size of files read","accumulatorId":284,"metricType":"size"},{"name":"number of output rows","accumulatorId":280,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":278,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":272,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":279,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":273,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":270,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":267,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":276,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":269,"metricType":"sum"},{"name":"records read","accumulatorId":266,"metricType":"sum"},{"name":"local bytes read","accumulatorId":264,"metricType":"size"},{"name":"fetch wait time","accumulatorId":265,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":262,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":268,"metricType":"sum"},{"name":"local blocks read","accumulatorId":261,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":271,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":260,"metricType":"sum"},{"name":"data size","accumulatorId":258,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":274,"metricType":"size"},{"name":"number of partitions","accumulatorId":259,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":275,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":263,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":277,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":256,"metricType":"timing"},{"name":"number of written files","accumulatorId":252,"metricType":"sum"},{"name":"job commit time","accumulatorId":257,"metricType":"timing"},{"name":"number of output rows","accumulatorId":254,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":255,"metricType":"sum"},{"name":"written output","accumulatorId":253,"metricType":"size"}]}],"metadata":{},"metrics":[]},"time":1750996691341,"modifiedConfigs":{},"jobTags":[]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (12)\n+- == Current Plan ==\n Execute InsertIntoHadoopFsRelationCommand (7)\n +- WriteFiles (6)\n +- ShuffleQueryStage (5)\n +- Exchange (4)\n +- * Project (3)\n +- * ColumnarToRow (2)\n +- Scan parquet (1)\n+- == Initial Plan ==\n Execute InsertIntoHadoopFsRelationCommand (11)\n +- WriteFiles (10)\n +- Exchange (9)\n +- Project (8)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct\n\n(2) ColumnarToRow [codegen id : 1]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(3) Project [codegen id : 1]\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 03:57:59.158854 AS current_date#38]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(4) Exchange\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=80]\n\n(5) ShuffleQueryStage\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: 0\n\n(6) WriteFiles\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\n\n(7) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]\n\n(8) Project\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 03:57:59.158854 AS current_date#38]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(9) Exchange\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=65]\n\n(10) WriteFiles\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\n\n(11) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]\n\n(12) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=80]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 03:57:59.158854 AS current_date#38]","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [VendorID#0L,tpep_pickup_datetime#1,tpep_dropoff_datetime#2,passenger_count#3,trip_distance#4,RatecodeID#5,store_and_fwd_flag#6,PULocationID#7L,DOLocationID#8L,payment_type#9L,fare_amount#10,extra#11,mta_tax#12,tip_amount#13,tolls_amount#14,improvement_surcharge#15,total_amount#16,congestion_surcharge#17,airport_fee#18] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":282,"metricType":"sum"},{"name":"scan time","accumulatorId":281,"metricType":"timing"},{"name":"metadata time","accumulatorId":283,"metricType":"timing"},{"name":"size of files read","accumulatorId":284,"metricType":"size"},{"name":"number of output rows","accumulatorId":280,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":308,"metricType":"sum"},{"name":"number of input batches","accumulatorId":309,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":307,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":305,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":299,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":306,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":300,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":297,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":294,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":303,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":296,"metricType":"sum"},{"name":"records read","accumulatorId":293,"metricType":"sum"},{"name":"local bytes read","accumulatorId":291,"metricType":"size"},{"name":"fetch wait time","accumulatorId":292,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":289,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":295,"metricType":"sum"},{"name":"local blocks read","accumulatorId":288,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":298,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":287,"metricType":"sum"},{"name":"data size","accumulatorId":285,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":301,"metricType":"size"},{"name":"number of partitions","accumulatorId":286,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":302,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":290,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":304,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":256,"metricType":"timing"},{"name":"number of written files","accumulatorId":252,"metricType":"sum"},{"name":"job commit time","accumulatorId":257,"metricType":"timing"},{"name":"number of output rows","accumulatorId":254,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":255,"metricType":"sum"},{"name":"written output","accumulatorId":253,"metricType":"size"}]}],"metadata":{},"metrics":[]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[282,101],[283,0],[284,3852134849]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[286,2]]} +{"Event":"SparkListenerJobStart","Job ID":4,"Submission Time":1750996691452,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":34,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"FileScanRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"36\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[5],"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750996612238","spark.rdd.scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_03_56_52","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.63.89","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-77d59097af8544ae-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-cc4d115f011443d787f03a71a476a745","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","__fetch_continuous_blocks_in_batch_enabled":"true","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"2","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.63.89","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750996611643","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"2","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.rdd.scope.noOverride":"true","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":34,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"FileScanRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"36\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750996691514,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750996612238","spark.rdd.scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_03_56_52","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.63.89","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-77d59097af8544ae-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-cc4d115f011443d787f03a71a476a745","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","resource.executor.cores":"1","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","__fetch_continuous_blocks_in_batch_enabled":"true","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"2","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.63.89","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750996611643","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"2","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.rdd.scope.noOverride":"true","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":37,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750996691521,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":38,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1750996691522,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":39,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1750996691522,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":40,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1750996691522,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":41,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1750996708099,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":40,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1750996691522,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996708100,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"7391793","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2741","Value":"2741","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"1299239496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"316938494","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"7391793","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"304769159","Value":"304769159","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6744","Value":"6744","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"7391793","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"1806","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10215504,"Value":10215504,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":16553,"Value":16553,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":14954702480,"Value":14954702480,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":2036,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":316938494,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":7391793,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":304769159,"Value":304769159,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":114444762,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":7391793,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2218104288,"JVMOffHeapMemory":118576640,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1515890,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1612128306,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16850645,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10042818560,"ProcessTreeJVMRSSMemory":2930069504,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":12,"MinorGCTime":77,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":77},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":10215504,"Executor Run Time":16553,"Executor CPU Time":14954702480,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":304769159,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":42,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1750996708271,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":38,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1750996691522,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996708272,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"14783586","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2980","Value":"5721","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"2598478992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"633876988","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"14783586","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"307321826","Value":"612090985","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6872","Value":"13616","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"14783586","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"3612","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":22,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":14252913,"Value":24468417,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":16719,"Value":33272,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":15343967441,"Value":30298669921,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2034,"Value":4070,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":59,"Value":96,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":633876988,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":14783586,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":307321826,"Value":612090985,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":228889524,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":14783586,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2231602448,"JVMOffHeapMemory":115249032,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10030317568,"ProcessTreeJVMRSSMemory":2759475200,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":12,"MinorGCTime":90,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":90},"Task Metrics":{"Executor Deserialize Time":22,"Executor Deserialize CPU Time":14252913,"Executor Run Time":16719,"Executor CPU Time":15343967441,"Peak Execution Memory":0,"Result Size":2034,"JVM GC Time":59,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":307321826,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":43,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1750996708359,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":37,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750996691521,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996708359,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"22175379","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2836","Value":"8557","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"3897718488","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"950815482","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"22175379","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"307715213","Value":"919806198","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"7021","Value":"20637","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"22175379","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"5418","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":21,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":14451312,"Value":38919729,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":16808,"Value":50080,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":15757252584,"Value":46055922505,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":6106,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":54,"Value":150,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":950815482,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":22175379,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":307715213,"Value":919806198,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":343334286,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":22175379,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2381026744,"JVMOffHeapMemory":115309536,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":2909048832,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":11,"MinorGCTime":86,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":86},"Task Metrics":{"Executor Deserialize Time":21,"Executor Deserialize CPU Time":14451312,"Executor Run Time":16808,"Executor CPU Time":15757252584,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":54,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":307715213,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":44,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1750996708500,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":39,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1750996691522,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996708501,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"29567172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3282","Value":"11839","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"5196957984","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"1267753976","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"29567172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"303237004","Value":"1223043202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"7153","Value":"27790","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"29567172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"7224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":79,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":13336857,"Value":52256586,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":16951,"Value":67031,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":15240034620,"Value":61295957125,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":8142,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":45,"Value":195,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":1267753976,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":29567172,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":303237004,"Value":1223043202,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":457779048,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":29567172,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2335468952,"JVMOffHeapMemory":115177272,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":2884853760,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":10,"MinorGCTime":76,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":76},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":13336857,"Executor Run Time":16951,"Executor CPU Time":15240034620,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":45,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":303237004,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":45,"Index":8,"Attempt":0,"Partition ID":8,"Launch Time":1750996723439,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":42,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1750996708271,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996723439,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"36958965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2640","Value":"14479","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"6496197480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"1584692470","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"36958965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"304232921","Value":"1527276123","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5799","Value":"33589","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"36958965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"9030","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":81,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2893929,"Value":55150515,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15158,"Value":82189,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13758236893,"Value":75054194018,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2034,"Value":10176,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":9,"Value":204,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":1584692470,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":36958965,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":304232921,"Value":1527276123,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":572223810,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":36958965,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2144591136,"JVMOffHeapMemory":113726568,"OnHeapExecutionMemory":1342177024,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1343196652,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10030317568,"ProcessTreeJVMRSSMemory":4062052352,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":13,"MinorGCTime":95,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":95},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2893929,"Executor Run Time":15158,"Executor CPU Time":13758236893,"Peak Execution Memory":0,"Result Size":2034,"JVM GC Time":9,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":304232921,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":46,"Index":9,"Attempt":0,"Partition ID":9,"Launch Time":1750996723535,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":43,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1750996708359,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996723535,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"44350758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2445","Value":"16924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"7795436976","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"1901630964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"44350758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"302607245","Value":"1829883368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5764","Value":"39353","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"44350758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"10836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":84,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3058608,"Value":58209123,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15167,"Value":97356,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13938440422,"Value":88992634440,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":12212,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":3,"Value":207,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":1901630964,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":44350758,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":302607245,"Value":1829883368,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":686668572,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":44350758,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4222326200,"JVMOffHeapMemory":116745880,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4466999296,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":11,"MinorGCTime":86,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":86},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3058608,"Executor Run Time":15167,"Executor CPU Time":13938440422,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":3,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":302607245,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":47,"Index":10,"Attempt":0,"Partition ID":10,"Launch Time":1750996723989,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":41,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1750996708099,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996723990,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"51742551","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3089","Value":"20013","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"9094676472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"2218569458","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"51742551","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"305865371","Value":"2135748739","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6377","Value":"45730","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"51742551","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"12642","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":86,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2388910,"Value":60598033,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15881,"Value":113237,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13979487765,"Value":102972122205,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":14248,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":5,"Value":212,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":2218569458,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":51742551,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":305865371,"Value":2135748739,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":801113334,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":51742551,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3484784096,"JVMOffHeapMemory":119741560,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1515890,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1612128306,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16850645,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10042818560,"ProcessTreeJVMRSSMemory":4384817152,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":13,"MinorGCTime":82,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":82},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2388910,"Executor Run Time":15881,"Executor CPU Time":13979487765,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":5,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":305865371,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":48,"Index":11,"Attempt":0,"Partition ID":11,"Launch Time":1750996724041,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":44,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1750996708500,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996724042,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"59134344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2892","Value":"22905","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"10393915968","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"2535507952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"59134344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"306376295","Value":"2442125034","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6178","Value":"51908","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"59134344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"14448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":88,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2808342,"Value":63406375,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15532,"Value":128769,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13858966162,"Value":116831088367,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":16284,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":2,"Value":214,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":2535507952,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":59134344,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":306376295,"Value":2442125034,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":915558096,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":59134344,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4191448472,"JVMOffHeapMemory":113460104,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4441374720,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":10,"MinorGCTime":76,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":76},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2808342,"Executor Run Time":15532,"Executor CPU Time":13858966162,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":2,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":306376295,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":49,"Index":12,"Attempt":0,"Partition ID":12,"Launch Time":1750996738236,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":46,"Index":9,"Attempt":0,"Partition ID":9,"Launch Time":1750996723535,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996738236,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"66526137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2636","Value":"25541","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"11693155464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"2852446446","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"66526137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"301789626","Value":"2743914660","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5258","Value":"57166","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"66526137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"16254","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":90,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2558466,"Value":65964841,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":14692,"Value":143461,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13114557069,"Value":129945645436,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":18320,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":5,"Value":219,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":2852446446,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":66526137,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":301789626,"Value":2743914660,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1030002858,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":66526137,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3711326752,"JVMOffHeapMemory":114566768,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4524027904,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":14,"MinorGCTime":94,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":94},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2558466,"Executor Run Time":14692,"Executor CPU Time":13114557069,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":5,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":301789626,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":50,"Index":13,"Attempt":0,"Partition ID":13,"Launch Time":1750996738497,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":45,"Index":8,"Attempt":0,"Partition ID":8,"Launch Time":1750996723439,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996738497,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"73917930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2854","Value":"28395","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"12992394960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"3169384940","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"73917930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"302291200","Value":"3046205860","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5558","Value":"62724","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"73917930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"18060","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":92,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2469302,"Value":68434143,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15050,"Value":158511,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13242501780,"Value":143188147216,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2034,"Value":20354,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":7,"Value":226,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":3169384940,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":73917930,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":302291200,"Value":3046205860,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1144447620,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":73917930,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2056085088,"JVMOffHeapMemory":114760728,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10030317568,"ProcessTreeJVMRSSMemory":4115197952,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":18,"MinorGCTime":106,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":106},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2469302,"Executor Run Time":15050,"Executor CPU Time":13242501780,"Peak Execution Memory":0,"Result Size":2034,"JVM GC Time":7,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":302291200,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":51,"Index":14,"Attempt":0,"Partition ID":14,"Launch Time":1750996738632,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":47,"Index":10,"Attempt":0,"Partition ID":10,"Launch Time":1750996723989,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996738633,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"81309723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2393","Value":"30788","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"14291634456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"3486323434","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"81309723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"305092006","Value":"3351297866","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5140","Value":"67864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"81309723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"19866","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":95,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3000128,"Value":71434271,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":14632,"Value":173143,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13237134126,"Value":156425281342,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2079,"Value":22433,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":56,"Value":282,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":3486323434,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":81309723,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":305092006,"Value":3351297866,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1258892382,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":81309723,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2082934152,"JVMOffHeapMemory":120819192,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1515890,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1612128306,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16850645,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10042818560,"ProcessTreeJVMRSSMemory":4412616704,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":36,"MinorGCTime":136,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":136},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3000128,"Executor Run Time":14632,"Executor CPU Time":13237134126,"Peak Execution Memory":0,"Result Size":2079,"JVM GC Time":56,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":305092006,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":52,"Index":15,"Attempt":0,"Partition ID":15,"Launch Time":1750996738813,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":48,"Index":11,"Attempt":0,"Partition ID":11,"Launch Time":1750996724041,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996738813,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"88701516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2843","Value":"33631","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"15590873952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"3803261928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"88701516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"304585792","Value":"3655883658","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5484","Value":"73348","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"88701516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"21672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":97,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2607975,"Value":74042246,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":14763,"Value":187906,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":12972366326,"Value":169397647668,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":24469,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":4,"Value":286,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":3803261928,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":88701516,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":304585792,"Value":3655883658,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1373337144,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":88701516,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3640604448,"JVMOffHeapMemory":114657440,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4528726016,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":13,"MinorGCTime":82,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":82},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2607975,"Executor Run Time":14763,"Executor CPU Time":12972366326,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":4,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":304585792,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":53,"Index":16,"Attempt":0,"Partition ID":16,"Launch Time":1750996753325,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":49,"Index":12,"Attempt":0,"Partition ID":12,"Launch Time":1750996738236,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996753327,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"96093309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2905","Value":"36536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"16890113448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"4120200422","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"96093309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"303518348","Value":"3959402006","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5744","Value":"79092","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"96093309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"23478","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":99,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2386145,"Value":76428391,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15081,"Value":202987,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13254074629,"Value":182651722297,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":26505,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":9,"Value":295,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":4120200422,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":96093309,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":303518348,"Value":3959402006,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1487781906,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":96093309,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2931768344,"JVMOffHeapMemory":115321952,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4580102144,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":17,"MinorGCTime":103,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":103},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2386145,"Executor Run Time":15081,"Executor CPU Time":13254074629,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":9,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":303518348,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":54,"Index":17,"Attempt":0,"Partition ID":17,"Launch Time":1750996753434,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":50,"Index":13,"Attempt":0,"Partition ID":13,"Launch Time":1750996738497,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996753434,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"103485102","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2651","Value":"39187","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"18189352944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"4437138916","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"103485102","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"428849145","Value":"4388251151","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5336","Value":"84428","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"103485102","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"25284","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":102,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3661505,"Value":80089896,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":14925,"Value":217912,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13209385671,"Value":195861107968,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2077,"Value":28582,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":9,"Value":304,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":4437138916,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":103485102,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":428849145,"Value":4388251151,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1602226668,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":103485102,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3832311384,"JVMOffHeapMemory":115439216,"OnHeapExecutionMemory":1476394720,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1477414348,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10030317568,"ProcessTreeJVMRSSMemory":4182876160,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":19,"MinorGCTime":112,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":112},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3661505,"Executor Run Time":14925,"Executor CPU Time":13209385671,"Peak Execution Memory":0,"Result Size":2077,"JVM GC Time":9,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":428849145,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":55,"Index":18,"Attempt":0,"Partition ID":18,"Launch Time":1750996753637,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":51,"Index":14,"Attempt":0,"Partition ID":14,"Launch Time":1750996738632,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996753637,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"110876895","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2725","Value":"41912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"19488592440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"4754077410","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"110876895","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"299436815","Value":"4687687966","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5605","Value":"90033","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"110876895","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"27090","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":104,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2675254,"Value":82765150,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":14996,"Value":232908,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13301690841,"Value":209162798809,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":30618,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":47,"Value":351,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":4754077410,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":110876895,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":299436815,"Value":4687687966,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1716671430,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":110876895,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2396019080,"JVMOffHeapMemory":121600440,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1515890,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1612128306,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16850645,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10042818560,"ProcessTreeJVMRSSMemory":4431302656,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":58,"MinorGCTime":185,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":185},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2675254,"Executor Run Time":14996,"Executor CPU Time":13301690841,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":47,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":299436815,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":56,"Index":19,"Attempt":0,"Partition ID":19,"Launch Time":1750996753789,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":52,"Index":15,"Attempt":0,"Partition ID":15,"Launch Time":1750996738813,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996753789,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"118268688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2858","Value":"44770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"20787831936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"5071015904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"118268688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"306737741","Value":"4994425707","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5625","Value":"95658","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"118268688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"28896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":106,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2945977,"Value":85711127,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":14966,"Value":247874,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13137484200,"Value":222300283009,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2079,"Value":32697,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":18,"Value":369,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":5071015904,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":118268688,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":306737741,"Value":4994425707,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1831116192,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":118268688,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2052061272,"JVMOffHeapMemory":115303696,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4573601792,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":20,"MinorGCTime":99,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":99},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2945977,"Executor Run Time":14966,"Executor CPU Time":13137484200,"Peak Execution Memory":0,"Result Size":2079,"JVM GC Time":18,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":306737741,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":57,"Index":20,"Attempt":0,"Partition ID":20,"Launch Time":1750996768392,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":53,"Index":16,"Attempt":0,"Partition ID":16,"Launch Time":1750996753325,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996768393,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2860","Value":"47630","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"22087071432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"5387954398","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"300324327","Value":"5294750034","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5723","Value":"101381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"125660481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"30702","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":108,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2849052,"Value":88560179,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15057,"Value":262931,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13265129341,"Value":235565412350,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":34733,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":6,"Value":375,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":5387954398,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":125660481,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":300324327,"Value":5294750034,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":1945560954,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":125660481,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3185219944,"JVMOffHeapMemory":116173480,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4603052032,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":20,"MinorGCTime":109,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":109},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2849052,"Executor Run Time":15057,"Executor CPU Time":13265129341,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":6,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":300324327,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":58,"Index":21,"Attempt":0,"Partition ID":21,"Launch Time":1750996768594,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":55,"Index":18,"Attempt":0,"Partition ID":18,"Launch Time":1750996753637,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996768595,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"133052274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2769","Value":"50399","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"23386310928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"5704892892","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"133052274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"300406015","Value":"5595156049","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5432","Value":"106813","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"133052274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"32508","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":110,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2181040,"Value":90741219,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":14949,"Value":277880,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13232060046,"Value":248797472396,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":36769,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":2,"Value":377,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":5704892892,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":133052274,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":300406015,"Value":5595156049,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":2060005716,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":133052274,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4153432456,"JVMOffHeapMemory":122255896,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1515890,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1612128306,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16850645,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10042818560,"ProcessTreeJVMRSSMemory":4501340160,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":58,"MinorGCTime":185,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":185},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2181040,"Executor Run Time":14949,"Executor CPU Time":13232060046,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":2,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":300406015,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":59,"Index":22,"Attempt":0,"Partition ID":22,"Launch Time":1750996768852,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":54,"Index":17,"Attempt":0,"Partition ID":17,"Launch Time":1750996753434,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996768853,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"140444067","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3119","Value":"53518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"24685550424","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"6021831386","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"140444067","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"305639722","Value":"5900795771","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5871","Value":"112684","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"140444067","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"34314","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":112,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2310572,"Value":93051791,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15410,"Value":293290,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13335489308,"Value":262132961704,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":1991,"Value":38760,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":6021831386,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":140444067,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":305639722,"Value":5900795771,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":2174450478,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":140444067,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3749760944,"JVMOffHeapMemory":116142344,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10030317568,"ProcessTreeJVMRSSMemory":4195008512,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":21,"MinorGCTime":115,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":115},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2310572,"Executor Run Time":15410,"Executor CPU Time":13335489308,"Peak Execution Memory":0,"Result Size":1991,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":305639722,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":60,"Index":23,"Attempt":0,"Partition ID":23,"Launch Time":1750996769604,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":56,"Index":19,"Attempt":0,"Partition ID":19,"Launch Time":1750996753789,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996769605,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"147835860","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3438","Value":"56956","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"25984789920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"6338769880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"147835860","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"521547915","Value":"6422343686","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6255","Value":"118939","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"147835860","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"36120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":114,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2582278,"Value":95634069,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15806,"Value":309096,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13203177048,"Value":275336138752,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":40796,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":38,"Value":415,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":6338769880,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":147835860,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":521547915,"Value":6422343686,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":2288895240,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":147835860,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3217468656,"JVMOffHeapMemory":116054576,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4602855424,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":39,"MinorGCTime":138,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":138},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2582278,"Executor Run Time":15806,"Executor CPU Time":13203177048,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":38,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":521547915,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":61,"Index":24,"Attempt":0,"Partition ID":24,"Launch Time":1750996783198,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":58,"Index":21,"Attempt":0,"Partition ID":21,"Launch Time":1750996768594,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996783199,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"155227653","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2493","Value":"59449","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"27284029416","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"6655708374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"155227653","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"298931240","Value":"6721274926","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5130","Value":"124069","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"155227653","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"37926","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":116,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2067419,"Value":97701488,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":14596,"Value":323692,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13116836386,"Value":288452975138,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":42832,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":5,"Value":420,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":6655708374,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":155227653,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":298931240,"Value":6721274926,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":2403340002,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":155227653,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3627126144,"JVMOffHeapMemory":122824880,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1515890,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1612128306,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16850645,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10042818560,"ProcessTreeJVMRSSMemory":4598587392,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":61,"MinorGCTime":192,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":192},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2067419,"Executor Run Time":14596,"Executor CPU Time":13116836386,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":5,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":298931240,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":62,"Index":25,"Attempt":0,"Partition ID":25,"Launch Time":1750996783503,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":57,"Index":20,"Attempt":0,"Partition ID":20,"Launch Time":1750996768392,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996783503,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"162619446","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2923","Value":"62372","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"28583268912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"6972646868","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"162619446","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"301581287","Value":"7022856213","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5778","Value":"129847","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"162619446","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"39732","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":118,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2249735,"Value":99951223,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15103,"Value":338795,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13232395225,"Value":301685370363,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":44868,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":7,"Value":427,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":6972646868,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":162619446,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":301581287,"Value":7022856213,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":2517784764,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":162619446,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3023950416,"JVMOffHeapMemory":116692792,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4613107712,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":23,"MinorGCTime":116,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":116},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2249735,"Executor Run Time":15103,"Executor CPU Time":13232395225,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":7,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":301581287,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":63,"Index":26,"Attempt":0,"Partition ID":26,"Launch Time":1750996784335,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":59,"Index":22,"Attempt":0,"Partition ID":22,"Launch Time":1750996768852,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996784336,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"170011239","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3111","Value":"65483","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"29882508408","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"7289585362","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"170011239","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"304881007","Value":"7327737220","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6029","Value":"135876","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"170011239","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"41538","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":120,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2411413,"Value":102362636,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15475,"Value":354270,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13442155213,"Value":315127525576,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2034,"Value":46902,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":9,"Value":436,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":7289585362,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":170011239,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":304881007,"Value":7327737220,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":2632229526,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":170011239,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3485595360,"JVMOffHeapMemory":117078568,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10030317568,"ProcessTreeJVMRSSMemory":4427907072,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":24,"MinorGCTime":124,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":124},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2411413,"Executor Run Time":15475,"Executor CPU Time":13442155213,"Peak Execution Memory":0,"Result Size":2034,"JVM GC Time":9,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":304881007,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":64,"Index":27,"Attempt":0,"Partition ID":27,"Launch Time":1750996784667,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":60,"Index":23,"Attempt":0,"Partition ID":23,"Launch Time":1750996769604,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996784668,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"177403032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3025","Value":"68508","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"31181747904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"7606523856","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"177403032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"310387485","Value":"7638124705","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5742","Value":"141618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"177403032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"43344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":122,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2347076,"Value":104709712,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15054,"Value":369324,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13078789059,"Value":328206314635,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":48938,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":6,"Value":442,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":7606523856,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":177403032,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":310387485,"Value":7638124705,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":2746674288,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":177403032,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1979568704,"JVMOffHeapMemory":116885312,"OnHeapExecutionMemory":1409285872,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1410305500,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4648292352,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":42,"MinorGCTime":144,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":144},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2347076,"Executor Run Time":15054,"Executor CPU Time":13078789059,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":6,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":310387485,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":65,"Index":28,"Attempt":0,"Partition ID":28,"Launch Time":1750996798772,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":61,"Index":24,"Attempt":0,"Partition ID":24,"Launch Time":1750996783198,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996798773,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"184794825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3483","Value":"71991","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"32480987400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"7923462350","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"184794825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"302338794","Value":"7940463499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6143","Value":"147761","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"184794825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"45150","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":124,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2133841,"Value":106843553,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15566,"Value":384890,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13065084981,"Value":341271399616,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":50974,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":48,"Value":490,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":7923462350,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":184794825,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":302338794,"Value":7940463499,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":2861119050,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":184794825,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1988366472,"JVMOffHeapMemory":123328512,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1515890,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1612128306,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16850645,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10042818560,"ProcessTreeJVMRSSMemory":4634320896,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":83,"MinorGCTime":240,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":240},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2133841,"Executor Run Time":15566,"Executor CPU Time":13065084981,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":48,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":302338794,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":66,"Index":29,"Attempt":0,"Partition ID":29,"Launch Time":1750996798867,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":62,"Index":25,"Attempt":0,"Partition ID":25,"Launch Time":1750996783503,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996798868,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"192186618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3257","Value":"75248","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"33780226896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"8240400844","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"192186618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"302418190","Value":"8242881689","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"6087","Value":"153848","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"192186618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"46956","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":126,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2207037,"Value":109050590,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15357,"Value":400247,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13182309591,"Value":354453709207,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":53010,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":7,"Value":497,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":8240400844,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":192186618,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":302418190,"Value":8242881689,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":2975563812,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":192186618,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3839742544,"JVMOffHeapMemory":117652728,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4622299136,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":26,"MinorGCTime":123,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":123},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2207037,"Executor Run Time":15357,"Executor CPU Time":13182309591,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":7,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":302418190,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":67,"Index":30,"Attempt":0,"Partition ID":30,"Launch Time":1750996799508,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":64,"Index":27,"Attempt":0,"Partition ID":27,"Launch Time":1750996784667,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996799508,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"199578411","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2856","Value":"78104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"35079466392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"8557339338","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"199578411","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"303902644","Value":"8546784333","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5510","Value":"159358","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"199578411","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"48762","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":128,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2285744,"Value":111336334,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":14831,"Value":415078,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":12997917848,"Value":367451627055,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2079,"Value":55089,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":7,"Value":504,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":8557339338,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":199578411,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":303902644,"Value":8546784333,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":3090008574,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":199578411,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2594034240,"JVMOffHeapMemory":117508232,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4660244480,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":44,"MinorGCTime":149,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":149},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2285744,"Executor Run Time":14831,"Executor CPU Time":12997917848,"Peak Execution Memory":0,"Result Size":2079,"JVM GC Time":7,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":303902644,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":68,"Index":31,"Attempt":0,"Partition ID":31,"Launch Time":1750996799682,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":63,"Index":26,"Attempt":0,"Partition ID":26,"Launch Time":1750996784335,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996799683,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"206970204","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3020","Value":"81124","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"36378705888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"8874277832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"206970204","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"305894104","Value":"8852678437","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5843","Value":"165201","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"206970204","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"50568","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":130,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2422663,"Value":113758997,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15338,"Value":430416,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13395843592,"Value":380847470647,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2034,"Value":57123,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":9,"Value":513,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":8874277832,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":206970204,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":305894104,"Value":8852678437,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":3204453336,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":206970204,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2847312288,"JVMOffHeapMemory":117659264,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611517,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10030317568,"ProcessTreeJVMRSSMemory":4553195520,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":27,"MinorGCTime":133,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":133},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2422663,"Executor Run Time":15338,"Executor CPU Time":13395843592,"Peak Execution Memory":0,"Result Size":2034,"JVM GC Time":9,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":305894104,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":69,"Index":32,"Attempt":0,"Partition ID":32,"Launch Time":1750996813919,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":66,"Index":29,"Attempt":0,"Partition ID":29,"Launch Time":1750996798867,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996813919,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"214361997","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3016","Value":"84140","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"37677945384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"9191216326","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"214361997","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"301311856","Value":"9153990293","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5790","Value":"170991","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"214361997","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"52374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":132,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2114997,"Value":115873994,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15043,"Value":445459,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13062339528,"Value":393909810175,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":59159,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":7,"Value":520,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":9191216326,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":214361997,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":301311856,"Value":9153990293,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":3318898098,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":214361997,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2401175616,"JVMOffHeapMemory":118048016,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611559,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4630704128,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":29,"MinorGCTime":130,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":130},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2114997,"Executor Run Time":15043,"Executor CPU Time":13062339528,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":7,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":301311856,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":70,"Index":33,"Attempt":0,"Partition ID":33,"Launch Time":1750996814053,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":65,"Index":28,"Attempt":0,"Partition ID":28,"Launch Time":1750996798772,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996814053,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"221753790","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3204","Value":"87344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"38977184880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"9508154820","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"221753790","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"301152660","Value":"9455142953","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5862","Value":"176853","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"221753790","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"54180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":134,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2178503,"Value":118052497,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15272,"Value":460731,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13111608178,"Value":407021418353,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":1993,"Value":61152,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":9508154820,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":221753790,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":301152660,"Value":9455142953,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":3433342860,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":221753790,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4064546952,"JVMOffHeapMemory":123784536,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1515890,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1612128306,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16850685,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10042818560,"ProcessTreeJVMRSSMemory":4645195776,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":83,"MinorGCTime":240,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":240},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2178503,"Executor Run Time":15272,"Executor CPU Time":13111608178,"Peak Execution Memory":0,"Result Size":1993,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":301152660,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":68,"Index":31,"Attempt":0,"Partition ID":31,"Launch Time":1750996799682,"Executor ID":"4","Host":"100.64.247.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996814450,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"229145583","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2579","Value":"89923","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"40276424376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"9825093314","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"229145583","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"308000390","Value":"9763143343","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5262","Value":"182115","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"229145583","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"55986","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":137,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3217330,"Value":121269827,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":14758,"Value":475489,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13195814104,"Value":420217232457,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2034,"Value":63186,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":8,"Value":528,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":9825093314,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":229145583,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":308000390,"Value":9763143343,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":3547787622,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":229145583,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2427037448,"JVMOffHeapMemory":118192696,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611557,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10030317568,"ProcessTreeJVMRSSMemory":4587307008,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":30,"MinorGCTime":141,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":141},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3217330,"Executor Run Time":14758,"Executor CPU Time":13195814104,"Peak Execution Memory":0,"Result Size":2034,"JVM GC Time":8,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":308000390,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":67,"Index":30,"Attempt":0,"Partition ID":30,"Launch Time":1750996799508,"Executor ID":"3","Host":"100.64.131.144","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996814483,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"236537376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2810","Value":"92733","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"41575663872","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"10142031808","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"236537376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"307776944","Value":"10070920287","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5561","Value":"187676","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"236537376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"57792","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":140,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3213542,"Value":124483369,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":14966,"Value":490455,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":13209797066,"Value":433427029523,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":1993,"Value":65179,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":10142031808,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":236537376,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":307776944,"Value":10070920287,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":3662232384,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":236537376,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3575011568,"JVMOffHeapMemory":118024760,"OnHeapExecutionMemory":1476394720,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1477414348,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611519,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4672458752,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":45,"MinorGCTime":151,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":151},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3213542,"Executor Run Time":14966,"Executor CPU Time":13209797066,"Peak Execution Memory":0,"Result Size":1993,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":307776944,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":70,"Index":33,"Attempt":0,"Partition ID":33,"Launch Time":1750996814053,"Executor ID":"2","Host":"100.64.240.100","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996824414,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"4927862","Value":"241465238","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"2388","Value":"95121","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"866159664","Value":"42441823536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"299195580","Value":"10441227388","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"4927862","Value":"241465238","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"299465410","Value":"10370385697","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"4240","Value":"191916","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"4927862","Value":"241465238","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1204","Value":"58996","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":142,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2026060,"Value":126509429,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":10351,"Value":500806,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":8633247772,"Value":442060277295,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":67215,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":6,"Value":534,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":299195580,"Value":10441227388,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":4927862,"Value":241465238,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":299465410,"Value":10370385697,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":76296508,"Value":3738528892,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":4927862,"Value":241465238,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2224608720,"JVMOffHeapMemory":124429480,"OnHeapExecutionMemory":1073741600,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1515890,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1075257490,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16850685,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10042818560,"ProcessTreeJVMRSSMemory":4666298368,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":86,"MinorGCTime":246,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":246},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2026060,"Executor Run Time":10351,"Executor CPU Time":8633247772,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":6,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":299195580,"Shuffle Write Time":299465410,"Shuffle Records Written":4927862},"Input Metrics":{"Bytes Read":76296508,"Records Read":4927862},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":69,"Index":32,"Attempt":0,"Partition ID":32,"Launch Time":1750996813919,"Executor ID":"1","Host":"100.64.190.213","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750996828930,"Failed":false,"Killed":false,"Accumulables":[{"ID":280,"Name":"number of output rows","Update":"7391793","Value":"248857031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Update":"3271","Value":"98392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Update":"1299239496","Value":"43741063032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Update":"316938494","Value":"10758165882","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Update":"7391793","Value":"248857031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Update":"370508733","Value":"10740894430","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Update":"5915","Value":"197831","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Update":"7391793","Value":"248857031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Update":"1806","Value":"60802","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":145,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3090750,"Value":129600179,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Update":15002,"Value":515808,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Update":12699278405,"Value":454759555700,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Update":2036,"Value":69251,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Update":6,"Value":540,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":316938494,"Value":10758165882,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7391793,"Value":248857031,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Update":370508733,"Value":10740894430,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Update":114444762,"Value":3852973654,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Update":7391793,"Value":248857031,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3277785152,"JVMOffHeapMemory":118720136,"OnHeapExecutionMemory":1610612416,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1019628,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1611632044,"OffHeapUnifiedMemory":0,"DirectPoolMemory":12611559,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":9966354432,"ProcessTreeJVMRSSMemory":4647936000,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":32,"MinorGCTime":136,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":136},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3090750,"Executor Run Time":15002,"Executor CPU Time":12699278405,"Peak Execution Memory":0,"Result Size":2036,"JVM GC Time":6,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":316938494,"Shuffle Write Time":370508733,"Shuffle Records Written":7391793},"Input Metrics":{"Bytes Read":114444762,"Records Read":7391793},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":34,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"FileScanRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"36\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750996691514,"Completion Time":1750996828931,"Accumulables":[{"ID":280,"Name":"number of output rows","Value":"248857031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":281,"Name":"scan time","Value":"98392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":285,"Name":"data size","Value":"43741063032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":304,"Name":"shuffle bytes written","Value":"10758165882","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"shuffle records written","Value":"248857031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"shuffle write time","Value":"10740894430","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"duration","Value":"197831","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":308,"Name":"number of output rows","Value":"248857031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"number of input batches","Value":"60802","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":310,"Name":"internal.metrics.executorDeserializeTime","Value":145,"Internal":true,"Count Failed Values":true},{"ID":311,"Name":"internal.metrics.executorDeserializeCpuTime","Value":129600179,"Internal":true,"Count Failed Values":true},{"ID":312,"Name":"internal.metrics.executorRunTime","Value":515808,"Internal":true,"Count Failed Values":true},{"ID":313,"Name":"internal.metrics.executorCpuTime","Value":454759555700,"Internal":true,"Count Failed Values":true},{"ID":314,"Name":"internal.metrics.resultSize","Value":69251,"Internal":true,"Count Failed Values":true},{"ID":315,"Name":"internal.metrics.jvmGCTime","Value":540,"Internal":true,"Count Failed Values":true},{"ID":316,"Name":"internal.metrics.resultSerializationTime","Value":8,"Internal":true,"Count Failed Values":true},{"ID":338,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":10758165882,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":248857031,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.shuffle.write.writeTime","Value":10740894430,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.input.bytesRead","Value":3852973654,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.input.recordsRead","Value":248857031,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":4,"Completion Time":1750996828932,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (12)\n+- == Final Plan ==\n Execute InsertIntoHadoopFsRelationCommand (7)\n +- WriteFiles (6)\n +- ShuffleQueryStage (5), Statistics(sizeInBytes=40.7 GiB, rowCount=2.49E+8)\n +- Exchange (4)\n +- * Project (3)\n +- * ColumnarToRow (2)\n +- Scan parquet (1)\n+- == Initial Plan ==\n Execute InsertIntoHadoopFsRelationCommand (11)\n +- WriteFiles (10)\n +- Exchange (9)\n +- Project (8)\n +- Scan parquet (1)\n\n\n(1) Scan parquet \nOutput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\nBatched: true\nLocation: InMemoryFileIndex [s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input]\nReadSchema: struct\n\n(2) ColumnarToRow [codegen id : 1]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(3) Project [codegen id : 1]\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 03:57:59.158854 AS current_date#38]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(4) Exchange\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=80]\n\n(5) ShuffleQueryStage\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: 0\n\n(6) WriteFiles\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\n\n(7) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]\n\n(8) Project\nOutput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 03:57:59.158854 AS current_date#38]\nInput [19]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18]\n\n(9) Exchange\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\nArguments: RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=65]\n\n(10) WriteFiles\nInput [20]: [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, current_date#38]\n\n(11) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]\n\n(12) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output, false, Parquet, [path=s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/output/], Overwrite, [VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, RatecodeID, store_and_fwd_flag, PULocationID, DOLocationID, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, improvement_surcharge, total_amount, congestion_surcharge, airport_fee, current_date]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange RoundRobinPartitioning(2), REPARTITION_BY_NUM, [plan_id=80]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [VendorID#0L, tpep_pickup_datetime#1, tpep_dropoff_datetime#2, passenger_count#3, trip_distance#4, RatecodeID#5, store_and_fwd_flag#6, PULocationID#7L, DOLocationID#8L, payment_type#9L, fare_amount#10, extra#11, mta_tax#12, tip_amount#13, tolls_amount#14, improvement_surcharge#15, total_amount#16, congestion_surcharge#17, airport_fee#18, 2025-06-27 03:57:59.158854 AS current_date#38]","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [VendorID#0L,tpep_pickup_datetime#1,tpep_dropoff_datetime#2,passenger_count#3,trip_distance#4,RatecodeID#5,store_and_fwd_flag#6,PULocationID#7L,DOLocationID#8L,payment_type#9L,fare_amount#10,extra#11,mta_tax#12,tip_amount#13,tolls_amount#14,improvement_surcharge#15,total_amount#16,congestion_surcharge#17,airport_fee#18] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://sparksense-ai-spark-20250624175205897100000006/taxi-trip/input], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":282,"metricType":"sum"},{"name":"scan time","accumulatorId":281,"metricType":"timing"},{"name":"metadata time","accumulatorId":283,"metricType":"timing"},{"name":"size of files read","accumulatorId":284,"metricType":"size"},{"name":"number of output rows","accumulatorId":280,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":308,"metricType":"sum"},{"name":"number of input batches","accumulatorId":309,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":307,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":305,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":299,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":306,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":300,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":297,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":294,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":303,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":296,"metricType":"sum"},{"name":"records read","accumulatorId":293,"metricType":"sum"},{"name":"local bytes read","accumulatorId":291,"metricType":"size"},{"name":"fetch wait time","accumulatorId":292,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":289,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":295,"metricType":"sum"},{"name":"local blocks read","accumulatorId":288,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":298,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":287,"metricType":"sum"},{"name":"data size","accumulatorId":285,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":301,"metricType":"size"},{"name":"number of partitions","accumulatorId":286,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":302,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":290,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":304,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":349,"metricType":"timing"},{"name":"number of written files","accumulatorId":345,"metricType":"sum"},{"name":"job commit time","accumulatorId":350,"metricType":"timing"},{"name":"number of output rows","accumulatorId":347,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":348,"metricType":"sum"},{"name":"written output","accumulatorId":346,"metricType":"size"}]}],"metadata":{},"metrics":[]}} +{"Event":"SparkListenerJobStart","Job ID":5,"Submission Time":1750996829446,"Stage Infos":[{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":34,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"FileScanRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"36\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":34,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":2,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"41\",\"name\":\"WriteFiles\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"43\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[6,7],"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750996612238","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_03_56_52","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.63.89","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-77d59097af8544ae-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-cc4d115f011443d787f03a71a476a745","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","__fetch_continuous_blocks_in_batch_enabled":"true","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"2","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.63.89","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750996611643","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"2","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":2,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"41\",\"name\":\"WriteFiles\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"43\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750996829448,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.submit.pyFiles":"","spark.local.dir":"/data","spark.app.startTime":"1750996612238","spark.rdd.compress":"True","spark.network.timeout":"2400","spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","spark.kubernetes.authenticate.executor.serviceAccountName":"spark-team-a","spark.hadoop.fs.s3a.connection.timeout":"1200000","spark.kubernetes.submitInDriver":"true","spark.hadoop.fs.s3a.aws.credentials.provider":"*********(redacted)","spark.kubernetes.authenticate.driver.serviceAccountName":"spark-team-a","spark.kubernetes.executor.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.sql.warehouse.dir":"file:/opt/spark/spark-warehouse","spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name":"taxi-trip","spark.driver.memory":"4g","spark.master":"k8s://https://172.20.0.1:443","spark.kubernetes.driver.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.driver.cores":"1","spark.kubernetes.driver.limit.cores":"1200m","spark.kubernetes.driver.label.version":"3.5.3","spark.kubernetes.submission.waitAppCompletion":"false","spark.app.name":"NewYorkTaxiData_2025_06_27_03_56_52","spark.executorEnv.SPARK_DRIVER_POD_IP":"100.64.63.89","spark.speculation":"false","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"taxi-trip-77d59097af8544ae-driver-svc.spark-team-a.svc","spark.driver.blockManager.port":"7079","spark.app.id":"spark-cc4d115f011443d787f03a71a476a745","spark.kubernetes.memoryOverheadFactor":"0.4","spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name":"taxi-trip","resource.executor.cores":"1","spark.kubernetes.driver.label.applicationId":"taxi-trip-nvme","spark.metrics.conf.executor.sink.prometheusServlet.path":"/metrics/executors/prometheus/","__fetch_continuous_blocks_in_batch_enabled":"true","spark.kubernetes.executor.label.applicationId":"taxi-trip-nvme","spark.kubernetes.driver.label.app":"taxi-trip","spark.kubernetes.executor.label.queue":"root.test","spark.sql.execution.root.id":"2","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.app":"taxi-trip","spark.kubernetes.container.image.pullPolicy":"IfNotPresent","spark.driver.port":"7078","spark.driver.bindAddress":"100.64.63.89","spark.kubernetes.resource.type":"python","spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet","spark.hadoop.fs.s3a.aws.credentials.provider.mapping":"*********(redacted)","spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.eventLog.rolling.maxFileSize":"64m","spark.kubernetes.namespace":"spark-team-a","spark.app.submitTime":"1750996611643","spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id":"77ac9ebd-3039-4134-be4c-26e238e4814f","spark.hadoop.fs.s3a.fast.upload":"true","spark.eventLog.dir":"s3a://sparksense-ai-spark-20250624175205897100000006/spark-event-logs","spark.executor.memoryOverhead":"4g","spark.kubernetes.driver.pod.name":"taxi-trip-driver","spark.executor.instances":"4","spark.sql.execution.id":"2","spark.executor.processTreeMetrics.enabled":"true","spark.hadoop.fs.s3a.readahead.range":"256K","spark.executor.memory":"4g","spark.kubernetes.driver.label.queue":"root.test","spark.eventLog.rolling.enabled":"true","spark.ui.prometheus.enabled":"true","spark.kubernetes.executor.limit.cores":"3400m","spark.executor.id":"driver","spark.executor.cores":"1","spark.hadoop.fs.s3a.input.fadvise":"random","spark.submit.deployMode":"client","spark.kubernetes.driver.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.container.image":"public.ecr.aws/data-on-eks/spark:3.5.3-scala2.12-java17-python3-ubuntu","spark.kubernetes.executor.podNamePrefix":"taxi-trip","spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator":"true","spark.driver.memoryOverhead":"4g","spark.metrics.conf.driver.sink.prometheusServlet.path":"/metrics/driver/prometheus/","spark.eventLog.enabled":"true","spark.hadoop.fs.s3a.connection.maximum":"200","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.kubernetes.executor.label.sparkoperator.k8s.io/mutated-by-spark-operator":"true","spark.kubernetes.executor.label.version":"3.5.3","spark.serializer.objectStreamReset":"100"}} +{"Event":"SparkListenerTaskStart","Stage ID":7,"Stage Attempt ID":0,"Task Info":{"Task ID":71,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750996829476,"Executor ID":"3","Host":"100.64.131.144","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":7,"Stage Attempt ID":0,"Task Info":{"Task ID":72,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1750996829476,"Executor ID":"1","Host":"100.64.190.213","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":7,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":72,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1750996829476,"Executor ID":"1","Host":"100.64.190.213","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750997072749,"Failed":false,"Killed":false,"Accumulables":[{"ID":287,"Name":"remote blocks read","Update":"25","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":288,"Name":"local blocks read","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":289,"Name":"remote bytes read","Update":"3952870588","Value":"3952870588","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":291,"Name":"local bytes read","Update":"1426239528","Value":"1426239528","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":292,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":293,"Name":"records read","Update":"124428513","Value":"124428513","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":302,"Name":"remote reqs duration","Update":"2963","Value":"2963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":349,"Name":"task commit time","Update":"3019","Value":"3019","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":351,"Name":"internal.metrics.executorDeserializeTime","Update":77,"Value":77,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.executorDeserializeCpuTime","Update":43500686,"Value":43500686,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.executorRunTime","Update":243182,"Value":243182,"Internal":true,"Count Failed Values":true},{"ID":354,"Name":"internal.metrics.executorCpuTime","Update":239137189565,"Value":239137189565,"Internal":true,"Count Failed Values":true},{"ID":355,"Name":"internal.metrics.resultSize","Update":4826,"Value":4826,"Internal":true,"Count Failed Values":true},{"ID":356,"Name":"internal.metrics.jvmGCTime","Update":199,"Value":199,"Internal":true,"Count Failed Values":true},{"ID":357,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":362,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":25,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":363,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":9,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":364,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":3952870588,"Value":3952870588,"Internal":true,"Count Failed Values":true},{"ID":365,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":366,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1426239528,"Value":1426239528,"Internal":true,"Count Failed Values":true},{"ID":367,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":368,"Name":"internal.metrics.shuffle.read.recordsRead","Update":124428513,"Value":124428513,"Internal":true,"Count Failed Values":true},{"ID":369,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":370,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":371,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":372,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":373,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":374,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":375,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":376,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":377,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":2963,"Value":2963,"Internal":true,"Count Failed Values":true},{"ID":378,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":384,"Name":"internal.metrics.output.bytesWritten","Update":2686645943,"Value":2686645943,"Internal":true,"Count Failed Values":true},{"ID":385,"Name":"internal.metrics.output.recordsWritten","Update":124428513,"Value":124428513,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3134745800,"JVMOffHeapMemory":134666456,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1364880,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1364880,"OffHeapUnifiedMemory":0,"DirectPoolMemory":330417329,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":10578984960,"ProcessTreeJVMRSSMemory":5163847680,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":61,"MinorGCTime":335,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":335},"Task Metrics":{"Executor Deserialize Time":77,"Executor Deserialize CPU Time":43500686,"Executor Run Time":243182,"Executor CPU Time":239137189565,"Peak Execution Memory":0,"Result Size":4826,"JVM GC Time":199,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":25,"Local Blocks Fetched":9,"Fetch Wait Time":0,"Remote Bytes Read":3952870588,"Remote Bytes Read To Disk":0,"Local Bytes Read":1426239528,"Total Records Read":124428513,"Remote Requests Duration":2963,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":2686645943,"Records Written":124428513},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":7,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":71,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1750996829476,"Executor ID":"3","Host":"100.64.131.144","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1750997116439,"Failed":false,"Killed":false,"Accumulables":[{"ID":287,"Name":"remote blocks read","Update":"26","Value":"51","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":288,"Name":"local blocks read","Update":"8","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":289,"Name":"remote bytes read","Update":"4111312660","Value":"8064183248","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":291,"Name":"local bytes read","Update":"1267743106","Value":"2693982634","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":292,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":293,"Name":"records read","Update":"124428518","Value":"248857031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":302,"Name":"remote reqs duration","Update":"3430","Value":"6393","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":349,"Name":"task commit time","Update":"2796","Value":"5815","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":351,"Name":"internal.metrics.executorDeserializeTime","Update":76,"Value":153,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.executorDeserializeCpuTime","Update":43670336,"Value":87171022,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.executorRunTime","Update":286876,"Value":530058,"Internal":true,"Count Failed Values":true},{"ID":354,"Name":"internal.metrics.executorCpuTime","Update":282995316871,"Value":522132506436,"Internal":true,"Count Failed Values":true},{"ID":355,"Name":"internal.metrics.resultSize","Update":4826,"Value":9652,"Internal":true,"Count Failed Values":true},{"ID":356,"Name":"internal.metrics.jvmGCTime","Update":169,"Value":368,"Internal":true,"Count Failed Values":true},{"ID":357,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":362,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":26,"Value":51,"Internal":true,"Count Failed Values":true},{"ID":363,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":8,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":364,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":4111312660,"Value":8064183248,"Internal":true,"Count Failed Values":true},{"ID":365,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":366,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1267743106,"Value":2693982634,"Internal":true,"Count Failed Values":true},{"ID":367,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":368,"Name":"internal.metrics.shuffle.read.recordsRead","Update":124428518,"Value":248857031,"Internal":true,"Count Failed Values":true},{"ID":369,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":370,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":371,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":372,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":373,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":374,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":375,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":376,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":377,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":3430,"Value":6393,"Internal":true,"Count Failed Values":true},{"ID":378,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":384,"Name":"internal.metrics.output.bytesWritten","Update":2686894618,"Value":5373540561,"Internal":true,"Count Failed Values":true},{"ID":385,"Name":"internal.metrics.output.recordsWritten","Update":124428518,"Value":248857031,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3631075952,"JVMOffHeapMemory":146413864,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1364880,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1364880,"OffHeapUnifiedMemory":0,"DirectPoolMemory":330367181,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":11808964608,"ProcessTreeJVMRSSMemory":5120544768,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":78,"MinorGCTime":320,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":320},"Task Metrics":{"Executor Deserialize Time":76,"Executor Deserialize CPU Time":43670336,"Executor Run Time":286876,"Executor CPU Time":282995316871,"Peak Execution Memory":0,"Result Size":4826,"JVM GC Time":169,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":26,"Local Blocks Fetched":8,"Fetch Wait Time":0,"Remote Bytes Read":4111312660,"Remote Bytes Read To Disk":0,"Local Bytes Read":1267743106,"Total Records Read":124428518,"Remote Requests Duration":3430,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":2686894618,"Records Written":124428518},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":2,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"41\",\"name\":\"WriteFiles\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"43\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":2,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:802)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1750996829448,"Completion Time":1750997116440,"Accumulables":[{"ID":287,"Name":"remote blocks read","Value":"51","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":288,"Name":"local blocks read","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":289,"Name":"remote bytes read","Value":"8064183248","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":291,"Name":"local bytes read","Value":"2693982634","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":292,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":293,"Name":"records read","Value":"248857031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":302,"Name":"remote reqs duration","Value":"6393","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":349,"Name":"task commit time","Value":"5815","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":351,"Name":"internal.metrics.executorDeserializeTime","Value":153,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.executorDeserializeCpuTime","Value":87171022,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.executorRunTime","Value":530058,"Internal":true,"Count Failed Values":true},{"ID":354,"Name":"internal.metrics.executorCpuTime","Value":522132506436,"Internal":true,"Count Failed Values":true},{"ID":355,"Name":"internal.metrics.resultSize","Value":9652,"Internal":true,"Count Failed Values":true},{"ID":356,"Name":"internal.metrics.jvmGCTime","Value":368,"Internal":true,"Count Failed Values":true},{"ID":357,"Name":"internal.metrics.resultSerializationTime","Value":5,"Internal":true,"Count Failed Values":true},{"ID":362,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":51,"Internal":true,"Count Failed Values":true},{"ID":363,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":17,"Internal":true,"Count Failed Values":true},{"ID":364,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":8064183248,"Internal":true,"Count Failed Values":true},{"ID":365,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":366,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2693982634,"Internal":true,"Count Failed Values":true},{"ID":367,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":368,"Name":"internal.metrics.shuffle.read.recordsRead","Value":248857031,"Internal":true,"Count Failed Values":true},{"ID":369,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":370,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":371,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":372,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":373,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":374,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":375,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":376,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":377,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":6393,"Internal":true,"Count Failed Values":true},{"ID":378,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":384,"Name":"internal.metrics.output.bytesWritten","Value":5373540561,"Internal":true,"Count Failed Values":true},{"ID":385,"Name":"internal.metrics.output.recordsWritten","Value":248857031,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}} +{"Event":"SparkListenerJobEnd","Job ID":5,"Completion Time":1750997116440,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[345,2],[350,4403],[347,248857031],[348,0],[346,5373540561]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":2,"time":1750997120851,"errorMessage":""} +{"Event":"SparkListenerApplicationEnd","Timestamp":1750997120852} diff --git a/examples/basic/history-server.conf b/examples/basic/history-server.conf index 29c5248..f5c070d 100644 --- a/examples/basic/history-server.conf +++ b/examples/basic/history-server.conf @@ -1 +1 @@ -spark.history.fs.logDirectory=/mnt/data/events \ No newline at end of file +spark.history.fs.logDirectory=/mnt/data/events diff --git a/examples/integrations/README.md b/examples/integrations/README.md new file mode 100644 index 0000000..795a592 --- /dev/null +++ b/examples/integrations/README.md @@ -0,0 +1,107 @@ +# AI Agent Integration Examples + +This directory contains comprehensive guides for integrating the Spark History Server MCP with various AI agent frameworks and platforms. + +## Available Integrations + +### ๐Ÿ”ง **Production AI Framework Integrations** +- **[LlamaIndex](llamaindex.md)** - RAG systems and knowledge bases for Spark data + - Vector indexing of Spark application data + - Query engines for performance analysis + - Real-time monitoring chat systems + - Custom embeddings for technical content + +- **[LangGraph](langgraph.md)** - Multi-agent workflows and state machines + - Complex analysis workflows + - Multi-agent monitoring systems + - Optimization recommendation pipelines + - State-based failure investigations + +## Quick Start Guide + +1. **Choose Your Platform**: Start with Claude Desktop for immediate interactive analysis +2. **Review Integration Guide**: Each guide includes complete setup instructions +3. **Test Locally**: Use the provided sample data and local Spark History Server +4. **Customize**: Adapt the examples to your specific use cases + +## Common Integration Patterns + +### **Interactive Analysis** +Perfect for ad-hoc investigation and exploration: +- Claude Desktop integration +- Jupyter notebook workflows +- Real-time query interfaces + +### **Automated Monitoring** +Ideal for production monitoring and alerting: +- LangChain monitoring agents +- Custom alerting systems +- Integration with existing monitoring tools + +### **Knowledge Systems** +Great for building organizational knowledge bases: +- LlamaIndex RAG systems +- Historical pattern analysis +- Performance regression detection + +### **Complex Workflows** +For sophisticated analysis pipelines: +- LangGraph state machines +- Multi-step optimization workflows +- Batch failure investigations + +## ๐Ÿงช Local Testing and Development + +For local testing and development, use the **MCP Inspector** instead of complex AI agent setups: + +- **[TESTING.md](../../TESTING.md)** - Complete guide for testing with MCP Inspector +- **Interactive Testing** - Use browser-based MCP Inspector for immediate tool testing +- **No Configuration Required** - Simple one-command setup for development + +The MCP Inspector provides the fastest way to test your MCP server locally before deploying to production with AI agents. + +## Best Practices + +### **Development** +1. Start with MCP Inspector for local testing +2. Use the sample Spark applications for development +3. Implement error handling and retries +4. Log all interactions for debugging + +### **Production** +1. Deploy using Kubernetes + Helm charts +2. Implement proper authentication +3. Add rate limiting and timeouts +4. Monitor agent performance and set up alerting + +### **Performance** +1. Batch API calls when possible +2. Cache frequently accessed data +3. Use appropriate similarity thresholds +4. Optimize query patterns + +## Sample Data + +All integration examples work with the provided sample data: +- **spark-bcec39f6201b42b9925124595baad260** - Successful ETL job +- **spark-110be3a8424d4a2789cb88134418217b** - Data processing job +- **spark-cc4d115f011443d787f03a71a476a745** - Multi-stage analytics job + +Use these applications to test your integrations before connecting to production data. + +## Contributing + +We welcome contributions to expand the integration examples: + +1. **New Framework Integrations**: Add support for additional AI frameworks +2. **Production Examples**: Share real-world deployment patterns +3. **Specialized Agents**: Contribute domain-specific analysis agents +4. **Best Practices**: Document lessons learned from production deployments + +See the main project [Contributing Guide](../../README.md#-contributing) for details. + +## Support + +- ๐Ÿ› **Issues**: [GitHub Issues](https://github.com/DeepDiagnostix-AI/spark-history-server-mcp/issues) +- ๐Ÿ’ก **Discussions**: [GitHub Discussions](https://github.com/DeepDiagnostix-AI/spark-history-server-mcp/discussions) +- ๐Ÿ“– **Documentation**: [Project Wiki](https://github.com/DeepDiagnostix-AI/spark-history-server-mcp/wiki) diff --git a/examples/integrations/langgraph.md b/examples/integrations/langgraph.md new file mode 100644 index 0000000..b930298 --- /dev/null +++ b/examples/integrations/langgraph.md @@ -0,0 +1,764 @@ +# LangGraph Integration + +This guide demonstrates how to build sophisticated Spark analysis workflows using LangGraph's state management and multi-agent capabilities. + +## Installation + +```bash +pip install langgraph langchain-openai +# or +uv add langgraph langchain-openai +``` + +## Basic Multi-Agent Spark Analysis + +### 1. Spark Analysis State Machine + +```python +from typing import Annotated, List, Dict, Any +from langgraph.graph import StateGraph, END +from langgraph.graph.message import add_messages +from langchain_core.messages import BaseMessage, HumanMessage, AIMessage +from langchain_openai import ChatOpenAI +import asyncio +from mcp import ClientSession + +class SparkAnalysisState: + """State for Spark analysis workflow.""" + messages: Annotated[List[BaseMessage], add_messages] + app_id: str + analysis_results: Dict[str, Any] + recommendations: List[str] + current_step: str + +class SparkAnalysisWorkflow: + def __init__(self, mcp_server_url: str): + self.mcp_url = mcp_server_url + self.llm = ChatOpenAI(model="gpt-4", temperature=0) + self.graph = self._build_graph() + + def _build_graph(self): + """Build the analysis workflow graph.""" + workflow = StateGraph(SparkAnalysisState) + + # Add nodes + workflow.add_node("collect_basic_info", self.collect_basic_info) + workflow.add_node("analyze_performance", self.analyze_performance) + workflow.add_node("identify_bottlenecks", self.identify_bottlenecks) + workflow.add_node("compare_with_historical", self.compare_with_historical) + workflow.add_node("generate_recommendations", self.generate_recommendations) + workflow.add_node("format_report", self.format_report) + + # Add edges + workflow.add_edge("collect_basic_info", "analyze_performance") + workflow.add_edge("analyze_performance", "identify_bottlenecks") + workflow.add_edge("identify_bottlenecks", "compare_with_historical") + workflow.add_edge("compare_with_historical", "generate_recommendations") + workflow.add_edge("generate_recommendations", "format_report") + workflow.add_edge("format_report", END) + + # Set entry point + workflow.set_entry_point("collect_basic_info") + + return workflow.compile() + + async def collect_basic_info(self, state: SparkAnalysisState): + """Collect basic application information.""" + async with ClientSession(self.mcp_url) as session: + app_info = await session.call_tool("get_application_info", {"app_id": state["app_id"]}) + jobs = await session.call_tool("get_jobs", {"spark_id": state["app_id"]}) + + state["analysis_results"]["basic_info"] = app_info + state["analysis_results"]["jobs"] = jobs + state["current_step"] = "basic_info_collected" + + state["messages"].append(AIMessage(content=f"Collected basic info for {state['app_id']}")) + return state + + async def analyze_performance(self, state: SparkAnalysisState): + """Analyze application performance metrics.""" + async with ClientSession(self.mcp_url) as session: + try: + executor_summary = await session.call_tool("get_executor_summary", {"spark_id": state["app_id"]}) + stages = await session.call_tool("get_stages", {"spark_id": state["app_id"]}) + + state["analysis_results"]["performance"] = { + "executor_summary": executor_summary, + "stages": stages + } + + # Analyze with LLM + analysis_prompt = f""" + Analyze the performance of this Spark application: + + Basic Info: {state['analysis_results']['basic_info']} + Executor Summary: {executor_summary} + Stages: {stages} + + Identify key performance metrics and potential issues. + """ + + analysis = self.llm.invoke([HumanMessage(content=analysis_prompt)]) + state["analysis_results"]["performance_analysis"] = analysis.content + + except Exception as e: + state["analysis_results"]["performance_analysis"] = f"Performance analysis failed: {e}" + + state["current_step"] = "performance_analyzed" + state["messages"].append(AIMessage(content="Completed performance analysis")) + return state + + async def identify_bottlenecks(self, state: SparkAnalysisState): + """Identify performance bottlenecks.""" + async with ClientSession(self.mcp_url) as session: + try: + bottlenecks = await session.call_tool("get_job_bottlenecks", {"spark_id": state["app_id"]}) + slowest_stages = await session.call_tool("get_slowest_stages", {"spark_id": state["app_id"]}) + + state["analysis_results"]["bottlenecks"] = { + "job_bottlenecks": bottlenecks, + "slowest_stages": slowest_stages + } + + # Analyze bottlenecks with LLM + bottleneck_prompt = f""" + Identify and prioritize bottlenecks in this Spark application: + + Job Bottlenecks: {bottlenecks} + Slowest Stages: {slowest_stages} + + Rank bottlenecks by impact and provide specific areas for optimization. + """ + + analysis = self.llm.invoke([HumanMessage(content=bottleneck_prompt)]) + state["analysis_results"]["bottleneck_analysis"] = analysis.content + + except Exception as e: + state["analysis_results"]["bottleneck_analysis"] = f"Bottleneck analysis failed: {e}" + + state["current_step"] = "bottlenecks_identified" + state["messages"].append(AIMessage(content="Identified performance bottlenecks")) + return state + + async def compare_with_historical(self, state: SparkAnalysisState): + """Compare with historical applications.""" + async with ClientSession(self.mcp_url) as session: + try: + # Get list of applications to find similar ones + apps = await session.call_tool("list_applications") + + # Find similar applications (simplified logic) + similar_apps = [] + current_app = state["analysis_results"]["basic_info"] + current_name = current_app.get("name", "") + + for app in apps.get("applications", []): + if (app["id"] != state["app_id"] and + app.get("name", "").startswith(current_name.split("-")[0])): + similar_apps.append(app["id"]) + if len(similar_apps) >= 3: # Limit comparisons + break + + # Compare with similar applications + comparisons = [] + for similar_app_id in similar_apps: + try: + comparison = await session.call_tool("compare_job_performance", { + "spark_id1": state["app_id"], + "spark_id2": similar_app_id + }) + comparisons.append({ + "compared_with": similar_app_id, + "comparison": comparison + }) + except: + continue + + state["analysis_results"]["historical_comparison"] = comparisons + + except Exception as e: + state["analysis_results"]["historical_comparison"] = f"Historical comparison failed: {e}" + + state["current_step"] = "historical_compared" + state["messages"].append(AIMessage(content="Completed historical comparison")) + return state + + async def generate_recommendations(self, state: SparkAnalysisState): + """Generate optimization recommendations.""" + # Combine all analysis results + all_analysis = state["analysis_results"] + + recommendation_prompt = f""" + Based on comprehensive analysis of Spark application {state['app_id']}, generate specific optimization recommendations: + + Basic Info: {all_analysis.get('basic_info', {})} + Performance Analysis: {all_analysis.get('performance_analysis', '')} + Bottleneck Analysis: {all_analysis.get('bottleneck_analysis', '')} + Historical Comparisons: {all_analysis.get('historical_comparison', [])} + + Provide: + 1. Top 3 optimization opportunities + 2. Specific configuration changes + 3. Resource allocation recommendations + 4. Expected performance improvements + 5. Implementation priority + """ + + recommendations = self.llm.invoke([HumanMessage(content=recommendation_prompt)]) + state["recommendations"] = recommendations.content.split("\n") + state["current_step"] = "recommendations_generated" + + state["messages"].append(AIMessage(content="Generated optimization recommendations")) + return state + + async def format_report(self, state: SparkAnalysisState): + """Format final analysis report.""" + report_prompt = f""" + Create a comprehensive Spark application analysis report: + + Application ID: {state['app_id']} + Analysis Results: {state['analysis_results']} + Recommendations: {state['recommendations']} + + Format as a professional report with: + - Executive Summary + - Key Findings + - Performance Metrics + - Recommendations with Priority + - Next Steps + """ + + report = self.llm.invoke([HumanMessage(content=report_prompt)]) + state["analysis_results"]["final_report"] = report.content + state["current_step"] = "report_completed" + + state["messages"].append(AIMessage(content="Analysis report completed")) + return state + + async def analyze_application(self, app_id: str): + """Run complete analysis workflow.""" + initial_state = { + "messages": [HumanMessage(content=f"Starting analysis of {app_id}")], + "app_id": app_id, + "analysis_results": {}, + "recommendations": [], + "current_step": "starting" + } + + # Execute the workflow + result = await self.graph.ainvoke(initial_state) + return result + +# Usage +async def analyze_spark_app(): + workflow = SparkAnalysisWorkflow("http://localhost:18888") + result = await workflow.analyze_application("spark-application-12345") + + print("Analysis Complete!") + print("Final Report:") + print(result["analysis_results"]["final_report"]) + +asyncio.run(analyze_spark_app()) +``` + +### 2. Multi-Agent Spark Monitoring System + +```python +from langgraph.graph import StateGraph, END +from typing import Dict, List +import asyncio + +class MonitoringState: + """State for monitoring workflow.""" + applications: List[Dict] + alerts: List[Dict] + analysis_results: Dict[str, Any] + current_time: str + monitoring_enabled: bool + +class SparkMonitoringSystem: + def __init__(self, mcp_server_url: str): + self.mcp_url = mcp_server_url + self.llm = ChatOpenAI(model="gpt-4", temperature=0) + self.monitoring_graph = self._build_monitoring_graph() + + def _build_monitoring_graph(self): + """Build monitoring workflow graph.""" + workflow = StateGraph(MonitoringState) + + # Monitoring agents + workflow.add_node("discovery_agent", self.discovery_agent) + workflow.add_node("health_agent", self.health_agent) + workflow.add_node("performance_agent", self.performance_agent) + workflow.add_node("alert_agent", self.alert_agent) + workflow.add_node("report_agent", self.report_agent) + + # Workflow + workflow.add_edge("discovery_agent", "health_agent") + workflow.add_edge("health_agent", "performance_agent") + workflow.add_edge("performance_agent", "alert_agent") + workflow.add_edge("alert_agent", "report_agent") + workflow.add_edge("report_agent", END) + + workflow.set_entry_point("discovery_agent") + return workflow.compile() + + async def discovery_agent(self, state: MonitoringState): + """Discover current Spark applications.""" + async with ClientSession(self.mcp_url) as session: + apps_result = await session.call_tool("list_applications") + state["applications"] = apps_result.get("applications", []) + + print(f"๐Ÿ” Discovered {len(state['applications'])} applications") + return state + + async def health_agent(self, state: MonitoringState): + """Check health status of applications.""" + health_results = {} + + async with ClientSession(self.mcp_url) as session: + for app in state["applications"]: + app_id = app["id"] + try: + app_info = await session.call_tool("get_application_info", {"app_id": app_id}) + + # Simple health check + attempts = app_info.get("attempts", []) + is_healthy = True + health_issues = [] + + if attempts: + attempt = attempts[0] + if not attempt.get("completed", True): + is_healthy = False + health_issues.append("Application failed") + + duration = app_info.get("duration", 0) + if duration > 7200000: # 2 hours + is_healthy = False + health_issues.append("Long running application") + + health_results[app_id] = { + "healthy": is_healthy, + "issues": health_issues, + "app_info": app_info + } + + except Exception as e: + health_results[app_id] = { + "healthy": False, + "issues": [f"Health check failed: {e}"], + "app_info": {} + } + + state["analysis_results"]["health"] = health_results + unhealthy_count = sum(1 for h in health_results.values() if not h["healthy"]) + print(f"๐Ÿฅ Health check complete: {unhealthy_count} unhealthy applications") + return state + + async def performance_agent(self, state: MonitoringState): + """Analyze performance of applications.""" + performance_results = {} + + async with ClientSession(self.mcp_url) as session: + for app in state["applications"]: + app_id = app["id"] + try: + # Get performance bottlenecks + bottlenecks = await session.call_tool("get_job_bottlenecks", {"spark_id": app_id}) + + # Analyze with LLM + perf_prompt = f""" + Quickly assess the performance of Spark application {app_id}: + + Bottlenecks: {bottlenecks} + + Provide: + - Performance score (1-10) + - Key issue (if any) + - Severity (low/medium/high) + """ + + analysis = self.llm.invoke([HumanMessage(content=perf_prompt)]) + performance_results[app_id] = { + "bottlenecks": bottlenecks, + "analysis": analysis.content + } + + except Exception as e: + performance_results[app_id] = { + "error": str(e), + "analysis": "Performance analysis failed" + } + + state["analysis_results"]["performance"] = performance_results + print(f"โšก Performance analysis complete for {len(performance_results)} applications") + return state + + async def alert_agent(self, state: MonitoringState): + """Generate alerts based on health and performance.""" + alerts = [] + + health_results = state["analysis_results"].get("health", {}) + performance_results = state["analysis_results"].get("performance", {}) + + for app_id in state["applications"]: + app_id_str = app_id["id"] + + # Health-based alerts + health = health_results.get(app_id_str, {}) + if not health.get("healthy", True): + alerts.append({ + "type": "health", + "app_id": app_id_str, + "severity": "high", + "message": f"Health issues: {', '.join(health.get('issues', []))}", + "timestamp": state.get("current_time", "") + }) + + # Performance-based alerts + perf = performance_results.get(app_id_str, {}) + if "high" in perf.get("analysis", "").lower(): + alerts.append({ + "type": "performance", + "app_id": app_id_str, + "severity": "medium", + "message": "Performance degradation detected", + "timestamp": state.get("current_time", "") + }) + + state["alerts"] = alerts + print(f"๐Ÿšจ Generated {len(alerts)} alerts") + return state + + async def report_agent(self, state: MonitoringState): + """Generate monitoring report.""" + total_apps = len(state["applications"]) + total_alerts = len(state["alerts"]) + + # Generate summary report + report_prompt = f""" + Create a monitoring summary report: + + Total Applications: {total_apps} + Total Alerts: {total_alerts} + Health Results: {state['analysis_results'].get('health', {})} + Performance Results: {state['analysis_results'].get('performance', {})} + Alerts: {state['alerts']} + + Provide: + - Executive summary + - Key issues requiring attention + - Overall system health score + - Recommended actions + """ + + report = self.llm.invoke([HumanMessage(content=report_prompt)]) + state["analysis_results"]["monitoring_report"] = report.content + + print("๐Ÿ“Š Monitoring report generated") + return state + + async def run_monitoring_cycle(self): + """Run one complete monitoring cycle.""" + from datetime import datetime + + initial_state = { + "applications": [], + "alerts": [], + "analysis_results": {}, + "current_time": datetime.now().isoformat(), + "monitoring_enabled": True + } + + result = await self.monitoring_graph.ainvoke(initial_state) + return result + + async def continuous_monitoring(self, interval_minutes: int = 5): + """Run continuous monitoring.""" + while True: + try: + print(f"\n{'='*50}") + print(f"๐Ÿ”„ Starting monitoring cycle at {datetime.now()}") + + result = await self.run_monitoring_cycle() + + # Print summary + print(f"๐Ÿ“‹ Monitoring Summary:") + print(f" Applications: {len(result['applications'])}") + print(f" Alerts: {len(result['alerts'])}") + + if result["alerts"]: + print("๐Ÿšจ Active Alerts:") + for alert in result["alerts"]: + print(f" - {alert['app_id']}: {alert['message']}") + + print(f"\n๐Ÿ“Š Full Report:") + print(result["analysis_results"]["monitoring_report"]) + + # Wait for next cycle + await asyncio.sleep(interval_minutes * 60) + + except Exception as e: + print(f"โŒ Monitoring error: {e}") + await asyncio.sleep(60) # Wait before retry + +# Usage +async def start_monitoring(): + monitor = SparkMonitoringSystem("http://localhost:18888") + await monitor.continuous_monitoring(interval_minutes=5) + +# Run monitoring +asyncio.run(start_monitoring()) +``` + +### 3. Spark Optimization Workflow + +```python +class OptimizationState: + """State for optimization workflow.""" + target_app_id: str + baseline_metrics: Dict + optimization_opportunities: List[Dict] + proposed_changes: List[Dict] + expected_improvements: Dict + implementation_plan: List[str] + +class SparkOptimizationWorkflow: + def __init__(self, mcp_server_url: str): + self.mcp_url = mcp_server_url + self.llm = ChatOpenAI(model="gpt-4", temperature=0) + self.graph = self._build_optimization_graph() + + def _build_optimization_graph(self): + """Build optimization workflow.""" + workflow = StateGraph(OptimizationState) + + workflow.add_node("baseline_analysis", self.baseline_analysis) + workflow.add_node("identify_opportunities", self.identify_opportunities) + workflow.add_node("propose_changes", self.propose_changes) + workflow.add_node("estimate_impact", self.estimate_impact) + workflow.add_node("create_implementation_plan", self.create_implementation_plan) + + workflow.add_edge("baseline_analysis", "identify_opportunities") + workflow.add_edge("identify_opportunities", "propose_changes") + workflow.add_edge("propose_changes", "estimate_impact") + workflow.add_edge("estimate_impact", "create_implementation_plan") + workflow.add_edge("create_implementation_plan", END) + + workflow.set_entry_point("baseline_analysis") + return workflow.compile() + + async def baseline_analysis(self, state: OptimizationState): + """Establish baseline metrics.""" + async with ClientSession(self.mcp_url) as session: + app_info = await session.call_tool("get_application_info", {"app_id": state["target_app_id"]}) + bottlenecks = await session.call_tool("get_job_bottlenecks", {"spark_id": state["target_app_id"]}) + executor_summary = await session.call_tool("get_executor_summary", {"spark_id": state["target_app_id"]}) + + state["baseline_metrics"] = { + "app_info": app_info, + "bottlenecks": bottlenecks, + "executor_summary": executor_summary, + "duration": app_info.get("duration", 0), + "resource_usage": executor_summary + } + + print(f"๐Ÿ“Š Baseline analysis complete for {state['target_app_id']}") + return state + + async def identify_opportunities(self, state: OptimizationState): + """Identify optimization opportunities.""" + opportunities_prompt = f""" + Identify optimization opportunities for this Spark application: + + Baseline Metrics: {state['baseline_metrics']} + + Focus on: + 1. Configuration optimizations + 2. Resource allocation improvements + 3. Algorithm/approach changes + 4. Infrastructure optimizations + + Rank opportunities by potential impact. + """ + + analysis = self.llm.invoke([HumanMessage(content=opportunities_prompt)]) + + # Parse opportunities (simplified) + opportunities = [ + {"type": "configuration", "description": "Optimize executor memory", "impact": "high"}, + {"type": "resource", "description": "Adjust parallelism", "impact": "medium"}, + {"type": "algorithm", "description": "Enable adaptive query execution", "impact": "high"} + ] + + state["optimization_opportunities"] = opportunities + print(f"๐ŸŽฏ Identified {len(opportunities)} optimization opportunities") + return state + + async def propose_changes(self, state: OptimizationState): + """Propose specific configuration changes.""" + changes_prompt = f""" + Based on these optimization opportunities, propose specific configuration changes: + + Opportunities: {state['optimization_opportunities']} + Current Configuration: {state['baseline_metrics']['app_info']} + + Provide specific Spark configuration parameters and values. + """ + + analysis = self.llm.invoke([HumanMessage(content=changes_prompt)]) + + # Example proposed changes + proposed_changes = [ + { + "parameter": "spark.executor.memory", + "current_value": "2g", + "proposed_value": "4g", + "rationale": "Reduce memory spilling" + }, + { + "parameter": "spark.sql.adaptive.enabled", + "current_value": "false", + "proposed_value": "true", + "rationale": "Enable adaptive query execution" + } + ] + + state["proposed_changes"] = proposed_changes + print(f"โš™๏ธ Proposed {len(proposed_changes)} configuration changes") + return state + + async def estimate_impact(self, state: OptimizationState): + """Estimate impact of proposed changes.""" + impact_prompt = f""" + Estimate the performance impact of these proposed changes: + + Baseline Duration: {state['baseline_metrics']['duration']} ms + Proposed Changes: {state['proposed_changes']} + Current Bottlenecks: {state['baseline_metrics']['bottlenecks']} + + Estimate: + 1. Expected duration reduction (%) + 2. Resource efficiency improvements + 3. Risk factors + 4. Implementation complexity + """ + + analysis = self.llm.invoke([HumanMessage(content=impact_prompt)]) + + state["expected_improvements"] = { + "duration_reduction_percent": 25, + "resource_efficiency_improvement": "15% better CPU utilization", + "risk_level": "low", + "implementation_complexity": "medium" + } + + print("๐Ÿ“ˆ Impact estimation complete") + return state + + async def create_implementation_plan(self, state: OptimizationState): + """Create step-by-step implementation plan.""" + plan_prompt = f""" + Create a detailed implementation plan for these optimizations: + + Proposed Changes: {state['proposed_changes']} + Expected Impact: {state['expected_improvements']} + + Include: + 1. Step-by-step implementation + 2. Testing strategy + 3. Rollback plan + 4. Monitoring approach + """ + + analysis = self.llm.invoke([HumanMessage(content=plan_prompt)]) + + implementation_plan = [ + "1. Create test environment with proposed configurations", + "2. Run validation tests with sample data", + "3. Monitor performance metrics during test runs", + "4. Compare results with baseline", + "5. Implement in production with gradual rollout", + "6. Monitor production performance", + "7. Document learnings and update optimization playbook" + ] + + state["implementation_plan"] = implementation_plan + print("๐Ÿ“‹ Implementation plan created") + return state + + async def optimize_application(self, app_id: str): + """Run complete optimization workflow.""" + initial_state = { + "target_app_id": app_id, + "baseline_metrics": {}, + "optimization_opportunities": [], + "proposed_changes": [], + "expected_improvements": {}, + "implementation_plan": [] + } + + result = await self.graph.ainvoke(initial_state) + return result + +# Usage +async def optimize_spark_app(): + optimizer = SparkOptimizationWorkflow("http://localhost:18888") + result = await optimizer.optimize_application("spark-application-12345") + + print("\n๐ŸŽฏ Optimization Plan Complete!") + print(f"Proposed Changes: {len(result['proposed_changes'])}") + print(f"Expected Improvements: {result['expected_improvements']}") + print("\nImplementation Plan:") + for step in result['implementation_plan']: + print(f" {step}") + +asyncio.run(optimize_spark_app()) +``` + +## Configuration and Best Practices + +### Graph Visualization + +```python +from langgraph.graph import StateGraph +import matplotlib.pyplot as plt + +# Visualize workflow graphs +def visualize_workflow(graph): + """Create visual representation of the workflow.""" + # Implementation would create workflow diagrams + pass +``` + +### Error Handling and Resilience + +```python +class RobustSparkWorkflow: + def __init__(self, mcp_server_url: str): + self.mcp_url = mcp_server_url + self.max_retries = 3 + self.retry_delay = 5 + + async def robust_mcp_call(self, tool_name: str, parameters: dict): + """MCP call with retry logic and error handling.""" + for attempt in range(self.max_retries): + try: + async with ClientSession(self.mcp_url) as session: + return await session.call_tool(tool_name, parameters) + except Exception as e: + if attempt == self.max_retries - 1: + raise + await asyncio.sleep(self.retry_delay) + + return None +``` + +## Advanced Examples + +The `/examples/langgraph/` directory contains more sophisticated examples: +- Multi-tenant Spark monitoring across different environments +- Automated performance regression detection pipelines +- Intelligent resource scaling recommendations +- Integration with CI/CD for Spark application optimization diff --git a/examples/integrations/llamaindex.md b/examples/integrations/llamaindex.md new file mode 100644 index 0000000..e48de16 --- /dev/null +++ b/examples/integrations/llamaindex.md @@ -0,0 +1,529 @@ +# LlamaIndex Integration + +This guide shows how to integrate the Spark History Server MCP with LlamaIndex to build intelligent Spark job analysis and retrieval systems. + +## Installation + +```bash +pip install llama-index llama-index-tools-requests +# or +uv add llama-index llama-index-tools-requests +``` + +## Basic Integration + +### 1. MCP Tool Integration with LlamaIndex + +```python +from llama_index.core.agent import ReActAgent +from llama_index.core.tools import FunctionTool +from llama_index.llms.openai import OpenAI +import asyncio +import aiohttp + +class SparkMCPTools: + def __init__(self, mcp_server_url: str): + self.mcp_url = mcp_server_url + + async def _call_mcp_tool(self, tool_name: str, parameters: dict = None): + """Call MCP tool via HTTP API.""" + async with aiohttp.ClientSession() as session: + payload = { + "tool": tool_name, + "parameters": parameters or {} + } + async with session.post(f"{self.mcp_url}/tools", json=payload) as response: + return await response.json() + + def get_application_info(self, app_id: str) -> str: + """Get detailed information about a Spark application.""" + async def _get_info(): + result = await self._call_mcp_tool("get_application_info", {"app_id": app_id}) + return str(result) + + return asyncio.run(_get_info()) + + def list_applications(self) -> str: + """List all Spark applications.""" + async def _list_apps(): + result = await self._call_mcp_tool("list_applications") + return str(result) + + return asyncio.run(_list_apps()) + + def compare_job_performance(self, spark_id1: str, spark_id2: str) -> str: + """Compare performance between two Spark applications.""" + async def _compare(): + result = await self._call_mcp_tool("compare_job_performance", { + "spark_id1": spark_id1, + "spark_id2": spark_id2 + }) + return str(result) + + return asyncio.run(_compare()) + + def get_job_bottlenecks(self, spark_id: str) -> str: + """Analyze job bottlenecks and performance issues.""" + async def _analyze(): + result = await self._call_mcp_tool("get_job_bottlenecks", {"spark_id": spark_id}) + return str(result) + + return asyncio.run(_analyze()) + +# Create LlamaIndex tools from MCP tools +def create_spark_tools(mcp_server_url: str): + spark_tools = SparkMCPTools(mcp_server_url) + + tools = [ + FunctionTool.from_defaults( + fn=spark_tools.get_application_info, + name="get_application_info", + description="Get detailed information about a Spark application by app_id" + ), + FunctionTool.from_defaults( + fn=spark_tools.list_applications, + name="list_applications", + description="List all available Spark applications" + ), + FunctionTool.from_defaults( + fn=spark_tools.compare_job_performance, + name="compare_job_performance", + description="Compare performance metrics between two Spark applications" + ), + FunctionTool.from_defaults( + fn=spark_tools.get_job_bottlenecks, + name="get_job_bottlenecks", + description="Analyze performance bottlenecks for a Spark application" + ), + ] + + return tools + +# Create the agent +llm = OpenAI(model="gpt-4", temperature=0) +tools = create_spark_tools("http://localhost:18888") + +agent = ReActAgent.from_tools( + tools, + llm=llm, + verbose=True, + system_prompt=""" + You are a Spark performance analysis expert. You have access to tools that can: + - List and retrieve Spark application details + - Compare performance between different jobs + - Analyze bottlenecks and performance issues + + Always provide actionable insights and specific recommendations. + When analyzing performance issues, look for patterns in resource usage, + stage execution times, and configuration differences. + """ +) + +# Use the agent +response = agent.chat("Analyze the performance of application spark-12345 and suggest optimizations") +print(response) +``` + +### 2. Building a Spark Knowledge Index + +```python +from llama_index.core import VectorStoreIndex, Document, Settings +from llama_index.core.node_parser import SentenceSplitter +from llama_index.embeddings.openai import OpenAIEmbedding +from llama_index.core.storage.storage_context import StorageContext +from llama_index.vector_stores.faiss import FaissVectorStore +import faiss + +class SparkKnowledgeIndex: + def __init__(self, mcp_server_url: str): + self.mcp_url = mcp_server_url + self.index = None + + # Configure LlamaIndex settings + Settings.embed_model = OpenAIEmbedding() + Settings.node_parser = SentenceSplitter(chunk_size=512, chunk_overlap=50) + + async def build_index(self): + """Build knowledge index from Spark application data.""" + documents = [] + spark_tools = SparkMCPTools(self.mcp_url) + + # Get all applications + apps_result = await spark_tools._call_mcp_tool("list_applications") + applications = apps_result.get('applications', []) + + for app in applications: + app_id = app['id'] + + # Get detailed application info + app_info = await spark_tools._call_mcp_tool("get_application_info", {"app_id": app_id}) + + # Get performance analysis + try: + bottlenecks = await spark_tools._call_mcp_tool("get_job_bottlenecks", {"spark_id": app_id}) + executor_summary = await spark_tools._call_mcp_tool("get_executor_summary", {"spark_id": app_id}) + except: + bottlenecks = {} + executor_summary = {} + + # Create comprehensive document + doc_content = f""" + Application ID: {app_id} + Application Name: {app_info.get('name', 'Unknown')} + Status: {app_info.get('attempts', [{}])[0].get('completed', 'Unknown')} + Duration: {app_info.get('duration', 'Unknown')} ms + + Application Details: + {app_info} + + Performance Analysis: + {bottlenecks} + + Executor Summary: + {executor_summary} + """ + + documents.append(Document( + text=doc_content, + metadata={ + "app_id": app_id, + "app_name": app_info.get('name', 'Unknown'), + "status": app_info.get('attempts', [{}])[0].get('completed', 'Unknown'), + "type": "spark_application" + } + )) + + # Create FAISS vector store + d = 1536 # OpenAI embedding dimension + faiss_index = faiss.IndexFlatL2(d) + vector_store = FaissVectorStore(faiss_index=faiss_index) + storage_context = StorageContext.from_defaults(vector_store=vector_store) + + # Build the index + self.index = VectorStoreIndex.from_documents( + documents, + storage_context=storage_context + ) + + return self.index + + def query(self, question: str, similarity_top_k: int = 5): + """Query the knowledge index.""" + if not self.index: + raise ValueError("Index not built. Call build_index() first.") + + query_engine = self.index.as_query_engine( + similarity_top_k=similarity_top_k, + response_mode="tree_summarize" + ) + + return query_engine.query(question) + +# Usage +async def main(): + kb = SparkKnowledgeIndex("http://localhost:18888") + await kb.build_index() + + # Query the knowledge base + response = kb.query("What are the most common performance bottlenecks in failed Spark jobs?") + print(response) + + response = kb.query("Show me applications that had memory-related issues") + print(response) + +asyncio.run(main()) +``` + +### 3. Advanced RAG with Spark Metrics + +```python +from llama_index.core.retrievers import VectorIndexRetriever +from llama_index.core.query_engine import RetrieverQueryEngine +from llama_index.core.postprocessor import SimilarityPostprocessor, KeywordNodePostprocessor + +class SparkRAGSystem: + def __init__(self, mcp_server_url: str): + self.mcp_url = mcp_server_url + self.index = None + + async def setup_advanced_rag(self): + """Setup advanced RAG system with custom retrievers and post-processors.""" + # Build knowledge index + kb = SparkKnowledgeIndex(self.mcp_url) + self.index = await kb.build_index() + + # Custom retriever with higher recall + retriever = VectorIndexRetriever( + index=self.index, + similarity_top_k=10, + ) + + # Post-processors for better relevance + postprocessors = [ + KeywordNodePostprocessor( + keywords=["error", "failure", "bottleneck", "performance", "memory", "cpu"], + exclude_keywords=["success", "completed"] + ), + SimilarityPostprocessor(similarity_cutoff=0.7) + ] + + # Create query engine + self.query_engine = RetrieverQueryEngine( + retriever=retriever, + node_postprocessors=postprocessors, + response_mode="tree_summarize" + ) + + async def analyze_with_context(self, app_id: str): + """Analyze application with historical context.""" + if not self.query_engine: + await self.setup_advanced_rag() + + # Get current app details + spark_tools = SparkMCPTools(self.mcp_url) + current_app = await spark_tools._call_mcp_tool("get_application_info", {"app_id": app_id}) + + # Query for similar issues + query = f""" + Find similar Spark applications that had comparable issues to this one: + + Current Application: {app_id} + Status: {current_app.get('attempts', [{}])[0].get('completed', 'Unknown')} + Duration: {current_app.get('duration', 'Unknown')} + + Look for patterns in: + - Similar execution times + - Memory usage patterns + - Common failure modes + - Configuration similarities + """ + + similar_cases = self.query_engine.query(query) + + return { + "current_application": current_app, + "similar_cases": similar_cases, + "recommendations": self._generate_recommendations(current_app, similar_cases) + } + + def _generate_recommendations(self, current_app, similar_cases): + """Generate recommendations based on historical patterns.""" + # This would use LLM to analyze patterns and generate recommendations + llm = OpenAI(model="gpt-4") + + prompt = f""" + Based on the current application and similar historical cases, provide recommendations: + + Current Application: {current_app} + Similar Historical Cases: {similar_cases} + + Provide: + 1. Root cause analysis + 2. Specific configuration changes + 3. Resource optimization suggestions + 4. Monitoring recommendations + """ + + return llm.complete(prompt) + +# Usage +async def analyze_application(): + rag_system = SparkRAGSystem("http://localhost:18888") + analysis = await rag_system.analyze_with_context("spark-application-12345") + + print("Analysis Results:") + print(f"Current App: {analysis['current_application']}") + print(f"Similar Cases: {analysis['similar_cases']}") + print(f"Recommendations: {analysis['recommendations']}") + +asyncio.run(analyze_application()) +``` + +### 4. Real-time Spark Monitoring with LlamaIndex + +```python +from llama_index.core.chat_engine import SimpleChatEngine +from llama_index.core.memory import ChatMemoryBuffer +import asyncio +import schedule +import time + +class SparkMonitoringChat: + def __init__(self, mcp_server_url: str): + self.mcp_url = mcp_server_url + self.chat_engine = None + self.memory = ChatMemoryBuffer.from_defaults(token_limit=3000) + + async def setup_monitoring_chat(self): + """Setup chat engine for interactive monitoring.""" + # Build knowledge base + kb = SparkKnowledgeIndex(self.mcp_url) + index = await kb.build_index() + + # Create chat engine with memory + self.chat_engine = SimpleChatEngine.from_defaults( + llm=OpenAI(model="gpt-4", temperature=0), + memory=self.memory, + system_prompt=""" + You are a Spark monitoring assistant. You help users understand: + - Current application status and performance + - Historical patterns and trends + - Performance optimization opportunities + - Failure predictions and prevention + + Always provide actionable insights and be proactive about potential issues. + """ + ) + + async def monitor_and_alert(self): + """Continuous monitoring with intelligent alerts.""" + spark_tools = SparkMCPTools(self.mcp_url) + + while True: + try: + # Get current applications + apps = await spark_tools._call_mcp_tool("list_applications") + + for app in apps.get('applications', []): + app_id = app['id'] + + # Analyze current state + analysis = await self._analyze_current_state(app_id) + + if analysis['needs_attention']: + alert_message = f""" + ๐Ÿšจ ALERT: Application {app_id} needs attention! + + Issue: {analysis['issue']} + Severity: {analysis['severity']} + Recommendation: {analysis['recommendation']} + """ + + print(alert_message) + + # Add to chat memory for context + self.memory.put(f"ALERT: {alert_message}") + + await asyncio.sleep(300) # Check every 5 minutes + + except Exception as e: + print(f"Monitoring error: {e}") + await asyncio.sleep(60) + + async def _analyze_current_state(self, app_id: str): + """Analyze current application state.""" + spark_tools = SparkMCPTools(self.mcp_url) + + # Get app info and bottlenecks + app_info = await spark_tools._call_mcp_tool("get_application_info", {"app_id": app_id}) + + try: + bottlenecks = await spark_tools._call_mcp_tool("get_job_bottlenecks", {"spark_id": app_id}) + except: + bottlenecks = {} + + # Simple heuristics for demonstration + needs_attention = False + issue = None + severity = "low" + recommendation = "" + + # Check duration + duration = app_info.get('duration', 0) + if duration > 3600000: # > 1 hour + needs_attention = True + issue = "Long running application" + severity = "medium" + recommendation = "Check for performance bottlenecks" + + # Check completion status + attempts = app_info.get('attempts', []) + if attempts and not attempts[0].get('completed', True): + needs_attention = True + issue = "Application failed" + severity = "high" + recommendation = "Investigate failure cause" + + return { + 'needs_attention': needs_attention, + 'issue': issue, + 'severity': severity, + 'recommendation': recommendation + } + + async def interactive_chat(self): + """Start interactive chat session.""" + if not self.chat_engine: + await self.setup_monitoring_chat() + + print("Spark Monitoring Chat Started. Type 'quit' to exit.") + + while True: + user_input = input("You: ") + if user_input.lower() == 'quit': + break + + response = self.chat_engine.chat(user_input) + print(f"Assistant: {response}") + +# Usage +async def start_monitoring_system(): + monitor = SparkMonitoringChat("http://localhost:18888") + + # Start background monitoring + monitoring_task = asyncio.create_task(monitor.monitor_and_alert()) + + # Start interactive chat + await monitor.interactive_chat() + + # Cancel monitoring when chat ends + monitoring_task.cancel() + +# Run the monitoring system +asyncio.run(start_monitoring_system()) +``` + +## Configuration Examples + +### Custom Embeddings for Spark Metrics + +```python +from llama_index.embeddings.huggingface import HuggingFaceEmbedding + +# Use specialized embeddings for technical content +Settings.embed_model = HuggingFaceEmbedding( + model_name="sentence-transformers/all-MiniLM-L6-v2" +) +``` + +### Custom Node Parsing for Spark Logs + +```python +from llama_index.core.node_parser import SimpleNodeParser + +class SparkLogNodeParser(SimpleNodeParser): + def get_nodes_from_documents(self, documents): + # Custom parsing logic for Spark application data + nodes = [] + for doc in documents: + # Extract key metrics and create focused nodes + # Implementation would parse Spark-specific structures + pass + return nodes +``` + +## Best Practices + +1. **Index Management**: Regularly rebuild indices with fresh Spark data +2. **Query Optimization**: Use specific keywords related to Spark performance +3. **Memory Management**: Implement proper cleanup for long-running monitoring +4. **Error Handling**: Robust handling of MCP connection issues +5. **Caching**: Cache frequently accessed application data + +## Advanced Examples + +See the `/examples/llamaindex/` directory for more examples including: +- Multi-modal analysis with Spark UI screenshots +- Integration with Spark streaming applications +- Custom evaluation metrics for RAG performance +- Automated performance regression detection diff --git a/main.py b/main.py index 60bb8a7..da445ab 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,26 @@ +"""Main entry point for Spark History Server MCP.""" + +import logging +import sys + from app import mcp +# Configure logging +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + + +def main(): + """Main entry point.""" + try: + logger.info("Starting Spark History Server MCP...") + mcp.run(transport="streamable-http") + except Exception as e: + logger.error(f"Failed to start MCP server: {e}") + sys.exit(1) + + if __name__ == "__main__": - mcp.run( - transport="streamable-http", - ) + main() diff --git a/pyproject.toml b/pyproject.toml index 9a547b6..235a852 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,19 +1,76 @@ [project] -name = "mcp-spark-events" +name = "spark-history-server-mcp" version = "0.1.0" -description = "Add your description here" +description = "Model Context Protocol (MCP) server for Apache Spark History Server with job comparison and analytics" readme = "README.md" +authors = [ + {name = "Manabu McCloskey", email = "Manabu.McCloskey@gmail.com"}, + {name = "Vara Bonthu", email = "vara.bonthu@gmail.com"} +] +license = {text = "Apache-2.0"} requires-python = ">=3.12" +keywords = ["spark", "mcp", "analytics", "performance", "history-server", "model-context-protocol"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: System :: Monitoring", + "Topic :: Scientific/Engineering :: Information Analysis", +] dependencies = [ "mcp[cli]>=1.9.4", "pyyaml>=6.0.2", "requests>=2.32.4", + "pydantic>=2.0.0", ] +# Development dependencies moved to [dependency-groups] section below + +[project.scripts] +spark-mcp = "main:main" + +[project.urls] +Homepage = "https://github.com/DeepDiagnostix-AI/spark-history-server-mcp" +Repository = "https://github.com/DeepDiagnostix-AI/spark-history-server-mcp" +Issues = "https://github.com/DeepDiagnostix-AI/spark-history-server-mcp/issues" + [tool.ruff] target-version = "py312" +line-length = 88 + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "N", "B", "A", "S", "T20"] +ignore = ["E501", "S101"] + +[tool.ruff.lint.per-file-ignores] +"test_*.py" = ["S101"] + +[tool.black] +line-length = 88 +target-version = ['py312'] + +[tool.mypy] +python_version = "3.12" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false # Start with false, gradually enable + +[tool.pytest.ini_options] +testpaths = ["tests", "."] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = "--cov=. --cov-report=term-missing" +pythonpath = ["."] [dependency-groups] dev = [ "pytest>=8.4.1", + "pytest-cov>=4.0.0", + "black>=23.0.0", + "ruff>=0.1.0", + "mypy>=1.7.0", + "pre-commit>=3.0.0", ] diff --git a/screenshots/README.md b/screenshots/README.md new file mode 100644 index 0000000..69f9efe --- /dev/null +++ b/screenshots/README.md @@ -0,0 +1,63 @@ +# ๐Ÿ“ธ Screenshots + +This directory contains screenshots for the README.md file. + +## ๐Ÿ“‹ Required Screenshots + +### ๐Ÿ” Get Applications Screenshot +**Filename**: `get-applications.png` + +**What to capture**: +1. Open MCP Inspector in browser (http://localhost:6274) +2. Click on `list_applications` tool in the sidebar +3. Click "Call Tool" button +4. Capture the results showing all 3 Spark applications: + - spark-bcec39f6201b42b9925124595baad260 + - spark-110be3a8424d4a2789cb88134418217b + - spark-cc4d115f011443d787f03a71a476a745 + +**Screenshot should show**: +- Tool name and parameters (if any) +- JSON response with application list +- Application IDs, names, and status + +### โšก Job Comparison Screenshot +**Filename**: `job-comparison.png` + +**What to capture**: +1. In MCP Inspector, click on `compare_job_performance` tool +2. Fill in parameters: + - `spark_id1`: `spark-bcec39f6201b42b9925124595baad260` + - `spark_id2`: `spark-110be3a8424d4a2789cb88134418217b` +3. Click "Call Tool" button +4. Capture the comparison results + +**Screenshot should show**: +- Tool name and input parameters +- Performance comparison results +- Metrics like duration, stages, tasks, etc. +- Any performance differences highlighted + +## ๐ŸŽฏ Screenshot Guidelines + +- **Resolution**: Capture at high resolution (retina/2x if possible) +- **Browser**: Use clean browser window, hide bookmarks bar +- **Focus**: Ensure the tool execution and results are clearly visible +- **Format**: Save as PNG for best quality +- **Content**: Make sure all important data is visible and readable + +## ๐Ÿ“ Usage in README + +These screenshots are referenced in the main README.md file: + +```markdown +### ๐Ÿ” Get Spark Applications +![Get Applications](screenshots/get-applications.png) +*Browse all available Spark applications with filtering options* + +### โšก Job Performance Comparison +![Job Comparison](screenshots/job-comparison.png) +*Compare performance metrics between different Spark jobs* +``` + +Replace the placeholder screenshots with actual MCP Inspector captures to show users what to expect when testing the tools. diff --git a/screenshots/get-application.png b/screenshots/get-application.png new file mode 100644 index 0000000..de22b3b Binary files /dev/null and b/screenshots/get-application.png differ diff --git a/screenshots/job-compare.png b/screenshots/job-compare.png new file mode 100644 index 0000000..3e58519 Binary files /dev/null and b/screenshots/job-compare.png differ diff --git a/spark_client.py b/spark_client.py index a240ce4..027aec5 100644 --- a/spark_client.py +++ b/spark_client.py @@ -1,27 +1,28 @@ -import requests -from typing import Dict, List, Optional, Any, Type, TypeVar +from typing import Any, Dict, List, Optional, Type, TypeVar from urllib.parse import urljoin + +import requests from pydantic import BaseModel +from config import ServerConfig from spark_types import ( - ApplicationInfo, ApplicationAttemptInfo, - JobData, + ApplicationEnvironmentInfo, + ApplicationInfo, + ExecutionData, ExecutorSummary, + JobData, + JobExecutionStatus, + ProcessSummary, + RDDStorageInfo, StageData, + StageStatus, TaskData, - RDDStorageInfo, - ApplicationEnvironmentInfo, + TaskMetricDistributions, + TaskStatus, ThreadStackTrace, VersionInfo, - JobExecutionStatus, - StageStatus, - TaskStatus, - ProcessSummary, - TaskMetricDistributions, - ExecutionData, ) -from config import ServerConfig T = TypeVar("T", bound=BaseModel) @@ -31,7 +32,7 @@ class SparkRestClient: Python client for the Spark REST API. """ - def __init__(self, sever_config: ServerConfig): + def __init__(self, server_config: ServerConfig): """ Initialize the Spark REST client. @@ -40,7 +41,7 @@ def __init__(self, sever_config: ServerConfig): config_path: Path to configuration file """ - self.config = sever_config + self.config = server_config self.base_url = self.config.url.rstrip("/") + "/api/v1" self.auth = None @@ -67,7 +68,9 @@ def _get(self, endpoint: str, params: Optional[Dict[str, Any]] = None) -> Any: if self.config.auth and self.config.auth.token: headers["Authorization"] = f"Bearer {self.config.auth.token}" - response = requests.get(url, params=params, headers=headers, auth=self.auth) + response = requests.get( + url, params=params, headers=headers, auth=self.auth, timeout=30 + ) response.raise_for_status() return response.json() @@ -504,7 +507,7 @@ def get_metrics_prometheus(self, app_id: str) -> str: url = urljoin( self.base_url.replace("/api/v1", "/metrics/executors"), "prometheus" ) - response = requests.get(url) + response = requests.get(url, timeout=30) response.raise_for_status() return response.text diff --git a/spark_html_client.py b/spark_html_client.py index 57eb00d..f74f2b0 100644 --- a/spark_html_client.py +++ b/spark_html_client.py @@ -3,15 +3,16 @@ import uuid from urllib.parse import urljoin -from config import ServerConfig from playwright.async_api import async_playwright -# very experimental html client +from config import ServerConfig + +# HTML client for Spark History Server web interface class SparkHtmlClient: - def __init__(self, sever_config: ServerConfig): - self.config = sever_config + def __init__(self, server_config: ServerConfig): + self.config = server_config self.base_url = self.config.url.rstrip("/") + "/history/" self.auth = None self.browser = None @@ -73,7 +74,6 @@ async def get_screenshot(self, path, save_path=None): page = await self.browser.new_page() await page.set_viewport_size({"width": 2560, "height": 800}) url = urljoin(self.base_url, path) - print(url) await page.goto(url) # Wait for network to be idle @@ -81,7 +81,13 @@ async def get_screenshot(self, path, save_path=None): await page.wait_for_timeout(3000) # 3 seconds # Use provided save_path or generate a random filename - filename = save_path if save_path else f"/tmp/{uuid.uuid4()}.jpg" + import tempfile + + filename = ( + save_path + if save_path + else f"{tempfile.gettempdir()}/{uuid.uuid4()}.jpg" + ) # Ensure directory exists os.makedirs(os.path.dirname(os.path.abspath(filename)), exist_ok=True) @@ -111,4 +117,5 @@ async def main(): # print("HTML content saved to rendered_page.html") -asyncio.run(main()) +if __name__ == "__main__": + asyncio.run(main()) diff --git a/spark_types.py b/spark_types.py index 23cac46..bca836e 100644 --- a/spark_types.py +++ b/spark_types.py @@ -1,7 +1,8 @@ from datetime import datetime from enum import Enum -from typing import Dict, Optional, Sequence, Set, Any -from pydantic import BaseModel, Field, field_validator, ConfigDict +from typing import Any, Dict, Optional, Sequence, Set + +from pydantic import BaseModel, ConfigDict, Field, field_validator class JobExecutionStatus(str, Enum): diff --git a/start_local_spark_history.sh b/start_local_spark_history.sh new file mode 100755 index 0000000..911ac72 --- /dev/null +++ b/start_local_spark_history.sh @@ -0,0 +1,157 @@ +#!/bin/bash + +# Help function +show_help() { + cat << EOF +๐Ÿš€ Starting Local Spark History Server for MCP Testing +======================================================= + +USAGE: + ./start_local_spark_history.sh [OPTIONS] + +OPTIONS: + -h, --help Show this help message + --dry-run Validate prerequisites without starting the server + +DESCRIPTION: + This script starts a local Spark History Server using Docker for testing + the Spark History Server MCP. It uses sample Spark event data provided + in the examples/basic/events/ directory. + +PREREQUISITES: + - Docker must be running + - Must be run from the project root directory + - Sample event data must exist in examples/basic/events/ + +ENDPOINTS: + - Web UI: http://localhost:18080 + - REST API: http://localhost:18080/api/v1/ + +EXAMPLES: + ./start_local_spark_history.sh # Start the server + ./start_local_spark_history.sh --help # Show this help + ./start_local_spark_history.sh --dry-run # Validate setup only + +EOF +} + +# Parse command line arguments +DRY_RUN=false +for arg in "$@"; do + case $arg in + -h|--help) + show_help + exit 0 + ;; + --dry-run) + DRY_RUN=true + shift + ;; + *) + echo "Unknown option: $arg" + echo "Use --help for usage information." + exit 1 + ;; + esac +done + +echo "๐Ÿš€ Starting Local Spark History Server for MCP Testing" +echo "=======================================================" + +# Function to check if Docker is running +check_docker() { + if ! docker info >/dev/null 2>&1; then + echo "โŒ Error: Docker is not running. Please start Docker first." + exit 1 + fi +} + +# Function to validate test data +validate_test_data() { + if [ ! -d "examples/basic/events" ]; then + echo "โŒ Error: Test data directory 'examples/basic/events' not found." + echo " Please ensure you're running this script from the project root directory." + exit 1 + fi + + if [ ! -f "examples/basic/history-server.conf" ]; then + echo "โŒ Error: Spark History Server configuration file not found." + echo " Expected: examples/basic/history-server.conf" + exit 1 + fi +} + +# Check prerequisites +echo "๐Ÿ” Checking prerequisites..." +check_docker +validate_test_data + +# Stop any existing spark-history-server container +echo "๐Ÿ›‘ Stopping any existing Spark History Server containers..." +docker stop spark-history-server 2>/dev/null && echo " Stopped existing container" || echo " No existing container found" +docker rm spark-history-server 2>/dev/null && echo " Removed existing container" || true + +echo "" +echo "๐Ÿ“Š Available Test Applications:" +echo "===============================" + +# Get actual event directories and their sizes +event_dirs=$(ls -1 examples/basic/events/ 2>/dev/null | grep "eventlog_v2_" | head -10) +if [ -z "$event_dirs" ]; then + echo "โŒ No Spark event logs found in examples/basic/events/" + exit 1 +fi + +# Display available applications with actual sizes +for dir in $event_dirs; do + app_id=$(echo "$dir" | sed 's/eventlog_v2_//') + size=$(du -sh "examples/basic/events/$dir" | cut -f1) + echo "๐Ÿ“‹ $app_id ($size)" +done + +echo "" +echo "๐Ÿ“ Event directories found:" +ls -1 examples/basic/events/ | grep eventlog | sed 's/^/ /' + +echo "" +echo "๐Ÿ“‹ Configuration:" +echo " Log Directory: $(cat examples/basic/history-server.conf)" +echo " Port: 18080" +echo " Docker Image: apache/spark:3.5.5" + +echo "" +echo "๐Ÿš€ Starting Spark History Server..." +echo "๐Ÿ“ Will be available at: http://localhost:18080" +echo "๐Ÿ“ Web UI: http://localhost:18080" +echo "๐Ÿ“ API: http://localhost:18080/api/v1/" +echo "" +echo "โš ๏ธ Keep this terminal open - Press Ctrl+C to stop the server" +echo "โš ๏ธ It may take 30-60 seconds for the server to fully start" +echo "" + +# Check if this is a dry run +if [ "$DRY_RUN" = true ]; then + echo "โœ… Dry run completed successfully!" + echo " All prerequisites are met. Ready to start Spark History Server." + echo "" + echo "To start the server, run:" + echo " ./start_local_spark_history.sh" + exit 0 +fi + +# Start Spark History Server with proper container name and error handling +echo "๐Ÿณ Starting Docker container..." +docker run -it \ + --name spark-history-server \ + --rm \ + -v "$(pwd)/examples/basic:/mnt/data" \ + -p 18080:18080 \ + docker.io/apache/spark:3.5.5 \ + /opt/java/openjdk/bin/java \ + -cp '/opt/spark/conf:/opt/spark/jars/*' \ + -Xmx1g \ + org.apache.spark.deploy.history.HistoryServer \ + --properties-file /mnt/data/history-server.conf + +echo "" +echo "๐Ÿ›‘ Spark History Server stopped." diff --git a/test_spark_client.py b/test_spark_client.py index 4ae0580..0c68fac 100644 --- a/test_spark_client.py +++ b/test_spark_client.py @@ -1,6 +1,5 @@ import unittest -from unittest.mock import patch, MagicMock - +from unittest.mock import MagicMock, patch from config import ServerConfig from spark_client import SparkRestClient @@ -49,6 +48,7 @@ def test_get_applications(self, mock_get): params={"status": ["COMPLETED"], "limit": 10}, headers={"Accept": "application/json"}, auth=None, + timeout=30, ) self.assertEqual(len(apps), 1) @@ -100,6 +100,7 @@ def test_get_applications_with_filters(self, mock_get): }, headers={"Accept": "application/json"}, auth=None, + timeout=30, ) self.assertEqual(len(apps), 1) diff --git a/test_tools.py b/test_tools.py index 16fc0fb..8705e0d 100644 --- a/test_tools.py +++ b/test_tools.py @@ -1,10 +1,25 @@ import unittest +from datetime import datetime, timedelta from unittest.mock import MagicMock, patch -from tools import get_client_or_default, get_stage, get_slowest_jobs from spark_client import SparkRestClient -from datetime import datetime, timedelta -from spark_types import JobData, StageData, TaskMetricDistributions +from spark_types import ( + ApplicationInfo, + ExecutionData, + JobData, + StageData, + TaskMetricDistributions, +) +from tools import ( + get_application, + get_client_or_default, + get_jobs, + get_slowest_jobs, + get_slowest_sql_queries, + get_stage, + get_stage_task_summary, + get_stages, +) class TestTools(unittest.TestCase): @@ -333,3 +348,380 @@ def test_get_stage_no_stages_found(self, mock_get_client): get_stage("app-123", stage_id=1) self.assertIn("No stage found with ID 1", str(context.exception)) + + # Tests for get_application tool + @patch("tools.get_client_or_default") + def test_get_application_success(self, mock_get_client): + """Test successful application retrieval""" + # Setup mock client + mock_client = MagicMock() + mock_app = MagicMock(spec=ApplicationInfo) + mock_app.id = "spark-app-123" + mock_app.name = "Test Application" + mock_client.get_application.return_value = mock_app + mock_get_client.return_value = mock_client + + # Call the function + result = get_application("spark-app-123") + + # Verify results + self.assertEqual(result, mock_app) + mock_client.get_application.assert_called_once_with("spark-app-123") + mock_get_client.assert_called_once_with(unittest.mock.ANY, None) + + @patch("tools.get_client_or_default") + def test_get_application_with_server(self, mock_get_client): + """Test application retrieval with specific server""" + # Setup mock client + mock_client = MagicMock() + mock_app = MagicMock(spec=ApplicationInfo) + mock_client.get_application.return_value = mock_app + mock_get_client.return_value = mock_client + + # Call the function with server + get_application("spark-app-123", server="production") + + # Verify server parameter is passed + mock_get_client.assert_called_once_with(unittest.mock.ANY, "production") + + @patch("tools.get_client_or_default") + def test_get_application_not_found(self, mock_get_client): + """Test application retrieval when app doesn't exist""" + # Setup mock client to raise exception + mock_client = MagicMock() + mock_client.get_application.side_effect = Exception("Application not found") + mock_get_client.return_value = mock_client + + # Verify exception is propagated + with self.assertRaises(Exception) as context: + get_application("non-existent-app") + + self.assertIn("Application not found", str(context.exception)) + + # Tests for get_jobs tool + @patch("tools.get_client_or_default") + def test_get_jobs_no_filter(self, mock_get_client): + """Test job retrieval without status filter""" + # Setup mock client + mock_client = MagicMock() + mock_jobs = [MagicMock(spec=JobData), MagicMock(spec=JobData)] + mock_client.get_jobs.return_value = mock_jobs + mock_get_client.return_value = mock_client + + # Call the function + result = get_jobs("spark-app-123") + + # Verify results + self.assertEqual(result, mock_jobs) + mock_client.get_jobs.assert_called_once_with( + app_id="spark-app-123", status=None + ) + + @patch("tools.get_client_or_default") + def test_get_jobs_with_status_filter(self, mock_get_client): + """Test job retrieval with status filter""" + # Setup mock client + mock_client = MagicMock() + mock_jobs = [MagicMock(spec=JobData)] + mock_jobs[0].status = "SUCCEEDED" + mock_client.get_jobs.return_value = mock_jobs + mock_get_client.return_value = mock_client + + # Call the function with status filter + result = get_jobs("spark-app-123", status=["SUCCEEDED"]) + + # Verify results + self.assertEqual(len(result), 1) + self.assertEqual(result[0].status, "SUCCEEDED") + + @patch("tools.get_client_or_default") + def test_get_jobs_empty_result(self, mock_get_client): + """Test job retrieval with empty result""" + # Setup mock client + mock_client = MagicMock() + mock_client.get_jobs.return_value = [] + mock_get_client.return_value = mock_client + + # Call the function + result = get_jobs("spark-app-123") + + # Verify results + self.assertEqual(result, []) + + @patch("tools.get_client_or_default") + def test_get_jobs_status_filtering(self, mock_get_client): + """Test job status filtering logic""" + # Setup mock client + mock_client = MagicMock() + + # Create jobs with different statuses + job1 = MagicMock(spec=JobData) + job1.status = "RUNNING" + job2 = MagicMock(spec=JobData) + job2.status = "SUCCEEDED" + job3 = MagicMock(spec=JobData) + job3.status = "FAILED" + + # Mock client to return only SUCCEEDED job when filtered + mock_client.get_jobs.return_value = [job2] # Only return SUCCEEDED job + mock_get_client.return_value = mock_client + + # Test filtering for SUCCEEDED jobs + result = get_jobs("spark-app-123", status=["SUCCEEDED"]) + + # Should only return SUCCEEDED job + self.assertEqual(len(result), 1) + self.assertEqual(result[0].status, "SUCCEEDED") + + # Tests for get_stages tool + @patch("tools.get_client_or_default") + def test_get_stages_no_filter(self, mock_get_client): + """Test stage retrieval without filters""" + # Setup mock client + mock_client = MagicMock() + mock_stages = [MagicMock(spec=StageData), MagicMock(spec=StageData)] + mock_client.get_stages.return_value = mock_stages + mock_get_client.return_value = mock_client + + # Call the function + result = get_stages("spark-app-123") + + # Verify results + self.assertEqual(result, mock_stages) + mock_client.get_stages.assert_called_once_with( + app_id="spark-app-123", status=None, with_summaries=False + ) + + @patch("tools.get_client_or_default") + def test_get_stages_with_status_filter(self, mock_get_client): + """Test stage retrieval with status filter""" + # Setup mock client + mock_client = MagicMock() + + # Create stages with different statuses + stage1 = MagicMock(spec=StageData) + stage1.status = "COMPLETE" + stage2 = MagicMock(spec=StageData) + stage2.status = "ACTIVE" + stage3 = MagicMock(spec=StageData) + stage3.status = "FAILED" + + # Mock client to return only COMPLETE stage when filtered + mock_client.get_stages.return_value = [stage1] # Only return COMPLETE stage + mock_get_client.return_value = mock_client + + # Call with status filter + result = get_stages("spark-app-123", status=["COMPLETE"]) + + # Should only return COMPLETE stage + self.assertEqual(len(result), 1) + self.assertEqual(result[0].status, "COMPLETE") + + @patch("tools.get_client_or_default") + def test_get_stages_with_summaries(self, mock_get_client): + """Test stage retrieval with summaries enabled""" + # Setup mock client + mock_client = MagicMock() + mock_stages = [MagicMock(spec=StageData)] + mock_client.get_stages.return_value = mock_stages + mock_get_client.return_value = mock_client + + # Call with summaries enabled + get_stages("spark-app-123", with_summaries=True) + + # Verify summaries parameter is passed + mock_client.get_stages.assert_called_once_with( + app_id="spark-app-123", status=None, with_summaries=True + ) + + @patch("tools.get_client_or_default") + def test_get_stages_empty_result(self, mock_get_client): + """Test stage retrieval with empty result""" + # Setup mock client + mock_client = MagicMock() + mock_client.get_stages.return_value = [] + mock_get_client.return_value = mock_client + + # Call the function + result = get_stages("spark-app-123") + + # Verify results + self.assertEqual(result, []) + + # Tests for get_stage_task_summary tool + @patch("tools.get_client_or_default") + def test_get_stage_task_summary_success(self, mock_get_client): + """Test successful stage task summary retrieval""" + # Setup mock client + mock_client = MagicMock() + mock_summary = MagicMock(spec=TaskMetricDistributions) + mock_client.get_stage_task_summary.return_value = mock_summary + mock_get_client.return_value = mock_client + + # Call the function + result = get_stage_task_summary("spark-app-123", 1, 0) + + # Verify results + self.assertEqual(result, mock_summary) + mock_client.get_stage_task_summary.assert_called_once_with( + app_id="spark-app-123", + stage_id=1, + attempt_id=0, + quantiles="0.05,0.25,0.5,0.75,0.95", + ) + + @patch("tools.get_client_or_default") + def test_get_stage_task_summary_with_quantiles(self, mock_get_client): + """Test stage task summary with custom quantiles""" + # Setup mock client + mock_client = MagicMock() + mock_summary = MagicMock(spec=TaskMetricDistributions) + mock_client.get_stage_task_summary.return_value = mock_summary + mock_get_client.return_value = mock_client + + # Call with custom quantiles + get_stage_task_summary("spark-app-123", 1, 0, quantiles="0.25,0.5,0.75") + + # Verify quantiles parameter is passed + mock_client.get_stage_task_summary.assert_called_once_with( + app_id="spark-app-123", stage_id=1, attempt_id=0, quantiles="0.25,0.5,0.75" + ) + + @patch("tools.get_client_or_default") + def test_get_stage_task_summary_not_found(self, mock_get_client): + """Test stage task summary when stage doesn't exist""" + # Setup mock client to raise exception + mock_client = MagicMock() + mock_client.get_stage_task_summary.side_effect = Exception("Stage not found") + mock_get_client.return_value = mock_client + + # Verify exception is propagated + with self.assertRaises(Exception) as context: + get_stage_task_summary("spark-app-123", 999, 0) + + self.assertIn("Stage not found", str(context.exception)) + + # Tests for get_slowest_sql_queries tool + @patch("tools.get_client_or_default") + def test_get_slowest_sql_queries_success(self, mock_get_client): + """Test successful SQL query retrieval and sorting""" + # Setup mock client + mock_client = MagicMock() + + # Create mock SQL executions with different durations + sql1 = MagicMock(spec=ExecutionData) + sql1.id = 1 + sql1.duration = 5000 # 5 seconds + sql1.status = "COMPLETED" + + sql2 = MagicMock(spec=ExecutionData) + sql2.id = 2 + sql2.duration = 10000 # 10 seconds + sql2.status = "COMPLETED" + + sql3 = MagicMock(spec=ExecutionData) + sql3.id = 3 + sql3.duration = 2000 # 2 seconds + sql3.status = "COMPLETED" + + mock_client.get_sql_list.return_value = [sql1, sql2, sql3] + mock_get_client.return_value = mock_client + + # Call the function + result = get_slowest_sql_queries("spark-app-123", top_n=2) + + # Verify results are sorted by duration (descending) + self.assertEqual(len(result), 2) + self.assertEqual(result[0].duration, 10000) # Slowest first + self.assertEqual(result[1].duration, 5000) # Second slowest + + @patch("tools.get_client_or_default") + def test_get_slowest_sql_queries_exclude_running(self, mock_get_client): + """Test SQL query retrieval excluding running queries""" + # Setup mock client + mock_client = MagicMock() + + # Create mock SQL executions with different statuses + sql1 = MagicMock(spec=ExecutionData) + sql1.id = 1 + sql1.duration = 5000 + sql1.status = "RUNNING" + + sql2 = MagicMock(spec=ExecutionData) + sql2.id = 2 + sql2.duration = 10000 + sql2.status = "COMPLETED" + + mock_client.get_sql_list.return_value = [sql1, sql2] + mock_get_client.return_value = mock_client + + # Call the function (include_running=False by default) + result = get_slowest_sql_queries("spark-app-123") + + # Should exclude running query + self.assertEqual(len(result), 1) + self.assertEqual(result[0].status, "COMPLETED") + + @patch("tools.get_client_or_default") + def test_get_slowest_sql_queries_include_running(self, mock_get_client): + """Test SQL query retrieval including running queries""" + # Setup mock client + mock_client = MagicMock() + + # Create mock SQL executions + sql1 = MagicMock(spec=ExecutionData) + sql1.id = 1 + sql1.duration = 5000 + sql1.status = "RUNNING" + + sql2 = MagicMock(spec=ExecutionData) + sql2.id = 2 + sql2.duration = 10000 + sql2.status = "COMPLETED" + + mock_client.get_sql_list.return_value = [sql1, sql2] + mock_get_client.return_value = mock_client + + # Call the function with include_running=True and top_n=2 + result = get_slowest_sql_queries("spark-app-123", include_running=True, top_n=2) + + # Should include both queries + self.assertEqual(len(result), 2) + + @patch("tools.get_client_or_default") + def test_get_slowest_sql_queries_empty_result(self, mock_get_client): + """Test SQL query retrieval with empty result""" + # Setup mock client + mock_client = MagicMock() + mock_client.get_sql_list.return_value = [] + mock_get_client.return_value = mock_client + + # Call the function + result = get_slowest_sql_queries("spark-app-123") + + # Verify results + self.assertEqual(result, []) + + @patch("tools.get_client_or_default") + def test_get_slowest_sql_queries_limit(self, mock_get_client): + """Test SQL query retrieval with limit""" + # Setup mock client + mock_client = MagicMock() + + # Create 5 mock SQL executions + sql_queries = [] + for i in range(5): + sql = MagicMock(spec=ExecutionData) + sql.id = i + sql.duration = (i + 1) * 1000 # Different durations + sql.status = "COMPLETED" + sql_queries.append(sql) + + mock_client.get_sql_list.return_value = sql_queries + mock_get_client.return_value = mock_client + + # Call the function with limit + result = get_slowest_sql_queries("spark-app-123", top_n=3) + + # Should return only 3 results + self.assertEqual(len(result), 3) diff --git a/tools.py b/tools.py index 0ea0ce3..e58fd9e 100644 --- a/tools.py +++ b/tools.py @@ -1,15 +1,15 @@ -from typing import Optional, List +from typing import Any, Dict, List, Optional from app import mcp from spark_types import ( - JobExecutionStatus, + ApplicationInfo, + ExecutionData, JobData, + JobExecutionStatus, + SQLExecutionStatus, StageData, StageStatus, TaskMetricDistributions, - ExecutionData, - SQLExecutionStatus, - ApplicationInfo, ) @@ -405,6 +405,325 @@ def get_executor_summary(spark_id: str, server: Optional[str] = None): return summary +@mcp.tool() +def compare_job_environments( + spark_id1: str, spark_id2: str, server: Optional[str] = None +) -> Dict[str, Any]: + """ + Compare Spark environment configurations between two jobs. + + Identifies differences in Spark properties, JVM settings, system properties, + and other configuration parameters between two Spark applications. + + Args: + spark_id1: First Spark application ID + spark_id2: Second Spark application ID + server: Optional server name to use (uses default if not specified) + + Returns: + Dictionary containing configuration differences and similarities + """ + ctx = mcp.get_context() + client = get_client_or_default(ctx, server) + + env1 = client.get_environment(app_id=spark_id1) + env2 = client.get_environment(app_id=spark_id2) + + def props_to_dict(props): + return {k: v for k, v in props} if props else {} + + spark_props1 = props_to_dict(env1.spark_properties) + spark_props2 = props_to_dict(env2.spark_properties) + + system_props1 = props_to_dict(env1.system_properties) + system_props2 = props_to_dict(env2.system_properties) + + comparison = { + "applications": {"app1": spark_id1, "app2": spark_id2}, + "runtime_comparison": { + "app1": { + "java_version": env1.runtime.java_version, + "java_home": env1.runtime.java_home, + "scala_version": env1.runtime.scala_version, + }, + "app2": { + "java_version": env2.runtime.java_version, + "java_home": env2.runtime.java_home, + "scala_version": env2.runtime.scala_version, + }, + }, + "spark_properties": { + "common": { + k: {"app1": v, "app2": spark_props2.get(k)} + for k, v in spark_props1.items() + if k in spark_props2 and v == spark_props2[k] + }, + "different": { + k: {"app1": v, "app2": spark_props2.get(k, "NOT_SET")} + for k, v in spark_props1.items() + if k in spark_props2 and v != spark_props2[k] + }, + "only_in_app1": { + k: v for k, v in spark_props1.items() if k not in spark_props2 + }, + "only_in_app2": { + k: v for k, v in spark_props2.items() if k not in spark_props1 + }, + }, + "system_properties": { + "key_differences": { + k: { + "app1": system_props1.get(k, "NOT_SET"), + "app2": system_props2.get(k, "NOT_SET"), + } + for k in [ + "java.version", + "java.runtime.version", + "os.name", + "os.version", + "user.timezone", + "file.encoding", + ] + if system_props1.get(k) != system_props2.get(k) + } + }, + } + + return comparison + + +@mcp.tool() +def compare_job_performance( + spark_id1: str, spark_id2: str, server: Optional[str] = None +) -> Dict[str, Any]: + """ + Compare performance metrics between two Spark jobs. + + Analyzes execution times, resource usage, task distribution, and other + performance indicators to identify differences between jobs. + + Args: + spark_id1: First Spark application ID + spark_id2: Second Spark application ID + server: Optional server name to use (uses default if not specified) + + Returns: + Dictionary containing detailed performance comparison + """ + ctx = mcp.get_context() + client = get_client_or_default(ctx, server) + + # Get application info + app1 = client.get_application(spark_id1) + app2 = client.get_application(spark_id2) + + # Get executor summaries + exec_summary1 = get_executor_summary(spark_id1, server) + exec_summary2 = get_executor_summary(spark_id2, server) + + # Get job data + jobs1 = client.get_jobs(app_id=spark_id1) + jobs2 = client.get_jobs(app_id=spark_id2) + + # Calculate job duration statistics + def calc_job_stats(jobs): + if not jobs: + return {"count": 0, "total_duration": 0, "avg_duration": 0} + + completed_jobs = [j for j in jobs if j.completion_time and j.submission_time] + if not completed_jobs: + return {"count": len(jobs), "total_duration": 0, "avg_duration": 0} + + durations = [ + (j.completion_time - j.submission_time).total_seconds() + for j in completed_jobs + ] + + return { + "count": len(jobs), + "completed_count": len(completed_jobs), + "total_duration": sum(durations), + "avg_duration": sum(durations) / len(durations), + "min_duration": min(durations), + "max_duration": max(durations), + } + + job_stats1 = calc_job_stats(jobs1) + job_stats2 = calc_job_stats(jobs2) + + comparison = { + "applications": { + "app1": {"id": spark_id1, "name": app1.name}, + "app2": {"id": spark_id2, "name": app2.name}, + }, + "resource_allocation": { + "app1": { + "cores_granted": app1.cores_granted, + "max_cores": app1.max_cores, + "cores_per_executor": app1.cores_per_executor, + "memory_per_executor_mb": app1.memory_per_executor_mb, + }, + "app2": { + "cores_granted": app2.cores_granted, + "max_cores": app2.max_cores, + "cores_per_executor": app2.cores_per_executor, + "memory_per_executor_mb": app2.memory_per_executor_mb, + }, + }, + "executor_metrics": { + "app1": exec_summary1, + "app2": exec_summary2, + "comparison": { + "executor_count_ratio": exec_summary2["total_executors"] + / max(exec_summary1["total_executors"], 1), + "memory_usage_ratio": exec_summary2["memory_used"] + / max(exec_summary1["memory_used"], 1), + "task_completion_ratio": exec_summary2["completed_tasks"] + / max(exec_summary1["completed_tasks"], 1), + "gc_time_ratio": exec_summary2["total_gc_time"] + / max(exec_summary1["total_gc_time"], 1), + }, + }, + "job_performance": { + "app1": job_stats1, + "app2": job_stats2, + "comparison": { + "job_count_ratio": job_stats2["count"] / max(job_stats1["count"], 1), + "avg_duration_ratio": job_stats2["avg_duration"] + / max(job_stats1["avg_duration"], 1) + if job_stats1["avg_duration"] > 0 + else 0, + "total_duration_ratio": job_stats2["total_duration"] + / max(job_stats1["total_duration"], 1) + if job_stats1["total_duration"] > 0 + else 0, + }, + }, + } + + return comparison + + +@mcp.tool() +def compare_sql_execution_plans( + spark_id1: str, + spark_id2: str, + execution_id1: Optional[int] = None, + execution_id2: Optional[int] = None, + server: Optional[str] = None, +) -> Dict[str, Any]: + """ + Compare SQL execution plans between two Spark jobs. + + Analyzes the logical and physical plans, identifies differences in operations, + and compares execution metrics between SQL queries. + + Args: + spark_id1: First Spark application ID + spark_id2: Second Spark application ID + execution_id1: Optional specific execution ID for first app (uses longest if not specified) + execution_id2: Optional specific execution ID for second app (uses longest if not specified) + server: Optional server name to use (uses default if not specified) + + Returns: + Dictionary containing SQL execution plan comparison + """ + ctx = mcp.get_context() + client = get_client_or_default(ctx, server) + + # Get SQL executions for both applications + sql_execs1 = client.get_sql_list( + app_id=spark_id1, details=True, plan_description=True + ) + sql_execs2 = client.get_sql_list( + app_id=spark_id2, details=True, plan_description=True + ) + + # If specific execution IDs not provided, use the longest running ones + if execution_id1 is None and sql_execs1: + execution_id1 = max(sql_execs1, key=lambda x: x.duration or 0).id + if execution_id2 is None and sql_execs2: + execution_id2 = max(sql_execs2, key=lambda x: x.duration or 0).id + + if execution_id1 is None or execution_id2 is None: + return { + "error": "No SQL executions found in one or both applications", + "app1_sql_count": len(sql_execs1), + "app2_sql_count": len(sql_execs2), + } + + # Get specific execution details + exec1 = client.get_sql_execution( + spark_id1, execution_id1, details=True, plan_description=True + ) + exec2 = client.get_sql_execution( + spark_id2, execution_id2, details=True, plan_description=True + ) + + # Analyze nodes and operations + def analyze_nodes(execution): + node_types = {} + for node in execution.nodes: + node_type = node.node_name + if node_type not in node_types: + node_types[node_type] = 0 + node_types[node_type] += 1 + return node_types + + nodes1 = analyze_nodes(exec1) + nodes2 = analyze_nodes(exec2) + + all_node_types = set(nodes1.keys()) | set(nodes2.keys()) + + comparison = { + "applications": {"app1": spark_id1, "app2": spark_id2}, + "executions": { + "app1": { + "execution_id": execution_id1, + "duration": exec1.duration, + "status": exec1.status, + "node_count": len(exec1.nodes), + "edge_count": len(exec1.edges), + }, + "app2": { + "execution_id": execution_id2, + "duration": exec2.duration, + "status": exec2.status, + "node_count": len(exec2.nodes), + "edge_count": len(exec2.edges), + }, + }, + "plan_structure": { + "node_type_comparison": { + node_type: { + "app1_count": nodes1.get(node_type, 0), + "app2_count": nodes2.get(node_type, 0), + } + for node_type in sorted(all_node_types) + }, + "complexity_metrics": { + "node_count_ratio": len(exec2.nodes) / max(len(exec1.nodes), 1), + "edge_count_ratio": len(exec2.edges) / max(len(exec1.edges), 1), + "duration_ratio": (exec2.duration or 0) / max(exec1.duration or 1, 1), + }, + }, + "job_associations": { + "app1": { + "running_jobs": exec1.running_job_ids, + "success_jobs": exec1.success_job_ids, + "failed_jobs": exec1.failed_job_ids, + }, + "app2": { + "running_jobs": exec2.running_job_ids, + "success_jobs": exec2.success_job_ids, + "failed_jobs": exec2.failed_job_ids, + }, + }, + } + + return comparison + + @mcp.tool() def get_stage_task_summary( spark_id: str, @@ -497,4 +816,285 @@ def get_slowest_sql_queries( # Sort by duration (descending) and take top N sorted_executions = sorted(all_executions, key=lambda e: e.duration, reverse=True) - return sorted_executions[:2] + return sorted_executions[:top_n] + + +@mcp.tool() +def get_job_bottlenecks( + spark_id: str, server: Optional[str] = None, top_n: int = 5 +) -> Dict[str, Any]: + """ + Identify performance bottlenecks in a Spark job. + + Analyzes stages, tasks, and executors to find the most time-consuming + operations and resource-intensive components. + + Args: + spark_id: The Spark application ID + server: Optional server name to use (uses default if not specified) + top_n: Number of top bottlenecks to return + + Returns: + Dictionary containing identified bottlenecks and recommendations + """ + ctx = mcp.get_context() + client = get_client_or_default(ctx, server) + + # Get slowest stages + slowest_stages = get_slowest_stages(spark_id, server, False, top_n) + + # Get slowest jobs + slowest_jobs = get_slowest_jobs(spark_id, server, False, top_n) + + # Get executor summary + exec_summary = get_executor_summary(spark_id, server) + + # Get all stages for detailed analysis + all_stages = client.get_stages(app_id=spark_id, details=True) + + # Identify stages with high spill + high_spill_stages = [] + for stage in all_stages: + if ( + stage.memory_bytes_spilled + and stage.memory_bytes_spilled > 100 * 1024 * 1024 + ): # > 100MB + high_spill_stages.append( + { + "stage_id": stage.stage_id, + "attempt_id": stage.attempt_id, + "name": stage.name, + "memory_spilled_mb": stage.memory_bytes_spilled / (1024 * 1024), + "disk_spilled_mb": stage.disk_bytes_spilled / (1024 * 1024) + if stage.disk_bytes_spilled + else 0, + } + ) + + # Sort by memory spilled + high_spill_stages.sort(key=lambda x: x["memory_spilled_mb"], reverse=True) + + # Identify GC pressure + gc_pressure = ( + exec_summary["total_gc_time"] / max(exec_summary["total_duration"], 1) + if exec_summary["total_duration"] > 0 + else 0 + ) + + bottlenecks = { + "application_id": spark_id, + "performance_bottlenecks": { + "slowest_stages": [ + { + "stage_id": stage.stage_id, + "attempt_id": stage.attempt_id, + "name": stage.name, + "duration_seconds": ( + stage.completion_time - stage.submission_time + ).total_seconds() + if stage.completion_time and stage.submission_time + else 0, + "task_count": stage.num_tasks, + "failed_tasks": stage.num_failed_tasks, + } + for stage in slowest_stages[:top_n] + ], + "slowest_jobs": [ + { + "job_id": job.job_id, + "name": job.name, + "duration_seconds": ( + job.completion_time - job.submission_time + ).total_seconds() + if job.completion_time and job.submission_time + else 0, + "failed_tasks": job.num_failed_tasks, + "status": job.status, + } + for job in slowest_jobs[:top_n] + ], + }, + "resource_bottlenecks": { + "memory_spill_stages": high_spill_stages[:top_n], + "gc_pressure_ratio": gc_pressure, + "executor_utilization": { + "total_executors": exec_summary["total_executors"], + "active_executors": exec_summary["active_executors"], + "utilization_ratio": exec_summary["active_executors"] + / max(exec_summary["total_executors"], 1), + }, + }, + "recommendations": [], + } + + # Generate recommendations + if gc_pressure > 0.1: # More than 10% time in GC + bottlenecks["recommendations"].append( + { + "type": "memory", + "priority": "high", + "issue": f"High GC pressure ({gc_pressure:.1%})", + "suggestion": "Consider increasing executor memory or reducing memory usage", + } + ) + + if high_spill_stages: + bottlenecks["recommendations"].append( + { + "type": "memory", + "priority": "high", + "issue": f"Memory spilling detected in {len(high_spill_stages)} stages", + "suggestion": "Increase executor memory or optimize data partitioning", + } + ) + + if exec_summary["failed_tasks"] > 0: + bottlenecks["recommendations"].append( + { + "type": "reliability", + "priority": "medium", + "issue": f"{exec_summary['failed_tasks']} failed tasks", + "suggestion": "Investigate task failures and consider increasing task retry settings", + } + ) + + return bottlenecks + + +@mcp.tool() +def get_resource_usage_timeline( + spark_id: str, server: Optional[str] = None +) -> Dict[str, Any]: + """ + Get resource usage timeline for a Spark application. + + Provides a chronological view of resource allocation and usage patterns + including executor additions/removals and stage execution overlap. + + Args: + spark_id: The Spark application ID + server: Optional server name to use (uses default if not specified) + + Returns: + Dictionary containing timeline of resource usage + """ + ctx = mcp.get_context() + client = get_client_or_default(ctx, server) + + # Get application info + app = client.get_application(spark_id) + + # Get all executors + executors = client.get_all_executors(app_id=spark_id) + + # Get stages + stages = client.get_stages(app_id=spark_id, details=True) + + # Create timeline events + timeline_events = [] + + # Add executor events + for executor in executors: + if executor.add_time: + timeline_events.append( + { + "timestamp": executor.add_time, + "type": "executor_add", + "executor_id": executor.id, + "cores": executor.total_cores, + "memory_mb": executor.max_memory / (1024 * 1024) + if executor.max_memory + else 0, + } + ) + + if executor.remove_time: + timeline_events.append( + { + "timestamp": executor.remove_time, + "type": "executor_remove", + "executor_id": executor.id, + "reason": executor.remove_reason, + } + ) + + # Add stage events + for stage in stages: + if stage.submission_time: + timeline_events.append( + { + "timestamp": stage.submission_time, + "type": "stage_start", + "stage_id": stage.stage_id, + "attempt_id": stage.attempt_id, + "name": stage.name, + "task_count": stage.num_tasks, + } + ) + + if stage.completion_time: + timeline_events.append( + { + "timestamp": stage.completion_time, + "type": "stage_end", + "stage_id": stage.stage_id, + "attempt_id": stage.attempt_id, + "status": stage.status, + "duration_seconds": ( + stage.completion_time - stage.submission_time + ).total_seconds() + if stage.submission_time + else 0, + } + ) + + # Sort events by timestamp + timeline_events.sort(key=lambda x: x["timestamp"]) + + # Calculate resource utilization over time + active_executors = 0 + total_cores = 0 + total_memory = 0 + + resource_timeline = [] + + for event in timeline_events: + if event["type"] == "executor_add": + active_executors += 1 + total_cores += event["cores"] + total_memory += event["memory_mb"] + elif event["type"] == "executor_remove": + active_executors -= 1 + # Note: We don't have cores/memory info in remove events + + resource_timeline.append( + { + "timestamp": event["timestamp"], + "active_executors": active_executors, + "total_cores": total_cores, + "total_memory_mb": total_memory, + "event": event, + } + ) + + return { + "application_id": spark_id, + "application_name": app.name, + "timeline": resource_timeline, + "summary": { + "total_events": len(timeline_events), + "executor_additions": len( + [e for e in timeline_events if e["type"] == "executor_add"] + ), + "executor_removals": len( + [e for e in timeline_events if e["type"] == "executor_remove"] + ), + "stage_executions": len( + [e for e in timeline_events if e["type"] == "stage_start"] + ), + "peak_executors": max( + [r["active_executors"] for r in resource_timeline] + [0] + ), + "peak_cores": max([r["total_cores"] for r in resource_timeline] + [0]), + }, + } diff --git a/uv.lock b/uv.lock index 579c4e0..1b03fa3 100644 --- a/uv.lock +++ b/uv.lock @@ -25,6 +25,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" }, ] +[[package]] +name = "black" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "mypy-extensions" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/49/26a7b0f3f35da4b5a65f081943b7bcd22d7002f5f0fb8098ec1ff21cb6ef/black-25.1.0.tar.gz", hash = "sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666", size = 649449, upload-time = "2025-01-29T04:15:40.373Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/71/3fe4741df7adf015ad8dfa082dd36c94ca86bb21f25608eb247b4afb15b2/black-25.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4b60580e829091e6f9238c848ea6750efed72140b91b048770b64e74fe04908b", size = 1650988, upload-time = "2025-01-29T05:37:16.707Z" }, + { url = "https://files.pythonhosted.org/packages/13/f3/89aac8a83d73937ccd39bbe8fc6ac8860c11cfa0af5b1c96d081facac844/black-25.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e2978f6df243b155ef5fa7e558a43037c3079093ed5d10fd84c43900f2d8ecc", size = 1453985, upload-time = "2025-01-29T05:37:18.273Z" }, + { url = "https://files.pythonhosted.org/packages/6f/22/b99efca33f1f3a1d2552c714b1e1b5ae92efac6c43e790ad539a163d1754/black-25.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b48735872ec535027d979e8dcb20bf4f70b5ac75a8ea99f127c106a7d7aba9f", size = 1783816, upload-time = "2025-01-29T04:18:33.823Z" }, + { url = "https://files.pythonhosted.org/packages/18/7e/a27c3ad3822b6f2e0e00d63d58ff6299a99a5b3aee69fa77cd4b0076b261/black-25.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:ea0213189960bda9cf99be5b8c8ce66bb054af5e9e861249cd23471bd7b0b3ba", size = 1440860, upload-time = "2025-01-29T04:19:12.944Z" }, + { url = "https://files.pythonhosted.org/packages/98/87/0edf98916640efa5d0696e1abb0a8357b52e69e82322628f25bf14d263d1/black-25.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f0b18a02996a836cc9c9c78e5babec10930862827b1b724ddfe98ccf2f2fe4f", size = 1650673, upload-time = "2025-01-29T05:37:20.574Z" }, + { url = "https://files.pythonhosted.org/packages/52/e5/f7bf17207cf87fa6e9b676576749c6b6ed0d70f179a3d812c997870291c3/black-25.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:afebb7098bfbc70037a053b91ae8437c3857482d3a690fefc03e9ff7aa9a5fd3", size = 1453190, upload-time = "2025-01-29T05:37:22.106Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ee/adda3d46d4a9120772fae6de454c8495603c37c4c3b9c60f25b1ab6401fe/black-25.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:030b9759066a4ee5e5aca28c3c77f9c64789cdd4de8ac1df642c40b708be6171", size = 1782926, upload-time = "2025-01-29T04:18:58.564Z" }, + { url = "https://files.pythonhosted.org/packages/cc/64/94eb5f45dcb997d2082f097a3944cfc7fe87e071907f677e80788a2d7b7a/black-25.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:a22f402b410566e2d1c950708c77ebf5ebd5d0d88a6a2e87c86d9fb48afa0d18", size = 1442613, upload-time = "2025-01-29T04:19:27.63Z" }, + { url = "https://files.pythonhosted.org/packages/09/71/54e999902aed72baf26bca0d50781b01838251a462612966e9fc4891eadd/black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717", size = 207646, upload-time = "2025-01-29T04:15:38.082Z" }, +] + [[package]] name = "certifi" version = "2025.6.15" @@ -90,6 +114,48 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "coverage" +version = "7.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/e0/98670a80884f64578f0c22cd70c5e81a6e07b08167721c7487b4d70a7ca0/coverage-7.9.1.tar.gz", hash = "sha256:6cf43c78c4282708a28e466316935ec7489a9c487518a77fa68f716c67909cec", size = 813650, upload-time = "2025-06-13T13:02:28.627Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/d9/7f66eb0a8f2fce222de7bdc2046ec41cb31fe33fb55a330037833fb88afc/coverage-7.9.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a8de12b4b87c20de895f10567639c0797b621b22897b0af3ce4b4e204a743626", size = 212336, upload-time = "2025-06-13T13:01:10.909Z" }, + { url = "https://files.pythonhosted.org/packages/20/20/e07cb920ef3addf20f052ee3d54906e57407b6aeee3227a9c91eea38a665/coverage-7.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5add197315a054e92cee1b5f686a2bcba60c4c3e66ee3de77ace6c867bdee7cb", size = 212571, upload-time = "2025-06-13T13:01:12.518Z" }, + { url = "https://files.pythonhosted.org/packages/78/f8/96f155de7e9e248ca9c8ff1a40a521d944ba48bec65352da9be2463745bf/coverage-7.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:600a1d4106fe66f41e5d0136dfbc68fe7200a5cbe85610ddf094f8f22e1b0300", size = 246377, upload-time = "2025-06-13T13:01:14.87Z" }, + { url = "https://files.pythonhosted.org/packages/3e/cf/1d783bd05b7bca5c10ded5f946068909372e94615a4416afadfe3f63492d/coverage-7.9.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a876e4c3e5a2a1715a6608906aa5a2e0475b9c0f68343c2ada98110512ab1d8", size = 243394, upload-time = "2025-06-13T13:01:16.23Z" }, + { url = "https://files.pythonhosted.org/packages/02/dd/e7b20afd35b0a1abea09fb3998e1abc9f9bd953bee548f235aebd2b11401/coverage-7.9.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81f34346dd63010453922c8e628a52ea2d2ccd73cb2487f7700ac531b247c8a5", size = 245586, upload-time = "2025-06-13T13:01:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/4e/38/b30b0006fea9d617d1cb8e43b1bc9a96af11eff42b87eb8c716cf4d37469/coverage-7.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:888f8eee13f2377ce86d44f338968eedec3291876b0b8a7289247ba52cb984cd", size = 245396, upload-time = "2025-06-13T13:01:19.164Z" }, + { url = "https://files.pythonhosted.org/packages/31/e4/4d8ec1dc826e16791f3daf1b50943e8e7e1eb70e8efa7abb03936ff48418/coverage-7.9.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9969ef1e69b8c8e1e70d591f91bbc37fc9a3621e447525d1602801a24ceda898", size = 243577, upload-time = "2025-06-13T13:01:22.433Z" }, + { url = "https://files.pythonhosted.org/packages/25/f4/b0e96c5c38e6e40ef465c4bc7f138863e2909c00e54a331da335faf0d81a/coverage-7.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:60c458224331ee3f1a5b472773e4a085cc27a86a0b48205409d364272d67140d", size = 244809, upload-time = "2025-06-13T13:01:24.143Z" }, + { url = "https://files.pythonhosted.org/packages/8a/65/27e0a1fa5e2e5079bdca4521be2f5dabf516f94e29a0defed35ac2382eb2/coverage-7.9.1-cp312-cp312-win32.whl", hash = "sha256:5f646a99a8c2b3ff4c6a6e081f78fad0dde275cd59f8f49dc4eab2e394332e74", size = 214724, upload-time = "2025-06-13T13:01:25.435Z" }, + { url = "https://files.pythonhosted.org/packages/9b/a8/d5b128633fd1a5e0401a4160d02fa15986209a9e47717174f99dc2f7166d/coverage-7.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:30f445f85c353090b83e552dcbbdad3ec84c7967e108c3ae54556ca69955563e", size = 215535, upload-time = "2025-06-13T13:01:27.861Z" }, + { url = "https://files.pythonhosted.org/packages/a3/37/84bba9d2afabc3611f3e4325ee2c6a47cd449b580d4a606b240ce5a6f9bf/coverage-7.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:af41da5dca398d3474129c58cb2b106a5d93bbb196be0d307ac82311ca234342", size = 213904, upload-time = "2025-06-13T13:01:29.202Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a7/a027970c991ca90f24e968999f7d509332daf6b8c3533d68633930aaebac/coverage-7.9.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:31324f18d5969feef7344a932c32428a2d1a3e50b15a6404e97cba1cc9b2c631", size = 212358, upload-time = "2025-06-13T13:01:30.909Z" }, + { url = "https://files.pythonhosted.org/packages/f2/48/6aaed3651ae83b231556750280682528fea8ac7f1232834573472d83e459/coverage-7.9.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0c804506d624e8a20fb3108764c52e0eef664e29d21692afa375e0dd98dc384f", size = 212620, upload-time = "2025-06-13T13:01:32.256Z" }, + { url = "https://files.pythonhosted.org/packages/6c/2a/f4b613f3b44d8b9f144847c89151992b2b6b79cbc506dee89ad0c35f209d/coverage-7.9.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef64c27bc40189f36fcc50c3fb8f16ccda73b6a0b80d9bd6e6ce4cffcd810bbd", size = 245788, upload-time = "2025-06-13T13:01:33.948Z" }, + { url = "https://files.pythonhosted.org/packages/04/d2/de4fdc03af5e4e035ef420ed26a703c6ad3d7a07aff2e959eb84e3b19ca8/coverage-7.9.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d4fe2348cc6ec372e25adec0219ee2334a68d2f5222e0cba9c0d613394e12d86", size = 243001, upload-time = "2025-06-13T13:01:35.285Z" }, + { url = "https://files.pythonhosted.org/packages/f5/e8/eed18aa5583b0423ab7f04e34659e51101135c41cd1dcb33ac1d7013a6d6/coverage-7.9.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:34ed2186fe52fcc24d4561041979a0dec69adae7bce2ae8d1c49eace13e55c43", size = 244985, upload-time = "2025-06-13T13:01:36.712Z" }, + { url = "https://files.pythonhosted.org/packages/17/f8/ae9e5cce8885728c934eaa58ebfa8281d488ef2afa81c3dbc8ee9e6d80db/coverage-7.9.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:25308bd3d00d5eedd5ae7d4357161f4df743e3c0240fa773ee1b0f75e6c7c0f1", size = 245152, upload-time = "2025-06-13T13:01:39.303Z" }, + { url = "https://files.pythonhosted.org/packages/5a/c8/272c01ae792bb3af9b30fac14d71d63371db227980682836ec388e2c57c0/coverage-7.9.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:73e9439310f65d55a5a1e0564b48e34f5369bee943d72c88378f2d576f5a5751", size = 243123, upload-time = "2025-06-13T13:01:40.727Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d0/2819a1e3086143c094ab446e3bdf07138527a7b88cb235c488e78150ba7a/coverage-7.9.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:37ab6be0859141b53aa89412a82454b482c81cf750de4f29223d52268a86de67", size = 244506, upload-time = "2025-06-13T13:01:42.184Z" }, + { url = "https://files.pythonhosted.org/packages/8b/4e/9f6117b89152df7b6112f65c7a4ed1f2f5ec8e60c4be8f351d91e7acc848/coverage-7.9.1-cp313-cp313-win32.whl", hash = "sha256:64bdd969456e2d02a8b08aa047a92d269c7ac1f47e0c977675d550c9a0863643", size = 214766, upload-time = "2025-06-13T13:01:44.482Z" }, + { url = "https://files.pythonhosted.org/packages/27/0f/4b59f7c93b52c2c4ce7387c5a4e135e49891bb3b7408dcc98fe44033bbe0/coverage-7.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:be9e3f68ca9edb897c2184ad0eee815c635565dbe7a0e7e814dc1f7cbab92c0a", size = 215568, upload-time = "2025-06-13T13:01:45.772Z" }, + { url = "https://files.pythonhosted.org/packages/09/1e/9679826336f8c67b9c39a359352882b24a8a7aee48d4c9cad08d38d7510f/coverage-7.9.1-cp313-cp313-win_arm64.whl", hash = "sha256:1c503289ffef1d5105d91bbb4d62cbe4b14bec4d13ca225f9c73cde9bb46207d", size = 213939, upload-time = "2025-06-13T13:01:47.087Z" }, + { url = "https://files.pythonhosted.org/packages/bb/5b/5c6b4e7a407359a2e3b27bf9c8a7b658127975def62077d441b93a30dbe8/coverage-7.9.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0b3496922cb5f4215bf5caaef4cf12364a26b0be82e9ed6d050f3352cf2d7ef0", size = 213079, upload-time = "2025-06-13T13:01:48.554Z" }, + { url = "https://files.pythonhosted.org/packages/a2/22/1e2e07279fd2fd97ae26c01cc2186e2258850e9ec125ae87184225662e89/coverage-7.9.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9565c3ab1c93310569ec0d86b017f128f027cab0b622b7af288696d7ed43a16d", size = 213299, upload-time = "2025-06-13T13:01:49.997Z" }, + { url = "https://files.pythonhosted.org/packages/14/c0/4c5125a4b69d66b8c85986d3321520f628756cf524af810baab0790c7647/coverage-7.9.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2241ad5dbf79ae1d9c08fe52b36d03ca122fb9ac6bca0f34439e99f8327ac89f", size = 256535, upload-time = "2025-06-13T13:01:51.314Z" }, + { url = "https://files.pythonhosted.org/packages/81/8b/e36a04889dda9960be4263e95e777e7b46f1bb4fc32202612c130a20c4da/coverage-7.9.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bb5838701ca68b10ebc0937dbd0eb81974bac54447c55cd58dea5bca8451029", size = 252756, upload-time = "2025-06-13T13:01:54.403Z" }, + { url = "https://files.pythonhosted.org/packages/98/82/be04eff8083a09a4622ecd0e1f31a2c563dbea3ed848069e7b0445043a70/coverage-7.9.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a25f814591a8c0c5372c11ac8967f669b97444c47fd794926e175c4047ece", size = 254912, upload-time = "2025-06-13T13:01:56.769Z" }, + { url = "https://files.pythonhosted.org/packages/0f/25/c26610a2c7f018508a5ab958e5b3202d900422cf7cdca7670b6b8ca4e8df/coverage-7.9.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2d04b16a6062516df97969f1ae7efd0de9c31eb6ebdceaa0d213b21c0ca1a683", size = 256144, upload-time = "2025-06-13T13:01:58.19Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8b/fb9425c4684066c79e863f1e6e7ecebb49e3a64d9f7f7860ef1688c56f4a/coverage-7.9.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7931b9e249edefb07cd6ae10c702788546341d5fe44db5b6108a25da4dca513f", size = 254257, upload-time = "2025-06-13T13:01:59.645Z" }, + { url = "https://files.pythonhosted.org/packages/93/df/27b882f54157fc1131e0e215b0da3b8d608d9b8ef79a045280118a8f98fe/coverage-7.9.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:52e92b01041151bf607ee858e5a56c62d4b70f4dac85b8c8cb7fb8a351ab2c10", size = 255094, upload-time = "2025-06-13T13:02:01.37Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/cad1c3dbed8b3ee9e16fa832afe365b4e3eeab1fb6edb65ebbf745eabc92/coverage-7.9.1-cp313-cp313t-win32.whl", hash = "sha256:684e2110ed84fd1ca5f40e89aa44adf1729dc85444004111aa01866507adf363", size = 215437, upload-time = "2025-06-13T13:02:02.905Z" }, + { url = "https://files.pythonhosted.org/packages/99/4d/fad293bf081c0e43331ca745ff63673badc20afea2104b431cdd8c278b4c/coverage-7.9.1-cp313-cp313t-win_amd64.whl", hash = "sha256:437c576979e4db840539674e68c84b3cda82bc824dd138d56bead1435f1cb5d7", size = 216605, upload-time = "2025-06-13T13:02:05.638Z" }, + { url = "https://files.pythonhosted.org/packages/1f/56/4ee027d5965fc7fc126d7ec1187529cc30cc7d740846e1ecb5e92d31b224/coverage-7.9.1-cp313-cp313t-win_arm64.whl", hash = "sha256:18a0912944d70aaf5f399e350445738a1a20b50fbea788f640751c2ed9208b6c", size = 214392, upload-time = "2025-06-13T13:02:07.642Z" }, + { url = "https://files.pythonhosted.org/packages/08/b8/7ddd1e8ba9701dea08ce22029917140e6f66a859427406579fd8d0ca7274/coverage-7.9.1-py3-none-any.whl", hash = "sha256:66b974b145aa189516b6bf2d8423e888b742517d37872f6ee4c5be0073bd9a3c", size = 204000, upload-time = "2025-06-13T13:02:27.173Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -193,37 +259,47 @@ cli = [ ] [[package]] -name = "mcp-spark-events" -version = "0.1.0" -source = { virtual = "." } -dependencies = [ - { name = "mcp", extra = ["cli"] }, - { name = "pyyaml" }, - { name = "requests" }, +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] -[package.dev-dependencies] -dev = [ - { name = "pytest" }, +[[package]] +name = "mypy" +version = "1.16.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, ] - -[package.metadata] -requires-dist = [ - { name = "mcp", extras = ["cli"], specifier = ">=1.9.4" }, - { name = "pyyaml", specifier = ">=6.0.2" }, - { name = "requests", specifier = ">=2.32.4" }, +sdist = { url = "https://files.pythonhosted.org/packages/81/69/92c7fa98112e4d9eb075a239caa4ef4649ad7d441545ccffbd5e34607cbb/mypy-1.16.1.tar.gz", hash = "sha256:6bd00a0a2094841c5e47e7374bb42b83d64c527a502e3334e1173a0c24437bab", size = 3324747, upload-time = "2025-06-16T16:51:35.145Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/d6/39482e5fcc724c15bf6280ff5806548c7185e0c090712a3736ed4d07e8b7/mypy-1.16.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:af4792433f09575d9eeca5c63d7d90ca4aeceda9d8355e136f80f8967639183d", size = 11066493, upload-time = "2025-06-16T16:47:01.683Z" }, + { url = "https://files.pythonhosted.org/packages/e6/e5/26c347890efc6b757f4d5bb83f4a0cf5958b8cf49c938ac99b8b72b420a6/mypy-1.16.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:66df38405fd8466ce3517eda1f6640611a0b8e70895e2a9462d1d4323c5eb4b9", size = 10081687, upload-time = "2025-06-16T16:48:19.367Z" }, + { url = "https://files.pythonhosted.org/packages/44/c7/b5cb264c97b86914487d6a24bd8688c0172e37ec0f43e93b9691cae9468b/mypy-1.16.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:44e7acddb3c48bd2713994d098729494117803616e116032af192871aed80b79", size = 11839723, upload-time = "2025-06-16T16:49:20.912Z" }, + { url = "https://files.pythonhosted.org/packages/15/f8/491997a9b8a554204f834ed4816bda813aefda31cf873bb099deee3c9a99/mypy-1.16.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ab5eca37b50188163fa7c1b73c685ac66c4e9bdee4a85c9adac0e91d8895e15", size = 12722980, upload-time = "2025-06-16T16:37:40.929Z" }, + { url = "https://files.pythonhosted.org/packages/df/f0/2bd41e174b5fd93bc9de9a28e4fb673113633b8a7f3a607fa4a73595e468/mypy-1.16.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dedb6229b2c9086247e21a83c309754b9058b438704ad2f6807f0d8227f6ebdd", size = 12903328, upload-time = "2025-06-16T16:34:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/61/81/5572108a7bec2c46b8aff7e9b524f371fe6ab5efb534d38d6b37b5490da8/mypy-1.16.1-cp312-cp312-win_amd64.whl", hash = "sha256:1f0435cf920e287ff68af3d10a118a73f212deb2ce087619eb4e648116d1fe9b", size = 9562321, upload-time = "2025-06-16T16:48:58.823Z" }, + { url = "https://files.pythonhosted.org/packages/28/e3/96964af4a75a949e67df4b95318fe2b7427ac8189bbc3ef28f92a1c5bc56/mypy-1.16.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ddc91eb318c8751c69ddb200a5937f1232ee8efb4e64e9f4bc475a33719de438", size = 11063480, upload-time = "2025-06-16T16:47:56.205Z" }, + { url = "https://files.pythonhosted.org/packages/f5/4d/cd1a42b8e5be278fab7010fb289d9307a63e07153f0ae1510a3d7b703193/mypy-1.16.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:87ff2c13d58bdc4bbe7dc0dedfe622c0f04e2cb2a492269f3b418df2de05c536", size = 10090538, upload-time = "2025-06-16T16:46:43.92Z" }, + { url = "https://files.pythonhosted.org/packages/c9/4f/c3c6b4b66374b5f68bab07c8cabd63a049ff69796b844bc759a0ca99bb2a/mypy-1.16.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a7cfb0fe29fe5a9841b7c8ee6dffb52382c45acdf68f032145b75620acfbd6f", size = 11836839, upload-time = "2025-06-16T16:36:28.039Z" }, + { url = "https://files.pythonhosted.org/packages/b4/7e/81ca3b074021ad9775e5cb97ebe0089c0f13684b066a750b7dc208438403/mypy-1.16.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:051e1677689c9d9578b9c7f4d206d763f9bbd95723cd1416fad50db49d52f359", size = 12715634, upload-time = "2025-06-16T16:50:34.441Z" }, + { url = "https://files.pythonhosted.org/packages/e9/95/bdd40c8be346fa4c70edb4081d727a54d0a05382d84966869738cfa8a497/mypy-1.16.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d5d2309511cc56c021b4b4e462907c2b12f669b2dbeb68300110ec27723971be", size = 12895584, upload-time = "2025-06-16T16:34:54.857Z" }, + { url = "https://files.pythonhosted.org/packages/5a/fd/d486a0827a1c597b3b48b1bdef47228a6e9ee8102ab8c28f944cb83b65dc/mypy-1.16.1-cp313-cp313-win_amd64.whl", hash = "sha256:4f58ac32771341e38a853c5d0ec0dfe27e18e27da9cdb8bbc882d2249c71a3ee", size = 9573886, upload-time = "2025-06-16T16:36:43.589Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d3/53e684e78e07c1a2bf7105715e5edd09ce951fc3f47cf9ed095ec1b7a037/mypy-1.16.1-py3-none-any.whl", hash = "sha256:5fc2ac4027d0ef28d6ba69a0343737a23c4d1b83672bf38d1fe237bdc0643b37", size = 2265923, upload-time = "2025-06-16T16:48:02.366Z" }, ] -[package.metadata.requires-dev] -dev = [{ name = "pytest", specifier = ">=8.4.1" }] - [[package]] -name = "mdurl" -version = "0.1.2" +name = "mypy-extensions" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] [[package]] @@ -235,6 +311,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/8b/3c73abc9c759ecd3f1f7ceff6685840859e8070c4d947c93fae71f6a0bf2/platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc", size = 21362, upload-time = "2025-05-07T22:47:42.121Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4", size = 18567, upload-time = "2025-05-07T22:47:40.376Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -340,6 +434,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, ] +[[package]] +name = "pytest-cov" +version = "6.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/99/668cade231f434aaa59bbfbf49469068d2ddd945000621d3d165d2e7dd7b/pytest_cov-6.2.1.tar.gz", hash = "sha256:25cc6cc0a5358204b8108ecedc51a9b57b34cc6b8c967cc2c01a4e00d8a67da2", size = 69432, upload-time = "2025-06-12T10:47:47.684Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/16/4ea354101abb1287856baa4af2732be351c7bee728065aed451b678153fd/pytest_cov-6.2.1-py3-none-any.whl", hash = "sha256:f5bc4c23f42f1cdd23c70b1dab1bbaef4fc505ba950d53e0081d0730dd7e86d5", size = 24644, upload-time = "2025-06-12T10:47:45.932Z" }, +] + [[package]] name = "python-dotenv" version = "1.1.0" @@ -412,6 +520,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0d/9b/63f4c7ebc259242c89b3acafdb37b41d1185c07ff0011164674e9076b491/rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0", size = 243229, upload-time = "2025-03-30T14:15:12.283Z" }, ] +[[package]] +name = "ruff" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/97/38/796a101608a90494440856ccfb52b1edae90de0b817e76bfade66b12d320/ruff-0.12.1.tar.gz", hash = "sha256:806bbc17f1104fd57451a98a58df35388ee3ab422e029e8f5cf30aa4af2c138c", size = 4413426, upload-time = "2025-06-26T20:34:14.784Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/bf/3dba52c1d12ab5e78d75bd78ad52fb85a6a1f29cc447c2423037b82bed0d/ruff-0.12.1-py3-none-linux_armv6l.whl", hash = "sha256:6013a46d865111e2edb71ad692fbb8262e6c172587a57c0669332a449384a36b", size = 10305649, upload-time = "2025-06-26T20:33:39.242Z" }, + { url = "https://files.pythonhosted.org/packages/8c/65/dab1ba90269bc8c81ce1d499a6517e28fe6f87b2119ec449257d0983cceb/ruff-0.12.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b3f75a19e03a4b0757d1412edb7f27cffb0c700365e9d6b60bc1b68d35bc89e0", size = 11120201, upload-time = "2025-06-26T20:33:42.207Z" }, + { url = "https://files.pythonhosted.org/packages/3f/3e/2d819ffda01defe857fa2dd4cba4d19109713df4034cc36f06bbf582d62a/ruff-0.12.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9a256522893cb7e92bb1e1153283927f842dea2e48619c803243dccc8437b8be", size = 10466769, upload-time = "2025-06-26T20:33:44.102Z" }, + { url = "https://files.pythonhosted.org/packages/63/37/bde4cf84dbd7821c8de56ec4ccc2816bce8125684f7b9e22fe4ad92364de/ruff-0.12.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:069052605fe74c765a5b4272eb89880e0ff7a31e6c0dbf8767203c1fbd31c7ff", size = 10660902, upload-time = "2025-06-26T20:33:45.98Z" }, + { url = "https://files.pythonhosted.org/packages/0e/3a/390782a9ed1358c95e78ccc745eed1a9d657a537e5c4c4812fce06c8d1a0/ruff-0.12.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a684f125a4fec2d5a6501a466be3841113ba6847827be4573fddf8308b83477d", size = 10167002, upload-time = "2025-06-26T20:33:47.81Z" }, + { url = "https://files.pythonhosted.org/packages/6d/05/f2d4c965009634830e97ffe733201ec59e4addc5b1c0efa035645baa9e5f/ruff-0.12.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bdecdef753bf1e95797593007569d8e1697a54fca843d78f6862f7dc279e23bd", size = 11751522, upload-time = "2025-06-26T20:33:49.857Z" }, + { url = "https://files.pythonhosted.org/packages/35/4e/4bfc519b5fcd462233f82fc20ef8b1e5ecce476c283b355af92c0935d5d9/ruff-0.12.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:70d52a058c0e7b88b602f575d23596e89bd7d8196437a4148381a3f73fcd5010", size = 12520264, upload-time = "2025-06-26T20:33:52.199Z" }, + { url = "https://files.pythonhosted.org/packages/85/b2/7756a6925da236b3a31f234b4167397c3e5f91edb861028a631546bad719/ruff-0.12.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84d0a69d1e8d716dfeab22d8d5e7c786b73f2106429a933cee51d7b09f861d4e", size = 12133882, upload-time = "2025-06-26T20:33:54.231Z" }, + { url = "https://files.pythonhosted.org/packages/dd/00/40da9c66d4a4d51291e619be6757fa65c91b92456ff4f01101593f3a1170/ruff-0.12.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6cc32e863adcf9e71690248607ccdf25252eeeab5193768e6873b901fd441fed", size = 11608941, upload-time = "2025-06-26T20:33:56.202Z" }, + { url = "https://files.pythonhosted.org/packages/91/e7/f898391cc026a77fbe68dfea5940f8213622474cb848eb30215538a2dadf/ruff-0.12.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fd49a4619f90d5afc65cf42e07b6ae98bb454fd5029d03b306bd9e2273d44cc", size = 11602887, upload-time = "2025-06-26T20:33:58.47Z" }, + { url = "https://files.pythonhosted.org/packages/f6/02/0891872fc6aab8678084f4cf8826f85c5d2d24aa9114092139a38123f94b/ruff-0.12.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ed5af6aaaea20710e77698e2055b9ff9b3494891e1b24d26c07055459bb717e9", size = 10521742, upload-time = "2025-06-26T20:34:00.465Z" }, + { url = "https://files.pythonhosted.org/packages/2a/98/d6534322c74a7d47b0f33b036b2498ccac99d8d8c40edadb552c038cecf1/ruff-0.12.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:801d626de15e6bf988fbe7ce59b303a914ff9c616d5866f8c79eb5012720ae13", size = 10149909, upload-time = "2025-06-26T20:34:02.603Z" }, + { url = "https://files.pythonhosted.org/packages/34/5c/9b7ba8c19a31e2b6bd5e31aa1e65b533208a30512f118805371dbbbdf6a9/ruff-0.12.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:2be9d32a147f98a1972c1e4df9a6956d612ca5f5578536814372113d09a27a6c", size = 11136005, upload-time = "2025-06-26T20:34:04.723Z" }, + { url = "https://files.pythonhosted.org/packages/dc/34/9bbefa4d0ff2c000e4e533f591499f6b834346025e11da97f4ded21cb23e/ruff-0.12.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:49b7ce354eed2a322fbaea80168c902de9504e6e174fd501e9447cad0232f9e6", size = 11648579, upload-time = "2025-06-26T20:34:06.766Z" }, + { url = "https://files.pythonhosted.org/packages/6f/1c/20cdb593783f8f411839ce749ec9ae9e4298c2b2079b40295c3e6e2089e1/ruff-0.12.1-py3-none-win32.whl", hash = "sha256:d973fa626d4c8267848755bd0414211a456e99e125dcab147f24daa9e991a245", size = 10519495, upload-time = "2025-06-26T20:34:08.718Z" }, + { url = "https://files.pythonhosted.org/packages/cf/56/7158bd8d3cf16394928f47c637d39a7d532268cd45220bdb6cd622985760/ruff-0.12.1-py3-none-win_amd64.whl", hash = "sha256:9e1123b1c033f77bd2590e4c1fe7e8ea72ef990a85d2484351d408224d603013", size = 11547485, upload-time = "2025-06-26T20:34:11.008Z" }, + { url = "https://files.pythonhosted.org/packages/91/d0/6902c0d017259439d6fd2fd9393cea1cfe30169940118b007d5e0ea7e954/ruff-0.12.1-py3-none-win_arm64.whl", hash = "sha256:78ad09a022c64c13cc6077707f036bab0fac8cd7088772dcd1e5be21c5002efc", size = 10691209, upload-time = "2025-06-26T20:34:12.928Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -430,6 +563,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "spark-history-server-mcp" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "mcp", extra = ["cli"] }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "requests" }, +] + +[package.dev-dependencies] +dev = [ + { name = "black" }, + { name = "mypy" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "mcp", extras = ["cli"], specifier = ">=1.9.4" }, + { name = "pydantic", specifier = ">=2.0.0" }, + { name = "pyyaml", specifier = ">=6.0.2" }, + { name = "requests", specifier = ">=2.32.4" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "black", specifier = ">=23.0.0" }, + { name = "mypy", specifier = ">=1.7.0" }, + { name = "pytest", specifier = ">=8.4.1" }, + { name = "pytest-cov", specifier = ">=4.0.0" }, + { name = "ruff", specifier = ">=0.1.0" }, +] + [[package]] name = "sse-starlette" version = "2.3.6"