Merge branch 'main' into 5864-fix-rust-cargo-workspace-dep-discovery

.devcontainer/core-dev/devcontainer.json

@@ -2,7 +2,7 @@
   "name": "core-dev",
   "build": {
     "dockerfile": "../../Dockerfile.development",
-    "cacheFrom": "ghcr.io/dependabot/dependabot-core-development"
+    "cacheFrom": "ghcr.io/dependabot/dependabot-updater-core"
   },
 
   "workspaceFolder": "/home/dependabot/dependabot-core",

.github/dependabot.yml

@@ -4,44 +4,80 @@ updates:
     directory: "/updater"
     schedule:
       interval: "weekly"
+      day: "sunday"
+      time: "16:00"
 
   # Watch the per-ecosystem native helpers
   - package-ecosystem: "composer"
     directory: "/composer/helpers/v1"
     schedule:
       interval: "weekly"
+      day: "sunday"
+      time: "16:00"      
   - package-ecosystem: "composer"
     directory: "/composer/helpers/v2"
     schedule:
       interval: "weekly"
+      day: "sunday"
+      time: "16:00"
   - package-ecosystem: "docker"
     directory: "/"
     schedule:
       interval: "weekly"
+      day: "sunday"
+      time: "16:00"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "weekly"
+      day: "sunday"
+      time: "16:00"
   - package-ecosystem: "gomod"
     directory: "/go_modules/helpers"
     schedule:
       interval: "weekly"
+      day: "sunday"
+      time: "16:00"
   - package-ecosystem: "mix"
     directory: "/hex/helpers"
     schedule:
       interval: "weekly"
+      day: "sunday"
+      time: "16:00"
   - package-ecosystem: "npm"
     directory: "/npm_and_yarn/helpers"
     schedule:
       interval: "weekly"
+      day: "sunday"
+      time: "16:00"
+    groups:
+      dependencies:
+        patterns:
+          - "@dependabot/yarn-lib"
+          - "@npmcli/arborist"
+          - "detect-indent"
+          - "nock"
+          - "npm"
+          - "@pnpm/lockfile-file"
+          - "@pnpm/dependency-path"
+          - "semver"
+      dev-dependencies:
+        patterns:
+          - "*eslint*"
+          - "*jest*"
+          - "*prettier"
     ignore:
       - dependency-name: "npm"
         update-types: ["version-update:semver-major"]
   - package-ecosystem: "pip"
     directory: "/python/helpers"
     schedule:
       interval: "weekly"
+      day: "sunday"
+      time: "16:00"
   - package-ecosystem: "pub"
     directory: "/pub/helpers"
     schedule:
       interval: "weekly"
+      day: "sunday"
+      time: "16:00"

.github/workflows/ci.yml

@@ -7,6 +7,10 @@ on:
   schedule:
     - cron: "0 0 * * *"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   ci:
     runs-on: ubuntu-latest

.github/workflows/codeql-analysis.yml

@@ -23,6 +23,10 @@ on:
   schedule:
     - cron: '41 4 * * 3'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze
@@ -45,20 +49,21 @@ jobs:
       uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
+    - name: Initialize CodeQL (ruby)
       uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
-        # TODO: Unpin when https://github.com/github/codeql/issues/13103 fixed.
-        tools: https://github.com/github/codeql-action/releases/download/codeql-bundle-20230418/codeql-bundle.tar.gz
         config: |
           paths-ignore:
             - 'bundler/spec/fixtures/projects/bundler1/invalid_ruby/Gemfile'
             - 'bundler/spec/fixtures/projects/bundler2/invalid_ruby/Gemfile'
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+      if: matrix.language == 'ruby'
+
+    - name: Initialize CodeQL (others)
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+      if: matrix.language != 'ruby'
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)

.github/workflows/dependency-review.yml

@@ -4,6 +4,10 @@ on: [pull_request]
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   dependency-review:
     name: Dependency Review

.github/workflows/gems-bump-version.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Bump the version
         # Cron runs with no inputs, so version_type will default to 'minor'
         run: |
-          NEW_VERSION=$(bin/bump-version.rb ${{ github.event.inputs.version_type || 'minor' }})
+          NEW_VERSION=$(bin/bump-version.rb ${{ inputs.version_type || 'minor' }})
           echo "New version is: $NEW_VERSION"
           echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_ENV
 

.github/workflows/images-branch.yml

@@ -55,7 +55,7 @@ jobs:
         if: github.event_name != 'workflow_dispatch'
 
       - name: Set PR (forks)
-        run: echo "PR=${{ github.event.inputs.pr }}" >> $GITHUB_ENV
+        run: echo "PR=${{ inputs.pr }}" >> $GITHUB_ENV
         if: github.event_name == 'workflow_dispatch'
 
         # sets DECISION to the PR's review decision, handling the push-after-approval case
@@ -76,7 +76,7 @@ jobs:
 
       - name: Prepare tag (forks)
         run: |
-          gh pr checkout ${{ github.event.inputs.pr }}
+          gh pr checkout ${{ inputs.pr }}
           git fetch origin main
           git merge origin/main --ff-only || exit 1
           echo "TAG=$(git rev-parse HEAD)" >> $GITHUB_ENV

.github/workflows/smoke.yml

@@ -9,6 +9,11 @@ on:
     paths-ignore:
       - docs/**
       - README.md
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 jobs:
@@ -20,6 +25,7 @@ jobs:
         suite:
           - { path: bundler, name: bundler, ecosystem: bundler }
           - { path: bundler, name: bundler-group-rules, ecosystem: bundler }
+          - { path: bundler, name: bundler-group-vendoring, ecosystem: bundler }
           - { path: cargo, name: cargo, ecosystem: cargo }
           - { path: composer, name: composer, ecosystem: composer }
           - { path: docker, name: docker, ecosystem: docker }
@@ -28,13 +34,15 @@ jobs:
           - { path: github_actions, name: actions, ecosystem: github-actions }
           - { path: go_modules, name: go, ecosystem: gomod }
           - { path: go_modules, name: go-close-pr, ecosystem: gomod }
+          - { path: go_modules, name: go-group-rules, ecosystem: gomod }
           - { path: go_modules, name: go-security, ecosystem: gomod }
           - { path: go_modules, name: go-update-pr, ecosystem: gomod }
           - { path: gradle, name: gradle, ecosystem: gradle }
           - { path: gradle, name: gradle-version-catalog, ecosystem: gradle }
           - { path: hex, name: hex, ecosystem: mix }
           - { path: maven, name: maven, ecosystem: maven }
           - { path: npm_and_yarn, name: npm, ecosystem: npm}
+          - { path: npm_and_yarn, name: npm-group-rules, ecosystem: npm}
           - { path: npm_and_yarn, name: npm-remove-transitive, ecosystem: npm}
           - { path: npm_and_yarn, name: pnpm, ecosystem: npm}
           - { path: npm_and_yarn, name: yarn, ecosystem: npm}
@@ -239,19 +247,20 @@ jobs:
         curl $(gh api $URL --jq .download_url) -o smoke.yaml
 
     # Download the Proxy cache. The job is ideally 100% cached so no real calls are made.
+    # Allowed to fail to get out of checking and egg situations, for example, when adding a new ecosystem.
     - name: Download cache
       if: steps.changes.outputs[matrix.suite.name] == 'true'
       run: |
-        mkdir cache
-        cd cache
-        gh run download --repo dependabot/smoke-tests --name cache-${{ matrix.suite.name }}
+        gh run download --repo dependabot/smoke-tests --name cache-${{ matrix.suite.name }} --dir cache
+      continue-on-error: true
 
     - name: Build ecosystem image
       if: steps.changes.outputs[matrix.suite.name] == 'true'
       run: script/build ${{ matrix.suite.path }}
 
     - name: ${{ matrix.suite.name }}
       if: steps.changes.outputs[matrix.suite.name] == 'true'
+      id: test
       env:
         LOCAL_GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
@@ -265,7 +274,7 @@ jobs:
           2>&1 | tee -a log.txt
 
     - name: Diff
-      if: always()
+      if: steps.test.outcome != 'skipped'
       continue-on-error: true
       run: diff --ignore-space-change smoke.yaml result.yaml && echo "Contents are identical"
 

Dockerfile.development

@@ -1,4 +1,4 @@
-ARG FROM_IMAGE=dependabot/dependabot-core
+ARG FROM_IMAGE=ghcr.io/dependabot/dependabot-updater-core
 FROM $FROM_IMAGE
 
 # Temporarily switch to root user in order to install packages
@@ -17,8 +17,6 @@ RUN apt-get update \
   && rm -rf /var/lib/apt/lists/*
 USER dependabot
 
-RUN git config --global user.name dependabot-ci \
-  && git config --global user.email no-reply@github.com
 RUN curl -L -o ~/.vimrc https://github.com/hmarr/dotfiles/raw/main/vimrc-vanilla.vim && \
   echo 'export PS1="[dependabot-core-dev] \w \[$(tput setaf 4)\]$ \[$(tput sgr 0)\]"' >> ~/.bashrc
 

Dockerfile.updater-core

@@ -46,6 +46,9 @@ USER dependabot
 ENV DEPENDABOT_HOME="/home/dependabot"
 WORKDIR $DEPENDABOT_HOME
 
+# For users to determine if dependabot is running
+ENV DEPENDABOT=true
+
 # Disable automatic pulling of files stored with Git LFS
 # This avoids downloading large files not necessary for the dependabot scripts
 ENV GIT_LFS_SKIP_SMUDGE=1
@@ -88,12 +91,12 @@ WORKDIR $DEPENDABOT_HOME/dependabot-updater
 
 # Install Ruby from official Docker image
 # When bumping Ruby minor, need to also add the previous version to `bundler/helpers/v{1,2}/monkey_patches/definition_ruby_version_patch.rb`
-COPY --from=ruby:3.1.4 --chown=dependabot:dependabot /usr/local /usr/local
+COPY --from=ruby:3.1.4-bullseye --chown=dependabot:dependabot /usr/local /usr/local
 
 # When bumping Bundler, need to also regenerate `updater/Gemfile.lock` via `bundle update --bundler`
 # Generally simplest to match the bundler version to the one that comes by default with whatever Ruby version we install.
 # This way other projects that import this library don't have to futz around with installing new / unexpected bundler versions.
-ARG BUNDLER_V2_VERSION=2.4.13
+ARG BUNDLER_V2_VERSION=2.4.14
 
 # We had to explicitly bump this as the bundled version `0.2.2` in ubuntu 20.04 has a bug.
 # Once Ubuntu base image pulls in a new enough yaml version, we may not need to

README.md

@@ -57,12 +57,18 @@ started.
 The [dependabot-script](https://github.com/dependabot/dependabot-script) repo provides a collection of example scripts for configuring the Dependabot-Core library.
 It is intended as a starting point for advanced users to run a self-hosted version of Dependabot within their own projects.
 
+>**Note:** We recently refactored the monolithic docker image used within the Dependabot Core library into one-image-per-ecosystem. Unfortunately, that broke dependabot-scritps, and we haven't had time to update them yet. We are aware of the problem and hope to provide a solution soon.
+
 ## Dependabot CLI
 
 The [Dependabot CLI](https://github.com/dependabot/cli) is a newer tool that may eventually replace [`dependabot-script`](#dependabot-script) for standalone use cases.
 While it creates dependency diffs, it's currently missing the logic to turn those diffs into actual PR's. Nevertheless, it
 may be useful for advanced users looking for examples of how to hack on Dependabot.
 
+## Dependabot on CI
+
+In an environment such as GitHub where Dependabot is running in a container, if you want to change your build or installation process depending on whether Dependabot is checking, you can determine it by the existence of `DEPENDABOT` environment variable.
+
 # Contributing to Dependabot
 
 ## Reporting issues and Feature Requests

bin/docker-dev-shell

@@ -235,6 +235,7 @@ docker run --rm -ti \
   -v "$(pwd)/terraform/spec:$CODE_DIR/terraform/spec" \
   -v "$(pwd)/tmp:/$CODE_DIR/tmp" \
   -v "$(pwd)/updater/.rubocop.yml:$CODE_DIR/updater/.rubocop.yml" \
+  -v "$(pwd)/updater/bin:$CODE_DIR/updater/bin" \
   -v "$(pwd)/updater/Gemfile.lock:$CODE_DIR/updater/Gemfile.lock" \
   -v "$(pwd)/updater/Gemfile:$CODE_DIR/updater/Gemfile" \
   -v "$(pwd)/updater/lib:$CODE_DIR/updater/lib" \

bin/dry-run.rb

@@ -799,8 +799,8 @@ def security_fix?(dependency)
 StackProf.results("tmp/stackprof-#{Time.now.strftime('%Y-%m-%d-%H:%M')}.dump") if $options[:profile]
 
 puts "🌍 Total requests made: '#{$network_trace_count}'"
-package_manager = fetcher.package_manager_version
-puts "🎈 Package manager version log: #{package_manager}" unless package_manager.nil?
+ecosystem_versions = fetcher.ecosystem_versions
+puts "🎈 Ecosystem Versions log: #{ecosystem_versions}" unless ecosystem_versions.nil?
 
 # rubocop:enable Metrics/BlockLength
 

bundler/README.md

@@ -4,21 +4,15 @@ Ruby (bundler) support for [`dependabot-core`][core-repo].
 
 ### Running locally
 
-1. Install native helpers
-    ```
-    $ export DEPENDABOT_NATIVE_HELPERS_PATH=$PWD/helpers/install-dir
-    $ helpers/v1/build
-    $ helpers/v2/build
-    ```
+1. Start a development shell
 
-2. Install Ruby dependencies
-   ```
-   $ bundle install
-   ```
+  ```
+  $ bin/docker-dev-shell bundler
+  ```
 
 2. Run tests
    ```
-   $ bundle exec rspec spec
+   [dependabot-core-dev] ~/dependabot-core $ cd bundler && rspec
    ```
 
 [core-repo]: https://github.com/dependabot/dependabot-core

bundler/helpers/v1/lib/functions/lockfile_updater.rb

@@ -64,7 +64,7 @@ def cache_vendored_gems(definition)
       # Dependencies that have been unlocked for the update (including
       # sub-dependencies)
       unlocked_gems = definition.instance_variable_get(:@unlock).
-                      fetch(:gems).reject { |gem| __keep_on_prune?(gem) }
+                      fetch(:gems)
       bundler_opts = {
         cache_all: true,
         cache_all_platforms: true,
@@ -84,15 +84,6 @@ def cache_vendored_gems(definition)
       end
     end
 
-    # This is not officially supported and may be removed without notice.
-    def __keep_on_prune?(spec_name)
-      unless (specs = Bundler.settings[:persistent_gems_after_clean])
-        return false
-      end
-
-      specs.include?(spec_name)
-    end
-
     # Copied from Bundler::Runtime: Modified to only prune gems that have
     # been unlocked
     def prune_gem_cache(resolve, cache_path, unlocked_gems)