sagemathgh-37988: CI Build&Test: Replace use of sage -t --new

    
<!-- ^ Please provide a concise and informative title. -->
<!-- ^ Don't put issue numbers in the title, do this in the PR
description below. -->
<!-- ^ For example, instead of "Fixes sagemath#12345" use "Introduce new method
to calculate 1 + 2". -->
<!-- v Describe your changes below in detail. -->
<!-- v Why is this change required? What problem does it solve? -->
<!-- v If this PR resolves an open issue, please link to it here. For
example, "Fixes sagemath#12345". -->

Fixes sagemath#37985.

We also turn on  `--long --format github` for this run, so that
annotations are shown in the "Files changed" tab.
(Previously, only the annotations from the full testsuite run were
shown, out of the concern of duplicating the numerous doctest style
warnings. But these warnings have been much reduced, and the last two
warnings are fixed in sagemath#38029.)

Example run: https://github.com/sagemath/sage/actions/runs/9198070834/jo
b/25299950120?pr=37988

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->

- [x] The title is concise and informative.
- [ ] The description explains in detail what this PR is about.
- [x] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation and checked the documentation
preview.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on. For example,
-->
<!-- - sagemath#12345: short description why this is a dependency -->
<!-- - sagemath#34567: ... -->

- Depends on sagemath#37926 (merged here)
    
URL: sagemath#37988
Reported by: Matthias Köppe
Reviewer(s): Kwankyu Lee, Matthias Köppe

.ci/retrofit-worktree.sh

@@ -32,18 +32,14 @@ git tag -f new
 # But $WORKTREE_DIRECTORY is not a git repository.
 # We make $WORKTREE_DIRECTORY a worktree whose index is at tag "new".
 # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
-# Then we update worktree and index with "git reset --hard new".
+# Then we update worktree and index with "git checkout new".
 # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
-# Finally we reset the index to "old". (This keeps all mtimes unchanged.)
-# The changed files now show up as uncommitted changes.
-# The final "git add -N" makes sure that files that were added in "new" do not show
-# as untracked files, which would be removed by "git clean -fx".
 if [ -L $WORKTREE_NAME ]; then
     rm -f $WORKTREE_NAME
-    git worktree prune --verbose
 fi
+git worktree prune --verbose
 git worktree add --detach $WORKTREE_NAME
 rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
 rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME
 if [ ! -f $WORKTREE_NAME/.gitignore ]; then cp .gitignore $WORKTREE_NAME/; fi
-(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
+(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git checkout new && git status)

.ci/write-dockerfile.sh

@@ -272,13 +272,14 @@ $ADD src/Pipfile.m4 src/pyproject.toml src/requirements.txt.m4 src/setup.cfg.m4
 $ADD m4 /new/m4
 $ADD pkgs /new/pkgs
 $ADD build /new/build
-$ADD .ci /new/.ci
 $ADD .upstream.d /new/.upstream.d
+ADD .ci /.ci
 RUN if [ -d /sage ]; then                                               \
         echo "### Incremental build from \$(cat /sage/VERSION.txt)" &&  \
         printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /sage/.gitignore && \
         printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /new/.gitignore && \
-        if ! (cd /new && ./.ci/retrofit-worktree.sh worktree-image /sage); then \
+        if ! (cd /new && /.ci/retrofit-worktree.sh worktree-image /sage); then \
+            echo "retrofit-worktree.sh failed, falling back to replacing /sage"; \
             for a in local logs; do                                     \
                 if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi;       \
             done;                                                       \
@@ -337,8 +338,12 @@ ENV SAGE_CHECK=warn
 ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst"
 $ADD .gitignore /new/.gitignore
 $ADD src /new/src
+ADD .ci /.ci
 RUN cd /new && rm -rf .git && \
-    if ! /sage/.ci/retrofit-worktree.sh worktree-pre /sage; then \
+    if /.ci/retrofit-worktree.sh worktree-pre /sage; then \
+        cd /sage && touch configure build/make/Makefile; \
+    else \
+        echo "retrofit-worktree.sh failed, falling back to replacing /sage/src"; \
         rm -rf /sage/src;                                    \
         mv src /sage/src;                                    \
         cd /sage && ./bootstrap && ./config.status;          \

.github/workflows/build.yml

@@ -175,11 +175,20 @@ jobs:
           ./sage -python -m pytest -c tox.ini -qq --doctest --collect-only || true
         shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v44
+        with:
+          # File extensions per sage.doctest.control.skipfile
+          files: src/**/*.{py,pyx,pxd,pxi,sage,spyx,rst,tex}
+          files_ignore: src/{setup,conftest*}.py
+
       - name: Test changed files (sage -t --new)
+        if: steps.changed-files.outputs.all_changed_files
         run: |
           export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=4
-          # We run tests with "sage -t --new"; this only tests the uncommitted changes.
-          ./sage -t --new -p4
+          # https://github.com/tj-actions/changed-files?tab=readme-ov-file#outputs-
+          ./sage -t --long --format github -p4 ${{ steps.changed-files.outputs.all_changed_files }}
         shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
   test-mod: