build helper, take 2

.azure-pipelines/azure-pipelines-linux.yml

@@ -17,6 +17,7 @@ jobs:
         BUILD_TYPE: 'release'
         APT_INSTALL: 'gfortran gcc-7 g++-7'
         vmImage: 'ubuntu-20.04'
+        PYTEST_MARKER_EXPR: "quick"
 
       gcc_latest:
         CXX_COMPILER: 'g++'
@@ -25,7 +26,8 @@ jobs:
         F_COMPILER: 'gfortran'
         BUILD_TYPE: 'release'
         APT_INSTALL: 'gfortran'
-        vmImage: 'ubuntu-20.04'
+        vmImage: 'ubuntu-latest'
+        PYTEST_MARKER_EXPR: 'api and not stdsuite and not (smoke or quick or medlong or long)'
 
       clang_latest:
         CXX_COMPILER: 'clang++'
@@ -35,6 +37,7 @@ jobs:
         BUILD_TYPE: 'release'
         APT_INSTALL: 'gfortran clang-6.0'
         vmImage: 'ubuntu-20.04'
+        PYTEST_MARKER_EXPR: 'cli and not (smoke or quick or medlong or long)'
 
   pool:
     vmImage: $(vmImage)
@@ -79,61 +82,54 @@ jobs:
   - bash: |
         echo "##vso[task.prependpath]$CONDA/bin"
         conda config --set always_yes yes
+        conda config --set solver libmamba
     displayName: "Add Conda to PATH"
 
   - bash: |
-      conda create -q \
-        -n p4env \
-        python=$PYTHON_VER \
-        psi4/label/dev::gau2grid \
-        psi4/label/testing::libint2=2.7.2=h2fe1556_0 \
-        psi4/label/dev::libxc=5 \
-        psi4/label/dev::dftd3 \
-        psi4/label/dev::gcp \
-        psi4/label/dev::pybind11-headers \
-        blas=*=mkl \
-        mkl-include \
-        networkx \
-        conda-forge::pytest>=7.0.1 \
-        eigen \
-        libboost-headers \
-        conda-forge::pytest-xdist \
-        msgpack-python \
-        conda-forge::qcelemental \
-        conda-forge::qcengine \
-        conda-forge::optking \
-        scipy
+      conda install pyyaml -c conda-forge
+      conda/psi4-path-advisor.py \
+        env \
+        --name p4env \
+        --python $PYTHON_VER \
+        --disable compilers addons docs
+      echo "\n<<<  env_p4env.yaml  >>>\n"
+      cat env_p4env.yaml
+      # test some alternative packages
+      if [[ "${PYTHON_VER}" != "3.12" ]]; then
+        echo "  - psi4::gcp" >> env_p4env.yaml
+        echo "  - psi4::dftd3" >> env_p4env.yaml
+      fi
+      sed -i "s;- libxc-c;- psi4::libxc=5;g" env_p4env.yaml
+      sed -i "s;- conda-forge/label/libint_dev;#- conda-forge/label/libint_dev;g" env_p4env.yaml
+      sed -i "s;#- psi4/label/testing::libint2;- psi4/label/testing::libint2;g" env_p4env.yaml
+      echo "\n<<<  env_p4env.yaml  >>>\n"
+      cat env_p4env.yaml
+      conda env create -n p4env -f env_p4env.yaml
       source activate p4env
-      which python
+      conda info
       conda list
     displayName: 'Build Environment'
 
   - bash: |
       source activate p4env
       python -V
       python -c 'import numpy; print(numpy.version.version)'
-      export CXX=${CXX_COMPILER}
-      export CC=${C_COMPILER}
-      export FC=${F_COMPILER}
-      # * can't use conda dist of the more complicated lib pkgs (e.g., CheMPS2, PCMSolver, v2rdm)
-      #   b/c their c++ symbols don't mix with the different Travis compilers. for other
-      #   reasons, pkgs are being compiled less statically, sad for CI.
-      # * can't enable trivial plugins b/c no psi4 for them to detect at start (e.g., snsmp2)
-      cmake -Bbuild -S. \
-        -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \
-        -DCMAKE_C_COMPILER=${C_COMPILER} \
-        -DCMAKE_Fortran_COMPILER=${F_COMPILER} \
-        -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-        -DMAX_AM_ERI=5 \
-        -DCMAKE_PREFIX_PATH=${CONDA}/envs/p4env \
-        -DPython_EXECUTABLE="${CONDA}/envs/p4env/bin/python" \
-        -DMPFR_ROOT=${CONDA}/envs/p4env \
-        -DCMAKE_INSIST_FIND_PACKAGE_gau2grid=ON \
-        -DCMAKE_INSIST_FIND_PACKAGE_Libint2=ON \
-        -DCMAKE_INSIST_FIND_PACKAGE_Libxc=ON \
-        -DENABLE_PLUGIN_TESTING=ON \
-        -DBUILD_Libint2_GENERATOR=OFF \
-        -DCMAKE_INSTALL_PREFIX=$(Agent.BuildDirectory)/Install
+      conda/psi4-path-advisor.py \
+        cmake \
+        --objdir build \
+        --insist
+      echo "\n<<<  cache_p4env@build.cmake  >>>\n"
+      cat cache_p4env@build.cmake
+      cmake \
+        -S. \
+        -B build \
+        -C $(Build.SourcesDirectory)/cache_p4env@build.cmake \
+        -D CMAKE_CXX_COMPILER=${CXX_COMPILER} \
+        -D CMAKE_C_COMPILER=${C_COMPILER} \
+        -D CMAKE_Fortran_COMPILER=${F_COMPILER} \
+        -D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
+        -D ENABLE_PLUGIN_TESTING=ON \
+        -D CMAKE_INSTALL_PREFIX=$(Agent.BuildDirectory)/Install
     displayName: 'Configure Build'
 
   - bash: |
@@ -160,5 +156,5 @@ jobs:
   - bash: |
       cd build
       source activate p4env
-      PYTHONPATH=stage/lib/ pytest -v -rws --durations=15 -n 2 -m 'quick' stage/lib/psi4/tests/
+      PYTHONPATH=stage/lib/ pytest -v -rws --durations=15 --durations-min=40 --strict-markers --color yes -n 2 -m "$PYTEST_MARKER_EXPR" stage/lib/psi4/tests/
     displayName: 'PyTest Tests'

.azure-pipelines/azure-pipelines-windows.yml

@@ -107,11 +107,13 @@ jobs:
 
       # Configure Miniconda
       #    conda config --append channels conda-forge
+      #    conda=23.10 forced a bugfix at conda-build stage
       - script: |
           conda config --set always_yes yes
           conda config --add channels conda-forge/label/libint_dev
           conda config --set channel_priority strict
           conda config --set solver libmamba
+          conda install conda=23.10 -n base -c conda-forge
           conda info
           conda list
         displayName: "Configure Miniconda"
@@ -123,6 +125,7 @@ jobs:
           conda install cmake ^
                         anaconda-project ^
                         conda-build ^
+                        m2-sed ^
                         eigen ^
                         libboost-headers ^
                         conda-verify ^

.github/workflows/docs-pr.yml

@@ -7,8 +7,13 @@ on:
 jobs:
 
   docs:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        cfg:
+          - runs-on: ubuntu-latest
+            python-version: "3.10"
     name: Propose Docs
+    runs-on: ${{ matrix.cfg.runs-on }}
     defaults:
       run:
         shell: bash -l {0}
@@ -31,17 +36,37 @@ jobs:
         path: docs
         ref: master
 
+    - name: Write Conda environment files
+      run: |
+        echo "::group::Prepare OS Python"
+        which python
+        python --version
+        pip install PyYAML
+        echo "::endgroup::"
+        #
+        code/conda/psi4-path-advisor.py \
+          env \
+          --name p4docs \
+          --python ${{ matrix.cfg.python-version }} \
+          --disable addons \
+          --offline-conda
+        #
+        echo "::group::View Env Spec File for Conda"
+        printf "\n<<<  env_p4docs.yaml  >>>\n"
+        pygmentize -g -O style=friendly,linenos=1 env_p4docs.yaml
+        echo "::endgroup::"
+
     - name: Create Conda Environment
       if: ${{ github.repository == 'psi4/psi4' }}
       uses: conda-incubator/setup-miniconda@v2
       with:
         miniforge-variant: Mambaforge
         use-mamba: true
-        activate-environment: test
+        activate-environment: p4docs
         add-pip-as-python-dependency: true
         auto-activate-base: false
         channels: conda-forge
-        environment-file: code/devtools/conda-envs/docs-cf.yaml
+        environment-file: env_p4docs.yaml
         show-channel-urls: true
 
     - name: Environment Information
@@ -51,47 +76,39 @@ jobs:
         conda list
         which conda python cmake $CXX
 
-    # This patch and the jinja2<3 restriction were needed as recently as March 2023, but when I try a py311 env with newest everything, both can be released.
-    #- name: Patch Sphinx for Pybind11
-    #  if: ${{ github.repository == 'psi4/psi4' }}
-    #  run: sed -i "s/app\.add_autodoc_attrgetter/# app.add_autodoc_attrgetter/g" ${CONDA_PREFIX}/lib/python3.10/site-packages/sphinx_automodapi/autodoc_enhancements.py
-
-    # docs are not Ninja ready
+    # docs are finally Ninja ready
     - name: Configure with CMake (Conda Gnu + MKL)
       if: ${{ github.repository == 'psi4/psi4' }}
       working-directory: ./code
       run: |
+        conda/psi4-path-advisor.py \
+          cache \
+          --objdir objdir \
+          --insist
+        echo "::group::View Cache File for CMake"
+        printf "\n<<<  cache_p4docs@objdir.cmake  >>>\n"
+        pygmentize -g -O style=colorful,linenos=1 cache_p4docs@objdir.cmake
+        echo "::endgroup::"
+        #
         cmake \
           -S. \
-          -Bobjdir \
-          -G"Unix Makefiles" \
-          -DCMAKE_BUILD_TYPE=Debug \
-          -DCMAKE_C_COMPILER=${CC} \
-          -DCMAKE_CXX_COMPILER=${CXX} \
-          -DCMAKE_CXX_FLAGS="-O0" \
-          -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \
-          -DCMAKE_INSIST_FIND_PACKAGE_gau2grid=ON \
-          -DCMAKE_INSIST_FIND_PACKAGE_Libint2=ON \
-          -DCMAKE_INSIST_FIND_PACKAGE_pybind11=ON \
-          -DCMAKE_INSIST_FIND_PACKAGE_Libxc=ON \
-          -DENABLE_ecpint=ON \
-          -DCMAKE_INSIST_FIND_PACKAGE_ecpint=ON \
-          -DCMAKE_INSIST_FIND_PACKAGE_optking=ON \
-          -DCMAKE_INSIST_FIND_PACKAGE_qcelemental=ON \
-          -DCMAKE_INSIST_FIND_PACKAGE_qcengine=ON \
-          -DSPHINXMAN_STRICT=ON \
-          -DSPHINX_ROOT=${CONDA_PREFIX}
+          -B objdir \
+          -G Ninja \
+          -C ${{ github.workspace }}/code/cache_p4docs@objdir.cmake \
+          -D CMAKE_BUILD_TYPE=Debug \
+          -D CMAKE_CXX_FLAGS="-O0" \
+          -D SPHINXMAN_STRICT=ON
 
     - name: Compile Psi4
       if: ${{ github.repository == 'psi4/psi4' }}
       working-directory: ./code
-      run: cmake --build objdir -j`getconf _NPROCESSORS_ONLN`
+      run: cmake --build objdir
 
     - name: Compile & Pack Docs
       if: ${{ github.repository == 'psi4/psi4' }}
       working-directory: ./code
       run: |
-        cmake --build objdir --target sphinxman -j`getconf _NPROCESSORS_ONLN`
+        cmake --build objdir --target sphinxman
         cd objdir/doc/sphinxman
         tar -zcf sphinxman.tar.gz html/