Separate out python dependencies for running scripts (#28582)

* Separate oput requirements for various python scripts

* Update certbins to install requirements for test script running instead of hard-coding dependencies

* Update text for help in build_python

* Fix the requirements install - need to copy the dependencies locally

* Update the sha to something that supports the latest options. This is the TE2 for 1.2 SHA

* A few more dependencies for py_matter_yamltests ... dependencies there seem off

* Revert submodule updates

* Update dependencies for chip repl to allow py_matter_idl

* Fix dependencies in yaml: need dependencies at runtime not at build time

---------

Co-authored-by: Andrei Litvin <andreilitvin@google.com>

.github/workflows/tests.yaml

@@ -427,7 +427,7 @@ jobs:
 
             - name: Build Python REPL and example apps
               run: |
-                  scripts/run_in_build_env.sh './scripts/build_python.sh --install_virtual_env out/venv --extra_packages "mobly"'
+                  scripts/run_in_build_env.sh './scripts/build_python.sh --install_virtual_env out/venv'
                   ./scripts/run_in_build_env.sh \
                    "./scripts/build/build_examples.py \
                       --target linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test \

integrations/docker/images/chip-cert-bins/Dockerfile

@@ -3,7 +3,7 @@ FROM ubuntu:22.04 as chip-build-cert
 LABEL org.opencontainers.image.source https://github.com/project-chip/connectedhomeip
 ARG TARGETPLATFORM
 # COMMITHASH defines the target commit to build from. May be passed in using --build-arg.
-ARG COMMITHASH=7b99e6399c6069037c613782d78132c69b9dcabb
+ARG COMMITHASH=c1ec2d777456924dcaa59b53351b00d73caf378f
 
 # Ensure TARGETPLATFORM is set
 RUN case ${TARGETPLATFORM} in \
@@ -268,5 +268,11 @@ COPY --from=chip-build-cert-bins /root/connectedhomeip/out/chip-app1 chip-app1
 # Stage 3.1 Setup the Matter Python environment
 COPY --from=chip-build-cert-bins /root/connectedhomeip/out/python_lib python_lib
 COPY --from=chip-build-cert-bins /root/connectedhomeip/src/python_testing python_testing
-RUN pip install click websockets lark diskcache
+
+COPY --from=chip-build-cert-bins /root/connectedhomeip/scripts/tests/requirements.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt && rm /tmp/requirements.txt
+
+COPY --from=chip-build-cert-bins /root/connectedhomeip/src/python_testing/requirements.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt && rm /tmp/requirements.txt
+
 RUN pip install --no-cache-dir python_lib/controller/python/chip*.whl

scripts/build_python.sh

@@ -64,7 +64,8 @@ Input Options:
                                                             <path> represents where the virtual environment is to be created.
   -c, --clean_virtual_env  <yes|no>                         When installing a virtual environment, create/clean it first.
                                                             Defaults to yes.
-  --include_pytest_deps  <yes|no>                           Install requirements.python_tests.txt.
+  --include_pytest_deps  <yes|no>                           Install requirements.txt for running scripts/tests and
+                                                            src/python_testing scripts.
                                                             Defaults to yes.
   --extra_packages PACKAGES                                 Install extra Python packages from PyPI
   --include_yamltests                                       Whether to install the matter_yamltests wheel.
@@ -214,7 +215,8 @@ if [ -n "$install_virtual_env" ]; then
 
     if [ "$install_pytest_requirements" = "yes" ]; then
         echo_blue "Installing python test dependencies ..."
-        "$ENVIRONMENT_ROOT"/bin/pip install -r "$CHIP_ROOT/scripts/setup/requirements.python_tests.txt"
+        "$ENVIRONMENT_ROOT"/bin/pip install -r "$CHIP_ROOT/scripts/tests/requirements.txt"
+        "$ENVIRONMENT_ROOT"/bin/pip install -r "$CHIP_ROOT/src/python_testing/requirements.txt"
     fi
 
     echo ""

scripts/py_matter_idl/setup.cfg

@@ -21,6 +21,10 @@ description = Parse matter idl files
 [options]
 packages = find:
 zip_safe = False
+install_requires=
+   lark
+   jinja2
+   stringcase
 
 [options.package_data]
 matter_idl =

scripts/py_matter_yamltests/setup.cfg

@@ -20,5 +20,7 @@ description = Parse matter yaml tests files
 
 [options]
 packages = find:
-
 zip_safe = False
+install_requires=
+   websockets
+   lark

scripts/tests/requirements.txt

@@ -0,0 +1,5 @@
+# Python requirements for scripts in this location
+click
+colorama
+diskcache
+websockets

src/python_testing/requirements.txt

@@ -0,0 +1,3 @@
+mobly
+pyasn1
+pyasn1_modules