tools: Add CSV support to idf_size.py

This adds CSV support to idf_size.py and idf.py size actions and using the --format argument which accepts 'text', 'json' or 'csv' as input.

idf_size.py --json argument is deprecated but left to avoid a breaking change.

For idf.py size actions OUTPUT_JSON environment variable set at configuration time is overriden at target build time if --format is used.

Additionally, this commit refactors big parts of code, unified usage of json_dict and manually generated dictionaries for textual output and improves code quality in many parts.

docs/en/api-guides/performance/size.rst

@@ -16,6 +16,9 @@ To optimize both firmware binary size and memory usage it's necessary to measure
 
 Using the :ref:`idf.py` sub-commands ``size``, ``size-components`` and ``size-files`` provides a summary of memory used by the project:
 
+.. note::
+    It is possible to add ``-DOUTPUT_FORMAT=csv`` or ``-DOUTPUT_FORMAT=json`` to get the output in CSV or JSON format.
+
 Size Summary (idf.py size)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -188,9 +191,9 @@ Comparing Two Binaries
 
 If making some changes that affect binary size, it's possible to use an ESP-IDF tool to break down the exact differences in size.
 
-This operation isn't part of ``idf.py``, it's necessary to run the ``idf-size.py`` Python tool directly.
+This operation isn't part of ``idf.py``, it's necessary to run the ``idf_size.py`` Python tool directly.
 
-To do so, first locate the linker map file in the build directory. It will have the name ``PROJECTNAME.map``. The ``idf-size.py`` tool performs its analysis based on the output of the linker map file.
+To do so, first locate the linker map file in the build directory. It will have the name ``PROJECTNAME.map``. The ``idf_size.py`` tool performs its analysis based on the output of the linker map file.
 
 To compare with another binary, you will also need its corresponding ``.map`` file saved from the build directory.
 
@@ -215,6 +218,9 @@ We can see from the "Difference" column that changing this one setting caused th
 
 It's also possible to use the "diff" mode to output a table of component-level (static library archive) differences:
 
+.. note::
+    To get the output in JSON or CSV format using ``idf_size.py`` it is possible to use the ``--format`` option.
+
 .. code-block:: bash
 
     $IDF_PATH/tools/idf_size.py --archives --diff build_Og/https_request.map build_Oshttps_request.map

docs/en/api-guides/tools/idf-py.rst

@@ -201,7 +201,7 @@ Similarly, this will print the same information for each component used in the p
 
 Will print size information per source file in the project.
 
-If you define variable ``-DOUTPUT_JSON=1`` when running CMake (or ``idf.py``), the output will be formatted as JSON not as human readable text. See ``idf.py-size`` for more information.
+If you define the ``OUTPUT_FORMAT`` variable as ``csv`` or ``json`` when running CMake (or ``idf.py``), the output will be formatted in the specified format and not as human readable text. See ``idf.py-size`` for more information.
 
 Reconfigure the project: reconfigure
 ------------------------------------

tools/cmake/project.cmake

@@ -531,23 +531,44 @@ macro(project project_name)
     idf_build_get_property(python PYTHON)
 
     set(idf_size ${python} ${idf_path}/tools/idf_size.py)
-    if(DEFINED OUTPUT_JSON AND OUTPUT_JSON)
-        list(APPEND idf_size "--json")
-    endif()
 
     # Add size targets, depend on map file, run idf_size.py
+    # OUTPUT_JSON is passed for compatibility reasons, SIZE_OUTPUT_FORMAT
+    # environment variable is recommended and has higher priority
     add_custom_target(size
+        COMMAND ${CMAKE_COMMAND}
+        -D "IDF_SIZE_TOOL=${idf_size}"
+        -D "MAP_FILE=${mapfile}"
+        -D "OUTPUT_JSON=${OUTPUT_JSON}"
+        -P "${idf_path}/tools/cmake/run_size_tool.cmake"
         DEPENDS ${mapfile}
-        COMMAND ${idf_size} ${mapfile}
-        )
+        USES_TERMINAL
+        VERBATIM
+    )
+
     add_custom_target(size-files
+        COMMAND ${CMAKE_COMMAND}
+        -D "IDF_SIZE_TOOL=${idf_size}"
+        -D "IDF_SIZE_MODE=--files"
+        -D "MAP_FILE=${mapfile}"
+        -D "OUTPUT_JSON=${OUTPUT_JSON}"
+        -P "${idf_path}/tools/cmake/run_size_tool.cmake"
         DEPENDS ${mapfile}
-        COMMAND ${idf_size} --files ${mapfile}
-        )
+        USES_TERMINAL
+        VERBATIM
+    )
+
     add_custom_target(size-components
+        COMMAND ${CMAKE_COMMAND}
+        -D "IDF_SIZE_TOOL=${idf_size}"
+        -D "IDF_SIZE_MODE=--archives"
+        -D "MAP_FILE=${mapfile}"
+        -D "OUTPUT_JSON=${OUTPUT_JSON}"
+        -P "${idf_path}/tools/cmake/run_size_tool.cmake"
         DEPENDS ${mapfile}
-        COMMAND ${idf_size} --archives ${mapfile}
-        )
+        USES_TERMINAL
+        VERBATIM
+    )
 
     unset(idf_size)
 

tools/cmake/run_size_tool.cmake

@@ -0,0 +1,33 @@
+# A CMake script to run size tool commands supporting OUTPUT_FORMAT and
+# OUTPUT_JSON environment variables from within ninja or make or another
+# cmake-based build runner.
+#
+# It is recommended to NOT USE this CMake script if you have the option of
+# running the tool directly. This script exists only for use inside CMake builds.
+cmake_minimum_required(VERSION 3.16)
+
+# Main purpose of this script: we can't expand these environment variables in the main IDF CMake build,
+# because we want to expand them at CMake target build time not at CMake configuration time
+# (so they can change without needing a CMake re-run)
+
+set(IDF_SIZE_CMD ${IDF_SIZE_TOOL})
+
+if(DEFINED ENV{SIZE_OUTPUT_FORMAT})
+    list(APPEND IDF_SIZE_CMD "--format=$ENV{SIZE_OUTPUT_FORMAT}")
+elseif(DEFINED OUTPUT_JSON AND OUTPUT_JSON)
+    list(APPEND IDF_SIZE_CMD "--format=json")
+endif()
+
+if(DEFINED IDF_SIZE_MODE)
+    list(APPEND IDF_SIZE_CMD ${IDF_SIZE_MODE})
+endif()
+
+list(APPEND IDF_SIZE_CMD ${MAP_FILE})
+
+execute_process(COMMAND ${IDF_SIZE_CMD}
+    RESULT_VARIABLE result
+    )
+
+if(${result})
+    message(FATAL_ERROR "${IDF_SIZE_TOOL} failed")
+endif()

tools/idf_py_actions/core_ext.py

@@ -33,17 +33,18 @@ def build_target(target_name: str, ctx: Context, args: PropertyDict) -> None:
         ensure_build_directory(args, ctx.info_name)
         run_target(target_name, args, force_progression=GENERATORS[args.generator].get('force_progression', False))
 
-    def size_target(target_name: str, ctx: Context, args: PropertyDict) -> None:
+    def size_target(target_name: str, ctx: Context, args: PropertyDict, output_format: str) -> None:
         """
         Builds the app and then executes a size-related target passed in 'target_name'.
         `tool_error_handler` handler is used to suppress errors during the build,
         so size action can run even in case of overflow.
-
         """
 
         def tool_error_handler(e: int, stdout: str, stderr: str) -> None:
             print_hints(stdout, stderr)
 
+        if output_format:
+            os.environ['SIZE_OUTPUT_FORMAT'] = output_format
         ensure_build_directory(args, ctx.info_name)
         run_target('all', args, force_progression=GENERATORS[args.generator].get('force_progression', False),
                    custom_error_handler=tool_error_handler)
@@ -336,6 +337,13 @@ def help_and_exit(action: str, ctx: Context, param: List, json_option: bool, add
         'global_action_callbacks': [validate_root_options],
     }
 
+    # Default value is intentionally blank, so that we know if the user explicitly specified
+    # the format and override the OUTPUT_JSON variable if it is set
+    size_options = [{'names': ['--format', 'output_format'],
+                     'type': click.Choice(['text', 'csv', 'json']),
+                     'help': 'Specify output format: text, csv or json.',
+                     'default': ''}]
+
     build_actions = {
         'actions': {
             'all': {
@@ -388,17 +396,17 @@ def help_and_exit(action: str, ctx: Context, param: List, json_option: bool, add
             'size': {
                 'callback': size_target,
                 'help': 'Print basic size information about the app.',
-                'options': global_options,
+                'options': global_options + size_options,
             },
             'size-components': {
                 'callback': size_target,
                 'help': 'Print per-component size information.',
-                'options': global_options,
+                'options': global_options + size_options,
             },
             'size-files': {
                 'callback': size_target,
                 'help': 'Print per-source-file size information.',
-                'options': global_options,
+                'options': global_options + size_options,
             },
             'bootloader': {
                 'callback': build_target,