Linking multiple components in custom repo.

[horsemann07]

Hello developers,
For my project, I build the repository based on this documentation where I m using the amazon-freertos as an external library.
Specification.
Amazon-freertos version 202006 and esp-idf v4.2.2.
Repository looks like

- freertos
- freertos-configs
  - aws_clientcredential.h
  - aws_clientcredential_keys.h
  - iot_mqtt_agent_config.h
  - iot_config.h
- components
- src
-        include
- CMakeLists.txt

• Top-level CmakeLists.txt

cmake_minimum_required(VERSION 3.13)

set(esp_idf_dir "${CMAKE_CURRENT_LIST_DIR}/freertos/vendors/espressif/esp-idf")

include(${esp_idf_dir}/tools/cmake/idf.cmake)

string(FIND "${CMAKE_TOOLCHAIN_FILE}" "esp32s2" SOC_TOOLCHAIN_ESP32S2)
string(FIND "${CMAKE_TOOLCHAIN_FILE}" "esp32" SOC_TOOLCHAIN_ESP32)
if (NOT(${SOC_TOOLCHAIN_ESP32S2} EQUAL -1))
    set(SOC_NAME "esp32s2")
elseif(NOT($SOC_TOOLCHAIN_ESP32) EQUAL -1)
    set(SOC_NAME "esp32")
endif()

project(freertos_examples)

# Tell IDF build to link against this target.
set(IDF_PROJECT_EXECUTABLE afr_demo)
get_filename_component(
    IDF_EXECUTABLE_SRCS
    "src/main.c" ABSOLUTE
    )

include_directories(afr_demo PRIVATE src/include)

# Add some extra components. IDF_EXTRA_COMPONENT_DIRS is an variable used by ESP-IDF
# to collect extra components.
get_filename_component(
    EXTRA_COMPONENT_DIRS
    "components" ABSOLUTE
)
idf_build_component(${EXTRA_COMPONENT_DIRS})

# As of now there's no official way to redefine config files outside of FreeRTOS source tree.
# This is a temporary approach to inject an include path so that this takes precedence over the
# config file directory inside FreeRTOS.
include_directories(BEFORE freertos-configs)

# Add freertos as an subdirectory. AFR_BOARD tells which board to target.
if ("${SOC_NAME}" STREQUAL "esp32s2")
    set(AFR_BOARD espressif.esp32s2_saola_1 CACHE INTERNAL "")
elseif("${SOC_NAME}" STREQUAL "esp32")
    set(AFR_BOARD espressif.esp32_devkitc CACHE INTERNAL "")
endif()
add_subdirectory(freertos)

# Link against the mqtt demo so that we can use it. Dependencies of this demo are transitively
# linked.
target_link_libraries(
    afr_demo
    PRIVATE
    AFR::demo_core_mqtt
    AFR::common_io
    AFR::demo_device_shadow
    AFR::demo_core_mqtt_agent
}

Here, when I building the demo, it throws the error.

horsemann/Desktop/WorkSpace/esp_toolchain/xtensa-esp32-elf/bin/../lib/gcc/xtensa-esp32-elf/8.4.0/../../../../xtensa-esp32-elf/bin/ld: freertos/CMakeFiles/afr_demo.dir/__/src/main.c.obj:(.literal.app_main+0xc): undefined reference to `runDemoTaskCustom'
/home/horsemann/Desktop/WorkSpace/esp_toolchain/xtensa-esp32-elf/bin/../lib/gcc/xtensa-esp32-elf/8.4.0/../../../../xtensa-esp32-elf/bin/ld: freertos/CMakeFiles/afr_demo.dir/__/src/main.c.obj: in function `app_main':
/home/horsemann/Desktop/WorkSpace/aws_latest_repository/amazon-freertos-examples/build/../src/main.c:158: undefined reference to `runDemoTaskCustom'
collect2: error: ld returned 1 exit status
ninja: build stopped: subcommand failed.

Now the runDemoTaskCustom function in src/ directory and the header file is in src/include directory but still shows the undefined reference. Also, tried to use extern but throws the same error.

So, I made some small changes in the top-level CMake

cmake_minimum_required(VERSION 3.13)

set(esp_idf_dir "${CMAKE_CURRENT_LIST_DIR}/freertos/vendors/espressif/esp-idf")

include(${esp_idf_dir}/tools/cmake/idf.cmake)

string(FIND "${CMAKE_TOOLCHAIN_FILE}" "esp32s2" SOC_TOOLCHAIN_ESP32S2)
string(FIND "${CMAKE_TOOLCHAIN_FILE}" "esp32" SOC_TOOLCHAIN_ESP32)
if (NOT(${SOC_TOOLCHAIN_ESP32S2} EQUAL -1))
    set(SOC_NAME "esp32s2")
elseif(NOT($SOC_TOOLCHAIN_ESP32) EQUAL -1)
    set(SOC_NAME "esp32")
endif()

project(freertos_examples)

file(GLOB SOURCES "src/*.c")
add_executable(afr_demo ${SOURCES})

include_directories(afr_demo PRIVATE src/include)

include_directories(afr_demo PRIVATE "freertos/vendors/espressif/esp-idf/components/esp_wifi/include")
include_directories(afr_demo PRIVATE "freertos/vendors/espressif/esp-idf/components/esp_netif/include")
include_directories(afr_demo PRIVATE "freertos/vendors/espressif/esp-idf/components/esp_eth/include")

# Tell IDF build to link against this target.
# set(IDF_PROJECT_EXECUTABLE afr_demo)

list(APPEND IDF_EXTRA_COMPONENT_DIRS ${EXTRA_COMPONENT_DIRS})
get_filename_component(
    EXTRA_COMPONENT_DIRS
    "components" ABSOLUTE
)
list(APPEND IDF_EXTRA_COMPONENT_DIRS ${EXTRA_COMPONENT_DIRS})

# As of now there's no official way to redefine config files outside of the FreeRTOS source tree.
# This is a temporary approach to inject an include path so that this takes precedence over the
# config file directory inside FreeRTOS.
include_directories(BEFORE freertos-configs)

# Add freertos as an subdirectory. AFR_BOARD tells which board to target.
if ("${SOC_NAME}" STREQUAL "esp32s2")
    set(AFR_BOARD espressif.esp32s2_saola_1 CACHE INTERNAL "")
elseif("${SOC_NAME}" STREQUAL "esp32")
    set(AFR_BOARD espressif.esp32_devkitc CACHE INTERNAL "")
endif()
add_subdirectory(freertos)

# Link against the mqtt demo so that we can use it. Dependencies of this demo are transitively
# linked.
target_link_libraries(
    afr_demo
    PRIVATE
    AFR::demo_core_mqtt
    AFR::common_io
    AFR::demo_device_shadow
    AFR::demo_core_mqtt_agent
    AFR::ble
    AFR::demo_gatt_server
)

Here, at build time I got error
src/main.c.obj -MF CMakeFiles/afr_demo.dir/src/main.c.obj.d -o CMakeFiles/afr_demo.dir/src/main.c.obj -c ../src/main.c
../src/main.c:49:10: fatal error: esp_wifi.h: No such file or directory
#include "esp_wifi.h"
^~~~~~~~~~~~
compilation terminated.

For that, I include the directory of esp_wifi. h file which we can see in the Cmake file after I got another error for missing file

../freertos/vendors/espressif/esp-idf/components/esp_event/include/esp_event_legacy.h:22:10: fatal error: esp_netif.h: No such file or directory
 #include "esp_netif.h"

So, I think esp-idf components are not linked with the top-level CMake. But in CMake, we already set the esp-idf directory.
set(esp_idf_dir "${CMAKE_CURRENT_LIST_DIR}/freertos/vendors/espressif/esp-idf")

So, why esp-idf components files are not linking with the top-level CMakeList.txt file?

[shubhamkulkarni97]

@Raghav3107 You need to manually link the IDF components that are used externally. This can be done by passing idf::component_name in target_link_libraries.

Please find working example at https://github.com/shubhamkulkarni97/amazon-freertos-examples/tree/feature/idf_v4.2

[horsemann07]

Hello @shubhamkulkarni97
Thanks for the reply.

I saw your repository of the above link. I tried to use that. I replaced your freertos v202012 with 202106. But the issue is the same.
As you reply, I understand what I need to do.
I just want small clarification.

Q: How we will find the components name?
Question Explanation:->
For example, to link the amazon AFR library we can get the module name from CMakeList.txt or at build time we get the names of the modules like demo_core_mqtt, demo_core_mqtt_agent, etc.
Same way if we want to link the esp-idf components where we find the name of components.
for example. I m getting the error : /src/main.c:51:10: fatal error: esp_bt.h: No such file or directory which is in /components/bt/include/ directory. So, I want to link the esp-idf bt components file. I searched the CMakeLists.txt file to find the components name. But I did not found a thing, from which I understand what I need to replace at the components_name place.

So can you explain, from where can get the components name?

Thanks in advance for your reply and clarification.

---

I manage to link the esp-idf components files. But It will be better if you explain to us how we will find the components name?
Now, the Build program proceeds for further process. At the end, when it is linking the afr_demo some linker issue I found

storage_impl  esp-idf/app_trace/libapp_trace.a  -lgcov  esp-idf/app_trace/libapp_trace.a  -lgcov  -lc  -u vfs_include_syscalls_impl  -u esp_app_desc  -L/home/horsemann/Desktop/WorkSpace/aws_latest_repository/amazon-freertos-examples/freertos/vendors/espressif/esp-idf/components/bt/controller/lib  -lbtdm_app  freertos/libraries/3rdparty/libafr_3rdparty_tinycbor.a  freertos/afr_wifi.a  freertos/libraries/3rdparty/libafr_3rdparty_mbedtls.a  freertos/afr_kernel.a && :
/home/horsemann/Desktop/WorkSpace/esp_toolchain/xtensa-esp32-elf/bin/../lib/gcc/xtensa-esp32-elf/8.4.0/../../../../xtensa-esp32-elf/bin/ld: cannot open linker script file esp32.project.ld: No such file or directory
collect2: error: ld returned 1 exit status
[1719/1776] Linking CXX static library freertos/afr_ota_demo_dependencies.a
ninja: build stopped: subcommand failed.

[shubhamkulkarni97]

Hi @Raghav3107,

• In case of ESP-IDF, component name is same is the directory name of component. For example, for linking bt component, you can use idf::bt as directory name is bt. Similarly, if you want to use driver APIs, you can use idf::driver.

• esp32.project.ld linker script file is generated by IDF build system in the build directory. You can check if ld file is generated at <path-to-build>/build/esp-idf/esp32/ld/esp32.project.ld. These issues generally occur if build directory is not specified correctly or some issues configuring the project (CMakeLists.txt implementation).

[horsemann07]

Thanks, @shubhamkulkarni97 for the answer.

The esp32.project.ld file is not generated at your specified directory. Actually, the ld/ directory is not generated and my top level CMakeLists.txt looks like this

cmake_minimum_required(VERSION 3.13)

set(esp_idf_dir "${CMAKE_CURRENT_LIST_DIR}/freertos/vendors/espressif/esp-idf")

include(${esp_idf_dir}/tools/cmake/idf.cmake)

string(FIND "${CMAKE_TOOLCHAIN_FILE}" "esp32s2" SOC_TOOLCHAIN_ESP32S2)
string(FIND "${CMAKE_TOOLCHAIN_FILE}" "esp32" SOC_TOOLCHAIN_ESP32)
if (NOT(${SOC_TOOLCHAIN_ESP32S2} EQUAL -1))
    set(SOC_NAME "esp32s2")
elseif(NOT($SOC_TOOLCHAIN_ESP32) EQUAL -1)
    set(SOC_NAME "esp32")
endif()

project(freertos_examples)

file(GLOB SOURCES "src/*.c")
add_executable(afr_demo ${SOURCES})
include_directories(afr_demo PRIVATE src/include)

list(APPEND IDF_EXTRA_COMPONENT_DIRS ${EXTRA_COMPONENT_DIRS})
get_filename_component(
    EXTRA_COMPONENT_DIRS
    "components" ABSOLUTE
)
list(APPEND IDF_EXTRA_COMPONENT_DIRS ${EXTRA_COMPONENT_DIRS})

# As of now there's no official way to redefine config files outside of the FreeRTOS source tree.
# This is a temporary approach to inject an include path so that this takes precedence over the
# config file directory inside FreeRTOS.
include_directories(BEFORE freertos-configs)

# Add freertos as an subdirectory. AFR_BOARD tells which board to target.
if ("${SOC_NAME}" STREQUAL "esp32s2")
    set(AFR_BOARD espressif.esp32s2_saola_1 CACHE INTERNAL "")
elseif("${SOC_NAME}" STREQUAL "esp32")
    set(AFR_BOARD espressif.esp32_devkitc CACHE INTERNAL "")
endif()
add_subdirectory(freertos)

# Link against the mqtt demo so that we can use it. Dependencies of this demo are transitively
# linked.
target_link_libraries(
    afr_demo
    PRIVATE
    AFR::demo_core_mqtt
    AFR::common_io
    AFR::demo_device_shadow
    AFR::demo_core_mqtt_agent
    AFR::ble
    AFR::demo_gatt_server
    idf::esp_wifi
    idf::bt
    idf:esp32
)

The command I m using for build
cmake -S . -B build -DIDF_SDKCONFIG_DEFAULTS=./sdkconfig -DCMAKE_TOOLCHAIN_FILE=freertos/tools/cmake/toolchains/xtensa-esp32.cmake -GNinja

[shubhamkulkarni97]

@Raghav3107 I don't find any obvious issue in your CMakeLists.txt. Could try running verbose CMake build to check if there is some issue while build?

[horsemann07]

Hello @shubhamkulkarni97
I run the cmake --build build --verbose as you said. I got tons of output and I don't know what I m looking for.
I have the log can you tell me what is the issue or what should I search in log.
Log.txt

Also, here the repository on which I m working? link

[shubhamkulkarni97]

Hi @Raghav3107,

There are a few issues with the implementation in CMakeLists.txt in your repo.

• Appending component names (as done here) in IDF_EXTRA_COMPONENTS_DIR is not supported in IDF v4.2. You need to call idf_build_component and pass absolute path to the component.
• add_executable call is not supported in the root CMakeLists.txt. Instead you need to set IDF_PROJECT_EXECUTABLE to executable name and IDF_EXECUTABLE_SRCS to executable srcs.
• Include directories are not supported in main. You can create a new component instead of adding all sources and include directories in main.
• In target_link_libraries, you should not link idf::esp_wifi, idf::bt and idf:esp32 because these components are handled by build system internally. Here you should only link custom added components like idf::foo

Please review your CMakeLists.txt as per https://github.com/shubhamkulkarni97/amazon-freertos-examples/blob/feature/idf_v4.2/CMakeLists.txt

Hope this helps!

Thanks,
Shubham

[horsemann07]

Hello @shubhamkulkarni97

Actually, I want to run the demos file outside of the amazon-freertos(in main folder) and instead of config the demo I want to call direct the function which I want. For that, I created the demoRunner.c and demoRunner.h file in which works the same as iot_demo_freertos.c and instead of calling the DEMO_RUNNER_RunDemos() I m directly calling the runMyProgram() and it's calling the mqtt_agent_demo function .

You can see all this file is in same folder i.e main but when I m building it. It is not able to find the runMyProgram() which is in the main demoRunner.c file. and throwing the error

freertos/CMakeFiles/afr_demo.dir/__/src/main.c.obj: in function `app_main':
/home/horsemann/Desktop/WorkSpace/aws_latest_repository/amazon-freertos-custom-v202106/build/../src/main.c:159: undefined reference to `runDemoTaskCustom'

I also tried to build the program by putting demoRunner.c and .h file in the components folder.

[horsemann07]

Thanks, @shubhamkulkarni97 for your help

I fixed the issue by making a change in a top-level CMakeList.

project(freertos_examples)

file(GLOB SOURCES "src/*.c")
get_filename_component(
    IDF_EXECUTABLE_SRCS
    "${SOURCES}" ABSOLUTE
    )

include_directories(afr_demo PRIVATE src/include)
# Tell IDF build to link against this target.
set(IDF_PROJECT_EXECUTABLE afr_demo)

And all thing is working perfectly.
You can check the repository here.

If possible can you give any input on this issue -> #3215

Thanks!

[horsemann07]

Hello @shubhamkulkarni97

• Appending component names (as done here) in IDF_EXTRA_COMPONENTS_DIR is not supported in IDF v4.2. You need to call idf_build_component and pass absolute path to the component.
• add_executable call is not supported in the root CMakeLists.txt. Instead you need to set IDF_PROJECT_EXECUTABLE to executable name and IDF_EXECUTABLE_SRCS to executable srcs.

Can you tell me if we have the multiple components in components directory, then how we will link this all this components with top level CMakeLists.txt

             - components/ - component1/ - CMakeLists.txt
                                         - src1.c
                           - component2/ - CMakeLists.txt
                                         - src1.c
                                         - include/ - component2.h
  

I tried using this step

list(APPEND extra_components_dir "components/foo1" "components/foo2")

get_filename_component(
    EXTRA_COMPONENT_DIRS
    "${extra_components_dir}" ABSOLUTE
)
idf_build_component(${EXTRA_COMPONENT_DIRS})

But that did not work.

[shubhamkulkarni97]

@Raghav3107 You can implement in the following way:

list(APPEND extra_components_dir "components/foo1" "components/foo2")

foreach(dir ${extra_components_dir})
    get_filename_component(
        EXTRA_COMPONENT_DIRS
        "${dir}" ABSOLUTE
    )
    idf_build_component(${EXTRA_COMPONENT_DIRS})
endforeach()

Hope this helps!