diff --git a/codesamples/apis.json b/codesamples/apis.json index 9cac7d040..292668a56 100644 --- a/codesamples/apis.json +++ b/codesamples/apis.json @@ -3,6 +3,11 @@ "client": "Arm", "func": "get_end_position", "args": [] + }, + "audio_in": { + "client": "AudioIn", + "func": "get_properties", + "args": [] }, "button": { "client": "Button", diff --git a/src/viam/api/CMakeLists.txt b/src/viam/api/CMakeLists.txt index 88328b802..e153a5995 100644 --- a/src/viam/api/CMakeLists.txt +++ b/src/viam/api/CMakeLists.txt @@ -132,6 +132,14 @@ if (VIAMCPPSDK_USE_DYNAMIC_PROTOS) ${PROTO_GEN_DIR}/component/arm/v1/arm.grpc.pb.h ${PROTO_GEN_DIR}/component/arm/v1/arm.pb.cc ${PROTO_GEN_DIR}/component/arm/v1/arm.pb.h + ${PROTO_GEN_DIR}/component/audioin/v1/audioin.grpc.pb.cc + ${PROTO_GEN_DIR}/component/audioin/v1/audioin.grpc.pb.h + ${PROTO_GEN_DIR}/component/audioin/v1/audioin.pb.cc + ${PROTO_GEN_DIR}/component/audioin/v1/audioin.pb.h + ${PROTO_GEN_DIR}/component/audioout/v1/audioout.grpc.pb.cc + ${PROTO_GEN_DIR}/component/audioout/v1/audioout.grpc.pb.h + ${PROTO_GEN_DIR}/component/audioout/v1/audioout.pb.cc + ${PROTO_GEN_DIR}/component/audioout/v1/audioout.pb.h ${PROTO_GEN_DIR}/component/base/v1/base.grpc.pb.cc ${PROTO_GEN_DIR}/component/base/v1/base.grpc.pb.h ${PROTO_GEN_DIR}/component/base/v1/base.pb.cc @@ -294,6 +302,10 @@ target_sources(viamapi ${PROTO_GEN_DIR}/common/v1/common.pb.cc ${PROTO_GEN_DIR}/component/arm/v1/arm.grpc.pb.cc ${PROTO_GEN_DIR}/component/arm/v1/arm.pb.cc + ${PROTO_GEN_DIR}/component/audioin/v1/audioin.grpc.pb.cc + ${PROTO_GEN_DIR}/component/audioin/v1/audioin.pb.cc + ${PROTO_GEN_DIR}/component/audioout/v1/audioout.grpc.pb.cc + ${PROTO_GEN_DIR}/component/audioout/v1/audioout.pb.cc ${PROTO_GEN_DIR}/component/base/v1/base.grpc.pb.cc ${PROTO_GEN_DIR}/component/base/v1/base.pb.cc ${PROTO_GEN_DIR}/component/board/v1/board.grpc.pb.cc @@ -358,6 +370,10 @@ target_sources(viamapi ${PROTO_GEN_DIR}/../../viam/api/common/v1/common.pb.h ${PROTO_GEN_DIR}/../../viam/api/component/arm/v1/arm.grpc.pb.h ${PROTO_GEN_DIR}/../../viam/api/component/arm/v1/arm.pb.h + ${PROTO_GEN_DIR}/../../viam/api/component/audioin/v1/audioin.grpc.pb.h + ${PROTO_GEN_DIR}/../../viam/api/component/audioin/v1/audioin.pb.h + ${PROTO_GEN_DIR}/../../viam/api/component/audioout/v1/audioout.grpc.pb.h + ${PROTO_GEN_DIR}/../../viam/api/component/audioout/v1/audioout.pb.h ${PROTO_GEN_DIR}/../../viam/api/component/base/v1/base.grpc.pb.h ${PROTO_GEN_DIR}/../../viam/api/component/base/v1/base.pb.h ${PROTO_GEN_DIR}/../../viam/api/component/board/v1/board.grpc.pb.h diff --git a/src/viam/examples/modules/CMakeLists.txt b/src/viam/examples/modules/CMakeLists.txt index 1714eb933..94a355840 100644 --- a/src/viam/examples/modules/CMakeLists.txt +++ b/src/viam/examples/modules/CMakeLists.txt @@ -15,3 +15,4 @@ add_subdirectory(tflite) add_subdirectory(simple) add_subdirectory(complex) +add_subdirectory(audioin) diff --git a/src/viam/examples/modules/audioin/CMakeLists.txt b/src/viam/examples/modules/audioin/CMakeLists.txt new file mode 100644 index 000000000..9ef219e7c --- /dev/null +++ b/src/viam/examples/modules/audioin/CMakeLists.txt @@ -0,0 +1,43 @@ +# Copyright 2023 Viam Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_executable(audioin_module + main.cpp +) + +target_link_libraries(audioin_module + PRIVATE Threads::Threads + viam-cpp-sdk::viamsdk +) + +install( + TARGETS audioin_module + COMPONENT examples +) + +add_executable(audioin_client) +target_sources(audioin_client + PRIVATE + client.cpp +) + + +target_link_libraries(audioin_client + viam-cpp-sdk::viamsdk +) + +install( + TARGETS audioin_client + COMPONENT examples +) diff --git a/src/viam/examples/modules/audioin/README.md b/src/viam/examples/modules/audioin/README.md new file mode 100644 index 000000000..c7c23e1bd --- /dev/null +++ b/src/viam/examples/modules/audioin/README.md @@ -0,0 +1,34 @@ +# VIAM Simple Module Example +This example goes through how to create a custom audio input resource using Viam's C++ SDK, and how to connect it to a Robot. + +This is a limited document. For a more in-depth understanding of modules, see the [documentation](https://docs.viam.com/registry/). + +Refer to main.cpp and the comments throughout for more information. For other C++ module examples, refer to the [simple module example](https://github.com/viamrobotics/viam-cpp-sdk/tree/main/src/viam/examples/modules/simple) or [complex module example](https://github.com/viamrobotics/viam-cpp-sdk/tree/main/src/viam/examples/modules/complex). + +For a fully fleshed-out example of a C++ module that uses Github CI to upload to the Viam Registry, take a look at [module-example-cpp](https://github.com/viamrobotics/module-example-cpp). For a list of example modules in different Viam SDKs, take a look [here](https://github.com/viamrobotics/upload-module/#example-repos). + +This is a limited document. For a more in-depth understanding of modules generally, see the [documentation](https://docs.viam.com/program/extend/modular-resources/). + + +Example Configuration: +```json{ + "components": [ + { + "name": "sinewave-audio", + "api": "rdk:component:audio_in", + "model": "viam:audio_in:sinewave", + "attributes": { + "frequency": 440 + } + } + ], + "modules": [ + { + "type": "local", + "name": "my-module", + "executable_path": "/home/viam-cpp-sdk/build/src/viam/examples/modules/audioin/audioin_module" + } + ] +} +``` + diff --git a/src/viam/examples/modules/audioin/client.cpp b/src/viam/examples/modules/audioin/client.cpp new file mode 100644 index 000000000..507ed8143 --- /dev/null +++ b/src/viam/examples/modules/audioin/client.cpp @@ -0,0 +1,96 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace viam::sdk; + +int main() { + // Every Viam C++ SDK program must have one and only one Instance object which is created before + // any other C++ SDK objects and stays alive until all Viam C++ SDK objects are destroyed. + Instance inst; + const char* uri = "http://localhost:8080/"; // replace with your URI if connecting securely + DialOptions dial_options; + dial_options.set_allow_insecure_downgrade(true); // set to false if connecting securely + + // Uncomment and fill out your credentials details if connecting securely + // std::string type = "api-key"; + // std::string payload = "your-api-key-here"; + // Credentials credentials(type, payload); + // dial_options.set_credentials(credentials); + + boost::optional opts(dial_options); + std::string address(uri); + Options options(1, opts); + + std::shared_ptr robot = RobotClient::at_address(address, options); + + // Print resources + VIAM_SDK_LOG(info) << "Resources:"; + std::vector resource_names = robot->resource_names(); + for (const Name& resource : resource_names) { + VIAM_SDK_LOG(info) << " " << resource; + } + + // Get the AudioIn component (update with your component name) + auto audio_in = robot->resource_by_name("sinewave-audio"); + if (!audio_in) { + VIAM_SDK_LOG(error) << "could not get 'sinewave-audio' resource from robot"; + return EXIT_FAILURE; + } + + VIAM_SDK_LOG(info) << "Getting audio properties..."; + audio_properties props = audio_in->get_properties(); + VIAM_SDK_LOG(info) << "Audio properties:"; + VIAM_SDK_LOG(info) << " sample_rate_hz: " << props.sample_rate_hz; + VIAM_SDK_LOG(info) << " num_channels: " << props.num_channels; + VIAM_SDK_LOG(info) << " supported_codecs: " << props.supported_codecs.size() << " codecs"; + + VIAM_SDK_LOG(info) << "Retrieving 2 seconds of audio..."; + + std::vector all_audio_data; + int chunk_count = 0; + + // Define chunk handler to collect audio data + auto chunk_handler = [&](AudioIn::audio_chunk&& chunk) -> bool { + chunk_count++; + VIAM_SDK_LOG(info) << "Received chunk " << chunk_count + << " - length: " << chunk.audio_data.size() << " bytes"; + + for (const auto& byte : chunk.audio_data) { + all_audio_data.push_back(static_cast(byte)); + } + + return true; // Continue receiving chunks + }; + + // Get 2 seconds of audio (with previous_timestamp = 0 to start from now) + audio_in->get_audio(audio_codecs::PCM_16, chunk_handler, 2.0, 0); + + VIAM_SDK_LOG(info) << "Total audio data received: " << all_audio_data.size() << " bytes"; + VIAM_SDK_LOG(info) << "Total chunks: " << chunk_count; + + std::string filename = "sine_wave_audio.wav"; + try { + write_wav_file(filename, + all_audio_data, + audio_codecs::PCM_16, + props.sample_rate_hz, + props.num_channels); + VIAM_SDK_LOG(info) << "Audio saved to " << filename; + VIAM_SDK_LOG(info) << "To play: open " << filename << " (or use any audio player)"; + } catch (const std::exception& e) { + VIAM_SDK_LOG(error) << "Failed to write WAV file: " << e.what(); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} diff --git a/src/viam/examples/modules/audioin/main.cpp b/src/viam/examples/modules/audioin/main.cpp new file mode 100644 index 000000000..75b4d9d8c --- /dev/null +++ b/src/viam/examples/modules/audioin/main.cpp @@ -0,0 +1,205 @@ +#define _USE_MATH_DEFINES +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace viam::sdk; + +// Implements an AudioIn component that generates a sine wave for testing +class SineWaveAudioIn : public AudioIn, public Reconfigurable { + public: + SineWaveAudioIn(const ResourceConfig& cfg) : AudioIn(cfg.name()) { + this->reconfigure({}, cfg); + } + + static std::vector validate(const ResourceConfig&); + + void reconfigure(const Dependencies&, const ResourceConfig&) override; + + ProtoStruct do_command(const ProtoStruct&) override; + + std::vector get_geometries(const ProtoStruct&) override { + throw Exception("method not supported"); + } + + audio_properties get_properties(const ProtoStruct&) override; + + void get_audio(std::string const& codec, + std::function const& chunk_handler, + double const& duration_seconds, + int64_t const& previous_timestamp, + const ProtoStruct& extra) override; + + private: + double frequency_{440.0}; + + static double generate_sine_sample(double frequency, double amplitude, double time_seconds); + static int16_t float_to_pcm16(double sample_value); + static std::vector pcm16_samples_to_bytes(const std::vector& samples); + static audio_chunk create_audio_chunk(const std::vector& samples, + const std::string& codec, + int sample_rate_hz, + int num_channels, + int sequence_number); +}; + +std::vector SineWaveAudioIn::validate(const ResourceConfig& cfg) { + auto itr = cfg.attributes().find("frequency"); + if (itr != cfg.attributes().end()) { + const double* freq = itr->second.get(); + if (!freq) { + throw Exception("frequency must be a number value"); + } + if (*freq <= 0.0 || *freq > 20000.0) { + throw Exception("frequency must be between 0 and 20000 Hz"); + } + } + return {}; +} + +void SineWaveAudioIn::reconfigure(const Dependencies&, const ResourceConfig& cfg) { + auto itr = cfg.attributes().find("frequency"); + if (itr != cfg.attributes().end()) { + const double* freq = itr->second.get(); + if (freq) { + frequency_ = *freq; + } + } +} + +ProtoStruct SineWaveAudioIn::do_command(const ProtoStruct& command) { + for (const auto& entry : command) { + VIAM_RESOURCE_LOG(info) << "Command entry " << entry.first; + } + return command; +} + +// Generates a single audio sample representing a sine wave at the given frequency, amplitude, and +// time. +double SineWaveAudioIn::generate_sine_sample(double frequency, + double amplitude, + double time_seconds) { + return amplitude * std::sin(2.0 * M_PI * frequency * time_seconds); +} + +// Converts a normalized floating-point sample (-1.0 to 1.0) to 16-bit PCM format. +int16_t SineWaveAudioIn::float_to_pcm16(double sample_value) { + return static_cast(sample_value * 32767.0); +} + +std::vector SineWaveAudioIn::pcm16_samples_to_bytes(const std::vector& samples) { + std::vector bytes(samples.size() * sizeof(int16_t)); + std::copy(reinterpret_cast(samples.data()), + reinterpret_cast(samples.data()) + bytes.size(), + bytes.begin()); + return bytes; +} + +AudioIn::audio_chunk SineWaveAudioIn::create_audio_chunk(const std::vector& samples, + const std::string& codec, + int sample_rate_hz, + int num_channels, + int sequence_number) { + audio_chunk chunk; + chunk.audio_data = pcm16_samples_to_bytes(samples); + chunk.info.codec = codec; + chunk.info.sample_rate_hz = sample_rate_hz; + chunk.info.num_channels = num_channels; + + auto now = std::chrono::system_clock::now(); + auto nanos = std::chrono::duration_cast(now.time_since_epoch()); + // Set both start and end timestamps to the current system time in nanoseconds. + // In a real application, start_timestamp_ns would mark the time the audio chunk begins, + // and end_timestamp_ns would mark when it ends. Here they are equal for simplicity. + chunk.start_timestamp_ns = nanos; + chunk.end_timestamp_ns = nanos; + chunk.sequence_number = sequence_number; + + return chunk; +} + +audio_properties SineWaveAudioIn::get_properties(const ProtoStruct&) { + audio_properties props; + props.supported_codecs = {audio_codecs::PCM_16}; + props.sample_rate_hz = 44100; + props.num_channels = 1; + return props; +} + +void SineWaveAudioIn::get_audio(std::string const& codec, + std::function const& chunk_handler, + double const& duration_seconds, + int64_t const& previous_timestamp, + const ProtoStruct& extra) { + const int sample_rate = 44100; + const double amplitude = 0.5; + const int chunk_size = 1024; + + int total_samples = static_cast(duration_seconds * sample_rate); + int num_chunks = (total_samples + chunk_size - 1) / chunk_size; + + VIAM_RESOURCE_LOG(info) << "Generating sine wave: " << frequency_ << "Hz, " << duration_seconds + << "s, " << num_chunks << " chunks"; + + for (int chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) { + int samples_in_chunk = std::min(chunk_size, total_samples - (chunk_idx * chunk_size)); + std::vector samples; + samples.reserve(samples_in_chunk); + + // Create each sample and put in chunk. + for (int i = 0; i < samples_in_chunk; ++i) { + int sample_idx = chunk_idx * chunk_size + i; + double time_seconds = static_cast(sample_idx) / sample_rate; + double sample_value = generate_sine_sample(frequency_, amplitude, time_seconds); + samples.push_back(float_to_pcm16(sample_value)); + } + + audio_chunk chunk = create_audio_chunk(samples, codec, sample_rate, 1, chunk_idx); + + if (!chunk_handler(std::move(chunk))) { + VIAM_RESOURCE_LOG(info) << "Chunk handler returned false, stopping"; + break; + } + } + + VIAM_RESOURCE_LOG(info) << "Finished generating sine wave"; +} + +int main(int argc, char** argv) try { + // Every Viam C++ SDK program must have one and only one Instance object which is created before + // any other C++ SDK objects and stays alive until all Viam C++ SDK objects are destroyed. + Instance inst; + + Model sinewave_model("viam", "audio_in", "sinewave"); + + std::shared_ptr mr = std::make_shared( + API::get(), + sinewave_model, + [](Dependencies, ResourceConfig cfg) { return std::make_unique(cfg); }, + &SineWaveAudioIn::validate); + + std::vector> mrs = {mr}; + auto my_mod = std::make_shared(argc, argv, mrs); + my_mod->serve(); + + return EXIT_SUCCESS; +} catch (const viam::sdk::Exception& ex) { + std::cerr << "main failed with exception: " << ex.what() << "\n"; + return EXIT_FAILURE; +} diff --git a/src/viam/sdk/CMakeLists.txt b/src/viam/sdk/CMakeLists.txt index b9af46039..0b1b12480 100644 --- a/src/viam/sdk/CMakeLists.txt +++ b/src/viam/sdk/CMakeLists.txt @@ -60,6 +60,7 @@ endif() target_sources(viamsdk PRIVATE + common/audio.cpp common/client_helper.cpp common/exception.cpp common/instance.cpp @@ -71,6 +72,7 @@ target_sources(viamsdk common/world_state.cpp common/private/service_helper.cpp components/arm.cpp + components/audio_in.cpp components/base.cpp components/board.cpp components/button.cpp @@ -86,6 +88,8 @@ target_sources(viamsdk components/power_sensor.cpp components/private/arm_client.cpp components/private/arm_server.cpp + components/private/audio_in_server.cpp + components/private/audio_in_client.cpp components/private/base_client.cpp components/private/base_server.cpp components/private/board_client.cpp @@ -163,6 +167,7 @@ target_sources(viamsdk ../.. ${CMAKE_CURRENT_BINARY_DIR}/../.. FILES + ../../viam/sdk/common/audio.hpp ../../viam/sdk/common/client_helper.hpp ../../viam/sdk/common/exception.hpp ../../viam/sdk/common/instance.hpp @@ -174,6 +179,7 @@ target_sources(viamsdk ../../viam/sdk/common/version_metadata.hpp ../../viam/sdk/common/world_state.hpp ../../viam/sdk/components/arm.hpp + ../../viam/sdk/components/audio_in.hpp ../../viam/sdk/components/base.hpp ../../viam/sdk/components/board.hpp ../../viam/sdk/components/button.hpp diff --git a/src/viam/sdk/common/audio.cpp b/src/viam/sdk/common/audio.cpp new file mode 100644 index 000000000..081c58fee --- /dev/null +++ b/src/viam/sdk/common/audio.cpp @@ -0,0 +1,71 @@ +#include + +#include +#include +#include + +namespace viam { +namespace sdk { + +namespace { +template +void write_value(std::ofstream& out, const T& value) { + out.write(reinterpret_cast(&value), sizeof(value)); +} +} // anonymous namespace + +bool operator==(const audio_properties& lhs, const audio_properties& rhs) { + return std::tie(lhs.supported_codecs, lhs.sample_rate_hz, lhs.num_channels) == + std::tie(rhs.supported_codecs, rhs.sample_rate_hz, rhs.num_channels); +} + +uint16_t get_bits_per_sample(const std::string& codec) { + if (codec == audio_codecs::PCM_16) { + return 16; + } + if (codec == audio_codecs::PCM_32 || codec == audio_codecs::PCM_32_FLOAT) { + return 32; + } + throw std::runtime_error("Unsupported codec for WAV file: " + codec); +} + +void write_wav_file(const std::string& filename, + const std::vector& audio_data, + const std::string& codec, + uint32_t sample_rate_hz, + uint16_t num_channels) { + std::ofstream outfile(filename, std::ios::binary); + if (!outfile.is_open()) { + throw std::runtime_error("Failed to open file for writing: " + filename); + } + + const uint16_t bits_per_sample = get_bits_per_sample(codec); + const uint32_t data_size = audio_data.size(); + const uint32_t byte_rate = sample_rate_hz * num_channels * (bits_per_sample / 8); + const uint16_t block_align = num_channels * (bits_per_sample / 8); + + outfile.write("RIFF", 4); + const uint32_t chunk_size = 36 + data_size; + write_value(outfile, chunk_size); + outfile.write("WAVE", 4); + + outfile.write("fmt ", 4); + const uint32_t subchunk1_size = 16; + write_value(outfile, subchunk1_size); + const uint16_t audio_format = 1; + write_value(outfile, audio_format); + write_value(outfile, num_channels); + write_value(outfile, sample_rate_hz); + write_value(outfile, byte_rate); + write_value(outfile, block_align); + write_value(outfile, bits_per_sample); + + outfile.write("data", 4); + write_value(outfile, data_size); + outfile.write(reinterpret_cast(audio_data.data()), data_size); + + outfile.close(); +} + +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/common/audio.hpp b/src/viam/sdk/common/audio.hpp new file mode 100644 index 000000000..505a5cf9a --- /dev/null +++ b/src/viam/sdk/common/audio.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include +#include +#include +#include + +namespace viam { +namespace sdk { + +/// @brief Common audio codecs +namespace audio_codecs { +constexpr const char* PCM_16 = "pcm16"; +constexpr const char* PCM_32 = "pcm32"; +constexpr const char* PCM_32_FLOAT = "pcm32_float"; +constexpr const char* MP3 = "mp3"; +constexpr const char* AAC = "aac"; +constexpr const char* OPUS = "opus"; +constexpr const char* FLAC = "flac"; +} // namespace audio_codecs + +/// @struct audio_properties +/// @brief Properties of an audio component (input or output) +struct audio_properties { + std::vector supported_codecs; + int sample_rate_hz; + int num_channels; +}; + +/// @struct audio_info +/// @brief Information about a piece of audio data +struct audio_info { + std::string codec; + int sample_rate_hz; + int num_channels; +}; + +/// @brief Equality operator for properties +bool operator==(const audio_properties& lhs, const audio_properties& rhs); + +uint16_t get_bits_per_sample(const std::string& codec); + +void write_wav_file(const std::string& filename, + const std::vector& audio_data, + const std::string& codec, + uint32_t sample_rate_hz, + uint16_t num_channels); + +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/components/audio_in.cpp b/src/viam/sdk/components/audio_in.cpp new file mode 100644 index 000000000..54c5e3347 --- /dev/null +++ b/src/viam/sdk/components/audio_in.cpp @@ -0,0 +1,21 @@ +#include + +#include +#include +#include + +namespace viam { +namespace sdk { + +API AudioIn::api() const { + return API::get(); +} + +API API::traits::api() { + return {kRDK, kComponent, "audio_in"}; +} + +AudioIn::AudioIn(std::string name) : Component(std::move(name)) {} + +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/components/audio_in.hpp b/src/viam/sdk/components/audio_in.hpp new file mode 100644 index 000000000..6a5592170 --- /dev/null +++ b/src/viam/sdk/components/audio_in.hpp @@ -0,0 +1,111 @@ +/// @file components/audio_in.hpp +/// +/// @brief Defines a `AudioIn` component. +#pragma once + +#include + +#include +#include +#include +#include + +namespace viam { +namespace sdk { + +/// @defgroup Base Classes related to the AudioIn component. + +/// @class AudioIn audio_in.hpp "components/audio_in.hpp" +/// @brief An `AudioIn` is a deivce that can take audio input. +/// @ingroup AudioIn +/// +/// This acts as an abstract parent class to be inherited from by any drivers representing +/// specific AudioIn implementations. This class cannot be used on its own. +class AudioIn : public Component { + public: + /// @struct audio_chunk + /// @brief A sequential chunk of audio data with timing information for continuous audio + /// streams. + struct audio_chunk { + std::vector audio_data; + audio_info info; + std::chrono::nanoseconds start_timestamp_ns; + std::chrono::nanoseconds end_timestamp_ns; + int sequence_number; // sequential chunk number + std::string request_id; + }; + + /// @brief Get a stream of audio from the device + /// until completed or cancelled + /// @param codec requested codec of the audio data + /// @param chunk_handler callback function to call when an audio response is received. + /// This should return true to keep streaming audio and false to indicate + /// that the stream should terminate. The callback function should not be blocking. + /// @param duration_seconds duration of audio stream. If not set, stream duration is indefinite. + /// @param previous_timestamp timestamp to start the audio stream from for continuity between + /// multiple calls. If not set, will stream data + // starting from the time the request was received by the server. + inline void get_audio(std::string const& codec, + std::function const& chunk_handler, + double const& duration_seconds, + int64_t const& previous_timestamp) { + return get_audio(codec, chunk_handler, duration_seconds, previous_timestamp, {}); + } + + /// @brief Get a stream of audio from the device + /// until completed or cancelled + /// @param codec requested codec of the audio data + /// @param chunk_handler callback function to call when an audio response is received. + /// For an infinite stream this should return true to keep streaming audio and false to indicate + /// that the stream should terminate. The callback function should not be blocking. + /// @param duration_seconds duration of audio stream. If zero, stream duration is indefinite. + /// @param previous_timestamp timestamp to start the audio stream from for continuity between + /// multiple calls. If zero, will stream data + // starting from the time the request was received by the server. + /// @param extra Any additional arguments to the method + virtual void get_audio(std::string const& codec, + std::function const& chunk_handler, + double const& duration_seconds, + int64_t const& previous_timestamp, + const ProtoStruct& extra) = 0; + + /// @brief Returns properties of the audio in device (supported codecs, sample rate, number of + /// channels) + inline audio_properties get_properties() { + return get_properties({}); + } + + /// @brief Returns properties of the audio in device (supported codecs, sample rate, number of + /// channels) + /// @param extra Any additional arguments to the method + virtual audio_properties get_properties(const ProtoStruct& extra) = 0; + + // @brief Send/receive arbitrary commands to the resource. + /// @param Command the command to execute. + /// @return The result of the executed command. + virtual ProtoStruct do_command(const ProtoStruct& command) = 0; + + /// @brief Returns `GeometryConfig`s associated with the calling AudioIn. + /// @return The requested `GeometryConfig`s associated with the component. + inline std::vector get_geometries() { + return get_geometries({}); + } + + /// @brief Returns `GeometryConfig`s associated with the calling AudioIn. + /// @param extra Any additional arguments to the method. + /// @return The requested `GeometryConfig`s associated with the component. + virtual std::vector get_geometries(const ProtoStruct& extra) = 0; + + API api() const override; + + protected: + explicit AudioIn(std::string name); +}; + +template <> +struct API::traits { + static API api(); +}; + +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/components/private/audio_in_client.cpp b/src/viam/sdk/components/private/audio_in_client.cpp new file mode 100644 index 000000000..b7c58c2c6 --- /dev/null +++ b/src/viam/sdk/components/private/audio_in_client.cpp @@ -0,0 +1,91 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace viam { +namespace sdk { +namespace impl { + +AudioInClient::AudioInClient(std::string name, std::shared_ptr channel) + : AudioIn(std::move(name)), + stub_(viam::component::audioin::v1::AudioInService::NewStub(channel)), + channel_(std::move(channel)) {} + +void AudioInClient::get_audio(std::string const& codec, + std::function const& chunk_handler, + double const& duration_seconds, + int64_t const& previous_timestamp, + const ProtoStruct& extra) { + return make_client_helper(this, *stub_, &StubType::GetAudio) + .with(extra, + [&](auto& request) { + request.set_codec(codec); + request.set_duration_seconds(duration_seconds); + request.set_previous_timestamp_nanoseconds(previous_timestamp); + }) + .invoke_stream([&](auto& response) { + // Create audio_chunk struct from proto response + audio_chunk chunk; + + // Convert audio_data from string to std::vector + const std::string& audio_data_str = response.audio().audio_data(); + chunk.audio_data.assign(audio_data_str.c_str(), + audio_data_str.c_str() + audio_data_str.size()); + + chunk.sequence_number = response.audio().sequence(); + chunk.request_id = response.request_id(); + + if (response.audio().has_audio_info()) { + chunk.info.codec = response.audio().audio_info().codec(); + chunk.info.sample_rate_hz = response.audio().audio_info().sample_rate_hz(); + chunk.info.num_channels = response.audio().audio_info().num_channels(); + } + return chunk_handler(std::move(chunk)); + }); +} +audio_properties AudioInClient::get_properties(const ProtoStruct& extra) { + return make_client_helper(this, *stub_, &StubType::GetProperties) + .with(extra) + .invoke([](auto& response) { + // Convert proto repeated field to vector + std::vector codecs; + codecs.reserve(response.supported_codecs_size()); + for (const auto& codec : response.supported_codecs()) { + codecs.push_back(codec); + } + + return audio_properties{ + std::move(codecs), response.sample_rate_hz(), response.num_channels()}; + }); +} + +ProtoStruct AudioInClient::do_command(const ProtoStruct& command) { + return make_client_helper(this, *stub_, &StubType::DoCommand) + .with([&](auto& request) { *request.mutable_command() = to_proto(command); }) + .invoke([](auto& response) { return from_proto(response.result()); }); +} + +std::vector AudioInClient::get_geometries(const ProtoStruct& extra) { + return make_client_helper(this, *stub_, &StubType::GetGeometries) + .with(extra) + .invoke([](auto& response) { return from_proto(response); }); +}; + +} // namespace impl +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/components/private/audio_in_client.hpp b/src/viam/sdk/components/private/audio_in_client.hpp new file mode 100644 index 000000000..da338f792 --- /dev/null +++ b/src/viam/sdk/components/private/audio_in_client.hpp @@ -0,0 +1,46 @@ +/// @file components/private/audio_in_client.hpp +/// +/// @brief Implements a gRPC client for the `AudioIn` component +#pragma once + +#include + +#include + +#include + +namespace viam { +namespace sdk { +namespace impl { + +/// @class AudioInClient +/// @brief gRPC client implementation of a `AudioIn` component. +/// @ingroup AudioIn +class AudioInClient : public AudioIn { + public: + using interface_type = AudioIn; + AudioInClient(std::string name, std::shared_ptr channel); + + void get_audio(std::string const& codec, + std::function const& chunk_handler, + double const& duration_seconds, + int64_t const& previous_timestamp, + const ProtoStruct& extra) override; + + audio_properties get_properties(const ProtoStruct& extra) override; + ProtoStruct do_command(const ProtoStruct& command) override; + std::vector get_geometries(const ProtoStruct& extra) override; + + using AudioIn::get_audio; + using AudioIn::get_geometries; + using AudioIn::get_properties; + + private: + using StubType = viam::component::audioin::v1::AudioInService::StubInterface; + std::unique_ptr stub_; + std::shared_ptr channel_; +}; + +} // namespace impl +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/components/private/audio_in_server.cpp b/src/viam/sdk/components/private/audio_in_server.cpp new file mode 100644 index 000000000..b5454c04a --- /dev/null +++ b/src/viam/sdk/components/private/audio_in_server.cpp @@ -0,0 +1,102 @@ +#include +#include +#include +#include + +#include + +namespace viam { +namespace sdk { +namespace impl { + +AudioInServer::AudioInServer(std::shared_ptr manager) + : ResourceServer(std::move(manager)) {} + +::grpc::Status AudioInServer::GetAudio( + ::grpc::ServerContext* context, + const ::viam::component::audioin::v1::GetAudioRequest* request, + ::grpc::ServerWriter<::viam::component::audioin::v1::GetAudioResponse>* writer) noexcept { + make_service_helper( + "AudioInServer::GetAudio", this, request)([&](auto& helper, auto& audio_in) { + const std::string request_id = boost::uuids::to_string(boost::uuids::random_generator()()); + auto writeChunk = [writer, context, request_id](AudioIn::audio_chunk&& chunk) { + if (context->IsCancelled()) { + // send bool to tell the resource to stop calling the callback function. + return false; + } + ::viam::component::audioin::v1::GetAudioResponse response; + auto* audio_chunk = response.mutable_audio(); + + // Convert audio_data from std::vector to string + std::string audio_data_str; + audio_data_str.reserve(chunk.audio_data.size()); + for (const auto& byte : chunk.audio_data) { + audio_data_str.push_back(static_cast(byte)); + } + audio_chunk->set_audio_data(std::move(audio_data_str)); + + // Set audio_info fields + auto* audio_info = audio_chunk->mutable_audio_info(); + audio_info->set_codec(std::move(chunk.info.codec)); + audio_info->set_sample_rate_hz(chunk.info.sample_rate_hz); + audio_info->set_num_channels(chunk.info.num_channels); + + audio_chunk->set_start_timestamp_nanoseconds(chunk.start_timestamp_ns.count()); + audio_chunk->set_end_timestamp_nanoseconds(chunk.end_timestamp_ns.count()); + audio_chunk->set_sequence(chunk.sequence_number); + response.set_request_id(request_id); + writer->Write(response); + return true; + }; + audio_in->get_audio(request->codec(), + writeChunk, + request->duration_seconds(), + request->previous_timestamp_nanoseconds(), + helper.getExtra()); + }); + + return ::grpc::Status(); +} + +::grpc::Status AudioInServer::DoCommand(::grpc::ServerContext*, + const ::viam::common::v1::DoCommandRequest* request, + ::viam::common::v1::DoCommandResponse* response) noexcept { + return make_service_helper( + "AudioInServer::DoCommand", this, request)([&](auto&, auto& audio_in) { + const ProtoStruct result = audio_in->do_command(from_proto(request->command())); + *response->mutable_result() = to_proto(result); + }); +} + +::grpc::Status AudioInServer::GetProperties( + grpc::ServerContext*, + const viam::common::v1::GetPropertiesRequest* request, + viam::common::v1::GetPropertiesResponse* response) noexcept { + return make_service_helper( + "AudioInServer::GetProperties", this, request)([&](auto& helper, auto& audio_in) { + const audio_properties result = audio_in->get_properties(helper.getExtra()); + for (const auto& codec : result.supported_codecs) { + response->add_supported_codecs(codec); + } + + response->set_sample_rate_hz(result.sample_rate_hz); + response->set_num_channels(result.num_channels); + }); +} + +::grpc::Status AudioInServer::GetGeometries( + ::grpc::ServerContext*, + const ::viam::common::v1::GetGeometriesRequest* request, + ::viam::common::v1::GetGeometriesResponse* response) noexcept { + return make_service_helper( + "AudioInServer::GetGeometries", this, request)([&](auto& helper, auto& audio_in) { + const std::vector geometries = audio_in->get_geometries(helper.getExtra()); + for (const auto& geometry : geometries) { + *response->mutable_geometries()->Add() = to_proto(geometry); + } + }); +} + +} // namespace impl +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/components/private/audio_in_server.hpp b/src/viam/sdk/components/private/audio_in_server.hpp new file mode 100644 index 000000000..46dc16dd9 --- /dev/null +++ b/src/viam/sdk/components/private/audio_in_server.hpp @@ -0,0 +1,48 @@ +/// @file components/private/audio_in_server.hpp +/// +/// @brief Implements a gRPC server for the `AudioIn` component +#pragma once + +#include +#include + +#include +#include +#include + +namespace viam { +namespace sdk { +namespace impl { + +/// @class AudioInServer +/// @brief gRPC server implementation of an `AudioIn` component. +/// @ingroup Arm + +class AudioInServer : public ResourceServer, + public viam::component::audioin::v1::AudioInService::Service { + public: + using interface_type = AudioIn; + using service_type = component::audioin::v1::AudioInService; + + explicit AudioInServer(std::shared_ptr manager); + + ::grpc::Status GetAudio(::grpc::ServerContext* context, + const ::viam::component::audioin::v1::GetAudioRequest*, + ::grpc::ServerWriter<::viam::component::audioin::v1::GetAudioResponse>* + writer) noexcept override; + ::grpc::Status DoCommand(::grpc::ServerContext* context, + const ::viam::common::v1::DoCommandRequest* request, + ::viam::common::v1::DoCommandResponse* response) noexcept override; + ::grpc::Status GetGeometries( + ::grpc::ServerContext* context, + const ::viam::common::v1::GetGeometriesRequest* request, + ::viam::common::v1::GetGeometriesResponse* response) noexcept override; + ::grpc::Status GetProperties( + ::grpc::ServerContext* context, + const ::viam::common::v1::GetPropertiesRequest* request, + ::viam::common::v1::GetPropertiesResponse* response) noexcept override; +}; + +} // namespace impl +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/registry/registry.cpp b/src/viam/sdk/registry/registry.cpp index 792ceb0fe..d689707a2 100644 --- a/src/viam/sdk/registry/registry.cpp +++ b/src/viam/sdk/registry/registry.cpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include #include @@ -201,6 +203,7 @@ const google::protobuf::ServiceDescriptor* ResourceServerRegistration::service_d void Registry::register_resources() { // Register all components register_resource(); + register_resource(); register_resource(); register_resource(); register_resource(); diff --git a/src/viam/sdk/tests/CMakeLists.txt b/src/viam/sdk/tests/CMakeLists.txt index a054ed5b4..ce59a75c6 100644 --- a/src/viam/sdk/tests/CMakeLists.txt +++ b/src/viam/sdk/tests/CMakeLists.txt @@ -22,6 +22,7 @@ target_sources(viamsdk_test mocks/generic_mocks.cpp mocks/mlmodel_mocks.cpp mocks/mock_arm.cpp + mocks/mock_audio_in.cpp mocks/mock_base.cpp mocks/mock_board.cpp mocks/mock_button.cpp @@ -55,6 +56,7 @@ target_link_libraries(viamsdk_test viamcppsdk_link_viam_api(viamsdk_test PUBLIC) viamcppsdk_add_boost_test(test_arm.cpp) +viamcppsdk_add_boost_test(test_audio_in.cpp) viamcppsdk_add_boost_test(test_base.cpp) viamcppsdk_add_boost_test(test_board.cpp) viamcppsdk_add_boost_test(test_button.cpp) diff --git a/src/viam/sdk/tests/mocks/mock_audio_in.cpp b/src/viam/sdk/tests/mocks/mock_audio_in.cpp new file mode 100644 index 000000000..6f0a894c4 --- /dev/null +++ b/src/viam/sdk/tests/mocks/mock_audio_in.cpp @@ -0,0 +1,99 @@ +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace viam { +namespace sdktests { +namespace audioin { + +using namespace viam::sdk; + +void MockAudioIn::get_audio(std::string const& codec, + std::function const& chunk_handler, + double const& duration_seconds, + int64_t const& previous_timestamp, + const ProtoStruct& extra) { + // Simulate streaming audio chunks + int chunk_count = 0; + int max_chunks = (duration_seconds == 0) ? 100 : static_cast(duration_seconds * 100); + + for (const auto& mock_chunk : mock_chunks_) { + if (chunk_count >= max_chunks) { + break; + } + + // Create a copy of the chunk to pass to handler + audio_chunk chunk = mock_chunk; + chunk.sequence_number = chunk_count; + + if (!chunk_handler(std::move(chunk))) { + break; // Handler requested to stop + } + + chunk_count++; + + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } +} + +audio_properties MockAudioIn::get_properties(const ProtoStruct& extra) { + return properties_; +} + +ProtoStruct MockAudioIn::do_command(const ProtoStruct& command) { + return map_; +} + +std::shared_ptr MockAudioIn::get_mock_audio_in() { + auto audio_in = std::make_shared("mock_audio_in"); + + audio_in->properties_ = fake_properties(); + audio_in->mock_chunks_ = fake_audio_chunks(); + audio_in->map_ = fake_map(); + audio_in->geometries_ = fake_geometries(); + + return audio_in; +} + +audio_properties fake_properties() { + audio_properties props; + props.supported_codecs = {audio_codecs::PCM_16, audio_codecs::PCM_32}; + props.sample_rate_hz = 48000; + props.num_channels = 1; + return props; +} + +std::vector fake_audio_chunks() { + std::vector chunks; + + for (int i = 0; i < 5; ++i) { + AudioIn::audio_chunk chunk; + chunk.audio_data = std::vector(1024, static_cast(i + 1)); + chunk.info.codec = audio_codecs::PCM_16; + chunk.info.sample_rate_hz = 48000; + chunk.info.num_channels = 1; + auto now = std::chrono::system_clock::now(); + chunk.start_timestamp_ns = + std::chrono::duration_cast(now.time_since_epoch()); + chunk.end_timestamp_ns = chunk.start_timestamp_ns; + chunk.sequence_number = i; + chunks.push_back(chunk); + } + + return chunks; +} + +std::vector MockAudioIn::get_geometries(const ProtoStruct&) { + return geometries_; +} + +} // namespace audioin +} // namespace sdktests +} // namespace viam diff --git a/src/viam/sdk/tests/mocks/mock_audio_in.hpp b/src/viam/sdk/tests/mocks/mock_audio_in.hpp new file mode 100644 index 000000000..f98e0d496 --- /dev/null +++ b/src/viam/sdk/tests/mocks/mock_audio_in.hpp @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + +namespace viam { +namespace sdktests { +namespace audioin { + +using viam::sdk::AudioIn; +using namespace viam::sdk; + +class MockAudioIn : public AudioIn { + public: + void get_audio(std::string const& codec, + std::function const& chunk_handler, + double const& duration_seconds, + int64_t const& previous_timestamp, + const sdk::ProtoStruct& extra) override; + + audio_properties get_properties(const sdk::ProtoStruct& extra) override; + + viam::sdk::ProtoStruct do_command(const viam::sdk::ProtoStruct& command) override; + + std::vector get_geometries(const sdk::ProtoStruct& extra) override; + + static std::shared_ptr get_mock_audio_in(); + + MockAudioIn(std::string name) : AudioIn(std::move(name)) {} + + using AudioIn::get_audio; + using AudioIn::get_properties; + + private: + audio_properties properties_; + viam::sdk::ProtoStruct map_; + std::vector mock_chunks_; + std::vector geometries_; +}; + +audio_properties fake_properties(); +std::vector fake_audio_chunks(); + +} // namespace audioin +} // namespace sdktests +} // namespace viam diff --git a/src/viam/sdk/tests/test_audio_in.cpp b/src/viam/sdk/tests/test_audio_in.cpp new file mode 100644 index 000000000..86b9e39bd --- /dev/null +++ b/src/viam/sdk/tests/test_audio_in.cpp @@ -0,0 +1,135 @@ +#define BOOST_TEST_MODULE test module test_audio_in +#include +#include +#include +#include +#include +#include +#include +#include + +namespace viam { +namespace sdktests { + +using namespace audioin; + +using namespace viam::sdk; + +BOOST_AUTO_TEST_SUITE(test_audio_in) + +BOOST_AUTO_TEST_CASE(mock_get_api) { + const MockAudioIn audio_in("mock_audio_in"); + auto api = audio_in.api(); + auto static_api = API::get(); + + BOOST_CHECK_EQUAL(api, static_api); + BOOST_CHECK_EQUAL(static_api.resource_subtype(), "audio_in"); +} + +BOOST_AUTO_TEST_CASE(test_get_audio) { + std::shared_ptr mock = MockAudioIn::get_mock_audio_in(); + + client_to_mock_pipeline(mock, [&](AudioIn& client) { + std::vector received_chunks; + + client.get_audio( + audio_codecs::PCM_16, + [&](AudioIn::audio_chunk&& chunk) -> bool { + received_chunks.push_back(std::move(chunk)); + // Stop stream after 3 chunks + return received_chunks.size() < 3; + }, + 1.0, // 1 second duration + 0 // No previous timestamp + ); + + BOOST_CHECK_EQUAL(received_chunks.size(), 3); + + // Check that request_id is consistent across all chunks + BOOST_CHECK(!received_chunks[0].request_id.empty()); + std::string first_request_id = received_chunks[0].request_id; + + for (size_t i = 0; i < received_chunks.size(); ++i) { + const auto& chunk = received_chunks[i]; + BOOST_CHECK_EQUAL(chunk.audio_data.size(), 1024); + BOOST_CHECK_EQUAL(chunk.info.codec, audio_codecs::PCM_16); + BOOST_CHECK_EQUAL(chunk.info.sample_rate_hz, 48000); + BOOST_CHECK_EQUAL(chunk.info.num_channels, 1); + BOOST_CHECK_EQUAL(chunk.request_id, first_request_id); + BOOST_CHECK_EQUAL(chunk.sequence_number, static_cast(i)); + BOOST_CHECK_GE(chunk.start_timestamp_ns.count(), 0); + BOOST_CHECK_GE(chunk.end_timestamp_ns.count(), 0); + } + }); +} + +BOOST_AUTO_TEST_CASE(test_get_audio_request_ids_differ_across_calls) { + std::shared_ptr mock = MockAudioIn::get_mock_audio_in(); + + client_to_mock_pipeline(mock, [&](AudioIn& client) { + std::string first_request_id; + std::string second_request_id; + + // First call + client.get_audio( + audio_codecs::PCM_16, + [&](AudioIn::audio_chunk&& chunk) -> bool { + first_request_id = chunk.request_id; + return false; // Stop after first chunk + }, + 1.0, + 0); + + // Second call + client.get_audio( + audio_codecs::PCM_16, + [&](AudioIn::audio_chunk&& chunk) -> bool { + second_request_id = chunk.request_id; + return false; // Stop after first chunk + }, + 1.0, + 0); + + // Request IDs should be different across separate calls + BOOST_CHECK(!first_request_id.empty()); + BOOST_CHECK(!second_request_id.empty()); + BOOST_CHECK_NE(first_request_id, second_request_id); + }); +} + +BOOST_AUTO_TEST_CASE(test_do_command) { + std::shared_ptr mock = MockAudioIn::get_mock_audio_in(); + client_to_mock_pipeline(mock, [](AudioIn& client) { + ProtoStruct expected = fake_map(); + + ProtoStruct command = fake_map(); + ProtoStruct result_map = client.do_command(command); + + BOOST_CHECK(result_map.at("test") == expected.at("test")); + }); +} + +BOOST_AUTO_TEST_CASE(test_get_properties) { + std::shared_ptr mock = MockAudioIn::get_mock_audio_in(); + client_to_mock_pipeline(mock, [](AudioIn& client) { + audio_properties props = client.get_properties(); + audio_properties expected = fake_properties(); + + BOOST_CHECK(expected == props); + }); +} + +BOOST_AUTO_TEST_CASE(test_get_geometries) { + std::shared_ptr mock = MockAudioIn::get_mock_audio_in(); + client_to_mock_pipeline(mock, [](AudioIn& client) { + std::vector result_geometries = client.get_geometries(); + std::vector expected_geometries = fake_geometries(); + + BOOST_CHECK(result_geometries == expected_geometries); + }); +} + +BOOST_AUTO_TEST_SUITE_END() + +} // namespace sdktests +} // namespace viam