ESPHome  2024.10.2
Public Member Functions | Protected Member Functions | Protected Attributes
esphome::micro_wake_word::MicroWakeWord Class Reference

#include <micro_wake_word.h>

Inheritance diagram for esphome::micro_wake_word::MicroWakeWord:
esphome::Component

Public Member Functions

void setup () override
 
void loop () override
 
float get_setup_priority () const override
 
void dump_config () override
 
void start ()
 
void stop ()
 
bool is_running () const
 
void set_features_step_size (uint8_t step_size)
 
void set_microphone (microphone::Microphone *microphone)
 
Trigger< std::string > * get_wake_word_detected_trigger () const
 
void add_wake_word_model (const uint8_t *model_start, float probability_cutoff, size_t sliding_window_average_size, const std::string &wake_word, size_t tensor_arena_size)
 
void add_vad_model (const uint8_t *model_start, float probability_cutoff, size_t sliding_window_size, size_t tensor_arena_size)
 
- Public Member Functions inherited from esphome::Component
float get_actual_setup_priority () const
 
void set_setup_priority (float priority)
 
virtual float get_loop_priority () const
 priority of loop(). More...
 
void call ()
 
virtual void on_shutdown ()
 
virtual void on_safe_shutdown ()
 
uint32_t get_component_state () const
 
virtual void mark_failed ()
 Mark this component as failed. More...
 
bool is_failed () const
 
bool is_ready () const
 
virtual bool can_proceed ()
 
bool status_has_warning () const
 
bool status_has_error () const
 
void status_set_warning (const char *message="unspecified")
 
void status_set_error (const char *message="unspecified")
 
void status_clear_warning ()
 
void status_clear_error ()
 
void status_momentary_warning (const std::string &name, uint32_t length=5000)
 
void status_momentary_error (const std::string &name, uint32_t length=5000)
 
bool has_overridden_loop () const
 
void set_component_source (const char *source)
 Set where this component was loaded from for some debug messages. More...
 
const char * get_component_source () const
 Get the integration where this component was declared as a string. More...
 

Protected Member Functions

void set_state_ (State state)
 
bool has_enough_samples_ ()
 Tests if there are enough samples in the ring buffer to generate new features. More...
 
size_t read_microphone_ ()
 Reads audio from microphone into the ring buffer. More...
 
bool allocate_buffers_ ()
 Allocates memory for input_buffer_, preprocessor_audio_buffer_, and ring_buffer_. More...
 
void deallocate_buffers_ ()
 Frees memory allocated for input_buffer_ and preprocessor_audio_buffer_. More...
 
bool load_models_ ()
 Loads streaming models and prepares the feature generation frontend. More...
 
void unload_models_ ()
 Deletes each model's TFLite interpreters and frees tensor arena memory. More...
 
void update_model_probabilities_ ()
 Performs inference with each configured model. More...
 
bool detect_wake_words_ ()
 Checks every model's recent probabilities to determine if the wake word has been predicted. More...
 
bool generate_features_for_window_ (int8_t features[PREPROCESSOR_FEATURE_SIZE])
 Generates features for a window of audio samples. More...
 
void reset_states_ ()
 Resets the ring buffer, ignore_windows_, and sliding window probabilities. More...
 
bool register_streaming_ops_ (tflite::MicroMutableOpResolver< 20 > &op_resolver)
 Returns true if successfully registered the streaming model's TensorFlow operations. More...
 
uint16_t new_samples_to_get_ ()
 
- Protected Member Functions inherited from esphome::Component
virtual void call_loop ()
 
virtual void call_setup ()
 
virtual void call_dump_config ()
 
void set_interval (const std::string &name, uint32_t interval, std::function< void()> &&f)
 Set an interval function with a unique name. More...
 
void set_interval (uint32_t interval, std::function< void()> &&f)
 
bool cancel_interval (const std::string &name)
 Cancel an interval function. More...
 
void set_retry (const std::string &name, uint32_t initial_wait_time, uint8_t max_attempts, std::function< RetryResult(uint8_t)> &&f, float backoff_increase_factor=1.0f)
 Set an retry function with a unique name. More...
 
void set_retry (uint32_t initial_wait_time, uint8_t max_attempts, std::function< RetryResult(uint8_t)> &&f, float backoff_increase_factor=1.0f)
 
bool cancel_retry (const std::string &name)
 Cancel a retry function. More...
 
void set_timeout (const std::string &name, uint32_t timeout, std::function< void()> &&f)
 Set a timeout function with a unique name. More...
 
void set_timeout (uint32_t timeout, std::function< void()> &&f)
 
bool cancel_timeout (const std::string &name)
 Cancel a timeout function. More...
 
void defer (const std::string &name, std::function< void()> &&f)
 Defer a callback to the next loop() call. More...
 
void defer (std::function< void()> &&f)
 Defer a callback to the next loop() call. More...
 
bool cancel_defer (const std::string &name)
 Cancel a defer callback using the specified name, name must not be empty. More...
 

Protected Attributes

microphone::Microphonemicrophone_ {nullptr}
 
Trigger< std::string > * wake_word_detected_trigger_ = new Trigger<std::string>()
 
State state_ {State::IDLE}
 
HighFrequencyLoopRequester high_freq_
 
std::unique_ptr< RingBufferring_buffer_
 
std::vector< WakeWordModelwake_word_models_
 
std::unique_ptr< VADModelvad_model_
 
tflite::MicroMutableOpResolver< 20 > streaming_op_resolver_
 
struct FrontendConfig frontend_config_
 
struct FrontendState frontend_state_
 
int16_t ignore_windows_ {-MIN_SLICES_BEFORE_DETECTION}
 
uint8_t features_step_size_
 
int16_t * input_buffer_ {nullptr}
 
int16_t * preprocessor_audio_buffer_ {nullptr}
 
bool detected_ {false}
 
std::string detected_wake_word_ {""}
 
- Protected Attributes inherited from esphome::Component
uint32_t component_state_ {0x0000}
 State of this component. More...
 
float setup_priority_override_ {NAN}
 
const char * component_source_ {nullptr}
 

Detailed Description

Definition at line 35 of file micro_wake_word.h.

Member Function Documentation

◆ add_vad_model()

void esphome::micro_wake_word::MicroWakeWord::add_vad_model ( const uint8_t *  model_start,
float  probability_cutoff,
size_t  sliding_window_size,
size_t  tensor_arena_size 
)

Definition at line 96 of file micro_wake_word.cpp.

◆ add_wake_word_model()

void esphome::micro_wake_word::MicroWakeWord::add_wake_word_model ( const uint8_t *  model_start,
float  probability_cutoff,
size_t  sliding_window_average_size,
const std::string &  wake_word,
size_t  tensor_arena_size 
)

Definition at line 88 of file micro_wake_word.cpp.

◆ allocate_buffers_()

bool esphome::micro_wake_word::MicroWakeWord::allocate_buffers_ ( )
protected

Allocates memory for input_buffer_, preprocessor_audio_buffer_, and ring_buffer_.

Returns
True if successful, false otherwise

Definition at line 219 of file micro_wake_word.cpp.

◆ deallocate_buffers_()

void esphome::micro_wake_word::MicroWakeWord::deallocate_buffers_ ( )
protected

Frees memory allocated for input_buffer_ and preprocessor_audio_buffer_.

Definition at line 249 of file micro_wake_word.cpp.

◆ detect_wake_words_()

bool esphome::micro_wake_word::MicroWakeWord::detect_wake_words_ ( )
protected

Checks every model's recent probabilities to determine if the wake word has been predicted.

Verifies the models have processed enough new samples for accurate predictions. Sets detected_wake_word_ to the wake word, if one is detected.

Returns
True if a wake word is predicted, false otherwise

Definition at line 312 of file micro_wake_word.cpp.

◆ dump_config()

void esphome::micro_wake_word::MicroWakeWord::dump_config ( )
overridevirtual

Reimplemented from esphome::Component.

Definition at line 50 of file micro_wake_word.cpp.

◆ generate_features_for_window_()

bool esphome::micro_wake_word::MicroWakeWord::generate_features_for_window_ ( int8_t  features[PREPROCESSOR_FEATURE_SIZE])
protected

Generates features for a window of audio samples.

Reads samples from the ring buffer and feeds them into the preprocessor frontend. Adapted from TFLite microspeech frontend.

Parameters
featuresint8_t array to store the audio features
Returns
True if successful, false otherwise.

Definition at line 345 of file micro_wake_word.cpp.

◆ get_setup_priority()

float esphome::micro_wake_word::MicroWakeWord::get_setup_priority ( ) const
overridevirtual

Reimplemented from esphome::Component.

Definition at line 29 of file micro_wake_word.cpp.

◆ get_wake_word_detected_trigger()

Trigger<std::string>* esphome::micro_wake_word::MicroWakeWord::get_wake_word_detected_trigger ( ) const
inline

Definition at line 51 of file micro_wake_word.h.

◆ has_enough_samples_()

bool esphome::micro_wake_word::MicroWakeWord::has_enough_samples_ ( )
protected

Tests if there are enough samples in the ring buffer to generate new features.

Returns
True if enough samples, false otherwise.

Definition at line 340 of file micro_wake_word.cpp.

◆ is_running()

bool esphome::micro_wake_word::MicroWakeWord::is_running ( ) const
inline

Definition at line 45 of file micro_wake_word.h.

◆ load_models_()

bool esphome::micro_wake_word::MicroWakeWord::load_models_ ( )
protected

Loads streaming models and prepares the feature generation frontend.

Returns
True if successful, false otherwise

Definition at line 257 of file micro_wake_word.cpp.

◆ loop()

void esphome::micro_wake_word::MicroWakeWord::loop ( )
overridevirtual

Reimplemented from esphome::Component.

Definition at line 102 of file micro_wake_word.cpp.

◆ new_samples_to_get_()

uint16_t esphome::micro_wake_word::MicroWakeWord::new_samples_to_get_ ( )
inlineprotected

Definition at line 155 of file micro_wake_word.h.

◆ read_microphone_()

size_t esphome::micro_wake_word::MicroWakeWord::read_microphone_ ( )
protected

Reads audio from microphone into the ring buffer.

Audio data (16000 kHz with int16 samples) is read into the input_buffer_. Verifies the ring buffer has enough space for all audio data. If not, it logs a warning and resets the ring buffer entirely.

Returns
Number of bytes written to the ring buffer

Definition at line 199 of file micro_wake_word.cpp.

◆ register_streaming_ops_()

bool esphome::micro_wake_word::MicroWakeWord::register_streaming_ops_ ( tflite::MicroMutableOpResolver< 20 > &  op_resolver)
protected

Returns true if successfully registered the streaming model's TensorFlow operations.

Definition at line 410 of file micro_wake_word.cpp.

◆ reset_states_()

void esphome::micro_wake_word::MicroWakeWord::reset_states_ ( )
protected

Resets the ring buffer, ignore_windows_, and sliding window probabilities.

Definition at line 398 of file micro_wake_word.cpp.

◆ set_features_step_size()

void esphome::micro_wake_word::MicroWakeWord::set_features_step_size ( uint8_t  step_size)
inline

Definition at line 47 of file micro_wake_word.h.

◆ set_microphone()

void esphome::micro_wake_word::MicroWakeWord::set_microphone ( microphone::Microphone microphone)
inline

Definition at line 49 of file micro_wake_word.h.

◆ set_state_()

void esphome::micro_wake_word::MicroWakeWord::set_state_ ( State  state)
protected

Definition at line 193 of file micro_wake_word.cpp.

◆ setup()

void esphome::micro_wake_word::MicroWakeWord::setup ( )
overridevirtual

Reimplemented from esphome::Component.

Definition at line 61 of file micro_wake_word.cpp.

◆ start()

void esphome::micro_wake_word::MicroWakeWord::start ( )

Definition at line 149 of file micro_wake_word.cpp.

◆ stop()

void esphome::micro_wake_word::MicroWakeWord::stop ( )

Definition at line 181 of file micro_wake_word.cpp.

◆ unload_models_()

void esphome::micro_wake_word::MicroWakeWord::unload_models_ ( )
protected

Deletes each model's TFLite interpreters and frees tensor arena memory.

Frees memory used by the feature generation frontend.

Definition at line 282 of file micro_wake_word.cpp.

◆ update_model_probabilities_()

void esphome::micro_wake_word::MicroWakeWord::update_model_probabilities_ ( )
protected

Performs inference with each configured model.

If enough audio samples are available, it will generate one slice of new features. It then loops through and performs inference with each of the loaded models.

Definition at line 293 of file micro_wake_word.cpp.

Field Documentation

◆ detected_

bool esphome::micro_wake_word::MicroWakeWord::detected_ {false}
protected

Definition at line 92 of file micro_wake_word.h.

◆ detected_wake_word_

std::string esphome::micro_wake_word::MicroWakeWord::detected_wake_word_ {""}
protected

Definition at line 93 of file micro_wake_word.h.

◆ features_step_size_

uint8_t esphome::micro_wake_word::MicroWakeWord::features_step_size_
protected

Definition at line 85 of file micro_wake_word.h.

◆ frontend_config_

struct FrontendConfig esphome::micro_wake_word::MicroWakeWord::frontend_config_
protected

Definition at line 78 of file micro_wake_word.h.

◆ frontend_state_

struct FrontendState esphome::micro_wake_word::MicroWakeWord::frontend_state_
protected

Definition at line 79 of file micro_wake_word.h.

◆ high_freq_

HighFrequencyLoopRequester esphome::micro_wake_word::MicroWakeWord::high_freq_
protected

Definition at line 65 of file micro_wake_word.h.

◆ ignore_windows_

int16_t esphome::micro_wake_word::MicroWakeWord::ignore_windows_ {-MIN_SLICES_BEFORE_DETECTION}
protected

Definition at line 83 of file micro_wake_word.h.

◆ input_buffer_

int16_t* esphome::micro_wake_word::MicroWakeWord::input_buffer_ {nullptr}
protected

Definition at line 88 of file micro_wake_word.h.

◆ microphone_

microphone::Microphone* esphome::micro_wake_word::MicroWakeWord::microphone_ {nullptr}
protected

Definition at line 62 of file micro_wake_word.h.

◆ preprocessor_audio_buffer_

int16_t* esphome::micro_wake_word::MicroWakeWord::preprocessor_audio_buffer_ {nullptr}
protected

Definition at line 90 of file micro_wake_word.h.

◆ ring_buffer_

std::unique_ptr<RingBuffer> esphome::micro_wake_word::MicroWakeWord::ring_buffer_
protected

Definition at line 67 of file micro_wake_word.h.

◆ state_

State esphome::micro_wake_word::MicroWakeWord::state_ {State::IDLE}
protected

Definition at line 64 of file micro_wake_word.h.

◆ streaming_op_resolver_

tflite::MicroMutableOpResolver<20> esphome::micro_wake_word::MicroWakeWord::streaming_op_resolver_
protected

Definition at line 75 of file micro_wake_word.h.

◆ vad_model_

std::unique_ptr<VADModel> esphome::micro_wake_word::MicroWakeWord::vad_model_
protected

Definition at line 72 of file micro_wake_word.h.

◆ wake_word_detected_trigger_

Trigger<std::string>* esphome::micro_wake_word::MicroWakeWord::wake_word_detected_trigger_ = new Trigger<std::string>()
protected

Definition at line 63 of file micro_wake_word.h.

◆ wake_word_models_

std::vector<WakeWordModel> esphome::micro_wake_word::MicroWakeWord::wake_word_models_
protected

Definition at line 69 of file micro_wake_word.h.


The documentation for this class was generated from the following files: