diff --git a/README.md b/README.md index 7b31fe2..c1b9615 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Scarecrow-Cam -A `Raspberry Pi` powered edge-computing camera setups that runs a `Tensorflow` object detection model to determine whether a person is on the camera and plays loud audio to scare them off. +A `Raspberry Pi` powered, distributed (edge) computing camera setups that runs a `Tensorflow` object detection model to determine whether a person is on the camera. The `Raspberry Pi` is used for video streaming and triggering actions (such as playing audio, turning on lights, or triggering an Arduino), whereas a server or laptop runs the object detection. With a suitable `TFLite` installation, this can happen locally on the `Raspberry` as well. + +Based on the detection criteria, a **plugin model** allows to trigger downstream actions. *Based on my [blog](https://chollinger.com/blog/2019/12/tensorflow-on-edge-building-a-smar-security-camera-with-a-raspberry-pi/).* @@ -8,9 +10,9 @@ A `Raspberry Pi` powered edge-computing camera setups that runs a `Tensorflow` o ![Sample](./docs/cam_1.png) -The setup shown here only fits the use-case of `edge` to a degree, as we run local detection on a separate machine; technically, the Raspberry Pi is capable of running Tensorflow on board, e.g. through `TFLite` or `esp32cam`. +**Side note**: *The setup shown here only fits the use-case of `edge` to a degree, as we run local detection on a separate machine; technically, the Raspberry Pi is capable of running Tensorflow on board, e.g. through `TFLite` or `esp32cam`.* -You can change this behavior by relying on a local `tensorflor` instance and having the `ZMQ` communication run over `localhost`. +*You can change this behavior by relying on a local `tensorflor` instance and having the `ZMQ` communication run over `localhost`.* ## Requirements This project requires: @@ -22,6 +24,8 @@ This project requires: ## Install A helper script is available: + +**Use a virtual environment** ``` python3 -m venv env source env/bin/activate @@ -32,15 +36,9 @@ Please see [INSTALL.md](./INSTALL.md) for details. ## Configuration and data -**Use a virtual environment** -``` -python3 -m venv env -source env/bin/activate -``` - Edit the `conf/config.ini` with the settings for your Raspberry and server. -For playing audio, please adjust +For playing audio, please adjust `conf/plugins.d/audio.ini`. ``` [Audio] @@ -68,5 +66,21 @@ python3 $PROJECT_LOCATION/client/sender.py --input '/path/to/video' # for local python3 $PROJECT_LOCATION/server/receiver.py ``` +## Plugins +A plugin model allows to trigger downstream actions. These actions are triggered based on the configuration. + +Plugins can be enabled by setting the following in `config.ini`: +``` +[Plugins] +Enabled=audio +Disabled= +``` + +Currently, the following plugins are avaibale: + +| Plugin | Description | Requirements | Configuration | Base | +|--------|---------------------------------------------|----------------------------------------------|----------------------------|-------| +| audio | Plays audio files once a person is detected | Either `playsound`, `pygame`, or `omxplayer` | `conf/plugins.d/audio.ini` | `ZMQ` | + ## License This project is licensed under the GNU GPLv3 License - see the [LICENSE](LICENSE) file for details. \ No newline at end of file diff --git a/client/sender.py b/client/sender.py index a99bce4..408a18b 100644 --- a/client/sender.py +++ b/client/sender.py @@ -5,11 +5,12 @@ import sys sys.path.append('..') from utilities.utils import * -from network.server import start_zmq_thread +from plugin_base.utils import * import argparse import configparser import time + def run_camera(input_str, address, port, protocol, fps=25): """Runs the camera, sends messages @@ -60,8 +61,8 @@ def run_camera(input_str, address, port, protocol, fps=25): conf = configparser.ConfigParser() conf.read('../conf/config.ini') - # Audio ZMQ thread - start_zmq_thread(conf['ZmqServer']['IP'], conf['ZmqServer']['Port'], conf['Audio']['Path'], conf['Audio']['Streamer']) + # Plugin ZMQ threads + start_receiver_plugins(load_plugins(conf['Plugins']['Enabled'].split(','))) print('Starting camera stream') run_camera(args.in_file, conf['Video']['IP'], conf['Video']['Port'], conf['Video']['Protocol'], int(conf['Video']['FPS'])) \ No newline at end of file diff --git a/conf/config.ini b/conf/config.ini index 4f12bd3..9226100 100644 --- a/conf/config.ini +++ b/conf/config.ini @@ -16,12 +16,12 @@ Port=5556 IP=192.168.1.240 Port=5556 -[Audio] -Streamer=pygame -Path=../audio_files - [General] UseSenderThread=False [Tensorflow] -ModelUrl=ssdlite_mobilenet_v2_coco_2018_05_09 \ No newline at end of file +ModelUrl=ssdlite_mobilenet_v2_coco_2018_05_09 + +[Plugins] +Enabled=audio +Disabled= \ No newline at end of file diff --git a/local_detector.py b/local_detector.py index 37de27d..84a5921 100644 --- a/local_detector.py +++ b/local_detector.py @@ -2,14 +2,12 @@ sys.path.append('./models/research/object_detection') sys.path.append('./tensor_detectors') from network.messages import Messages -from network.sender import send_command from tensor_detectors.detector import load_model, label_map_util, run_inference import configparser import argparse import cv2 - if __name__ == "__main__": # Args parser = argparse.ArgumentParser(description='Runs local image detection') @@ -45,4 +43,3 @@ conf['Detection']['min_confidence']), fps=int(conf['Video']['FPS'])): print('Received signal') - #send_command(conf['ZmqCamera']['IP'], conf['ZmqCamera']['Port'], Messages.WARN) diff --git a/network/sender.py b/network/sender.py deleted file mode 100644 index 03480c3..0000000 --- a/network/sender.py +++ /dev/null @@ -1,44 +0,0 @@ -import zmq -import multiprocessing as mp -from . import messages - - -def send_command(server, port, *args): - """Sends a command via `ZMQ` - - Args: - server (str): Server address - port (int): Server port - """ - context = zmq.Context() - - socket = context.socket(zmq.REQ) - socket.connect('tcp://{}:{}'.format(server, port)) - - for msg in args: - print('Sending message {} to server {}:{}'.format(msg, server, port)) - socket.send_string(str(msg.value)) - - # Get the reply. - response = socket.recv() - print('Received response: {}'.format(response)) - - -def start_sender_thread(server, port, msg=messages.Messages.WARN): - """Starts a sender thread for non-blocking network I/O - - Args: - server (str): Server address - port (int): Server port - msg (messages.Messages, optional): Message type. Defaults to messages.Messages.WARN. - - Returns: - Process: Process - """ - # TODO: *args msg - print('Starting sender thread') - p = mp.Process(target=send_command, args=(server, port, msg, )) - # Set as daemon, so it gets killed alongside the parent - p.daemon = True - p.start() - return p diff --git a/network/server.py b/network/server.py deleted file mode 100644 index e053938..0000000 --- a/network/server.py +++ /dev/null @@ -1,56 +0,0 @@ -import time -import zmq - -from . import messages -import multiprocessing as mp - -import sys -sys.path.append('..') -from audio.player import play_sound - -def receive_messages(server, port, audio_path, streamer, *args): - """Receives messages from `ZMQ` and plays audio - - Args: - server (str): Server address - port (int): Server port - audio_path (str): Path to audio files - streamer (str): [`pygame`, `playsound`, `os`] - """ - print('Starting receiver thread for ZMQ...') - context = zmq.Context() - socket = context.socket(zmq.REP) - socket.bind('tcp://*:{}'.format(port)) - - while True: - # Wait for next request from client - message = socket.recv() - print('Received request: {}'.format(message)) - print(str(messages.Messages.WARN.value)) - if message.decode('ascii') == str(messages.Messages.WARN.value): - print('Playing warning') - play_sound('{}/warning.mp3'.format(audio_path), streamer) - elif message.decode('ascii') == str(messages.Messages.MUSIC.value): - print('Playing Music') - play_sound('{}/music.mp3'.format(audio_path), streamer) - else: - print('Can\'t parse message!') - - # Send acknowlede - socket.send(b'Ack') - -def start_zmq_thread(server, port, audio_path, streamer='pygame', msg=messages.Messages.WARN): - """Starts a ZMQ listener thread - - Args: - server (str): Server address - port (int): Server port - streamer (str): [`pygame`, `playsound`, `os`]. Defaults to 'pygame'. - audio_path (str): Path to audio files - msg (messages.Messages, optional): Message type. Defaults to messages.Messages.WARN. - """ - # TODO: *args msg - p = mp.Process(target=receive_messages, args=(server, port, audio_path, streamer, msg, )) - # Set as daemon, so it gets killed alongside the parent - p.daemon = True - p.start() \ No newline at end of file diff --git a/plugin_base/base.py b/plugin_base/base.py index bf2745e..52aef57 100644 --- a/plugin_base/base.py +++ b/plugin_base/base.py @@ -2,6 +2,8 @@ class ZmqBasePlugin: + """ZMQ Base plugin to implement sender/receiver plugins + """ def __init__(self, configuration): self.configuration = configuration self.recv_server = configuration['ZmqReceiver']['IP'] @@ -9,21 +11,34 @@ def __init__(self, configuration): self.send_server = configuration['ZmqSender']['IP'] self.send_port = configuration['ZmqSender']['Port'] + print('Loaded plugin {}'.format(self.__class__.__name__)) + def on_receive(self, *args): + """Called on receving a message + """ print('on_receive is not implemented in {}'.format( self.__class__.__name__)) pass def send_ack(self, socket, *args): + """Sends acknowledgement. Called after `process` + + Args: + socket (socket): `ZMQ` socket + """ print('send_ack is not implemented in {}'.format(self.__class__.__name__)) # Send acknowlede socket.send(b'Ack') def process(self, *args): + """Processes the message. Called after `on_receive` + """ print('process is not implemented in {}'.format(self.__class__.__name__)) pass def start_receiver(self, *args): + """Starts the main reciver loop + """ print('Starting receiver thread for ZMQ in {}...'.format( self.__class__.__name__)) context = zmq.Context() @@ -38,6 +53,8 @@ def start_receiver(self, *args): self.send_ack(socket) def start_sender(self, *args): + """Starts the main sender loop + """ context = zmq.Context() socket = context.socket(zmq.REQ) socket.connect('tcp://{}:{}'.format(self.send_server, self.send_port)) @@ -45,6 +62,11 @@ def start_sender(self, *args): self.on_ack(socket) def send(self, socket, *args): + """Sends a message + + Args: + socket (socket): `ZMQ` socket + """ print('send is not implemented in {}'.format(self.__class__.__name__)) msg = 'no_implemented' print('Sending message {} to server {}:{}'.format(msg, self.send_server, self.send_port)) @@ -52,6 +74,11 @@ def send(self, socket, *args): def on_ack(self, socket, *args): + """Prases ack message. Called after `send` + + Args: + socket (socket): `ZMQ` socket + """ print('on_ack is not implemented in {}'.format(self.__class__.__name__)) # Send acknowlede # Get the reply. diff --git a/plugin_base/utils.py b/plugin_base/utils.py index 7d3d217..2b5f2b5 100644 --- a/plugin_base/utils.py +++ b/plugin_base/utils.py @@ -10,9 +10,18 @@ __allowed_plugins__ = { 'audio': AudioPlugin } -plugins = ['audio'] -def load_plugins(): +# Read config + +def load_plugins(plugins): + """Loads all plugins defined in `__allowed_plugins__` + + Raises: + NotImplementedError: If an invalid plugin was specified + + Returns: + list: loaded_plugins + """ # Load plugins loaded_plugins = [] for plugin in plugins: @@ -30,6 +39,14 @@ def load_plugins(): def start_receiver_plugins(loaded_plugins): + """Starts the daemon threads for the receiver plugins + + Args: + loaded_plugins (list): loaded_plugins + + Returns: + list: Started processes + """ # Execution procs = [] for plugin in loaded_plugins: @@ -41,5 +58,22 @@ def start_receiver_plugins(loaded_plugins): return procs def send_messages(loaded_plugins): + """Sends a message across all plugins + + Args: + loaded_plugins (list): loaded_plugins + """ + for se in loaded_plugins: + se.start_sender() + +def send_async_messages(loaded_plugins): + """Starts a separate thread to send all messages + + Args: + loaded_plugins (list): loaded_plugins + """ for se in loaded_plugins: - se.start_sender() \ No newline at end of file + p = mp.Process(target=se.start_sender) + # Set as daemon, so it gets killed alongside the parent + p.daemon = True + p.start() \ No newline at end of file diff --git a/server/receiver.py b/server/receiver.py index 11df1aa..4caf3b5 100644 --- a/server/receiver.py +++ b/server/receiver.py @@ -9,8 +9,8 @@ import cv2 from vidgear.gears import NetGear from tensor_detectors.detector import run_inference_for_single_image, load_model, detect -from network.sender import send_command, start_sender_thread from network.messages import Messages +from plugin_base.utils import * import configparser @@ -66,7 +66,7 @@ def receive(category_index, model, address, port, protocol, min_detections=10, m client.close() -def main(address, port, protocol, zmq_ip, zmq_port, min_detections, min_confidence, model_name, use_sender_thread): +def main(address, port, protocol, zmq_ip, zmq_port, min_detections, min_confidence, model_name, use_sender_thread, plugins): # List of the strings that is used to add correct label for each box. PATH_TO_LABELS = '../models/research/object_detection/data/mscoco_label_map.pbtxt' category_index = label_map_util.create_category_index_from_labelmap( @@ -74,12 +74,14 @@ def main(address, port, protocol, zmq_ip, zmq_port, min_detections, min_confiden detection_model = load_model(model_name) + # Plugins + plugins = load_plugins(plugins=plugins) for res in receive(category_index, detection_model, address, port, protocol, min_detections, min_confidence): print('Received signal') if use_sender_thread: - start_sender_thread(zmq_ip, zmq_port, Messages.WARN) + send_async_messages(plugins) else: - send_command(zmq_ip, zmq_port, Messages.WARN) + send_messages(plugins) if __name__ == "__main__": @@ -88,7 +90,8 @@ def main(address, port, protocol, zmq_ip, zmq_port, min_detections, min_confiden conf.read('../conf/config.ini') main(conf['Video']['IP'], conf['Video']['Port'], conf['Video']['Protocol'], conf['ZmqCamera']['IP'], conf['ZmqCamera']['Port'], - float(conf['Detection']['min_detections']), float( - conf['Detection']['min_confidence']), + float(conf['Detection']['min_detections']), + float(conf['Detection']['min_confidence']), model_name=conf['Tensorflow']['ModelUrl'], - use_sender_thread=conf.getboolean('General', 'UseSenderThread')) + use_sender_thread=conf.getboolean('General', 'UseSenderThread'), + plugins=conf['Plugins']['Enabled'].split(','))