diff --git a/README.rst b/README.rst index fb5bff4..4cc6034 100644 --- a/README.rst +++ b/README.rst @@ -50,7 +50,7 @@ Python 3 versions may also work. 3) Create new environment for Birdwatcher (name is up to you, in example here 'mybirdwatcher'). We install Jupter lab and ffmpeg at the same time:: - $ conda create -n mybirdwatcher python=3.9 jupyterlab ffmpeg + $ conda create -n mybirdwatcher python=3.9 jupyterlab ffmpeg git 4) Switch to this new environment: @@ -73,7 +73,9 @@ The following dependencies are automatically taken care of when you install Birdwatcher from GitHub using the pip method above: - numpy +- pandas - matplotlib +- seaborn - darr - opencv-python - opencv-contrib-python diff --git a/birdwatcher/__init__.py b/birdwatcher/__init__.py index ecce6b3..0df422e 100644 --- a/birdwatcher/__init__.py +++ b/birdwatcher/__init__.py @@ -1,19 +1,10 @@ from .video import * -from .movementdetection import * from .backgroundsubtraction import * from .coordinatearrays import * from .frames import * -from .utils import * -try: - import pypylon - from .recording import * -except ImportError: - pass -#from .plotting import * from ._version import get_versions __version__ = get_versions()['version'] del get_versions - from .tests import test \ No newline at end of file diff --git a/birdwatcher/backgroundsubtraction.py b/birdwatcher/backgroundsubtraction.py index 6e5809f..0ecd24a 100644 --- a/birdwatcher/backgroundsubtraction.py +++ b/birdwatcher/backgroundsubtraction.py @@ -139,7 +139,7 @@ class BackgroundSubtractorMOG2(BaseBackgroundSubtractor): Parameters ---------- - History : int, default=5 + History : int, default=3 Length of the history. ComplexityReductionThreshold : float, default=0.5 This parameter defines the number of samples needed to accept to prove @@ -155,11 +155,11 @@ class BackgroundSubtractorMOG2(BaseBackgroundSubtractor): The number of gaussian components in the background model. VarInit : int, default=15 The initial variance of each gaussian component. - VarMin : int, default=4 + VarMin : int, default=10 The minimum variance of each gaussian component. VarMax : int, default=75 The maximum variance of each gaussian component. - VarThreshold : int, default=10 + VarThreshold : int, default=70 The variance threshold for the pixel-model match. The main threshold on the squared Mahalanobis distance to decide if the sample is well described by the background model or not. Related to Cthr from the @@ -180,24 +180,24 @@ class BackgroundSubtractorMOG2(BaseBackgroundSubtractor): The shadow threshold is a threshold defining how much darker the shadow can be. 0.5 means that if a pixel is more than twice darker then it is not shadow. - ShadowValue : int, default=127 + ShadowValue : int, default=0 Shadow value is the value used to mark shadows in the foreground mask. Value 0 in the mask always means background, 255 means foreground. """ - _setparams = {'History': 5, + _setparams = {'History': 3, 'ComplexityReductionThreshold': 0.05, 'BackgroundRatio': 0.1, 'NMixtures': 7, 'VarInit': 15, - 'VarMin': 4, + 'VarMin': 10, 'VarMax': 75, - 'VarThreshold': 10, + 'VarThreshold': 70, 'VarThresholdGen': 9, 'DetectShadows': False, 'ShadowThreshold': 0.5, - 'ShadowValue': 127} + 'ShadowValue': 0} _bgsubtractorcreatefunc = cv.createBackgroundSubtractorMOG2 diff --git a/birdwatcher/coordinatearrays.py b/birdwatcher/coordinatearrays.py index 04b5313..a01406a 100644 --- a/birdwatcher/coordinatearrays.py +++ b/birdwatcher/coordinatearrays.py @@ -1,28 +1,31 @@ """This module provides objects and functions for coordinate data. -Coordinates are pixel positions (x,y). This is a convenient way of storing -output from image algorithms that determine whether or not a pixel is -categorized as something (e.g. crossing some threshold). Since the number of -pixels per frame may be different, we use disk-based ragged arrays from the -python library Darr to store them. This is not the most disk-space efficient -way of doing this, but it is fast and the data can easily be read in any -computing platform. Coordinate files can be archived in compressed form -(lzma) when data is large. +Coordinates are pixel positions (x,y) in a Frame. Coordinate Arrays are a +convenient way of storing output from image algorithms that determine if a +pixel is categorized as something (e.g. crossing some threshold). +Information is memory-mapped from disk, because data can easily become very +large and will not fit in RAM memory. Since the number of pixels per frame may +be variable, we use Ragged Arrays from the python library Darr to store them. +This is not the most disk-space efficient way of doing this (no compression), +but it is fast and the data can easily be read in any scientific computing +environement (Python, Matlab, R, Mathematica, etc.) Coordinate files can be +archived in compressed form (lzma) to save disk space. """ import os -from contextlib import contextmanager -from pathlib import Path import shutil -import numpy as np import tarfile +from contextlib import contextmanager +from pathlib import Path +import numpy as np from darr import RaggedArray, delete_raggedarray, create_raggedarray from ._version import get_versions from .utils import tempdir from .frames import frameiterator + __all__ = ['CoordinateArrays', 'open_archivedcoordinatedata', 'create_coordarray'] diff --git a/birdwatcher/ffmpeg.py b/birdwatcher/ffmpeg.py index 207a71e..7d2d096 100644 --- a/birdwatcher/ffmpeg.py +++ b/birdwatcher/ffmpeg.py @@ -1,10 +1,12 @@ -import subprocess -import numpy as np import json +import subprocess from pathlib import Path +import numpy as np + from .utils import peek_iterable + __all__ = ['arraytovideo'] @@ -104,7 +106,7 @@ def videofileinfo(filepath, ffprobepath='ffprobe'): ## FIXME inform before raising StopIteration that file has no frames def iterread_videofile(filepath, startat=None, nframes=None, color=True, - ffmpegpath='ffmpeg', loglevel= 'quiet'): + ffmpegpath='ffmpeg', loglevel='quiet'): """ Parameters ---------- @@ -199,11 +201,11 @@ def get_frame(filepath, framenumber, color=True, ffmpegpath='ffmpeg', dtype=np.uint8).reshape(frameshape) -def get_frameat(filepath, time, color=True, ffmpegpath='ffmpeg', loglevel= - 'quiet'): - return next(iterread_videofile(filepath, startat=time, nframes=1, \ - color=color, ffmpegpath=ffmpegpath), - loglevel=loglevel) +def get_frameat(filepath, time, color=True, ffmpegpath='ffmpeg', + loglevel='quiet'): + return next(iterread_videofile(filepath, startat=time, nframes=1, + color=color, ffmpegpath=ffmpegpath, + loglevel=loglevel)) # FIXME do not assume things on audio (i.e. number of channels) and make more versatile diff --git a/birdwatcher/frames.py b/birdwatcher/frames.py index 23ba9e5..c59ef88 100644 --- a/birdwatcher/frames.py +++ b/birdwatcher/frames.py @@ -8,16 +8,19 @@ """ -import numpy as np -import cv2 as cv from functools import wraps from pathlib import Path +import numpy as np +import cv2 as cv + from .utils import peek_iterable + __all__ = ['Frames', 'FramesColor', 'FramesGray', 'FramesNoise', 'framecolor', 'framegray', 'framenoise'] + def frameiterator(func): @wraps(func) def wrapper(*args, **kwargs): @@ -778,7 +781,7 @@ def calc_meanframe(self, dtype=None): (0, 0, 0), dtype='float64') for i, frame in enumerate(self._frames): meanframe += frame - meanframe /= i + meanframe /= (i+1) if dtype is None: dtype = self._dtype return meanframe.astype(dtype) @@ -815,17 +818,18 @@ class FramesColor(Frames): """ - def __init__(self, nframes, width, height, color=(0, 0, 0), dtype='uint8'): + def __init__(self, nframes, height, width, color=(0, 0, 0), + dtype='uint8'): """Creates an iterator that yields color frames. Parameters ---------- nframes : int Number of frames to be produced. - width : int - Width of frame. height : int Height of frame. + width : int + Width of frame. color : tuple of ints, optional Fill value of frame (r, g, b). The default (0, 0, 0) color is black. @@ -837,7 +841,7 @@ def __init__(self, nframes, width, height, color=(0, 0, 0), dtype='uint8'): Iterator of numpy ndarrays """ - frame = framecolor(width=width, height=height, color=color, + frame = framecolor(height=height, width=width, color=color, dtype=dtype) frames = (frame.copy() for _ in range(nframes)) super().__init__(frames=frames) @@ -850,17 +854,17 @@ class FramesGray(Frames): """ - def __init__(self, nframes, width, height, value=0, dtype='uint8'): + def __init__(self, nframes, height, width, value=0, dtype='uint8'): """Creates an iterator that yields gray frames. Parameters ---------- nframes : int Number of frames to be produced. - width : int - Width of frame. height : int Height of frame. + width : int + Width of frame. value : int, optional Fill value of frame. The default (0) is black. dtype : numpy dtype, default='uint8' @@ -872,9 +876,8 @@ def __init__(self, nframes, width, height, value=0, dtype='uint8'): """ - frame = framegray(width=width, height=height, value=value, + frame = framegray(height=height, width=width, value=value, dtype=dtype) - frames = (frame.copy() for _ in range(nframes)) super().__init__(frames=frames) @@ -886,17 +889,17 @@ class FramesNoise(Frames): """ - def __init__(self, nframes,width, height, dtype='uint8'): + def __init__(self, nframes, height, width, dtype='uint8'): """Creates an iterator that yields gray frames. Parameters ---------- nframes : int Number of frames to be produced. - width : int - Width of frame. height : int Height of frame. + width : int + Width of frame. dtype : numpy dtype, default='uint8' Dtype of frame. @@ -911,15 +914,15 @@ def __init__(self, nframes,width, height, dtype='uint8'): super().__init__(frames=frames) -def framegray(width, height, value=0, dtype='uint8'): +def framegray(height, width, value=0, dtype='uint8'): """Creates a gray frame. Parameters ---------- - width : int - Width of frame. height : int Height of frame. + width : int + Width of frame. value : int, optional Fill value of frame. The default (0) is black. dtype : numpy dtype, default='uint8' @@ -933,15 +936,15 @@ def framegray(width, height, value=0, dtype='uint8'): return np.ones((height, width), dtype=dtype) * value -def framecolor(width, height, color=(0, 0, 0), dtype='uint8'): +def framecolor(height, width, color=(0, 0, 0), dtype='uint8'): """Creates a color frame. Parameters ---------- - width : int - Width of frame. height : int Height of frame. + width : int + Width of frame. color : tuple of ints, optional Fill value of frame (r, g, b). The default (0, 0, 0) color is black. dtype : numpy dtype, default='uint8' @@ -952,19 +955,18 @@ def framecolor(width, height, color=(0, 0, 0), dtype='uint8'): numpy ndarray """ - return np.ones((height, width, 3), dtype=dtype) * np.asanyarray(color, - dtype=dtype) + return np.ones((height, width, 3), dtype=dtype) * np.asanyarray(color, dtype=dtype) -def framenoise(width, height, dtype='uint8'): +def framenoise(height, width, dtype='uint8'): """Creates a noise frame. Parameters ---------- - width : int - Width of frame. height : int Height of frame. + width : int + Width of frame. dtype : numpy dtype, default='uint8' Dtype of frame. diff --git a/birdwatcher/movementdetection.py b/birdwatcher/movementdetection.py deleted file mode 100644 index 22a9aba..0000000 --- a/birdwatcher/movementdetection.py +++ /dev/null @@ -1,358 +0,0 @@ -from pathlib import Path -import numpy as np -import darr - -from .video import VideoFileStream -from .coordinatearrays import create_coordarray -from .backgroundsubtraction import BackgroundSubtractorMOG2, \ - BackgroundSubtractorKNN, BackgroundSubtractorLSBP -from .utils import derive_filepath -from ._version import get_versions - -__all__ = ['batch_detect_movement', 'detect_movement', 'detect_movementmog2', - 'detect_movementknn', 'detect_movementlsbp', - 'create_movementvideo'] - - -def _f(rar): - rar.archive(overwrite=True) - darr.delete_raggedarray(rar) - - -def batch_detect_movement(videofilepaths, bgs, nprocesses=6, morphologyex=2, - color=False, roi=None, nroi=None, analysispath='.', - overwrite=False, ignore_firstnframes=10, - resultvideo=False): - """The reason for having a special batch function, instead of just - applying functions in a loop, is that compression of coordinate results - takes a long time and is single-threaded. We therefore do this in - parallel. Use the `nprocesses` parameter to specify the number of cores - devoted to this. - - """ - from multiprocessing.pool import ThreadPool - - tobearchived = [] - for i, videofilepath in enumerate(videofilepaths): - cd, cc, cm = detect_movement(videofilepath, bgs=bgs, - morphologyex=morphologyex, color=color, - roi=roi, nroi=nroi, - analysispath=analysispath, - overwrite=overwrite, - ignore_firstnframes=ignore_firstnframes, - resultvideo=resultvideo) - tobearchived.append(cd) - if (len(tobearchived) == nprocesses) or (i == (len(videofilepaths) - 1)): - with ThreadPool(processes=nprocesses) as pool: - list([i for i in pool.imap_unordered(_f, tobearchived)]) - tobearchived = [] - - -def detect_movement(videofilestream, bgs, morphologyex=2, color=False, - roi=None, nroi=None, analysispath='.', - ignore_firstnframes=10, - overwrite=False, resultvideo=False): - """Detects movement based on a background subtraction algorithm. - - The backgound subtractor should be provided as a parameter. - - Parameters - ---------- - videofilestream : VideoFileStream - A Birdwatcher VideoFileStream object - bgs : BaseBackgroundSubtractor - Can be any instance of child from BaseBackgroundSubtractor. - Currently included in Birdwatcher are BackgroundSubtractorMOG2, - BackgroundSubtractorKNN, BackgroundSubtractorLSBP. - morphologyex : int, default=2 - Kernel size of MorphologeEx open processing. - color : bool, default=False - Should detection be done on color frames (True) or on gray frames - (False). - roi : (int, int, int, int), optional - Region of interest. Only look at this rectangular region. h1, - h2, w1, w2. - nroi : (int, int, int, int), optional - Not region of interest. Exclude this rectangular region. h1, - h2, w1, w2. - analysispath : Path or str, optional - Where to write results to. The default writes to the current working - directory. - ignore_firstnframes : int, default=10 - Do not provide coordinates for the first n frames. These often have - a lot of false positives. - overwrite : bool, default=False - Overwrite results or not. - resultvideo : bool, default=False - Automatically generate a video with results, yes or no. - - Returns - ------- - tuple of arrays (coordinates, coordinate count, coordinate mean) - These are Darr arrays that are disk-based. - - """ - if isinstance(videofilestream, VideoFileStream): - vfs = videofilestream - else: - raise TypeError(f"`videofilestream` parameter not a VideoFileStream " - f"object ({type(videofilestream)}).") - - Path(analysispath).mkdir(parents=True, exist_ok=True) - movementpath = Path(analysispath) / f'{vfs.filepath.stem}_movement' - Path(movementpath).mkdir(parents=True, exist_ok=True) - - metadata = {} - metadata['backgroundsegmentclass'] = str(bgs) - metadata['backgroundsegmentparams'] = bgs.get_params() - metadata['morphologyex'] = morphologyex - metadata['roi'] = roi - metadata['nroi'] = nroi - metadata['birdwatcherversion'] = get_versions()['version'] - - frames = (vfs.iter_frames(color=color) - .apply_backgroundsegmenter(bgs, roi=roi, nroi=nroi)) - if morphologyex is not None: - frames = frames.morphologyex(kernelsize=morphologyex) - - cd = frames.save_nonzero(movementpath / 'coords.darr', - metadata = metadata, - ignore_firstnframes = ignore_firstnframes, - overwrite = overwrite) - cc = darr.asarray(movementpath / 'coordscount.darr', cd.get_coordcount(), - metadata=metadata, overwrite=True) - cm = darr.asarray(movementpath / 'coordsmean.darr', cd.get_coordmean(), - metadata=metadata, overwrite=True) - - if resultvideo: - ovfilepath = Path(movementpath) / f'{ vfs.filepath.stem}_movement.mp4' - cframes = cd.iter_frames(nchannels=3, value=(0, 0, 255)) - (vfs.iter_frames().add_weighted(0.7, cframes, 0.8) - .draw_framenumbers() - .tovideo(ovfilepath, framerate=vfs.avgframerate, crf=25)) - return cd, cc, cm - - -def detect_movementknn(videofilestream, morphologyex=2, color=False, - roi=None, nroi=None, analysispath='.', - ignore_firstnframes=10, overwrite=False, - **kwargs): - """Detects movement based on a KNN background segmentation algorithm. - - The parameters for the algorithm should be provided as keyword arguments. - There are, with their defaults {'History': 5, 'kNNSamples': 10, - 'NSamples': 6, 'Dist2Threshold': 500, 'DetectShadows': False, - 'ShadowThreshold': 0.5, 'ShadowValue': 127} - - Parameters - ---------- - videofilestream : VideoFileStream - A Birdwatcher VideoFileStream object - morphologyex : int, default=2 - Kernel size of MorphologeEx open processing. - color : bool, default=False - Should detection be done on color frames (True) or on gray frames - (False). - roi : (int, int, int, int), optional - Region of interest. Only look at this rectangular region. h1, - h2, w1, w2. - nroi : (int, int, int, int), optional - Not region of interest. Exclude this rectangular region. h1, - h2, w1, w2. - analysispath : Path or str, optional - Where to write results to. The default writes to the current working - directory. - ignore_firstnframes : int, default=10 - Do not provide coordinates for the first n frames. These often have - a lot of false positives. - overwrite : bool, default=False - Overwrite results or not. - **kwargs : dict or additional keyword arguments - Parameters for the background segmentation algorithm. - - Returns - ------- - tuple of arrays (coordinates, coordinate count, coordinate mean) - These are Darr arrays that are disk-based. - - """ - bgs = BackgroundSubtractorKNN(**kwargs) - cd, cc, cm = detect_movement(videofilestream=videofilestream, - bgs=bgs, - morphologyex=morphologyex, - color=color, - roi=roi, - nroi=nroi, - analysispath=analysispath, - ignore_firstnframes=ignore_firstnframes, - overwrite=overwrite) - return cd, cc, cm - - -def detect_movementmog2(videofilestream, morphologyex=2, color=False, - roi=None, nroi=None, analysispath='.', - ignore_firstnframes=10, overwrite=False, - **kwargs): - """Detects movement based on a MOG2 background segmentation algorithm. - - The parameters for the algorithm should be provided as keyword arguments. - There are, with their defaults {'History': 5, - 'ComplexityReductionThreshold': 0.05, 'BackgroundRatio': 0.1, 'NMixtures': - 7, 'VarInit': 15, 'VarMin': 4, 'VarMax': 75, 'VarThreshold': 10, - 'VarThresholdGen': 9, 'DetectShadows': False, 'ShadowThreshold': 0.5, - 'ShadowValue': 127} - - Parameters - ---------- - videofilestream : VideoFileStream - A Birdwatcher VideoFileStream object - morphologyex : int, default=2 - Kernel size of MorphologeEx open processing. - color : bool, default=False - Should detection be done on color frames (True) or on gray frames - (False). - roi : (int, int, int, int), optional - Region of interest. Only look at this rectangular region. h1, - h2, w1, w2. - nroi : (int, int, int, int), optional - Not region of interest. Exclude this rectangular region. h1, - h2, w1, w2. - analysispath : Path or str, optional - Where to write results to. The default writes to the current working - directory. - ignore_firstnframes : int, default=10 - Do not provide coordinates for the first n frames. These often have - a lot of false positives. - overwrite : bool, default=False - Overwrite results or not. - **kwargs : dict or additional keyword arguments - Parameters for the background segmentation algorithm. - - Returns - ------- - tuple of arrays (coordinates, coordinate count, coordinate mean) - These are Darr arrays that are disk-based. - - """ - bgs = BackgroundSubtractorMOG2(**kwargs) - cd, cc, cm = detect_movement(videofilestream=videofilestream, - bgs=bgs, - morphologyex=morphologyex, - color=color, - roi=roi, - nroi=nroi, - analysispath=analysispath, - ignore_firstnframes=ignore_firstnframes, - overwrite=overwrite) - return cd, cc, cm - -def detect_movementlsbp(videofilestream, morphologyex=2, color=False, - roi=None, nroi=None, analysispath='.', - ignore_firstnframes=10, overwrite=False, - **kwargs): - """Detects movement based on a LSBP background segmentation algorithm. - - The parameters for the algorithm should be provided as keyword arguments. - There are, with their defaults {'mc': 0, 'nSamples': 20, 'LSBPRadius': 16, - 'Tlower': 2.0, 'Tupper': 32.0, 'Tinc': 1.0, 'Tdec': 0.05, 'Rscale': 10.0, - 'Rincdec': 0.005, 'noiseRemovalThresholdFacBG': 0.0004, - 'noiseRemovalThresholdFacFG': 0.0008, 'LSBPthreshold': 8, 'minCount': 2} - - Parameters - ---------- - videofilestream : VideoFileStream - A Birdwatcher VideoFileStream object - morphologyex : int, default=2 - Kernel size of MorphologeEx open processing. - color : bool, default=False - Should detection be done on color frames (True) or on gray frames - (False). - roi : (int, int, int, int), optional - Region of interest. Only look at this rectangular region. h1, - h2, w1, w2. - nroi : (int, int, int, int), optional - Not region of interest. Exclude this rectangular region. h1, - h2, w1, w2. - analysispath : Path or str, optional - Where to write results to. The default writes to the current working - directory. - ignore_firstnframes : int, default=10 - Do not provide coordinates for the first n frames. These often have - a lot of false positives. - overwrite : bool, default=False - Overwrite results or not. - **kwargs : dict or additional keyword arguments - Parameters for the background segmentation algorithm. - - Returns - ------- - tuple of arrays (coordinates, coordinate count, coordinate mean) - These are Darr arrays that are disk-based. - - """ - bgs = BackgroundSubtractorLSBP(**kwargs) - cd, cc, cm = detect_movement(videofilestream=videofilestream, - bgs=bgs, - morphologyex=morphologyex, - color=color, - roi=roi, - nroi=nroi, - analysispath=analysispath, - ignore_firstnframes=ignore_firstnframes, - overwrite=overwrite) - return cd, cc, cm - - -def create_movementvideo(videofilestream, coordinatearrays, - videofilepath=None, draw_mean=True, - draw_framenumbers=(2, 25), crf=17, scale=None): - """Create a nice video from the original video with movement detection - results superimposed. - - Parameters - ---------- - videofilestream : VideoFileStream - A Birdwatcher VideoFileStream object - coordinatearrays : CoordinateArrays - CoordinateArrays object with movement results. - videofilepath : Path or str, optional - Output path. If None, writes to filepath of videofilestream. - draw_mean : bool, default=True - Draw the mean of the coordinates per frame, or not. - draw_framenumbers : tuple, optional - Draw frame numbers. A tuple of ints indicates where to draw - them. The default (2, 25) draws numbers in the top left corner. - To remove the framenumbers use None. - crf : int, default=17 - Quality factor output video for ffmpeg. The default 17 is high - quality. Use 23 for good quality. - scale : tuple, optional - (width, height). The default (None) does not change width and height. - - Returns - ------- - VideoFileStream - Videofilestream object of the output video. - - """ - if isinstance(videofilestream, VideoFileStream): - vfs = videofilestream - else: - raise TypeError(f"`videofilestream` parameter not a VideoFileStream " - f"object ({type(videofilestream)}).") - if videofilepath is None: - videofilepath = derive_filepath(vfs.filepath, 'results', - suffix='.mp4') - frames = coordinatearrays.iter_frames(nchannels=3, value=(0, 0, 255)).add_weighted(0.8, vfs.iter_frames(), 0.7) - if draw_framenumbers is not None: - frames = frames.draw_framenumbers(org=draw_framenumbers) - if draw_mean: - centers = coordinatearrays.get_coordmean() - frames = frames.draw_circles(centers=centers, radius=6, color=(255, 100, 0), thickness=2, linetype=16, shift=0) - # centers_lp = np.array( - # [np.convolve(centers[:, 0], np.ones(7) / 7, 'same'), - # np.convolve(centers[:, 1], np.ones(7) / 7, 'same')]).T - # frames = frames.draw_circles(centers=centers_lp, radius=6, color=(100, 255, 0), thickness=2, linetype=16, shift=0) - vfs = frames.tovideo(videofilepath, framerate=vfs.avgframerate, crf=crf, - scale=scale) - return vfs \ No newline at end of file diff --git a/birdwatcher/movementdetection/__init__.py b/birdwatcher/movementdetection/__init__.py new file mode 100644 index 0000000..74618e6 --- /dev/null +++ b/birdwatcher/movementdetection/__init__.py @@ -0,0 +1,2 @@ +from .movementdetection import * +from .parameterselection import * \ No newline at end of file diff --git a/birdwatcher/movementdetection/movementdetection.py b/birdwatcher/movementdetection/movementdetection.py new file mode 100644 index 0000000..b4b3368 --- /dev/null +++ b/birdwatcher/movementdetection/movementdetection.py @@ -0,0 +1,273 @@ +from pathlib import Path + +import numpy as np +import darr + +import birdwatcher as bw +from birdwatcher.utils import derive_filepath + + +__all__ = ['batch_detect_movement', 'detect_movement', 'apply_settings', + 'create_movementvideo'] + + +default_settings = {'processing':{'color': False, # booleans only + 'blur': 0, # use '0' for no blur + 'morphologyex': True, # booleans only + 'resizebyfactor': 1}} # use '1' for no change in size + + +def _f(rar): + rar.archive(overwrite=True) + darr.delete_raggedarray(rar) + + +def batch_detect_movement(vfs_list, settings=None, startat=None, nframes=None, + roi=None, nroi=None, + bgs_type=bw.BackgroundSubtractorMOG2, + analysispath='.', ignore_firstnframes=10, + overwrite=False, resultvideo=False, nprocesses=6): + """The reason for having a special batch function, instead of just + applying functions in a loop, is that compression of coordinate results + takes a long time and is single-threaded. We therefore do this in + parallel. Use the `nprocesses` parameter to specify the number of cores + devoted to this. + + """ + from multiprocessing.pool import ThreadPool + + tobearchived = [] + for i, vfs in enumerate(vfs_list): + cd, cc, cm = detect_movement(vfs, settings=settings, startat=startat, + nframes=nframes, roi=roi, nroi=nroi, + bgs_type=bgs_type, + analysispath=analysispath, + ignore_firstnframes=ignore_firstnframes, + overwrite=overwrite, + resultvideo=resultvideo) + tobearchived.append(cd) + if (len(tobearchived) == nprocesses) or (i == (len(vfs_list) - 1)): + with ThreadPool(processes=nprocesses) as pool: + list([i for i in pool.imap_unordered(_f, tobearchived)]) + tobearchived = [] + + +def detect_movement(vfs, settings=None, startat=None, nframes=None, roi=None, + nroi=None, bgs_type=bw.BackgroundSubtractorMOG2, + analysispath='.', ignore_firstnframes=10, + overwrite=False, resultvideo=False): + """Detects movement based on a background subtraction algorithm. + + High-level function to perform movement detection with default parameters, + but also the option to modify many parameter settings. + + Parameters + ---------- + vfs : VideoFileStream + A Birdwatcher VideoFileStream object. + settings : {dict, dict}, optional + Dictionary with two dictionaries. One 'bgs_params' with the parameter + settings from the BackgroundSubtractor and another 'processing' + dictionary with settings for applying color, resizebyfactor, blur and + morphologyex manipulations. If None, the default settings of the + BackgroundSubtractor are used on grey color frames, including + morphological transformation to reduce noise. + startat : str, optional + If specified, start at this time point in the video file. You can use + two different time unit formats: sexagesimal + (HOURS:MM:SS.MILLISECONDS, as in 01:23:45.678), or in seconds. + nframes : int, optional + Read a specified number of frames. + roi : (int, int, int, int), optional + Region of interest. Only look at this rectangular region. h1, h2, w1, + w2. + nroi : (int, int, int, int), optional + Not region of interest. Exclude this rectangular region. h1, h2, w1, + w2. + bgs_type: BackgroundSubtractor, default=bw.BackgroundSubtractorMOG2 + This can be any of the BackgroundSubtractors in Birdwatcher, e.g. + BackgroundSubtractorMOG2, BackgroundSubtractorKNN, + BackgroundSubtractorLSBP. + analysispath : Path or str, optional + Where to write results to. The default writes to the current working + directory. + ignore_firstnframes : int, default=10 + Do not provide coordinates for the first n frames. These often have a + lot of false positives. + overwrite : bool, default=False + Overwrite results or not. + resultvideo : bool, default=False + Automatically generate a video with results, yes or no. + + Returns + ------- + tuple of arrays (coordinates, coordinate count, coordinate mean) + These are Darr arrays that are disk-based. + + """ + if not isinstance(vfs, bw.VideoFileStream): + raise TypeError(f"`vfs` parameter not a VideoFileStream object.") + + output_settings = {**bgs_type().get_params(), + **default_settings['processing']} # get flat dict + if settings is not None: + settings = {**settings['bgs_params'], **settings['processing']} + output_settings.update(settings) + + movementpath = Path(analysispath) / f'movement_{vfs.filepath.stem}' + Path(movementpath).mkdir(parents=True, exist_ok=True) + + metadata = {} + metadata['backgroundsegmentclass'] = str(bgs_type) + metadata['settings'] = output_settings + metadata['startat'] = startat + metadata['nframes'] = nframes + metadata['roi'] = roi + metadata['nroi'] = nroi + metadata['birdwatcherversion'] = bw.__version__ + + frames = apply_settings(vfs, output_settings, startat, nframes, roi, nroi, + bgs_type) + + cd = frames.save_nonzero(Path(movementpath) / 'coords.darr', + metadata = metadata, + ignore_firstnframes = ignore_firstnframes, + overwrite = overwrite) + cc = darr.asarray(Path(movementpath) / 'coordscount.darr', + cd.get_coordcount(), + metadata=metadata, overwrite=True) + cm = darr.asarray(Path(movementpath) / 'coordsmean.darr', + cd.get_coordmean(), + metadata=metadata, overwrite=True) + + if resultvideo: + create_movementvideo(vfs, cd, startat=startat, nframes=nframes, + videofilepath=Path(movementpath) / + 'movementvideo.mp4') + return cd, cc, cm + + +def apply_settings(vfs, settings, startat=None, nframes=None, roi=None, + nroi=None, bgs_type=bw.BackgroundSubtractorMOG2): + """Applies movement detection based on various parameter settings. + + The background subtractor should be provided as a parameter. + + Parameters + ---------- + vfs : VideoFileStream + A Birdwatcher VideoFileStream object. + settings : dict + Dictionary with parameter settings from the BackgroundSubtractor and + settings for applying color, resizebyfactor, blur and morphologyex + manipulations. + startat : str, optional + If specified, start at this time point in the video file. You can use + two different time unit formats: sexagesimal + (HOURS:MM:SS.MILLISECONDS, as in 01:23:45.678), or in seconds. + nframes : int, optional + Read a specified number of frames. + roi : (int, int, int, int), optional + Region of interest. Only look at this rectangular region. h1, h2, w1, + w2. + nroi : (int, int, int, int), optional + Not region of interest. Exclude this rectangular region. h1, h2, w1, + w2. + bgs_type: BackgroundSubtractor, default=bw.BackgroundSubtractorMOG2 + This can be any of the BackgroundSubtractors in Birdwatcher, e.g. + BackgroundSubtractorMOG2, BackgroundSubtractorKNN, + BackgroundSubtractorLSBP. + + Yields + ------ + Frames + Iterator that generates numpy array frames (height x width x color + channel). + + """ + frames = vfs.iter_frames(startat=startat, nframes=nframes, + color=settings['color']) + + if settings['resizebyfactor'] != 1: + val = settings['resizebyfactor'] + frames = frames.resizebyfactor(val,val) + + if settings['blur']: + val = settings['blur'] + frames = frames.blur((val,val)) + + bgs_params = bgs_type().get_params() + bgs_params.update((k, v) for k, v in settings.items() if k in bgs_params) + bgs = bgs_type(**bgs_params) + + frames = frames.apply_backgroundsegmenter(bgs, learningRate=-1, + roi=roi, nroi=nroi) + if settings['morphologyex']: + frames = frames.morphologyex(morphtype='open', kernelsize=2) + + return frames + + +def create_movementvideo(vfs, coords, startat=None, nframes=None, + videofilepath=None, draw_mean=True, + draw_framenumbers=(2, 25), crf=17, scale=None): + """Create a nice video from the original video with movement detection + results superimposed. + + Parameters + ---------- + vfs : VideoFileStream + A Birdwatcher VideoFileStream object + coords : CoordinateArrays + CoordinateArrays object with movement results. + startat : str, optional + If specified, start at this time point in the video file. You can use + two different time unit formats: sexagesimal + (HOURS:MM:SS.MILLISECONDS, as in 01:23:45.678), or in seconds. + nframes : int, optional + Read a specified number of frames. + videofilepath : Path or str, optional + Output path. If None, writes to filepath of videofilestream. + draw_mean : bool, default=True + Draw the mean of the coordinates per frame, or not. + draw_framenumbers : tuple, optional + Draw frame numbers. A tuple of ints indicates where to draw + them. The default (2, 25) draws numbers in the top left corner. + To remove the framenumbers use None. + crf : int, default=17 + Quality factor output video for ffmpeg. The default 17 is high + quality. Use 23 for good quality. + scale : tuple, optional + (width, height). The default (None) does not change width and height. + + Returns + ------- + VideoFileStream + Videofilestream object of the output video. + + """ + if not isinstance(vfs, bw.VideoFileStream): + raise TypeError(f"`vfs` parameter not a VideoFileStream object.") + + if videofilepath is None: + videofilepath = derive_filepath(vfs.filepath, 'results', + suffix='.mp4') + + videoframes = vfs.iter_frames(startat=startat, nframes=nframes) + frames = (coords.iter_frames(nchannels=3, value=(0, 0, 255)) + .add_weighted(0.8, videoframes, 0.7)) + + if draw_framenumbers is not None: + frames = frames.draw_framenumbers(org=draw_framenumbers) + if draw_mean: + centers = coords.get_coordmean() + frames = frames.draw_circles(centers=centers, radius=6, + color=(255, 100, 0), thickness=2, + linetype=16, shift=0) + # centers_lp = np.array( + # [np.convolve(centers[:, 0], np.ones(7) / 7, 'same'), + # np.convolve(centers[:, 1], np.ones(7) / 7, 'same')]).T + # frames = frames.draw_circles(centers=centers_lp, radius=6, color=(100, 255, 0), thickness=2, linetype=16, shift=0) + vfs = frames.tovideo(videofilepath, framerate=vfs.avgframerate, crf=crf, + scale=scale) + return vfs \ No newline at end of file diff --git a/birdwatcher/movementdetection/parameterselection.py b/birdwatcher/movementdetection/parameterselection.py new file mode 100644 index 0000000..2446924 --- /dev/null +++ b/birdwatcher/movementdetection/parameterselection.py @@ -0,0 +1,406 @@ +"""This module contains objects and functions helpfull for determining which settings result in optimal movement detection. + +""" + +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import seaborn as sns +import matplotlib.pyplot as plt + +import birdwatcher as bw +import birdwatcher.movementdetection as md +from birdwatcher.utils import product_dict + + +__all__ = ['ParameterSelection', 'apply_all_parameters', + 'load_parameterselection', 'product_dict'] + + +class ParameterSelection(): + """A Pandas dataframe with movement detection results of various parameter + settings associated with a (fragment of a) Videofilestream. + + """ + # colors in BGR + colors = [('orange', [0, 100, 255]), + ('blue', [255, 0, 0]), + ('red', [0, 0, 255]), + ('lime', [0, 255, 0]), + ('cyan', [255, 255, 0]), + ('magenta', [255, 0, 255])] + + def __init__(self, df, videofilepath, bgs_type, + startat, nframes, roi, nroi, path=None): + self.df = df + self.vfs = bw.VideoFileStream(videofilepath) + self.bgs_type = bgs_type + self.startat = startat + self.nframes = nframes + self.roi = roi + self.nroi = nroi + self.path = path + + def get_info(self): + return {'vfs': str(self.vfs.filepath), + 'bgs_type': self.bgs_type, + 'startat': self.startat, + 'nframes': self.nframes, + 'roi': self.roi, + 'nroi': self.nroi} + + def get_videofragment(self): + """Returns video fragment as Frames. + + NOTE: the whole frames are returned. If a region of interest (roi or + nroi) is specified, this is not visible in the videofragment. + + """ + return self.vfs.iter_frames(startat=self.startat, + nframes=self.nframes) + + def get_parameters(self, selection='multi_only'): + """Returns the parameter settings used for movement detection. + + Parameters + ---------- + selection : {'all', multi_only'} + Specify which selection of parameters is returned: + all : returns all parameters and their settings. + multi_only : returns only parameters for which multiple values + have been used to run movement detection. + + Returns + ------ + dict + With parameters as keys, and each value contains a list of the + settings used for movement detection. + + """ + paramkeys = (set(self.df.columns) - + set(['framenumber', 'pixel', 'coords'])) + all_parameters = {k:list(self.df[k].unique()) for k in paramkeys} + + if selection == 'all': + return all_parameters + elif selection == 'multi_only': + return {k:all_parameters[k] for k in paramkeys if + len(all_parameters[k])>1} + else: + raise Exception(f"'{selection}' is not recognized. Please " + "choose between 'all' and 'multi_only'.") + + def plot_parameters(self, rows, cols, default_values): + """Returns a figure from seaborn with subplots. + + Usefull to look at different location detection results from two + parameters with various values. + + Parameters + ---------- + rows : str + One parameter tested with multiple values. + cols : str + A second parameter tested with multiple values. + default_values : dict + All parameters that are tested with multiple settings, should be + added to a dictionary with each parameter as key, and the default + as value. + + Returns + ------ + FacedGrid + A seaborn object managing multiple subplots. + + """ + self._check_multi_only(default_values) + other_values = {key:default_values[key] for key in + default_values.keys()-[rows, cols]} + df_selection = self._select_data(**other_values) + + # plot with seaborn + g = sns.relplot(x="framenumber", y="pixel", hue="coords", style=None, + col=cols, row=rows, kind="line", data=df_selection, + height=3, aspect=2) + g.figure.suptitle(str(other_values), fontsize=15, x=0.51, y=1.05) + + return g + + def batch_plot_parameters(self, default_values, overwrite=False): + """Saves multiple figures with subplots of all combinations of + parameters. + + The figures are saved in a folder 'figures' in the same directory as + the associated ParameterSelection file. Multiple rounds of plotting + with different default values can be easily saved in new folders with + a number added to the foldername. + + Parameters + ---------- + default_values : dict + All parameters that are tested with multiple settings, should be + added to a dictionary with each parameter as key, and the default + as value. + overwrite : bool, default=False + If False, an integer number (1,2,3,etc.) will be added as suffix + to the foldername, if the filepath already exists. + + """ + path = self._create_path(self.path, 'figures', overwrite) + + # save default values as txt file + with open(path / 'default_values.txt', 'w') as f: + for key, val in default_values.items(): + f.write(f"{key}: {val}\n") + + # plot and save each combination of two parameters + settings = self.get_parameters('multi_only') + for rows, _ in settings.items(): + for cols, _ in settings.items(): + if rows != cols: + filename = f'{self.vfs.filepath.stem}_{rows}_{cols}.png' + g = self.plot_parameters(rows, cols, default_values) + g.savefig(path / filename) + plt.close(g.figure) + print(f"The figures are saved in {path}") + + def draw_multiple_circles(self, all_settings, radius=60, thickness=2): + """Returns a Frames object with circles on the videofragment. + + It is possible to plot multiple circles on the videofragment to see + the results from different parameter settings. + + Parameters + ---------- + settings : dict + All parameters that are tested with multiple settings, should be + added to a dictionary with each parameter as key, and the value(s) + that should be superimposed on the video added as list. + radius : int, default=60 + Radius of circle. + thickness : int, default=2 + Line thickness. + + Returns + ------- + Frames, DataFrame + Iterator that generates frames with multiple circles, and a Pandas + DataFrame with the settings for each color of the circles. + + """ + self._check_multi_only(all_settings) + self._check_number_of_colorcombinations(all_settings) + + frames = self.get_videofragment().draw_framenumbers() + colorspecs = {} + for i, settings in enumerate(product_dict(**all_settings)): + colorspecs[self.colors[i][0]] = settings + df_selection = self._select_data(**settings) + iterdata = (df_selection.set_index(['framenumber', 'coords']) + .loc[:, 'pixel'].unstack().values) + frames = frames.draw_circles(iterdata, radius=radius, + color=self.colors[i][1], + thickness=thickness) + + return frames, pd.DataFrame(colorspecs) + + def save_parameters(self, path, foldername=None, overwrite=False): + """Save results of all parameter settings as .csv file. + + Often several rounds of parameter selection per videofragment will be + done with different parameter settings. For this, the same foldername + could be used, in which case a number is added automatically as suffix + to display the round. + + Parameters + ---------- + path : str + Path to disk-based directory that should be written to. + foldername : str, optional + Name of the folder the data should be written to. + overwrite : bool, default=False + If False, an integer number (1,2,3,etc.) will be added as suffix + to the foldername, if the filepath already exists. + + """ + if foldername is None: + foldername = f'params_{self.vfs.filepath.stem}' + path = self._create_path(path, foldername, overwrite) + self.path = str(path) + + # add extra information to saved .csv file + json_info = json.dumps(self.get_info()) + self.df.to_csv(path / 'parameterselection.csv', index_label=json_info) + + # save parameter settings as readme file + with open(path / 'readme.txt', 'w') as f: + for key, val in self.get_parameters('all').items(): + f.write(f"{key}: {val}\n") + + + def _create_path(self, path, foldername, overwrite): + """Useful for creating a path with a number added as suffix in case + the folder already exists. + + Parameters + ---------- + path : str + Path to disk-based directory that should be written to. + foldername : str, optional + Name of the folder the data should be written to. + overwrite : bool, default=False + If False, an integer number (1,2,3,etc.) will be added as suffix + to the foldername, if the filepath already exists. + + """ + path = Path(path) / foldername + + if not overwrite: + i = 1 + while path.exists(): + i += 1 + path = path.parent / f'{foldername}_{i}' + Path(path).mkdir(parents=True, exist_ok=overwrite) + + return path + + def _select_data(self, **kwargs): + """Returns a copy with a selection of the dataframe. + + **kwargs can be a dictionary, with keys matching column names. The + value of each key will be selected from a copy of the dataframe. + + """ + df = self.df.copy() + for key, value in kwargs.items(): + df = df.loc[df[key]==value] + + return df + + def _check_multi_only(self, inputdict): + """Check if all parameters that are tested with multiple settings, are + included in the inputdic. + + """ + multi_only = self.get_parameters('multi_only') + if multi_only.keys() != inputdict.keys(): + raise Exception("Make sure the input dictionary contains all keys" + f" with multiple values: {multi_only.keys()}") + + def _check_number_of_colorcombinations(self, settings): + """Check if the number of settings combinations does not exceed the + number of possible colors. + + """ + n_combinations = len(list(product_dict(**settings))) + n_colors = len(self.colors) + if n_combinations > n_colors: + raise Exception( + f"The number of settings combinations is {n_combinations}, " + f"but a maximum of {n_colors} circles can be plotted. Reduce " + "the number of settings combinations, or add new colors to " + "the class attribute 'colors' to be able to plot more " + "circles.") + + +def apply_all_parameters(vfs, all_settings, startat=None, nframes=None, + roi=None, nroi=None, + bgs_type=bw.BackgroundSubtractorMOG2, + reportprogress=50): + """Run movement detection with each set of parameters. + + Parameters + ---------- + vfs : VideoFileStream + A Birdwatcher VideoFileStream object. + all_settings : {dict, dict} + Dictionary with two dictionaries. One 'bgs_params' with the parameter + settings from the BackgroundSubtractor and another 'processing' + dictionary with settings for applying color, resizebyfactor, blur and + morphologyex manipulations. + startat : str, optional + If specified, start at this time point in the video file. You can use + two different time unit formats: sexagesimal + (HOURS:MM:SS.MILLISECONDS, as in 01:23:45.678), or in seconds. + nframes : int, optional + Read a specified number of frames. + roi : (int, int, int, int), optional + Region of interest. Only look at this rectangular region. h1, + h2, w1, w2. + nroi : (int, int, int, int), optional + Not region of interest. Exclude this rectangular region. h1, + h2, w1, w2. + bgs_type: BackgroundSubtractor + This can be any of the BackgroundSubtractors in Birdwatcher, e.g. + BackgroundSubtractorMOG2, BackgroundSubtractorKNN, + BackgroundSubtractorLSBP. + reportprogress: int or bool, default=50 + The input integer represents how often the progress of applying each + combination of settings is printed. Use False, to turn off + reportprogress. + + """ + if reportprogress: + import datetime + starttime = datetime.datetime.now() + n = 0 + + list_with_dfs = [] + all_combinations = product_dict(**all_settings['bgs_params'], + **all_settings['processing']) + + for settings in all_combinations: + frames = md.apply_settings(vfs, settings, startat, nframes, roi, nroi, + bgs_type) + + # find mean of nonzero coordinates + coordinates = frames.find_nonzero() + coordsmean = np.array([c.mean(0) if c.size>0 else (np.nan, np.nan) for + c in coordinates]) + + # add results as pandas DataFrame + settings['coords'] = ['x', 'y'] + columns = pd.MultiIndex.from_frame(pd.DataFrame(settings)) + df = pd.DataFrame(coordsmean, columns=columns) + list_with_dfs.append(df) + + if reportprogress: + n += 1 + if n % reportprogress == 0: + diff = datetime.datetime.now() - starttime + print(f'{n} combinations of settings applied in' + f' {str(diff).split(".")[0]} hours:min:sec') + + # create long-format DataFrame + df = pd.concat(list_with_dfs, axis=1) + df.index.name = 'framenumber' + df = (df.stack(list(range(df.columns.nlevels)), dropna=False) + .reset_index() # stack all column levels + .rename({0: 'pixel'}, axis=1)) + + return ParameterSelection(df, vfs.filepath, str(bgs_type), startat, + nframes, roi, nroi) + +def load_parameterselection(path): + """Load a parameterselection.csv file. + + Parameters + ---------- + path : str + Name of the directory where parameterselection.csv is saved. + + Returns + ------- + ParameterSelection + + """ + filepath = Path(path) / 'parameterselection.csv' + df = pd.read_csv(filepath, index_col=0, engine='python') + info = json.loads(df.index.names[0]) + df.index.name = None + + return ParameterSelection(df, info['vfs'], info['bgs_type'], + info['startat'], info['nframes'], + info['roi'], info['nroi'], path) \ No newline at end of file diff --git a/birdwatcher/plotting.py b/birdwatcher/plotting.py index bc5251a..083a5ec 100644 --- a/birdwatcher/plotting.py +++ b/birdwatcher/plotting.py @@ -5,12 +5,14 @@ """ +import cv2 as cv import matplotlib.pyplot as plt import matplotlib.patches as patches -import cv2 as cv + __all__ = ['imshow_frame'] + def imshow_frame(frame, fig=None, ax=None, figsize=None, cmap=None, draw_rectangle=None ): """Create an matplotlib image plot of the frame. diff --git a/birdwatcher/tests/__init__.py b/birdwatcher/tests/__init__.py index 7c28230..bea44f3 100644 --- a/birdwatcher/tests/__init__.py +++ b/birdwatcher/tests/__init__.py @@ -1,18 +1,21 @@ from unittest import TestLoader, TextTestRunner, TestSuite -from . import test_coordinatearrays +from . import test_videoinput from . import test_frameprocessing -from . import test_movementdetection +from . import test_backgroundsubtraction +from . import test_coordinatearrays from . import test_plotting -from . import test_videoinput +from . import test_movementdetection +from . import test_parameterselection +modules = [test_videoinput, test_frameprocessing, test_backgroundsubtraction, + test_coordinatearrays, test_plotting, test_movementdetection, + test_parameterselection] -modules = [test_coordinatearrays, test_frameprocessing, - test_movementdetection, test_plotting, test_videoinput] def test(verbosity=1): - suite =TestSuite() + suite = TestSuite() for module in modules: suite.addTests(TestLoader().loadTestsFromModule(module)) return TextTestRunner(verbosity=verbosity).run(suite) \ No newline at end of file diff --git a/birdwatcher/tests/test_backgroundsubtraction.py b/birdwatcher/tests/test_backgroundsubtraction.py new file mode 100644 index 0000000..8858094 --- /dev/null +++ b/birdwatcher/tests/test_backgroundsubtraction.py @@ -0,0 +1,58 @@ +import unittest +import birdwatcher as bw +from birdwatcher.frames import create_frameswithmovingcircle + + +class TestBackgroundSubtractorKNN(unittest.TestCase): + + def test_KNNdefaultinstantiation(self): + bw.BackgroundSubtractorKNN() + + def test_KNNparams(self): + bgs = bw.BackgroundSubtractorKNN(History=10) + self.assertEqual(bgs.get_params()['History'], 10) + + def test_KNNapply(self): + bgs = bw.BackgroundSubtractorKNN(History=10) + frames = create_frameswithmovingcircle(nframes=5, width=1080, + height=720) + for fg in frames.apply_backgroundsegmenter(bgs, + roi=(10, 710, 10, 500), + nroi=(20,30,20,30)): + pass + + +class TestBackgroundSubtractorMOG2(unittest.TestCase): + + def test_MOG2defaultinstantiation(self): + bw.BackgroundSubtractorMOG2() + + def test_MOG2params(self): + bgs = bw.BackgroundSubtractorMOG2(History=10) + self.assertEqual(bgs.get_params()['History'], 10) + + def test_MOG2apply(self): + bgs = bw.BackgroundSubtractorMOG2(History=10) + frames = create_frameswithmovingcircle(nframes=5, width=1080, + height=720) + for fg in frames.apply_backgroundsegmenter(bgs, + roi=(10, 710, 10, 500), + nroi=(20, 30, 20, 30)): + pass + + +class TestBackgroundSubtractorLSBP(unittest.TestCase): + + def test_LSBPdefaultinstantiation(self): + bw.BackgroundSubtractorLSBP() + + def test_LSBPparams(self): + bgs = bw.BackgroundSubtractorLSBP(nSamples=10) + self.assertEqual(bgs.get_params()['nSamples'], 10) + + def test_LSBPapply(self): + bgs = bw.BackgroundSubtractorLSBP() + frames = create_frameswithmovingcircle(nframes=5, width=1080, + height=720) + for fg in frames.apply_backgroundsegmenter(bgs): + pass \ No newline at end of file diff --git a/birdwatcher/tests/test_frameprocessing.py b/birdwatcher/tests/test_frameprocessing.py index c045442..f426eb6 100644 --- a/birdwatcher/tests/test_frameprocessing.py +++ b/birdwatcher/tests/test_frameprocessing.py @@ -9,11 +9,11 @@ from birdwatcher.frames import Frames, FramesColor, framecolor, framegray -colorlist = [framecolor(width=640, height=480, color=(0,0,0)), - framecolor(width=640, height=480, color=(1,1,1))] +colorlist = [framecolor(height=480, width=640, color=(0,0,0)), + framecolor(height=480, width=640, color=(1,1,1))] -graylist = [framegray(width=640, height=480, value=0), - framegray(width=640, height=480, value=1)] +graylist = [framegray(height=480, width=640, value=0), + framegray(height=480, width=640, value=1)] class TestFrames(unittest.TestCase): @@ -26,6 +26,13 @@ def test_size(self): def test_color(self): for frame in FramesColor(5, height=480, width=640, color= (0,0,0)): self.assertEqual(frame.sum(), 0) + + def test_calcmeanframe(self): + frames = Frames([framecolor(height=480, width=640, color=(10,10,10)), + framecolor(height=480, width=640, color=(30,30,30))]) + meanframe = frames.calc_meanframe() + self.assertTupleEqual(meanframe.shape, (480, 640, 3)) + self.assertEqual(meanframe[0,0].sum(), 60) class TestPeekFrame(unittest.TestCase): diff --git a/birdwatcher/tests/test_movementdetection.py b/birdwatcher/tests/test_movementdetection.py index a7722b7..d86bd14 100644 --- a/birdwatcher/tests/test_movementdetection.py +++ b/birdwatcher/tests/test_movementdetection.py @@ -1,64 +1,37 @@ import unittest import tempfile -import birdwatcher as bw import shutil -import time from pathlib import Path -from birdwatcher.frames import create_frameswithmovingcircle - - -class TestBackgroundSubtractorKNN(unittest.TestCase): - - def test_KNNdefaultinstantiation(self): - bw.BackgroundSubtractorKNN() - - def test_KNNparams(self): - bgs = bw.BackgroundSubtractorKNN(History=10) - self.assertEqual(bgs.get_params()['History'], 10) - - def test_KNNapply(self): - bgs = bw.BackgroundSubtractorKNN(History=10) - frames = create_frameswithmovingcircle(nframes=5, width=1080, - height=720) - for fg in frames.apply_backgroundsegmenter(bgs, roi=(10, 710, 10, 500), - nroi=(20,30,20,30)): - pass - - -class TestBackgroundSubtractorMOG2(unittest.TestCase): - def test_MOG2defaultinstantiation(self): - bw.BackgroundSubtractorMOG2() - - def test_MOG2params(self): - bgs = bw.BackgroundSubtractorMOG2(History=10) - self.assertEqual(bgs.get_params()['History'], 10) - - - def test_MOG2apply(self): - bgs = bw.BackgroundSubtractorMOG2(History=10) - frames = create_frameswithmovingcircle(nframes=5, width=1080, - height=720) - for fg in frames.apply_backgroundsegmenter(bgs, roi=(10, 710, 10, 500), - nroi=(20, 30, 20, 30)): - pass +import birdwatcher as bw +import birdwatcher.movementdetection as md -class TestBackgroundSubtractorLSBP(unittest.TestCase): +settings = {'bgs_params': {'History': 12, + 'ComplexityReductionThreshold': 0.05, + 'BackgroundRatio': 0.1, + 'NMixtures': 7, + 'VarInit': 15, + 'VarMin': 10, + 'VarMax': 75, + 'VarThreshold': 70, + 'VarThresholdGen': 9, + 'DetectShadows': False, + 'ShadowThreshold': 0.5, + 'ShadowValue': 0}, + 'processing': {'color': False, + 'resizebyfactor': 1, + 'blur': 10, + 'morphologyex': True}} - def test_LSBPdefaultinstantiation(self): - bw.BackgroundSubtractorLSBP() - def test_LSBPparams(self): - bgs = bw.BackgroundSubtractorLSBP(nSamples=10) - self.assertEqual(bgs.get_params()['nSamples'], 10) +class TestApplySettingst(unittest.TestCase): - def test_LSBPapply(self): - bgs = bw.BackgroundSubtractorLSBP() - frames = create_frameswithmovingcircle(nframes=5, width=1080, - height=720) - for fg in frames.apply_backgroundsegmenter(bgs): - pass + def test_applysettings(self): + vfs = bw.testvideosmall() + settings_flat = {**settings['bgs_params'], **settings['processing']} + frames = md.apply_settings(vfs, settings_flat) + self.assertIsInstance(frames, bw.Frames) class TestDetectMovement(unittest.TestCase): @@ -72,21 +45,23 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tempdirname1) - def test_MOG2(self): - bgs = bw.BackgroundSubtractorMOG2(History=2) - bw.detect_movement(self.vfs, bgs=bgs, - analysispath=self.tempdirname1, overwrite=True) - - def test_KNN(self): - bgs = bw.BackgroundSubtractorKNN(History=2) - bw.detect_movement(self.vfs, bgs=bgs, - analysispath=self.tempdirname1, overwrite=True) - - def test_LSBP(self): - bgs = bw.BackgroundSubtractorLSBP(History=2) - bw.detect_movement(self.vfs, bgs=bgs, - analysispath=self.tempdirname1, - overwrite=True) + def test_detectmovement(self): + cd, _, _ = md.detect_movement(self.vfs, nframes=200, + analysispath=self.tempdirname1, + overwrite=True) + self.assertIsInstance(cd, bw.CoordinateArrays) + + def test_movementsettings(self): + cd, _, _ = md.detect_movement(self.vfs, settings=settings, + nframes=200, + analysispath=self.tempdirname1, + overwrite=True) + self.assertEqual(cd.metadata['settings']['History'], 12) + self.assertEqual(cd.metadata['settings']['blur'], 10) + + def test_exception(self): + with self.assertRaises(TypeError): + md.detect_movement('not_a_videofilestream_object') class TestBatchDetectMovement(unittest.TestCase): @@ -104,7 +79,6 @@ def test_batchdetection(self): p1 = self.vfs p2 = p1.iter_frames().tovideo(self.tempdirname1 / 'even2.mp4', framerate=p1.avgframerate) - bgs = bw.BackgroundSubtractorKNN(History=2) - bw.batch_detect_movement([p1,p2], bgs=bgs, - analysispath=self.tempdirname1, overwrite=True) - + md.batch_detect_movement([p1,p2], nframes=200, + analysispath=self.tempdirname1, + overwrite=True) \ No newline at end of file diff --git a/birdwatcher/tests/test_parameterselection.py b/birdwatcher/tests/test_parameterselection.py new file mode 100644 index 0000000..4ba15c8 --- /dev/null +++ b/birdwatcher/tests/test_parameterselection.py @@ -0,0 +1,88 @@ +import unittest +import tempfile +import shutil +from pathlib import Path + +import pandas as pd +import matplotlib.pyplot as plt +import seaborn as sns + +import birdwatcher as bw +import birdwatcher.movementdetection as md + + +settings = {'bgs_params': {'History': [4, 12], + 'ComplexityReductionThreshold': [0.05], + 'BackgroundRatio': [0.1], + 'NMixtures': [7], + 'VarInit': [15], + 'VarMin': [4], + 'VarMax': [75], + 'VarThreshold': [10], + 'VarThresholdGen': [9], + 'DetectShadows': [False], + 'ShadowThreshold': [0.5], + 'ShadowValue': [0]}, + 'processing': {'color': [True], + 'resizebyfactor': [1], + 'blur': [0,], + 'morphologyex': [True]}} + + +class TestParameterSelection(unittest.TestCase): + + def setUp(self): + self.tempdirname1 = Path(tempfile.mkdtemp()) + try: + self.params = md.apply_all_parameters(bw.testvideosmall(), + settings, nframes=200) + except: + self.tearDown() + raise + + def tearDown(self): + shutil.rmtree(self.tempdirname1) + + def test_attributes(self): + self.assertIsInstance(self.params, md.ParameterSelection) + self.assertIsInstance(self.params.df, pd.DataFrame) + self.assertIsInstance(self.params.vfs, bw.VideoFileStream) + self.assertEqual(self.params.bgs_type, + str(bw.BackgroundSubtractorMOG2)) + self.assertEqual(self.params.startat, None) + self.assertEqual(self.params.nframes, 200) + self.assertEqual(self.params.path, None) + + def test_getvideofragment(self): + frames = self.params.get_videofragment() + self.assertIsInstance(frames, bw.Frames) + + def test_getparams(self): + all = self.params.get_parameters('all') + settings_flat = {**settings['bgs_params'], **settings['processing']} + self.assertEqual(all, settings_flat) + multi_only = self.params.get_parameters('multi_only') + self.assertEqual(multi_only, {'History': [4, 12]}) + + def test_plotparams(self): + g = self.params.plot_parameters(rows='History', cols=None, + default_values={'History': 4}) + self.assertIsInstance(g, sns.axisgrid.FacetGrid) + self.assertEqual(g.axes.shape, (2,1)) + plt.close(g.figure) + + def test_drawcircles(self): + frames, colorspecs = self.params.draw_multiple_circles({'History': + [4,12]}) + self.assertIsInstance(frames, bw.Frames) + self.assertEqual(colorspecs.shape, (1,2)) + + def test_saveparams(self): + self.assertIsNone(self.params.path) + self.params.save_parameters(self.tempdirname1) + self.assertIsNotNone(self.params.path) + + def test_saveanotherparams(self): + self.params.save_parameters(self.tempdirname1) + self.params.save_parameters(self.tempdirname1) + self.assertEqual(self.params.path[-1], '2') \ No newline at end of file diff --git a/birdwatcher/tests/test_videoinput.py b/birdwatcher/tests/test_videoinput.py index a8da9c9..a2abd5a 100644 --- a/birdwatcher/tests/test_videoinput.py +++ b/birdwatcher/tests/test_videoinput.py @@ -37,6 +37,11 @@ def test_getframe(self): vf = bw.testvideosmall() frame = vf.get_frame(100) self.assertSequenceEqual(frame.shape, (720,1280,3)) + + def test_getframeat(self): + vf = bw.testvideosmall() + frame = vf.get_frameat('00:10.') + self.assertSequenceEqual(frame.shape, (720,1280,3)) def test_extractaudio(self): d = Path(tempfile.mkdtemp()) diff --git a/birdwatcher/utils.py b/birdwatcher/utils.py index 279b40a..f580e48 100644 --- a/birdwatcher/utils.py +++ b/birdwatcher/utils.py @@ -1,19 +1,26 @@ -import os,sys -import shutil -import tempfile import itertools +import os, sys import pathlib +import shutil +import tempfile +import time from contextlib import contextmanager -__all__ = ['derive_filepath', 'peek_iterable', 'datetimestring'] - - -import time def datetimestring(): return time.strftime('%Y%m%d%H%M%S') +def product_dict(**kwargs): + """Generates a Cartesian product of dictionary values. + + """ + keys = kwargs.keys() + vals = kwargs.values() + for instance in itertools.product(*vals): + yield dict(zip(keys, instance)) + + @contextmanager def tempdir(dirname='.', keep=False, report=False): """Yields a temporary directory which is removed when context is diff --git a/birdwatcher/video.py b/birdwatcher/video.py index fc31489..4682251 100644 --- a/birdwatcher/video.py +++ b/birdwatcher/video.py @@ -4,16 +4,20 @@ """ -import pathlib +from pathlib import Path + import numpy as np import cv2 as cv + from .ffmpeg import videofileinfo, iterread_videofile, count_frames, \ get_frame, get_frameat, extract_audio from .frames import frameiterator -from .utils import progress, walk_paths +from .utils import progress + __all__ = ['VideoFileStream', 'testvideosmall'] + class VideoFileStream(): """Video stream from file. @@ -40,7 +44,7 @@ class VideoFileStream(): def __init__(self, filepath, streamnumber=0): - self.filepath = fp = pathlib.Path(filepath) + self.filepath = fp = Path(filepath) self.streamnumber = streamnumber if not fp.exists(): raise FileNotFoundError(f'"{filepath}" does not exist') @@ -293,7 +297,7 @@ def testvideosmall(): """ file = 'zf20s_low.mp4' - path = pathlib.Path(__file__).parent / 'testvideos' / file + path = Path(__file__).parent / 'testvideos' / file return VideoFileStream(path) @@ -310,7 +314,7 @@ def walk_videofiles(dirpath, extension='.avi'): """ - dirpath = pathlib.Path(dirpath) + dirpath = Path(dirpath) if extension.startswith('.'): extension = extension[1:] for file in dirpath.rglob(f'*.{extension}'): diff --git a/docs/api.rst b/docs/api.rst index 20e4c69..dc5684d 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -13,6 +13,7 @@ Video input from file .. autofunction:: birdwatcher.testvideosmall + Frame processing ================ @@ -66,7 +67,6 @@ Frames methods :members: - Background subtraction ====================== @@ -94,35 +94,49 @@ LSBP :inherited-members: +Coordinate Arrays +================= + +.. automodule:: birdwatcher.coordinatearrays + +.. autoclass:: birdwatcher.CoordinateArrays + :members: + :inherited-members: + +.. autofunction:: open_archivedcoordinatedata + +.. autofunction:: create_coordarray + + Movement detection ================== -.. automodule:: birdwatcher.movementdetection +Movement detection contains top-level functionality. The classes, methods and functions provided in these submodules, are written to help the user get the most out of Birdwatcher. Also see the notebooks for examples how to use it and how to find the optimal parameter settings for movement detection. -.. autofunction:: birdwatcher.detect_movement +.. automodule:: birdwatcher.movementdetection.movementdetection -.. autofunction:: detect_movementmog2 +.. autofunction:: birdwatcher.movementdetection.detect_movement -.. autofunction:: detect_movementknn +.. autofunction:: birdwatcher.movementdetection.detect_movementmog2 -.. autofunction:: detect_movementlsbp +.. autofunction:: birdwatcher.movementdetection.detect_movementknn -.. autofunction:: create_movementvideo +.. autofunction:: birdwatcher.movementdetection.detect_movementlsbp +.. autofunction:: birdwatcher.movementdetection.create_movementvideo -Coordinate Arrays -================= +Parameter selection +------------------- -.. automodule:: birdwatcher.coordinatearrays +.. automodule:: birdwatcher.movementdetection.parameters -.. autoclass:: birdwatcher.CoordinateArrays +.. autoclass:: birdwatcher.movementdetection.ParameterSelection :members: :inherited-members: -.. autofunction:: open_archivedcoordinatedata - -.. autofunction:: create_coordarray +.. autofunction:: birdwatcher.movementdetection.apply_all_parameters +.. autofunction:: birdwatcher.movementdetection.load_parameterselection Plotting @@ -132,8 +146,9 @@ Plotting .. autofunction:: birdwatcher.plotting.imshow_frame + Utils -======== +===== .. automodule:: birdwatcher.utils diff --git a/docs/images/banner.gif b/docs/images/banner.gif index 2687aa2..378141d 100644 Binary files a/docs/images/banner.gif and b/docs/images/banner.gif differ diff --git a/docs/installation.rst b/docs/installation.rst index 6eeef3e..4056a39 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -42,7 +42,9 @@ The following dependencies are automatically taken care of when you install Birdwatcher from GitHub using the pip method above: - numpy +- pandas - matplotlib +- seaborn - darr - opencv-python - opencv-contrib-python diff --git a/docs/releasenotes.rst b/docs/releasenotes.rst index 33fd9f7..a7f7935 100644 --- a/docs/releasenotes.rst +++ b/docs/releasenotes.rst @@ -1,6 +1,44 @@ Release notes ============= +Version 0.4 +----------- + +Commits by Carien Mol and Gabriel Beckers. + +New submodule `parameterselection`, part of module `movementdetection`: + - `ParameterSelection` class with the results as Pandas DataFrame, and + methods to easily view and compare the results of various parameters. + - `apply_all_parameters` and `load_parameterselection` function. + - `parameterselection` notebook. + +Big change to `movementdetection` module: + - There is now one high-level function `detect_movement`, in which the + type of background subtraction algorithm that should be used can be added + as optional parameter. + - There is an easy-to-use default parameters setting for this function + that takes care of many of the pre- and postprocessing steps. + - There is also a optional `settings` parameters in which you can easily + set various processing steps, as wel as the parameters values for the + background subtraction algorithm, to be in full control of all settings. + +Tutorial with five notebooks: + - The notebooks of previous versions are modified into a more cohesive + tutorial. The first three notebooks demonstrate some basic functionalities + of Birdwatcher. The fourth and fifth notebook are specifically designed to + apply movement detection on the user's own videos. + +Other changes: + - `product_dict` function in utils module. + - some restrictions in what is imported automatically via the init file. + - modified existing tests and added new tests. + +Some corrections: + - switch frameheight/framewidth when calling framecolor or framegray. + - also include first frame when calculating the mean frame in + `calc_meanframe`. + + Version 0.3 ----------- @@ -9,12 +47,14 @@ Commits by Carien Mol and Gabriel Beckers. New methods: - `peek_frames` method for Frames for peeking the first frame - `show` method for Frames, VideoFileStream and CoordinateArrays - - 'save_nonzero' method for Frames to directly save nonzero pixel coordinates as CoordinateArrays - - 'get_coordmedian' method for CoordinateArrays - - 'edge_detection' method for Frames + - `save_nonzero` method for Frames to directly save nonzero pixel + coordinates as CoordinateArrays + - `get_coordmedian` method for CoordinateArrays + - `edge_detection` method for Frames Other changes: - - bug correction: switch frameheight/framewidth when initializing Frames object + - bug correction: switch frameheight/framewidth when initializing Frames + object - `find_nonzero` methods now can work with color frames - `apply_backgroundsegmenter` method on Frames for background segmentation - improved logging of processing steps diff --git a/docs/testing.rst b/docs/testing.rst index d7e846d..fd0d5d4 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -10,10 +10,9 @@ To run the test suite: >>> import birdwatcher as bw >>> bw.test() - ........................................ + .................................................. ---------------------------------------------------------------------- - Ran 40 tests in 21.621s + Ran 50 tests in 75.858s OK - - + \ No newline at end of file diff --git a/notebooks/videofile.ipynb b/notebooks/1_videoframes.ipynb similarity index 53% rename from notebooks/videofile.ipynb rename to notebooks/1_videoframes.ipynb index 7a5e0e0..cda95ad 100644 --- a/notebooks/videofile.ipynb +++ b/notebooks/1_videoframes.ipynb @@ -1,5 +1,14 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Welcome to the tutorial of Birdwatcher! \n", + "\n", + "In this first notebook, we'll introduce how you can load a video as VideoFileStream object. We also show how to look at the video frames, explain what a Frames object is, and how to apply some basic manipulations to the video frames." + ] + }, { "cell_type": "code", "execution_count": null, @@ -15,7 +24,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "**A short test video of a zebra finch is distributed with Birdwatcher**" + "### Create a video object" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A short videofile is distributed with Birdwatcher which you can use to run the notebooks. But you can also enter a pathname to your own videofile." ] }, { @@ -24,7 +40,7 @@ "metadata": {}, "outputs": [], "source": [ - "vfs = bw.testvideosmall()\n", + "vfs = bw.VideoFileStream(r'..\\videos\\zf20s_low.mp4')\n", "vfs # this is a VideoFileStream" ] }, @@ -41,7 +57,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Some useful properties and methods." + "Some useful properties and methods:" ] }, { @@ -63,14 +79,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "**Look at video frames**" + "### Look at video frames" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Look at video in separate window." + "Look at video in separate window:" ] }, { @@ -79,14 +95,14 @@ "metadata": {}, "outputs": [], "source": [ - "vfs.show() # press 'q' if you want to quit video before end" + "vfs.show() # press 'q' if you want to quit video before end" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Get a frame at a given time, say at 10 s." + "Get a frame at a given time, say at 10 s:" ] }, { @@ -128,7 +144,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "**Iterate Frames in video file**" + "### Iterate Frames in video file" ] }, { @@ -137,7 +153,8 @@ "metadata": {}, "outputs": [], "source": [ - "for frame in vfs.iter_frames(nframes=20): # only iterate first 20 frames\n", + "# only iterate first 20 frames\n", + "for frame in vfs.iter_frames(nframes=20):\n", " print(frame.shape, end = ', ')" ] }, @@ -172,7 +189,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Get information, including the methods applied to a Frames object." + "Get information, including the methods applied to a Frames object:" ] }, { @@ -188,7 +205,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Get a sneak preview of the manipulations." + "Get a sneak preview of the manipulations:" ] }, { @@ -205,7 +222,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Save as a video file." + "Save as a video file:" ] }, { @@ -230,7 +247,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Note that, because a Frames object is an iterator, the object will be exhausted after iterating through a Frames object, such as when writing to a video or calculating the mean frame (see below). If you try to apply another method on an 'empty' Frames iterator, this will raise a `StopIteration`. To set-up a new analysis, you will need to run the `iter_frames` method again to create a new Frames iterator." + "Note that, because a Frames object is an iterator, the object will be exhausted after iterating through a Frames object, such as when writing to a video or calculating the mean frame (see below). If you try to apply another method on an 'empty' Frames iterator, this will raise a `StopIteration`. To set-up a new analysis, you will need to create the Frames iterator again, e.g. by running the `iter_frames` method again." ] }, { @@ -243,6 +260,139 @@ "imshow_frame(frame)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Select video fragment" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To select a video fragment specify the start time and the number of frames when using `iter_frames`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "startat = '00:00:02' # in HOURS:MM:SS\n", + "nframes = 100\n", + "\n", + "frames = vfs.iter_frames(startat=startat, nframes=nframes)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using the number of frames to specificy the duration of the video fragment is precise. But, if you want to use the duration in time to select a video fragment, just approximate the number of frames:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "duration = 15 # in seconds\n", + "nframes = int(vfs.avgframerate*duration)\n", + "print(f'A video fragment of {duration} seconds with framerate {vfs.avgframerate} per second corresponds with {nframes} frames.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Look at the chosen video fragment:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "frames.show(framerate=vfs.avgframerate)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Select region of interest" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If there is a region in the video that you don't want to track, e.g. where the subject cannot move to, or a region with movement (such as a timer), you can exclude this part of each frame." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To specify the coordinates of the rectangle that should be excluded or included, it's useful to know the framewidth and height of the video:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Vertical coordinates h1, h2 = 0, {vfs.frameheight}\")\n", + "print(f\"Horizontal coordinates w1, w2 = 0, {vfs.framewidth}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Specify region of interest, so that only in this rectangular region movement detection is done. Using`imshow_frame` you can also see the coordinates along the horizontal and vertical axes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# specify h1, h2, w1, w2\n", + "roi = (10, 570, 10, 1250) # or choose None to use the whole frame\n", + "\n", + "# show roi in frame\n", + "frame = vfs.iter_frames().peek_frame()\n", + "imshow_frame(frame, draw_rectangle=roi)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Or, define a region that should be excluded:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# specify h1, h2, w1, w2\n", + "nroi = (600, 720, 0, 1280) # or choose None to use the whole frame\n", + "\n", + "# show nroi in frame\n", + "frame = vfs.iter_frames().peek_frame()\n", + "imshow_frame(frame, draw_rectangle=nroi)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -267,7 +417,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" + "version": "3.9.16" } }, "nbformat": 4, diff --git a/notebooks/movementdetection_detailed.ipynb b/notebooks/2_backgroundsubtraction.ipynb similarity index 63% rename from notebooks/movementdetection_detailed.ipynb rename to notebooks/2_backgroundsubtraction.ipynb index 136161c..a95b548 100644 --- a/notebooks/movementdetection_detailed.ipynb +++ b/notebooks/2_backgroundsubtraction.ipynb @@ -4,7 +4,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook shows the basic steps that are involved in movement detection. Much of what's in here can be encaspulated by higher-order functions or classes, but if you want to have full control over things, have a look at the following." + "To detect movements in a video, we use background subtraction. There are many different background subtraction algorithms, and each algorithm has various parameters. You can use the default parameter settings in Birdwatcher, but you can also modify the parameters.\n", + "\n", + "This notebook introduces some of background subtractors that are implemented in Birdwatcher. How to access them, and look at the various parameters they have. And how they are used for movement detection. Much of what's in here can be encaspulated by higher-order functions or classes, but if you want to have full control over things, have a look at the following." ] }, { @@ -20,7 +22,23 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Create a background subtractor object with suitable parameters" + "### Create a video object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vfs = bw.VideoFileStream(r'..\\videos\\zf20s_low.mp4')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a background subtractor object" ] }, { @@ -59,7 +77,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "You can use non-default paramaters by specifying them at intstantiation." + "In the docstrings, you can see the definition of each parameter:" ] }, { @@ -68,23 +86,23 @@ "metadata": {}, "outputs": [], "source": [ - "bgs = bw.BackgroundSubtractorMOG2(VarThreshold=70, NMixtures=8, History=3)" + "bw.BackgroundSubtractorMOG2?" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "bgs.get_params()" + "You can use non-default parameters by specifying them at intstantiation:" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "## Create a video object (see separate notebook on this)" + "bgs = bw.BackgroundSubtractorMOG2(VarThreshold=50, NMixtures=8, History=4)" ] }, { @@ -93,14 +111,14 @@ "metadata": {}, "outputs": [], "source": [ - "vfs = bw.testvideosmall()" + "bgs.get_params()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Do movement detection by hand based on this background subtractor" + "### Apply background subtractor to video Frames" ] }, { @@ -197,16 +215,41 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Create a coordinate array for storage of the results" + "### Another example" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We wrote the movement detection results (suprathreshold pixels) above to a video so that we could view the results. However, if you want to save the results for further analyses, it is much better to save them as a *coordinate array*.\n", - "\n", - "A coordinate array really is just a Darr ragged array (see separate library). This makes it easy to read the data in other environments, e.g. R. We save some metadata so that we later know what we did. Instead of saving the frames to video, we now detect non-zero pixel (i.e. foreground) coordinates, and save that to the coordinate array." + "Here's another example of a pipeline. We have added a blur manipulation to the videoframes before appying the background segmenter. Also, we have added a region of interest (roi)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "frames = (vfs.iter_frames(color=False)\n", + " .blur((10,10))\n", + " .apply_backgroundsegmenter(bgs, learningRate=-1, roi=(10, 570, 10, 1250))\n", + " .morphologyex(morphtype='open', kernelsize=2)\n", + " .tovideo('output/test2_MOG2.mp4', framerate=vfs.avgframerate))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a coordinate array for storage of the results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We wrote the movement detection results (suprathreshold pixels) above to a video so that we could view the results. However, if you want to save the results for further analyses, it is much better to detect non-zero pixel (i.e. foreground) coordinates, and save that to a `CoordinateArrays`. We also save some metadata." ] }, { @@ -219,7 +262,7 @@ "coordsarray = (vfs.iter_frames(color=False)\n", " .apply_backgroundsegmenter(bgs, learningRate=-1)\n", " .morphologyex(morphtype='open', kernelsize=2)\n", - " .save_nonzero(filepath='output/testcoords.darr',\n", + " .save_nonzero(filepath='output/testcoordsMOG.darr',\n", " metadata={'bgsparams': bgs.get_params(),\n", " 'morphologyex': ('open', 2),\n", " 'learningrate': -1,\n", @@ -234,14 +277,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Accessing coordinate arrays" + "For more information on `CoordinateArrays`, and how to access and view the them, just take a look at the next notebook." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The coordinate array can be accessed in other python sessions like so:" + "# A different algorithm: KNN" ] }, { @@ -250,7 +293,15 @@ "metadata": {}, "outputs": [], "source": [ - "coordsarray = bw.CoordinateArrays('output/testcoords.darr')" + "bgs = bw.BackgroundSubtractorKNN()\n", + "bgs.get_params()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A different background algorithm with different parameters. You can change these parameters the same way as in the example above." ] }, { @@ -259,7 +310,14 @@ "metadata": {}, "outputs": [], "source": [ - "coordsarray.metadata" + "bgs = bw.BackgroundSubtractorKNN(kNNSamples=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And run the whole pipeline again with the other background subtractor, and save the results as video." ] }, { @@ -268,14 +326,17 @@ "metadata": {}, "outputs": [], "source": [ - "coordsarray[100] # coordinates of the 101th frame " + "frames = (vfs.iter_frames(color=False)\n", + " .apply_backgroundsegmenter(bgs, learningRate=-1)\n", + " .morphologyex(morphtype='open', kernelsize=2)\n", + " .tovideo('output/test_KNN.mp4', framerate=vfs.avgframerate))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "You can also have a look at the results as a video in a separate window: (press 'q' to quit)" + "Look at results:" ] }, { @@ -284,21 +345,24 @@ "metadata": {}, "outputs": [], "source": [ - "coordsarray.show()" + "frames.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Take it together and look at a range of parameters" + "# Yet another algorithm: LSBP" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "As an example we'll look at a range of history settings. Note that we do not have to run the analysis pipeline twice in order to get both coordinate results and a video. We just create a coordinate array first. This can then be saved as a video for inspection." + "bgs = bw.BackgroundSubtractorLSBP()\n", + "bgs.get_params()" ] }, { @@ -307,22 +371,28 @@ "metadata": {}, "outputs": [], "source": [ - "%%time\n", - "vfs = bw.testvideosmall()\n", - "for history in (2,3,4):\n", - " bgs = bw.BackgroundSubtractorMOG2(History=history, VarThreshold=50)\n", - " basefilename = f'testcoords_hist{history}'\n", - " coordsarray = (vfs.iter_frames(color=False)\n", - " .apply_backgroundsegmenter(bgs, learningRate=-1)\n", - " .morphologyex(morphtype='open', kernelsize=2)\n", - " .save_nonzero(f'output/{basefilename}.darr', \n", - " metadata={'bgsparams': bgs.get_params(),\n", - " 'morphologyex': ('open', 2),\n", - " 'learningrate': -1,\n", - " 'avgframerate': vfs.avgframerate},\n", - " ignore_firstnframes=10,\n", - " overwrite=True))\n", - " coordsarray.tovideo(f'output/{basefilename}.mp4', framerate=vfs.avgframerate)" + "bgs = bw.BackgroundSubtractorLSBP(nSamples=10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "frames = (vfs.iter_frames(color=False)\n", + " .apply_backgroundsegmenter(bgs, learningRate=-1)\n", + " .morphologyex(morphtype='open', kernelsize=2)\n", + " .tovideo('output/test_LSBP.mp4', framerate=vfs.avgframerate))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "frames.show()" ] }, { @@ -349,7 +419,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.9.16" } }, "nbformat": 4, diff --git a/notebooks/3_coordinatearrays.ipynb b/notebooks/3_coordinatearrays.ipynb new file mode 100644 index 0000000..0394910 --- /dev/null +++ b/notebooks/3_coordinatearrays.ipynb @@ -0,0 +1,391 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The results of background subtraction are non-zero pixel (i.e. foreground) coordinates. These can saved as coordinate arrays. A coordinate array really is just a Darr ragged array (see separate library). This makes it easy to read the data in other environments, e.g. R. But also Birdwatcher provides functionality to access coordinate arrays, and look at the results in videos and plots." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import birdwatcher as bw\n", + "import birdwatcher.movementdetection as md\n", + "\n", + "from birdwatcher.plotting import imshow_frame\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a video object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vfs = bw.VideoFileStream(r'..\\videos\\zf20s_low.mp4')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create coordinate arrays" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the previous notebook, we showed how to use background subtraction and save the results into a coordinate array step-by-step. In this notebook, we will do movement detection the easy way, by using a high-level function, with the results saved as coordinate arrays.\n", + "\n", + "We will use the default settings, meaning that the default values for the background subtractor MOG2 and some pre- and postprocessing steps are automatically taken care of. For the example video this will work great, but for other videos you might need more control of the various settings. Have a look at the next notebooks to find out how to modify the default settings of this function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "coords, coordscount, coordsmean = md.detect_movement(vfs, bgs_type=bw.BackgroundSubtractorMOG2,\n", + " analysispath='output/', ignore_firstnframes=50, \n", + " overwrite=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This high-level function returns three arrays, which are disk-based Darr arrays. They can be very large and hence not fit in RAM. The coordinate arrays are saved within a 'movement' folder with the name of the VideoFileStream." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Accessing coordinate arrays" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The coordinate array can be accessed in other python sessions like so:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "coords = bw.CoordinateArrays('output/movement_zf20s_low/coords.darr')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "coords.metadata" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To access coordscount and coordsmean just run:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "coordscount = coords.get_coordcount()\n", + "coordsmean = coords.get_coordmean()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Look at `coords`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "'Coords' provide the detected foreground pixels in a ragged array. Therefore, you will have the coordinates of all 'movement pixels' per frame. You can have a look at the results as a video in a separate window: (press 'q' to quit)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "coords.show(framerate=vfs.avgframerate)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's look at the results of frame 131:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "coords[131]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can see a simple numpy array with the x, y coordinates of all 'movement pixels'." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can view the results as a black and white image by the `get_frame` method, which returns a frame instead of coordinates." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "frame = coords.get_frame(131)\n", + "imshow_frame(frame)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's look at the original frame as comparison:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "imshow_frame(vfs.get_frame(131))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can create a video of the results as well:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "coords.tovideo('output/zf20s_coords.mp4', framerate=vfs.avgframerate)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Or a selection of the results by indicating start and end frame numbers:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "coords.tovideo('output/zf20s_coords_selection.mp4', startframe=100, endframe=200, framerate=vfs.avgframerate)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you want to do more things before saving to video, just use `iter_frames` which turns it into a Frames object with many more methods. Make sure you use three color channels and set coordinates to value 255 if you want them white." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "frames = (coords.iter_frames(startframe=100, endframe=200, nchannels=3, value=255)\n", + " .draw_framenumbers()\n", + " .tovideo('output/zf20s_coords_selection_framenumbers.mp4', framerate=vfs.avgframerate))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Look at `coordscount`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The 'coordscount' shows the number of pixels that belong to the foreground, e.g. 'movement pixels', per frame. Thus, higher peaks means more movement." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.plot(coordscount)\n", + "plt.title('number of pixels above treshold')\n", + "plt.xlabel('frame number')\n", + "plt.ylabel('number of pixels')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Look at `coordsmean`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The 'coordsmean' shows the mean coordinates per frame. This could be used to look at the location of the subject during the video. The blue line shows the horizontal coordinates (left-rigth) and the orange line show the vertical coordinates (top-bottom)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.plot(coordsmean)\n", + "plt.title('coordinates of pixels above treshold')\n", + "plt.xlabel('frame number')\n", + "plt.ylabel('pixel coordinate')\n", + "plt.legend(['left-right', 'top-bottom'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also look at the mean coordinates in a video using the original frames + the mean coordinate per frame superimposed on it as a circle:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vfs_circle = (vfs.iter_frames()\n", + " .draw_framenumbers()\n", + " .draw_circles(coordsmean)\n", + " .tovideo('output/zf20s_coords_center.mp4', framerate=vfs.avgframerate))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It's also possible to change the settings of the circle, such as the radius and color:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vfs_circle = (vfs.iter_frames()\n", + " .draw_framenumbers()\n", + " .draw_circles(coordsmean, radius=50, color=(0, 100, 255))\n", + " .tovideo('output/zf20s_coords_center_orange.mp4', framerate=vfs.avgframerate))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There is also a high-level function that creates a similar video, but better! It will produce a video of the original one with coordinate results ánd the mean results superimposed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vfs_results = md.create_movementvideo(vfs, coords, videofilepath='output/movementvideoexample.mp4')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/4_parameterselection.ipynb b/notebooks/4_parameterselection.ipynb new file mode 100644 index 0000000..ed934aa --- /dev/null +++ b/notebooks/4_parameterselection.ipynb @@ -0,0 +1,684 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c25aad0b-650b-451b-b26d-3a118939caab", + "metadata": {}, + "source": [ + "This notebook can be used to play around with all options and various parameters settings of background subtraction algorithms when using Birdwatcher for movement detection, and see how this influences the results. You can use this notebook to optimize the settings for your own videos.\n", + "\n", + "**NOTE:** this notebook is specifically usefull for finding optimal settings for **location detection**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bbf1bc75-f463-48ff-b040-42acf78b1f9c", + "metadata": {}, + "outputs": [], + "source": [ + "import birdwatcher as bw\n", + "import birdwatcher.movementdetection as md\n", + "from birdwatcher.plotting import imshow_frame # birdwatcher has vizualization tools\n", + "\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "id": "fbd81528-f308-4d16-8820-948ab9379c34", + "metadata": {}, + "source": [ + "### Select video fragment" + ] + }, + { + "cell_type": "markdown", + "id": "7f1709e7-b855-452e-a1d3-369db3a82086", + "metadata": {}, + "source": [ + "Choose a short representative video fragment where the object of interest is moving quite a lot. See notebook 1 for more information of how to select a video fragment or a region of interest (roi and nroi)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f018f97e-9b49-496b-8947-5f19b1d4f558", + "metadata": {}, + "outputs": [], + "source": [ + "vfs = bw.VideoFileStream(r'..\\videos\\zf20s_low.mp4')\n", + "\n", + "# select video fragment\n", + "startat = '00:00:00' # in HOURS:MM:SS\n", + "nframes = 375 # is 15 seconds\n", + "\n", + "# specify h1, h2, w1, w2, or choose None to use the whole frame\n", + "roi = (10, 570, 10, 1250) # region of interest\n", + "nroi = (600, 720, 0, 1280) # nót region of interest" + ] + }, + { + "cell_type": "markdown", + "id": "c83aeb16-5c01-4b9e-8165-023420d82b5b", + "metadata": {}, + "source": [ + "Check roi and nroi in frame:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5c28204-e28d-4e38-b9f1-a0c85f07f274", + "metadata": {}, + "outputs": [], + "source": [ + "frame = vfs.iter_frames(startat=startat, nframes=nframes).peek_frame()\n", + "\n", + "if roi is not None:\n", + " frame = vfs.iter_frames(startat=startat, nframes=1).draw_text(['roi'], org=(roi[2],roi[1]))\n", + " imshow_frame(frame.peek_frame(), draw_rectangle=roi)\n", + "\n", + "if nroi is not None:\n", + " frame = vfs.iter_frames(startat=startat, nframes=1).draw_text(['nroi'], org=(nroi[2],nroi[1]))\n", + " imshow_frame(frame.peek_frame(), draw_rectangle=nroi)" + ] + }, + { + "cell_type": "markdown", + "id": "7c0682b4-7114-4ec6-975a-f2051c94fe0b", + "metadata": {}, + "source": [ + "Look at the chosen video fragment:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6077e13-76c9-4682-8df9-36c97a2475d7", + "metadata": {}, + "outputs": [], + "source": [ + "vfs.iter_frames(startat=startat, nframes=nframes).show(framerate=vfs.avgframerate)" + ] + }, + { + "cell_type": "markdown", + "id": "cf1d975d-9b23-4c22-80fa-78825e107720", + "metadata": {}, + "source": [ + "### Choose parameters" + ] + }, + { + "cell_type": "markdown", + "id": "5c654fe0-102b-4f2e-bf73-0c48ce3ddc8a", + "metadata": {}, + "source": [ + "In this example, we will use background subtractor MOG2. For more information of the background subtraction algorithms make sure to look at notebook 2. To choose parameters from another algorithm just modify the dictionary below with the appropriate parameters.\n", + "\n", + "To get a better feeling of the effect of the various background subtraction parameters, you can play around with different values. Also, some processing steps before or after performing background subtraction might improve location detection, and therefore, you can compare the settings of those as well.\n", + "\n", + "In the dictionary below, decide which settings you would like by adding one or more values in the list after each parameter." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd5f94bc-531e-4aea-aae3-0747a0481265", + "metadata": {}, + "outputs": [], + "source": [ + "settings = {'bgs_params': {'History': [3, 6, 12],\n", + " 'ComplexityReductionThreshold': [0.05],\n", + " 'BackgroundRatio': [0.1],\n", + " 'NMixtures': [7],\n", + " 'VarInit': [15],\n", + " 'VarMin': [4, 10],\n", + " 'VarMax': [75],\n", + " 'VarThreshold': [30, 70],\n", + " 'VarThresholdGen': [9],\n", + " 'DetectShadows': [False],\n", + " 'ShadowThreshold': [0.5],\n", + " 'ShadowValue': [0]},\n", + "\n", + " 'processing': {'color': [True, False], # booleans only\n", + " 'resizebyfactor': [1, (2/3)], # use '1' for no change in size\n", + " 'blur': [0, 10], # use '0' for no blur\n", + " 'morphologyex': [True, False]}} # booleans only\n", + "\n", + "all_combinations = list(md.product_dict(**settings['bgs_params'], **settings['processing']))\n", + "print(f'There are {len(all_combinations)} different combinations of settings to perform movement detection.')" + ] + }, + { + "cell_type": "markdown", + "id": "8385f831-e9da-4344-a42f-7050353bb213", + "metadata": {}, + "source": [ + "The higher the number of combinations, the longer the next step will take. Another option is to start by tweaking some parameters, and fine-tune in next rounds by running this notebook again with different settings." + ] + }, + { + "cell_type": "markdown", + "id": "51d86475-5e5e-4af8-ac8d-3f8b33f2599d", + "metadata": {}, + "source": [ + "### Run movemement detection per combination of settings" + ] + }, + { + "cell_type": "markdown", + "id": "91da47a1-4d92-4afa-976a-cae814d21be8", + "metadata": {}, + "source": [ + "Movement detection is done for each combination of settings, and the mean coordinate per frame is saved in a Pandas dataframe." + ] + }, + { + "cell_type": "markdown", + "id": "6ba40a81-3d53-4f08-a6a5-19e1aaf81bd3", + "metadata": {}, + "source": [ + "**WARNING:** This step might take a while, depending on the number of settings combinations! To shorten runtime, reduce the number of combinations and/or choose a shorter videofragment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "07aed424-15f0-4d44-8889-34a29749d83f", + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "params = md.apply_all_parameters(vfs, settings, bgs_type=bw.BackgroundSubtractorMOG2, \n", + " startat=startat, nframes=nframes, roi=roi, nroi=nroi,\n", + " reportprogress=25)\n", + "\n", + "params.save_parameters(f'output/')" + ] + }, + { + "cell_type": "markdown", + "id": "9178513b-89f6-43d4-a1bf-d1fa4153de68", + "metadata": {}, + "source": [ + "The results are saved in a folder with the name of the VideoFileStream. Also, a readme.txt file with the parameter settings is saved to quickly look up which settings were used." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "85908368-e917-4ae1-8e29-fc4b2f821c31", + "metadata": {}, + "outputs": [], + "source": [ + "params.path" + ] + }, + { + "cell_type": "markdown", + "id": "53282e6d-83b8-4e62-a820-43eb43fcec9a", + "metadata": {}, + "source": [ + "To get the optimal parameter settings, you'll probably do several rounds with a different combination of settings. Then, a new project folder is created with a number added as suffix to the foldername to display the round." + ] + }, + { + "cell_type": "markdown", + "id": "0926fe3b-2c4d-4976-b7e5-a029fe08afa2", + "metadata": {}, + "source": [ + "The output of applying all parameters is a `ParameterSelection` object, which contains information of the videofragment and the results of all setting combinations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4aeccdae-13b9-48fc-b00b-dffb9c493262", + "metadata": {}, + "outputs": [], + "source": [ + "params.get_info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e112709-ed44-48d2-a070-1b113d05c8e1", + "metadata": {}, + "outputs": [], + "source": [ + "params.df" + ] + }, + { + "cell_type": "markdown", + "id": "e33bdcd9-7535-4e48-8c3c-60c4bd7a95e2", + "metadata": {}, + "source": [ + "Here, you see a pandas dataframe with in the columns all parameters that are used to run movement detection. The rows show the specific value of each parameter and the resulted mean x,y coordinates per frame (NaN means there were no nonzero pixels found for that frame). " + ] + }, + { + "cell_type": "markdown", + "id": "dd445386-2274-4072-8f55-6b352cc4a35a", + "metadata": {}, + "source": [ + "### Load ParameterSelection" + ] + }, + { + "cell_type": "markdown", + "id": "6e505592-7ea4-48ca-9874-baefbefbb707", + "metadata": {}, + "source": [ + "You can run and save `apply_all_parameters`, and later look at the results by loading a `ParameterSelection` object like this:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "563df4aa-1df6-4f2a-a068-93e353b7ec71", + "metadata": {}, + "outputs": [], + "source": [ + "params = md.load_parameterselection(f'output\\params_zf20s_low')" + ] + }, + { + "cell_type": "markdown", + "id": "fe93a70f-7932-4642-a696-460651abcf2b", + "metadata": {}, + "source": [ + "Make sure the location of the original video where the `ParameterSelection` object is based on, has not changed. Then, it is also possible to load the associated videofilestream directly:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bcdcdcfe-8d01-4687-84df-fbf4e0644cf4", + "metadata": {}, + "outputs": [], + "source": [ + "params.vfs" + ] + }, + { + "cell_type": "markdown", + "id": "97d7e904-410d-4ae7-ac4a-27c14c0568b8", + "metadata": {}, + "source": [ + "Or watch the videofragment of which the `ParameterSelection` object is based on:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "668dc440-edbc-4884-9e05-1a4d12bb0694", + "metadata": {}, + "outputs": [], + "source": [ + "frames_fragment = params.get_videofragment()\n", + "frames_fragment.show()" + ] + }, + { + "cell_type": "markdown", + "id": "f6b30c40-2c2b-482f-a295-ac57ecb990fb", + "metadata": {}, + "source": [ + "To access the data as Pandas dataframe, run:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3c5605f5-4fd6-4cc1-87de-eb8340c1f6d8", + "metadata": {}, + "outputs": [], + "source": [ + "params.df" + ] + }, + { + "cell_type": "markdown", + "id": "2069949c-0544-4007-ab1a-592c6ad8d5df", + "metadata": {}, + "source": [ + "### Correction resizebyfactor" + ] + }, + { + "cell_type": "markdown", + "id": "aecd1ecc-7e79-42aa-bd2a-32d049745693", + "metadata": {}, + "source": [ + "If you've used setting 'resizebyfactor' this has changed the width and height of the frames. Below, we correct for this change in pixel resolution, so that it's easier to see and compare the effects of different settings on the movementdetection results below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "94a03a31-bf90-4457-a7b3-dda961100605", + "metadata": {}, + "outputs": [], + "source": [ + "params.df['pixel'] = params.df['pixel'] / params.df['resizebyfactor']\n", + "params.df.loc[:, ('resizebyfactor', 'coords', 'pixel')]" + ] + }, + { + "cell_type": "markdown", + "id": "e5df1e6e-86af-41b4-a876-672e3de643db", + "metadata": {}, + "source": [ + "### Visualize results" + ] + }, + { + "cell_type": "markdown", + "id": "e74dfa98-038c-456f-8249-0ed3e1d4c07a", + "metadata": {}, + "source": [ + "Before visualizing the results, look again at all settings." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39440b99-518f-4f00-a8de-5f692161c4e2", + "metadata": {}, + "outputs": [], + "source": [ + "# the following settings have been used for backgroundsubstraction in this dataframe\n", + "params.get_parameters('all')" + ] + }, + { + "cell_type": "markdown", + "id": "260dfaf1-26d0-4192-b497-297fe5dae7d6", + "metadata": {}, + "source": [ + "Here, you see for which settings multiple values have been used to run movement detection. So, these are also the settings that are interesting to compare in plots or superimpose on the video fragment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a0e789e-a632-4fab-a14e-dce06d391643", + "metadata": {}, + "outputs": [], + "source": [ + "# the following settings have been tested with multiple values\n", + "params.get_parameters('multi_only')" + ] + }, + { + "cell_type": "markdown", + "id": "810b7d5e-5e18-40de-8d80-e4bd16d71cb5", + "metadata": {}, + "source": [ + "#### Plots" + ] + }, + { + "cell_type": "markdown", + "id": "dcdf6823-af93-45df-acbe-9ca555f0efa9", + "metadata": {}, + "source": [ + "First, choose for each parameter with multiple values which value is the default. TIP: you can copy the output dictionary above and choose one of the values in each list. Use the value of which you think will provide the best location detection. If you have no idea, don't worry, just choose one." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fab9933b-70cc-4f41-864e-0db8e277bb3c", + "metadata": {}, + "outputs": [], + "source": [ + "default_values = {'resizebyfactor': 1.0,\n", + " 'color': True,\n", + " 'VarThreshold': 30,\n", + " 'blur': 0,\n", + " 'morphologyex': False,\n", + " 'History': 4,\n", + " 'VarMin': 4}" + ] + }, + { + "cell_type": "markdown", + "id": "f2be4f4d-d320-4487-b59c-0ff4cd73930f", + "metadata": {}, + "source": [ + "You can plot the results of two parameters in one figure. The different values of one parameter is outlined in the rows and the other parameter in the columns of the subplots." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bb2dcde0-156b-4588-86bd-d12a90b40397", + "metadata": {}, + "outputs": [], + "source": [ + "rows = 'blur'\n", + "cols = 'color'\n", + "\n", + "g = params.plot_parameters(rows, cols, default_values)" + ] + }, + { + "cell_type": "markdown", + "id": "880e054a-79af-4abb-94eb-04e51eff815c", + "metadata": {}, + "source": [ + "Here you see the results of using different settings for 'blur' and 'color'. The settings for the other parameters are the ones you've specified as default." + ] + }, + { + "cell_type": "markdown", + "id": "d4d91041-6898-43dc-b927-e59e7bd9e391", + "metadata": {}, + "source": [ + "To save the plots of all combinations of parameters, use the function below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53698f3d-70e7-4739-9937-3fbc2d77c6e5", + "metadata": {}, + "outputs": [], + "source": [ + "params.batch_plot_parameters(default_values)" + ] + }, + { + "cell_type": "markdown", + "id": "174fc315-5944-4759-90e5-c077f75526e8", + "metadata": {}, + "source": [ + "The figures are saved in the same directory as where the associated ParameterSelection dataframe is saved. You can go to the folder where the figures are saved and walk through the figures. That way you get a sense of the influence of various parameter-value combinations on location detection." + ] + }, + { + "cell_type": "markdown", + "id": "17856e6e-f611-42d6-b1a9-c02d0376b718", + "metadata": {}, + "source": [ + "For certain parameters, you might see large noise differences for the different values. For these parameters, choose the best value (the one with the least noise), and use these values as default. Run the above cells again with the new default values. The figures will be saved in a new folder (figures_2). Look again at the figures. Do this several rounds, untill you get an idea of which parameter-value combinations provide the best (least noisy) location detection." + ] + }, + { + "cell_type": "markdown", + "id": "b1f30d74-5ec0-40b8-83fd-059db462f2e5", + "metadata": {}, + "source": [ + "#### Superimpose on video" + ] + }, + { + "cell_type": "markdown", + "id": "73304678-2c96-4494-bc24-91ca881d114e", + "metadata": {}, + "source": [ + "In the plots you get an idea of which paramater-value combinations result in the least noisy graphs. However, it is not possible to see whether the pixel coordinates also accurately match the location of the bird. For this, it is usefull to plot the mean coordinates directly on top of the video." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cb6c7b9e-3b05-457a-8a4f-e75881d878ab", + "metadata": {}, + "outputs": [], + "source": [ + "params.get_parameters('multi_only')" + ] + }, + { + "cell_type": "markdown", + "id": "d3e9cc79-5e30-48f3-bd02-5ba49f6218ef", + "metadata": {}, + "source": [ + "Again, look at the parameters with multiple values. Choose from these parameters which values you would like to see plotted as circle on the videofragment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d998cc74-af96-4b93-9886-012b9a573a47", + "metadata": {}, + "outputs": [], + "source": [ + "# choose which settings to superimpose on the videofragment\n", + "settings = {'resizebyfactor': [1.0],\n", + " 'color': [False],\n", + " 'VarThreshold': [30, 70],\n", + " 'blur': [0, 10],\n", + " 'morphologyex': [True],\n", + " 'History': [4],\n", + " 'VarMin': [10]}\n", + "\n", + "all_combinations = list(md.product_dict(**settings))\n", + "print(f'There are {len(all_combinations)} combinations of settings to superimpose on a video.')" + ] + }, + { + "cell_type": "markdown", + "id": "c5174725-edea-4aaa-b973-51b24c0edb41", + "metadata": {}, + "source": [ + "Too many circles plotted on the video are hard to follow. As default, a maximum of 6 circles can be superimposed on one videofragment, but often you'll probably want to plot less circles." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "763143ba-8793-47db-a510-d249bf8e36d3", + "metadata": {}, + "outputs": [], + "source": [ + "# draw circles on videofragment\n", + "frames, colorspecs = params.draw_multiple_circles(settings)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f4db08-36f7-426b-b087-5cf14ec119bb", + "metadata": {}, + "outputs": [], + "source": [ + "# show the settings for each color of the circles\n", + "colorspecs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43dc1785-c4db-4b1a-affb-97c868727e19", + "metadata": {}, + "outputs": [], + "source": [ + "# look at the video using show()\n", + "frames.show(framerate=20)" + ] + }, + { + "cell_type": "markdown", + "id": "34736487-9913-46de-97b3-1206bb61b7db", + "metadata": {}, + "source": [ + "TIP: a lower framerate makes it easier to follow the circles." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "be15b7d5-e18e-4399-9a69-8e56bd73a670", + "metadata": {}, + "outputs": [], + "source": [ + "# or, save as video with circles superimposed\n", + "vfs_circles = frames.tovideo(f'{params.path}/multicircles.mp4', framerate=params.vfs.avgframerate)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e21b809a-42cd-449d-b033-b8f7bfba5bd2", + "metadata": {}, + "outputs": [], + "source": [ + "# you can also save the color specification\n", + "colorspecs.to_csv(f'{params.path}/multicircles_colorspecs.csv')" + ] + }, + { + "cell_type": "markdown", + "id": "584e0543-1087-428b-b084-bf870dfc875b", + "metadata": {}, + "source": [ + "Now, you have an idea which parameters have a large influence on movement detection. You might want to run the notebook again and test some other values or the parameters to fine-tune your results even more. Just repeat all the steps above." + ] + }, + { + "cell_type": "markdown", + "id": "863b24b0-2e7b-4276-a546-3376f41c093d", + "metadata": {}, + "source": [ + "Also, repeat these steps with a second short representative videofragment to make sure the same parameter-value combinations provide the best results. After that, you could use these settings to run movement detection on all your videos. For this, have a look at the next notebook!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae23c7d4-6284-4d92-bbfe-84523146f707", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/5_movementdetection.ipynb b/notebooks/5_movementdetection.ipynb new file mode 100644 index 0000000..dd2fbee --- /dev/null +++ b/notebooks/5_movementdetection.ipynb @@ -0,0 +1,367 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c8a55aa1-ba0d-44a7-a771-ad2c3bf6765f", + "metadata": {}, + "source": [ + "Again, we will use the high-level detect movement function, but instead of using the default values, you can use the optimal settings found in notebook 4 'parameterselection'. Besides viewing location detection results, it is possible to see the effects of the settings on raw movement detection. Also, we will show you how to run movement detection on a list of videofiles." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "274b5d58-76e5-4eee-b48a-0a4b599c75a0", + "metadata": {}, + "outputs": [], + "source": [ + "import birdwatcher as bw\n", + "import birdwatcher.movementdetection as md\n", + "from birdwatcher.plotting import imshow_frame # birdwatcher has vizualization tools\n", + "\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "id": "c8a97e60-1527-44f8-83cb-b8163761a298", + "metadata": {}, + "source": [ + "### Select video fragment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e9ee6186-7a33-4e8e-9f4b-f21add4df022", + "metadata": {}, + "outputs": [], + "source": [ + "vfs = bw.VideoFileStream(r'..\\videos\\zf20s_low.mp4')\n", + "\n", + "# optional: if you want to do movement detection only on part of the video\n", + "startat = '00:00:00' # in HOURS:MM:SS\n", + "nframes = None\n", + "\n", + "# specify h1, h2, w1, w2, or choose None to use the whole frame\n", + "roi = None # region of interest\n", + "nroi = None # nót region of interest" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe10009d-7bab-427e-9f54-4d333791b00a", + "metadata": {}, + "outputs": [], + "source": [ + "# show roi and nroi in frame\n", + "if roi is not None:\n", + " frame = vfs.iter_frames(startat=startat, nframes=1).draw_text(['roi'], org=(roi[0],roi[3]))\n", + " imshow_frame(frame.peek_frame(), draw_rectangle=roi)\n", + "\n", + "if nroi is not None:\n", + " frame = vfs.iter_frames(startat=startat, nframes=1).draw_text(['nroi'], org=(nroi[0],nroi[3]))\n", + " imshow_frame(frame.peek_frame(), draw_rectangle=nroi)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "828d67ad-f3f9-4907-b6b0-e267294d6fcc", + "metadata": {}, + "outputs": [], + "source": [ + "# look at the chosen video fragment\n", + "vfs.iter_frames(startat=startat, nframes=nframes).show(framerate=150)" + ] + }, + { + "cell_type": "markdown", + "id": "7996ec8a-4352-47ca-a2e9-c14cfd927d54", + "metadata": {}, + "source": [ + "### Set parameters" + ] + }, + { + "cell_type": "markdown", + "id": "9e261f92-84ff-45e3-9f42-76bcc35fced5", + "metadata": {}, + "source": [ + "First, decide which settings you would like, by adding one value in the list after each parameter. You could enter here the optimal settings you have found in the notebook 'parameterselection'. NOTE: that the values in this dictionary don't contain lists!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9e372b4e-4432-46bd-b51a-c08c2943b101", + "metadata": {}, + "outputs": [], + "source": [ + "settings = {'bgs_params': {'History': 3,\n", + " 'ComplexityReductionThreshold': 0.05,\n", + " 'BackgroundRatio': 0.1,\n", + " 'NMixtures': 7,\n", + " 'VarInit': 15,\n", + " 'VarMin': 10,\n", + " 'VarMax': 75,\n", + " 'VarThreshold': 70,\n", + " 'VarThresholdGen': 9,\n", + " 'DetectShadows': False,\n", + " 'ShadowThreshold': 0.5,\n", + " 'ShadowValue': 0},\n", + "\n", + " 'processing': {'color': False, # booleans only\n", + " 'resizebyfactor': 1, # use '1' for no change in size\n", + " 'blur': 10, # use '0' for no blur\n", + " 'morphologyex': True}} # booleans only" + ] + }, + { + "cell_type": "markdown", + "id": "10a79d50-e4ec-4d4a-bce3-a67b6dede612", + "metadata": {}, + "source": [ + "To use a different background subtraction algorithm, just replace the parameters of the background subtractor with parameters of another algorithm (e.g. from BackgroundSubtractorKNN or BackgroundSubtractorLSBP)." + ] + }, + { + "cell_type": "markdown", + "id": "67533a6a-faef-4f4d-812a-867964947a19", + "metadata": {}, + "source": [ + "### Run movemement detection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1599bf4a-73cf-4aaf-9e2b-d2df1cb8bfcb", + "metadata": {}, + "outputs": [], + "source": [ + "coords, coordscount, coordsmean = md.detect_movement(vfs, settings, startat, nframes, roi, nroi,\n", + " bgs_type=bw.BackgroundSubtractorMOG2,\n", + " analysispath='output/', ignore_firstnframes=50, \n", + " overwrite=True, resultvideo=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a6bbbaed-c3d4-4a4d-bd7b-3bdb29518947", + "metadata": {}, + "outputs": [], + "source": [ + "movementpath = f'output/movement_{vfs.filepath.stem}'\n", + "movementpath" + ] + }, + { + "cell_type": "markdown", + "id": "fe8ebece-7cdb-455c-974f-77c78aeb2f49", + "metadata": {}, + "source": [ + "The coordinate arrays are saved in the output/ directory within a 'movement' folder with the name of the videofilestream. Also a movementvideo (of the videofragment) is directly saved in the movement folder." + ] + }, + { + "cell_type": "markdown", + "id": "545be2dd-543d-4e8a-8768-4b32d76705a1", + "metadata": {}, + "source": [ + "We can create a video of the coordinates results as well:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8abe3683-328b-42cc-905f-8e55354dc1c2", + "metadata": {}, + "outputs": [], + "source": [ + "coords.tovideo(f'{movementpath}/coordsvideo.mp4', framerate=vfs.avgframerate)" + ] + }, + { + "cell_type": "markdown", + "id": "aed571fd-385f-4658-bef2-69a86df057ab", + "metadata": { + "tags": [] + }, + "source": [ + "### Load coordinate arrays" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec7ccf41-8404-4c4c-aeae-095d072247ff", + "metadata": {}, + "outputs": [], + "source": [ + "coords = bw.CoordinateArrays(f'{movementpath}/coords.darr')\n", + "coordscount = coords.get_coordcount()\n", + "coordsmean = coords.get_coordmean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "da66aacf-786d-4c35-b622-e511fca0880c", + "metadata": {}, + "outputs": [], + "source": [ + "coords.metadata" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3a856f4-a2b9-45c1-8ab2-4e1992d5ce60", + "metadata": {}, + "outputs": [], + "source": [ + "coords.metadata['settings']" + ] + }, + { + "cell_type": "markdown", + "id": "ef346d52-78cc-497a-bc63-6a1beb031b72", + "metadata": {}, + "source": [ + "### Plot results" + ] + }, + { + "cell_type": "markdown", + "id": "59743992-857b-4286-81e6-4f22a911047b", + "metadata": {}, + "source": [ + "The coordscount shows the number of pixels that belong to the foreground, e.g. 'movement pixels', per frame. Higher peaks means more movement." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0582cc5f-ed41-4e1d-beac-8929b2f3421b", + "metadata": {}, + "outputs": [], + "source": [ + "plt.plot(coordscount)\n", + "plt.title('number of pixels above treshold')\n", + "plt.xlabel('framenumber')\n", + "plt.ylabel('number of pixels')" + ] + }, + { + "cell_type": "markdown", + "id": "03471ac0-11e4-4534-af34-7367f3744200", + "metadata": {}, + "source": [ + "The coordsmean shows the mean coordinates per frame. This could be used to look at the location of the subject during the video. Note, there is a different graph to see the horizontal coordinates (left-rigth) and the vertical coordinates (top-bottom)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b426fe1-771d-4680-a5b6-c38015aa7ae0", + "metadata": {}, + "outputs": [], + "source": [ + "plt.plot(coordsmean)\n", + "plt.title('coordinates of pixels above treshold')\n", + "plt.xlabel('frame number')\n", + "plt.ylabel('pixel coordinate')\n", + "plt.legend(['left-right', 'top-bottom'])" + ] + }, + { + "cell_type": "markdown", + "id": "5ebf3bd6-f305-4401-a352-de808f0bd87e", + "metadata": {}, + "source": [ + "### Look at range of parameter values" + ] + }, + { + "cell_type": "markdown", + "id": "70f6f613-0169-4c94-91ee-749e3e965e6c", + "metadata": {}, + "source": [ + "If you still have some doubt about the optimal parameter values, or you just want to compare several settings by looking at the raw coordinates, you can run movement detection with several values for a specific parameter. For each setting a coordinate array and a movementvideo per value are directly saved. We also save a coordinate video." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a0331f8-505a-4d9b-b73f-43605e744fbb", + "metadata": {}, + "outputs": [], + "source": [ + "parameter = 'blur'\n", + "values = [0, 10]\n", + "\n", + "s = settings.copy()\n", + "\n", + "for value in values: \n", + " pathname = f'{movementpath}_{parameter}{value}'\n", + " s.update({parameter: value})\n", + " coords, _, _ = md.detect_movement(vfs, s, startat, nframes, roi, nroi,\n", + " bgs_type=bw.BackgroundSubtractorMOG2,\n", + " analysispath=pathname, ignore_firstnframes=50, \n", + " overwrite=True, resultvideo=True)\n", + " coords.tovideo(f'{pathname}/coordsvideo.mp4', framerate=vfs.avgframerate)" + ] + }, + { + "cell_type": "markdown", + "id": "39783968-eab8-4551-80cc-bea83e45b45c", + "metadata": {}, + "source": [ + "### Batch detect movement" + ] + }, + { + "cell_type": "markdown", + "id": "c95894b7-4792-48d6-b41e-74712f2c8284", + "metadata": {}, + "source": [ + "Do movement detection on multiple videos, in which the same parameters values will be applied to each video. From each video a coordsarray and movementvideo will be saved." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1aa6161d-638f-4bbd-9ce3-511f86008378", + "metadata": {}, + "outputs": [], + "source": [ + "# to be implemented" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/banner.ipynb b/notebooks/banner.ipynb index 8d518cc..83db71b 100644 --- a/notebooks/banner.ipynb +++ b/notebooks/banner.ipynb @@ -9,13 +9,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import birdwatcher as bw\n", - "from birdwatcher.movementdetection import detect_movementmog2\n", + "import birdwatcher.movementdetection as md\n", "%matplotlib inline" ] }, @@ -28,7 +28,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -44,11 +44,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "coords, coordscount, coordsmean = detect_movementmog2(vfs, morphologyex=2, analysispath='output/',\n", + "coords, coordscount, coordsmean = md.detect_movementmog2(vfs, morphologyex=2, analysispath='output/',\n", " ignore_firstnframes=20, overwrite=True,\n", " VarThreshold=200, NMixtures=8, History=3)" ] @@ -75,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -90,7 +90,7 @@ "diff[1:] = np.diff(coordsmean, axis=0)\n", "# we include for speed only when many pixels change (>400), which is when bird moves around\n", "speed = [(x**2+y**2)*0.5 if c>400 else 0 for ((x,y),c) in zip(diff,coordscount) ]\n", - "speedtextlist = [f'speed : {int(s):4}' for s in speed]\n", + "speedtextlist = [f'velocity : {int(s):4}' for s in speed]\n", "\n", "location = coordsmean.copy()\n", "location[speed==0]\n", @@ -126,7 +126,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" + "version": "3.9.11" } }, "nbformat": 4, diff --git a/notebooks/movementdetection.ipynb b/notebooks/movementdetection.ipynb deleted file mode 100644 index 33b160e..0000000 --- a/notebooks/movementdetection.ipynb +++ /dev/null @@ -1,353 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import birdwatcher as bw\n", - "from birdwatcher.movementdetection import detect_movementmog2, detect_movementknn, \\\n", - " detect_movementlsbp, create_movementvideo\n", - "from birdwatcher.plotting import imshow_frame\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook shows movement detection the easy way, using a high-level function." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vfs = bw.testvideosmall() # get a videofilestream object" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Example with MOG2 algorithm" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "One high-level function can detect movement based on background subtraction. It returns three arrays, which are disk-based darr arrays. They can be very large and hence not fit in RAM. 'coords' provided the coordinates of detected pixels in a ragged array. 'coordscount' the number of pixels per frame, 'coordsmean' the spatial average." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "coords, coordscount, coordsmean = detect_movementmog2(vfs, morphologyex=2, analysispath='output/',\n", - " ignore_firstnframes=50, overwrite=True,\n", - " VarThreshold=70, NMixtures=8, History=3)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's look at frame 200" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "coords[200]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "we can view the results as a black and white image by the `get_frame` method, which returns a frame instead of coordinates." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "frame = coords.get_frame(130)\n", - "imshow_frame(frame)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "let's look at the original frame" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "imshow_frame(vfs.get_frame(130))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "we can create a video of the results as well" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "coords.tovideo('output/zf20s_coords.mp4', framerate=vfs.avgframerate)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "or a selection of the results by indicating start and end frame numbers" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "coords.tovideo('output/zf20s_coords_selection.mp4', startframe=100, endframe=200, framerate=vfs.avgframerate)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "if you want to do more things before saving to video, just use `iter_frames` which turns it into a Frames object with many more methods. Make sure you use three color channels and set coordinates to value 255 if you want them white." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "frames = coords.iter_frames(startframe=100, endframe=200, nchannels=3, value=255)\n", - "_ = frames.draw_framenumbers().tovideo('output/zf20s_coords_selection_framenumbers.mp4', framerate=vfs.avgframerate)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "or we can look at the data in plots" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.plot(coordscount)\n", - "plt.title('number of pixels above treshold')\n", - "plt.xlabel('framenumber')\n", - "plt.ylabel('number of pixels')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.plot(coordsmean)\n", - "plt.title('coordinates of pixels above treshold')\n", - "plt.xlabel('x coordinate')\n", - "plt.ylabel('y coordinate')\n", - "plt.legend(['left-right', 'top-bottom'])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "let's make a video with the original frames + the mean coordinate superimposed on it as a circle" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vfs_circle = vfs.iter_frames().draw_framenumbers().draw_circles(coordsmean).tovideo('output/zf20s_coords_center.mp4', framerate=vfs.avgframerate)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "frame = coords.get_frame(130)\n", - "imshow_frame(frame)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "coords.tovideo('output/zf20s_coords_lsbp.mp4', framerate=vfs.avgframerate)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "There is also a high-level function that does this for you, and better ;-) It will produce a video next to the original one with results superimposed." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vfs_results = create_movementvideo(vfs, coords, videofilepath='output/movementvideoexample.mp4')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "imshow_frame(vfs_results.get_frame(130))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "coords.iter_frames?" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# A different algorithm: KNN" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "coords, coordscount, coordsmean = detect_movementknn(vfs, morphologyex=2, analysispath='output/',\n", - " ignore_firstnframes=50, overwrite=True,\n", - " kNNSamples=0, NSamples=6, Dist2Threshold=500, \n", - " DetectShadows=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "frame = coords.get_frame(130)\n", - "imshow_frame(frame)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Yet another algorithm, LSBP" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "coords, coordscount, coordsmean = detect_movementlsbp(vfs, morphologyex=2, analysispath='output/',\n", - " ignore_firstnframes=50, overwrite=True,\n", - " mc=0, nSamples=20, LSBPRadius=16)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "frame = coords.get_frame(130)\n", - "imshow_frame(frame)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "coords.tovideo('output/zf20s_coords_lsbp.mp4', framerate=vfs.avgframerate)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/requirements.txt b/requirements.txt index f9518dc..1c50360 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ numpy -darr >= 0.4.1 +pandas matplotlib +seaborn +darr >= 0.4.1 opencv-python opencv-contrib-python diff --git a/setup.py b/setup.py index 0ea4871..a041661 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ name='birdwatcher', version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), - packages=['birdwatcher', 'birdwatcher.tests'], + packages=['birdwatcher', 'birdwatcher.movementdetection', 'birdwatcher.tests'], package_data={'birdwatcher.testvideos': ['*.mp4']}, include_package_data=True, url='https://github.com/gbeckers/birdwatcher', @@ -51,9 +51,8 @@ description='A Python computer vision library for animal behavior', long_description=long_description, long_description_content_type="text/x-rst", - requires=['numpy', 'darr', 'opencv'], - install_requires=['numpy', 'darr','matplotlib', 'opencv-python', - 'opencv-contrib-python'], + install_requires=['numpy', 'pandas', 'matplotlib', 'seaborn', 'darr', + 'opencv-python', 'opencv-contrib-python'], data_files=[("", ["LICENSE"])], classifiers=[ "Programming Language :: Python :: 3",