Merge branch 'master' of github.com:lambdaloop/anipose

docs/sphinx/Makefile

@@ -1,10 +1,10 @@
 # Minimal makefile for Sphinx documentation
 #
 
-# You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-SPHINXPROJ    = Anipose
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build
 
@@ -17,4 +17,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/sphinx/conf.py

@@ -1,10 +1,8 @@
-# -*- coding: utf-8 -*-
-#
 # Configuration file for the Sphinx documentation builder.
 #
-# This file does only contain a selection of the most common options. For a
-# full list see the documentation:
-# http://www.sphinx-doc.org/en/master/config
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
 
 # -- Path setup --------------------------------------------------------------
 
@@ -20,57 +18,29 @@
 # -- Project information -----------------------------------------------------
 
 project = 'Anipose'
-copyright = '2020, Anipose authors'
+copyright = 'Anipose authors, 2020'
 author = 'Anipose authors'
 
-# The short X.Y version
-version = ''
 # The full version, including alpha/beta/rc tags
 release = '0.6.6'
 
 
 # -- General configuration ---------------------------------------------------
 
-# If your documentation needs a minimal Sphinx version, state it here.
-#
-# needs_sphinx = '1.0'
-
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.doctest',
-    'sphinx.ext.mathjax',
-]
+master_doc = 'index'
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.coverage', 'sphinx.ext.napoleon']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-#
-# source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
-
-# The master toctree document.
-master_doc = 'index'
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path .
+# This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
 
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
 
 # -- Options for HTML output -------------------------------------------------
 
@@ -79,87 +49,7 @@
 #
 html_theme = 'alabaster'
 
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-# html_theme_options = {}
-
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
-# Custom sidebar templates, must be a dictionary that maps document names
-# to template names.
-#
-# The default sidebars (for documents that don't match any pattern) are
-# defined by theme itself.  Builtin themes are using these templates by
-# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
-# 'searchbox.html']``.
-#
-# html_sidebars = {}
-html_sidebars = {
-   '**': ['globaltoc.html', 'sourcelink.html', 'searchbox.html'],
-   'using/windows': ['windowssidebar.html', 'searchbox.html'],
-}
-
-
-# -- Options for HTMLHelp output ---------------------------------------------
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'Aniposedoc'
-
-
-# -- Options for LaTeX output ------------------------------------------------
-
-latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
-    #
-    # 'papersize': 'letterpaper',
-
-    # The font size ('10pt', '11pt' or '12pt').
-    #
-    # 'pointsize': '10pt',
-
-    # Additional stuff for the LaTeX preamble.
-    #
-    # 'preamble': '',
-
-    # Latex figure (float) alignment
-    #
-    # 'figure_align': 'htbp',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-latex_documents = [
-    (master_doc, 'Anipose.tex', 'Anipose Documentation',
-     'Anipose authors', 'manual'),
-]
-
-
-# -- Options for manual page output ------------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'anipose', 'Anipose Documentation',
-     [author], 1)
-]
-
-
-# -- Options for Texinfo output ----------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
-texinfo_documents = [
-    (master_doc, 'Anipose', 'Anipose Documentation',
-     author, 'Anipose', 'One line description of project.',
-     'Miscellaneous'),
-]
-
-
-# -- Extension configuration -------------------------------------------------
+html_static_path = ['_static']

docs/sphinx/index.rst

@@ -4,6 +4,8 @@ Welcome to Anipose's documentation!
 .. toctree::
    :maxdepth: 2
 
+   params
+   nesting
    tutorial 
 
    :caption: Contents:

docs/sphinx/make.bat

@@ -9,7 +9,6 @@ if "%SPHINXBUILD%" == "" (
 )
 set SOURCEDIR=.
 set BUILDDIR=_build
-set SPHINXPROJ=Anipose
 
 if "%1" == "" goto help
 
@@ -26,11 +25,11 @@ if errorlevel 9009 (
 	exit /b 1
 )
 
-%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
 goto end
 
 :help
-%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
 
 :end
 popd

docs/sphinx/nesting.rst

@@ -0,0 +1,145 @@
+Setting up Anipose for 2D tracking
+##################################
+
+So you want to setup Anipose for 2D tracking at scale? Well, you've come
+to the right place!
+
+Just follow these steps:
+
+1) Train a network to label your data `using
+   DeepLabCut <https://github.com/AlexEMG/DeepLabCut/blob/master/docs/UseOverviewGuide.md>`__
+2) Setup your folder structure for the experiment in the appropriate
+   format
+3) Create a ``config.toml`` file with the parameters for your experiment
+
+Folder structure
+================
+
+Anipose tries to follow the organization that people tend to come to
+naturally for organizing behavioral videos, with one key modification.
+
+The modification is that the *final folder* should be ``videos-raw``.
+This allows Anipose to know these are the input videos, and to create
+further folders with the processed data (e.g. ``pose-2d``,
+``pose-2d-filtered``, ``videos-labeled``).
+
+Here is the general layout of files for videos for 2D tracking
+
+.. image:: anipose-tutorial/folder_structure.png
+   :align: center
+   :height: 300px
+
+.. code-block:: text
+
+    experiment/config.toml
+    experiment/trial 1/videos-raw/vid1-trial1.avi
+    experiment/trial 1/videos-raw/vid2-trial1.avi
+    experiment/trial 2/videos-raw/vid1-trial2.avi
+    experiment/trial 2/videos-raw/vid2-trial2.avi
+
+There is one main experiment folder, and some subfolders under that. The
+names for the experiment and session folders can be whatever you like.
+
+Furthermore, the nesting can be arbitrarily large. Thus, an equally
+valid structure could be (here with nesting of 2 folders instead of 1 as
+above, note the arbitrary names as well):
+
+.. image:: anipose-tutorial/folder_structure_2.png
+   :align: center
+   :height: 300px
+
+
+.. code-block:: text
+
+    flypose/config.toml
+    flypose/2019-02-06/fly1/videos-raw/vid-2019-02-06-01.avi
+    flypose/2019-02-06/fly2/videos-raw/vid-2019-02-06-02.avi
+    flypose/2019-02-07/fly1/videos-raw/vid-2019-02-07-01.avi
+
+Configuration
+=============
+
+Got your folder structure setup? Great! Let's setup the configuration!
+
+Anipose uses the `TOML <https://github.com/toml-lang/toml>`__
+configuration format, as it is simple and can be read/written easily
+from Python and Matlab. Consider reading through the
+`specification <https://github.com/toml-lang/toml>`__ to get a sense of
+it.
+
+Below is an annotated example config file for 2D tracking. Create a file
+called ``config.toml`` within your experiment folder with this
+structure, and update the parameters.
+
+.. code:: yaml
+
+    # Project name
+    project = "flypose"
+
+    # Change this to match deeplabcut folder (one with trained network)
+    model_folder = '/Data/Videos/DLC_Analysis/Running-Brandon-2019-01-29'
+
+    # How many folders are nested in structure?
+    nesting = 1
+
+    [pipeline]
+    videos_raw = "videos-raw" # change this if you'd like to change name of "videos-raw" folder
+
+    # labeling scheme...specify lines that you want to draw for visualizing labels in videos
+    [labeling]
+    scheme = [
+      ["head", "thorax", "abdomen"],
+      ["thorax", "leg-1"]
+    ]
+
+Filtering the data
+==================
+
+You may enable filtering of the data (it is off by default). This
+compares the data to a median filtered version and looks for outliers.
+We found this helps find and remove tracking errors, even if the score
+is high for these.
+
+To do this, you can edit your config with these parameters:
+
+.. code:: yaml
+
+    # Settings for a threshold filter
+    # Removes data outside threshold (probably errors in tracking), and interpolates
+    [filter]
+    enabled = true
+    medfilt = 13 # length of median filter
+    offset_threshold = 25 # offset from median filter to count as jump
+    score_threshold = 0.8 # score below which to count as bad
+    spline = true # interpolate using cubic spline instead of linear
+
+Running Anipose
+===============
+
+With the configuration file and folder structure setup, you can now run
+Anipose on your full dataset!
+
+For 2D tracking, the relevant commands are as follows:
+
+.. code-block:: text
+
+   anipose analyze # analyze the poses for each video
+   anipose filter # filter out the tracked poses
+   anipose label_videos # create videos for each pose
+   anipose summarize_2d # summarize the 2d tracked data (both raw and filtered)
+   anipose summarize_errors # summarize the errors in the tracking
+
+To run these, open a command line window, go to the folder where your
+experiment is stored, and run one of the commands above.
+
+Summarizing the data
+====================
+
+After computing the whole pipeline for all videos, the final outputs of
+interest (the 2D coordinates) are scattered across a lot of folders.
+
+For further processing and analysis, it is often useful to have one
+central file with all the data. Hence, Anipose provides the command
+“summarize\_2d”, which summarizes everything into a “summaries” folder.
+The output csv for the 2d tracking coordinates has all the data from all
+sessions, and a few extra columns to show where the data comes from.