Implement Registries, Plugins, and Wiki Integration (#109)

Very late now, but I believe i have done this: #39

I will return tomorrow to write the docs and the PR, but yes it is time.

dump anything that inherits from the appropriate metaclass into `PLUGINDIR` and...

```python
import autopilot

# list all hardware
autopilot.get('hardware')
# or an alias
autopilot.get_hardware()

# get hardware (including from plugins)
autopilot.get('hardware', 'PWM')

# get anything
autopilot.get('task')
autopilot.get('graduation')
autopilot.get('transform')
autopilot.get('children')
```

and as a little bonus...

```python
>>> from autopilot.utils import wiki
>>> wiki.ask('[[HiFiBerry Amp2]]', "Uses GPIO Pin")
[{'Uses GPIO Pin': [3, 5, 7, 12, 35, 38, 40],
  'name': 'HiFiBerry Amp2',
  'url': 'https://wiki.auto-pi-lot.com/index.php/HiFiBerry_Amp2'}]
```

which will serve as the means of submitting plugins and doing some stuff that is truly messed up how cool it is... more soon ;)

* deduplicate utils

formerly had core/utils and a utils module, just have the module now.

* draft simpler registry :)

* moving common utility functions out of __init__

* task and hardware aliases

* proper sorting

* got excited and made a wiki API access module in this branch by accident whoops

* want to get one good build off!

* want to get one good build off!

* tests for registry module

* whoops reverting stuff that was broken at the time of adding it lmao jonny watch the results of the tests jesus christ

* basic test of a plugin

* implementing get throughout rest of library

removing imports and references to hardware and task objects, and finally killed the godforsaken Task List once and for all.
also adding other assorted improvements and bugfixes from the registries branch

* just for shits lets see if other versions work

* fix path traversal through modules whose base class is not in __init__

* proof of concept plugins manager ;)

.travis.yml

@@ -3,6 +3,8 @@ sudo: required
 language: python
 python:
   - "3.7"
+  - "3.8"
+  - "3.9"
 
 arch:
   - amd64

autopilot/__init__.py

@@ -1,2 +1,5 @@
 __author__  = 'Jonny Saunders <j@nny.fyi>'
 __version__ = '0.3.5'
+
+from autopilot.setup import setup_autopilot
+from autopilot.utils.registry import get, get_task, get_hardware, get_names

autopilot/core/gui.py

@@ -16,7 +16,6 @@
     This will be corrected before v1.0
 
 """
-
 import sys
 import typing
 import os
@@ -36,15 +35,16 @@
 from functools import reduce
 
 # adding autopilot parent directory to path
+import autopilot
 from autopilot.core.subject import Subject
-from autopilot import tasks, prefs
+from autopilot import prefs
 from autopilot.stim.sound import sounds
 from autopilot.networking import Net_Node
 from functools import wraps
-from autopilot.core.utils import InvokeEvent
+from autopilot.utils.invoker import InvokeEvent, get_invoker
 from autopilot.core import styles
-from autopilot.core.utils import get_invoker
 from autopilot.core.loggers import init_logger
+from autopilot.utils import plugins, registry, wiki
 
 _MAPS = {
     'dialog': {
@@ -77,7 +77,6 @@ def gui_event(fn):
     """
     @wraps(fn)
     def wrapper_gui_event(*args, **kwargs):
-        # type: (object, object) -> None
         """
 
         Args:
@@ -868,7 +867,7 @@ class Protocol_Wizard(QtWidgets.QDialog):
 
     This widget is composed of three windows:
 
-    * **left**: possible task types from :py:data:`.tasks.TASK_LIST`
+    * **left**: possible task types from :func:`autopilot.get_task()`
     * **center**: current steps in task
     * **right**: :class:`.Parameters` for currently selected step.
 
@@ -915,7 +914,7 @@ def __init__(self):
         addstep_label = QtWidgets.QLabel("Add Step")
         addstep_label.setFixedHeight(40)
         self.task_list = QtWidgets.QListWidget()
-        self.task_list.insertItems(0, tasks.TASK_LIST.keys())
+        self.task_list.insertItems(0, autopilot.get_names('task'))
         self.add_button = QtWidgets.QPushButton("+")
         self.add_button.setFixedHeight(40)
         self.add_button.clicked.connect(self.add_step)
@@ -981,7 +980,7 @@ def add_step(self):
         task_type = self.task_list.currentItem().text()
         new_item = QtWidgets.QListWidgetItem()
         new_item.setText(task_type)
-        task_params = copy.deepcopy(tasks.TASK_LIST[task_type].PARAMS)
+        task_params = copy.deepcopy(autopilot.get_task(task_type).PARAMS)
 
         # Add params that are non-task specific
         # Name of task type
@@ -1203,7 +1202,7 @@ class Graduation_Widget(QtWidgets.QWidget):
 
     Attributes:
         type_selection (:class:`QtWidgets.QComboBox`): A box to select from the available
-            graduation types listed in :py:data:`.tasks.GRAD_LIST` . Has its `currentIndexChanged`
+            graduation types listed in :func:`autopilot.get_task()` . Has its `currentIndexChanged`
             signal connected to :py:meth:`.Graduation_Widget.populate_params`
         param_dict (dict): Stores the type of graduation and the relevant params,
             fetched by :class:`.Protocol_Wizard` when defining a protocol.
@@ -1216,7 +1215,7 @@ def __init__(self):
         # Grad type dropdown
         type_label = QtWidgets.QLabel("Graduation Criterion:")
         self.type_selection = QtWidgets.QComboBox()
-        self.type_selection.insertItems(0, tasks.GRAD_LIST.keys())
+        self.type_selection.insertItems(0, autopilot.get_names('graduation'))
         self.type_selection.currentIndexChanged.connect(self.populate_params)
 
         # Param form
@@ -1259,7 +1258,7 @@ def populate_params(self, params=None):
         self.type = self.type_selection.currentText()
         self.param_dict['type'] = self.type
 
-        for k in tasks.GRAD_LIST[self.type].PARAMS:
+        for k in autopilot.get_task(self.type).PARAMS:
             edit_box = QtWidgets.QLineEdit()
             edit_box.setObjectName(k)
             edit_box.editingFinished.connect(self.store_param)
@@ -2628,7 +2627,7 @@ def init_ui(self):
                 self.setItem(row, j, item)
 
         # make headers
-        self.setHorizontalHeaderLabels(self.colnames.values())
+        self.setHorizontalHeaderLabels(list(self.colnames.values()))
         self.resizeColumnsToContents()
         self.updateGeometry()
         self.adjustSize()
@@ -2661,6 +2660,114 @@ def set_weight(self, row, column):
             column_name = self.colnames.keys()[column] # recall colnames is an ordered dictionary
             self.subjects[subject_name].set_weight(date, column_name, new_val)
 
+class Plugins(QtWidgets.QDialog):
+    """
+    Dialog window that allows plugins to be viewed and installed.
+
+    Works by querying the `wiki <https://wiki.auto-pi-lot.com>`_ ,
+    find anything in the category ``Autopilot Plugins`` , clone the
+    related repo, and reload plugins.
+
+    At the moment this widget is a proof of concept and will be made functional
+    asap :)
+    """
+
+    def __init__(self):
+        super(Plugins, self).__init__()
+
+        self.logger = init_logger(self)
+        self.plugins = {}
+
+        self.init_ui()
+        self.list_plugins()
+
+    def init_ui(self):
+        self.layout = QtWidgets.QGridLayout()
+
+        # top combobox for selecting plugin type
+        self.plugin_type = QtWidgets.QComboBox()
+        self.plugin_type.addItem("Plugin Type")
+        self.plugin_type.addItem('All')
+        for ptype in registry.REGISTRIES:
+            self.plugin_type.addItem(str(ptype.name).capitalize())
+        self.plugin_type.currentIndexChanged.connect(self.select_plugin_type)
+
+        # left panel for listing plugins
+        self.plugin_list = QtWidgets.QListWidget()
+        self.plugin_list.currentItemChanged.connect(self.select_plugin)
+        self.plugin_details = QtWidgets.QFormLayout()
+
+        self.plugin_list.setMinimumWidth(200)
+        self.plugin_list.setMinimumHeight(600)
+
+        self.status = QtWidgets.QLabel()
+        self.download_button = QtWidgets.QPushButton('Download')
+        self.download_button.setDisabled(True)
+
+        # --------------------------------------------------
+        # layout
+
+        self.layout.addWidget(self.plugin_type, 0, 0, 1, 2)
+        self.layout.addWidget(self.plugin_list, 1, 0, 1, 1)
+        self.layout.addLayout(self.plugin_details, 1, 1, 1, 1)
+        self.layout.addWidget(self.status, 2, 0, 1, 1)
+        self.layout.addWidget(self.download_button, 2, 1, 1, 1)
+
+        self.layout.setRowStretch(0, 1)
+        self.layout.setRowStretch(1, 10)
+        self.layout.setRowStretch(2, 1)
+
+        self.setLayout(self.layout)
+
+    def list_plugins(self):
+        self.status.setText('Querying wiki for plugin list...')
+
+        self.plugins = plugins.list_wiki_plugins()
+        self.logger.info(f'got plugins: {self.plugins}')
+
+        self.status.setText(f'Got {len(self.plugins)} plugins')
+
+    def download_plugin(self):
+        pass
+
+    def select_plugin_type(self):
+        nowtype = self.plugin_type.currentText()
+
+
+        if nowtype == "Plugin Type":
+            return
+        elif nowtype == "All":
+            plugins = self.plugins.copy()
+        else:
+            plugins = [plug for plug in self.plugins if plug['Is Autopilot Plugin Type'] == nowtype]
+
+        self.logger.debug(f'showing plugin type {nowtype}, matched {plugins}')
+
+        self.plugin_list.clear()
+        for plugin in plugins:
+            self.plugin_list.addItem(plugin['name'])
+
+    def select_plugin(self):
+        if self.plugin_list.currentItem() is None:
+            self.download_button.setDisabled(True)
+        else:
+            self.download_button.setDisabled(False)
+
+        plugin_name = self.plugin_list.currentItem().text()
+        plugin = [p for p in self.plugins if p['name'] == plugin_name][0]
+
+        while self.plugin_details.rowCount() > 0:
+            self.plugin_details.removeRow(0)
+
+        for k, v in plugin.items():
+            if k == 'name':
+                continue
+            self.plugin_details.addRow(k, QtWidgets.QLabel(v))
+
+
+
+
+
 
 #####################################################
 # Custom Autopilot Qt Style

autopilot/core/pilot.py

@@ -21,6 +21,7 @@
 import tables
 warnings.simplefilter('ignore', category=tables.NaturalNameWarning)
 
+import autopilot
 from autopilot import prefs
 from autopilot.core.loggers import init_logger
 
@@ -54,7 +55,6 @@
 
 from autopilot.networking import Message, Net_Node, Pilot_Station
 from autopilot import external
-from autopilot import tasks
 from autopilot.hardware import gpio
 
 
@@ -273,7 +273,7 @@ def l_start(self, value):
         Start running a task.
 
         Get the task object by using `value['task_type']` to select from
-        :data:`.tasks.TASK_LIST` , then feed the rest of `value` as kwargs
+        :func:`autopilot.get_task()` , then feed the rest of `value` as kwargs
         into the task object.
 
         Calls :meth:`.autopilot.run_task` in a new thread
@@ -294,9 +294,9 @@ def l_start(self, value):
         try:
             # Get the task object by its type
             if 'child' in value.keys():
-                task_class = tasks.CHILDREN_LIST[value['task_type']]
+                task_class = autopilot.get('children', value['task_type'])
             else:
-                task_class = tasks.TASK_LIST[value['task_type']]
+                task_class = autopilot.get_task(value['task_type'])
             # Instantiate the task
             self.stage_block.clear()
 

autopilot/core/plots.py

@@ -12,32 +12,26 @@
 """
 
 # Classes for plots
-import sys
 import logging
-import os
 import numpy as np
 import PySide2 # have to import to tell pyqtgraph to use it
 import pandas as pd
-from PySide2 import QtGui
 from PySide2 import QtCore
-from PySide2 import QtOpenGL
 from PySide2 import QtWidgets
 import pyqtgraph as pg
 from time import time, sleep
 from itertools import count
 from functools import wraps
 from threading import Event, Thread
-import multiprocessing as mp
-import pdb
 from queue import Queue, Empty, Full
 #import cv2
 pg.setConfigOptions(antialias=True)
 # from pyqtgraph.widgets.RawImageWidget import RawImageWidget, RawImageGLWidget
 
-from autopilot import tasks, prefs
+import autopilot
+from autopilot import prefs
 from autopilot.core import styles
-from autopilot.core.utils import get_invoker
-from .utils import InvokeEvent, Invoker
+from ..utils.invoker import InvokeEvent, Invoker, get_invoker
 from autopilot.networking import Net_Node
 from autopilot.core.loggers import init_logger
 
@@ -329,7 +323,7 @@ def l_start(self, value):
         self.info['Protocol'].setText(value['step_name'])
 
         # We're sent a task dict, we extract the plot params and send them to the plot object
-        self.plot_params = tasks.TASK_LIST[value['task_type']].PLOT
+        self.plot_params = autopilot.get_task(value['task_type']).PLOT
 
         # if we got no plot params, that's fine, just set as running and return
         if not self.plot_params:

autopilot/core/subject.py

@@ -19,7 +19,7 @@
 import typing
 import warnings
 from copy import copy
-from autopilot.tasks import GRAD_LIST, TASK_LIST
+import autopilot
 from autopilot import prefs
 from autopilot.stim.sound.sounds import STRING_PARAMS
 from autopilot.core.loggers import init_logger
@@ -481,7 +481,7 @@ def assign_protocol(self, protocol, step_n=0):
         # memory, we can just keep appending to keep things simple.
         for i, step in enumerate(self.current):
             # First we get the task class for this step
-            task_class = TASK_LIST[step['task_type']]
+            task_class = autopilot.get_task(step['task_type'])
             step_name = step['step_name']
             # group name is S##_'step_name'
             group_name = "S{:02d}_{}".format(i, step_name)
@@ -709,7 +709,7 @@ def prepare_run(self):
         h5f.flush()
 
         # prepare continuous data group and tables
-        task_class = TASK_LIST[task_params['task_type']]
+        task_class = autopilot.get_task(task_params['task_type'])
         cont_group = None
         if hasattr(task_class, 'ContinuousData'):
             cont_group = h5f.get_node(group_name, 'continuous_data')
@@ -730,7 +730,7 @@ def prepare_run(self):
                 grad_params = task_params['graduation']['value'].copy()
 
                 # add other params asked for by the task class
-                grad_obj = GRAD_LIST[grad_type]
+                grad_obj = autopilot.get('graduation', grad_type)
 
                 if grad_obj.PARAMS:
                     # these are params that should be set in the protocol settings

autopilot/core/terminal.py

@@ -43,8 +43,8 @@
 from autopilot.core.subject import Subject
 from autopilot.core.plots import Plot_Widget
 from autopilot.networking import Net_Node, Terminal_Station
-from autopilot.core.utils import InvokeEvent, Invoker, get_invoker
-from autopilot.core.gui import Control_Panel, Protocol_Wizard, Weights, Reassign, Calibrate_Water, Bandwidth_Test, pop_dialog
+from autopilot.utils.invoker import get_invoker
+from autopilot.core.gui import Control_Panel, Protocol_Wizard, Weights, Reassign, Calibrate_Water, Bandwidth_Test, pop_dialog, Plugins
 from autopilot.core.loggers import init_logger
 
 # Try to import viz, but continue if that doesn't work
@@ -291,6 +291,11 @@ def initUI(self):
         bandwidth_test_act = QtWidgets.QAction("Test Bandwidth", self, triggered=self.test_bandwidth)
         self.tests_menu.addAction(bandwidth_test_act)
 
+        # Create a Plugins menu to manage plugins and provide a hook to give them additional terminal actions
+        self.plugins_menu = self.menuBar().addMenu("Plugins")
+        plugin = QtWidgets.QAction("Manage Plugins", self, triggered=self.manage_plugins)
+        self.plugins_menu.addAction(plugin)
+
 
         ## Init main panels and add to layout
         # Control panel sits on the left, controls pilots & subjects
@@ -893,6 +898,10 @@ def plot_psychometric(self):
             #viz.plot_psychometric(self.subjects_protocols)
         #result = psychometric_dialog.exec_()
 
+    def manage_plugins(self):
+        plugs = Plugins()
+        plugs.exec_()
+
     def closeEvent(self, event):
         """
         When Closing the Terminal Window, close any running subject objects,