Merge pull request #225 from earthgecko/py3

upload_data - flux and webapp
earthgecko · May 20, 2020 · 1da5870 · 1da5870
2 parents c717030 + d1f20eb
commit 1da5870
Show file tree

Hide file tree

Showing 10 changed files with 88 additions and 34 deletions.
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -523,4 +523,7 @@ falcon==2.0.0
 # @added 20200517 - Feature #3550: flux.uploaded_data_worker
 #                   Feature #3538: webapp - upload_data endpoint
 xlrd==1.2.0
-pandas_log==0.1.7; python_version >= '3.2'
+# @added 20200520 - Feature #3550: flux.uploaded_data_worker
+# pin to 0.1.6 until pandas-1.0.3 has been thoroughly tested
+#pandas_log==0.1.7; python_version >= '3.2'
+pandas_log==0.1.6; python_version >= '3.2'
diff --git a/docs/getting-data-into-skyline.rst b/docs/getting-data-into-skyline.rst
@@ -118,7 +118,7 @@ Skyline, see the `Flux <flux.html>`__ page.
 upload_data to Flux
 ~~~~~~~~~~~~~~~~~~~
 
-See the `upload_data to Flux <uploa-data-to-flux.html>`__ page.
+See the `upload_data to Flux <upload-data-to-flux.html>`__ page.
 
 Vista
 ~~~~~

diff --git a/docs/skyline.analyzer.rst b/docs/skyline.analyzer.rst
@@ -12,6 +12,14 @@ skyline.analyzer.agent module
     :undoc-members:
     :show-inheritance:
 
+skyline.analyzer.agent_batch module
+-----------------------------------
+
+.. automodule:: analyzer.agent_batch
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 skyline.analyzer.alerters module
 --------------------------------
 
@@ -28,6 +36,22 @@ skyline.analyzer.algorithms module
     :undoc-members:
     :show-inheritance:
 
+skyline.analyzer.algorithms_batch module
+----------------------------------------
+
+.. automodule:: analyzer.algorithms_batch
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+skyline.analyzer.analyzer.debug.nonNegativeDerivative module
+------------------------------------------------------------
+
+.. automodule:: analyzer.analyzer.debug.nonNegativeDerivative
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 skyline.analyzer.analyzer module
 --------------------------------
 
@@ -36,6 +60,14 @@ skyline.analyzer.analyzer module
     :undoc-members:
     :show-inheritance:
 
+skyline.analyzer.analyzer_batch module
+--------------------------------------
+
+.. automodule:: analyzer.analyzer_batch
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 
 Module contents
 ---------------

diff --git a/docs/skyline.crucible.rst b/docs/skyline.crucible.rst
@@ -28,6 +28,14 @@ skyline.crucible.crucible_algorithms module
     :undoc-members:
     :show-inheritance:
 
+skyline.crucible.crucible_firings module
+----------------------------------------
+
+.. automodule:: crucible.crucible_firings
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 
 Module contents
 ---------------

diff --git a/docs/skyline.flux.rst b/docs/skyline.flux.rst
@@ -52,6 +52,14 @@ skyline.flux.populate_metric_worker module
     :undoc-members:
     :show-inheritance:
 
+skyline.flux.uploaded_data_worker module
+----------------------------------------
+
+.. automodule:: flux.uploaded_data_worker
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 skyline.flux.worker module
 --------------------------
 

diff --git a/docs/skyline.rst b/docs/skyline.rst
@@ -31,6 +31,14 @@ skyline.algorithm_exceptions module
     :undoc-members:
     :show-inheritance:
 
+skyline.create_matplotlib_graph module
+--------------------------------------
+
+.. automodule:: create_matplotlib_graph
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 skyline.database module
 -----------------------
 
@@ -55,6 +63,14 @@ skyline.ionosphere_functions module
     :undoc-members:
     :show-inheritance:
 
+skyline.matched_or_regexed_in_list module
+-----------------------------------------
+
+.. automodule:: matched_or_regexed_in_list
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 skyline.settings module
 -----------------------
 

diff --git a/docs/skyline.webapp.rst b/docs/skyline.webapp.rst
@@ -12,6 +12,14 @@ skyline.webapp.backend module
     :undoc-members:
     :show-inheritance:
 
+skyline.webapp.crucible_backend module
+--------------------------------------
+
+.. automodule:: webapp.crucible_backend
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 skyline.webapp.gunicorn module
 ------------------------------
 

diff --git a/docs/upload-data-to-flux.rst b/docs/upload-data-to-flux.rst
@@ -165,33 +165,3 @@ curl request.
          -F "info_file=@<FULL_PATH_TO_FILE>/info.json" \
          -F "json_response=true" \
          https://$SKYLINE_HOST/upload_data
-
-
-Vista
-~~~~~
-
-Metrics to be fetched by Vista which submits to Flux, see the
-`Vista <vista.html>`__ page.
-
-Adding a Listener
-=================
-
-If neither of these listeners are acceptable, it's easy enough to extend
-them. Add a method in listen.py and add a line in the horizon-agent that
-points to your new listener.
-
-:mod:`settings.FULL_DURATION`
-=============================
-
-Once you get real data flowing through your system, the Analyzer will be
-able start analyzing for anomalies.
-
-.. note:: Do not expect to see anomalies or anything in the Webapp immediately
-  after starting the Skyline services. Realistically :mod:`settings.FULL_DURATION`
-  should have been passed, before you begin to assess any triggered anomalies,
-  after all :mod:`settings.FULL_DURATION` is the baseline.  Although not all
-  algorithms utilize all the :mod:`settings.FULL_DURATION` data points, some do
-  and some use only 1 hour's worth.  However the Analyzer log should still report
-  values in the exception stats, reporting how many metrics were boring, too
-  short, etc as soon as it is getting data for metrics that Horizon is populating
-  into Redis.
diff --git a/requirements.txt b/requirements.txt
@@ -53,4 +53,4 @@ graphyte==1.6.0
 statsd==3.3.0
 falcon==2.0.0
 xlrd==1.2.0
-pandas_log==0.1.7; python_version >= '3.2'
+pandas_log==0.1.6; python_version >= '3.2'
diff --git a/tests/algorithms_test.py b/tests/algorithms_test.py
@@ -75,7 +75,16 @@ def test_run_selected_algorithm(self, timeMock):
         # Added the airgapped_metrics list
         # result, ensemble, datapoint = algorithms.run_selected_algorithm(timeseries, "test.metric")
         airgapped_metrics = ['test.metric.airgapped.1', 'test.metric.airgapped.2']
-        result, ensemble, datapoint = algorithms.run_selected_algorithm(timeseries, "test.metric", airgapped_metrics)
+        # @modified 20200520 - Feature #3400: Identify air gaps in the metric data
+        #                      Feature #3508: ionosphere.untrainable_metrics
+        #                      Feature #3504: Handle airgaps in batch metrics
+        # Added airgapped_metrics_filled, run_negatives_present and
+        # check_for_airgaps_only
+        # result, ensemble, datapoint = algorithms.run_selected_algorithm(timeseries, "test.metric", airgapped_metrics)
+        airgapped_metrics_filled = []
+        run_negatives_present = False
+        check_for_airgaps_only = False
+        result, ensemble, datapoint, negatives_found = algorithms.run_selected_algorithm(timeseries, 'test.metric', airgapped_metrics, airgapped_metrics_filled, run_negatives_present, check_for_airgaps_only)
 
         self.assertTrue(result)
         self.assertTrue(len(filter(None, ensemble)) >= settings.CONSENSUS)