add plotter

.gitignore

@@ -1,6 +1,7 @@
 .idea/
 tmp/
 data/
+output/
 
 ### PyCharm ###
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm

docs/source/peeper.analysis.rst

@@ -0,0 +1,28 @@
+peeper.analysis package
+=======================
+
+.. automodule:: peeper.analysis
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Submodules
+----------
+
+peeper.analysis.cli module
+--------------------------
+
+.. automodule:: peeper.analysis.cli
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+peeper.analysis.models module
+-----------------------------
+
+.. automodule:: peeper.analysis.models
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+

docs/source/peeper.rst

@@ -11,6 +11,7 @@ Subpackages
 
 .. toctree::
 
+    peeper.analysis
     peeper.ml
     peeper.preprocessing
 

peeper/analysis/cli.py

@@ -41,16 +41,32 @@ def parse_args(parser):
     return file
 
 
-def main():
-    file = parse_args(create_args())
-    log_message("Using file", file)
+def get_output_file(file):
+    """Finds output file suitable for input file
 
-    output_file = "plot.png"
-    output_file = os.path.join(os.path.dirname(file), output_file)
+    :param file: input file
+    :return: output file
+    """
+
+    output_file = "sensors.png"
+    output_folder = os.path.dirname(file)
+    output_file = os.path.join(output_folder, output_file)
+
+    if not os.path.exists(output_folder):  # create necessary folders
+        os.makedirs(output_folder)
 
     if os.path.exists(output_file):  # remove any previous outputs
         os.remove(output_file)
 
+    return output_file
+
+
+def main():
+    file = parse_args(create_args())
+    log_message("Using file", file)
+
+    output_file = get_output_file(file)
+
     driver = Plotter(file)
     driver.save(output_file)
 

peeper/analysis/models.py

@@ -2,9 +2,8 @@
 
 """Module containing analysis models"""
 
+import matplotlib.pyplot as plt
 import pandas as pd
-from hal.files.models.files import Document
-from hal.files.models.system import ls_dir
 
 
 class Plotter:
@@ -16,64 +15,87 @@ def __init__(self, file):
         """
 
         self.path = file
-        self.data = {
-            file: pd.read_csv(file)
-            for file in self._find_files()
-        }  # dictionary file name -> file data (as pandas data frame)
+        self.data = self._parse()
+        self.plots = self._create_plots()
 
-    def _find_files(self):
-        """Finds files in folder
+    def _parse(self):
+        """Parses input file
 
-        :return: list of input files in folder
+        :return: data in file
         """
 
-        files = ls_dir(self.path)
-        files = [
-            file
-            for file in files
-            if Document(file).extension == ".csv"  # just csv files
-        ]
-        return files
+        data = pd.read_csv(self.path)
+        data = data.set_index("Milliseconds")
+        data.index.names = ["Seconds"]
+        data = data.rename(index={
+            x: x / 1000.0
+            for x in data.index
+        })  # convert to s
 
-    def _merge(self):
-        """Merges data frames into one big
+        return data
+
+    def _create_plots(self):
+        """Create plots from data
 
-        :return: one big data frame with data from all input files
+        :return: dictionary with title and data to plot
         """
 
-        dfs = []  # list of data frames
-        for file, data in self.data.items():
-            file_name = Document(file).name
-            data = data.drop(["Timestamp"], axis=1)  # remove column
-            data = data.set_index("Milliseconds")  # set index
+        labels = {
+            "Compass": [
+                key
+                for key in self.data.keys()
+                if key.startswith("Compass")
+            ],
+            "RotationVector": [
+                key
+                for key in self.data.keys()
+                if key.startswith("RotationVector")
+            ],
+            "AccelerometerLinear": [
+                key
+                for key in self.data.keys()
+                if key.startswith("AccelerometerLinear")
+            ],
+            "Gyroscope": [
+                key
+                for key in self.data.keys()
+                if key.startswith("Gyroscope")
+            ]
+        }
 
-            # rename columns
+        plots = {
+            key: self.data[column_names]
+            for key, column_names in labels.items()
+        }
+        for key, plot in plots.items():
             new_columns = {
-                col: file_name + " " + col
-                for col in data.keys()
+                column: column.replace(key, "").strip()  # remove unnecessary
+                for column in plot.keys()
             }
-            data = data.rename(index=str, columns=new_columns)  # rename columns
-
-            dfs.append(data)
+            plots[key] = plot.rename(columns=new_columns)
 
-        data = pd.concat(dfs, axis=1, sort=True)  # merge
-
-        # rename rows (convert to float)
-        new_rows = {
-            row: float(row)
-            for row in data.index
-        }
-        data = data.rename(index=new_rows)
-        data = data.sort_index()  # sort by index
-
-        return data
+        return plots
 
     def save(self, output_file):
-        """Merges all inputs files into one
+        """Saves plot to output
 
         :param output_file: output file (where to write data)
         """
 
-        data = self._merge()
-        data = sample_by_frequency(data, 5)
-        data.to_csv(output_file, index_label="Milliseconds")
+        fig, ax = plt.subplots(2, 2, sharex="all")
+
+        self.plots["Compass"].plot(ax=ax[0, 0], title="Compass")
+        self.plots["RotationVector"].plot(ax=ax[0, 1], title="Rotation vector")
+        self.plots["AccelerometerLinear"] \
+            .plot(ax=ax[1, 0], title="Accelerations")
+        self.plots["Gyroscope"].plot(ax=ax[1, 1], title="Gyro")
+
+        plt.savefig(
+            output_file,
+            dpi=400,
+            quality=100,
+            orientation="landscape",
+            papertype="a4",
+            format="png"
+        )
+        plt.show()

peeper/preprocessing/cli.py

@@ -39,16 +39,38 @@ def parse_args(parser):
     return directory
 
 
-def main():
-    folder = parse_args(create_args())
-    log_message("Using folder", folder)
+def get_output_file(folder):
+    """Finds output file suitable for input folder
 
-    output_file = "Merged.csv"
-    output_file = os.path.join(os.path.dirname(folder), output_file)
+    :param folder: input folder
+    :return: output file
+    """
+
+    folders = folder.split(os.path.sep)
+    data_time = folders[-3]
+    data_day = folders[-4]
+    output_file = "sensors.csv"
+    output_folder = folder
+    for _ in range(5):
+        output_folder = os.path.dirname(output_folder)
+    output_folder = os.path.join(output_folder, "output", data_day, data_time)
+    output_file = os.path.join(output_folder, output_file)
+
+    if not os.path.exists(output_folder):  # create necessary folders
+        os.makedirs(output_folder)
 
     if os.path.exists(output_file):  # remove any previous outputs
         os.remove(output_file)
 
+    return output_file
+
+
+def main():
+    folder = parse_args(create_args())
+    log_message("Using folder", folder)
+
+    output_file = get_output_file(folder)
+
     driver = Merger(folder)
     driver.merge_into(output_file)
 

peeper/preprocessing/models.py

@@ -37,6 +37,10 @@ def sample_by_frequency(data, hertz):
 
             averages = sample_data.apply(np.nanmean, axis=0)  # average sample
             row = [sampled_time] + averages.tolist()
+            for j, val in enumerate(row):
+                if np.isnan(val):
+                    row[j] = sampled_data[-1][j]  # last known value
+
             sampled_data.append(row)
 
             start_sample_index = end_sample_index + 1
@@ -89,6 +93,7 @@ def _merge(self):
             file_name = Document(file).name
             data = data.drop(["Timestamp"], axis=1)  # remove column
             data = data.set_index("Milliseconds")  # set index
+            data = data.groupby(data.index).first()  # remove duplicate index
 
             # rename columns
             new_columns = {