From 33a0a2aa188af2b5a8a732a420a67fc6f2055c0d Mon Sep 17 00:00:00 2001 From: Phillipp Drieger Date: Sat, 6 Feb 2021 11:32:17 +0100 Subject: [PATCH] Example for stumpy Fixes #11 --- app/model/stumpy.py | 141 +++++++++++++++ notebooks/stumpy.ipynb | 400 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 541 insertions(+) create mode 100644 app/model/stumpy.py create mode 100644 notebooks/stumpy.ipynb diff --git a/app/model/stumpy.py b/app/model/stumpy.py new file mode 100644 index 0000000..9353e45 --- /dev/null +++ b/app/model/stumpy.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +# coding: utf-8 + + + +# In[1]: + + +# this definition exposes all python module imports that should be available in all subsequent commands +import json +import numpy as np +import pandas as pd +import datetime as dt +import stumpy +# ... +# global constants +MODEL_DIRECTORY = "/srv/app/model/data/" + + + + + + + + +# In[36]: + + +# this cell is not executed from MLTK and should only be used for staging data into the notebook environment +def stage(name): + with open("data/"+name+".csv", 'r') as f: + df = pd.read_csv(f) + with open("data/"+name+".json", 'r') as f: + param = json.load(f) + return df, param + + + + + + + + +# In[38]: + + +# initialize your model +# available inputs: data and parameters +# returns the model object which will be used as a reference to call fit, apply and summary subsequently +def init(df,param): + model = {} + return model + + + + + + + + +# In[40]: + + +# train your model +# returns a fit info json object and may modify the model object +def fit(model,df,param): + # model.fit() + info = {"message": "model created"} + return info + + + + + + + + +# In[42]: + + +# apply your model +# returns the calculated results +def apply(model,df,param): + m = 24 + if 'options' in param: + if 'params' in param['options']: + if 'm' in param['options']['params']: + m = int(param['options']['params']['m']) + target = param['target_variables'][0] + mp = stumpy.stump(df[target], m) + result = pd.DataFrame(mp[:, 0], columns=['matrix_profile']) + return pd.concat([df, result], axis=1) + + + + + + + + +# In[33]: + + +# save model to name in expected convention "_" +def save(model,name): + with open(MODEL_DIRECTORY + name + ".json", 'w') as file: + json.dump(model, file) + return model + + + + + + +# In[34]: + + +# load model from name in expected convention "_" +def load(name): + model = {} + with open(MODEL_DIRECTORY + name + ".json", 'r') as file: + model = json.load(file) + return model + + + + + + +# In[35]: + + +# return a model summary +def summary(model=None): + returns = {"version": {"numpy": np.__version__, "pandas": pd.__version__} } + return returns + + + + + diff --git a/notebooks/stumpy.ipynb b/notebooks/stumpy.ipynb new file mode 100644 index 0000000..1d36cfb --- /dev/null +++ b/notebooks/stumpy.ipynb @@ -0,0 +1,400 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deep Learning Toolkit for Splunk - Barebone Notebook" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook contains a barebone example workflow how to work on custom containerized code that seamlessly interfaces with the Deep Learning Toolkit for Splunk." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: By default every time you save this notebook the cells are exported into a python module which is then invoked by Splunk MLTK commands like | fit ... | apply ... | summary . Please read the Model Development Guide in the Deep Learning Toolkit app for more information." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Stage 0 - import libraries\n", + "At stage 0 we define all imports necessary to run our subsequent code depending on various libraries." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "deletable": false, + "name": "mltkc_import" + }, + "outputs": [], + "source": [ + "# this definition exposes all python module imports that should be available in all subsequent commands\n", + "import json\n", + "import numpy as np\n", + "import pandas as pd\n", + "import datetime as dt\n", + "import stumpy\n", + "# ...\n", + "# global constants\n", + "MODEL_DIRECTORY = \"/srv/app/model/data/\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "numpy version: 1.18.1\n", + "pandas version: 1.0.1\n", + "stumpy version: 1.7.2\n" + ] + } + ], + "source": [ + "# THIS CELL IS NOT EXPORTED - free notebook cell for testing or development purposes\n", + "print(\"numpy version: \" + np.__version__)\n", + "print(\"pandas version: \" + pd.__version__)\n", + "print(\"stumpy version: \" + stumpy.__version__)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Stage 1 - get a data sample from Splunk\n", + "In Splunk run a search to pipe a dataset into your notebook environment. Note: mode=stage is used in the | fit command to do this." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "| inputlookup cyclical_business_process.csv
\n", + "| eval _time=strptime(_time, \"%Y-%m-%dT%H:%M:%S\")
\n", + "| timechart span=15m avg(logons) as logons
\n", + "| fit MLTKContainer mode=stage algo=stumpy logons from _time into app:stumpy_anomalies
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After you run this search your data set sample is available as a csv inside the container to develop your model. The name is taken from the into keyword or set to \"default\" if no into keyword is present. This step is intended to work with a subset of your data to create your custom model." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "deletable": false, + "name": "mltkc_stage" + }, + "outputs": [], + "source": [ + "# this cell is not executed from MLTK and should only be used for staging data into the notebook environment\n", + "def stage(name):\n", + " with open(\"data/\"+name+\".csv\", 'r') as f:\n", + " df = pd.read_csv(f)\n", + " with open(\"data/\"+name+\".json\", 'r') as f:\n", + " param = json.load(f)\n", + " return df, param" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " _time logons\n", + "count 3.681000e+03 3681.000000\n", + "mean 1.534990e+09 55.685511\n", + "std 9.564817e+05 42.976066\n", + "min 1.533334e+09 0.666667\n", + "25% 1.534162e+09 18.666667\n", + "50% 1.534990e+09 43.333333\n", + "75% 1.535818e+09 85.000000\n", + "max 1.536646e+09 245.266667\n", + "{'options': {'params': {'mode': 'stage', 'algo': 'stumpy'}, 'args': ['logons', '_time'], 'target_variable': ['logons'], 'feature_variables': ['_time'], 'model_name': 'stumpy_anomalies', 'algo_name': 'MLTKContainer', 'mlspl_limits': {'disabled': False, 'handle_new_cat': 'default', 'max_distinct_cat_values': '10000', 'max_distinct_cat_values_for_classifiers': '10000', 'max_distinct_cat_values_for_scoring': '10000', 'max_fit_time': '6000', 'max_inputs': '100000000', 'max_memory_usage_mb': '4000', 'max_model_size_mb': '150', 'max_score_time': '6000', 'streaming_apply': '0', 'use_sampling': '1'}, 'kfold_cv': None}, 'feature_variables': ['_time'], 'target_variables': ['logons']}\n" + ] + } + ], + "source": [ + "# THIS CELL IS NOT EXPORTED - free notebook cell for testing or development purposes\n", + "df, param = stage(\"stumpy_anomalies\")\n", + "print(df.describe())\n", + "print(param)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Stage 2 - create and initialize a model" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "deletable": false, + "name": "mltkc_init" + }, + "outputs": [], + "source": [ + "# initialize your model\n", + "# available inputs: data and parameters\n", + "# returns the model object which will be used as a reference to call fit, apply and summary subsequently\n", + "def init(df,param):\n", + " model = {}\n", + " return model" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{}\n" + ] + } + ], + "source": [ + "# THIS CELL IS NOT EXPORTED - free notebook cell for testing or development purposes\n", + "model = init(df,param)\n", + "print(model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Stage 3 - fit the model" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "deletable": false, + "name": "mltkc_fit" + }, + "outputs": [], + "source": [ + "# train your model\n", + "# returns a fit info json object and may modify the model object\n", + "def fit(model,df,param):\n", + " # model.fit()\n", + " info = {\"message\": \"model created\"}\n", + " return info" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'message': 'model created'}\n" + ] + } + ], + "source": [ + "# THIS CELL IS NOT EXPORTED - free notebook cell for testing or development purposes\n", + "print(fit(model,df,param))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Stage 4 - apply the model" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "deletable": false, + "name": "mltkc_apply" + }, + "outputs": [], + "source": [ + "# apply your model\n", + "# returns the calculated results\n", + "def apply(model,df,param):\n", + " m = 24\n", + " if 'options' in param:\n", + " if 'params' in param['options']:\n", + " if 'm' in param['options']['params']:\n", + " m = int(param['options']['params']['m'])\n", + " target = param['target_variables'][0]\n", + " mp = stumpy.stump(df[target], m) \n", + " result = pd.DataFrame(mp[:, 0], columns=['matrix_profile'])\n", + " return pd.concat([df, result], axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " _time logons matrix_profile\n", + "0 1533333600 0.666667 1.32494\n", + "1 1533334500 1.233333 1.6778\n", + "2 1533335400 2.000000 1.95066\n", + "3 1533336300 1.233333 2.03543\n", + "4 1533337200 3.000000 2.05146\n", + "... ... ... ...\n", + "3676 1536642000 50.000000 NaN\n", + "3677 1536642900 50.666667 NaN\n", + "3678 1536643800 47.333333 NaN\n", + "3679 1536644700 58.533333 NaN\n", + "3680 1536645600 17.100000 NaN\n", + "\n", + "[3681 rows x 3 columns]\n" + ] + } + ], + "source": [ + "# THIS CELL IS NOT EXPORTED - free notebook cell for testing or development purposes\n", + "print(apply(model,df,param))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Stage 5 - save the model" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "deletable": false, + "name": "mltkc_save" + }, + "outputs": [], + "source": [ + "# save model to name in expected convention \"_\"\n", + "def save(model,name):\n", + " with open(MODEL_DIRECTORY + name + \".json\", 'w') as file:\n", + " json.dump(model, file)\n", + " return model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Stage 6 - load the model" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "deletable": false, + "name": "mltkc_load" + }, + "outputs": [], + "source": [ + "# load model from name in expected convention \"_\"\n", + "def load(name):\n", + " model = {}\n", + " with open(MODEL_DIRECTORY + name + \".json\", 'r') as file:\n", + " model = json.load(file)\n", + " return model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Stage 7 - provide a summary of the model" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "deletable": false, + "name": "mltkc_summary" + }, + "outputs": [], + "source": [ + "# return a model summary\n", + "def summary(model=None):\n", + " returns = {\"version\": {\"numpy\": np.__version__, \"pandas\": pd.__version__} }\n", + " return returns" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## End of Stages\n", + "All subsequent cells are not tagged and can be used for further freeform code" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}