base stuff

.gitignore

@@ -0,0 +1,66 @@
+integration_testing/*.csv
+integration_testing/*.tsv
+integration_testing/*.sql
+integration_testing/__*
+test_data.csv
+train_data.csv
+*.ipy*
+*.test.*
+.cache*
+storage/*
+mindsdb_storage/*
+config/personal_config.py
+*.jar
+data/*
+mindsdb.egg-info
+clean_data
+.pypirc
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# visual studio code
+.DStore
+.DS_Store
+.idea
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# pyenv
+.python-version
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+docs/_build
+docs/_static
+docs/_templates
+docs/index.rst
+docs/make.bat
+docs/Makefile
+docs/source

.travis.yml

@@ -0,0 +1,15 @@
+language: python
+matrix:
+  include:
+    - python: 3.5
+    - python: 3.6
+    - python: 3.7
+      dist: xenial
+      sudo: true 
+cache: pip
+# install dependencies
+install:
+  - travis_wait pip install lightwood
+# run tests
+script:
+  - pytest

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018-  MindsDB, Inc   (Jorge Torres)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,3 @@
+
+![Lightwood](https://raw.githubusercontent.com/mindsdb/lightwood/master/assets/img/logo.png "Lightwood") v.0.9.0 
+#

TODO.md

@@ -0,0 +1,3 @@
+* Start repo mindsdb server
+* Respond to Alex 
+* 

__init__.py

@@ -0,0 +1,2 @@
+from lightwood.constants import lightwood
+name = "lightwood"

install.sh

@@ -0,0 +1,38 @@
+rm -rf build
+rm -rf dist
+rm -rf lightwood.egg-info
+# python3 setup.py --help-commands
+
+echo "mode (prod/dev)?"
+
+read mode
+
+if [ "$mode" = "prod" ]; then
+
+    python3 setup.py develop --uninstall
+    python3 setup.py clean
+    python3 setup.py build
+    #python3 setup.py install
+    #python3 setup.py bdist_egg -p win32
+    python3 setup.py sdist bdist_wheel
+
+    echo "Do you want to publish this version (yes/no)?"
+
+    read publish
+
+    if [ "$publish" = "yes" ]; then
+        echo "Publishing lightwood to Pypi"
+        python3 -m twine upload dist/*
+    fi
+
+
+fi
+
+if [ "$mode" = "dev" ]; then
+    pip3 uninstall lightwood
+    python3 setup.py develop --uninstall
+    python3 setup.py develop
+fi
+
+
+

lightwood/__init__.py

@@ -0,0 +1,6 @@
+import sys
+if sys.version_info < (3,3):
+    sys.exit('Sorry, For Lightwood Python < 3.3 is not supported')
+
+import lightwood.constants.lightwood as  CONST
+from lightwood.api.predictor import Predictor

lightwood/api/predictor.py

@@ -0,0 +1,101 @@
+import traceback
+
+from lightwood.data_schemas.definition import definition_schema
+from lightwood.constants.lightwood import COLUMN_DATA_TYPES, HISTOGRAM_TYPES
+from lightwood.classes.dataset import Dataset
+
+class Predictor:
+
+
+
+    def __init__(self, definition):
+        """
+        Start a predictor pass the
+
+        :param definition: a predictor definition object (can be a dictionary or a PredictorDefinition object)
+        :type definition: dictionary
+        """
+        try:
+            definition_schema.validate(definition)
+        except:
+            error = traceback.format_exc(1)
+            raise ValueError('[BAD DEFINITION] argument has errors: {err}'.format(err=error))
+
+        self.definition = definition
+        self._model = None
+        self._encoders = None
+
+
+    def learn(self, from_data, test_data, validation_data):
+        """
+        Train and save a model (you can use this to retrain model from data)
+
+        :param from_data:
+        :param test_data:
+        :param validation_data:
+        :return:
+        """
+
+        model, encoders = self._load()
+
+        validation_dataset = Dataset(df=validation_data, parent_predictor_object=self)
+        test_dataset = Dataset(df=test_data, parent_predictor_object=self)
+
+        train_dataset = Dataset(df=from_data, parent_predictor_object=self)
+        train_dataset.load_encoders(encoders) # if encoders exist start from there
+        train_dataset.encode(validation_dataset=validation_dataset)
+
+        if model is None: # build model if no model exists
+            model = Model(definition=self.definition)
+
+        model.train(train_dataset, validation_dataset = test_dataset)
+
+        self._save(model = model, encoders = train_dataset.get_encoders())
+
+
+    def predict(self, when_data):
+        """
+        Predict given when conditions
+        :param when: a dataframe
+        :return: a complete dataframe
+        """
+
+        model, encoders = self._load()
+
+        when_dataset = Dataset(df=when_data, parent_predictor_object=self)
+        when_dataset.load_encoders(encoders)
+        when_dataset.encode(train_encoders = False)
+
+        complete_dataset = model.forward(when_dataset)
+
+        return complete_dataset.get_df()
+
+
+    def _save(self, model, encoders):
+        """
+
+        :param model:
+        :param encoders:
+        :return:
+        """
+
+        pass
+
+    def _load(self, use_cache = True):
+        """
+
+        :param use_cache: (default True) if the model already in memory use it
+        :return: model, encoders
+        """
+        model = self._model
+        encoders = self._encoders
+
+        return model, encoders
+
+
+
+
+# only run the test if this file is called from debugger
+if __name__ == "__main__":
+    Predictor(definition = {'name':'Will', 'input_features': [{'name':'trio', 'type': COLUMN_DATA_TYPES.TEXT}]} )
+

lightwood/classes/column_abstract.py

@@ -0,0 +1,30 @@
+
+class ColumnAbstract:
+
+    def __init__(self, type):
+        """
+        This initializes the column, and has a pointer to the parent dataset,
+        which includes the dataframe as well as the predictor
+
+        :param column_name: the column that we are analyzing
+        :param parent_dataset: the parent dataset
+        """
+
+        self._type = None # This column is
+        self._encoded_status = None # None, Encoding, Done
+        pass
+
+    def encode(self):
+        """
+        This is meant to run asynchronously, and once its done encoding it should change teh self._encoded status to Done
+        :return: None
+        """
+        pass
+
+
+    def decode(self):
+        """
+
+        :return:
+        """
+        pass

lightwood/classes/dataset.py

@@ -0,0 +1,15 @@
+
+class Dataset:
+
+    def __init__(self, df, parent_predictor_object):
+        """
+        initialize the dataset
+        :param df: the dataframe that defines the dataset
+        :param parent_predictor_object:  the parent predictor that owns this dataset
+        """
+
+        self.df = df
+
+        pass
+
+

lightwood/column_data_types/text/encoders/rnn.py

@@ -0,0 +1,3 @@
+
+
+class RnnEncoder():

lightwood/column_data_types/text/text.py

@@ -0,0 +1,2 @@
+
+class Text