Merge pull request #21 from ralphg6/devel

Architecture Evolution

.gitignore

@@ -5,6 +5,8 @@
 *.eggs
 *.egg-info
 *.log
+*.swp
+*.swo
 *build/
 *dat
 .env

david/__init__.py

@@ -1,4 +1,4 @@
 from david import version
 
 # define the version before the other imports since these need it
-__version__ = version.__version__
+__version__ = version.__version__

david/__main__.py

@@ -3,44 +3,51 @@
 from david.assistant import Assistant
 from david.googleadap import GoogleWebHook
 from david.dialog import fetch_dialog
-from david.brain import fetch_model, fetch_know
+
+# from david.brain import fetch_model, fetch_know
 
 app = Flask(__name__)
 CORS(app)
 
 assistant = Assistant()
 googleWH = GoogleWebHook(assistant)
 
-@app.route('/')
+
+@app.route("/")
 def hi():
-    return 'Hi, am i David!'
+    return "Hi, am i David!"
 
-@app.route('/train')
+
+@app.route("/train")
 def train():
     assistant.train()
-    return "OK"   
+    return "OK"
+
 
-@app.route('/dialog', methods=['POST'])
+@app.route("/dialog", methods=["POST"])
 def dialog():
     data = request.get_json()
-    return jsonify(assistant.respond(data['input']))    
-
-@app.route('/google', methods=['POST'])
+    return jsonify(assistant.respond(data["input"]))
+
+
+@app.route("/google", methods=["POST"])
 def google():
     data = request.get_json()
-    return jsonify(googleWH.handle(data))  
+    return jsonify(googleWH.handle(data))
+
+
+# @app.route('/data/dialog')
+# def data_dialog():
+# return jsonify(fetch_dialog())
 
-@app.route('/data/dialog')
-def data_dialog():
-    return jsonify(fetch_dialog())  
+# @app.route('/data/know')
+# def data_know():
+# return jsonify(fetch_know())
 
-@app.route('/data/know')
-def data_know():
-    return jsonify(fetch_know())           
 
 def main() -> None:
-    app.run(host='0.0.0.0')
+    app.run(host="0.0.0.0")
 
 
 if __name__ == "__main__":
-    main()
+    main()

david/assistant.py

@@ -1,8 +1,10 @@
 from david.brain import Brain
 from david.dialog import Dialog
 
-class Assistant:
+from david.constants import INTENTS_ATTRIBUTE, ENTITIES_ATTRIBUTE, CONTEXT_ATTRIBUTE
+
 
+class Assistant:
     def __init__(self):
         self.brain = Brain()
         self.dialog = Dialog()
@@ -12,14 +14,18 @@ def train(self):
         self.brain.train()
         self.dialog.train()
 
-    def respond(self, input, context = {}):
-        intents = self.brain.classify(input)
-        entities = self.brain.nlp(input)
-        #print("intents", intents)
+    def respond(self, input, context={}):
+        message = self.brain.process(input)
+
+        intents = message.get(INTENTS_ATTRIBUTE)
+        entities = message.get(ENTITIES_ATTRIBUTE)
+        context = message.get(CONTEXT_ATTRIBUTE)
+
+        # print("intents", intents)
         dialog_node = self.dialog.dialog(input, context, intents, entities)
         return {
             "context": context,
             "intents": intents,
             "entities": entities,
-            "output": dialog_node["output"]
-        }   
+            "output": dialog_node["output"],
+        }

david/brain.py

@@ -1,33 +1,18 @@
 import json
 import david.util as util
 import os
-from Levenshtein import distance
+
 
 from david.components.nlu.simplenlu import SimpleNLU
 
 from david.config import DavidConfig
 
-SIMMILARITY_ERROR_ACCEPTED = 0.3
-INTENT_MODEL_FILE = './models/intent_model.json'
-KNOW_FILE = './data/know.json'
-
-
-def fetch_model():
-    try:
-        with open(INTENT_MODEL_FILE) as f:
-            return json.load(f)
-    except:
-        return
+from david.training_data import Message
 
 
-def persist_model(model):
-
-    model_folder = os.path.dirname(INTENT_MODEL_FILE)
-    if not os.path.exists(model_folder):
-        os.makedirs(model_folder)
-
-    with open(INTENT_MODEL_FILE, 'w') as outfile:
-        json.dump(model, outfile)
+MODEL_DIR = "./models/"
+MODEL_FILE = "intent_model.json"
+KNOW_FILE = "./data/know.json"
 
 
 def fetch_know():
@@ -36,91 +21,35 @@ def fetch_know():
 
 
 def persist_know(data):
-    with open(KNOW_FILE, 'w') as outfile:
+    with open(KNOW_FILE, "w") as outfile:
         json.dump(data, outfile)
 
-# def fetch_stopwords():
-#     return set(line.strip() for line in open('./data/stopwords.txt', 'r'))
-
-
-def simmilarity(a, b):
-    d = distance(a, b)
-    t = float(len(a) + len(b)) / 2
-    if (d / t <= SIMMILARITY_ERROR_ACCEPTED):
-        return (t - d) / t
-    return 0
 
 # def persist_stopwords(data):
 #     with open('./data/stopwords.json', 'w') as outfile:
 #         json.dump(data, outfile)
 
 
 class Brain:
-
     def __init__(self):
         # print("brain inited")
-        # from david.training_data import TrainingData
-        # training_data = TrainingData()
-        # config = DavidConfig()
-        # self.nlu = SimpleNLU()
-        # self.nlu.train(training_data, config)
-
-        self.intent_model = fetch_model()
-        if self.intent_model is None:
-            self.train()
-        # self.stopwords = fetch_stopwords()
+        self.train()
 
-    def train(self):
+    def train(self) -> None:
+        from david.training_data import TrainingData
 
         know = fetch_know()
-        # self.stopwords = fetch_stopwords()
-
-        self.intent_model = {}
-
-        for intent, samples in know["intents"].items():
-            self.intent_model[intent] = {}
-            for sample in samples:
-                self.intent_model[intent][sample] = {
-                    "total" : 0,
-                    "tokens" : {}
-                }
-                for t in util.tokenize(sample):
-                    self.intent_model[intent][sample]["total"] += 1
-                    if t in self.intent_model[intent][sample]["tokens"]:
-                        self.intent_model[intent][sample]["tokens"][t] += 1
-                    else:
-                        self.intent_model[intent][sample]["tokens"][t] = 1
-
-        persist_model(self.intent_model)
-
-        return self.intent_model
-
-    def classify(self, input):
-        tokens = util.tokenize(input)
-        #print ("tokens", tokens)
-        intents = {}
-        for intent, samples in self.intent_model.items():
-            intents[intent] = 0
-            for s, smeta in samples.items():
-                brutal_score = 0
-                stokens = smeta["tokens"]
-                for t in tokens:
-                    for st, value in stokens.items():
-                        #print t, st, simmilarity(t, st), value
-                        brutal_score += simmilarity(t, st) * value
-                score = float(brutal_score) / smeta["total"]
-                #print("brutal_score", s, brutal_score, smeta, intents[intent], score)        
-                if intents[intent] < score:
-                    intents[intent] = score
-
-        intents = [{
-            "intent": intent,
-            "confidence": intents[intent]
-        } for intent in sorted(intents, key=intents.__getitem__, reverse = True)]
-
-        intents = list(filter(lambda i: i["confidence"] > 0, intents))
-
-        return intents[:10]
+
+        training_data = TrainingData(know)
+        config = DavidConfig()
+        self.nlu = SimpleNLU.create({}, config)
+        self.nlu.train(training_data, config)
+        self.nlu.persist(file_name=MODEL_FILE, model_dir=MODEL_DIR)
+
+    def process(self, text) -> Message:
+        message = Message.build(text)
+        self.nlu.process(message)
+        return message
 
     def nlp(self, input):
-        return []
+        return []

david/components/__init__.py

@@ -1 +1 @@
-from david.components.component import Component
+from david.components.component import Component

david/components/component.py

@@ -5,6 +5,7 @@
 
 ## Component class inspired by RasaHQ/rasa
 
+
 class ComponentMetaclass(type):
     """Metaclass with `name` class property"""