Initialize the agent (#34)

* First approach at reading/writing from stdin/stdout

* Refactor to avoid request-response

* Move handler into better named package

* Marshal messages

* Add demo message types

* Remove executor for StdStreams.write

* Use a thread instead of an executor for StdStreams

* Create Vim Plugin (#29)

* Created initial vim plugin (python-supported versions of vim only)

* Use test client code in vim plugin. Spawn and communicate with agent instance

* Add python3 check

* Fix lint errors and use double quotes

* Hide thread code from StdStreams

* Remove unnecessary do not call comment

* Add missing init files

agent/main.py

@@ -0,0 +1,35 @@
+import signal
+import logging
+import threading
+from tandem.executables.agent import TandemAgent
+
+should_shutdown = threading.Event()
+
+
+def signal_handler(signal, frame):
+    global should_shutdown
+    should_shutdown.set()
+
+
+def set_up_logging():
+    logging.basicConfig(
+        level=logging.DEBUG,
+        format="%(asctime)s %(levelname)-8s %(message)s",
+        datefmt="%Y-%m-%d %H:%M",
+        filename="/tmp/tandem-agent.log",
+        filemode="w",
+    )
+
+
+def main():
+    set_up_logging()
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    # Run the agent until asked to terminate
+    with TandemAgent() as agent:
+        should_shutdown.wait()
+
+
+if __name__ == "__main__":
+    main()

agent/tandem/executables/agent.py

@@ -0,0 +1,31 @@
+import logging
+from tandem.io.std_streams import StdStreams
+from tandem.protocol.editor.handler import EditorProtocolHandler
+from concurrent.futures import ThreadPoolExecutor
+
+
+class TandemAgent:
+    def __init__(self):
+        self._std_streams = StdStreams(self._on_std_input)
+        self._editor_protocol = EditorProtocolHandler(self._std_streams)
+        self._main_executor = ThreadPoolExecutor(max_workers=1)
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.stop()
+
+    def start(self):
+        self._std_streams.start()
+        logging.info("Tandem Agent has started")
+
+    def stop(self):
+        self._std_streams.stop()
+        self._main_executor.shutdown()
+        logging.info("Tandem Agent has shut down")
+
+    def _on_std_input(self, data):
+        # Called by _std_streams after receiving a new message from the plugin
+        self._main_executor.submit(self._editor_protocol.handle_message, data)

agent/tandem/io/std_streams.py

@@ -0,0 +1,31 @@
+import sys
+import logging
+from threading import Thread
+
+
+class StdStreams:
+    def __init__(self, handler_function):
+        self._handler_function = handler_function
+        self._reader = self._get_read_thread()
+
+    def start(self):
+        self._reader.start()
+
+    def stop(self):
+        self._reader.join()
+        sys.stdout.close()
+
+    def write(self, data):
+        sys.stdout.write(data)
+        sys.stdout.write("\n")
+        sys.stdout.flush()
+
+    def _get_read_thread(self):
+        def stdin_read():
+            try:
+                for line in sys.stdin:
+                    self._handler_function(line)
+            except:
+                logging.exception("Exception when reading from stdin:")
+                raise
+        return Thread(target=stdin_read)

agent/tandem/protocol/editor/handler.py

@@ -0,0 +1,19 @@
+import logging
+import tandem.protocol.editor.messages as m
+
+
+class EditorProtocolHandler:
+    def __init__(self, std_streams):
+        self._std_streams = std_streams
+
+    def handle_message(self, data):
+        try:
+            message = m.deserialize(data)
+            response = m.serialize(message)
+            self._std_streams.write(response)
+        except m.EditorProtocolMarshalError:
+            pass
+        except:
+            logging.exception(
+                "Exception when handling editor protocol message:")
+            raise

agent/tandem/protocol/editor/messages.py

@@ -0,0 +1,77 @@
+import json
+import enum
+
+
+class EditorProtocolMarshalError:
+    pass
+
+
+class EditorProtocolMessageType(enum.Enum):
+    UserChangedEditorText = "user-changed-editor-text"
+    ApplyText = "apply-text"
+
+
+class UserChangedEditorText:
+    """
+    Sent by the editor plugin to the agent to
+    notify it that the user changed the text buffer.
+    """
+    def __init__(self, contents):
+        self.type = EditorProtocolMessageType.UserChangedEditorText
+        self.contents = contents
+
+    def to_payload(self):
+        return {
+            "contents": self.contents,
+        }
+
+    @staticmethod
+    def from_payload(payload):
+        return UserChangedEditorText(payload["contents"])
+
+
+class ApplyText:
+    """
+    Sent by the agent to the editor plugin to
+    notify it that someone else edited the text buffer.
+    """
+    def __init__(self, contents):
+        self.type = EditorProtocolMessageType.ApplyText
+        self.contents = contents
+
+    def to_payload(self):
+        return {
+            "contents": self.contents,
+        }
+
+    @staticmethod
+    def from_payload(payload):
+        return ApplyText(payload["contents"])
+
+
+def serialize(message):
+    as_dict = {
+        "type": message.type.value,
+        "payload": message.to_payload(),
+        "version": 1,
+    }
+    return json.dumps(as_dict)
+
+
+def deserialize(data):
+    try:
+        as_dict = json.loads(data)
+        type = as_dict["type"]
+        payload = as_dict["payload"]
+
+        if type == EditorProtocolMessageType.UserChangedEditorText.value:
+            return UserChangedEditorText.from_payload(payload)
+
+        elif type == EditorProtocolMessageType.ApplyTextBuffer.value:
+            return ApplyTextBuffer.from_payload(payload)
+
+        else:
+            raise EditorProtocolMarshalError
+
+    except JSONDecodeError:
+        raise EditorProtocolMarshalError

agent/test_client.py

@@ -0,0 +1,50 @@
+import sys
+from subprocess import Popen, PIPE
+import tandem.protocol.editor.messages as m
+
+
+def start_agent():
+    return Popen(
+        ["python3", "main.py"],
+        stdin=PIPE,
+        stdout=PIPE,
+        encoding="utf-8",
+    )
+
+
+def send_user_changed(agent_stdin, text):
+    message = m.UserChangedEditorText(text)
+    agent_stdin.write(m.serialize(message))
+    agent_stdin.write("\n")
+    agent_stdin.flush()
+
+
+def print_raw_message(agent_stdout):
+    resp = agent_stdout.readline()
+    print("Received: " + resp)
+
+
+def main():
+    # Spawn the agent process
+    agent = start_agent()
+
+    # Send the agent a dummy message
+    send_user_changed(agent.stdin, "Hello world!")
+
+    # The agent currently just echos messages
+    # so just print the response
+    print_raw_message(agent.stdout)
+
+    # Repeat
+    send_user_changed(agent.stdin, "Hello world again!")
+    print_raw_message(agent.stdout)
+
+    # Stop the agent and wait for it to
+    # shutdown gracefully
+    agent.stdin.close()
+    agent.terminate()
+    agent.wait()
+
+
+if __name__ == "__main__":
+    main()

plugins/vim/tandem.py

@@ -0,0 +1,56 @@
+import os
+import sys
+
+from subprocess import Popen, PIPE
+
+# For now, add the tandem agent path to the system path so that we can use the
+# existing messages protocol implementation
+tandem_agent_path = os.path.abspath('../../agent')
+if tandem_agent_path not in sys.path:
+    sys.path.insert(0, tandem_agent_path)
+
+import tandem.protocol.editor.messages as m
+
+
+def start_agent():
+    return Popen(
+        ["python3", "../../agent/main.py"],
+        stdin=PIPE,
+        stdout=PIPE,
+    )
+
+
+def send_user_changed(agent_stdin, text):
+    message = m.UserChangedEditorText(text)
+    agent_stdin.write(m.serialize(message))
+    agent_stdin.write("\n")
+    agent_stdin.flush()
+
+
+def print_raw_message(agent_stdout):
+    resp = agent_stdout.readline()
+    print "Received: " + resp
+
+
+def main():
+    # Spawn the agent process
+    agent = start_agent()
+
+    # Send the agent a dummy message
+    send_user_changed(agent.stdin, "Hello world!")
+
+    # The agent currently just echos messages so just print the response
+    print_raw_message(agent.stdout)
+
+    # Repeat
+    send_user_changed(agent.stdin, "Hello world again!")
+    print_raw_message(agent.stdout)
+
+    # Stop the agent and wait for it to shutdown gracefully
+    agent.stdin.close()
+    agent.terminate()
+    agent.wait()
+
+
+if __name__ == "__main__":
+    main()

plugins/vim/tandem.vim

@@ -0,0 +1,14 @@
+if !has('python')
+  " :echom is persistent messaging. See
+  " http://learnvimscriptthehardway.stevelosh.com/chapters/01.html
+  :echom "ERROR: Please use a version of Vim with Python support"
+  finish
+endif
+
+if !executable('python3')
+  :echom "ERROR: Global python3 install required."
+  finish
+endif
+
+" TODO: Use file path relative to .vim file
+pyfile tandem.py