Webpack, website that queries backend

.dockerignore

@@ -0,0 +1,4 @@
+.git
+*Dockerfile*
+*docker-compose*
+node_modules

.gitignore

@@ -6,3 +6,6 @@ __pycache__/
 build/
 libraries/
 .swp
+node_modules/
+package-lock.json
+deploy/secret_env_vars.sh

cpu_deploy_environment.yml

@@ -7,7 +7,6 @@ channels:
 dependencies:
   - beautifulsoup4>=4.8.2
   - google-auth>=1.14.1
-  - grpcio>=1.23.0
   - grpcio-tools>=1.16.1
   - numpy>=1.18.1
   - cpuonly>=1.0.0
@@ -23,3 +22,4 @@ dependencies:
   - pip:
     - transformers==2.8.0
     - pyhyphen>=3.0.1
+    - grpcio>=1.28.1

deploy/WebsiteDockerfile

@@ -28,9 +28,10 @@ COPY ./deploy/website/supervisord.conf /etc/supervisor/supervisord.conf
 COPY ./website website
 COPY ./title_maker_pro title_maker_pro
 COPY ./word_service word_service
+COPY ./deploy/secret_env_vars.sh /app/secret_env_vars.sh
 
 # Expose ports.
 EXPOSE 8080
 
-ENTRYPOINT ["supervisord"]
+ENTRYPOINT ["/bin/bash", "-c", "source secret_env_vars.sh && supervisord"]
 # ENTRYPOINT ["/bin/bash"]

deploy/website/supervisord.conf

@@ -20,7 +20,12 @@ numprocs = 2
 numprocs_start = 1
 process_name = app_%(process_num)s
 environment=PYTHONPATH=.
-command=python3.7 -u website/main.py --path=/tmp/app_%(process_num)s.sock
+command=python3.7 
+    -u website/main.py 
+    --path=/tmp/app_%(process_num)s.sock
+    --word-service-address %(ENV_WORD_SERVICE_ADDRESS)s
+    --word-service-port %(ENV_WORD_SERVICE_PORT)s
+    --word-index-path website/data/words.json
 user=root
 autostart=true
 autorestart=true

deploy/word_service/deploy_service_definition.sh

@@ -1,2 +1,3 @@
 #!/bin/sh
-gcloud endpoints services deploy ../word_service_proto/api_descriptor.pb api_config.yaml
+gcloud container clusters get-credentials word-service --zone us-central1-c
+gcloud endpoints services deploy ../../word_service/word_service_proto/api_descriptor.pb api_config.yaml

deploy/word_service/deploy_service_server.sh

@@ -1 +1,2 @@
+gcloud container clusters get-credentials word-service --zone us-central1-c
 kubectl apply -f grpc-wordservice.yaml

scripts/gen_stubs.sh

@@ -2,8 +2,9 @@ python -m grpc_tools.protoc \
     --include_imports \
     --include_source_info \
     --proto_path=word_service/word_service_proto \
+    --python_grpc_out=word_service/word_service_proto \
     --python_out=word_service/word_service_proto \
     --grpc_python_out=word_service/word_service_proto \
     --descriptor_set_out=word_service/word_service_proto/api_descriptor.pb \
     wordservice.proto
-sed -i -r 's/import (.+_pb2.*)/from . import \1/g' word_service/word_service_proto/*_pb2*.py
+sed -i -r 's/import (.+_pb2.*)/from . import \1/g' word_service/word_service_proto/*_pb2.py word_service/word_service_proto/*_grpc.py

scripts/website_dev.sh

@@ -0,0 +1,2 @@
+export PYTHONPATH=.: 
+adev runserver website --root website --port 8001

title_maker_pro/word_generator.py

@@ -56,7 +56,6 @@ def generate_word(self, user_filter=None):
             generation_args=dict(
                 top_k=300, num_return_sequences=10, max_length=self.approx_max_length, do_sample=True,
             ),
-            num_expansion_candidates=20,
             example_match_pos_pipeline=self.stanza_pos_pipeline,
             user_filter=user_filter,
             dedupe_titles=True,

website/main.py

@@ -1,35 +1,74 @@
+import json
 import jinja2
 import aiohttp_jinja2
 from aiohttp import web
 import words
-import sys
 import argparse
 from urllib.parse import quote_plus
 from word_service.word_service_proto import wordservice_pb2
-from word_service.word_service_proto import wordservice_pb2_grpc
+from word_service.word_service_proto import wordservice_grpc
+from grpclib.client import Channel
 
-routes = web.RouteTableDef()
-word_index = words.WordIndex.load("./website/data/words.json")
 
 
-@routes.get("/")
-@aiohttp_jinja2.template("index.jinja2")
-async def index(request):
-    w = word_index.random()
-    return {"word": w}
+def _word_to_dict(w: wordservice_pb2.WordDefinition):
+    if not w.word:
+        return None
 
+    return {
+        "word": w.word,
+        "definition": w.definition,
+        "examples": list(w.examples),
+        "pos": w.pos
+    }
 
-@routes.get("/favicon.ico")
-async def favicon(request):
-    return web.FileResponse("./static/favicon.ico")
 
+def _dev_handlers():
+    return Handlers(
+        word_service_address="localhost",
+        word_service_port=8000,
+        word_index=words.WordIndex.load("./website/data/words.json"),
+    )
+
+
+class Handlers:
+    def __init__(self, word_service_address, word_service_port, word_index):
+        channel = Channel(word_service_address, word_service_port)
+        self.word_service = wordservice_grpc.WordServiceStub(channel)
+        self.word_index = word_index
+
+    @aiohttp_jinja2.template("index.jinja2")
+    async def index(self, request):
+        w = self.word_index.random()
+        return {"word": w}
+
+    async def define_word(self, request):
+        word = request.query["word"]
+        response = await self.word_service.DefineWord(wordservice_pb2.DefineWordRequest(
+            word=word
+        ))
+        return web.Response(
+            text=json.dumps({
+                "word": _word_to_dict(response.word),
+            }),
+            content_type="application/json",
+        )
 
-def app(argv=()):
+    async def favicon(self, request):
+        return web.FileResponse("./website/static/favicon.ico")
+
+
+def app(handlers=None):
+    handlers = handlers or _dev_handlers()
     app = web.Application()
-    app.add_routes(routes)
-    app.add_routes([web.static("/static", "./website/static")])
+    app.add_routes([
+        web.get('/', handlers.index),
+        web.get('/define_word', handlers.define_word),
+        web.get('/favicon.ico', handlers.favicon),
+        web.static("/static", "./website/static"),
+    ])
     aiohttp_jinja2.setup(
-        app, 
+        app,
         loader=jinja2.FileSystemLoader("./website/templates"),
         filters={
             'quote_plus': quote_plus
@@ -41,5 +80,16 @@ def app(argv=()):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--path", type=str, help="Unix socket path")
+    parser.add_argument("--word-service-address", type=str, help="Word service address", required=True)
+    parser.add_argument("--word-service-port", type=int, help="Word service port", required=True)
+    parser.add_argument("--word-index-path", type=str, help="Path to word index", required=True)
     args = parser.parse_args()
-    web.run_app(app(sys.argv), path=args.path)
+
+    wi = words.WordIndex.load(args.word_index_path)
+    handlers = Handlers(
+        args.word_service_address,
+        args.word_service_port,
+        word_index=wi,
+    )
+
+    web.run_app(app(handlers), path=args.path)

website/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "website",
+  "version": "1.0.0",
+  "description": "",
+  "private": true,
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build": "webpack",
+    "watch": "webpack -d --watch"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@babel/core": "^7.9.6",
+    "@babel/preset-env": "^7.9.6",
+    "babel-loader": "^8.1.0",
+    "mini-css-extract-plugin": "^0.9.0",
+    "webpack": "^4.43.0",
+    "webpack-cli": "^3.3.11"
+  },
+  "dependencies": {
+    "css-loader": "^3.5.3",
+    "lodash": "^4.17.15",
+    "optimize-css-assets-webpack-plugin": "^5.0.3",
+    "whatwg-fetch": "^3.0.0"
+  }
+}

website/static/index.css → website/static_src/index.css

@@ -66,6 +66,33 @@ a.darken:hover img {
   color: white;
 }
 
+.write-your-own {
+  margin-top: 20vh;
+  margin-bottom: 50px;
+  margin-left: auto;
+  margin-right: auto;
+  height: 50vh;
+  color: white;
+}
+
+.hint-text {
+    color: rgba(255, 255, 255, 0.5);
+}
+
+.word-loading {
+    color: rgba(255, 255, 255, 0.5);
+}
+
+.word-entry {
+    border: 0px;
+    color: #51b8b8;
+    background-color: rgba(255, 255, 255, 0.0);
+    resize: none;
+    width: 100%;
+    font-size: 2em;
+    outline: none;
+}
+
 .definition-zone ol {
     padding-inline-start: 20px;
 }
@@ -131,13 +158,21 @@ a.darken:hover img {
     .definition-zone {
         margin-top: 10vh;
     }
+
+    .write-your-own {
+        margin-top: 10vh;
+    }
 }
 
 @media screen and (max-height: 550px) {
     .definition-zone {
         height: auto;
     }
 
+    .write-your-own {
+        height: auto;
+    }
+
     .attribution {
         width: 100%;
         position: static;

website/static_src/index.js

@@ -0,0 +1,95 @@
+import 'whatwg-fetch'
+
+function syncToWord(word) {
+  let posEl = document.getElementById("definition-pos");
+  let wordEl = document.getElementById("definition-word");
+  let syllablesEl = document.getElementById("definition-syllables");
+  let definitionEl = document.getElementById("definition-definition");
+  let exampleEl = document.getElementById("definition-example");
+
+  posEl.innerHTML = word.pos;
+  if (!posEl.innerHTML.endsWith("]")) {
+    posEl.innerHTML += ".";
+  }
+
+  wordEl.innerHTML = word.word;
+  definitionEl.innerHTML = word.definition;
+  if (word.examples && word.examples.length > 0) {
+    exampleEl.innerHTML = word.examples[0];
+  } else {
+    exampleEl.innerHTML = "";
+  }
+
+  if (word.syllables && word.syllables.length > 1) {
+    syllablesEl.innerHTML = word.syllables.join("<span class='syllable-separator'>&middot;</span>");
+  } else {
+    syllablesEl.innerHTML = "";
+  }
+}
+
+window.onload = () => {
+  let definitionEl = document.getElementById("definition-zone");
+  let writeYourOwnEl = document.getElementById("write-your-own");
+  let wordEntry = document.getElementById("word-entry");
+  let writeButton = document.getElementById("write-button");
+  let hintText = document.getElementById("hint-text");
+  let loadingText = document.getElementById("word-loading");
+  let wordEntryForm = document.getElementById("word-entry-form");
+  let cancelButton = document.getElementById("word-entry-cancel")
+  let hintTextValue = document.getElementById("hint-text-value");
+  let defaultHintText = hintTextValue.innerHTML;
+  var errorText = "something went wrong, try again?"
+
+  document.addEventListener('click', event => {
+    if (event.target == writeButton) {
+      event.preventDefault();
+
+      errorText = "something went wrong, try again?"
+      definitionEl.style.display = "none";
+      writeYourOwnEl.style.display = "";
+      hintTextValue.innerHTML = defaultHintText;
+      wordEntry.value = "";
+      wordEntry.disabled = false;
+      wordEntry.focus();
+    } else if (event.target == cancelButton) {
+      event.preventDefault();
+
+      definitionEl.style.display = "";
+      writeYourOwnEl.style.display = "none";
+      wordEntry.blur();
+    }
+  }, false);
+
+
+  wordEntryForm.addEventListener("submit", e => {
+    e.preventDefault();
+    wordEntry.disabled = true;
+    let word = wordEntry.value;
+    hintText.style.display = "none";
+    hintTextValue.innerHTML = defaultHintText;
+    loadingText.style.display = "";
+    document.activeElement.blur()
+    window.fetch(`/define_word?word=${encodeURIComponent(word)}`)
+      .then(res => res.json())
+      .then(json => {
+        console.log(json);
+        if (!json.word) {
+          errorText = "we couldn't define that word"
+          throw new Error("couldn't define word");
+        }
+
+        syncToWord(json.word);
+        definitionEl.style.display = "";
+        writeYourOwnEl.style.display = "none";
+        hintText.style.display = "";
+        loadingText.style.display = "none";
+      })
+      .catch((error) => {
+        hintTextValue.innerHTML = errorText;
+        hintText.style.display = "";
+        loadingText.style.display = "none";
+        wordEntry.disabled = false;
+        wordEntry.focus();
+      });
+  }, false);
+};