add data

Makefile

@@ -1,26 +1,25 @@
-all: clean build deploy
+all: clean build push
 build:
 	cd encode && docker build --network=host -t gnes/demo:poem-encode . && cd - && \
 	cd preprocess && docker build --network=host -t gnes/demo:poem-preprocess . && cd - && \
 	cd vector-index && docker build --network=host -t gnes/demo:poem-vector-index . && cd - && \
-	cd fulltext-index && docker build --network=host -t gnes/demo:poem-fulltext-index . && cd -
 	cd client && docker build --network=host -t gnes/demo:poem-client . && cd -
 push:
 	docker push gnes/demo:poem-encode && \
 	docker push gnes/demo:poem-preprocess && \
 	docker push gnes/demo:poem-vector-index && \
-	docker push gnes/demo:poem-fulltext-index && \
 	docker push gnes/demo:poem-client
 pull:
 	docker pull gnes/demo:poem-encode && \
 	docker pull gnes/demo:poem-preprocess && \
 	docker pull gnes/demo:poem-vector-index && \
-	docker pull gnes/demo:poem-fulltext-index && \
 	docker pull gnes/demo:poem-client
-clean: ; rm -rf .cache && mkdir -p .cache && docker stack rm my-gnes
-deploy: ; mkdir -p .cache && docker stack deploy --compose-file demo-poem-index.yml my-gnes
-index: ; unset https_proxy && unset http_proxy && docker run --rm --network host -v ${PWD}/data:/data/ gnes/demo:poem-client --mode index --batch_size 4 --txt_file /data/kaggle_poem_dataset.csv
-query:
-	docker stack rm my-gnes && \
-	docker stack deploy --compose-file demo-poem-query.yml my-gnes && \
-	unset https_proxy && unset http_proxy && docker run --rm --network host -v ${PWD}/data:/data/ gnes/demo:poem-client --mode query --txt_file /data/kaggle_poem_dataset.csv
+clean_data: ; rm -rf .cache && mkdir -p .cache
+clean_stack: ; docker stack rm my-gnes
+clean: clean_data clean_stack
+deploy_index: ; mkdir -p .cache && docker stack deploy --compose-file demo-poem-index.yml my-gnes
+deploy_query: ; docker stack rm my-gnes && docker stack deploy --compose-file demo-poem-query.yml my-gnes
+client_index: ; unset https_proxy && unset http_proxy && docker run --rm --network host -v ${PWD}/data:/data/ gnes/demo:poem-client --mode index --batch_size 4 --txt_file /data/kaggle_poem_dataset.csv
+client_query: ; unset https_proxy && unset http_proxy && docker run --rm --network host -v ${PWD}/data:/data/ gnes/demo:poem-client --mode query --txt_file /data/kaggle_poem_dataset.csv
+index: clean deploy_index
+query: clean_stack deploy_query

README.md

@@ -1,13 +1,70 @@
-# demo-poems-ir
-Demo built with GNES framework
+# Semantic Poem Search Demo using GNES
 
+This demo shows you how to build a semantic poem search using GNES. 
 
+## Prerequisite
+
+You need to have [Docker](https://docs.docker.com/install/) installed on your machine.
+
+As a start, run the following command to initialize the Docker Swarm environment.
+
+```bash
 docker swarm init
+``` 
+
+## 1. Download required GNES images 
+
+```bash
+make pull
+```
+
+### (Optional) Build images from local
+
+```bash
+make build
+```
+
+## 2. Index all poems
 
+```bash
+make index
+# wait for a minute until all services started and grpc is ready
+make client_index
+```
 
-Error
+## 3. Query and show top-10 poems
 
+```bash
+make query
+# wait for a minute until all services started and grpc is ready
+make client_query
+```
+
+
+## Other commands
+
+| Command | Description |
+|---|---|
+| `make clean` | Clean index files in `.cache` and remove all docker stacks |
+| `make clean_stack` | Remove all docker stacks |
+| `make clean_data` | Remove all index files in `.cache` |
+| `make deploy_query` | Deploy the stack described in `demo-poem-query.yml` for query |
+| `make deploy_index` | Deploy the stack described in `demo-poem-index.yml` for index |
+
+
+## Troubleshooting
+
+If you encounter the following errors when doing `make`, simply wait for couple of seconds. Docker is sometimes slow on recycling network device.
+
+```bash
 rm -rf .cache && mkdir .cache && docker stack rm my-gnes
 Removing network my-gnes_default
 Failed to remove network b0ei205abak98pn84yj9f6u70: Error response from daemon: network b0ei205abak98pn84yj9f6u70 not foundFailed to remove some resources from stack: my-gnes
-make: *** [clean] Error 1
+make: *** [clean] Error 1
+```
+
+```bash
+Creating service my-gnes_Router40
+failed to create service my-gnes_Router40: Error response from daemon: network my-gnes_default not found
+make: *** [deploy] Error 1
+```

client/app.py

@@ -10,7 +10,7 @@
 class MyClient(CLIClient):
     def read_all(self):
         return [json.dumps(rr).encode() for rr in csv.DictReader(self.args.txt_file, delimiter=',', quotechar='"')][
-               :100]
+               :10]
 
     def index(self, all_bytes: List[bytes], stub):
         with ProgressBar(all_bytes, self.args.batch_size, task_name='index') as p_bar:

demo-poem-index.yml

@@ -21,11 +21,12 @@ services:
     volumes:
       - ./.cache:/workspace
   Indexer21:
-    image: gnes/demo:poem-fulltext-index
-    command: --port_in 56264 --socket_in SUB_CONNECT --port_out 57909 --socket_out
-      PUSH_CONNECT --host_out Router40 --host_in Preprocessor10
+    image: gnes/gnes:latest-alpine
+    command: index --port_in 56264 --socket_in SUB_CONNECT --port_out 57909 --socket_out
+      PUSH_CONNECT --host_out Router40 --host_in Preprocessor10 --yaml_path /yaml/index.yml
     volumes:
       - ./.cache:/workspace
+      - ./fulltext-index:/yaml
   Router40:
     image: gnes/gnes:latest-alpine
     command: route --num_part 2 --port_in 57909 --socket_in PULL_BIND --port_out 57908 --socket_out

demo-poem-query.yml

@@ -17,12 +17,19 @@ services:
   Indexer30:
     image: gnes/demo:poem-vector-index
     command: --port_in 64886 --socket_in PULL_BIND --port_out 57909 --socket_out
-      PUSH_BIND
+      PUSH_CONNECT --host_out Router40
     volumes:
       - ./.cache:/workspace
+  Router40:
+    image: gnes/gnes:latest-alpine
+    command: route --socket_in PULL_BIND --socket_out PUSH_BIND --port_in 57909 --port_out 57910 --yaml_path ChunkToDocRouter
   Indexer21:
-    image: gnes/demo:poem-fulltext-index
-    command: --port_in 57909 --socket_in PULL_CONNECT --port_out 57908 --socket_out
-      PUSH_CONNECT --host_out Frontend00 --host_in Indexer30
+    image: gnes/gnes:latest-alpine
+    command: index --port_in 57910 --socket_in PULL_CONNECT --port_out 57911 --socket_out
+      PUSH_CONNECT --host_out Router50 --host_in Router40 --yaml_path /yaml/index.yml
     volumes:
       - ./.cache:/workspace
+      - ./fulltext-index:/yaml
+  Router50:
+    image: gnes/gnes:latest-alpine
+    command: route --socket_in PULL_BIND --socket_out PUSH_CONNECT --port_in 57911 --port_out 57908 --yaml_path DocFillRouter

fulltext-index/index.yml

@@ -1,4 +1,4 @@
-!SimpleDictIndexer
+!DictIndexer
 gnes_config:
   name: my_fulltext_indexer  # a customized name
   work_dir: /workspace