Port UI to React (#159)

Dockerfile

@@ -23,7 +23,21 @@ COPY package-lock.json /maniwani
 COPY Gulpfile.js /maniwani
 COPY scss /maniwani/scss
 RUN npm install && npm run gulp && rm -rf node_modules
+# build react render sidecar
+WORKDIR /maniwani-frontend
+COPY frontend/package.json /maniwani-frontend
+COPY frontend/package-lock.json /maniwani-frontend
+RUN npm install
+COPY frontend/src /maniwani-frontend/src
+COPY frontend/Gulpfile.js /maniwani-frontend
+RUN npm run gulp
+RUN cp -r build/* /maniwani-frontend/
+COPY frontend/devmode-entrypoint.sh /maniwani-frontend
+# TODO: how do we do this when running/deploying without docker?
+RUN mkdir -p /maniwani/static/js
+RUN cp /maniwani-frontend/build/client-bundle/*.js /maniwani/static/js/
 # copy source files over
+WORKDIR /maniwani
 COPY migrations /maniwani/migrations
 COPY *.py /maniwani/
 COPY blueprints /maniwani/blueprints
@@ -44,6 +58,7 @@ WORKDIR /maniwani
 # clean up dev image bootstrapping
 RUN rm ./deploy-configs/test.db
 RUN rm -r uploads
+RUN apt-get -y autoremove npm nodejs
 ENV MANIWANI_CFG=./deploy-configs/maniwani.cfg
 # chown and switch users for security purposes
 RUN adduser --disabled-login maniwani

README.md

@@ -77,8 +77,7 @@ requires `docker-compose`. In this directory, type:
 
 The last command will only need to be run once per clean installation of the production
 environment. If you ever want to remove all database and storage data, remove the
-`compose-data` and `compose-captchouli` directories, though you'll likely need root
-permissions to do so since some subdirectories are created by other users. At this point,
+`compose-minio`, `compose-postgres`, and `compose-captchouli` volumes. At this point,
 you can use the normal `docker-compose start` and `docker-compose stop` to start and stop the production
 environment, navigating to http://127.0.0.1:5000 as per usual to view Maniwani. If you
 want additional info on deploying Maniwani in production, see `doc/deploying.md` for more.

app.py

@@ -17,7 +17,7 @@
 from resources import (
     BoardCatalogResource, BoardListResource, FirehoseResource,
     NewPostResource, NewThreadResource, PostRemovalResource,
-    ThreadPostsResource,
+    ThreadPostsResource, SinglePostResource
 )
 from shared import app, rest_api
 
@@ -37,6 +37,7 @@
     rest_api.add_resource(NewThreadResource, "/api/v1/thread/new")
     rest_api.add_resource(PostRemovalResource, "/api/v1/thread/post/<int:post_id>")
     rest_api.add_resource(NewPostResource, "/api/v1/thread/<int:thread_id>/new")
+    rest_api.add_resource(SinglePostResource, "/api/v1/post/<int:post_id>")
     rest_api.add_resource(FirehoseResource, "/api/v1/firehose")
     app.register_blueprint(live_blueprint, url_prefix="/api/v1")
 

blueprints/boards.py

@@ -6,6 +6,9 @@
 from werkzeug.http import parse_etags
 
 import cache
+import captchouli
+import renderer
+from model.Media import storage
 from model.Board import Board
 from model.BoardList import BoardList
 from model.BoardListCatalog import BoardCatalog
@@ -36,7 +39,12 @@ def get_tags(threads):
 
 @boards_blueprint.route("/")
 def list():
-    return render_template("board-index.html", boards=BoardList().get())
+    boards = BoardList().get()
+    for board in boards:
+        board["catalog_url"] = url_for("boards.catalog", board_id=board["id"])
+        if board["media"]:
+            board["thumb_url"] = storage.get_thumb_url(board["media"])
+    return renderer.render_board_index(boards)
 
 
 @boards_blueprint.route("/<int:board_id>")
@@ -63,7 +71,15 @@ def catalog(board_id):
     board_name = board.name
     render_for_catalog(threads)
     tag_styles = get_tags(threads)
-    template = render_template("catalog.html", threads=threads, board=board, board_name=board_name, tag_styles=tag_styles)
+    catalog_data = {}
+    catalog_data["tag_styles"] = tag_styles
+    for thread in threads:
+        del thread["last_updated"]
+    catalog_data["threads"] = threads
+    extra_data = {}
+    if app.config.get("CAPTCHA_METHOD") == "CAPTCHOULI":
+        extra_data = renderer.captchouli_to_json(captchouli.request_captcha())
+    template = renderer.render_catalog(catalog_data, board_name, board_id, extra_data)
     uncached_response = make_response(template)
     uncached_response.set_etag(etag_value, weak=True)
     uncached_response.headers["Cache-Control"] = "public,must-revalidate"

blueprints/main.py

@@ -4,6 +4,7 @@
 from markdown import markdown
 from werkzeug.http import parse_etags
 
+import renderer
 import cache
 from blueprints.boards import get_tags
 from model.Firehose import Firehose
@@ -34,8 +35,15 @@ def index():
         return cached_response
     greeting = open("deploy-configs/index-greeting.html").read()
     threads = Firehose().get_impl()
+    # fix to avoid needing to serialize the datetime
+    for thread in threads:
+        del thread["last_updated"]
     tag_styles = get_tags(threads)
-    template = render_template("index.html", greeting=greeting, threads=threads, tag_styles=tag_styles)
+    firehose_data = {
+        "threads": threads,
+        "tag_styles": tag_styles}
+    template = renderer.render_firehose(firehose_data, greeting)
+    #template = render_template("index.html", greeting=greeting, threads=threads, tag_styles=tag_styles)
     uncached_response = make_response(template)
     uncached_response.set_etag(etag_value, weak=True)
     uncached_response.headers["Cache-Control"] = "public,must-revalidate"

blueprints/threads.py

@@ -6,7 +6,7 @@
 import cache
 import captchouli
 import cooldown
-from model.Media import upload_size
+from model.Media import upload_size, storage
 from model.Board import Board
 from model.NewPost import NewPost
 from model.NewThread import NewThread
@@ -19,6 +19,7 @@
 from model.Thread import Thread
 from model.ThreadPosts import ThreadPosts
 from post import InvalidMimeError, CaptchaError
+import renderer
 from shared import db, app
 from thread import invalidate_board_cache
 
@@ -36,8 +37,10 @@ def _get_captchouli():
 
 @threads_blueprint.route("/new/<int:board_id>")
 def new(board_id):
-    board = db.session.query(Board).get(board_id)
-    return render_template("new-thread.html", board=board)
+    extra_data = {}
+    if app.config.get("CAPTCHA_METHOD") == "CAPTCHOULI":
+        extra_data = renderer.captchouli_to_json(captchouli.request_captcha())
+    return renderer.render_new_thread_form(board_id, extra_data)
 
 
 @threads_blueprint.route("/new", methods=["POST"])
@@ -95,8 +98,17 @@ def view(thread_id):
     num_posters = db.session.query(Poster).filter(Poster.thread == thread_id).count()
     num_media = thread.num_media()
     reply_urls = _get_reply_urls(posts)
-    template = render_template("thread.html", thread_id=thread_id, board=board, posts=posts, num_views=thread.views,
-                               num_media=num_media, num_posters=num_posters, reply_urls=reply_urls)
+    thread_data = {}
+    for post in posts:
+        post["datetime"] = post["datetime"].strftime("%a, %d %b %Y %H:%M:%S UTC")
+        if post["media"]:
+            post["media_url"] = storage.get_media_url(post["media"], post["media_ext"])
+            post["thumb_url"] = storage.get_thumb_url(post["media"])
+    thread_data["posts"] = posts
+    extra_data = {}
+    if app.config.get("CAPTCHA_METHOD") == "CAPTCHOULI":
+        extra_data = renderer.captchouli_to_json(captchouli.request_captcha())
+    template = renderer.render_thread(thread_data, thread_id, extra_data)
     uncached_response = make_response(template)
     uncached_response.set_etag(etag_value, weak=True)
     uncached_response.headers["Cache-Control"] = "public,must-revalidate"
@@ -146,7 +158,10 @@ def move_submit(thread_id):
 
 @threads_blueprint.route("/<int:thread_id>/new")
 def new_post(thread_id):
-    return render_template("new-post.html", thread_id=thread_id)
+    extra_data = {}
+    if app.config.get("CAPTCHA_METHOD") == "CAPTCHOULI":
+        extra_data = renderer.captchouli_to_json(captchouli.request_captcha())
+    return renderer.render_new_post_form(thread_id, extra_data)
 
 
 @threads_blueprint.route("/<int:thread_id>/new", methods=["POST"])
@@ -191,9 +206,15 @@ def render_post(post_id):
 @threads_blueprint.route("/<int:thread_id>/gallery")
 def view_gallery(thread_id):
     posts = ThreadPosts().retrieve(thread_id)
+    for post in posts:
+        # TODO: either streamline what gets sent to the frontend
+        # or automatically serialize datetimes so the below isn't necessary
+        del post["datetime"]
+        post["thumb_url"] = storage.get_thumb_url(post["media"])
+        post["media_url"] = storage.get_media_url(post["media"], post["media_ext"])
     thread = db.session.query(Thread).filter(Thread.id == thread_id).one()
     board = db.session.query(Board).get(thread.board)
-    return render_template("gallery.html", thread_id=thread_id, board=board, posts=posts)
+    return renderer.render_thread_gallery(board, thread_id, posts)
 
 
 def _get_reply_urls(posts):

build-helpers/docker-entrypoint.sh

@@ -4,6 +4,8 @@
 if [ $1 = "devmode" ]; then
 	# start up internal pubsub server
 	python3 storestub.py &
+	# start up react sidecar
+	/maniwani-frontend/devmode-entrypoint.sh &
 	uwsgi --ini ./deploy-configs/uwsgi-devmode.ini
 # attempting to bootstrap?
 elif [ $1 = "bootstrap" ]; then

deploy-configs/devmode.cfg

@@ -1,6 +1,7 @@
 # Configuration file for devmode
 INSTANCE_NAME = "Maniwani"
 SQLALCHEMY_DATABASE_URI = "sqlite:///./deploy-configs/test.db"
+RENDERER_HOST = "http://127.0.0.1:3000"
 STORAGE_PROVIDER = "FOLDER"
 STORE_PROVIDER = "INTERNAL"
 SERVE_STATIC = True

deploy-configs/maniwani.cfg

@@ -2,6 +2,7 @@
 # Make sure to change these settings before triggering a build intended to be public-facing!
 INSTANCE_NAME = "Maniwani"
 SQLALCHEMY_DATABASE_URI = "postgresql://dev:dev@postgres/dev"
+RENDERER_HOST = "http://maniwani-frontend:3000"
 STORAGE_PROVIDER = "S3"
 S3_ENDPOINT = "http://minio:9000"
 S3_ACCESS_KEY = "minio"

doc/deploying.md

@@ -93,6 +93,9 @@ but a brief description of each follows:
 * `SERVE_REST` - if `True`, REST endpoints will be served by Maniwani. Note that ideally, the REST API for
   your Maniwani installation is protected with HTTPS, since Maniwani uses HTTP Basic for authentication and
   authorization.
+* `RENDERER_HOST` - this is a required string indicating the URL of the Maniwani frontend. For a production
+  deployment, this should be simply be the URL of the Express/React frontend container. Keep in mind that
+  the frontend container serves traffic on port 3000 by default.
 * `DEFAULT_THEME` - this is an optional string representing the name of the theme Maniwani should use if
   the user has not otherwise selected a theme to use. If not present, the stock theme will be used.
 * `THEME_LIST` - this is an optional list of strings representing the list of installed themes available.
@@ -276,7 +279,7 @@ Running Maniwani
 
 Running Maniwani is highly dependent on how you've organized things in your particular production
 environment - whether you're using `docker-compose` or some other tool, etc. - but in essence comes
-down to more or less running `docker run dangerontheranger/maniwani` while ensuring the other
+down to more or less running both the backedn and frontend containers while ensuring the other
 pieces of your stack are also executing. If you went with the directory mount option for your custom
 configuration files, don't forget to mount that with `--mount` or `-v` before running the container.
 

docker-compose.yml

@@ -2,14 +2,17 @@ version: '3'
 services:
     maniwani:
         build: .
+    maniwani-frontend:
+        build: ./frontend
+        hostname: maniwani-frontend
     captchouli:
         build: ./build-helpers/captchouli
         hostname: captchouli
         restart: always
         environment:
             - CAPTCHOULI_FLAGS=-e -t izayoi_sakuya,yakumo_yukari,hong_meiling
         volumes:
-            - ./compose-captchouli:/home/captchouli/.captchouli
+            - compose-captchouli:/home/captchouli/.captchouli
     minio:
         image: "minio/minio"
         hostname: minio
@@ -20,7 +23,7 @@ services:
             - MINIO_ACCESS_KEY=minio
             - MINIO_SECRET_KEY=miniostorage
         volumes:
-            - ./compose-data/minio:/data
+            - compose-minio:/data
         command: server /data
     nginx:
         image: nginx
@@ -39,15 +42,20 @@ services:
             - POSTGRES_PASSWORD=dev
             - POSTGRES_DB=dev
         volumes:
-            - ./compose-data/postgres:/var/lib/postgresql/data
+            - compose-postgres:/var/lib/postgresql/data
     redis:
         image: redis
         hostname: redis
         restart: always
+        command: [sh, -c, "rm -f /data/dump.rdb && redis-server --save ''"]
     dnsproxy:
         image: "defreitas/dns-proxy-server"
         hostname: "dns.mageddo"
         network_mode: host
         volumes:
             - /var/run/docker.sock:/var/run/docker.sock
             - /etc/resolv.conf:/etc/resolv.conf
+volumes:
+  compose-captchouli:
+  compose-minio:
+  compose-postgres:

frontend/Dockerfile

@@ -0,0 +1,15 @@
+FROM node:12.12.0-alpine
+WORKDIR /frontend-src/
+
+COPY package.json /frontend-src
+COPY package-lock.json /frontend-src
+RUN npm install
+COPY src/ /frontend-src/src
+COPY Gulpfile.js /frontend-src
+RUN npm run gulp
+RUN mv /frontend-src/build /maniwani-frontend
+RUN cp -r /frontend-src/node_modules /maniwani-frontend/
+WORKDIR /maniwani-frontend
+COPY entrypoint.sh /maniwani-frontend
+EXPOSE 3000
+ENTRYPOINT ["sh", "./entrypoint.sh"]

frontend/Gulpfile.js

@@ -0,0 +1,40 @@
+const { series, parallel, src, dest } = require('gulp');
+var gulp = require('gulp');
+var babel = require('gulp-babel');
+var filter = require('gulp-filter');
+var log = require('gulplog');
+var tap = require('gulp-tap');
+var uglify = require('gulp-uglify');
+var browserify = require('browserify');
+var buffer = require('vinyl-buffer');
+
+function compile_jsx() {
+    return gulp.src('src/**/*.jsx').
+        pipe(babel({
+            plugins: ['@babel/transform-react-jsx', '@babel/plugin-transform-modules-commonjs'],
+			presets: ['@babel/env']
+        })).
+        pipe(gulp.dest('build/'));
+}
+function compile_js() {
+	return gulp.src('src/**/*.js').
+		pipe(babel({
+			presets: ['@babel/env']})). 
+		pipe(gulp.dest('build/'));
+}
+function compile_client() {
+	return gulp.src('build/client/*.js').
+		pipe(tap(function (file) {
+			log.info('bundling ' + file.path);
+			file.contents = browserify(file.path, {debug: true}).bundle();
+		})).
+		pipe(buffer()).
+		pipe(uglify()).
+		pipe(gulp.dest('build/client-bundle'));
+}
+
+exports.jsx = compile_jsx;
+exports.js = compile_js;
+exports.build = series(parallel(exports.jsx, exports.js), compile_client);
+exports.default = exports.build;
+

frontend/README.md

@@ -0,0 +1,22 @@
+Express/React frontend for Maniwani
+===================================
+
+
+Building
+--------
+
+The frontend builds with Docker, so the following should work:
+
+	docker build -t maniwani-frontend .
+
+
+Running
+-------
+
+The frontend currently takes no options, so running it should be as simple as:
+
+	docker run -p 3000:3000 maniwani-frontend
+
+Point to the hostname and port combo of the frontend in `maniwani.cfg`, and
+the backend should communicate with it correctly. See the `docker-compose.yml`
+and `maniwani.cfg` included with this repository for an example.

frontend/devmode-entrypoint.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+cd /maniwani-frontend
+node server.js

frontend/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+node server.js