Add developer support for using next.js to serve generated static files

.github/workflows/dockerize.yml

@@ -8,6 +8,7 @@ on:
       - master
     paths:
       - src/khoj/**
+      - src/interface/web/**
       - pyproject.toml
       - Dockerfile
       - prod.Dockerfile
@@ -30,8 +31,9 @@ on:
 env:
   # Tag Image with tag name on release
   # else with user specified tag (default 'dev') if triggered via workflow
+  # else with run_id if triggered via a pull request
   # else with 'pre' (if push to master)
-  DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || github.event_name == 'workflow_dispatch' && github.event.inputs.tag || 'pre' }}
+  DOCKER_IMAGE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || github.event_name == 'workflow_dispatch' && github.event.inputs.tag || github.event_name == 'push' && github.run_id || 'pre' }}
 
 jobs:
   build:

.github/workflows/pypi.yml

@@ -8,13 +8,15 @@ on:
       - 'master'
     paths:
       - src/khoj/**
+      - src/interface/web/**
       - pyproject.toml
       - .github/workflows/pypi.yml
   pull_request:
     branches:
       - 'master'
     paths:
       - src/khoj/**
+      - src/interface/web/**
       - pyproject.toml
       - .github/workflows/pypi.yml
 
@@ -37,6 +39,16 @@ jobs:
       - name: ⬇️ Install Application
         run: python -m pip install --upgrade pip && pip install --upgrade .
 
+      - name: Install the Next.js application
+        run: |
+          yarn install
+        working-directory: src/interface/web
+
+      - name: Build & export static Next.js app to Django static assets
+        run: |
+          yarn ciexport
+        working-directory: src/interface/web
+
       - name: ⚙️ Build Python Package
         run: |
           # Setup Environment for Reproducible Builds

.gitignore

@@ -16,6 +16,7 @@ todesktop.json
 
 # Build artifacts
 /src/khoj/interface/web/images
+/src/khoj/interface/built/
 /build/
 /dist/
 khoj_assistant.egg-info

Dockerfile

@@ -3,11 +3,17 @@ FROM ubuntu:jammy
 LABEL org.opencontainers.image.source https://github.com/khoj-ai/khoj
 
 # Install System Dependencies
-RUN apt update -y && apt -y install python3-pip swig
+RUN apt update -y && apt -y install python3-pip swig curl
 
-WORKDIR /app
+# Install Node.js and Yarn
+RUN curl -sL https://deb.nodesource.com/setup_22.x | bash -
+RUN apt -y install nodejs
+RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+RUN apt update && apt -y install yarn
 
 # Install Application
+WORKDIR /app
 COPY pyproject.toml .
 COPY README.md .
 ARG VERSION=0.0.0
@@ -20,6 +26,11 @@ COPY . .
 # Set the PYTHONPATH environment variable in order for it to find the Django app.
 ENV PYTHONPATH=/app/src:$PYTHONPATH
 
+# Go to the directory src/interface/web and export the built Next.js assets
+WORKDIR /app/src/interface/web
+RUN bash -c "yarn install && yarn ciexport"
+WORKDIR /app
+
 # Run the Application
 # There are more arguments required for the application to run,
 # but these should be passed in through the docker-compose.yml file.

prod.Dockerfile

@@ -3,7 +3,14 @@ FROM ubuntu:jammy
 LABEL org.opencontainers.image.source https://github.com/khoj-ai/khoj
 
 # Install System Dependencies
-RUN apt update -y && apt -y install python3-pip libsqlite3-0 ffmpeg libsm6 libxext6
+RUN apt update -y && apt -y install python3-pip libsqlite3-0 ffmpeg libsm6 libxext6 swig curl
+
+# Install Node.js and Yarn
+RUN curl -sL https://deb.nodesource.com/setup_22.x | bash -
+RUN apt -y install nodejs
+RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+RUN apt update && apt -y install yarn
 
 WORKDIR /app
 
@@ -20,6 +27,11 @@ COPY . .
 # Set the PYTHONPATH environment variable in order for it to find the Django app.
 ENV PYTHONPATH=/app/src:$PYTHONPATH
 
+# Go to the directory src/interface/web and export the built Next.js assets
+WORKDIR /app/src/interface/web
+RUN bash -c "yarn install && yarn ciexport"
+WORKDIR /app
+
 # Run the Application
 # There are more arguments required for the application to run,
 # but these should be passed in through the docker-compose.yml file.

src/interface/web/.env.development

@@ -0,0 +1 @@
+NEXT_PUBLIC_ENV='development'

src/interface/web/.env.production

@@ -0,0 +1 @@
+NEXT_PUBLIC_ENV='production'

src/interface/web/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

src/interface/web/.gitignore

@@ -0,0 +1,36 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+.yarn/install-state.gz
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

src/interface/web/README.md

@@ -0,0 +1,93 @@
+This is a [Next.js](https://nextjs.org/) project.
+
+## Getting Started
+
+First, install the dependencies:
+
+```bash
+yarn install
+```
+
+In case you run into any dependency linking issues, you can try running:
+
+```bash
+yarn add next
+```
+
+### Run the development server:
+
+```bash
+yarn dev
+```
+
+Make sure the `rewrites` in `next.config.mjs` are set up correctly for your environment. The rewrites are used to proxy requests to the API server.
+
+```js
+    rewrites: async () => {
+        return [
+            {
+                source: '/api/:path*',
+                destination: 'http://localhost:42110/api/:path*',
+            },
+        ];
+    },
+```
+
+The `destination` should be the URL of the API server.
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying any of the `.tsx` pages. The page auto-updates as you edit the file.
+
+### Testing built files
+
+We've setup a utility command for building and serving the built files. This is useful for testing the production build locally.
+
+1. Exporting code
+To build the files once and serve them, run:
+```bash
+yarn export
+```
+
+If you're using Windows:
+```bash
+yarn windowsexport
+```
+
+
+2. Continuously building code
+
+To keep building the files and serving them, run:
+```bash
+yarn watch
+```
+
+If you're using Windows:
+```bash
+yarn windowswatch
+```
+
+Now you should be able to load your custom pages from the Khoj app at http://localhost:42110/. To server any of the built files, you should update the routes in the `web_client.py` like so, where `new_file` is the new page you've added in this repo:
+
+```python
+@web_client.post("/new_route", response_class=FileResponse)
+@requires(["authenticated"], redirect="login_page")
+def index_post(request: Request):
+
+    return templates.TemplateResponse(
+        "new_file/index.html",
+        context={
+            "request": request,
+        },
+    )
+```
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Next.js App Router](https://nextjs.org/docs/app) - learn about the Next.js router.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!