Use local SQLite database

.env.example

@@ -1,9 +1,6 @@
 # When cloning the repository, copy this file as .env
 # For getting started, you don't need to change any values
 
-CATDAT_DB_URL=file:databases/catdat/catdat.db
-CATDAT_DB_AUTH_TOKEN=
-
 APP_DB_URL=file:databases/app/app.db
 APP_DB_AUTH_TOKEN=
 

.github/workflows/_deploy-reusable.yaml

@@ -6,14 +6,7 @@ on:
             is_prod:
                 required: true
                 type: boolean
-            skip_db:
-                required: true
-                type: boolean
         secrets:
-            CATDAT_DB_URL:
-                required: true
-            CATDAT_DB_AUTH_TOKEN:
-                required: true
             APP_DB_URL:
                 required: true
             APP_DB_AUTH_TOKEN:
@@ -62,30 +55,14 @@ jobs:
             - name: Install Dependencies
               run: pnpm i
 
-            - name: Generate SvelteKit config
-              run: pnpm svelte-kit sync
-
-            - name: Create snapshot of local database as static asset
-              env:
-                  CATDAT_DB_URL: file:databases/catdat/catdat.db
-                  APP_DB_URL: file:databases/app/app.db
-              run: |
-                  pnpm db:update
-                  pnpm db:snapshot
-
-            - name: Update production database
-              if: ${{ !inputs.skip_db }}
+            - name: Build database
               env:
-                  CATDAT_DB_URL: ${{ secrets.CATDAT_DB_URL }}
-                  CATDAT_DB_AUTH_TOKEN: ${{ secrets.CATDAT_DB_AUTH_TOKEN }}
                   APP_DB_URL: ${{ secrets.APP_DB_URL }}
                   APP_DB_AUTH_TOKEN: ${{ secrets.APP_DB_AUTH_TOKEN }}
               run: pnpm db:update
 
             - name: Build app
               env:
-                  CATDAT_DB_URL: ${{ secrets.CATDAT_DB_URL }}
-                  CATDAT_DB_AUTH_TOKEN: ${{ secrets.CATDAT_DB_AUTH_TOKEN }}
                   APP_DB_URL: ${{ secrets.APP_DB_URL }}
                   APP_DB_AUTH_TOKEN: ${{ secrets.APP_DB_AUTH_TOKEN }}
                   GITHUB_PRIVATE_KEY: ${{ secrets._GITHUB_PRIVATE_KEY }}

.github/workflows/deploy-preview.yaml

@@ -9,18 +9,4 @@ jobs:
             contents: write
         with:
             is_prod: false
-            skip_db: false
-        secrets:
-            CATDAT_DB_URL: ${{ secrets.CATDAT_DB_PREVIEW_URL }}
-            CATDAT_DB_AUTH_TOKEN: ${{ secrets.CATDAT_DB_PREVIEW_AUTH_TOKEN }}
-            NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
-            NETLIFY_ACCESS_TOKEN: ${{ secrets.NETLIFY_ACCESS_TOKEN }}
-            APP_DB_URL: ${{ secrets.APP_DB_URL }}
-            APP_DB_AUTH_TOKEN: ${{ secrets.APP_DB_AUTH_TOKEN }}
-            _GITHUB_PRIVATE_KEY: ${{ secrets._GITHUB_PRIVATE_KEY }}
-            REDIS_URL: ${{ secrets.REDIS_URL }}
-            ADMIN_PAGE_PASSWORD: ${{ secrets.ADMIN_PAGE_PASSWORD }}
-            EMAIL_ADDRESS: ${{ secrets.EMAIL_ADDRESS }}
-            EMAIL_PASSWORD: ${{ secrets.EMAIL_PASSWORD }}
-            ENABLE_EMAILS: ${{ secrets.ENABLE_EMAILS }}
-            APPROVAL_EMAIL: ${{ secrets.APPROVAL_EMAIL }}
+        secrets: inherit

.github/workflows/deploy-prod.yaml

@@ -9,18 +9,4 @@ jobs:
             contents: write
         with:
             is_prod: true
-            skip_db: false
-        secrets:
-            CATDAT_DB_URL: ${{ secrets.CATDAT_DB_URL }}
-            CATDAT_DB_AUTH_TOKEN: ${{ secrets.CATDAT_DB_AUTH_TOKEN }}
-            NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
-            NETLIFY_ACCESS_TOKEN: ${{ secrets.NETLIFY_ACCESS_TOKEN }}
-            APP_DB_URL: ${{ secrets.APP_DB_URL }}
-            APP_DB_AUTH_TOKEN: ${{ secrets.APP_DB_AUTH_TOKEN }}
-            _GITHUB_PRIVATE_KEY: ${{ secrets._GITHUB_PRIVATE_KEY }}
-            REDIS_URL: ${{ secrets.REDIS_URL }}
-            ADMIN_PAGE_PASSWORD: ${{ secrets.ADMIN_PAGE_PASSWORD }}
-            EMAIL_ADDRESS: ${{ secrets.EMAIL_ADDRESS }}
-            EMAIL_PASSWORD: ${{ secrets.EMAIL_PASSWORD }}
-            ENABLE_EMAILS: ${{ secrets.ENABLE_EMAILS }}
-            APPROVAL_EMAIL: ${{ secrets.APPROVAL_EMAIL }}
+        secrets: inherit

.github/workflows/test.yaml

@@ -29,11 +29,5 @@ jobs:
             - name: Install Dependencies
               run: pnpm i
 
-            - name: Generate SvelteKit config
-              run: pnpm svelte-kit sync
-
             - name: Update and Test database
-              env:
-                  CATDAT_DB_URL: file:databases/catdat/catdat.db
-                  APP_DB_URL: file:databases/app/app.db
-              run: pnpm db:update
+              run: pnpm db:update:catdat

CONTRIBUTING.md

@@ -73,7 +73,7 @@ to continuously run this update when a file in the subfolder [/databases/catdat/
 
 ### Troubleshooting
 
-- If the local database is corrupted, delete the `catdat.db` file and recreate it using `pnpm db:update`.
+- If the local database is corrupted, recreate it using `pnpm db:update`.
 - If the `pnpm db:update` command fails, examine the error message to determine the cause. It could be due to malformed SQL, a contradictory property, or a failing test in the `pnpm db:test` script (which also runs as part of the update command), as explained below.
 
 ### Tests for Data Quality

DATABASE.md

@@ -42,11 +42,11 @@ For functors there are similar tables, such as:
 - `functor_implications`
 - `functor_property_assignments`
 
-## Migrations vs. Data
+## Schema vs. Data
 
-Migrations update the database structure: tables, views, indexes, and triggers. They are defined in SQL files located in the subfolder [/databases/catdat/migrations](/databases/catdat/migrations/). The command `pnpm db:migrate` applies any new migrations.
+The schema defines the structure of the database: tables, views, indexes, and triggers. It is specified in several SQL files located in the subfolder [/databases/catdat/schema](/databases/catdat/schema/). The command `pnpm db:setup` deletes the old database file (if it exists) and creates a new one using this schema.
 
-Database entries (categories, properties, implications, etc.) are defined via SQL files in the subfolder [/databases/catdat/data](/databases/catdat/data). The command `pnpm db:seed` replaces the current database by clearing all existing data and inserting the entries defined in these SQL files.
+Database entries (categories, properties, implications, etc.) are defined in SQL files located in the subfolder [/databases/catdat/data](/databases/catdat/data/). The command `pnpm db:seed` replaces the current contents of the database by clearing all existing data and inserting the entries defined in these SQL files.
 
 ## Derived Data
 
@@ -60,7 +60,7 @@ The command `pnpm db:test` executes some tests and verifies that the data behave
 
 ## One command for everything
 
-Use `pnpm db:update` to run all the commands in sequence: `pnpm db:migrate`, `pnpm db:seed`,`pnpm db:deduce`, and `pnpm db:test`.
+Use `pnpm db:update` to run all the commands in sequence: `pnpm db:setup`, `pnpm db:seed`,`pnpm db:deduce`, and `pnpm db:test`.
 
 Use `pnpm db:watch` to run this command automatically every time a file in the subfolder [/databases/catdat/data](/databases/catdat/data) changes. This is useful in particular during development.
 

DEPLOYMENT.md

@@ -1,39 +1,31 @@
 # Deployment
 
-This page is only relevant for the maintainer(s) of this project.
-
-## Hosting
-
-The application is deployed on [Netlify](https://netlify.com). The database is hosted on [Turso](https://turso.tech).
+This page is only relevant to the maintainer(s) of this project.
 
 - [GitHub repository](https://github.com/ScriptRaccoon/catdat)
 - [Netlify project](https://app.netlify.com/sites/catdat/overview)
 
 ## Prerendering
 
-Most pages are prerendered for performance reasons. This implies that the database is consumed at build time to generate static HTML pages. The only non-prerendered pages are the search and comparison pages, since they involve a dynamic list of properties and categories.
+Most pages are prerendered for performance reasons. This implies that the database is consumed at build time to generate static HTML pages. The only non-prerendered pages are the search and comparison pages, since they involve dynamic lists of properties and categories.
+
+## Databases
 
-## Preview Database
+The CatDat SQLite database `catdat.db` is deployed as a file alongside the web application to Netlify and is located at `/var/task/databases/catdat/catdat.db` after deployment. The file system on Netlify is ephemeral, so this only works because the database is read-only during the runtime of the application.
 
-Since deployment requires running `pnpm db:update`, which temporarily clears all tables in the database, we maintain a separate remote database `catdat-preview` on Turso as a copy of the production database `catdat`. This prevents temporary disruptions and data inconsistencies in production.
+However, the SQLite database `app.db` (used for user submissions and page visits) requires writes and is hosted remotely on [Turso](https://turso.tech).
 
 ## Deployment Process
 
 Follow these steps to deploy to production:
 
-1. **Preview Deployment.** On [GitHub's actions page](https://github.com/ScriptRaccoon/CatDat/actions), manually trigger the preview pipeline [deploy-preview.yaml](.github/workflows/deploy-preview.yaml).
-
-It updates the remote preview database (`catdat-preview`) via `pnpm db:update`, builds the application, and creates a [deployment preview](https://docs.netlify.com/deploy/deploy-types/deploy-previews/) on Netlify. The production database and application remain unchanged in this step.
-
-2. **Promote to Production.** If the deployment preview looks correct, promote it to production in [Netlify's Deploys UI](https://app.netlify.com/projects/catdat/deploys).
-
-The updated application goes live and uses the preview database.
+1. **Preview Deployment.** On [GitHub's Actions page](https://github.com/ScriptRaccoon/CatDat/actions), manually trigger the preview pipeline [deploy-preview.yaml](.github/workflows/deploy-preview.yaml).
 
-3. **Production Deployment.** On [GitHub's actions page](https://github.com/ScriptRaccoon/CatDat/actions), manually trigger the pipeline [deploy-prod.yaml](.github/workflows/deploy-prod.yaml).
+    It creates the database via `pnpm db:update`, builds the application, and creates a [deployment preview](https://docs.netlify.com/deploy/deploy-types/deploy-previews/) on Netlify. For testing purposes, this step can also be used on branches other than `main`.
 
-This pipeline updates the remote production database (`catdat`) via `pnpm db:update` (hence, consolidates it with the preview database `catdat-preview`), builds the application, and deploys it on Netlify. The previous deployment preview is no longer live.
+2. **Production Deployment.** If the deployment preview looks correct, on [GitHub's Actions page](https://github.com/ScriptRaccoon/CatDat/actions), manually trigger the pipeline [deploy-prod.yaml](.github/workflows/deploy-prod.yaml).
 
-In case only the application, not the database has been updated since the last production deployment, the pipeline [deploy-prod-skip-db](.github/workflows/deploy-prod-skip-db.yaml) can be used. The preview deployment is not required.
+    This pipeline creates the database via `pnpm db:update`, builds the application, and deploys it on Netlify. The previous deployment preview is no longer live. Moreover, a tag is created on GitHub.
 
 ## Domains
 

README.md

@@ -55,7 +55,7 @@ Built with modern web technologies:
 
 - Language: [TypeScript](https://www.typescriptlang.org)
 - Framework: [SvelteKit](https://svelte.dev/docs/kit/introduction)
-- Database: [SQLite](https://sqlite.org/) / [Turso](https://turso.tech)
+- Database: [SQLite](https://sqlite.org/)
 - Deployment: [Netlify](https://netlify.com)
 - Math Rendering: [katex](https://www.npmjs.com/package/katex)
 

databases/app/scripts/migrate.ts → databases/app/scripts/setup.ts

@@ -13,13 +13,13 @@ const db = createClient({
 	authToken: APP_DB_AUTH_TOKEN,
 })
 
-migrate()
+setup()
 
 /**
  * Creates the tables in the app database.
  */
-async function migrate() {
-	console.info('\n--- Migrate App database ---')
+async function setup() {
+	console.info('\n--- Setup App database ---')
 	await create_visits_table()
 	await create_submissions_table()
 }

databases/catdat/migrations/004_category-properties.sql → databases/catdat/schema/004_category-properties.sql

@@ -20,15 +20,15 @@ CREATE TABLE related_category_properties (
 );
 
 CREATE TABLE category_property_assignments (
+    id INTEGER PRIMARY KEY,
     category_id TEXT NOT NULL,
     property_id TEXT NOT NULL,
     is_satisfied INTEGER NOT NULL
         CHECK (is_satisfied in (TRUE, FALSE)),
     reason TEXT NOT NULL CHECK (length(reason) > 0),
     is_deduced INTEGER NOT NULL DEFAULT FALSE
         CHECK (is_deduced in (TRUE, FALSE)),
-    position INTEGER NOT NULL DEFAULT 0,
-    PRIMARY KEY (category_id, property_id),
+    UNIQUE (category_id, property_id),
     FOREIGN KEY (category_id) REFERENCES categories (id) ON DELETE CASCADE,
     FOREIGN KEY (property_id) REFERENCES category_properties (id) ON DELETE CASCADE
 );

databases/catdat/migrations/010_functor-properties.sql → databases/catdat/schema/010_functor-properties.sql

@@ -12,15 +12,15 @@ CREATE TABLE functor_properties (
 CREATE UNIQUE INDEX functor_properties_lower_id_unique ON functor_properties (lower(id));
 
 CREATE TABLE functor_property_assignments (
+    id INTEGER PRIMARY KEY,
     functor_id TEXT NOT NULL,
     property_id TEXT NOT NULL,
     is_satisfied INTEGER NOT NULL
         CHECK (is_satisfied IN (TRUE, FALSE)),
     reason TEXT NOT NULL CHECK (length(reason) > 0),
     is_deduced INTEGER NOT NULL DEFAULT FALSE
         CHECK (is_deduced in (TRUE, FALSE)),
-    position INTEGER NOT NULL DEFAULT 0,
-    PRIMARY KEY (functor_id, property_id),
+    UNIQUE (functor_id, property_id),
     FOREIGN KEY (functor_id) REFERENCES functors (id) ON DELETE CASCADE,
     FOREIGN KEY (property_id) REFERENCES functor_properties (id) ON DELETE CASCADE
 );