Merge branch 'develop' into i18n-save-lang-for-auth-user

.dockerignore

@@ -1,6 +1,5 @@
 **/node_modules
 **/dist
-**/storybook-static
 **/.vscode
 **/npm-debug.log
 **/Dockerfile

.github/sync.yml

@@ -28,6 +28,16 @@ centerofci/mathesar-design:
   - .github/workflows/update-project-on-issue-label.yml
   - .github/workflows/update-project-on-pr-edit.yml
   - .github/workflows/update-project-on-pr-label.yml
+centerofci/mathesar-internal-crm:
+  - .github/actions/project_update/
+  - .github/workflows/add-to-project.yml
+  - .github/workflows/toc.yml
+  - .github/workflows/stale.yml
+  - .github/workflows/update-project-on-issue-close.yml
+  - .github/workflows/update-project-on-issue-edit.yml
+  - .github/workflows/update-project-on-issue-label.yml
+  - .github/workflows/update-project-on-pr-edit.yml
+  - .github/workflows/update-project-on-pr-label.yml
 centerofci/mathesar-private-notes:
   - .github/actions/project_update/
   - .github/workflows/add-to-project.yml

.github/workflows/run-lint-audit-tests-ui.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: 14
+          node-version: 18
       - id: npm-cache-dir
         run: echo "::set-output name=dir::$(npm config get cache)"
       - uses: actions/cache@v2
@@ -28,7 +28,7 @@ jobs:
           path: ${{ steps.npm-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: ${{ runner.os }}-node-
-      - run: npm install --no-audit --prefer-offline
+      - run: npm ci --no-audit --prefer-offline
       - run: npm run check-format
 
   lint:
@@ -41,7 +41,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: 14
+          node-version: 18
       - id: npm-cache-dir
         run: echo "::set-output name=dir::$(npm config get cache)"
       - uses: actions/cache@v2
@@ -50,7 +50,7 @@ jobs:
           path: ${{ steps.npm-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: ${{ runner.os }}-node-
-      - run: npm install --no-audit --prefer-offline
+      - run: npm ci --no-audit --prefer-offline
       - run: npm run lint
 
   typecheck:
@@ -63,7 +63,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: 14
+          node-version: 18
       - id: npm-cache-dir
         run: echo "::set-output name=dir::$(npm config get cache)"
       - uses: actions/cache@v2
@@ -72,7 +72,7 @@ jobs:
           path: ${{ steps.npm-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: ${{ runner.os }}-node-
-      - run: npm install --no-audit --prefer-offline
+      - run: npm ci --no-audit --prefer-offline
       - run: npm run typecheck
 
   audit:
@@ -85,7 +85,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: 14
+          node-version: 18
       - id: npm-cache-dir
         run: echo "::set-output name=dir::$(npm config get cache)"
       - uses: actions/cache@v2
@@ -94,7 +94,7 @@ jobs:
           path: ${{ steps.npm-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: ${{ runner.os }}-node-
-      - run: npm install
+      - run: npm ci
       - uses: oke-py/npm-audit-action@v1.8.2
         with:
           audit_level: moderate
@@ -116,7 +116,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: 14
+          node-version: 18
       - id: npm-cache-dir
         run: echo "::set-output name=dir::$(npm config get cache)"
       - uses: actions/cache@v2
@@ -125,5 +125,5 @@ jobs:
           path: ${{ steps.npm-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: ${{ runner.os }}-node-
-      - run: npm install --no-audit --prefer-offline
+      - run: npm ci --no-audit --prefer-offline
       - run: npm test

.github/workflows/run-pytest.yml

@@ -18,6 +18,9 @@ jobs:
     # We only want to run on external PRs, since internal PRs are covered by "push"
     # This prevents this from running twice on internal PRs
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      matrix:
+        pg-version: [13, 14, 15]
     steps:
       - uses: actions/checkout@v2
 
@@ -30,7 +33,11 @@ jobs:
         run: sudo chown -R 1000:1000 .
 
       - name: Build the stack
-        run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build -d test-service
-
+        run: docker build -t mathesar_service_test --build-arg BUILD_PG_MAJOR=${{ matrix.pg-version }} --build-arg PYTHON_REQUIREMENTS=requirements-dev.txt .
+        env:
+          PG_MAJOR: ${{ matrix.pg-version }}
+      - name: Run docker container
+        run: docker run -d --name mathesar_service_test mathesar_service_test
+      # Override setup.cfg docker-compose db service credentials with in built docker postgres server
       - name: Run tests with pytest
-        run: docker exec mathesar_service_test ./run_pytest.sh
+        run: docker exec -e DJANGO_DATABASE_URL="postgres://postgres:mathesar@localhost:5432/mathesar_django" -e MATHESAR_DATABASES="(mathesar_db_test|postgres://postgres:mathesar@localhost:5432/mathesar_db_test)" mathesar_service_test ./run_pytest.sh

.github/workflows/run-sql-tests.yml

@@ -14,6 +14,9 @@ jobs:
     # We only want to run on external PRs, since internal PRs are covered by "push"
     # This prevents this from running twice on internal PRs
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      matrix:
+        pg-version: [13, 14, 15]
     steps:
       - uses: actions/checkout@v2
 
@@ -27,6 +30,8 @@ jobs:
 
       - name: Build the test DB
         run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build -d dev-db
+        env:
+          PG_VERSION: ${{ matrix.pg-version }}
 
       - name: Run tests with pg_prove
         run: docker exec mathesar_dev_db /bin/bash /sql/run_tests.sh

.github/workflows/sync-github-labels-milestones.yml

@@ -16,6 +16,7 @@ jobs:
     - run: /home/runner/.composer/vendor/bin/github-sync labels -f centerofci/mathesar -t centerofci/mathesar-ansible -d
     - run: /home/runner/.composer/vendor/bin/github-sync labels -f centerofci/mathesar -t centerofci/mathesar-data-playground -d
     - run: /home/runner/.composer/vendor/bin/github-sync labels -f centerofci/mathesar -t centerofci/mathesar-design -d
+    - run: /home/runner/.composer/vendor/bin/github-sync labels -f centerofci/mathesar -t centerofci/mathesar-internal-crm -d
     - run: /home/runner/.composer/vendor/bin/github-sync labels -f centerofci/mathesar -t centerofci/mathesar-private-notes -d
     - run: /home/runner/.composer/vendor/bin/github-sync labels -f centerofci/mathesar -t centerofci/mathesar-scripts -d
     - run: /home/runner/.composer/vendor/bin/github-sync labels -f centerofci/mathesar -t centerofci/mathesar-update-companion -d
@@ -24,6 +25,7 @@ jobs:
     - run: /home/runner/.composer/vendor/bin/github-sync milestones -f centerofci/mathesar -t centerofci/mathesar-ansible -s open
     - run: /home/runner/.composer/vendor/bin/github-sync milestones -f centerofci/mathesar -t centerofci/mathesar-data-playground -s open
     - run: /home/runner/.composer/vendor/bin/github-sync milestones -f centerofci/mathesar -t centerofci/mathesar-design -s open
+    - run: /home/runner/.composer/vendor/bin/github-sync milestones -f centerofci/mathesar -t centerofci/mathesar-internal-crm -s open
     - run: /home/runner/.composer/vendor/bin/github-sync milestones -f centerofci/mathesar -t centerofci/mathesar-private-notes -s open
     - run: /home/runner/.composer/vendor/bin/github-sync milestones -f centerofci/mathesar -t centerofci/mathesar-scripts -s open
     - run: /home/runner/.composer/vendor/bin/github-sync milestones -f centerofci/mathesar -t centerofci/mathesar-update-companion -s open

.github/workflows/sync-workflows.yml

@@ -2,7 +2,7 @@ name: Sync GitHub workflows to other repos
 on:
   push:
     branches:
-      - master
+      - develop
   workflow_dispatch:
 jobs:
   sync_workflows:
@@ -13,4 +13,4 @@ jobs:
       - name: Run GitHub file sync
         uses: BetaHuhn/repo-file-sync-action@v1
         with:
-          GH_PAT: ${{secrets.MATHESAR_ORG_GITHUB_TOKEN}}
+          GH_PAT: ${{secrets.MATHESAR_ORG_GITHUB_TOKEN}}

.gitignore

@@ -183,7 +183,6 @@ docs/_build/
 
 # UI
 node_modules/
-storybook-static/
 
 # Client Build files
 mathesar/static/mathesar/

DEVELOPER_GUIDE.md

@@ -39,7 +39,7 @@ Mathesar is built using:
     - username: `admin`
     - password: `password`
 
-1. Keep Docker running while making your code changes. The app will update automatically with your new code.
+1. Keep Docker running while making your code changes. The app will update automatically with your new code. Please refer to our [Troubleshooting guide](#troubleshooting) if you are experiencing any issues.
 
 ## Contribution guidelines
 
@@ -143,6 +143,37 @@ See our [Live demo mode](./demo/README.md) guide for more information on enablin
     docker exec -it mathesar_db psql -U mathesar
     ```
 
+
+## Building Debian package
+
+- On a Debian machine, install the following dependencies
+
+    ```
+    sudo apt install debhelper-compat dh-virtualenv libsystemd-dev libpq-dev libicu-dev pkg-config lsb-release python3-dev python3 python3-setuptools python3-pip python3-venv tar
+    ```
+- Setup Mathesar build environment.
+   This step is useful only when testing locally is needed for building static files and for collecting them. We won't have a need for this step while using the build service as it will be using the source code from release assets which will contain these static files
+
+
+- Install Python and Nodejs preferably on a Linux machine
+- Run the following commands to set up the environment
+
+    ```
+    python3 -m venv ./mathesar-venv
+    source ./mathesar-venv/bin/activate
+    pip install -r requirements.txt
+    sudo npm install -g npm-force-resolutions
+    cd mathesar_ui && npm install --unsafe-perm && npm run build
+    cd ..
+    python manage.py collectstatic
+    ```
+
+- From the mathesar directory, run the build script to generate the debian package
+
+    ```
+    cd release-scripts && source build-debian.sh
+    ```
+
 ## Troubleshooting
 
 ### Permissions within Windows

Dockerfile

@@ -2,16 +2,58 @@ FROM python:3.9-buster
 ARG PYTHON_REQUIREMENTS=requirements.txt
 ENV PYTHONUNBUFFERED=1
 ENV DOCKERIZE_VERSION v0.6.1
+ENV NODE_MAJOR 18
+ARG BUILD_PG_MAJOR=15
+ENV PG_MAJOR=$BUILD_PG_MAJOR
 
-# Install dockerize
+# Install dependencies
+RUN apt-get update
+RUN apt-get install -y sudo ca-certificates curl gnupg
+RUN set -eux; \
+	apt-get update; apt-get install -y --no-install-recommends locales; rm -rf /var/lib/apt/lists/*; \
+	localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
+
+# Install PostgreSQL
+ENV LANG en_US.utf8
+RUN set -ex; \
+    curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - ; \
+    echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list; \
+    apt-get update -y; \
+     apt-get install -y --no-install-recommends \
+        postgresql-$PG_MAJOR postgresql-client-$PG_MAJOR postgresql-contrib-$PG_MAJOR \
+    ; \
+    apt-get clean; \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*; \
+    :
+
+ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin
+
+ENV PGDATA /var/lib/postgresql/mathesar
+VOLUME /etc/postgresql/
+VOLUME /var/lib/postgresql/
+
+
+# We set the default STOPSIGNAL to SIGINT, which corresponds to what PostgreSQL
+# calls "Fast Shutdown mode" wherein new connections are disallowed and any
+# in-progress transactions are aborted, allowing PostgreSQL to stop cleanly and
+# flush tables to disk, which is the best compromise available to avoid data
+# corruption.
+
+STOPSIGNAL SIGINT
+
+EXPOSE 5432
+
+# Install dockerize, we still need it when running Postgres using docker-compose
 RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
     && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
     && rm dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz
 
-# Install node
-RUN curl -fsSL https://deb.nodesource.com/setup_14.x | bash -
+# Install Node
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
 RUN apt-get update
-RUN apt install -y sudo nodejs && rm -rf /var/lib/apt/lists/*
+RUN apt-get install nodejs -y
 
 # Change work directory
 WORKDIR /code/
@@ -21,7 +63,6 @@ COPY requirements* ./
 RUN pip install --no-cache-dir -r ${PYTHON_REQUIREMENTS} --force-reinstall sqlalchemy-filters
 COPY . .
 
-RUN sudo npm install -g npm-force-resolutions
-RUN cd mathesar_ui && npm install --unsafe-perm && npm run build
+RUN cd mathesar_ui && npm ci && npm run build
 EXPOSE 8000 3000 6006
-ENTRYPOINT ["./run.sh"]
+ENTRYPOINT ["./run.sh"]

Dockerfile.devdb

@@ -1,4 +1,6 @@
-FROM postgres:13
+ARG PG_VERSION=13
+FROM postgres:${PG_VERSION}
+ARG PG_VERSION
 
 RUN apt update
-RUN apt install -y postgresql-13-pgtap postgresql-13-pldebugger && rm -rf /var/lib/apt/lists/*
+RUN apt install -y postgresql-${PG_VERSION}-pgtap postgresql-${PG_VERSION}-pldebugger && rm -rf /var/lib/apt/lists/*

Dockerfile.integ-tests

@@ -13,8 +13,17 @@ ENV PATH /usr/local/bin:$PATH
 # > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
 ENV LANG C.UTF-8
 
-# Download node14 source
-RUN curl -fsSL https://deb.nodesource.com/setup_14.x | bash -
+# Download node source
+ENV NODE_MAJOR 18
+
+RUN apt-get update
+RUN apt-get install -y ca-certificates curl gnupg
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+
+RUN apt-get update
+RUN apt-get install nodejs -y
 
 # extra dependencies (over what buildpack-deps already includes)
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -127,7 +136,6 @@ RUN pip install -r requirements-dev.txt
 RUN pip install -r requirements-demo.txt
 COPY . .
 
-RUN sudo npm install -g npm-force-resolutions
-RUN cd mathesar_ui && npm install --unsafe-perm && npm run build
+RUN cd mathesar_ui && npm ci && npm run build
 
 EXPOSE 8000 3000 6006

MANIFEST.in

@@ -0,0 +1 @@
+recursive-include static *