[Feature] Allow Building

.changeset/fuzzy-knives-brush.md

@@ -0,0 +1,5 @@
+---
+'@powersync/lib-services-framework': minor
+---
+
+Initial release for service utillities

.changeset/shaggy-parents-join.md

@@ -0,0 +1,5 @@
+---
+'@powersync/service-core': minor
+---
+
+Removed dependency for restricted packages

.env.template

@@ -0,0 +1,3 @@
+# Connections for tests
+MONGO_TEST_UR="mongodb://localhost:27017/powersync_test"
+PG_TEST_URL="postgres://postgres:postgres@localhost:5432/powersync_test"

.github/workflows/development_image_release.yaml

@@ -41,12 +41,6 @@ jobs:
           version: 9
           run_install: false
 
-      - name: Add NPM auth
-        run: |
-          echo //npm.pkg.github.com/:_authToken=\${{secrets.RESTRICTED_PACKAGES_TOKEN}} >> ~/.npmrc
-          echo "//npm.pkg.github.com/journeyapps-platform/:_authToken=${{secrets.RESTRICTED_PACKAGES_TOKEN}}" >> ~/.npmrc
-          echo "@journeyapps-platform:registry=https://npm.pkg.github.com/journeyapps-platform/" >> ~/.npmrc
-
       - name: Get pnpm store directory
         shell: bash
         run: |
@@ -86,6 +80,3 @@ jobs:
           tags: ${{vars.DOCKER_REGISTRY}}:${{steps.get_version.outputs.SERVICE_VERSION}}
           push: true
           file: ./service/Dockerfile
-          # TODO remove this when removing Journey Micro
-          build-args: |
-            GITHUB_TOKEN=${{secrets.RESTRICTED_PACKAGES_TOKEN}}

.github/workflows/development_packages_release.yaml

@@ -27,13 +27,6 @@ jobs:
           version: 9
           run_install: false
 
-      - name: Add NPM auth
-        run: |
-          echo "//registry.npmjs.org/:_authToken=${{secrets.NPM_TOKEN}}" >> ~/.npmrc
-          echo //npm.pkg.github.com/:_authToken=\${{secrets.RESTRICTED_PACKAGES_TOKEN}} >> ~/.npmrc
-          echo "//npm.pkg.github.com/journeyapps-platform/:_authToken=${{secrets.RESTRICTED_PACKAGES_TOKEN}}" >> ~/.npmrc
-          echo "@journeyapps-platform:registry=https://npm.pkg.github.com/journeyapps-platform/" >> ~/.npmrc
-
       - name: Get pnpm store directory
         shell: bash
         run: |
@@ -47,6 +40,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pnpm-store-
 
+      - name: Add NPM auth
+        run: |
+          echo "//registry.npmjs.org/:_authToken=${{secrets.NPM_TOKEN}}" >> ~/.npmrc
+
       - name: Install dependencies
         run: pnpm install
 

.github/workflows/image_release.yaml

@@ -2,60 +2,57 @@
 name: Docker Image Release
 
 on: workflow_dispatch
-  
+
 concurrency: ${{ github.workflow }}-${{ github.ref }}
 
 jobs:
-    release-docker-image:
-        name: Build and Release powersync-service Docker Image
-        runs-on: ubuntu-latest
-        if: github.ref == 'refs/heads/main' 
-        steps:
-          - name: Checkout
-            uses: actions/checkout@v4
-            with:
-              # check out full history
-              # Temporarily needed for changesets
-              fetch-depth: 0
-
-          - name: Set up QEMU
-            uses: docker/setup-qemu-action@v3
-
-          - name: Set up Docker Buildx
-            uses: docker/setup-buildx-action@v3
-
-          - name: Login to Docker Hub
-            uses: docker/login-action@v3
-            with:
-              username: ${{ secrets.DOCKERHUB_USERNAME }}
-              password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-            # This uses the service's package.json version for the Docker Image tag
-          - name: Get Service Version from package.json
-            id: get_version
-            run: echo "SERVICE_VERSION=$(node -p "require('./service/package.json').version")" >> $GITHUB_OUTPUT
-
-          - name: Build Image and Push
-            uses: docker/build-push-action@v5
-            with:
-              platforms: linux/arm64,linux/amd64
-              cache-from: type=registry,ref=${{vars.DOCKER_REGISTRY}}:latest
-              context: .
-              tags: ${{vars.DOCKER_REGISTRY}}:latest,${{vars.DOCKER_REGISTRY}}:${{steps.get_version.outputs.SERVICE_VERSION}}
-              push: true
-              file: ./service/Dockerfile
-              # TODO remove this when removing Journey Micro
-              build-args: |
-                GITHUB_TOKEN=${{secrets.RESTRICTED_PACKAGES_TOKEN}}
-
-          #   # Updates the README section on the DockerHub page
-          - name: Update repo description
-            #  Note that this 3rd party extention is recommended in the DockerHub docs:
-            #  https://docs.docker.com/build/ci/github-actions/update-dockerhub-desc/
-            uses: peter-evans/dockerhub-description@e98e4d1628a5f3be2be7c231e50981aee98723ae # v4.0.0
-            with:
-              username: ${{ secrets.DOCKERHUB_USERNAME }}
-              password: ${{ secrets.DOCKERHUB_TOKEN }}
-              repository: ${{vars.DOCKER_REGISTRY}}
-              #   This is the contents of what will be shown on DockerHub
-              readme-filepath: ./service/README.md
+  release-docker-image:
+    name: Build and Release powersync-service Docker Image
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # check out full history
+          # Temporarily needed for changesets
+          fetch-depth: 0
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+        # This uses the service's package.json version for the Docker Image tag
+      - name: Get Service Version from package.json
+        id: get_version
+        run: echo "SERVICE_VERSION=$(node -p "require('./service/package.json').version")" >> $GITHUB_OUTPUT
+
+      - name: Build Image and Push
+        uses: docker/build-push-action@v5
+        with:
+          platforms: linux/arm64,linux/amd64
+          cache-from: type=registry,ref=${{vars.DOCKER_REGISTRY}}:latest
+          context: .
+          tags: ${{vars.DOCKER_REGISTRY}}:latest,${{vars.DOCKER_REGISTRY}}:${{steps.get_version.outputs.SERVICE_VERSION}}
+          push: true
+          file: ./service/Dockerfile
+
+      #   # Updates the README section on the DockerHub page
+      - name: Update repo description
+        #  Note that this 3rd party extention is recommended in the DockerHub docs:
+        #  https://docs.docker.com/build/ci/github-actions/update-dockerhub-desc/
+        uses: peter-evans/dockerhub-description@e98e4d1628a5f3be2be7c231e50981aee98723ae # v4.0.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          repository: ${{vars.DOCKER_REGISTRY}}
+          #   This is the contents of what will be shown on DockerHub
+          readme-filepath: ./service/README.md

.github/workflows/packages_release.yaml

@@ -39,12 +39,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pnpm-store-
 
-      # TODO remove this when no longer needed
-      - name: Temporary Package Credentials
+      - name: Add NPM auth
         run: |
-          echo //npm.pkg.github.com/:_authToken=\${{secrets.RESTRICTED_PACKAGES_TOKEN}} >> ~/.npmrc
-          echo "//npm.pkg.github.com/journeyapps-platform/:_authToken=${{secrets.RESTRICTED_PACKAGES_TOKEN}}" >> ~/.npmrc
-          echo "@journeyapps-platform:registry=https://npm.pkg.github.com/journeyapps-platform/" >> ~/.npmrc
+          echo "//registry.npmjs.org/:_authToken=${{secrets.NPM_TOKEN}}" >> ~/.npmrc
 
       - name: Install dependencies
         run: pnpm install

.github/workflows/test.yml

@@ -70,13 +70,6 @@ jobs:
         with:
           node-version-file: '.nvmrc'
 
-      # TODO remove this when no longer needed
-      - name: Temporary Package Credentials
-        run: |
-          echo //npm.pkg.github.com/:_authToken=\${{secrets.RESTRICTED_PACKAGES_TOKEN}} >> ~/.npmrc
-          echo "//npm.pkg.github.com/journeyapps-platform/:_authToken=${{secrets.RESTRICTED_PACKAGES_TOKEN}}" >> ~/.npmrc
-          echo "@journeyapps-platform:registry=https://npm.pkg.github.com/journeyapps-platform/" >> ~/.npmrc
-
       - uses: pnpm/action-setup@v2
         name: Install pnpm
         with:

.gitignore

@@ -15,6 +15,8 @@ npm-error.log
 .pnpm-debug.log
 .local-dev
 .probes
+powersync.yaml
+sync-rules.yaml
 
 
 packages/*/manifest.json

DEVELOP.md

@@ -14,6 +14,69 @@ The project uses TypeScript. Build packages with
 pnpm build
 ```
 
+# Running Service
+
+## Dependent Services
+
+The PowerSync service requires Postgres and MongoDB server connections. These configuration details can be specified in a `powersync.yaml` (or JSON) configuration file.
+
+See the [Self hosting demo](https://github.com/powersync-ja/self-host-demo) for examples of starting these services.
+
+A quick method for running all required services with a handy backend and frontend is to run the following in a checked-out `self-host-demo` folder.
+
+```bash
+docker compose up --scale powersync=0
+```
+
+Note: The `mongo` hostname specified in the MongoDB replica set needs to be accessible by your host machine if using the Mongo service above.
+
+This will start all the services defined in the Self hosting demo except for the PowerSync service - which will be started from this repository.
+
+## Local Configuration
+
+The `./service` folder contains a NodeJS project which starts all PowerSync service operations.
+
+Copy the template configuration files and configure any changes to your local needs.
+
+```bash
+cd ./service
+cp local-dev/powersync-template.yaml powersync.yaml
+cp local-dev/sync-rules-template.yaml sync-rules.yaml
+```
+
+## Starting Service
+
+The service can be started with watching changes to any consumed source files by running the `pnpm watch:service` command in the repository root.
+
+# Running Tests
+
+Most packages should contain a `test` script which can be executed with `pnpm test`. Some packages may require additional setup to run tests.
+
+## Service Core
+
+Some tests for these packages require a connection to MongoDB and Postgres. Connection strings for these services should be available as environment variables. See [Running Tests](#running-services) for details on configuring those services.
+
+These can be set in a terminal/shell
+
+```bash
+export MONGO_TEST_UR="mongodb://localhost:27017/powersync_test"
+export PG_TEST_URL="postgres://postgres:postgres@localhost:5432/powersync_test"
+```
+
+or by copying the `.env.template` file and using a loader such as [Direnv](https://direnv.net/)
+
+```bash
+cp .env.template .env
+```
+
+## Postgres Configuration
+
+The default `PG_TEST_URL` points to a `powersync_test` database. Ensure this is created by executing the following SQL on your connection.
+
+```SQL
+CREATE DATABASE powersync_test;
+```
+
 # Releases
 
 This repository uses Changesets. Add changesets to changed packages before merging PRs.
@@ -22,4 +85,6 @@ This repository uses Changesets. Add changesets to changed packages before mergi
 changeset add
 ```
 
-Merging a PR with changeset files will automatically create a release PR. Merging the release PR will bump versions, tag and publish packages and the Docker image. The Docker image version is extracted from the `./service/package.json` `version` field.
+Merging a PR with changeset files will automatically create a release PR. Merging the release PR will bump versions, tag and publish packages.
+
+The Docker image is published by manually triggering the `Docker Image Release` Github Action. The Docker image version is extracted from the `./service/package.json` `version` field.

README.md

@@ -2,7 +2,7 @@
   <a href="https://www.powersync.com" target="_blank"><img src="https://github.com/powersync-ja/.github/assets/7372448/d2538c43-c1a0-4c47-9a76-41462dba484f"/></a>
 </p>
 
-*[PowerSync](https://www.powersync.com) is a Postgres-SQLite sync layer, which helps developers to create local-first real-time reactive apps that work seamlessly both online and offline.*
+_[PowerSync](https://www.powersync.com) is a Postgres-SQLite sync layer, which helps developers to create local-first real-time reactive apps that work seamlessly both online and offline._
 
 # PowerSync Service
 
@@ -35,11 +35,16 @@ The service can be started using the public Docker image. See the image [notes](
 - [packages/types](./packages/types/README.md)
   - Type definitions for the PowerSync service
 
+## Monorepo Structure: Libraries
+
+- [libs/lib-services](./libs/lib-services/README.md)
+
+  - A light-weight set of definitions and utilities for micro services
+
 ## Service
 
 The PowerSync service code is located in the `service` folder. This project is used to build the `journeyapps/powersync-service` Docker image.
 
 # Notes
 
 This mono repo currently relies on `restricted` packages. Currently this repo can only be built in CI. These dependencies will be removed soon.
-

libs/lib-services/README.md

@@ -0,0 +1,3 @@
+# Service Library
+
+A library containing base definitions for interacting with Micro services.

libs/lib-services/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@powersync/lib-services-framework",
+  "repository": "https://github.com/powersync-ja/powersync-service",
+  "version": "0.0.1",
+  "main": "dist/index.js",
+  "type": "module",
+  "types": "dist/index.d.ts",
+  "license": "FSL-1.1-Apache-2.0",
+  "files": [
+    "dist/**/*"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "clean": "rm -r ./dist && tsc -b --clean",
+    "build": "tsc -b",
+    "build:tests": "tsc -b test/tsconfig.json",
+    "test": "vitest"
+  },
+  "keywords": [],
+  "dependencies": {
+    "ajv": "^8.12.0",
+    "better-ajv-errors": "^1.2.0",
+    "bson": "^6.6.0",
+    "dotenv": "^16.4.5",
+    "lodash": "^4.17.21",
+    "ts-codec": "^1.2.2",
+    "winston": "^3.13.0",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@types/lodash": "^4.17.5",
+    "vitest": "^0.34.6"
+  }
+}

libs/lib-services/src/alerts/alerts-index.ts

@@ -0,0 +1,2 @@
+export * from './definitions.js';
+export * from './no-op-reporter.js';

libs/lib-services/src/alerts/definitions.ts

@@ -0,0 +1,17 @@
+import * as errors from '../errors/errors-index.js';
+
+export type Primitive = string | number | boolean;
+
+export type CaptureOptions = {
+  level?: errors.ErrorSeverity;
+  tags?: Record<string, Primitive | undefined>;
+  metadata?: Record<string, Primitive | undefined>;
+};
+
+export type CaptureErrorFunction = (error: any, options?: CaptureOptions) => void;
+export type CaptureMessageFunction = (message: string, options?: CaptureOptions) => void;
+
+export type ErrorReporter = {
+  captureException: CaptureErrorFunction;
+  captureMessage: CaptureMessageFunction;
+};

libs/lib-services/src/alerts/no-op-reporter.ts

@@ -0,0 +1,6 @@
+import { ErrorReporter } from './definitions.js';
+
+export const NoOpReporter: ErrorReporter = {
+  captureException: () => {},
+  captureMessage: () => {}
+};

libs/lib-services/src/codec/codec-index.ts

@@ -0,0 +1,2 @@
+export * from './codecs.js';
+export * from './parsers.js';