atproto self-host in docker

[syui]

version

v0.4.1

$ curl -sL ${pds}/xrpc/_health
{"version":"0.4.1"}

$ git clone https://github.com/bluesky-social
$ cd bluesky-social
$ git reset --hard 53122b4b3e2e3e764e152e9a98d7ccd215900ef9

docker compose

server	url	port	url
plc	https://github.com/did-method-plc/did-method-plc/tree/main/packages/server	2582	plc.example.com
pds	https://github.com/bluesky-social/atproto/tree/main/services/pds	2583	example.com
bsky	https://github.com/bluesky-social/atproto/tree/main/services/bsky	2584	api.example.com
mod	https://github.com/bluesky-social/atproto/tree/main/services/ozone	2585	mod.example.com
feed	https://github.com/bluesky-social/feed-generator	2586	feed.example.com
bgs	https://github.com/bluesky-social/indigo/tree/main/cmd/bigsky	2470	bgs.example.com
search	https://github.com/bluesky-social/indigo/tree/main/cmd/palomar	3999	search.example.com
web	https://github.com/bluesky-social/social-app/tree/main	8100	web.example.com

bsky=appview
host=example.com

git clone https://github.com/bluesky-social/atproto
cd atproto
git clone https://github.com/did-method-plc/did-method-plc ./repos/did-method-plc
git clone https://github.com/bluesky-social/indigo ./repos/indigo
git clone https://github.com/bluesky-social/social-app ./repos/social-app
touch .plc.env .bsky.env .bgs.env .pds.env .db.env .mod.env .web.env .feed.env

mkdir -p ./postgres/init
echo '-- plc
CREATE DATABASE plc;
GRANT ALL PRIVILEGES ON DATABASE plc TO postgres;

-- bgs
CREATE DATABASE bgs;
CREATE DATABASE carstore;
GRANT ALL PRIVILEGES ON DATABASE bgs TO postgres;
GRANT ALL PRIVILEGES ON DATABASE carstore TO postgres;

-- bsky(appview)
CREATE DATABASE bsky;
GRANT ALL PRIVILEGES ON DATABASE bsky TO postgres;

-- ozone(moderation)
CREATE DATABASE mod;
GRANT ALL PRIVILEGES ON DATABASE mod TO postgres;

-- pds
CREATE DATABASE pds;
GRANT ALL PRIVILEGES ON DATABASE pds TO postgres;
' >> ./postgres/init/init.sql

compose.yaml

services:
  plc:
    ports:
      - 2582:3000
    build:
      context: ./repos/did-method-plc/
      dockerfile: packages/server/Dockerfile
    env_file:
      - ./.plc.env
    depends_on:
      - db

  bgs:
    ports:
      - 2470:2470
    build:
      context: ./repos/indigo/
      dockerfile: cmd/bigsky/Dockerfile
    env_file:
      - ./.bgs.env
    volumes:
      - ./data/bgs/:/data/
    depends_on:
      - db

  bsky:
    ports:
      - 2584:3000
    build:
      context: ./
      dockerfile: services/bsky/Dockerfile
    env_file:
      - ./.bsky.env
    depends_on:
      - db
      - redis

  bsky-daemon:
    build:
      context: ./
      dockerfile: services/bsky/Dockerfile
    command: node --enable-source-maps daemon.js
    env_file:
      - ./.bsky.env
    depends_on:
      - bsky
      - db
      - redis

  bsky-indexer:
    build:
      context: ./
      dockerfile: services/bsky/Dockerfile
    command: node --enable-source-maps indexer.js
    env_file:
      - ./.bsky.env
    volumes:
      - ./data/bsky/cache/:/cache/
    depends_on:
      - bsky
      - db
      - redis

  bsky-ingester:
    build:
      context: ./
      dockerfile: services/bsky/Dockerfile
    command: node --enable-source-maps ingester.js
    env_file:
      - ./.bsky.env
    volumes:
      - ./data/bsky/cache/:/cache/
    depends_on:
      - bsky
      - db
      - redis

  mod:
    ports:
      - 2585:3000
    build:
      context: ./
      dockerfile: services/ozone/Dockerfile
    env_file:
      - ./.mod.env
    depends_on:
      - db

  mod-daemon:
    build:
      context: ./
      dockerfile: services/ozone/Dockerfile
    command: node --enable-source-maps daemon.js
    env_file:
      - ./.mod.env
    depends_on:
      - mod
      - db

  pds:
    ports:
      - 2583:3000
    build:
      context: ./
      dockerfile: services/pds/Dockerfile
    env_file:
      - ./.pds.env
    volumes:
      - ./data/pds/:/data/
    depends_on:
      - db

  social-app:
    ports:
      - 8100:8100
    build:
      context: ./repos/social-app/
      dockerfile: Dockerfile
    env_file:
      - ./.web.env
    command: "/usr/bin/bskyweb serve"

  db:
    image: postgres:latest
    env_file:
      - ./.db.env
    volumes:
      - ./postgres/init/:/docker-entrypoint-initdb.d/
      - ./data/db/:/var/lib/postgresql/data/

  redis:
    image: redis
    volumes:
      - ./data/redis/:/data/

other server

$ git clone https://github.com/bluesky-social/feed-generator ./repos/feed-generator

repos/feed-generator/Dockerfile

FROM node:18

WORKDIR /app
COPY . .
RUN yarn install

EXPOSE 3000
ENV PORT=3000
ENV NODE_ENV=production

CMD ["yarn", "start"]

LABEL org.opencontainers.image.source=https://github.com/bluesky-social/feed-generator

  feed-generator:
    ports:
      - 2586:3000
    build:
      context: ./repos/feed-generator/
    restart: always
    env_file:
      - ./.feed.env
    volumes:
      - ./data/feed-generator/:/data/

  search:
    ports:
      - 3999:3999
    build:
      context: ./repos/indigo/
      dockerfile: cmd/palomar/Dockerfile
    restart: always
    env_file:
      - ./.search.env
    depends_on:
      - database

hint

services:
    plc:
        depends_on:
            db:
                condition: service_started

services:
    plc:
        depends_on:
            db:
                condition: service_healthy

    db:
        healthcheck:
            # https://docs.docker.jp/compose/compose-file/compose-file-v3.html
            test: ["CMD-SHELL", "pg_isready"]
            interval: 10s
            timeout: 5s
            retries: 5

services:
    plc:
        # https://docs.docker.jp/v19.03/config/container/start-containers-automatically.html
        restart: always

services:
    db:
        # https://hub.docker.com/_/postgres
        image: postgres:16

services:
    db:
        # postgresql://postgres:postgres@127.0.0.1:5432
        ports:
          - 5432:5432

services:
    db:
        # postgresql://user:password@127.0.0.1/test
        environment:
          POSTGRES_USER: user
          POSTGRES_DB: test
          POSTGRES_PASSWORD: password 

$ docker compose up
$ docker compose down

postgres

POSTGRES_PORT=5432
POSTGRES_USER=postgres
POSTGRES_PASSWORD=postgres
POSTGRES_DB=healthcheck

env

server	env	body
bsky	DB_PRIMARY_POSTGRES_URL	postgres://postgres:postgres@db/bsky
bsky	DB_REPLICA_POSTGRES_URLS	postgres://postgres:postgres@db/bsky
bsky	DB_REPLICA_TAGS_ANY	0
bsky	PUBLIC_URL	https://api.${host}
bsky	SERVER_DID	did:web:api.${host}
bsky	DID_PLC_URL	https://plc.${host}
bsky	BLOB_CACHE_LOC	/cache/
bsky	SEARCH_ENDPOINT	https://search.${host}
bsky	REDIS_HOST	redis
bsky	INDEXER_PARTITION_IDS	0
bsky	INGESTER_PARTITION_COUNT	1
bsky	PUSH_NOTIFICATION_ENDPOINT	https://push.bsky.${host}/api/push
bsky	REPO_PROVIDER	wss://${host}
bsky	IMG_URI_ENDPOINT	https://cdn.${host}/img
bsky	ODERATION_SERVICE_DID	did:web:mod.${host}
bsky	MODERATION_PUSH_URL	https://admin:${OZONE_ADMIN_PASSWORD}@mod.${host}
bsky	ADMIN_PASSWORD	xxx
bsky	MODERATOR_PASSWORD	xxx
bsky	TRIAGE_PASSWORD	xxx
bsky	SERVICE_SIGNING_KEY	xxx
bsky	IMG_URI_SALT	xxx
bsky	IMG_URI_KEY	xxx

server	env	body
bgs	DATABASE_URL	postgres://postgres:postgres@db/bgs
bgs	CARSTORE_DATABASE_URL	postgres://postgres:postgres@db/carstore
bgs	DATA_DIR	/data
bgs	ATP_PLC_HOST	https://plc.${host}
bgs	BGS_ADMIN_KEY	xxx

server	env	body
mod	OZONE_PUBLIC_URL	https://mod.${host}
mod	OZONE_SERVER_DID	did:web:mod.${host}
mod	OZONE_APPVIEW_URL	https://api.${host}
mod	OZONE_APPVIEW_DID	did:web:api.${host}
mod	OZONE_PDS_URL	https://${host}
mod	OZONE_PDS_DID	did:web:${host}
mod	OZONE_DB_POSTGRES_URL	postgres://postgres:postgres@db/mod
mod	OZONE_DID_PLC_URL	https://plc.${host}
mod	MODERATION_PUSH_URL	https://admin:${OZONE_ADMIN_PASSWORD}@mod.${host}
mod	OZONE_ADMIN_PASSWORD	xxx
mod	OZONE_MODERATOR_PASSWORD	xxx
mod	OZONE_TRIAGE_PASSWORD	xxx
mod	OZONE_SIGNING_KEY_HEX	xxx

server	env	body
pds	PDS_HOSTNAME	${host}
pds	PDS_DATA_DIRECTORY	/data
pds	PDS_DB_POSTGRES_URL	postgresql://postgres:postgres@db/pds
pds	PDS_DID_PLC_URL	https://plc.${host}
pds	PDS_BSKY_APP_VIEW_URL	https://api.${host}
pds	PDS_BSKY_APP_VIEW_DID	did:web:api.${host}
pds	PDS_MOD_SERVICE_URL	https://mod.${host}
pds	PDS_MOD_SERVICE_DID	did:web:mod.${host}
pds	PDS_CRAWLERS	https://bgs.${host}
pds	PDS_EMAIL_SMTP_URL	smtps://${username}:${password}@smtp.gmail.com
pds	PDS_EMAIL_FROM_ADDRESS	no-reply@${host}
pds	PDS_INVITE_REQUIRED	false
pds	PDS_INVITE_INTERVAL	604800000
pds	PDS_BLOBSTORE_DISK_LOCATION	/data/img/static
pds	PDS_BLOBSTORE_DISK_TMP_LOCATION	/data/img/tmp
pds	PDS_JWT_SECRET	$ openssl rand --hex 16
pds	PDS_ADMIN_PASSWORD	xxx
pds	PDS_REPO_SIGNING_KEY_K256_PRIVATE_KEY_HEX	xxx
pds	PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX	xxx

server	env	body
plc	DATABASE_URL	postgres://postgres:postgres@db/plc
plc	DB_CREDS_JSON	'{"username":"postgres","password":"postgres","host":"db","port":"5432","database":"plc"}'
plc	DB_MIGRATE_CREDS_JSON	'{"username":"postgres","password":"postgres","host":"db","port":"5432","database":"plc"}'

server	env	body
*	DEBUG_MODE	1
*	LOG_ENABLED	true
*	LOG_LEVEL	debug
*	LOG_DESTINATION	1
*	ENABLE_MIGRATIONS	true

# xxx
$ openssl ecparam --name secp256k1 --genkey --noout --outform DER | tail --bytes=+8 | head --bytes=32 | xxd --plain --cols 32

appview : SERVICE_SIGNING_KEY/SERVER_DID

packages/dev-env/src/bsky.ts

//SERVICE_SIGNING_KEY=xxx
//SERVER_DID=did:plc:xxx
  static async create(cfg: BskyConfig): Promise<TestBsky> {
    // packages/crypto/tests/keypairs.test.ts
    const serviceKeypair = await Secp256k1Keypair.create({ exportable: true })
    console.log(`ROTATION_KEY=${serviceKeypair.did()}`)
    const exported = await serviceKeypair.export()
    const plcClient = new PlcClient(cfg.plcUrl)

    const port = cfg.port || (await getPort())
    const url = `http://localhost:${port}`
    const serverDid = await plcClient.createDid({
      signingKey: serviceKeypair.did(),
      rotationKeys: [serviceKeypair.did()],
      handle: 'bsky.test',
      pds: `http://localhost:${port}`,
      signer: serviceKeypair,
    })
    console.log(`SERVER_DID=${serverDid}`)

    const server = bsky.BskyAppView.create({
      db,
      redis: redisCache,
      config,
      algos: cfg.algos,
      imgInvalidator: cfg.imgInvalidator,
      signingKey: serviceKeypair,
    })

# https://web.plc.directory/api/redoc#operation/ResolveDid
url=https://plc.directory/did:plc:pyc2ihzpelxtg4cdkfzbhcv4
json='{ "type": "create", "signingKey": "did:key:zQ3shP5TBe1sQfSttXty15FAEHV1DZgcxRZNxvEWnPfLFwLxJ", "recoveryKey": "did:key:zQ3shhCGUqDKjStzuDxPkTxN6ujddP4RkEKJJouJGRRkaLGbg", "handle": "first-post.bsky.social", "service": "https://bsky.social", "prev": null, "sig": "yvN4nQYWTZTDl9nKSSyC5EC3nsF5g4S56OmRg9G6_-pM6FCItV2U2u14riiMGyHiCD86l6O-1xC5MPwf8vVsRw" }'
curl -X POST -H "Content-Type: application/json" -d "$json" $url | jq .

pds : invitecode

host=example.com
admin_password="admin-pass"
url=https://$host/xrpc/com.atproto.server.createInviteCode
json="{\"useCount\":1}"
curl -X POST -u admin:${admin_password} -H "Content-Type: application/json" -d "$json" -sL $url | jq .

bsky(appview) latest

v0.4.24

$ curl -sL ${pds}/xrpc/_health
{"version":"0.4.24"}

• patch : https://github.com/itaru2622/bluesky-selfhost-env/blob/master/patching/105-atproto-services-for-docker.diff
• discussion : anyone maintaining code for docker containers? #2334

compose.yml

  bsky:
    ports:
      - 2584:3000
    build:
      context: ./repos/atproto/
      dockerfile: services/bsky/Dockerfile
    restart: always
    env_file:
      - ./envs/appview
    volumes:
      - ./data/appview/cache/:/cache/
    command: node --enable-source-maps api.js
    depends_on:
      pds:
        condition: service_healthy
      db:
        condition: service_healthy
      redis:
        condition: service_healthy

#  bsky:
#    ports:
#      - 2584:3000
#    build:
#      context: ./repos/atproto/
#      dockerfile: services/bsky/Dockerfile
#    env_file:
#      - ./envs/appview
#    volumes:
#      - ./data/appview/cache/:/cache/
#    depends_on:
#      db:
#        condition: service_healthy
#      redis:
#        condition: service_healthy
#
#  bsky-daemon:
#    command: node --enable-source-maps daemon.js
#  bsky-indexer:
#    command: node --enable-source-maps indexer.js
#  bsky-ingester:
#    command: node --enable-source-maps ingester.js

# caddy and ssl3
BSKY_REPO_PROVIDER=ws://pds:3000
BSKY_REPO_PROVIDER=ws://${host}

BSKY_BSYNC_HTTP_VERSION=1.1
BSKY_BSYNC_PORT=3002
BSKY_BSYNC_URL=http://bsky:3002
BSKY_COURIER_URL=http://fake-courier.example.invalid/
BSKY_DATAPLANE_HTTP_VERSION=1.1
BSKY_DATAPLANE_PORT=3001
BSKY_DATAPLANE_URLS=http://bsky:3001
BSKY_DB_POSTGRES_SCHEMA=bsky
BSKY_DB_POSTGRES_URL=postgres://postgres:postgres@db
BSKY_DID_PLC_URL=https://plc.${host}
BSKY_PUBLIC_URL=https://api.${host}
BSKY_SEARCH_URL=https://palomar.${host}
BSKY_SERVER_DID=did:web:api.${host}
BSKY_SERVICE_SIGNING_KEY=xxx
DEBUG_MODE=1
DID_PLC_URL=https://plc.${host}
ENABLE_MIGRATIONS=false
LOG_DESTINATION=1
LOG_ENABLED=true
MOD_SERVICE_DID=did:web:mod.${host}
NODE_ENV=development
REDIS_HOST=redis
#NODE_TLS_REJECT_UNAUTHORIZED=${NODE_TLS_REJECT_UNAUTHORIZED}
#GOINSECURE=${GOINSECURE}
#BSKY_LABELS_FROM_ISSUER_DIDS=${BSKY_LABELS_FROM_ISSUER_DIDS}
#PORT=2584

BSKY_DB_POSTGRES_SCHEMA

postgres://postgres:postgres@db/${BSKY_DB_POSTGRES_SCHEMA}

BSKY_BSYNC_URL=http://bsky:3002

# compose.yml
  bsky:
    ports:
      - 2584:3000

special thanks

thx ikuradon -> https://github.com/ikuradon/atproto-starter-kit
thx itaru2622 -> https://github.com/itaru2622/bluesky-selfhost-env

[bnewbold]

There is an out-of-date docker-compose file and README in the indigo repo that might be helpful: https://github.com/bluesky-social/indigo/tree/main/cmd/fakermaker

Things are not in a great state for this right now. We plan to iterate on all the docs and dev support around self-hosting at the same time we open up federation, which means we aren't prioritizing updating docs on this until then.

In particular, I think the pds service is probably pretty out of date compared to the sandbox PDS "distribution", and we need to get those synced back up, and that may involve breaking changes to the PDS config env vars. Also the version of the PDS that we (bluesky) runs is now behind a "gateway service", which is unnecessary complexity for most folks (but needs documentation). The relay (BGS) and PLC are pretty stable. The appview is in the middle of several large refactors, with big changes in feature branches which have not been merged yet.

[itaru2622]

@syui did you succeed on self hosting with current codes? please update if you got progress.

[syui]

yes, my account https://web.syu.is/profile/syui.syu.is

[itaru2622]

could you share how you configure those? I also want to get self-hosting environment but failed configuring palomar (search service in indigo/cmd)

[syui]

I have compiled information for self-hosting here.
We recommend sandbox first.
https://github.com/bluesky-social/pds

[itaru2622]

hmm, finally I got my self-hosting environment for latest bluesky including bgs, plc, appview, pds, and others.
my codes are located https://github.com/itaru2622/bluesky-selfhost-env and self-hosting domain name is easy tune-able by environment variable. enjoy ;-)

[syui]

if bsky(appview) doesn't work

• anyone maintaining code for docker containers? #2334
• https://github.com/itaru2622/bluesky-selfhost-env/blob/master/patching/105-atproto-services-for-docker.diff