atproto self-host in docker #2026

syui · 2024-01-06T14:00:31Z

syui
Jan 6, 2024

version

v0.4.1

$ curl -sL ${pds}/xrpc/_health
{"version":"0.4.1"}

$ git clone https://github.com/bluesky-social
$ cd bluesky-social
$ git reset --hard 53122b4b3e2e3e764e152e9a98d7ccd215900ef9

docker compose

server	url	port	url
plc	https://github.com/did-method-plc/did-method-plc/tree/main/packages/server	2582	plc.example.com
pds	https://github.com/bluesky-social/atproto/tree/main/services/pds	2583	example.com
bsky	https://github.com/bluesky-social/atproto/tree/main/services/bsky	2584	api.example.com
mod	https://github.com/bluesky-social/atproto/tree/main/services/ozone	2585	mod.example.com
feed	https://github.com/bluesky-social/feed-generator	2586	feed.example.com
bgs	https://github.com/bluesky-social/indigo/tree/main/cmd/bigsky	2470	bgs.example.com
search	https://github.com/bluesky-social/indigo/tree/main/cmd/palomar	3999	search.example.com
web	https://github.com/bluesky-social/social-app/tree/main	8100	web.example.com

bsky=appview
host=example.com

git clone https://github.com/bluesky-social/atproto
cd atproto
git clone https://github.com/did-method-plc/did-method-plc ./repos/did-method-plc
git clone https://github.com/bluesky-social/indigo ./repos/indigo
git clone https://github.com/bluesky-social/social-app ./repos/social-app
touch .plc.env .bsky.env .bgs.env .pds.env .db.env .mod.env .web.env .feed.env

mkdir -p ./postgres/init
echo '-- plc
CREATE DATABASE plc;
GRANT ALL PRIVILEGES ON DATABASE plc TO postgres;

-- bgs
CREATE DATABASE bgs;
CREATE DATABASE carstore;
GRANT ALL PRIVILEGES ON DATABASE bgs TO postgres;
GRANT ALL PRIVILEGES ON DATABASE carstore TO postgres;

-- bsky(appview)
CREATE DATABASE bsky;
GRANT ALL PRIVILEGES ON DATABASE bsky TO postgres;

-- ozone(moderation)
CREATE DATABASE mod;
GRANT ALL PRIVILEGES ON DATABASE mod TO postgres;

-- pds
CREATE DATABASE pds;
GRANT ALL PRIVILEGES ON DATABASE pds TO postgres;
' >> ./postgres/init/init.sql

compose.yaml

services:
  plc:
    ports:
      - 2582:3000
    build:
      context: ./repos/did-method-plc/
      dockerfile: packages/server/Dockerfile
    env_file:
      - ./.plc.env
    depends_on:
      - db

  bgs:
    ports:
      - 2470:2470
    build:
      context: ./repos/indigo/
      dockerfile: cmd/bigsky/Dockerfile
    env_file:
      - ./.bgs.env
    volumes:
      - ./data/bgs/:/data/
    depends_on:
      - db

  bsky:
    ports:
      - 2584:3000
    build:
      context: ./
      dockerfile: services/bsky/Dockerfile
    env_file:
      - ./.bsky.env
    depends_on:
      - db
      - redis

  bsky-daemon:
    build:
      context: ./
      dockerfile: services/bsky/Dockerfile
    command: node --enable-source-maps daemon.js
    env_file:
      - ./.bsky.env
    depends_on:
      - bsky
      - db
      - redis

  bsky-indexer:
    build:
      context: ./
      dockerfile: services/bsky/Dockerfile
    command: node --enable-source-maps indexer.js
    env_file:
      - ./.bsky.env
    volumes:
      - ./data/bsky/cache/:/cache/
    depends_on:
      - bsky
      - db
      - redis

  bsky-ingester:
    build:
      context: ./
      dockerfile: services/bsky/Dockerfile
    command: node --enable-source-maps ingester.js
    env_file:
      - ./.bsky.env
    volumes:
      - ./data/bsky/cache/:/cache/
    depends_on:
      - bsky
      - db
      - redis

  mod:
    ports:
      - 2585:3000
    build:
      context: ./
      dockerfile: services/ozone/Dockerfile
    env_file:
      - ./.mod.env
    depends_on:
      - db

  mod-daemon:
    build:
      context: ./
      dockerfile: services/ozone/Dockerfile
    command: node --enable-source-maps daemon.js
    env_file:
      - ./.mod.env
    depends_on:
      - mod
      - db

  pds:
    ports:
      - 2583:3000
    build:
      context: ./
      dockerfile: services/pds/Dockerfile
    env_file:
      - ./.pds.env
    volumes:
      - ./data/pds/:/data/
    depends_on:
      - db

  social-app:
    ports:
      - 8100:8100
    build:
      context: ./repos/social-app/
      dockerfile: Dockerfile
    env_file:
      - ./.web.env
    command: "/usr/bin/bskyweb serve"

  db:
    image: postgres:latest
    env_file:
      - ./.db.env
    volumes:
      - ./postgres/init/:/docker-entrypoint-initdb.d/
      - ./data/db/:/var/lib/postgresql/data/

  redis:
    image: redis
    volumes:
      - ./data/redis/:/data/

other server

$ git clone https://github.com/bluesky-social/feed-generator ./repos/feed-generator

repos/feed-generator/Dockerfile

FROM node:18

WORKDIR /app
COPY . .
RUN yarn install

EXPOSE 3000
ENV PORT=3000
ENV NODE_ENV=production

CMD ["yarn", "start"]

LABEL org.opencontainers.image.source=https://github.com/bluesky-social/feed-generator

  feed-generator:
    ports:
      - 2586:3000
    build:
      context: ./repos/feed-generator/
    restart: always
    env_file:
      - ./.feed.env
    volumes:
      - ./data/feed-generator/:/data/

  search:
    ports:
      - 3999:3999
    build:
      context: ./repos/indigo/
      dockerfile: cmd/palomar/Dockerfile
    restart: always
    env_file:
      - ./.search.env
    depends_on:
      - database

hint

services:
    plc:
        depends_on:
            db:
                condition: service_started

services:
    plc:
        depends_on:
            db:
                condition: service_healthy

    db:
        healthcheck:
            # https://docs.docker.jp/compose/compose-file/compose-file-v3.html
            test: ["CMD-SHELL", "pg_isready"]
            interval: 10s
            timeout: 5s
            retries: 5

services:
    plc:
        # https://docs.docker.jp/v19.03/config/container/start-containers-automatically.html
        restart: always

services:
    db:
        # https://hub.docker.com/_/postgres
        image: postgres:16

services:
    db:
        # postgresql://postgres:postgres@127.0.0.1:5432
        ports:
          - 5432:5432

services:
    db:
        # postgresql://user:password@127.0.0.1/test
        environment:
          POSTGRES_USER: user
          POSTGRES_DB: test
          POSTGRES_PASSWORD: password

$ docker compose up
$ docker compose down

postgres

POSTGRES_PORT=5432
POSTGRES_USER=postgres
POSTGRES_PASSWORD=postgres
POSTGRES_DB=healthcheck

env

server	env	body
bsky	DB_PRIMARY_POSTGRES_URL	postgres://postgres:postgres@db/bsky
bsky	DB_REPLICA_POSTGRES_URLS	postgres://postgres:postgres@db/bsky
bsky	DB_REPLICA_TAGS_ANY	0
bsky	PUBLIC_URL	`https://api.${host}`
bsky	SERVER_DID	did:web:api.${host}
bsky	DID_PLC_URL	`https://plc.${host}`
bsky	BLOB_CACHE_LOC	/cache/
bsky	SEARCH_ENDPOINT	`https://search.${host}`
bsky	REDIS_HOST	redis
bsky	INDEXER_PARTITION_IDS	0
bsky	INGESTER_PARTITION_COUNT	1
bsky	PUSH_NOTIFICATION_ENDPOINT	`https://push.bsky.${host}/api/push`
bsky	REPO_PROVIDER	wss://${host}
bsky	IMG_URI_ENDPOINT	`https://cdn.${host}/img`
bsky	ODERATION_SERVICE_DID	did:web:mod.${host}
bsky	MODERATION_PUSH_URL	`https://admin:${OZONE_ADMIN_PASSWORD}@mod.${host}`
bsky	ADMIN_PASSWORD	xxx
bsky	MODERATOR_PASSWORD	xxx
bsky	TRIAGE_PASSWORD	xxx
bsky	SERVICE_SIGNING_KEY	xxx
bsky	IMG_URI_SALT	xxx
bsky	IMG_URI_KEY	xxx

server	env	body
bgs	DATABASE_URL	postgres://postgres:postgres@db/bgs
bgs	CARSTORE_DATABASE_URL	postgres://postgres:postgres@db/carstore
bgs	DATA_DIR	/data
bgs	ATP_PLC_HOST	`https://plc.${host}`
bgs	BGS_ADMIN_KEY	xxx

server	env	body
mod	OZONE_PUBLIC_URL	`https://mod.${host}`
mod	OZONE_SERVER_DID	did:web:mod.${host}
mod	OZONE_APPVIEW_URL	`https://api.${host}`
mod	OZONE_APPVIEW_DID	did:web:api.${host}
mod	OZONE_PDS_URL	`https://${host}`
mod	OZONE_PDS_DID	did:web:${host}
mod	OZONE_DB_POSTGRES_URL	postgres://postgres:postgres@db/mod
mod	OZONE_DID_PLC_URL	`https://plc.${host}`
mod	MODERATION_PUSH_URL	`https://admin:${OZONE_ADMIN_PASSWORD}@mod.${host}`
mod	OZONE_ADMIN_PASSWORD	xxx
mod	OZONE_MODERATOR_PASSWORD	xxx
mod	OZONE_TRIAGE_PASSWORD	xxx
mod	OZONE_SIGNING_KEY_HEX	xxx

server	env	body
pds	PDS_HOSTNAME	${host}
pds	PDS_DATA_DIRECTORY	/data
pds	PDS_DB_POSTGRES_URL	postgresql://postgres:postgres@db/pds
pds	PDS_DID_PLC_URL	`https://plc.${host}`
pds	PDS_BSKY_APP_VIEW_URL	`https://api.${host}`
pds	PDS_BSKY_APP_VIEW_DID	did:web:api.${host}
pds	PDS_MOD_SERVICE_URL	`https://mod.${host}`
pds	PDS_MOD_SERVICE_DID	did:web:mod.${host}
pds	PDS_CRAWLERS	`https://bgs.${host}`
pds	PDS_EMAIL_SMTP_URL	smtps://${username}:${password}@smtp.gmail.com
pds	PDS_EMAIL_FROM_ADDRESS	no-reply@${host}
pds	PDS_INVITE_REQUIRED	false
pds	PDS_INVITE_INTERVAL	604800000
pds	PDS_BLOBSTORE_DISK_LOCATION	/data/img/static
pds	PDS_BLOBSTORE_DISK_TMP_LOCATION	/data/img/tmp
pds	PDS_JWT_SECRET	`$ openssl rand --hex 16`
pds	PDS_ADMIN_PASSWORD	xxx
pds	PDS_REPO_SIGNING_KEY_K256_PRIVATE_KEY_HEX	xxx
pds	PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX	xxx

server	env	body
plc	DATABASE_URL	postgres://postgres:postgres@db/plc
plc	DB_CREDS_JSON	'{"username":"postgres","password":"postgres","host":"db","port":"5432","database":"plc"}'
plc	DB_MIGRATE_CREDS_JSON	'{"username":"postgres","password":"postgres","host":"db","port":"5432","database":"plc"}'

server	env	body
*	DEBUG_MODE	1
*	LOG_ENABLED	true
*	LOG_LEVEL	debug
*	LOG_DESTINATION	1
*	ENABLE_MIGRATIONS	true

# xxx
$ openssl ecparam --name secp256k1 --genkey --noout --outform DER | tail --bytes=+8 | head --bytes=32 | xxd --plain --cols 32

appview : SERVICE_SIGNING_KEY/SERVER_DID

packages/dev-env/src/bsky.ts

//SERVICE_SIGNING_KEY=xxx
//SERVER_DID=did:plc:xxx
  static async create(cfg: BskyConfig): Promise<TestBsky> {
    // packages/crypto/tests/keypairs.test.ts
    const serviceKeypair = await Secp256k1Keypair.create({ exportable: true })
    console.log(`ROTATION_KEY=${serviceKeypair.did()}`)
    const exported = await serviceKeypair.export()
    const plcClient = new PlcClient(cfg.plcUrl)

    const port = cfg.port || (await getPort())
    const url = `http://localhost:${port}`
    const serverDid = await plcClient.createDid({
      signingKey: serviceKeypair.did(),
      rotationKeys: [serviceKeypair.did()],
      handle: 'bsky.test',
      pds: `http://localhost:${port}`,
      signer: serviceKeypair,
    })
    console.log(`SERVER_DID=${serverDid}`)

    const server = bsky.BskyAppView.create({
      db,
      redis: redisCache,
      config,
      algos: cfg.algos,
      imgInvalidator: cfg.imgInvalidator,
      signingKey: serviceKeypair,
    })

# https://web.plc.directory/api/redoc#operation/ResolveDid
url=https://plc.directory/did:plc:pyc2ihzpelxtg4cdkfzbhcv4
json='{ "type": "create", "signingKey": "did:key:zQ3shP5TBe1sQfSttXty15FAEHV1DZgcxRZNxvEWnPfLFwLxJ", "recoveryKey": "did:key:zQ3shhCGUqDKjStzuDxPkTxN6ujddP4RkEKJJouJGRRkaLGbg", "handle": "first-post.bsky.social", "service": "https://bsky.social", "prev": null, "sig": "yvN4nQYWTZTDl9nKSSyC5EC3nsF5g4S56OmRg9G6_-pM6FCItV2U2u14riiMGyHiCD86l6O-1xC5MPwf8vVsRw" }'
curl -X POST -H "Content-Type: application/json" -d "$json" $url | jq .

pds : invitecode

host=example.com
admin_password="admin-pass"
url=https://$host/xrpc/com.atproto.server.createInviteCode
json="{\"useCount\":1}"
curl -X POST -u admin:${admin_password} -H "Content-Type: application/json" -d "$json" -sL $url | jq .

bsky(appview) latest

v0.4.24

$ curl -sL ${pds}/xrpc/_health
{"version":"0.4.24"}

compose.yml

  bsky:
    ports:
      - 2584:3000
    build:
      context: ./repos/atproto/
      dockerfile: services/bsky/Dockerfile
    restart: always
    env_file:
      - ./envs/appview
    volumes:
      - ./data/appview/cache/:/cache/
    command: node --enable-source-maps api.js
    depends_on:
      pds:
        condition: service_healthy
      db:
        condition: service_healthy
      redis:
        condition: service_healthy

#  bsky:
#    ports:
#      - 2584:3000
#    build:
#      context: ./repos/atproto/
#      dockerfile: services/bsky/Dockerfile
#    env_file:
#      - ./envs/appview
#    volumes:
#      - ./data/appview/cache/:/cache/
#    depends_on:
#      db:
#        condition: service_healthy
#      redis:
#        condition: service_healthy
#
#  bsky-daemon:
#    command: node --enable-source-maps daemon.js
#  bsky-indexer:
#    command: node --enable-source-maps indexer.js
#  bsky-ingester:
#    command: node --enable-source-maps ingester.js

# caddy and ssl3
BSKY_REPO_PROVIDER=ws://pds:3000
BSKY_REPO_PROVIDER=ws://${host}

BSKY_BSYNC_HTTP_VERSION=1.1
BSKY_BSYNC_PORT=3002
BSKY_BSYNC_URL=http://bsky:3002
BSKY_COURIER_URL=http://fake-courier.example.invalid/
BSKY_DATAPLANE_HTTP_VERSION=1.1
BSKY_DATAPLANE_PORT=3001
BSKY_DATAPLANE_URLS=http://bsky:3001
BSKY_DB_POSTGRES_SCHEMA=bsky
BSKY_DB_POSTGRES_URL=postgres://postgres:postgres@db
BSKY_DID_PLC_URL=https://plc.${host}
BSKY_PUBLIC_URL=https://api.${host}
BSKY_SEARCH_URL=https://palomar.${host}
BSKY_SERVER_DID=did:web:api.${host}
BSKY_SERVICE_SIGNING_KEY=xxx
DEBUG_MODE=1
DID_PLC_URL=https://plc.${host}
ENABLE_MIGRATIONS=false
LOG_DESTINATION=1
LOG_ENABLED=true
MOD_SERVICE_DID=did:web:mod.${host}
NODE_ENV=development
REDIS_HOST=redis
#NODE_TLS_REJECT_UNAUTHORIZED=${NODE_TLS_REJECT_UNAUTHORIZED}
#GOINSECURE=${GOINSECURE}
#BSKY_LABELS_FROM_ISSUER_DIDS=${BSKY_LABELS_FROM_ISSUER_DIDS}
#PORT=2584

BSKY_DB_POSTGRES_SCHEMA

postgres://postgres:postgres@db/${BSKY_DB_POSTGRES_SCHEMA}

BSKY_BSYNC_URL=http://bsky:3002

# compose.yml
  bsky:
    ports:
      - 2584:3000

special thanks

thx ikuradon -> https://github.com/ikuradon/atproto-starter-kit
thx itaru2622 -> https://github.com/itaru2622/bluesky-selfhost-env

Answered by bnewbold

Jan 8, 2024

There is an out-of-date docker-compose file and README in the indigo repo that might be helpful: https://github.com/bluesky-social/indigo/tree/main/cmd/fakermaker

Things are not in a great state for this right now. We plan to iterate on all the docs and dev support around self-hosting at the same time we open up federation, which means we aren't prioritizing updating docs on this until then.

In particular, I think the pds service is probably pretty out of date compared to the sandbox PDS "distribution", and we need to get those synced back up, and that may involve breaking changes to the PDS config env vars. Also the version of the PDS that we (bluesky) runs is now behind a "gateway servi…

View full answer

bnewbold · 2024-01-08T16:33:29Z

bnewbold
Jan 8, 2024
Maintainer

There is an out-of-date docker-compose file and README in the indigo repo that might be helpful: https://github.com/bluesky-social/indigo/tree/main/cmd/fakermaker

Things are not in a great state for this right now. We plan to iterate on all the docs and dev support around self-hosting at the same time we open up federation, which means we aren't prioritizing updating docs on this until then.

In particular, I think the pds service is probably pretty out of date compared to the sandbox PDS "distribution", and we need to get those synced back up, and that may involve breaking changes to the PDS config env vars. Also the version of the PDS that we (bluesky) runs is now behind a "gateway service", which is unnecessary complexity for most folks (but needs documentation). The relay (BGS) and PLC are pretty stable. The appview is in the middle of several large refactors, with big changes in feature branches which have not been merged yet.

0 replies

itaru2622 · 2024-02-07T02:13:50Z

itaru2622
Feb 7, 2024

@syui did you succeed on self hosting with current codes? please update if you got progress.

4 replies

syui Feb 7, 2024
Author

yes, my account https://web.syu.is/profile/syui.syu.is

itaru2622 Feb 7, 2024

could you share how you configure those? I also want to get self-hosting environment but failed configuring palomar (search service in indigo/cmd)

syui Feb 7, 2024
Author

I have compiled information for self-hosting here.
We recommend sandbox first.
https://github.com/bluesky-social/pds

itaru2622 Apr 3, 2024

hmm, finally I got my self-hosting environment for latest bluesky including bgs, plc, appview, pds, and others.
my codes are located https://github.com/itaru2622/bluesky-selfhost-env and self-hosting domain name is easy tune-able by environment variable. enjoy ;-)

syui · 2024-05-22T11:24:04Z

syui
May 22, 2024
Author

if bsky(appview) doesn't work

0 replies

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

atproto self-host in docker #2026

{{title}}

{{editor}}'s edit

{{editor}}'s edit

Replies: 3 comments 4 replies

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{editor}}'s edit

{{editor}}'s edit

Select a reply

atproto self-host in docker #2026

syui Jan 6, 2024

version

docker compose

other server

hint

postgres

env

appview : SERVICE_SIGNING_KEY/SERVER_DID

pds : invitecode

bsky(appview) latest

special thanks

Replies: 3 comments · 4 replies

bnewbold Jan 8, 2024 Maintainer

itaru2622 Feb 7, 2024

syui Feb 7, 2024 Author

itaru2622 Feb 7, 2024

syui Feb 7, 2024 Author

itaru2622 Apr 3, 2024

syui May 22, 2024 Author

syui
Jan 6, 2024

Replies: 3 comments 4 replies

bnewbold
Jan 8, 2024
Maintainer

itaru2622
Feb 7, 2024

syui Feb 7, 2024
Author

syui Feb 7, 2024
Author

syui
May 22, 2024
Author