Skip to content

Commit

Permalink
chore: Handle migrations with least permissions necessary (#630)
Browse files Browse the repository at this point in the history
* chore: use standard 'postgres' user for API and UI migrations
* chore: consolidate API and UI migration Dockerfiles into a single source
* chore: add migration component to supabase pacakge
    * The supabase package will be responsible for running migrations that require elevated permissions
  • Loading branch information
YrrepNoj committed Jun 18, 2024
1 parent eb65676 commit 37badae
Show file tree
Hide file tree
Showing 21 changed files with 363 additions and 259 deletions.
6 changes: 4 additions & 2 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ jobs:
- name: Build and Publish API
run: |
docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${{ steps.get_version.outputs.version-without-v }} --push packages/api
docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${{ steps.get_version.outputs.version-without-v }} --push packages/api/supabase
docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${{ steps.get_version.outputs.version-without-v }} --push -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/api/supabase/migrations" .
zarf package create packages/api --set=LEAPFROGAI_IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm
zarf package create packages/api --set=LEAPFROGAI_IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm
Expand All @@ -63,7 +63,7 @@ jobs:
- name: Build and Publish UI
run: |
docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-ui:${{ steps.get_version.outputs.version-without-v }} --push src/leapfrogai_ui
docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${{ steps.get_version.outputs.version-without-v }} --push src/leapfrogai_ui/supabase
docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${{ steps.get_version.outputs.version-without-v }} --push -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=src/leapfrogai_ui/supabase/migrations" .
zarf package create packages/ui --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm
zarf package create packages/ui --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm
Expand All @@ -76,6 +76,8 @@ jobs:
- name: Build and Publish Supabase
run: |
docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/supabase-migrations:${{ steps.get_version.outputs.version-without-v }} --push -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/supabase/migrations" .
zarf package create packages/supabase --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm
zarf package create packages/supabase --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm
Expand Down
5 changes: 4 additions & 1 deletion packages/api/supabase/Dockerfile → Dockerfile.migrations
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
FROM --platform=$TARGETPLATFORM cgr.dev/chainguard/bash:latest
ARG TARGETPLATFORM
ARG version="1.169.8"
ARG MIGRATIONS_DIR

RUN test -n "$MIGRATIONS_DIR" || (echo "MIGRATIONS_PATH is required" && false)

# Download the supabase cli
RUN mkdir -p /usr/local/bin
Expand All @@ -13,4 +16,4 @@ RUN ARCH=$(echo $TARGETPLATFORM | cut -d '/' -f2) \
USER 65532:65532

# Download the migration scripts
COPY --chown=65532:65532 migrations/*.sql supabase/migrations/
COPY --chown=65532:65532 ${MIGRATIONS_DIR}/*.sql supabase/migrations/
13 changes: 9 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,14 @@ sdk-wheel: ## build wheels for the leapfrogai_sdk package as a dependency for ot
-rm ${SDK_DEST}/*.whl
python -m pip wheel src/leapfrogai_sdk -w ${SDK_DEST}

build-supabase:
build-supabase: local-registry
## Build the migration container for this version of the supabase package
docker build -t ghcr.io/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/supabase/migrations" .
docker tag ghcr.io/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION}
docker push localhost:5000/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION}

## Build the Zarf package
uds zarf package create packages/supabase -o packages/supabase --set IMAGE_VERSION=${LOCAL_VERSION} --confirm
uds zarf package create packages/supabase -o packages/supabase --registry-override=ghcr.io=localhost:5000 --set IMAGE_VERSION=${LOCAL_VERSION} --confirm

setup-api-deps: sdk-wheel ## Download the wheels for the leapfrogai_api dependencies
-rm packages/api/build/*.whl
Expand All @@ -51,7 +56,7 @@ build-api: local-registry setup-api-deps ## Build the leapfrogai_api container a
docker tag ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION}

## Build the migration container for this version of the API
docker build -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} packages/api/supabase
docker build -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/api/supabase/migrations" .
docker tag ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION}

## Push the images to the local registry (Zarf is super slow if the image is only in the local daemon)
Expand All @@ -68,7 +73,7 @@ build-ui: local-registry ## Build the leapfrogai_ui container and Zarf package
docker tag ghcr.io/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION}

## Build the migration container for the version of the UI
docker build -t ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} src/leapfrogai_ui/supabase
docker build -t ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=src/leapfrogai_ui/supabase/migrations" .
docker tag ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION}

## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon)
Expand Down
4 changes: 2 additions & 2 deletions packages/api/chart/templates/migration-job.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ spec:
args:
- -c
- >-
supabase migration fetch --db-url="postgresql://supabase_admin:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT/postgres" --debug || true &&
supabase db push --db-url="postgresql://supabase_admin:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT/postgres" --include-all --debug
supabase migration fetch --db-url="postgresql://postgres:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT/postgres" --debug || true &&
supabase db push --db-url="postgresql://postgres:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT/postgres" --include-all --debug
securityContext:
runAsUser: {{ .Values.image.securityContext.runAsUser }}
runAsGroup: {{ .Values.image.securityContext.runAsGroup }}
Expand Down
17 changes: 0 additions & 17 deletions packages/api/supabase/migrations/20240419164109_assistant.sql

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
-- Create a table to store OpenAI Assistant Objects
create table
assistant_objects (
id uuid primary key DEFAULT uuid_generate_v4(),
created_at bigint default extract(epoch from now()) not null,
description varchar(512),
instructions text,
metadata jsonb,
model varchar(255) not null,
name varchar(255),
object text check (object in ('assistant')),
tools jsonb,
response_format jsonb,
temperature float,
tool_resources jsonb,
top_p float
);

-- Create a table to store the OpenAI Thread Objects
create table
thread_objects (
id uuid primary key DEFAULT uuid_generate_v4(),
user_id uuid references auth.users not null,
object text check (object in ('thread')),
created_at bigint default extract(epoch FROM NOW()) NOT NULL,
tool_resources jsonb,
metadata jsonb
);

-- Create a table to store the OpenAI Message Objects
create table
message_objects (
id uuid primary key DEFAULT uuid_generate_v4(),
user_id uuid references auth.users not null,
object text check (object in ('thread.message')),
created_at bigint default extract(epoch FROM NOW()) not null,
thread_id uuid references thread_objects (id) on delete cascade not null,
status text,
incomplete_details jsonb,
completed_at bigint,
incomplete_at bigint,
role text,
content jsonb,
assistant_id uuid, -- No foreign key constraint, can be null and doesn't have to refer to an assistant that exists
run_id uuid, -- No foreign key constraint, can be null and doesn't have to refer to a thread that exists
attachments jsonb,
metadata jsonb
);

-- RLS policies
alter table thread_objects enable row level security;
alter table message_objects enable row level security;

-- Policies for thread_objects
create policy "Individuals can view their own thread_objects." on thread_objects for
select using (auth.uid() = user_id);
create policy "Individuals can create thread_objects." on thread_objects for
insert with check (auth.uid() = user_id);
create policy "Individuals can update their own thread_objects." on thread_objects for
update using (auth.uid() = user_id);
create policy "Individuals can delete their own thread_objects." on thread_objects for
delete using (auth.uid() = user_id);

-- Policies for message_objects
create policy "Individuals can view their own message_objects." on message_objects for
select using (auth.uid() = user_id);
create policy "Individuals can create message_objects." on message_objects for
insert with check (auth.uid() = user_id);
create policy "Individuals can update their own message_objects." on message_objects for
update using (auth.uid() = user_id);
create policy "Individuals can delete their own message_objects." on message_objects for
delete using (auth.uid() = user_id);

-- Indexes for foreign keys for message_objects
CREATE INDEX message_objects_user_id ON message_objects (user_id);
CREATE INDEX message_objects_thread_id ON message_objects (thread_id);
CREATE INDEX message_objects_created_at ON thread_objects (created_at);

-- Indexes for common filtering and sorting for thread_objects
CREATE INDEX thread_objects_id ON thread_objects (id);
CREATE INDEX thread_objects_user_id ON thread_objects (user_id);
CREATE INDEX thread_objects_created_at ON thread_objects (created_at);

-- Add user_id column to assistant_objects and file_objects tables
alter table assistant_objects
add column user_id uuid references auth.users not null;
alter table file_objects
add column user_id uuid references auth.users not null;
-- Set buckets to private
update storage.buckets
set public = false
where id = 'file_bucket';

-- RLS policies
alter table assistant_objects enable row level security;
alter table file_objects enable row level security;

-- Policies for assistant_objects
create policy "Individuals can view their own assistant_objects. " on assistant_objects for
select using (auth.uid() = user_id);
create policy "Individuals can create assistant_objects." on assistant_objects for
insert with check (auth.uid() = user_id);
create policy "Individuals can update their own assistant_objects." on assistant_objects for
update using (auth.uid() = user_id);
create policy "Individuals can delete their own assistant_objects." on assistant_objects for
delete using (auth.uid() = user_id);

-- Policies for file_objects
create policy "Individuals can view their own file_objects." on file_objects for
select using (auth.uid() = user_id);
create policy "Individuals can create file_objects." on file_objects for
insert with check (auth.uid() = user_id);
create policy "Individuals can update their own file_objects." on file_objects for
update using (auth.uid() = user_id);
create policy "Individuals can delete their own file_objects." on file_objects for
delete using (auth.uid() = user_id);

-- Policies for file_bucket
create policy "Any authenticated individual can add files to file_bucket."
on storage.objects for
insert to authenticated with check (bucket_id = 'file_bucket');
create policy "Individuals can view their own files in the file_bucket."
on storage.objects for
select using (bucket_id = 'file_bucket' AND auth.uid() = owner);
create policy "Individuals can delete their own files."
on storage.objects for
delete using (bucket_id = 'file_bucket' AND auth.uid() = owner);
create policy "Individuals can update their own files in file_bucket."
on storage.objects for
update using (auth.uid() = owner) with check (bucket_id = 'file_bucket');

ALTER TABLE file_objects ALTER COLUMN created_at SET DEFAULT extract(epoch from now());
ALTER TABLE file_objects ALTER COLUMN created_at SET NOT NULL;
ALTER TABLE file_objects ADD CHECK (object in ('file'));

-- Create a table to store the OpenAI Run Objects
create table
run_objects (
id uuid primary key default uuid_generate_v4(),
user_id uuid references auth.users not null,
object text check (object in ('thread.run')),
created_at bigint default extract(epoch FROM NOW()) not null,
thread_id uuid references thread_objects (id) on delete cascade not null,
assistant_id uuid references assistant_objects (id) on delete cascade not null,
status text,
required_action jsonb,
last_error jsonb,
expires_at bigint,
started_at bigint,
cancelled_at bigint,
failed_at bigint,
completed_at bigint,
model text,
instructions text,
tools jsonb,
metadata jsonb,
parallel_tool_calls boolean,
stream boolean,
file_ids uuid[],
incomplete_details jsonb,
usage jsonb,
temperature float,
top_p float,
max_prompt_tokens int,
max_completion_tokens int,
truncation_strategy jsonb,
tool_choice jsonb,
response_format jsonb
);

-- RLS policies
alter table run_objects enable row level security;

-- Policies for run_objects
create policy "Individuals can view their own run_objects." on run_objects for
select using (auth.uid() = user_id);
create policy "Individuals can create run_objects." on run_objects for
insert with check (auth.uid() = user_id);
create policy "Individuals can update their own run_objects." on run_objects for
update using (auth.uid() = user_id);
create policy "Individuals can delete their own run_objects." on run_objects for
delete using (auth.uid() = user_id);

-- Indexes for common filtering and sorting for run_objects
CREATE INDEX run_objects_id ON run_objects (id);
CREATE INDEX run_objects_user_id ON run_objects (user_id);
CREATE INDEX run_objects_created_at ON run_objects (created_at);
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,38 @@ begin
limit match_limit;
end;
$$;

-- RLS policies
alter table vector_store enable row level security;
alter table vector_store_file enable row level security;
alter table vector_content enable row level security;

-- Policies for vector_store
create policy "Individuals can view their own vector_store." on vector_store for
select using (auth.uid() = user_id);
create policy "Individuals can create vector_store." on vector_store for
insert with check (auth.uid() = user_id);
create policy "Individuals can update their own vector_store." on vector_store for
update using (auth.uid() = user_id);
create policy "Individuals can delete their own vector_store." on vector_store for
delete using (auth.uid() = user_id);

-- Policies for vector_store_file
create policy "Individuals can view their own vector_store_file." on vector_store_file for
select using (auth.uid() = user_id);
create policy "Individuals can create vector_store_file." on vector_store_file for
insert with check (auth.uid() = user_id);
create policy "Individuals can update their own vector_store_file." on vector_store_file for
update using (auth.uid() = user_id);
create policy "Individuals can delete their own vector_store_file." on vector_store_file for
delete using (auth.uid() = user_id);

-- Policies for vector_content
create policy "Individuals can view their own vector_content." on vector_content for
select using (auth.uid() = user_id);
create policy "Individuals can create vector_content." on vector_content for
insert with check (auth.uid() = user_id);
create policy "Individuals can update their own vector_content." on vector_content for
update using (auth.uid() = user_id);
create policy "Individuals can delete their own vector_content." on vector_content for
delete using (auth.uid() = user_id);
47 changes: 0 additions & 47 deletions packages/api/supabase/migrations/20240516152530_enable_rls.sql

This file was deleted.

Loading

0 comments on commit 37badae

Please sign in to comment.