Permalink
Browse files

Merge pull request #115 from aquametalabs/feature/version-control-met…

…a-entities

Feature/version control meta entities
  • Loading branch information...
erichanson committed Feb 9, 2019
2 parents c735632 + fa961dc commit a1aad8b56a7a3516567de1b2ec3f173e9f508162
Showing with 757 additions and 104 deletions.
  1. +1 −1 .gitignore
  2. 0 bundles-available/org.aquameta.core.docs/ignored_row.csv
  3. 0 bundles-available/org.aquameta.core.endpoint/ignored_row.csv
  4. 0 bundles-available/org.aquameta.core.ide/ignored_row.csv
  5. +0 −1 bundles-available/org.aquameta.core.mimetypes/ignored_row.csv
  6. 0 bundles-available/org.aquameta.core.semantics/ignored_row.csv
  7. 0 bundles-available/org.aquameta.core.widget/ignored_row.csv
  8. 0 bundles-available/org.aquameta.games.snake/ignored_row.csv
  9. 0 bundles-available/org.aquameta.templates.simple/ignored_row.csv
  10. 0 bundles-available/org.aquameta.ui.admin/ignored_row.csv
  11. 0 bundles-available/org.aquameta.ui.auth/ignored_row.csv
  12. +0 −31 bundles-available/org.aquameta.ui.bundle/ignored_row.csv
  13. 0 bundles-available/org.aquameta.ui.dev/ignored_row.csv
  14. 0 bundles-available/org.aquameta.ui.event/ignored_row.csv
  15. 0 bundles-available/org.aquameta.ui.fsm/ignored_row.csv
  16. 0 bundles-available/org.aquameta.ui.layout/ignored_row.csv
  17. 0 bundles-available/org.aquameta.ui.tags/ignored_row.csv
  18. +176 −0 extension-to-bundle.sql
  19. +2 −0 install.sh
  20. +57 −33 src/pg-extension/bundle/000-data_model.sql
  21. +28 −2 src/pg-extension/bundle/001-functions.sql
  22. +0 −5 src/pg-extension/bundle/002-utils.sql
  23. +11 −0 src/pg-extension/bundle/004-ignored.sql
  24. +31 −0 src/pg-extension/bundle/005-tracked-nontable-relation.sql
  25. +11 −9 src/pg-extension/endpoint/001-server.sql
  26. +10 −0 src/pg-extension/meta/000-identifiers.sql
  27. +172 −21 src/pg-extension/meta/001-catalog.sql
  28. +29 −0 src/pg-extension/pg_catalog_get_defs/LICENSE
  29. +11 −0 src/pg-extension/pg_catalog_get_defs/Makefile
  30. +2 −0 src/pg-extension/pg_catalog_get_defs/README.md
  31. +2 −0 src/pg-extension/pg_catalog_get_defs/pg_catalog_get_defs.control
  32. +213 −0 src/pg-extension/pg_catalog_get_defs/pg_get_typedef.sql
  33. +1 −1 src/privileges/002-user.sql
@@ -1,7 +1,6 @@
*.swp
*.swo
*.pyc

src/pg-extension/bundle/bundle--*.*.*.sql
src/pg-extension/email/email--*.*.*.sql
src/pg-extension/endpoint/endpoint--*.*.*.sql
@@ -10,3 +9,4 @@ src/pg-extension/filesystem/filesystem--*.*.*.sql
src/pg-extension/meta/meta--*.*.*.sql
src/pg-extension/widget/widget--*.*.*.sql
src/pg-extension/semantics/semantics--*.*.*.sql
src/pg-extension/pg_catalog_get_defs/pg_catalog_get_defs--*.*.*.sql

This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.

Large diffs are not rendered by default.

Oops, something went wrong.
@@ -107,6 +107,7 @@ sudo -H pip install --upgrade .

# install extensions into PostgreSQL's extensions/ directory
echo "Building core PostgreSQL extensions..."
cd $SRC/src/pg-extension/pg_catalog_get_defs && make && make install
cd $SRC/src/pg-extension/meta && make && make install
cd $SRC/src/pg-extension/bundle && make && make install
cd $SRC/src/pg-extension/filesystem && make && make install
@@ -143,6 +144,7 @@ sudo -u postgres psql -c "create extension if not exists plv8" aquameta

# create aquameta core extensions
echo "Installing core Aquameta extensions..."
sudo -u postgres psql -c "create extension pg_catalog_get_defs schema pg_catalog" aquameta
sudo -u postgres psql -c "create extension meta" aquameta
sudo -u postgres psql -c "create extension bundle" aquameta
sudo -u postgres psql -c "create extension filesystem" aquameta
@@ -191,30 +191,23 @@ from bundle.bundle bundle
------------------------------------------------------------------------------
create table ignored_row (
id uuid default public.uuid_generate_v4() primary key,
bundle_id uuid not null references bundle(id) on delete cascade,
row_id meta.row_id,
unique (row_id) --TOO RESTRICTIVE?
row_id meta.row_id
);

-- ignored_schema: Ignored rows that are in meta.schema: If a schema's meta row
-- is ignored, the ignore cascades down to every row in that schema, effectively
-- ignoring everything in it.
create view ignored_schema as
select (row_id).pk_value::meta.schema_id as schema_id, row_id as meta_row_id
from bundle.ignored_row
where
(row_id::meta.schema_id).name = 'meta' and
(row_id::meta.relation_id).name = 'schema';
create table ignored_schema (
id uuid default public.uuid_generate_v4() primary key,
schema_id meta.schema_id not null
);

-- ignored_relation: Same as ignored_schema but for relation.
create view ignored_relation as
select (row_id).pk_value::meta.relation_id as relation_id, row_id as meta_row_id
from bundle.ignored_row
where
(row_id::meta.schema_id).name = 'meta' and
(row_id::meta.relation_id).name = 'relation';
create table ignored_relation (
id uuid default public.uuid_generate_v4() primary key,
relation_id meta.relation_id not null
);

-- TODO: ignored_column support...
create table ignored_column (
id uuid default public.uuid_generate_v4() primary key,
column_id meta.column_id not null
);


------------------------------------------------------------------------------
@@ -395,9 +388,9 @@ select
)
)::text as value

from bundle.stage_row sr
from bundle.stage_row sr -- this should be trackable_row or something
join meta.relation re on sr.row_id::meta.relation_id = re.id
join meta.column c on c.relation_id=re.id
join meta.relation_column c on c.relation_id=re.id
where sr.new_row=true

union all
@@ -546,25 +539,30 @@ where change_type != 'same'
-- stage_row_added [or stage_row_deleted?].
------------------------------------------------------------------------------

create table trackable_nontable_relation (
id uuid default public.uuid_generate_v4() primary key,
pk_column_id meta.column_id not null
);


-- Relations that are not specifically ignored, and not in a ignored schema
create or replace view not_ignored_relation as
-- TODO: why does this have schema_id and pk_column_id? should just be a realtion_id no?
create or replace view trackable_relation as
select relation_id, schema_id, primary_key_column_id from (
-- every single table
select t.id as relation_id, s.id as schema_id, r.primary_key_column_ids[1] as primary_key_column_id --TODO audit column
from meta.schema s
join meta.table t on t.schema_id=s.id
join meta.relation r on r.id=t.id
join meta.relation r on r.id=t.id -- only work with relations that have a primary key
where primary_key_column_ids[1] is not null

-- combined with every view in the meta schema
UNION
select v.id as relation_id, v.schema_id, meta.column_id('meta',v.name,'id') as primary_key_column_id
from meta.view v
where v.schema_name = 'meta'
select pk_column_id::meta.relation_id as relation_id, pk_column_id::meta.schema_id as schema_id, pk_column_id as primary_key_column_id
from bundle.trackable_nontable_relation
) r

-- ...that is not ignored
-- ...that is not ignored
where relation_id not in (
select relation_id from bundle.ignored_relation
)
@@ -580,19 +578,45 @@ ignored by schema- or relation-ignores. NOTE: We haven't pulled out
specifically ignored rows yet.
*/

create view not_ignored_row_stmt as
create or replace view bundle.not_ignored_row_stmt as
select *, 'select meta.row_id(' ||
quote_literal((r.schema_id).name) || ', ' ||
quote_literal((r.relation_id).name) || ', ' ||
quote_literal((r.primary_key_column_id).name) || ', ' ||
quote_ident((r.primary_key_column_id).name) || '::text ' ||
') as row_id from ' ||
quote_ident((r.schema_id).name) || '.' || quote_ident((r.relation_id).name) as stmt
from bundle.not_ignored_relation r;
-- join
quote_ident((r.schema_id).name) || '.' || quote_ident((r.relation_id).name) ||
-- special case meta rows so that ignored_* cascades down to all objects in it's scope
case
-- schemas
when (r.schema_id).name = 'meta' and ((r.relation_id).name) = 'schema' then
' where id not in (select schema_id from bundle.ignored_schema)'
-- relations
when (r.schema_id).name = 'meta' and ((r.relation_id).name) in ('table', 'view', 'relation') then
' where id not in (select relation_id from bundle.ignored_relation) and schema_id not in (select schema_id from bundle.ignored_schema)'
-- functions
when (r.schema_id).name = 'meta' and ((r.relation_id).name) = 'function_definition' then
' where id::meta.schema_id not in (select schema_id from bundle.ignored_schema)'
-- columns
when (r.schema_id).name = 'meta' and ((r.relation_id).name) = 'column' then
' where id not in (select column_id from bundle.ignored_column) and id::meta.relation_id not in (select relation_id from bundle.ignored_relation) and id::meta.schema_id not in (select schema_id from bundle.ignored_schema)'
-- objects that exist in schema scope
when (r.schema_id).name = 'meta' and ((r.relation_id).name) in ('operator') then
' where meta.schema_id(schema_name) not in (select schema_id from bundle.ignored_schema)'
-- objects that exist in schema scope
when (r.schema_id).name = 'meta' and ((r.relation_id).name) in ('type_definition') then
' where id::meta.schema_id not in (select schema_id from bundle.ignored_schema)'
-- objects that exist in table scope
when (r.schema_id).name = 'meta' and ((r.relation_id).name) in ('constraint_check','constraint_unique','table_privilege') then
' where meta.schema_id(schema_name) not in (select schema_id from bundle.ignored_schema) and table_id not in (select relation_id from bundle.ignored_relation)'
else ''
end
as stmt
from bundle.trackable_relation r;


create or replace view untracked_row as
select r.row_id, r.row_id::meta.relation_id as relation_id
select r.row_id /*, r.row_id::meta.relation_id as relation_id */
from bundle.exec((
select array_agg (stmt) from bundle.not_ignored_row_stmt
)) r(
@@ -24,6 +24,8 @@ create or replace function commit (bundle_name text, message text) returns void
new_commit_id uuid;
begin

raise notice 'bundle: Committing to %', bundle_name;

select id
into _bundle_id
from bundle.bundle
@@ -34,18 +36,21 @@ create or replace function commit (bundle_name text, message text) returns void
returning id into new_rowset_id;

-- STAGE
raise notice 'bundle: Committing rowset_rows...';
-- ROWS: copy everything in stage_row to the new rowset
insert into bundle.rowset_row (rowset_id, row_id)
select new_rowset_id, row_id from bundle.stage_row where bundle_id=_bundle_id;


raise notice 'bundle: Committing blobs...';
-- FIELDS: copy all the fields in stage_row_field to the new rowset's fields
insert into bundle.blob (value)
select f.value
from bundle.rowset_row rr
join bundle.rowset r on r.id=new_rowset_id and rr.rowset_id=r.id
join bundle.stage_row_field f on (f.field_id).row_id = rr.row_id;

raise notice 'bundle: Committing stage_row_fields...';
-- FIELDS: copy all the fields in stage_row_field to the new rowset's fields
insert into bundle.rowset_row_field (rowset_row_id, field_id, value_hash)
select rr.id, f.field_id, public.digest(value, 'sha256')
@@ -62,14 +67,17 @@ create or replace function commit (bundle_name text, message text) returns void
join bundle.blob b on (f.field_id).row_id = rr.row_id;
*/

raise notice 'bundle: Creating the commit...';
-- create the commit
insert into bundle.commit (bundle_id, parent_id, rowset_id, message)
values (_bundle_id, (select head_commit_id from bundle.bundle b where b.id=_bundle_id), new_rowset_id, message)
returning id into new_commit_id;

raise notice 'bundle: Updating bundle.head_commit_id...';
-- point HEAD at new commit
update bundle.bundle bundle set head_commit_id=new_commit_id where bundle.id=_bundle_id;

raise notice 'bundle: Cleaning up after commit...';
-- clear the stage
delete from bundle.stage_row_added where bundle_id=_bundle_id;
delete from bundle.stage_row_deleted where bundle_id=_bundle_id;
@@ -482,10 +490,26 @@ $$ language plpgsql;
create or replace function checkout (in commit_id uuid) returns void as $$
declare
commit_row record;
bundle_name text;
commit_message text;
_commit_id uuid;
commit_role text;
commit_time timestamp;
begin
set local search_path=bundle,meta,public;

raise notice 'bundle: Checking out bundle %', commit_id;
select b.name, c.id, c.message, c.time, (c.role_id).name
into bundle_name, _commit_id, commit_message, commit_time, commit_role
from bundle.bundle b
join bundle.commit c on c.bundle_id = b.id
where c.id = commit_id;

if _commit_id is null then
raise exception 'bundle.checkout() commit with id % does not exist', commit_id;
end if;

raise notice 'bundle.checkout(): % / % @ % by %: "%"', bundle_name, commit_id, commit_time, commit_role, commit_message;
-- raise notice 'bundle: Checking out bundle %', commit_id;
-- insert the meta-rows in this commit to the database
for commit_row in
select
@@ -502,19 +526,21 @@ create or replace function checkout (in commit_id uuid) returns void as $$
join bundle.rowset_row rr on rr.rowset_id=r.id
join bundle.rowset_row_field f on f.rowset_row_id=rr.id
join bundle.blob b on f.value_hash=b.hash
join meta.column col on (f.field_id).column_id = col.id
join meta.relation_column col on (f.field_id).column_id = col.id
where c.id=commit_id
and (rr.row_id::meta.schema_id).name = 'meta'
group by rr.id
-- add meta rows first, in sensible order
order by
case
when row_id::meta.relation_id = meta.relation_id('meta','schema') then 0
when row_id::meta.relation_id = meta.relation_id('meta','type_definition') then 1
when row_id::meta.relation_id = meta.relation_id('meta','table') then 2
when row_id::meta.relation_id = meta.relation_id('meta','column') then 3
when row_id::meta.relation_id = meta.relation_id('meta','sequence') then 4
when row_id::meta.relation_id = meta.relation_id('meta','constraint_check') then 4
when row_id::meta.relation_id = meta.relation_id('meta','constraint_unique') then 4
when row_id::meta.relation_id = meta.relation_id('meta','function_definition') then 5
else 100
end asc /*,
case
@@ -46,10 +46,6 @@ begin
join bundle.rowset_row_field rrf on rrf.rowset_row_id=rr.id
join bundle.blob on rrf.value_hash=blob.hash
where b.name=%L order by blob.hash) to ''%s/blob.csv''', bundle_name, directory);

execute format('copy (select distinct ir.* from bundle.bundle b
join bundle.ignored_row ir on ir.bundle_id=b.id
where b.name=%L order by ir.id) to ''%s/ignored_row.csv''', bundle_name, directory);
end
$$;

@@ -67,7 +63,6 @@ begin
execute format('copy bundle.rowset_row from ''%s/rowset_row.csv''', directory);
execute format('copy bundle.blob from ''%s/blob.csv''', directory);
execute format('copy bundle.rowset_row_field from ''%s/rowset_row_field.csv''', directory);
execute format('copy bundle.ignored_row from ''%s/ignored_row.csv''', directory);
execute format('alter table bundle.bundle enable trigger all');
execute format('alter table bundle.commit enable trigger all');
end
@@ -0,0 +1,11 @@
insert into bundle.ignored_relation(relation_id) values (meta.relation_id('bundle','bundle'));
insert into bundle.ignored_relation(relation_id) values (meta.relation_id('bundle','commit'));
insert into bundle.ignored_relation(relation_id) values (meta.relation_id('bundle','rowset'));
insert into bundle.ignored_relation(relation_id) values (meta.relation_id('bundle','rowset_row'));
insert into bundle.ignored_relation(relation_id) values (meta.relation_id('bundle','rowset_row_field'));
insert into bundle.ignored_relation(relation_id) values (meta.relation_id('bundle','blob'));
insert into bundle.ignored_relation(relation_id) values (meta.relation_id('bundle','tracked_row_added'));

insert into bundle.ignored_schema(schema_id) values (meta.schema_id('pg_catalog'));
insert into bundle.ignored_schema(schema_id) values (meta.schema_id('public'));
insert into bundle.ignored_schema(schema_id) values (meta.schema_id('information_schema'));
Oops, something went wrong.

0 comments on commit a1aad8b

Please sign in to comment.