diff --git a/sql/00-create-user.sql b/sql/00-create-user.sql index d3a0a16eb..50d7988a8 100644 --- a/sql/00-create-user.sql +++ b/sql/00-create-user.sql @@ -1,5 +1,6 @@ CREATE extension ltree; CREATE DATABASE blobber_meta; + \connect blobber_meta; CREATE USER blobber_user WITH ENCRYPTED PASSWORD 'blobber'; GRANT ALL PRIVILEGES ON DATABASE blobber_meta TO blobber_user; \ No newline at end of file diff --git a/sql/01-create-table.sql b/sql/01-create-table.sql index b8426aad1..4f8fdc787 100644 --- a/sql/01-create-table.sql +++ b/sql/01-create-table.sql @@ -11,16 +11,23 @@ $$ language 'plpgsql'; CREATE TABLE allocations( id VARCHAR (64) PRIMARY KEY, + tx VARCHAR (64) NOT NULL, size BIGINT NOT NULL DEFAULT 0, used_size BIGINT NOT NULL DEFAULT 0, owner_id VARCHAR(64) NOT NULL, - owner_public_key VARCHAR(256) NOT NULL, + payer_id VARCHAR(64) NOT NULL, + repairer_id VARCHAR(64) NOT NULL, + owner_public_key VARCHAR(512) NOT NULL, expiration_date BIGINT NOT NULL, allocation_root VARCHAR(255) NOT NULL DEFAULT '', blobber_size BIGINT NOT NULL DEFAULT 0, blobber_size_used BIGINT NOT NULL DEFAULT 0, latest_redeemed_write_marker VARCHAR(255), is_redeem_required BOOLEAN, + is_immutable BOOLEAN NOT NULL, + cleaned_up BOOLEAN NOT NULL DEFAULT FALSE, + finalized BOOLEAN NOT NULL DEFAULT FALSE, + time_unit BIGINT NOT NULL DEFAULT 172800000000000, created_at TIMESTAMP NOT NULL DEFAULT NOW(), updated_at TIMESTAMP NOT NULL DEFAULT NOW() ); @@ -52,6 +59,49 @@ CREATE TABLE allocation_changes( CREATE TRIGGER allocation_changes_modtime BEFORE UPDATE ON allocation_changes FOR EACH ROW EXECUTE PROCEDURE update_modified_column(); +CREATE TABLE terms ( + id bigserial, + blobber_id varchar(64) NOT NULL, + allocation_id varchar(64) REFERENCES allocations (id), + read_price bigint NOT NULL, + write_price bigint NOT NULL, + + PRIMARY KEY (id) +); + +-- clients' pending reads / writes +CREATE TABLE pendings ( + id bigserial, + client_id varchar(64) NOT NULL, + allocation_id varchar(64) NOT NULL, + blobber_id varchar(64) NOT NULL, + pending_write bigint NOT NULL DEFAULT 0, -- number of pending bytes + + PRIMARY KEY (id) +); + +CREATE TABLE read_pools ( + pool_id text NOT NULL, -- unique + client_id varchar(64) NOT NULL, + blobber_id varchar(64) NOT NULL, + allocation_id varchar(64) NOT NULL, + balance bigint NOT NULL, + expire_at bigint NOT NULL, + + PRIMARY KEY (pool_id) +); + +CREATE TABLE write_pools ( + pool_id text NOT NULL, -- unique + client_id varchar(64) NOT NULL, + blobber_id varchar(64) NOT NULL, + allocation_id varchar(64) NOT NULL, + balance bigint NOT NULL, + expire_at bigint NOT NULL, + + PRIMARY KEY (pool_id) +); + CREATE TABLE reference_objects ( id BIGSERIAL PRIMARY KEY, lookup_hash VARCHAR (64) NOT NULL, @@ -67,6 +117,7 @@ CREATE TABLE reference_objects ( custom_meta TEXT NOT NULL, content_hash VARCHAR(64) NOT NULL, size BIGINT NOT NULL DEFAULT 0, + chunk_size INT NOT NULL DEFAULT 65536, merkle_root VARCHAR(64) NOT NULL, actual_file_size BIGINT NOT NULL DEFAULT 0, actual_file_hash VARCHAR(64) NOT NULL, @@ -77,6 +128,8 @@ CREATE TABLE reference_objects ( actual_thumbnail_size BIGINT NOT NULL DEFAULT 0, actual_thumbnail_hash VARCHAR(64) NOT NULL, encrypted_key TEXT, + attributes JSON DEFAULT '{}'::jsonb, + on_cloud BOOLEAN DEFAULT FALSE, created_at TIMESTAMP NOT NULL DEFAULT NOW(), updated_at TIMESTAMP NOT NULL DEFAULT NOW(), deleted_at TIMESTAMP @@ -98,7 +151,7 @@ CREATE TABLE write_markers ( redeem_retries INT NOT NULL DEFAULT 0, close_txn_id VARCHAR(64), connection_id VARCHAR(64) NOT NULL, - client_key VARCHAR(256) NOT NULL, + client_key VARCHAR(512) NOT NULL, sequence BIGSERIAL UNIQUE, created_at TIMESTAMP NOT NULL DEFAULT NOW(), updated_at TIMESTAMP NOT NULL DEFAULT NOW() @@ -108,12 +161,15 @@ CREATE TRIGGER write_markers_modtime BEFORE UPDATE ON write_markers FOR EACH ROW CREATE TABLE read_markers ( client_id VARCHAR(64) NOT NULL PRIMARY KEY, - client_public_key VARCHAR(256) NOT NULL, + client_public_key VARCHAR(512) NOT NULL, blobber_id VARCHAR(64) NOT NULL, allocation_id VARCHAR(64) NOT NULL, owner_id VARCHAR(64) NOT NULL, + payer_id VARCHAR(64) NOT NULL, + auth_ticket JSON, timestamp BIGINT NOT NULL, counter BIGINT NOT NULL DEFAULT 0, + suspend BIGINT NOT NULL DEFAULT -1, signature VARCHAR(256) NOT NULL, latest_redeemed_rm JSON, redeem_required boolean, @@ -163,3 +219,30 @@ CREATE TABLE file_stats ( ); CREATE TRIGGER file_stats_modtime BEFORE UPDATE ON file_stats FOR EACH ROW EXECUTE PROCEDURE update_modified_column(); + +CREATE TABLE commit_meta_txns ( + ref_id BIGSERIAL NOT NULL, + txn_id VARCHAR(64) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT NOW() +); + +CREATE TABLE collaborators ( + ref_id BIGSERIAL NOT NULL, + client_id VARCHAR(64) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT NOW() +); + +CREATE TABLE marketplace_share_info ( + id BIGSERIAL PRIMARY KEY, + owner_id VARCHAR(64) NOT NULL, + client_id VARCHAR(64) NOT NULL, + file_path_hash TEXT NOT NULL, + re_encryption_key TEXT NOT NULL, + client_encryption_public_key TEXT NOT NULL, + expiry_at TIMESTAMP NULL, + revoked BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP NOT NULL DEFAULT NOW() +); + +CREATE TRIGGER share_info_modtime BEFORE UPDATE ON marketplace_share_info FOR EACH ROW EXECUTE PROCEDURE update_modified_column(); diff --git a/sql/02-create-indexes.sql b/sql/02-create-indexes.sql new file mode 100644 index 000000000..daf2fecd5 --- /dev/null +++ b/sql/02-create-indexes.sql @@ -0,0 +1,35 @@ +\connect blobber_meta; +BEGIN; +DROP INDEX IF EXISTS idx_unique_allocations_tx; +CREATE UNIQUE INDEX idx_unique_allocations_tx ON allocations (tx); + +DROP INDEX IF EXISTS idx_pendings_cab; +CREATE UNIQUE INDEX idx_pendings_cab ON pendings (client_id, allocation_id, blobber_id); + +DROP INDEX IF EXISTS idx_read_pools_cab; +CREATE INDEX idx_read_pools_cab ON read_pools (client_id, allocation_id, blobber_id); + +DROP INDEX IF EXISTS idx_write_pools_cab; +CREATE INDEX idx_write_pools_cab ON write_pools (client_id, allocation_id, blobber_id); + +-- TODO one of path_idx and idx_reference_objects_for_path is redundant +-- Create index on path column; It cannot be Unique index because of soft delete by gorm +DROP INDEX IF EXISTS path_idx; +CREATE INDEX path_idx ON reference_objects (path); + +DROP INDEX IF EXISTS update_idx; +CREATE INDEX update_idx ON reference_objects (updated_at); + +DROP INDEX IF EXISTS idx_reference_objects_for_lookup_hash; +CREATE INDEX idx_reference_objects_for_lookup_hash ON reference_objects(allocation_id, lookup_hash); + +DROP INDEX IF EXISTS idx_reference_objects_for_path; +CREATE INDEX idx_reference_objects_for_path ON reference_objects(allocation_id, path); + +DROP INDEX IF EXISTS idx_marketplace_share_info_for_owner; +CREATE INDEX idx_marketplace_share_info_for_owner ON marketplace_share_info(owner_id, file_path_hash); + +DROP INDEX IF EXISTS idx_marketplace_share_info_for_client; +CREATE INDEX idx_marketplace_share_info_for_client ON marketplace_share_info(client_id, file_path_hash); + +COMMIT; \ No newline at end of file diff --git a/sql/03-add-allocation-prices.sql b/sql/03-add-allocation-prices.sql deleted file mode 100644 index 846abeacb..000000000 --- a/sql/03-add-allocation-prices.sql +++ /dev/null @@ -1,104 +0,0 @@ -\connect blobber_meta; - -ALTER TABLE allocations ADD COLUMN tx varchar (64) NOT NULL; - -CREATE UNIQUE INDEX idx_unique_allocations_tx ON allocations (tx); - -CREATE TABLE terms ( - id bigserial, - - blobber_id varchar(64) NOT NULL, - allocation_tx varchar(64) REFERENCES allocations (tx), - - read_price bigint NOT NULL, - write_price bigint NOT NULL, - - PRIMARY KEY (id) -); - --- clients' pending reads / writes -CREATE TABLE pendings ( - id bigserial, - - client_id varchar(64) NOT NULL, - allocation_id varchar(64) NOT NULL, - blobber_id varchar(64) NOT NULL, - - pending_read bigint NOT NULL DEFAULT 0, -- number of pending blocks - pending_write bigint NOT NULL DEFAULT 0, -- number of pending bytes - - PRIMARY KEY (id) -); - -CREATE UNIQUE INDEX idx_pendings_cab - ON pendings (client_id, allocation_id, blobber_id); - -CREATE TABLE read_pools ( - pool_id text NOT NULL, -- unique - - client_id varchar(64) NOT NULL, - blobber_id varchar(64) NOT NULL, - allocation_id varchar(64) NOT NULL, - - balance bigint NOT NULL, - expire_at bigint NOT NULL, - - PRIMARY KEY (pool_id) -); - -CREATE UNIQUE INDEX idx_read_pools_cab - ON read_pools (client_id, allocation_id, blobber_id); - -CREATE TABLE write_pools ( - pool_id text NOT NULL, -- unique - - client_id varchar(64) NOT NULL, - blobber_id varchar(64) NOT NULL, - allocation_id varchar(64) NOT NULL, - - balance bigint NOT NULL, - expire_at bigint NOT NULL, - - PRIMARY KEY (pool_id) -); - -CREATE UNIQUE INDEX idx_write_pools_cab - ON write_pools (client_id, allocation_id, blobber_id); - - -CREATE TABLE read_redeems ( - id bigserial, - - read_counter bigint NOT NULL, - value bigint NOT NULL, - - client_id varchar(64) NOT NULL, - blobber_id varchar(64) NOT NULL, - allocation_id varchar(64) NOT NULL, - - PRIMARY KEY (id) -); - -CREATE TABLE write_redeems ( - id bigserial, - - signature varchar(256) NOT NULL, -- write marker signature - - size bigint NOT NULL, - value bigint NOT NULL, - - client_id varchar(64) NOT NULL, - blobber_id varchar(64) NOT NULL, - allocation_id varchar(64) NOT NULL, - - PRIMARY KEY (id) -); - -CREATE INDEX idx_write_redeems_signature ON write_redeems (signature); - --- --- grant again --- - -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO blobber_user; -GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO blobber_user; diff --git a/sql/02-grant-priv.sql b/sql/03-grant-privileges.sql similarity index 100% rename from sql/02-grant-priv.sql rename to sql/03-grant-privileges.sql diff --git a/sql/04-add-on-cloud.sql b/sql/04-add-on-cloud.sql deleted file mode 100644 index 4ad182657..000000000 --- a/sql/04-add-on-cloud.sql +++ /dev/null @@ -1,3 +0,0 @@ -\connect blobber_meta; - -ALTER TABLE reference_objects ADD COLUMN on_cloud BOOLEAN DEFAULT FALSE; \ No newline at end of file diff --git a/sql/05-add-commit-meta-txns-table.sql b/sql/05-add-commit-meta-txns-table.sql deleted file mode 100644 index 9560bde5b..000000000 --- a/sql/05-add-commit-meta-txns-table.sql +++ /dev/null @@ -1,7 +0,0 @@ -\connect blobber_meta; - -CREATE TABLE commit_meta_txns ( - ref_id BIGSERIAL NOT NULL, - txn_id VARCHAR(64) NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT NOW() -); \ No newline at end of file diff --git a/sql/06-add-cleaned_up-column-to-allocations.sql b/sql/06-add-cleaned_up-column-to-allocations.sql deleted file mode 100644 index b6551a670..000000000 --- a/sql/06-add-cleaned_up-column-to-allocations.sql +++ /dev/null @@ -1,7 +0,0 @@ -\connect blobber_meta; - -ALTER TABLE allocations - ADD COLUMN cleaned_up boolean NOT NULL DEFAULT false; - -ALTER TABLE allocations - ADD COLUMN finalized boolean NOT NULL DEFAULT false; diff --git a/sql/07-terms-belongs-to-allocation-id.sql b/sql/07-terms-belongs-to-allocation-id.sql deleted file mode 100644 index 4080ae32b..000000000 --- a/sql/07-terms-belongs-to-allocation-id.sql +++ /dev/null @@ -1,30 +0,0 @@ -\connect blobber_meta; - -BEGIN; - - ALTER TABLE terms - ADD COLUMN allocation_id varchar(64) REFERENCES allocations (id); - - UPDATE terms AS t - SET allocation_id = a.id - FROM allocations AS a - WHERE t.allocation_tx = a.tx; - - ALTER TABLE terms DROP COLUMN allocation_tx; - -COMMIT; - -BEGIN; - -- drop unique index - DROP INDEX idx_read_pools_cab; - DROP INDEX idx_write_pools_cab; - - -- create non-unique - CREATE INDEX idx_read_pools_cab - ON read_pools (client_id, allocation_id, blobber_id); - CREATE INDEX idx_write_pools_cab - ON write_pools (client_id, allocation_id, blobber_id); -COMMIT; - --- for the commit_meta_txns -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO blobber_user; diff --git a/sql/08-add-payer-id-to-allocations.sql b/sql/08-add-payer-id-to-allocations.sql deleted file mode 100644 index 525a97c12..000000000 --- a/sql/08-add-payer-id-to-allocations.sql +++ /dev/null @@ -1,4 +0,0 @@ -\connect blobber_meta; - -ALTER TABLE allocations ADD COLUMN payer_id VARCHAR(64) NOT NULL; - diff --git a/sql/10-add-time_unit-column-to-allocations.sql b/sql/10-add-time_unit-column-to-allocations.sql deleted file mode 100644 index 49a4e6d37..000000000 --- a/sql/10-add-time_unit-column-to-allocations.sql +++ /dev/null @@ -1,11 +0,0 @@ --- --- Add column time_unit to allocations. Default is 48h. --- - --- pew-pew -\connect blobber_meta; - -BEGIN; - ALTER TABLE allocations - ADD COLUMN time_unit BIGINT NOT NULL DEFAULT 172800000000000; -COMMIT; diff --git a/sql/11-add-payer_id-and-auth_tiket-columns-to-read_markers.sql b/sql/11-add-payer_id-and-auth_tiket-columns-to-read_markers.sql deleted file mode 100644 index af7865ea8..000000000 --- a/sql/11-add-payer_id-and-auth_tiket-columns-to-read_markers.sql +++ /dev/null @@ -1,11 +0,0 @@ --- --- add payer_id and auth_tiket columns to read_markers table --- - --- pew-pew -\connect blobber_meta; - -BEGIN; - ALTER TABLE read_markers ADD COLUMN payer_id VARCHAR(64) NOT NULL; - ALTER TABLE read_markers ADD COLUMN auth_ticket JSON; -COMMIT; diff --git a/sql/12-add-attributes-column-to-reference_objects.sql b/sql/12-add-attributes-column-to-reference_objects.sql deleted file mode 100644 index 47db2e20c..000000000 --- a/sql/12-add-attributes-column-to-reference_objects.sql +++ /dev/null @@ -1,12 +0,0 @@ --- --- Add who_pays column to reference_objects table. --- - --- pew-pew -\connect blobber_meta; - --- in a transaction -BEGIN; - ALTER TABLE reference_objects - ADD COLUMN attributes JSON DEFAULT '{}'::jsonb; -COMMIT; diff --git a/sql/13-add-collaborators-table.sql b/sql/13-add-collaborators-table.sql deleted file mode 100644 index 9dec684fe..000000000 --- a/sql/13-add-collaborators-table.sql +++ /dev/null @@ -1,10 +0,0 @@ -\connect blobber_meta; - - -CREATE TABLE collaborators ( - ref_id BIGSERIAL NOT NULL, - client_id VARCHAR(64) NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT NOW() -); - -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO blobber_user; \ No newline at end of file diff --git a/sql/14-increase_owner_pubkey.sql b/sql/14-increase_owner_pubkey.sql deleted file mode 100644 index f6cd40520..000000000 --- a/sql/14-increase_owner_pubkey.sql +++ /dev/null @@ -1,16 +0,0 @@ --- --- Increase the char limit of owner_public_key from 256 to 512. --- - --- pew-pew -\connect blobber_meta; - --- in a transaction -BEGIN; - ALTER TABLE allocations - ALTER COLUMN owner_public_key TYPE varchar(512); - ALTER TABLE read_markers - ALTER COLUMN client_public_key TYPE varchar(512); - ALTER TABLE write_markers - ALTER COLUMN client_key TYPE varchar(512); -COMMIT; diff --git a/sql/15-add-allocation-columns.sql b/sql/15-add-allocation-columns.sql deleted file mode 100644 index c983cf73f..000000000 --- a/sql/15-add-allocation-columns.sql +++ /dev/null @@ -1,6 +0,0 @@ -\connect blobber_meta; - -BEGIN; - ALTER TABLE allocations ADD COLUMN repairer_id VARCHAR(64) NOT NULL; - ALTER TABLE allocations ADD COLUMN is_immutable BOOLEAN NOT NULL; -COMMIT; \ No newline at end of file diff --git a/sql/15-add-chunk_size-to-reference_objects.sql b/sql/15-add-chunk_size-to-reference_objects.sql deleted file mode 100644 index a2530e3f0..000000000 --- a/sql/15-add-chunk_size-to-reference_objects.sql +++ /dev/null @@ -1,9 +0,0 @@ --- --- Add chunk_size column to reference_objects table. --- - --- pew-pew -\connect blobber_meta; - - -ALTER TABLE reference_objects ADD COLUMN chunk_size INT NOT NULL DEFAULT 65536; diff --git a/sql/16-add-marketplace-table.sql b/sql/16-add-marketplace-table.sql deleted file mode 100644 index bbfd4b043..000000000 --- a/sql/16-add-marketplace-table.sql +++ /dev/null @@ -1,23 +0,0 @@ -\connect blobber_meta; - - -CREATE TABLE marketplace_share_info ( - id BIGSERIAL PRIMARY KEY, - owner_id VARCHAR(64) NOT NULL, - client_id VARCHAR(64) NOT NULL, - file_path_hash TEXT NOT NULL, - re_encryption_key TEXT NOT NULL, - client_encryption_public_key TEXT NOT NULL, - expiry_at TIMESTAMP NULL, - revoked BOOLEAN NOT NULL DEFAULT false, - created_at TIMESTAMP NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP NOT NULL DEFAULT NOW() -); - -CREATE INDEX idx_marketplace_share_info_for_owner ON marketplace_share_info(owner_id, file_path_hash); -CREATE INDEX idx_marketplace_share_info_for_client ON marketplace_share_info(client_id, file_path_hash); - -CREATE TRIGGER share_info_modtime BEFORE UPDATE ON marketplace_share_info FOR EACH ROW EXECUTE PROCEDURE update_modified_column(); - -GRANT ALL PRIVILEGES ON TABLE marketplace_share_info TO blobber_user; -GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO blobber_user; diff --git a/sql/17-add-indexes-to-reference-objects.sql b/sql/17-add-indexes-to-reference-objects.sql deleted file mode 100644 index dc8146e84..000000000 --- a/sql/17-add-indexes-to-reference-objects.sql +++ /dev/null @@ -1,4 +0,0 @@ -\connect blobber_meta; - -CREATE INDEX idx_reference_objects_for_lookup_hash ON reference_objects(allocation_id, lookup_hash); -CREATE INDEX idx_reference_objects_for_path ON reference_objects(allocation_id, path); diff --git a/sql/09-add-suspend-column-to-read-markers.sql b/sql/comment.txt similarity index 76% rename from sql/09-add-suspend-column-to-read-markers.sql rename to sql/comment.txt index 39d8e3ec8..5944b8d2f 100644 --- a/sql/09-add-suspend-column-to-read-markers.sql +++ b/sql/comment.txt @@ -1,6 +1,6 @@ -\connect blobber_meta; +-- \connect blobber_meta; -BEGIN; +-- BEGIN; -- if the suspend is equal to 'counter' column, then we don't send the marker -- even if its 'redeem_required' column is set to true; for a case, where we @@ -17,15 +17,16 @@ BEGIN; -- expired -- - ALTER TABLE read_markers - ADD COLUMN suspend BIGINT NOT NULL DEFAULT -1; + -- Changes have been moved to table creation + -- ALTER TABLE read_markers + -- ADD COLUMN suspend BIGINT NOT NULL DEFAULT -1; -- -- we don't need to track pending reads anymore -- - - ALTER TABLE pendings - DROP COLUMN pending_read; + -- Changes have been moved to table creation + -- ALTER TABLE pendings + -- DROP COLUMN pending_read; -- -- pending values has changed from tokens to number of block for read markers @@ -39,7 +40,7 @@ BEGIN; -- the transaction; thus the pending will not be reset next time, that's -- expected) - UPDATE pendings SET pending_write = 0; + -- UPDATE pendings SET pending_write = 0 -- -- don't track every redeem, since update allocation or slow redeeming can @@ -47,7 +48,8 @@ BEGIN; -- to track pending redeems -- - DROP TABLE read_redeems CASCADE; - DROP TABLE write_redeems CASCADE; -- with indices + -- Changes have been moved to table creation; Table is simply never created + -- DROP TABLE read_redeems CASCADE; + -- DROP TABLE write_redeems CASCADE; -- with indices -COMMIT; +-- COMMIT; diff --git a/sql/create-indexes.sql b/sql/create-indexes.sql deleted file mode 100644 index f0d57dbe6..000000000 --- a/sql/create-indexes.sql +++ /dev/null @@ -1,8 +0,0 @@ -\connect blobber_meta; -BEGIN; -DROP INDEX IF EXISTS path_idx; -DROP INDEX IF EXISTS update_idx; --- Create index on path column; It cannot be Unique index because of soft delete by gorm -CREATE INDEX path_idx ON reference_objects (path); -CREATE INDEX update_idx ON reference_objects (updated_at); -COMMIT; \ No newline at end of file