/
migrate.go
2095 lines (2017 loc) · 70.4 KB
/
migrate.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package satellitedb
import (
"context"
"fmt"
"strings"
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/private/dbutil"
"storj.io/private/dbutil/cockroachutil"
"storj.io/private/dbutil/pgutil"
"storj.io/private/tagsql"
"storj.io/storj/private/migrate"
)
//go:generate go run migrate_gen.go
var (
// ErrMigrate is for tracking migration errors.
ErrMigrate = errs.Class("migrate")
// ErrMigrateMinVersion is for migration min version errors.
ErrMigrateMinVersion = errs.Class("migrate min version")
)
// MigrateToLatest migrates the database to the latest version.
func (db *satelliteDB) MigrateToLatest(ctx context.Context) error {
// First handle the idiosyncrasies of postgres and cockroach migrations. Postgres
// will need to create any schemas specified in the search path, and cockroach
// will need to create the database it was told to connect to. These things should
// not really be here, and instead should be assumed to exist.
// This is tracked in jira ticket SM-200
switch db.impl {
case dbutil.Postgres:
schema, err := pgutil.ParseSchemaFromConnstr(db.source)
if err != nil {
return errs.New("error parsing schema: %+v", err)
}
if schema != "" {
err = pgutil.CreateSchema(ctx, db, schema)
if err != nil {
return errs.New("error creating schema: %+v", err)
}
}
case dbutil.Cockroach:
var dbName string
if err := db.QueryRow(ctx, `SELECT current_database();`).Scan(&dbName); err != nil {
return errs.New("error querying current database: %+v", err)
}
_, err := db.Exec(ctx, fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %s;`,
pgutil.QuoteIdentifier(dbName)))
if err != nil {
return errs.Wrap(err)
}
}
switch db.impl {
case dbutil.Postgres, dbutil.Cockroach:
migration := db.PostgresMigration()
// since we merged migration steps 0-69, the current db version should never be
// less than 69 unless the migration hasn't run yet
const minDBVersion = 69
dbVersion, err := migration.CurrentVersion(ctx, db.log, db.DB)
if err != nil {
return errs.New("error current version: %+v", err)
}
if dbVersion > -1 && dbVersion < minDBVersion {
return ErrMigrateMinVersion.New("current database version is %d, it shouldn't be less than the min version %d",
dbVersion, minDBVersion,
)
}
return migration.Run(ctx, db.log.Named("migrate"))
default:
return migrate.Create(ctx, "database", db.DB)
}
}
// TestingMigrateToLatest is a method for creating all tables for database for testing.
func (db *satelliteDB) TestingMigrateToLatest(ctx context.Context) error {
switch db.impl {
case dbutil.Postgres:
schema, err := pgutil.ParseSchemaFromConnstr(db.source)
if err != nil {
return ErrMigrateMinVersion.New("error parsing schema: %+v", err)
}
if schema != "" {
err = pgutil.CreateSchema(ctx, db, schema)
if err != nil {
return ErrMigrateMinVersion.New("error creating schema: %+v", err)
}
}
case dbutil.Cockroach:
var dbName string
if err := db.QueryRow(ctx, `SELECT current_database();`).Scan(&dbName); err != nil {
return ErrMigrateMinVersion.New("error querying current database: %+v", err)
}
_, err := db.Exec(ctx, fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %s;`, pgutil.QuoteIdentifier(dbName)))
if err != nil {
return ErrMigrateMinVersion.Wrap(err)
}
}
switch db.impl {
case dbutil.Postgres, dbutil.Cockroach:
migration := db.PostgresMigration()
dbVersion, err := migration.CurrentVersion(ctx, db.log, db.DB)
if err != nil {
return ErrMigrateMinVersion.Wrap(err)
}
if dbVersion > -1 {
return ErrMigrateMinVersion.New("the database must be empty, got version %d", dbVersion)
}
testMigration := db.TestPostgresMigration()
return testMigration.Run(ctx, db.log.Named("migrate"))
default:
return migrate.Create(ctx, "database", db.DB)
}
}
// CheckVersion confirms the database is at the desired version.
func (db *satelliteDB) CheckVersion(ctx context.Context) error {
switch db.impl {
case dbutil.Postgres, dbutil.Cockroach:
migration := db.PostgresMigration()
return migration.ValidateVersions(ctx, db.log)
default:
return nil
}
}
// TestPostgresMigration returns steps needed for migrating test postgres database.
func (db *satelliteDB) TestPostgresMigration() *migrate.Migration {
return db.testMigration()
}
// PostgresMigration returns steps needed for migrating postgres database.
func (db *satelliteDB) PostgresMigration() *migrate.Migration {
return &migrate.Migration{
Table: "versions",
Steps: []*migrate.Step{
{
DB: &db.migrationDB,
Description: "Initial setup",
Version: 103,
Action: migrate.SQL{
`CREATE TABLE accounting_rollups (
id bigserial NOT NULL,
node_id bytea NOT NULL,
start_time timestamp with time zone NOT NULL,
put_total bigint NOT NULL,
get_total bigint NOT NULL,
get_audit_total bigint NOT NULL,
get_repair_total bigint NOT NULL,
put_repair_total bigint NOT NULL,
at_rest_total double precision NOT NULL,
PRIMARY KEY ( id )
);`,
`CREATE INDEX accounting_rollups_start_time_index ON accounting_rollups ( start_time );`,
`CREATE TABLE accounting_timestamps (
name text NOT NULL,
value timestamp with time zone NOT NULL,
PRIMARY KEY ( name )
);`,
`CREATE TABLE bucket_bandwidth_rollups (
bucket_name bytea NOT NULL,
interval_start timestamp with time zone NOT NULL,
interval_seconds integer NOT NULL,
action integer NOT NULL,
inline bigint NOT NULL,
allocated bigint NOT NULL,
settled bigint NOT NULL,
project_id bytea NOT NULL ,
CONSTRAINT bucket_bandwidth_rollups_pk PRIMARY KEY (bucket_name, project_id, interval_start, action)
);`,
`CREATE INDEX IF NOT EXISTS bucket_bandwidth_rollups_project_id_action_interval_index ON bucket_bandwidth_rollups ( project_id, action, interval_start );`,
`CREATE TABLE bucket_storage_tallies (
bucket_name bytea NOT NULL,
interval_start timestamp with time zone NOT NULL,
inline bigint NOT NULL,
remote bigint NOT NULL,
remote_segments_count integer NOT NULL,
inline_segments_count integer NOT NULL,
object_count integer NOT NULL,
metadata_size bigint NOT NULL,
project_id bytea NOT NULL,
CONSTRAINT bucket_storage_tallies_pk PRIMARY KEY (bucket_name, project_id, interval_start)
);`,
`CREATE TABLE injuredsegments (
data bytea NOT NULL,
attempted timestamp with time zone,
path bytea NOT NULL,
num_healthy_pieces integer DEFAULT 52 NOT NULL,
CONSTRAINT injuredsegments_pk PRIMARY KEY (path)
);`,
`CREATE INDEX injuredsegments_attempted_index ON injuredsegments ( attempted );`,
`CREATE INDEX injuredsegments_num_healthy_pieces_index ON injuredsegments ( num_healthy_pieces );`,
`CREATE TABLE irreparabledbs (
segmentpath bytea NOT NULL,
segmentdetail bytea NOT NULL,
pieces_lost_count bigint NOT NULL,
seg_damaged_unix_sec bigint NOT NULL,
repair_attempt_count bigint NOT NULL,
PRIMARY KEY ( segmentpath )
);`,
`CREATE TABLE nodes (
id bytea NOT NULL,
audit_success_count bigint NOT NULL DEFAULT 0,
total_audit_count bigint NOT NULL DEFAULT 0,
uptime_success_count bigint NOT NULL,
total_uptime_count bigint NOT NULL,
created_at timestamp with time zone NOT NULL DEFAULT current_timestamp,
updated_at timestamp with time zone NOT NULL DEFAULT current_timestamp,
wallet text NOT NULL,
email text NOT NULL,
address text NOT NULL DEFAULT '',
protocol INTEGER NOT NULL DEFAULT 0,
type INTEGER NOT NULL DEFAULT 0,
free_disk BIGINT NOT NULL DEFAULT -1,
latency_90 BIGINT NOT NULL DEFAULT 0,
last_contact_success TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT 'epoch',
last_contact_failure TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT 'epoch',
major bigint NOT NULL DEFAULT 0,
minor bigint NOT NULL DEFAULT 0,
patch bigint NOT NULL DEFAULT 0,
hash TEXT NOT NULL DEFAULT '',
timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT '0001-01-01 00:00:00+00',
release bool NOT NULL DEFAULT FALSE,
contained bool NOT NULL DEFAULT FALSE,
last_net text NOT NULL,
disqualified timestamp with time zone,
audit_reputation_alpha double precision NOT NULL DEFAULT 1,
audit_reputation_beta double precision NOT NULL DEFAULT 0,
uptime_reputation_alpha double precision NOT NULL DEFAULT 1,
uptime_reputation_beta double precision NOT NULL DEFAULT 0,
piece_count bigint NOT NULL DEFAULT 0,
exit_loop_completed_at timestamp with time zone,
exit_initiated_at timestamp with time zone,
exit_finished_at timestamp with time zone,
exit_success boolean NOT NULL DEFAULT FALSE,
unknown_audit_reputation_alpha double precision NOT NULL DEFAULT 1,
unknown_audit_reputation_beta double precision NOT NULL DEFAULT 0,
suspended timestamp with time zone,
last_ip_port text,
vetted_at timestamp with time zone,
PRIMARY KEY ( id )
);`,
`CREATE INDEX node_last_ip ON nodes ( last_net );`,
`CREATE TABLE offers (
id serial NOT NULL,
name text NOT NULL,
description text NOT NULL,
type integer NOT NULL,
award_credit_duration_days integer,
invitee_credit_duration_days integer,
redeemable_cap integer,
expires_at timestamp with time zone NOT NULL,
created_at timestamp with time zone NOT NULL,
status integer NOT NULL,
award_credit_in_cents integer NOT NULL DEFAULT 0,
invitee_credit_in_cents integer NOT NULL DEFAULT 0,
PRIMARY KEY ( id )
);`,
`CREATE TABLE peer_identities (
node_id bytea NOT NULL,
leaf_serial_number bytea NOT NULL,
chain bytea NOT NULL,
updated_at timestamp with time zone NOT NULL,
PRIMARY KEY ( node_id )
);`,
`CREATE TABLE pending_audits (
node_id bytea NOT NULL,
piece_id bytea NOT NULL,
stripe_index bigint NOT NULL,
share_size bigint NOT NULL,
expected_share_hash bytea NOT NULL,
reverify_count bigint NOT NULL,
path bytea NOT NULL,
PRIMARY KEY ( node_id )
);`,
`CREATE TABLE projects (
id bytea NOT NULL,
name text NOT NULL,
description text NOT NULL,
created_at timestamp with time zone NOT NULL,
usage_limit bigint NOT NULL DEFAULT 0,
partner_id bytea,
owner_id bytea NOT NULL,
rate_limit integer,
PRIMARY KEY ( id )
);`,
`CREATE TABLE registration_tokens (
secret bytea NOT NULL,
owner_id bytea UNIQUE,
project_limit integer NOT NULL,
created_at timestamp with time zone NOT NULL,
PRIMARY KEY ( secret )
);`,
`CREATE TABLE reset_password_tokens (
secret bytea NOT NULL,
owner_id bytea NOT NULL UNIQUE,
created_at timestamp with time zone NOT NULL,
PRIMARY KEY ( secret )
);`,
`CREATE TABLE serial_numbers (
id serial NOT NULL,
serial_number bytea NOT NULL,
bucket_id bytea NOT NULL,
expires_at timestamp with time zone NOT NULL,
PRIMARY KEY ( id )
);`,
`CREATE INDEX serial_numbers_expires_at_index ON serial_numbers ( expires_at );`,
`CREATE UNIQUE INDEX serial_number_index ON serial_numbers ( serial_number )`,
`CREATE TABLE storagenode_bandwidth_rollups (
storagenode_id bytea NOT NULL,
interval_start timestamp with time zone NOT NULL,
interval_seconds integer NOT NULL,
action integer NOT NULL,
allocated bigint DEFAULT 0,
settled bigint NOT NULL,
PRIMARY KEY ( storagenode_id, interval_start, action )
);`,
`CREATE TABLE storagenode_storage_tallies (
node_id bytea NOT NULL,
interval_end_time timestamp with time zone NOT NULL,
data_total double precision NOT NULL,
CONSTRAINT storagenode_storage_tallies_pkey PRIMARY KEY ( interval_end_time, node_id )
);`,
`CREATE INDEX storagenode_storage_tallies_node_id_index ON storagenode_storage_tallies ( node_id );`,
`CREATE TABLE users (
id bytea NOT NULL,
full_name text NOT NULL,
short_name text,
email text NOT NULL,
password_hash bytea NOT NULL,
status integer NOT NULL,
created_at timestamp with time zone NOT NULL,
partner_id bytea,
normalized_email text NOT NULL,
PRIMARY KEY ( id )
);`,
`CREATE TABLE value_attributions (
bucket_name bytea NOT NULL,
partner_id bytea NOT NULL,
last_updated timestamp with time zone NOT NULL,
project_id bytea NOT NULL,
PRIMARY KEY (project_id, bucket_name)
);`,
`CREATE TABLE api_keys (
id bytea NOT NULL,
project_id bytea NOT NULL REFERENCES projects( id ) ON DELETE CASCADE,
head bytea NOT NULL UNIQUE,
name text NOT NULL,
secret bytea NOT NULL,
created_at timestamp with time zone NOT NULL,
partner_id bytea,
PRIMARY KEY ( id ),
UNIQUE ( name, project_id )
);`,
`CREATE TABLE bucket_metainfos (
id bytea NOT NULL,
project_id bytea NOT NULL REFERENCES projects( id ),
name bytea NOT NULL,
path_cipher integer NOT NULL,
created_at timestamp with time zone NOT NULL,
default_segment_size integer NOT NULL,
default_encryption_cipher_suite integer NOT NULL,
default_encryption_block_size integer NOT NULL,
default_redundancy_algorithm integer NOT NULL,
default_redundancy_share_size integer NOT NULL,
default_redundancy_required_shares integer NOT NULL,
default_redundancy_repair_shares integer NOT NULL,
default_redundancy_optimal_shares integer NOT NULL,
default_redundancy_total_shares integer NOT NULL,
partner_id bytea,
PRIMARY KEY ( id ),
UNIQUE ( name, project_id )
);`,
`CREATE TABLE project_invoice_stamps (
project_id bytea NOT NULL REFERENCES projects( id ) ON DELETE CASCADE,
invoice_id bytea NOT NULL UNIQUE,
start_date timestamp with time zone NOT NULL,
end_date timestamp with time zone NOT NULL,
created_at timestamp with time zone NOT NULL,
PRIMARY KEY ( project_id, start_date, end_date )
);`,
`CREATE TABLE project_members (
member_id bytea NOT NULL REFERENCES users( id ) ON DELETE CASCADE,
project_id bytea NOT NULL REFERENCES projects( id ) ON DELETE CASCADE,
created_at timestamp with time zone NOT NULL,
PRIMARY KEY ( member_id, project_id )
);`,
`CREATE TABLE used_serials (
serial_number_id integer NOT NULL REFERENCES serial_numbers( id ) ON DELETE CASCADE,
storage_node_id bytea NOT NULL,
PRIMARY KEY ( serial_number_id, storage_node_id )
);`,
`CREATE TABLE user_credits (
id serial NOT NULL,
user_id bytea NOT NULL REFERENCES users( id ) ON DELETE CASCADE,
offer_id integer NOT NULL REFERENCES offers( id ),
referred_by bytea REFERENCES users( id ) ON DELETE SET NULL,
credits_earned_in_cents integer NOT NULL,
credits_used_in_cents integer NOT NULL,
expires_at timestamp with time zone NOT NULL,
created_at timestamp with time zone NOT NULL,
type text NOT NULL,
PRIMARY KEY ( id ),
UNIQUE (id, offer_id)
);`,
`CREATE UNIQUE INDEX credits_earned_user_id_offer_id ON user_credits (id, offer_id);`,
`INSERT INTO offers (
id,
name,
description,
award_credit_in_cents,
invitee_credit_in_cents,
expires_at,
created_at,
status,
type,
award_credit_duration_days,
invitee_credit_duration_days
)
VALUES (
1,
'Default referral offer',
'Is active when no other active referral offer',
300,
600,
'2119-03-14 08:28:24.636949+00',
'2019-07-14 08:28:24.636949+00',
1,
2,
365,
14
),
(
2,
'Default free credit offer',
'Is active when no active free credit offer',
0,
300,
'2119-03-14 08:28:24.636949+00',
'2019-07-14 08:28:24.636949+00',
1,
1,
NULL,
14
) ON CONFLICT DO NOTHING;`,
`CREATE TABLE graceful_exit_progress (
node_id bytea NOT NULL,
bytes_transferred bigint NOT NULL,
updated_at timestamp with time zone NOT NULL,
pieces_transferred bigint NOT NULL DEFAULT 0,
pieces_failed bigint NOT NULL DEFAULT 0,
PRIMARY KEY ( node_id )
);`,
`CREATE TABLE graceful_exit_transfer_queue (
node_id bytea NOT NULL,
path bytea NOT NULL,
piece_num integer NOT NULL,
durability_ratio double precision NOT NULL,
queued_at timestamp with time zone NOT NULL,
requested_at timestamp with time zone,
last_failed_at timestamp with time zone,
last_failed_code integer,
failed_count integer,
finished_at timestamp with time zone,
root_piece_id bytea,
order_limit_send_count integer NOT NULL DEFAULT 0,
PRIMARY KEY ( node_id, path, piece_num )
);`,
`CREATE TABLE stripe_customers (
user_id bytea NOT NULL,
customer_id text NOT NULL UNIQUE,
created_at timestamp with time zone NOT NULL,
PRIMARY KEY ( user_id )
);`,
`CREATE TABLE stripecoinpayments_invoice_project_records (
id bytea NOT NULL,
project_id bytea NOT NULL,
storage double precision NOT NULL,
egress bigint NOT NULL,
objects bigint NOT NULL,
period_start timestamp with time zone NOT NULL,
period_end timestamp with time zone NOT NULL,
state integer NOT NULL,
created_at timestamp with time zone NOT NULL,
PRIMARY KEY ( id ),
UNIQUE ( project_id, period_start, period_end )
);`,
`CREATE TABLE stripecoinpayments_tx_conversion_rates (
tx_id text NOT NULL,
rate bytea NOT NULL,
created_at timestamp with time zone NOT NULL,
PRIMARY KEY ( tx_id )
);`,
`CREATE TABLE coinpayments_transactions (
id text NOT NULL,
user_id bytea NOT NULL,
address text NOT NULL,
amount bytea NOT NULL,
received bytea NOT NULL,
status integer NOT NULL,
key text NOT NULL,
timeout integer NOT NULL,
created_at timestamp with time zone NOT NULL,
PRIMARY KEY ( id )
);`,
`CREATE TABLE stripecoinpayments_apply_balance_intents (
tx_id text NOT NULL REFERENCES coinpayments_transactions( id ) ON DELETE CASCADE,
state integer NOT NULL,
created_at timestamp with time zone NOT NULL,
PRIMARY KEY ( tx_id )
);`,
`CREATE TABLE nodes_offline_times (
node_id bytea NOT NULL,
tracked_at timestamp with time zone NOT NULL,
seconds integer NOT NULL,
PRIMARY KEY ( node_id, tracked_at )
);`,
`CREATE INDEX nodes_offline_times_node_id_index ON nodes_offline_times ( node_id );`,
`CREATE TABLE coupons (
id bytea NOT NULL,
project_id bytea NOT NULL,
user_id bytea NOT NULL,
amount bigint NOT NULL,
description text NOT NULL,
type integer NOT NULL,
status integer NOT NULL,
duration bigint NOT NULL,
created_at timestamp with time zone NOT NULL,
PRIMARY KEY ( id )
);`,
`CREATE TABLE coupon_usages (
coupon_id bytea NOT NULL,
amount bigint NOT NULL,
status integer NOT NULL,
period timestamp with time zone NOT NULL,
PRIMARY KEY ( coupon_id, period )
);`,
`CREATE TABLE reported_serials (
expires_at timestamp with time zone NOT NULL,
storage_node_id bytea NOT NULL,
bucket_id bytea NOT NULL,
action integer NOT NULL,
serial_number bytea NOT NULL,
settled bigint NOT NULL,
observed_at timestamp with time zone NOT NULL,
PRIMARY KEY ( expires_at, storage_node_id, bucket_id, action, serial_number )
);`,
`CREATE TABLE credits (
user_id bytea NOT NULL,
transaction_id text NOT NULL,
amount bigint NOT NULL,
created_at timestamp with time zone NOT NULL,
PRIMARY KEY ( transaction_id )
);`,
`CREATE TABLE credits_spendings (
id bytea NOT NULL,
user_id bytea NOT NULL,
project_id bytea NOT NULL,
amount bigint NOT NULL,
status int NOT NULL,
created_at timestamp with time zone NOT NULL,
PRIMARY KEY ( id )
);`,
`CREATE TABLE consumed_serials (
storage_node_id bytea NOT NULL,
serial_number bytea NOT NULL,
expires_at timestamp with time zone NOT NULL,
PRIMARY KEY ( storage_node_id, serial_number )
);`,
`CREATE INDEX consumed_serials_expires_at_index ON consumed_serials ( expires_at );`,
`CREATE TABLE pending_serial_queue (
storage_node_id bytea NOT NULL,
bucket_id bytea NOT NULL,
serial_number bytea NOT NULL,
action integer NOT NULL,
settled bigint NOT NULL,
expires_at timestamp with time zone NOT NULL,
PRIMARY KEY ( storage_node_id, bucket_id, serial_number )
);`,
`CREATE TABLE storagenode_payments (
id bigserial NOT NULL,
created_at timestamp with time zone NOT NULL,
node_id bytea NOT NULL,
period text NOT NULL,
amount bigint NOT NULL,
receipt text,
notes text,
PRIMARY KEY ( id )
);`,
`CREATE INDEX storagenode_payments_node_id_period_index ON storagenode_payments ( node_id, period );`,
`CREATE TABLE storagenode_paystubs (
period text NOT NULL,
node_id bytea NOT NULL,
created_at timestamp with time zone NOT NULL,
codes text NOT NULL,
usage_at_rest double precision NOT NULL,
usage_get bigint NOT NULL,
usage_put bigint NOT NULL,
usage_get_repair bigint NOT NULL,
usage_put_repair bigint NOT NULL,
usage_get_audit bigint NOT NULL,
comp_at_rest bigint NOT NULL,
comp_get bigint NOT NULL,
comp_put bigint NOT NULL,
comp_get_repair bigint NOT NULL,
comp_put_repair bigint NOT NULL,
comp_get_audit bigint NOT NULL,
surge_percent bigint NOT NULL,
held bigint NOT NULL,
owed bigint NOT NULL,
disposed bigint NOT NULL,
paid bigint NOT NULL,
PRIMARY KEY ( period, node_id )
);`,
`CREATE INDEX storagenode_paystubs_node_id_index ON storagenode_paystubs ( node_id );`,
},
},
{
DB: &db.migrationDB,
Description: "Add missing bucket_bandwidth_rollups_action_interval_project_id_index index",
Version: 104,
Action: migrate.SQL{
`CREATE INDEX IF NOT EXISTS bucket_bandwidth_rollups_action_interval_project_id_index ON bucket_bandwidth_rollups(action, interval_start, project_id );`,
},
},
{
DB: &db.migrationDB,
Description: "Remove all nodes from suspension mode.",
Version: 105,
Action: migrate.SQL{
`UPDATE nodes SET suspended=NULL;`,
},
},
{
DB: &db.migrationDB,
Description: "Add project_bandwidth_rollup table and populate with current months data",
Version: 106,
Action: migrate.SQL{
`CREATE TABLE IF NOT EXISTS project_bandwidth_rollups (
project_id bytea NOT NULL,
interval_month date NOT NULL,
egress_allocated bigint NOT NULL,
PRIMARY KEY ( project_id, interval_month )
);
INSERT INTO project_bandwidth_rollups(project_id, interval_month, egress_allocated) (
SELECT project_id, date_trunc('MONTH',now())::DATE, sum(allocated)::bigint FROM bucket_bandwidth_rollups
WHERE action = 2 AND interval_start >= date_trunc('MONTH',now())::timestamp group by project_id)
ON CONFLICT(project_id, interval_month) DO UPDATE SET egress_allocated = EXCLUDED.egress_allocated::bigint;`,
},
},
{
DB: &db.migrationDB,
Description: "add separate bandwidth column",
Version: 107,
Action: migrate.SQL{
`ALTER TABLE projects ADD COLUMN bandwidth_limit bigint NOT NULL DEFAULT 0;`,
},
},
{
DB: &db.migrationDB,
Description: "backfill bandwidth column with previous limits",
Version: 108,
SeparateTx: true,
Action: migrate.SQL{
`UPDATE projects SET bandwidth_limit = usage_limit;`,
},
},
{
DB: &db.migrationDB,
Description: "add period column to the credits_spendings table (step 1)",
Version: 109,
SeparateTx: true,
Action: migrate.SQL{
`ALTER TABLE credits_spendings ADD COLUMN period timestamp with time zone;`,
},
},
{
DB: &db.migrationDB,
Description: "add period column to the credits_spendings table (step 2)",
Version: 110,
SeparateTx: true,
Action: migrate.SQL{
`UPDATE credits_spendings SET period = 'epoch';`,
},
},
{
DB: &db.migrationDB,
Description: "add period column to the credits_spendings table (step 3)",
Version: 111,
SeparateTx: true,
Action: migrate.SQL{
`ALTER TABLE credits_spendings ALTER COLUMN period SET NOT NULL;`,
},
},
{
DB: &db.migrationDB,
Description: "fix incorrect calculations on backported paystub data",
Version: 112,
Action: migrate.SQL{`
UPDATE storagenode_paystubs SET
comp_at_rest = (
((owed + held - disposed)::float / GREATEST(surge_percent::float / 100, 1))::int
- comp_get - comp_get_repair - comp_get_audit
)
WHERE
(
abs(
((owed + held - disposed)::float / GREATEST(surge_percent::float / 100, 1))::int
- comp_get - comp_get_repair - comp_get_audit
) >= 10
OR comp_at_rest < 0
)
AND codes NOT LIKE '%O%'
AND codes NOT LIKE '%D%'
AND period < '2020-03'
`},
},
{
DB: &db.migrationDB,
Description: "drop project_id column from coupon table",
Version: 113,
Action: migrate.SQL{
`ALTER TABLE coupons DROP COLUMN project_id;`,
},
},
{
DB: &db.migrationDB,
Description: "add new columns for suspension to node tables",
Version: 114,
Action: migrate.SQL{
`ALTER TABLE nodes ADD COLUMN unknown_audit_suspended TIMESTAMP WITH TIME ZONE;`,
`ALTER TABLE nodes ADD COLUMN offline_suspended TIMESTAMP WITH TIME ZONE;`,
`ALTER TABLE nodes ADD COLUMN under_review TIMESTAMP WITH TIME ZONE;`,
},
},
{
DB: &db.migrationDB,
Description: "add revocations database",
Version: 115,
Action: migrate.SQL{`
CREATE TABLE revocations (
revoked bytea NOT NULL,
api_key_id bytea NOT NULL,
PRIMARY KEY ( revoked )
);
`},
},
{
DB: &db.migrationDB,
Description: "add audit histories database",
Version: 116,
Action: migrate.SQL{
`CREATE TABLE audit_histories (
node_id bytea NOT NULL,
history bytea NOT NULL,
PRIMARY KEY ( node_id )
);`,
},
},
{
DB: &db.migrationDB,
Description: "add node_api_versions table",
Version: 117,
Action: migrate.SQL{`
CREATE TABLE node_api_versions (
id bytea NOT NULL,
api_version integer NOT NULL,
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL,
PRIMARY KEY ( id )
);
`},
},
{
DB: &db.migrationDB,
Description: "add max_buckets field to projects and an implicit index on bucket_metainfos project_id,name",
SeparateTx: true,
Version: 118,
Action: migrate.SQL{
`ALTER TABLE projects ADD COLUMN max_buckets INTEGER NOT NULL DEFAULT 0;`,
`ALTER TABLE bucket_metainfos ADD UNIQUE (project_id, name);`,
},
},
{
DB: &db.migrationDB,
Description: "add project_limit field to users table",
Version: 119,
Action: migrate.SQL{
`ALTER TABLE users ADD COLUMN project_limit INTEGER NOT NULL DEFAULT 0;`,
},
},
{
DB: &db.migrationDB,
Description: "back fill user project limits from existing registration tokens",
Version: 120,
SeparateTx: true,
Action: migrate.SQL{
`UPDATE users SET project_limit = registration_tokens.project_limit FROM registration_tokens WHERE users.id = registration_tokens.owner_id;`,
},
},
{
DB: &db.migrationDB,
Description: "drop tables related to credits (old deposit bonuses)",
Version: 121,
Action: migrate.SQL{
`DROP TABLE credits;`,
`DROP TABLE credits_spendings;`,
},
},
{
DB: &db.migrationDB,
Description: "drop project_invoice_stamps table",
Version: 122,
Action: migrate.SQL{
`DROP TABLE project_invoice_stamps;`,
},
},
{
DB: &db.migrationDB,
Description: "drop project_invoice_stamps table",
Version: 123,
Action: migrate.SQL{
`ALTER TABLE nodes ADD COLUMN online_score double precision NOT NULL DEFAULT 1;`,
},
},
{
DB: &db.migrationDB,
Description: "add column and index updated_at to injuredsegments",
Version: 124,
Action: migrate.SQL{
`ALTER TABLE injuredsegments ADD COLUMN updated_at timestamp with time zone NOT NULL DEFAULT current_timestamp;`,
`CREATE INDEX injuredsegments_updated_at_index ON injuredsegments ( updated_at );`,
},
},
{
DB: &db.migrationDB,
Description: "make limit columns nullable",
Version: 125,
SeparateTx: true,
Action: migrate.SQL{
`ALTER TABLE projects ALTER COLUMN max_buckets DROP NOT NULL;`,
`ALTER TABLE projects ALTER COLUMN max_buckets SET DEFAULT 100;`,
`ALTER TABLE projects ALTER COLUMN usage_limit DROP NOT NULL;`,
`ALTER TABLE projects ALTER COLUMN usage_limit SET DEFAULT 50000000000;`,
`ALTER TABLE projects ALTER COLUMN bandwidth_limit DROP NOT NULL;`,
`ALTER TABLE projects ALTER COLUMN bandwidth_limit SET DEFAULT 50000000000;`,
},
},
{
DB: &db.migrationDB,
Description: "set 0 limits back to default",
Version: 126,
Action: migrate.SQL{
`UPDATE projects SET max_buckets = 100 WHERE max_buckets = 0;`,
`UPDATE projects SET usage_limit = 50000000000 WHERE usage_limit = 0;`,
`UPDATE projects SET bandwidth_limit = 50000000000 WHERE bandwidth_limit = 0;`,
},
},
{
DB: &db.migrationDB,
Description: "enable multiple projects for existing users",
Version: 127,
Action: migrate.SQL{
`UPDATE users SET project_limit=0 WHERE project_limit <= 10 AND project_limit > 0;`,
},
},
{
DB: &db.migrationDB,
Description: "drop default values for project limits",
Version: 128,
SeparateTx: true,
Action: migrate.SQL{
`ALTER TABLE projects ALTER COLUMN max_buckets DROP DEFAULT;`,
`ALTER TABLE projects ALTER COLUMN usage_limit DROP DEFAULT;`,
`ALTER TABLE projects ALTER COLUMN bandwidth_limit DROP DEFAULT;`,
},
},
{
DB: &db.migrationDB,
Description: "reset everyone with default rate limits to NULL",
Version: 129,
SeparateTx: true,
Action: migrate.SQL{
`UPDATE projects SET max_buckets = NULL WHERE max_buckets <= 100;`,
`UPDATE projects SET usage_limit = NULL WHERE usage_limit <= 50000000000;`,
`UPDATE projects SET bandwidth_limit = NULL WHERE bandwidth_limit <= 50000000000;`,
},
},
{
DB: &db.migrationDB,
Description: "add index for the gracefule exit transfer queue",
Version: 130,
SeparateTx: true,
Action: migrate.SQL{
`CREATE INDEX IF NOT EXISTS graceful_exit_transfer_queue_nid_dr_qa_fa_lfa_index ON graceful_exit_transfer_queue ( node_id, durability_ratio, queued_at, finished_at, last_failed_at );`,
},
},
{
DB: &db.migrationDB,
Description: "create table storagenode_bandwidth_rollups_phase2",
Version: 131,
SeparateTx: true,
Action: migrate.SQL{
`CREATE TABLE storagenode_bandwidth_rollups_phase2 (
storagenode_id bytea NOT NULL,
interval_start timestamp with time zone NOT NULL,
interval_seconds integer NOT NULL,
action integer NOT NULL,
allocated bigint DEFAULT 0,
settled bigint NOT NULL,
PRIMARY KEY ( storagenode_id, interval_start, action )
);`,
},
},
{
DB: &db.migrationDB,
Description: "add injuredsegments.segment_health",
Version: 132,
SeparateTx: true,
Action: migrate.SQL{
`ALTER TABLE injuredsegments ADD COLUMN segment_health double precision NOT NULL DEFAULT 1;`,
`CREATE INDEX injuredsegments_segment_health_index ON injuredsegments ( segment_health );`,
},
},
{
DB: &db.migrationDB,
Description: "Use node_id and start_time for accounting_rollups pkey instead of autogenerated id",
Version: 133,
SeparateTx: true,
Action: migrate.Func(func(ctx context.Context, log *zap.Logger, db tagsql.DB, tx tagsql.Tx) error {
if _, ok := db.Driver().(*cockroachutil.Driver); ok {
_, err := db.Exec(ctx,
`ALTER TABLE accounting_rollups RENAME TO accounting_rollups_original;`,
)
if err != nil {
return ErrMigrate.Wrap(err)
}
_, err = db.Exec(ctx,
`CREATE TABLE accounting_rollups (
node_id bytea NOT NULL,
start_time timestamp with time zone NOT NULL,