Skip to content

Commit

Permalink
shared/{dbutil|migrate|tagsql}: import db related packages from common
Browse files Browse the repository at this point in the history 
This patch moves db related utilities from common to this repository.

The spanner migration may require bigger changes (for example: spanner related implementation of tagsql interfaces).

It's way easier, if we can create the patch and integrate in one test.

Note: I decided to copy the files without keeping the history, as I wouldn't like to mess the git history with strange merges (even if it's very tempting).

History can be checked in storj.io/storj. Copied revision is 5f226fc9.

Note2: there are no licensing issues, as we are the only contributors:

git log --pretty=format:'%ae' -- tagsql | sort | uniq -c
      1 billt@storj.io
      7 egonelbre@gmail.com
      2 elek@apache.org
      1 ethan@storj.io
      1 michal@storj.io
git log --pretty=format:'%ae' -- dbutil | sort | uniq -c
      1 billt@storj.io
      1 damein@storj.io
     17 egonelbre@gmail.com
      7 elek@apache.org
      1 ivan@fraixed.es
      1 jennifer@storj.io
      2 michal@storj.io
      1 paul@thepaul.org
      2 yingrong.zhao@gmail.com
git log --pretty=format:'%ae' -- migrate | sort | uniq -c
      1 billt@storj.io
      6 egonelbre@gmail.com
      2 elek@apache.org

Change-Id: Id27e3152af0ad5bdf243bde7111ee82f055e5a22
  • Loading branch information
@elek
elek authored and Storj Robot committed Apr 25, 2024
1 parent 49663eb commit 71c0c2d
Show file tree
Hide file tree
Showing 146 changed files with 6,994 additions and 173 deletions.
2 changes: 1 addition & 1 deletion certificate/authorization/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ import (

"github.com/zeebo/errs"

"storj.io/common/dbutil"
"storj.io/common/identity"
"storj.io/common/peertls/extensions"
"storj.io/storj/private/kvstore"
"storj.io/storj/private/kvstore/boltdb"
"storj.io/storj/private/kvstore/redis"
"storj.io/storj/shared/dbutil"
)

var (
Expand Down
4 changes: 2 additions & 2 deletions cmd/storj-sim/network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,13 @@ import (
"golang.org/x/sync/errgroup"

"storj.io/common/base58"
"storj.io/common/dbutil"
"storj.io/common/dbutil/pgutil"
"storj.io/common/fpath"
"storj.io/common/identity"
"storj.io/common/pb"
"storj.io/common/processgroup"
"storj.io/common/storj"
"storj.io/storj/shared/dbutil"
"storj.io/storj/shared/dbutil/pgutil"
"storj.io/uplink"
)

Expand Down
2 changes: 1 addition & 1 deletion cmd/tools/metabase-listing-performance/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ import (
"go.uber.org/zap"
"golang.org/x/exp/slices"

_ "storj.io/common/dbutil/cockroachutil"
"storj.io/common/uuid"
"storj.io/storj/satellite/metabase"
_ "storj.io/storj/shared/dbutil/cockroachutil"
)

// Scenario declares a specific test scenario.
Expand Down
2 changes: 1 addition & 1 deletion cmd/tools/metabase-orphaned-segments/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"

"storj.io/common/dbutil/pgutil"
"storj.io/common/process"
"storj.io/common/uuid"
"storj.io/storj/satellite/metabase"
"storj.io/storj/shared/dbutil/pgutil"
)

var mon = monkit.Package()
Expand Down
4 changes: 2 additions & 2 deletions cmd/tools/metabase-orphaned-segments/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@ import (
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"

"storj.io/common/dbutil"
"storj.io/common/dbutil/tempdb"
"storj.io/common/memory"
"storj.io/common/testcontext"
cmd "storj.io/storj/cmd/tools/metabase-orphaned-segments"
"storj.io/storj/satellite/metabase"
"storj.io/storj/satellite/metabase/metabasetest"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
"storj.io/storj/shared/dbutil"
"storj.io/storj/shared/dbutil/tempdb"
)

func TestOrphanedSegment(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions cmd/tools/migrate-free-trial/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"

"storj.io/common/dbutil/cockroachutil"
"storj.io/common/dbutil/pgutil"
"storj.io/common/process"
"storj.io/common/uuid"
"storj.io/storj/shared/dbutil/cockroachutil"
"storj.io/storj/shared/dbutil/pgutil"
)

var mon = monkit.Package()
Expand Down
2 changes: 1 addition & 1 deletion cmd/tools/migrate-free-trial/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@ import (
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"

"storj.io/common/dbutil/tempdb"
"storj.io/common/testcontext"
"storj.io/common/testrand"
migrator "storj.io/storj/cmd/tools/migrate-free-trial"
"storj.io/storj/satellite"
"storj.io/storj/satellite/console"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
"storj.io/storj/shared/dbutil/tempdb"
)

// Test no entries in table doesn't error.
Expand Down
8 changes: 4 additions & 4 deletions cmd/tools/migrate-public-ids/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ import (
"go.uber.org/zap"
"go.uber.org/zap/zaptest"

"storj.io/common/dbutil"
"storj.io/common/dbutil/tempdb"
"storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/common/uuid"
migrator "storj.io/storj/cmd/tools/migrate-public-ids"
"storj.io/storj/satellite"
"storj.io/storj/satellite/console"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
"storj.io/storj/shared/dbutil"
"storj.io/storj/shared/dbutil/tempdb"
)

// Test no entries in table doesn't error.
Expand Down Expand Up @@ -168,8 +168,8 @@ func test(t *testing.T, prepare func(t *testing.T, ctx *testcontext.Context, raw
// * * * THIS IS ONLY FOR TESTING!!! * * *.
func testNullifyPublicIDs(ctx context.Context, log *zap.Logger, conn *pgx.Conn, exclude uuid.UUID) error {
_, err := conn.Exec(ctx, `
UPDATE projects
SET public_id = NULL
UPDATE projects
SET public_id = NULL
WHERE id != $1;
`, exclude.Bytes())
return err
Expand Down
4 changes: 2 additions & 2 deletions cmd/tools/migrate-public-ids/migrations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"

"storj.io/common/dbutil/cockroachutil"
"storj.io/common/dbutil/pgutil"
"storj.io/common/uuid"
"storj.io/storj/shared/dbutil/cockroachutil"
"storj.io/storj/shared/dbutil/pgutil"
)

// MigrateProjects updates all rows in the projects table, giving them a new UUID if they do not already have one.
Expand Down
4 changes: 2 additions & 2 deletions cmd/tools/migrate-token-payers/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"

"storj.io/common/dbutil/cockroachutil"
"storj.io/common/dbutil/pgutil"
"storj.io/common/process"
"storj.io/common/uuid"
"storj.io/storj/shared/dbutil/cockroachutil"
"storj.io/storj/shared/dbutil/pgutil"
)

var mon = monkit.Package()
Expand Down
2 changes: 1 addition & 1 deletion cmd/tools/migrate-token-payers/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"go.uber.org/zap/zaptest"

"storj.io/common/currency"
"storj.io/common/dbutil/tempdb"
"storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/common/uuid"
Expand All @@ -25,6 +24,7 @@ import (
"storj.io/storj/satellite/payments/coinpayments"
"storj.io/storj/satellite/payments/stripe"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
"storj.io/storj/shared/dbutil/tempdb"
)

// Test no entries in table doesn't error.
Expand Down
2 changes: 1 addition & 1 deletion cmd/tools/node-cleanup/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ import (

"storj.io/common/process"
"storj.io/common/storj"
"storj.io/common/tagsql"
"storj.io/storj/satellite/satellitedb"
"storj.io/storj/shared/tagsql"
)

var mon = monkit.Package()
Expand Down
2 changes: 1 addition & 1 deletion cmd/tools/node-cleanup/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ import (
"go.uber.org/zap/zaptest"

"storj.io/common/storj"
"storj.io/common/tagsql"
"storj.io/common/testcontext"
nodecleanup "storj.io/storj/cmd/tools/node-cleanup"
"storj.io/storj/satellite"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
"storj.io/storj/shared/tagsql"
)

func TestDelete(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions cmd/tools/segment-verify/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,15 @@ import (
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"

"storj.io/common/dbutil/cockroachutil"
"storj.io/common/memory"
"storj.io/common/storj"
"storj.io/common/tagsql"
"storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/common/uuid"
"storj.io/storj/private/testplanet"
"storj.io/storj/satellite/metabase"
"storj.io/storj/shared/dbutil/cockroachutil"
"storj.io/storj/shared/tagsql"
"storj.io/storj/storagenode/pieces"
)

Expand Down
6 changes: 3 additions & 3 deletions multinode/multinodedb/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,13 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"

"storj.io/common/dbutil"
"storj.io/common/dbutil/pgutil"
"storj.io/common/tagsql"
"storj.io/storj/multinode"
"storj.io/storj/multinode/multinodedb/dbx"
"storj.io/storj/multinode/nodes"
"storj.io/storj/private/migrate"
"storj.io/storj/shared/dbutil"
"storj.io/storj/shared/dbutil/pgutil"
"storj.io/storj/shared/tagsql"
)

var (
Expand Down
2 changes: 1 addition & 1 deletion multinode/multinodedb/dbx/multinodedb.dbx.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
_ "github.com/jackc/pgx/v5/stdlib"
"github.com/mattn/go-sqlite3"

"storj.io/common/tagsql"
"storj.io/storj/shared/tagsql"
)

// Prevent conditional imports from causing build failures.
Expand Down
10 changes: 5 additions & 5 deletions multinode/multinodedb/migrate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap/zaptest"

"storj.io/common/dbutil/dbschema"
"storj.io/common/dbutil/pgtest"
"storj.io/common/dbutil/pgutil"
"storj.io/common/dbutil/sqliteutil"
"storj.io/common/dbutil/tempdb"
"storj.io/common/testcontext"
"storj.io/storj/multinode/multinodedb"
"storj.io/storj/shared/dbutil/dbschema"
"storj.io/storj/shared/dbutil/pgtest"
"storj.io/storj/shared/dbutil/pgutil"
"storj.io/storj/shared/dbutil/sqliteutil"
"storj.io/storj/shared/dbutil/tempdb"
)

func TestMigrateSQLite3(t *testing.T) {
Expand Down
8 changes: 4 additions & 4 deletions multinode/multinodedb/multinodedbtest/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,14 @@ import (
"go.uber.org/zap"
"go.uber.org/zap/zaptest"

"storj.io/common/dbutil"
"storj.io/common/dbutil/pgtest"
"storj.io/common/dbutil/pgutil"
"storj.io/common/dbutil/tempdb"
"storj.io/common/testcontext"
"storj.io/storj/multinode"
"storj.io/storj/multinode/multinodedb"
"storj.io/storj/multinode/multinodedb/dbx"
"storj.io/storj/shared/dbutil"
"storj.io/storj/shared/dbutil/pgtest"
"storj.io/storj/shared/dbutil/pgutil"
"storj.io/storj/shared/dbutil/tempdb"
)

// Database describes a test database.
Expand Down
4 changes: 2 additions & 2 deletions private/migrate/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ import (

"github.com/zeebo/errs"

"storj.io/common/dbutil/txutil"
"storj.io/common/tagsql"
"storj.io/storj/shared/dbutil/txutil"
"storj.io/storj/shared/tagsql"
)

// Error is the default migrate errs class.
Expand Down
6 changes: 3 additions & 3 deletions private/migrate/create_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"storj.io/common/dbutil/pgtest"
"storj.io/common/dbutil/tempdb"
"storj.io/common/tagsql"
"storj.io/common/testcontext"
"storj.io/storj/private/migrate"
"storj.io/storj/shared/dbutil/pgtest"
"storj.io/storj/shared/dbutil/tempdb"
"storj.io/storj/shared/tagsql"
)

func TestCreate_Sqlite(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion private/migrate/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
package migrate

import (
"storj.io/common/tagsql"
"storj.io/storj/shared/tagsql"
)

// DBX contains additional methods for migrations.
Expand Down
4 changes: 2 additions & 2 deletions private/migrate/versions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"

"storj.io/common/dbutil/txutil"
"storj.io/common/tagsql"
"storj.io/storj/shared/dbutil/txutil"
"storj.io/storj/shared/tagsql"
)

var (
Expand Down
6 changes: 3 additions & 3 deletions private/migrate/versions_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"

"storj.io/common/dbutil/pgtest"
"storj.io/common/dbutil/tempdb"
"storj.io/common/tagsql"
"storj.io/common/testcontext"
"storj.io/storj/private/migrate"
"storj.io/storj/shared/dbutil/pgtest"
"storj.io/storj/shared/dbutil/tempdb"
"storj.io/storj/shared/tagsql"
)

func TestBasicMigrationSqliteNoRebind(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion private/revocation/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@ package revocation
import (
"context"

"storj.io/common/dbutil"
"storj.io/common/peertls/extensions"
"storj.io/common/peertls/tlsopts"
"storj.io/storj/private/kvstore/boltdb"
"storj.io/storj/private/kvstore/redis"
"storj.io/storj/shared/dbutil"
)

// OpenDBFromCfg is a convenience method to create a revocation DB
Expand Down
2 changes: 1 addition & 1 deletion private/testplanet/planet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"go.uber.org/zap"
"golang.org/x/sync/errgroup"

"storj.io/common/dbutil/pgutil"
"storj.io/common/identity"
"storj.io/common/identity/testidentity"
"storj.io/common/pb"
Expand All @@ -29,6 +28,7 @@ import (
"storj.io/common/testrand"
"storj.io/storj/satellite/overlay"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
"storj.io/storj/shared/dbutil/pgutil"
"storj.io/storj/versioncontrol"
)

Expand Down
4 changes: 2 additions & 2 deletions private/testplanet/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ import (

"go.uber.org/zap"

"storj.io/common/dbutil/pgtest"
"storj.io/common/dbutil/pgutil"
"storj.io/common/testcontext"
"storj.io/storj/private/testmonkit"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
"storj.io/storj/shared/dbutil/pgtest"
"storj.io/storj/shared/dbutil/pgutil"
)

// Run runs testplanet in multiple configurations.
Expand Down
2 changes: 1 addition & 1 deletion private/testplanet/run_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ import (

"go.uber.org/zap"

"storj.io/common/dbutil/pgtest"
"storj.io/common/testcontext"
"storj.io/storj/private/testmonkit"
"storj.io/storj/private/testplanet"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
"storj.io/storj/shared/dbutil/pgtest"
)

func TestRun(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion satellite/metabase/accounting.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"context"
"time"

"storj.io/common/tagsql"
"storj.io/storj/shared/tagsql"
)

// BucketTally contains information about aggregate data stored in a bucket.
Expand Down

0 comments on commit 71c0c2d

Please sign in to comment.