diff --git a/internal/migrate/contracts.go b/internal/migrate/contracts.go deleted file mode 100644 index cb0aff73..00000000 --- a/internal/migrate/contracts.go +++ /dev/null @@ -1,143 +0,0 @@ -package migrate - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "time" - - "gitlab.com/NebulousLabs/bolt" - rhp2 "go.sia.tech/core/rhp/v2" - "go.sia.tech/core/types" - "go.sia.tech/hostd/host/contracts" - stypes "go.sia.tech/siad/types" - "go.uber.org/zap" -) - -type ( - storageObligation struct { - // Storage obligations are broken up into ordered atomic sectors that are - // exactly 4MiB each. By saving the roots of each sector, storage proofs - // and modifications to the data can be made inexpensively by making use of - // the merkletree.CachedTree. Sectors can be appended, modified, or deleted - // and the host can recompute the Merkle root of the whole file without - // much computational or I/O expense. - SectorRoots []types.Hash256 - - // The negotiation height specifies the block height at which the file - // contract was negotiated. If the origin transaction set is not accepted - // onto the blockchain quickly enough, the contract is pruned from the - // host. The origin and revision transaction set contain the contracts + - // revisions as well as all parent transactions. The parents are necessary - // because after a restart the transaction pool may be emptied out. - NegotiationHeight uint64 - OriginTransactionSet []stypes.Transaction - RevisionTransactionSet []stypes.Transaction - } -) - -func migrateContracts(ctx context.Context, db Store, dir string, maxHeight uint64, log *zap.Logger) (success, skipped, failed int, err error) { - crapDB, err := bolt.Open(filepath.Join(dir, "host", "host.db"), 0600, &bolt.Options{Timeout: 3 * time.Second}) - if err != nil { - return 0, 0, 0, fmt.Errorf("failed to open host database: %w", err) - } - defer crapDB.Close() - - contractBucket := []byte("BucketStorageObligations") - - // iterate over all contracts - err = crapDB.View(func(tx *bolt.Tx) error { - select { - case <-ctx.Done(): - return ctx.Err() // abort - default: - } - - bucket := tx.Bucket(contractBucket) - if bucket == nil { - return fmt.Errorf("bucket %s not found", contractBucket) - } - - return bucket.ForEach(func(k, v []byte) error { - // contract parsing/corruption errors are logged and ignored - fcID := (types.FileContractID)(k) - - log := log.With(zap.Stringer("contractID", fcID)) - - var so storageObligation - if err := json.Unmarshal(v, &so); err != nil { - log.Warn("failed to decode storage obligation", zap.Error(err)) - failed++ - return nil - } else if len(so.RevisionTransactionSet) == 0 { - // all contracts should have at least one revision - log.Warn("contract has no revisions") - failed++ - return nil - } - - var revisionTxn types.Transaction - convertToCore(so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1], &revisionTxn) - if len(revisionTxn.FileContractRevisions) != 1 { - log.Warn("revision transaction has no revisions") - failed++ - return nil - } else if len(revisionTxn.Signatures) != 2 { - log.Warn("revision transaction has no signatures") - failed++ - return nil - } - - revision := revisionTxn.FileContractRevisions[0] - // validate that the revision makes sense - if revision.ParentID != fcID { - log.Warn("revision parent id does not match contract id", zap.Stringer("parentID", revision.ParentID)) - failed++ - return nil - } else if revision.WindowEnd < maxHeight { - log.Debug("skipping expired contract", zap.Uint64("proofWindowEnd", revision.WindowEnd)) - skipped++ - return nil - } - - expectedRoot := rhp2.MetaRoot(so.SectorRoots) - if revision.FileMerkleRoot != expectedRoot { - log.Warn("revision merkle root does not match expected root", zap.Stringer("root", revision.FileMerkleRoot), zap.Stringer("expected", expectedRoot)) - failed++ - return nil - } - - expectedFileSize := uint64(len(so.SectorRoots)) * rhp2.SectorSize - if revision.Filesize != expectedFileSize { - log.Warn("revision filesize does not match expected filesize", zap.Uint64("size", revision.Filesize), zap.Uint64("expected", expectedFileSize)) - failed++ - return nil - } - - signedRevision := contracts.SignedRevision{ - Revision: revision, - RenterSignature: types.Signature(revisionTxn.Signatures[0].Signature), - HostSignature: types.Signature(revisionTxn.Signatures[1].Signature), - } - - var formationTxnSet []types.Transaction - for _, stxn := range so.OriginTransactionSet { - var txn types.Transaction - convertToCore(stxn, &txn) - formationTxnSet = append(formationTxnSet, txn) - } - - // contract is added with no financials or collateral because we - // can't trust siad to have done the math correctly - if err := db.SiadMigrateContract(signedRevision, so.NegotiationHeight, formationTxnSet, so.SectorRoots); err != nil { - return fmt.Errorf("failed to revise contract: %w", err) - } - log.Info("migrated contract", zap.Int("sectors", len(so.SectorRoots))) - success++ - return nil - }) - }) - - return -} diff --git a/internal/migrate/migrate.go b/internal/migrate/migrate.go deleted file mode 100644 index 4ec53a7a..00000000 --- a/internal/migrate/migrate.go +++ /dev/null @@ -1,137 +0,0 @@ -package migrate - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - - "gitlab.com/NebulousLabs/encoding" - "go.sia.tech/core/types" - "go.sia.tech/hostd/host/contracts" - "go.sia.tech/hostd/host/settings" - "go.sia.tech/siad/crypto" - "go.sia.tech/siad/modules" - "go.uber.org/zap" -) - -type ( - jsonHostSettings struct { - BlockHeight uint64 `json:"blockheight"` - SecretKey crypto.SecretKey `json:"secretkey"` - InternalSettings modules.HostInternalSettings `json:"settings"` - } - - // A Store is used to migrate a siad host to hostd's database format. - Store interface { - // Sectors - HasSector(root types.Hash256) (bool, error) - - // Config - UpdateSettings(settings settings.Settings) error - - // Migration - SiadMigrateHostKey(pk types.PrivateKey) error - SiadMigrateContract(revision contracts.SignedRevision, negotiationHeight uint64, formationSet []types.Transaction, sectors []types.Hash256) error - SiadMigrateNextVolumeIndex(volumeID int64) (volumeIndex int64, err error) - SiadMigrateStoredSector(volumeID, volumeIndex int64, sectorID types.Hash256) (err error) - SiadMigrateVolume(localPath string, maxSectors uint64) (volumeID int64, err error) - SiadUpdateMetrics() error - } -) - -func readHostConfig(ctx context.Context, dir string) (jsonHostSettings, error) { - f, err := os.Open(filepath.Join(dir, "host", "host.json")) - if err != nil { - return jsonHostSettings{}, fmt.Errorf("failed to open host.json: %w", err) - } - defer f.Close() - - dec := json.NewDecoder(f) - - // decode the header, but ignore it - var header string - if err := dec.Decode(&header); err != nil { - return jsonHostSettings{}, fmt.Errorf("failed to decode header: %w", err) - } else if err := dec.Decode(&header); err != nil { - return jsonHostSettings{}, fmt.Errorf("failed to decode header: %w", err) - } else if err := dec.Decode(&header); err != nil { - return jsonHostSettings{}, fmt.Errorf("failed to decode header: %w", err) - } - - var settings jsonHostSettings - if err := dec.Decode(&settings); err != nil { - return jsonHostSettings{}, fmt.Errorf("failed to decode settings: %w", err) - } - return settings, nil -} - -func convertToCore(siad encoding.SiaMarshaler, core types.DecoderFrom) { - var buf bytes.Buffer - siad.MarshalSia(&buf) - d := types.NewBufDecoder(buf.Bytes()) - core.DecodeFrom(d) - if d.Err() != nil { - panic(d.Err()) - } -} - -// Siad migrates a siad host to hostd. -func Siad(ctx context.Context, db Store, dir string, destructive bool, log *zap.Logger) error { - oldConfig, err := readHostConfig(ctx, dir) - if err != nil { - return fmt.Errorf("failed to read host config: %w", err) - } - - // migrate host key - privateKey := types.PrivateKey(oldConfig.SecretKey[:]) - if err := db.SiadMigrateHostKey(privateKey); err != nil { - return fmt.Errorf("failed to set host key: %w", err) - } - - log.Info("Migrated host key", zap.Stringer("key", privateKey.PublicKey())) - - newSettings := settings.DefaultSettings - newSettings.AcceptingContracts = oldConfig.InternalSettings.AcceptingContracts - newSettings.NetAddress = string(oldConfig.InternalSettings.NetAddress) - // pricing - convertToCore(&oldConfig.InternalSettings.MinContractPrice, &newSettings.ContractPrice) - convertToCore(&oldConfig.InternalSettings.MinBaseRPCPrice, &newSettings.BaseRPCPrice) - convertToCore(&oldConfig.InternalSettings.MinSectorAccessPrice, &newSettings.SectorAccessPrice) - convertToCore(&oldConfig.InternalSettings.MinStoragePrice, &newSettings.StoragePrice) - convertToCore(&oldConfig.InternalSettings.MinUploadBandwidthPrice, &newSettings.IngressPrice) - convertToCore(&oldConfig.InternalSettings.MinDownloadBandwidthPrice, &newSettings.EgressPrice) - newSettings.CollateralMultiplier = 2 - // limits - convertToCore(&oldConfig.InternalSettings.MaxCollateral, &newSettings.MaxCollateral) - newSettings.MaxContractDuration = uint64(oldConfig.InternalSettings.MaxDuration) - - // migrate settings - if err := db.UpdateSettings(newSettings); err != nil { - return fmt.Errorf("failed to set settings: %w", err) - } - - log.Info("Migrated settings") - - // migrate contracts - success, skipped, failures, err := migrateContracts(ctx, db, dir, oldConfig.BlockHeight, log.Named("contracts")) - if err != nil { - return fmt.Errorf("failed to migrate contracts: %w", err) - } - log.Info("Migrated contracts", zap.Int("failed", failures), zap.Int("successful", success), zap.Int("skipped", skipped)) - - // migrate storage folders - if err := migrateStorageFolders(ctx, db, dir, destructive, log.Named("storage")); err != nil { - return fmt.Errorf("failed to migrate storage folders: %w", err) - } - log.Info("Migrated storage folders") - - if err := db.SiadUpdateMetrics(); err != nil { - return fmt.Errorf("failed to update metrics: %w", err) - } - - log.Info("Migration complete") - return nil -} diff --git a/internal/migrate/migrate_test.go b/internal/migrate/migrate_test.go deleted file mode 100644 index 9b856242..00000000 --- a/internal/migrate/migrate_test.go +++ /dev/null @@ -1,1295 +0,0 @@ -//go:build ignore - -// this file is ignored because the tests will not run outside of localhost. -// siad uses a different sector size for testing that is not compatible with -// hostd's sector size. Since the tests require a siad node, they will only run -// on localhost, after modifying siad's sector size to match hostd's. - -package migrate_test - -import ( - "context" - "fmt" - "os" - "path/filepath" - "testing" - "time" - - rhp2 "go.sia.tech/core/rhp/v2" - "go.sia.tech/core/types" - "go.sia.tech/hostd/host/contracts" - "go.sia.tech/hostd/internal/migrate" - "go.sia.tech/hostd/internal/test" - "go.sia.tech/hostd/persist/sqlite" - "go.sia.tech/siad/modules" - stypes "go.sia.tech/siad/types" - "go.uber.org/zap/zaptest" - "lukechampine.com/frand" -) - -func TestMigrate(t *testing.T) { - log := zaptest.NewLogger(t) - hostDir := t.TempDir() - siad, err := startSiad(hostDir) - if err != nil { - t.Fatal(err) - } - defer siad.Close() - - err = siad.Host().SetInternalSettings(modules.HostInternalSettings{ - AcceptingContracts: true, - WindowSize: 10, - MaxDuration: 100, - MaxCollateral: stypes.SiacoinPrecision.Mul64(10000), - CollateralBudget: stypes.SiacoinPrecision.Mul64(100000), - MinStoragePrice: stypes.NewCurrency64(1), - Collateral: stypes.NewCurrency64(1), - }) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 3; i++ { - if err := siad.Host().AddStorageFolder(t.TempDir(), rhp2.SectorSize*64); err != nil { - t.Fatal(err) - } - } - - hostKey := types.PublicKey(siad.Host().PublicKey().Key) - hostUC, err := siad.Wallet().NextAddress() - if err != nil { - t.Fatal(err) - } - hostWalletAddr := hostUC.UnlockHash() - hostAddr := string(siad.Host().ExternalSettings().NetAddress) - - // create a renter - renterKey := types.NewPrivateKeyFromSeed(frand.Bytes(32)) - renterNode, err := test.NewNode(t.TempDir()) - if err != nil { - t.Fatal(err) - } - defer renterNode.Close() - - renter, err := test.NewRenter(renterKey, t.TempDir(), renterNode, log.Named("renter")) - if err != nil { - t.Fatal(err) - } - defer renter.Close() - - if err := siad.Gateway().Connect(modules.NetAddress(renterNode.GatewayAddr())); err != nil { - t.Fatal(err) - } - - miner := test.NewMiner(renterNode.ChainManager()) - if err := renterNode.ChainManager().Subscribe(miner, modules.ConsensusChangeBeginning, nil); err != nil { - t.Fatal(err) - } - renterNode.TPool().Subscribe(miner) - - // mine enough blocks to fund both wallets - if err := miner.Mine(types.Address(hostWalletAddr), 20); err != nil { - t.Fatal(err) - } else if err := miner.Mine(types.Address(renter.Wallet().Address()), 20); err != nil { - t.Fatal(err) - } - - time.Sleep(time.Second) - - // form a few contracts with the host and upload some data with them - for i := 0; i < 10; i++ { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - revision, err := renter.FormContract(ctx, hostAddr, hostKey, types.Siacoins(1000), types.Siacoins(1000), 20) - if err != nil { - t.Fatal(err) - } - - err = func() error { - session, err := renter.NewRHP2Session(ctx, hostAddr, hostKey, revision.ID()) - if err != nil { - return fmt.Errorf("failed to create rhp2 session: %w", err) - } - defer session.Close() - - var sector [rhp2.SectorSize]byte - for j := 0; j < 10; j++ { - frand.Read(sector[:]) - - root := rhp2.SectorRoot(§or) - - returnedRoot, err := session.Append(ctx, §or, types.Siacoins(1), types.ZeroCurrency) - if err != nil { - return fmt.Errorf("failed to upload sector: %w", err) - } else if root != returnedRoot { - return fmt.Errorf("root mismatch: expected %v, got %v", root, returnedRoot) - } - } - return nil - }() - if err != nil { - t.Fatal(err) - } - } - - // mine a block to confirm all the contracts - if err := miner.Mine(types.Address(renter.Wallet().Address()), 1); err != nil { - t.Fatal(err) - } - - time.Sleep(time.Second) - - // mine until the contracts have expired - for i := 0; i < 50; i++ { - if err := miner.Mine(types.Address(renter.Wallet().Address()), 1); err != nil { - t.Fatal(err) - } - - time.Sleep(50 * time.Millisecond) - } - - // form additional contracts with the host and upload some data with them - var activeContracts []types.FileContractID - var activeSectors []types.Hash256 - for i := 0; i < 10; i++ { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - revision, err := renter.FormContract(ctx, hostAddr, hostKey, types.Siacoins(1000), types.Siacoins(1000), 20) - if err != nil { - t.Fatal(err) - } - activeContracts = append(activeContracts, revision.ID()) - - err = func() error { - session, err := renter.NewRHP2Session(ctx, hostAddr, hostKey, revision.ID()) - if err != nil { - return fmt.Errorf("failed to create rhp2 session: %w", err) - } - defer session.Close() - - var sector [rhp2.SectorSize]byte - for j := 0; j < 10; j++ { - frand.Read(sector[:]) - - root := rhp2.SectorRoot(§or) - - returnedRoot, err := session.Append(ctx, §or, types.Siacoins(1), types.ZeroCurrency) - if err != nil { - return fmt.Errorf("failed to upload sector: %w", err) - } else if root != returnedRoot { - return fmt.Errorf("root mismatch: expected %v, got %v", root, returnedRoot) - } - activeSectors = append(activeSectors, root) - } - return nil - }() - if err != nil { - t.Fatal(err) - } - } - - // mine a block to confirm all the contracts - if err := miner.Mine(types.Address(renter.Wallet().Address()), 1); err != nil { - t.Fatal(err) - } - - time.Sleep(time.Second) - - // shutdown the siad node - if err := siad.Close(); err != nil { - t.Fatal(err) - } - - // migrate the host - db, err := sqlite.OpenDatabase(filepath.Join(hostDir, "hostd.db"), log.Named("sqlite")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - if err := migrate.Siad(context.Background(), db, hostDir, false, log.Named("migrate")); err != nil { - t.Fatal(err) - } - - // start a hostd node using the renter's wallet key so it has existing funds - hostNode, err := test.NewEmptyHost(renterKey, hostDir, renterNode, log.Named("hostd")) - if err != nil { - t.Fatal(err) - } - defer hostNode.Close() - - settings, err := hostNode.RHP2Settings() - if err != nil { - t.Fatal(err) - } else if settings.TotalStorage != 3*rhp2.SectorSize*64 { - t.Fatalf("expected host to have %d total storage, got %d", 3*rhp2.SectorSize*64, settings.TotalStorage) - } else if settings.AcceptingContracts != true { - t.Fatal("expected host to be accepting contracts") - } else if settings.MaxDuration != 100 { - t.Fatalf("expected host to have max duration of 100, got %v", settings.MaxDuration) - } - - expectedUsage := uint64(10 * 10) // 10 contracts * 10 sectors - expectedTotal := uint64(3 * 64) // 3 folder * 64 sectors - - usedSectors, totalSectors, err := hostNode.Storage().Usage() - if err != nil { - t.Fatal(err) - } else if usedSectors != expectedUsage { - t.Fatalf("expected host to have %d used sectors, got %v", expectedUsage, usedSectors) - } else if totalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, totalSectors) - } - - // check the host's metrics - metrics, err := hostNode.Store().Metrics(time.Now()) - if err != nil { - t.Fatal(err) - } else if metrics.Storage.ContractSectors != expectedUsage { - t.Fatalf("expected host to have %d contract sectors, got %v", expectedUsage, metrics.Storage.ContractSectors) - } else if metrics.Storage.TotalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, metrics.Storage.TotalSectors) - } else if metrics.Storage.PhysicalSectors != expectedUsage { - t.Fatalf("expected host to have %d physical sectors, got %v", expectedUsage, metrics.Storage.PhysicalSectors) - } else if metrics.Contracts.Active != 10 { - t.Fatalf("expected host to have 10 active contracts, got %v", metrics.Contracts.Active) - } else if metrics.Contracts.Pending != 0 { - t.Fatalf("expected host to have 0 total contracts, got %v", metrics.Contracts.Pending) - } else if metrics.Contracts.Successful != 0 { - t.Fatalf("expected host to have 0 successful contracts, got %v", metrics.Contracts.Successful) - } - - if hostNode.Store().HostKey().PublicKey() != hostKey { - t.Fatal("host key mismatch") - } - - for _, root := range activeSectors { - data, err := hostNode.Storage().Read(root) - if err != nil { - t.Fatal(err) - } else if rhp2.SectorRoot(data) != root { - t.Fatalf("root mismatch: expected %v, got %v", root, rhp2.SectorRoot(data)) - } - } - - var maxWindowEnd uint64 - for _, contractID := range activeContracts { - fc, err := hostNode.Store().Contract(contractID) - if err != nil { - t.Fatal(err) - } else if fc.Revision.Filesize != 10*rhp2.SectorSize { - t.Fatalf("expected contract to have 10 sectors, got %v", fc.Revision.Filesize/rhp2.SectorSize) - } - - if fc.Revision.WindowEnd > maxWindowEnd { - maxWindowEnd = fc.Revision.WindowEnd - } - } - - waitForSync := func() { - // wait for the host node to sync - for { - if hostNode.Contracts().ScanHeight() == hostNode.ChainManager().TipState().Index.Height { - break - } - time.Sleep(100 * time.Millisecond) - } - } - - waitForSync() - - // mine a block and wait for it to be processed - for { - if err := miner.Mine(types.Address(renter.Wallet().Address()), 1); err != nil { - t.Fatal(err) - } - - waitForSync() - - if hostNode.Contracts().ScanHeight() >= maxWindowEnd+10 { - break - } - } - - time.Sleep(time.Second) - - // check that all contracts are now successful - c, _, err := hostNode.Store().Contracts(contracts.ContractFilter{}) - if err != nil { - t.Fatal(err) - } - - for _, contract := range c { - if contract.Status != contracts.ContractStatusSuccessful { - t.Fatalf("expected contract %v to be successful, got %v", contract.Revision.ParentID, contract.Status) - } - } - - // check that everything was cleaned up - usedSectors, totalSectors, err = hostNode.Storage().Usage() - if err != nil { - t.Fatal(err) - } else if usedSectors != 0 { - t.Fatalf("expected host to have %d used sectors, got %v", 0, usedSectors) - } else if totalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, totalSectors) - } - - // check the host's metrics - metrics, err = hostNode.Store().Metrics(time.Now()) - if err != nil { - t.Fatal(err) - } else if metrics.Storage.ContractSectors != 0 { - t.Fatalf("expected host to have %d contract sectors, got %v", 0, metrics.Storage.ContractSectors) - } else if metrics.Storage.TotalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, metrics.Storage.TotalSectors) - } else if metrics.Storage.PhysicalSectors != 0 { - t.Fatalf("expected host to have %d physical sectors, got %v", 0, metrics.Storage.PhysicalSectors) - } else if metrics.Contracts.Active != 0 { - t.Fatalf("expected host to have 0 active contracts, got %v", metrics.Contracts.Active) - } else if metrics.Contracts.Pending != 0 { - t.Fatalf("expected host to have 0 total contracts, got %v", metrics.Contracts.Pending) - } else if metrics.Contracts.Successful != 10 { - t.Fatalf("expected host to have 10 successful contracts, got %v", metrics.Contracts.Successful) - } -} - -func TestPartialMigrate(t *testing.T) { - log := zaptest.NewLogger(t) - hostDir := t.TempDir() - siad, err := startSiad(hostDir) - if err != nil { - t.Fatal(err) - } - defer siad.Close() - - err = siad.Host().SetInternalSettings(modules.HostInternalSettings{ - AcceptingContracts: true, - WindowSize: 10, - MaxDuration: 100, - MaxCollateral: stypes.SiacoinPrecision.Mul64(10000), - CollateralBudget: stypes.SiacoinPrecision.Mul64(100000), - MinStoragePrice: stypes.NewCurrency64(1), - Collateral: stypes.NewCurrency64(1), - }) - if err != nil { - t.Fatal(err) - } - - var folderPaths []string - for i := 0; i < 3; i++ { - fp := t.TempDir() - if err := siad.Host().AddStorageFolder(fp, rhp2.SectorSize*64); err != nil { - t.Fatal(err) - } - folderPaths = append(folderPaths, fp) - } - - hostKey := types.PublicKey(siad.Host().PublicKey().Key) - hostUC, err := siad.Wallet().NextAddress() - if err != nil { - t.Fatal(err) - } - hostWalletAddr := hostUC.UnlockHash() - hostAddr := string(siad.Host().ExternalSettings().NetAddress) - - // create a renter - renterKey := types.NewPrivateKeyFromSeed(frand.Bytes(32)) - renterNode, err := test.NewNode(t.TempDir()) - if err != nil { - t.Fatal(err) - } - defer renterNode.Close() - - renter, err := test.NewRenter(renterKey, t.TempDir(), renterNode, log.Named("renter")) - if err != nil { - t.Fatal(err) - } - defer renter.Close() - - if err := siad.Gateway().Connect(modules.NetAddress(renterNode.GatewayAddr())); err != nil { - t.Fatal(err) - } - - miner := test.NewMiner(renterNode.ChainManager()) - if err := renterNode.ChainManager().Subscribe(miner, modules.ConsensusChangeBeginning, nil); err != nil { - t.Fatal(err) - } - renterNode.TPool().Subscribe(miner) - - // mine enough blocks to fund both wallets - if err := miner.Mine(types.Address(hostWalletAddr), 20); err != nil { - t.Fatal(err) - } else if err := miner.Mine(types.Address(renter.Wallet().Address()), 20); err != nil { - t.Fatal(err) - } - - time.Sleep(5 * time.Second) - - var activeContracts []types.FileContractID - var activeSectors []types.Hash256 - - // form a few contracts with the host and upload some data with them - for i := 0; i < 10; i++ { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - revision, err := renter.FormContract(ctx, hostAddr, hostKey, types.Siacoins(1000), types.Siacoins(1000), 20) - if err != nil { - t.Fatal(err) - } - activeContracts = append(activeContracts, revision.ID()) - - err = func() error { - session, err := renter.NewRHP2Session(ctx, hostAddr, hostKey, revision.ID()) - if err != nil { - return fmt.Errorf("failed to create rhp2 session: %w", err) - } - defer session.Close() - - var sector [rhp2.SectorSize]byte - for j := 0; j < 10; j++ { - frand.Read(sector[:]) - - root := rhp2.SectorRoot(§or) - - returnedRoot, err := session.Append(ctx, §or, types.Siacoins(1), types.ZeroCurrency) - if err != nil { - return fmt.Errorf("failed to upload sector: %w", err) - } else if root != returnedRoot { - return fmt.Errorf("root mismatch: expected %v, got %v", root, returnedRoot) - } - activeSectors = append(activeSectors, root) - } - return nil - }() - if err != nil { - t.Fatal(err) - } - } - - // mine a block to confirm all the contracts - if err := miner.Mine(types.Address(renter.Wallet().Address()), 1); err != nil { - t.Fatal(err) - } - - // shutdown the siad node - if err := siad.Close(); err != nil { - t.Fatal(err) - } - - // rename one of the storage folders so it can't be found - oldPath := folderPaths[2] - newPath := oldPath + "missing" - if err := os.Rename(oldPath, newPath); err != nil { - t.Fatal(err) - } - - // migrate the host - db, err := sqlite.OpenDatabase(filepath.Join(hostDir, "hostd.db"), log.Named("sqlite")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - // migration will fail - if err := migrate.Siad(context.Background(), db, hostDir, false, log.Named("migrate")); err == nil { - t.Fatalf("migration should fail due to missing folder") - } - - // move the folder back - if err := os.Rename(newPath, oldPath); err != nil { - t.Fatal(err) - } - - dir, err := os.ReadDir(oldPath) - if err != nil { - t.Fatal(err) - } - for _, fi := range dir { - log.Info(fi.Name()) - } - - // try the migration again, should succeed - if err := migrate.Siad(context.Background(), db, hostDir, false, log.Named("migrate")); err != nil { - t.Fatal(err) - } - - // start a hostd node using the renter's wallet key so it has existing funds - hostNode, err := test.NewEmptyHost(renterKey, hostDir, renterNode, log.Named("hostd")) - if err != nil { - t.Fatal(err) - } - defer hostNode.Close() - - settings, err := hostNode.RHP2Settings() - if err != nil { - t.Fatal(err) - } else if settings.TotalStorage != 3*rhp2.SectorSize*64 { - t.Fatalf("expected host to have %d total storage, got %d", 3*rhp2.SectorSize*64, settings.TotalStorage) - } else if settings.AcceptingContracts != true { - t.Fatal("expected host to be accepting contracts") - } else if settings.MaxDuration != 100 { - t.Fatalf("expected host to have max duration of 100, got %v", settings.MaxDuration) - } - - expectedUsage := uint64(10 * 10) // 10 contracts * 10 sectors - expectedTotal := uint64(3 * 64) // 3 folder * 64 sectors - - usedSectors, totalSectors, err := hostNode.Storage().Usage() - if err != nil { - t.Fatal(err) - } else if usedSectors != expectedUsage { - t.Fatalf("expected host to have %d used sectors, got %v", expectedUsage, usedSectors) - } else if totalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, totalSectors) - } - - // check the host's metrics - metrics, err := hostNode.Store().Metrics(time.Now()) - if err != nil { - t.Fatal(err) - } else if metrics.Storage.ContractSectors != expectedUsage { - t.Fatalf("expected host to have %d contract sectors, got %v", expectedUsage, metrics.Storage.ContractSectors) - } else if metrics.Storage.TotalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, metrics.Storage.TotalSectors) - } else if metrics.Storage.PhysicalSectors != expectedUsage { - t.Fatalf("expected host to have %d physical sectors, got %v", expectedUsage, metrics.Storage.PhysicalSectors) - } else if metrics.Contracts.Active != 10 { - t.Fatalf("expected host to have 10 active contracts, got %v", metrics.Contracts.Active) - } else if metrics.Contracts.Pending != 0 { - t.Fatalf("expected host to have 0 total contracts, got %v", metrics.Contracts.Pending) - } else if metrics.Contracts.Successful != 0 { - t.Fatalf("expected host to have 0 successful contracts, got %v", metrics.Contracts.Successful) - } - - if hostNode.Store().HostKey().PublicKey() != hostKey { - t.Fatal("host key mismatch") - } - - for _, root := range activeSectors { - data, err := hostNode.Storage().Read(root) - if err != nil { - t.Fatal(err) - } else if rhp2.SectorRoot(data) != root { - t.Fatalf("root mismatch: expected %v, got %v", root, rhp2.SectorRoot(data)) - } - } - - var maxWindowEnd uint64 - for _, contractID := range activeContracts { - fc, err := hostNode.Store().Contract(contractID) - if err != nil { - t.Fatal(err) - } else if fc.Revision.Filesize != 10*rhp2.SectorSize { - t.Fatalf("expected contract to have 10 sectors, got %v", fc.Revision.Filesize/rhp2.SectorSize) - } - - if fc.Revision.WindowEnd > maxWindowEnd { - maxWindowEnd = fc.Revision.WindowEnd - } - } - - waitForSync := func() { - // wait for the host node to sync - for { - if hostNode.Contracts().ScanHeight() == hostNode.ChainManager().TipState().Index.Height { - break - } - time.Sleep(100 * time.Millisecond) - } - } - - waitForSync() - - // mine a block and wait for it to be processed - for { - if err := miner.Mine(types.Address(renter.Wallet().Address()), 1); err != nil { - t.Fatal(err) - } - - waitForSync() - - if hostNode.Contracts().ScanHeight() >= maxWindowEnd+10 { - break - } - } - - time.Sleep(time.Second) - - // check that all contracts are now successful - c, _, err := hostNode.Store().Contracts(contracts.ContractFilter{}) - if err != nil { - t.Fatal(err) - } - - for _, contract := range c { - if contract.Status != contracts.ContractStatusSuccessful { - t.Fatalf("expected contract %v to be successful, got %v", contract.Revision.ParentID, contract.Status) - } - } - - // check that everything was cleaned up - usedSectors, totalSectors, err = hostNode.Storage().Usage() - if err != nil { - t.Fatal(err) - } else if usedSectors != 0 { - t.Fatalf("expected host to have %d used sectors, got %v", 0, usedSectors) - } else if totalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, totalSectors) - } - - // check the host's metrics - metrics, err = hostNode.Store().Metrics(time.Now()) - if err != nil { - t.Fatal(err) - } else if metrics.Storage.ContractSectors != 0 { - t.Fatalf("expected host to have %d contract sectors, got %v", 0, metrics.Storage.ContractSectors) - } else if metrics.Storage.TotalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, metrics.Storage.TotalSectors) - } else if metrics.Storage.PhysicalSectors != 0 { - t.Fatalf("expected host to have %d physical sectors, got %v", 0, metrics.Storage.PhysicalSectors) - } else if metrics.Contracts.Active != 0 { - t.Fatalf("expected host to have 0 active contracts, got %v", metrics.Contracts.Active) - } else if metrics.Contracts.Pending != 0 { - t.Fatalf("expected host to have 0 total contracts, got %v", metrics.Contracts.Pending) - } else if metrics.Contracts.Successful != 10 { - t.Fatalf("expected host to have 10 successful contracts, got %v", metrics.Contracts.Successful) - } -} - -func TestDestructiveMigrate(t *testing.T) { - log := zaptest.NewLogger(t) - hostDir := t.TempDir() - siad, err := startSiad(hostDir) - if err != nil { - t.Fatal(err) - } - defer siad.Close() - - err = siad.Host().SetInternalSettings(modules.HostInternalSettings{ - AcceptingContracts: true, - WindowSize: 10, - MaxDuration: 100, - MaxCollateral: stypes.SiacoinPrecision.Mul64(10000), - CollateralBudget: stypes.SiacoinPrecision.Mul64(100000), - MinStoragePrice: stypes.NewCurrency64(1), - Collateral: stypes.NewCurrency64(1), - }) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 3; i++ { - if err := siad.Host().AddStorageFolder(t.TempDir(), rhp2.SectorSize*64); err != nil { - t.Fatal(err) - } - } - - hostKey := types.PublicKey(siad.Host().PublicKey().Key) - hostUC, err := siad.Wallet().NextAddress() - if err != nil { - t.Fatal(err) - } - hostWalletAddr := hostUC.UnlockHash() - hostAddr := string(siad.Host().ExternalSettings().NetAddress) - - // create a renter - renterKey := types.NewPrivateKeyFromSeed(frand.Bytes(32)) - renterNode, err := test.NewNode(t.TempDir()) - if err != nil { - t.Fatal(err) - } - defer renterNode.Close() - - renter, err := test.NewRenter(renterKey, t.TempDir(), renterNode, log.Named("renter")) - if err != nil { - t.Fatal(err) - } - defer renter.Close() - - if err := siad.Gateway().Connect(modules.NetAddress(renterNode.GatewayAddr())); err != nil { - t.Fatal(err) - } - - miner := test.NewMiner(renterNode.ChainManager()) - if err := renterNode.ChainManager().Subscribe(miner, modules.ConsensusChangeBeginning, nil); err != nil { - t.Fatal(err) - } - renterNode.TPool().Subscribe(miner) - - // mine enough blocks to fund both wallets - if err := miner.Mine(types.Address(hostWalletAddr), 20); err != nil { - t.Fatal(err) - } else if err := miner.Mine(types.Address(renter.Wallet().Address()), 20); err != nil { - t.Fatal(err) - } - - time.Sleep(5 * time.Second) - - var activeContracts []types.FileContractID - var activeSectors []types.Hash256 - - // form a few contracts with the host and upload some data with them - for i := 0; i < 10; i++ { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - revision, err := renter.FormContract(ctx, hostAddr, hostKey, types.Siacoins(1000), types.Siacoins(1000), 20) - if err != nil { - t.Fatal(err) - } - activeContracts = append(activeContracts, revision.ID()) - - err = func() error { - session, err := renter.NewRHP2Session(ctx, hostAddr, hostKey, revision.ID()) - if err != nil { - return fmt.Errorf("failed to create rhp2 session: %w", err) - } - defer session.Close() - - var sector [rhp2.SectorSize]byte - for j := 0; j < 10; j++ { - frand.Read(sector[:]) - - root := rhp2.SectorRoot(§or) - - returnedRoot, err := session.Append(ctx, §or, types.Siacoins(1), types.ZeroCurrency) - if err != nil { - return fmt.Errorf("failed to upload sector: %w", err) - } else if root != returnedRoot { - return fmt.Errorf("root mismatch: expected %v, got %v", root, returnedRoot) - } - activeSectors = append(activeSectors, root) - } - return nil - }() - if err != nil { - t.Fatal(err) - } - } - - // mine a block to confirm all the contracts - if err := miner.Mine(types.Address(renter.Wallet().Address()), 1); err != nil { - t.Fatal(err) - } - - // shutdown the siad node - if err := siad.Close(); err != nil { - t.Fatal(err) - } - - // migrate the host - db, err := sqlite.OpenDatabase(filepath.Join(hostDir, "hostd.db"), log.Named("sqlite")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - if err := migrate.Siad(context.Background(), db, hostDir, true, log.Named("migrate")); err != nil { - t.Fatal(err) - } - - // start a hostd node using the renter's wallet key so it has existing funds - hostNode, err := test.NewEmptyHost(renterKey, hostDir, renterNode, log.Named("hostd")) - if err != nil { - t.Fatal(err) - } - defer hostNode.Close() - - settings, err := hostNode.RHP2Settings() - if err != nil { - t.Fatal(err) - } else if settings.TotalStorage != 3*rhp2.SectorSize*64 { - t.Fatalf("expected host to have %d total storage, got %d", 3*rhp2.SectorSize*64, settings.TotalStorage) - } else if settings.AcceptingContracts != true { - t.Fatal("expected host to be accepting contracts") - } else if settings.MaxDuration != 100 { - t.Fatalf("expected host to have max duration of 100, got %v", settings.MaxDuration) - } - - expectedUsage := uint64(10 * 10) // 10 contracts * 10 sectors - expectedTotal := uint64(3 * 64) // 3 folder * 64 sectors - - usedSectors, totalSectors, err := hostNode.Storage().Usage() - if err != nil { - t.Fatal(err) - } else if usedSectors != expectedUsage { - t.Fatalf("expected host to have %d used sectors, got %v", expectedUsage, usedSectors) - } else if totalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, totalSectors) - } - - // check the host's metrics - metrics, err := hostNode.Store().Metrics(time.Now()) - if err != nil { - t.Fatal(err) - } else if metrics.Storage.ContractSectors != expectedUsage { - t.Fatalf("expected host to have %d contract sectors, got %v", expectedUsage, metrics.Storage.ContractSectors) - } else if metrics.Storage.TotalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, metrics.Storage.TotalSectors) - } else if metrics.Storage.PhysicalSectors != expectedUsage { - t.Fatalf("expected host to have %d physical sectors, got %v", expectedUsage, metrics.Storage.PhysicalSectors) - } else if metrics.Contracts.Active != 10 { - t.Fatalf("expected host to have 10 active contracts, got %v", metrics.Contracts.Active) - } else if metrics.Contracts.Pending != 0 { - t.Fatalf("expected host to have 0 total contracts, got %v", metrics.Contracts.Pending) - } else if metrics.Contracts.Successful != 0 { - t.Fatalf("expected host to have 0 successful contracts, got %v", metrics.Contracts.Successful) - } - - if hostNode.Store().HostKey().PublicKey() != hostKey { - t.Fatal("host key mismatch") - } - - for _, root := range activeSectors { - data, err := hostNode.Storage().Read(root) - if err != nil { - t.Fatal(err) - } else if rhp2.SectorRoot(data) != root { - t.Fatalf("root mismatch: expected %v, got %v", root, rhp2.SectorRoot(data)) - } - } - - var maxWindowEnd uint64 - for _, contractID := range activeContracts { - fc, err := hostNode.Store().Contract(contractID) - if err != nil { - t.Fatal(err) - } else if fc.Revision.Filesize != 10*rhp2.SectorSize { - t.Fatalf("expected contract to have 10 sectors, got %v", fc.Revision.Filesize/rhp2.SectorSize) - } - - if fc.Revision.WindowEnd > maxWindowEnd { - maxWindowEnd = fc.Revision.WindowEnd - } - } - - waitForSync := func() { - // wait for the host node to sync - for { - if hostNode.Contracts().ScanHeight() == hostNode.ChainManager().TipState().Index.Height { - break - } - time.Sleep(100 * time.Millisecond) - } - } - - waitForSync() - - // mine a block and wait for it to be processed - for { - if err := miner.Mine(types.Address(renter.Wallet().Address()), 1); err != nil { - t.Fatal(err) - } - - waitForSync() - - if hostNode.Contracts().ScanHeight() >= maxWindowEnd+10 { - break - } - } - - time.Sleep(time.Second) - - // check that all contracts are now successful - c, _, err := hostNode.Store().Contracts(contracts.ContractFilter{}) - if err != nil { - t.Fatal(err) - } - - for _, contract := range c { - if contract.Status != contracts.ContractStatusSuccessful { - t.Fatalf("expected contract %v to be successful, got %v", contract.Revision.ParentID, contract.Status) - } - } - - // check that everything was cleaned up - usedSectors, totalSectors, err = hostNode.Storage().Usage() - if err != nil { - t.Fatal(err) - } else if usedSectors != 0 { - t.Fatalf("expected host to have %d used sectors, got %v", 0, usedSectors) - } else if totalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, totalSectors) - } - - // check the host's metrics - metrics, err = hostNode.Store().Metrics(time.Now()) - if err != nil { - t.Fatal(err) - } else if metrics.Storage.ContractSectors != 0 { - t.Fatalf("expected host to have %d contract sectors, got %v", 0, metrics.Storage.ContractSectors) - } else if metrics.Storage.TotalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, metrics.Storage.TotalSectors) - } else if metrics.Storage.PhysicalSectors != 0 { - t.Fatalf("expected host to have %d physical sectors, got %v", 0, metrics.Storage.PhysicalSectors) - } else if metrics.Contracts.Active != 0 { - t.Fatalf("expected host to have 0 active contracts, got %v", metrics.Contracts.Active) - } else if metrics.Contracts.Pending != 0 { - t.Fatalf("expected host to have 0 total contracts, got %v", metrics.Contracts.Pending) - } else if metrics.Contracts.Successful != 10 { - t.Fatalf("expected host to have 10 successful contracts, got %v", metrics.Contracts.Successful) - } -} - -func TestDestructivePartialMigrate(t *testing.T) { - log := zaptest.NewLogger(t) - hostDir := t.TempDir() - siad, err := startSiad(hostDir) - if err != nil { - t.Fatal(err) - } - defer siad.Close() - - err = siad.Host().SetInternalSettings(modules.HostInternalSettings{ - AcceptingContracts: true, - WindowSize: 10, - MaxDuration: 100, - MaxCollateral: stypes.SiacoinPrecision.Mul64(10000), - CollateralBudget: stypes.SiacoinPrecision.Mul64(100000), - MinStoragePrice: stypes.NewCurrency64(1), - Collateral: stypes.NewCurrency64(1), - }) - if err != nil { - t.Fatal(err) - } - - var folderPaths []string - for i := 0; i < 3; i++ { - fp := t.TempDir() - if err := siad.Host().AddStorageFolder(fp, rhp2.SectorSize*64); err != nil { - t.Fatal(err) - } - folderPaths = append(folderPaths, fp) - } - - hostKey := types.PublicKey(siad.Host().PublicKey().Key) - hostUC, err := siad.Wallet().NextAddress() - if err != nil { - t.Fatal(err) - } - hostWalletAddr := hostUC.UnlockHash() - hostAddr := string(siad.Host().ExternalSettings().NetAddress) - - // create a renter - renterKey := types.NewPrivateKeyFromSeed(frand.Bytes(32)) - renterNode, err := test.NewNode(t.TempDir()) - if err != nil { - t.Fatal(err) - } - defer renterNode.Close() - - renter, err := test.NewRenter(renterKey, t.TempDir(), renterNode, log.Named("renter")) - if err != nil { - t.Fatal(err) - } - defer renter.Close() - - if err := siad.Gateway().Connect(modules.NetAddress(renterNode.GatewayAddr())); err != nil { - t.Fatal(err) - } - - miner := test.NewMiner(renterNode.ChainManager()) - if err := renterNode.ChainManager().Subscribe(miner, modules.ConsensusChangeBeginning, nil); err != nil { - t.Fatal(err) - } - renterNode.TPool().Subscribe(miner) - - // mine enough blocks to fund both wallets - if err := miner.Mine(types.Address(hostWalletAddr), 20); err != nil { - t.Fatal(err) - } else if err := miner.Mine(types.Address(renter.Wallet().Address()), 20); err != nil { - t.Fatal(err) - } - - time.Sleep(5 * time.Second) - - // form a few contracts with the host and upload some data with them - for i := 0; i < 10; i++ { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - revision, err := renter.FormContract(ctx, hostAddr, hostKey, types.Siacoins(1000), types.Siacoins(1000), 20) - if err != nil { - t.Fatal(err) - } - - err = func() error { - session, err := renter.NewRHP2Session(ctx, hostAddr, hostKey, revision.ID()) - if err != nil { - return fmt.Errorf("failed to create rhp2 session: %w", err) - } - defer session.Close() - - var sector [rhp2.SectorSize]byte - for j := 0; j < 10; j++ { - frand.Read(sector[:]) - - root := rhp2.SectorRoot(§or) - - returnedRoot, err := session.Append(ctx, §or, types.Siacoins(1), types.ZeroCurrency) - if err != nil { - return fmt.Errorf("failed to upload sector: %w", err) - } else if root != returnedRoot { - return fmt.Errorf("root mismatch: expected %v, got %v", root, returnedRoot) - } - } - return nil - }() - if err != nil { - t.Fatal(err) - } - } - - // mine a block to confirm all the contracts - if err := miner.Mine(types.Address(renter.Wallet().Address()), 1); err != nil { - t.Fatal(err) - } - - time.Sleep(time.Second) - - // mine until the contracts have expired - for i := 0; i < 50; i++ { - if err := miner.Mine(types.Address(renter.Wallet().Address()), 1); err != nil { - t.Fatal(err) - } - - time.Sleep(time.Second) - } - - var activeContracts []types.FileContractID - var activeSectors []types.Hash256 - - // form additional contracts with the host and upload some data with them - for i := 0; i < 10; i++ { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - revision, err := renter.FormContract(ctx, hostAddr, hostKey, types.Siacoins(1000), types.Siacoins(1000), 20) - if err != nil { - t.Fatal(err) - } - activeContracts = append(activeContracts, revision.ID()) - - err = func() error { - session, err := renter.NewRHP2Session(ctx, hostAddr, hostKey, revision.ID()) - if err != nil { - return fmt.Errorf("failed to create rhp2 session: %w", err) - } - defer session.Close() - - var sector [rhp2.SectorSize]byte - for j := 0; j < 10; j++ { - frand.Read(sector[:]) - - root := rhp2.SectorRoot(§or) - - returnedRoot, err := session.Append(ctx, §or, types.Siacoins(1), types.ZeroCurrency) - if err != nil { - return fmt.Errorf("failed to upload sector: %w", err) - } else if root != returnedRoot { - return fmt.Errorf("root mismatch: expected %v, got %v", root, returnedRoot) - } - activeSectors = append(activeSectors, root) - } - return nil - }() - if err != nil { - t.Fatal(err) - } - } - - // mine a block to confirm all the contracts - if err := miner.Mine(types.Address(renter.Wallet().Address()), 1); err != nil { - t.Fatal(err) - } - - time.Sleep(time.Second) - - // shutdown the siad node - if err := siad.Close(); err != nil { - t.Fatal(err) - } - - // rename one of the storage folders so it can't be found - oldPath := folderPaths[2] - newPath := oldPath + "missing" - if err := os.Rename(oldPath, newPath); err != nil { - t.Fatal(err) - } - - // migrate the host - db, err := sqlite.OpenDatabase(filepath.Join(hostDir, "hostd.db"), log.Named("sqlite")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - // migration will fail - if err := migrate.Siad(context.Background(), db, hostDir, true, log.Named("migrate")); err == nil { - t.Fatalf("migration should fail due to missing folder") - } - - // move the folder back - if err := os.Rename(newPath, oldPath); err != nil { - t.Fatal(err) - } - - dir, err := os.ReadDir(oldPath) - if err != nil { - t.Fatal(err) - } - for _, fi := range dir { - log.Info(fi.Name()) - } - - // try the migration again, should succeed - if err := migrate.Siad(context.Background(), db, hostDir, true, log.Named("migrate")); err != nil { - t.Fatal(err) - } - - // start a hostd node using the renter's wallet key so it has existing funds - hostNode, err := test.NewEmptyHost(renterKey, hostDir, renterNode, log.Named("hostd")) - if err != nil { - t.Fatal(err) - } - defer hostNode.Close() - - settings, err := hostNode.RHP2Settings() - if err != nil { - t.Fatal(err) - } else if settings.TotalStorage != 3*rhp2.SectorSize*64 { - t.Fatalf("expected host to have %d total storage, got %d", 3*rhp2.SectorSize*64, settings.TotalStorage) - } else if settings.AcceptingContracts != true { - t.Fatal("expected host to be accepting contracts") - } else if settings.MaxDuration != 100 { - t.Fatalf("expected host to have max duration of 100, got %v", settings.MaxDuration) - } - - expectedUsage := uint64(10 * 10) // 10 contracts * 10 sectors - expectedTotal := uint64(3 * 64) // 3 folder * 64 sectors - - usedSectors, totalSectors, err := hostNode.Storage().Usage() - if err != nil { - t.Fatal(err) - } else if usedSectors != expectedUsage { - t.Fatalf("expected host to have %d used sectors, got %v", expectedUsage, usedSectors) - } else if totalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, totalSectors) - } - - // check the host's metrics - metrics, err := hostNode.Store().Metrics(time.Now()) - if err != nil { - t.Fatal(err) - } else if metrics.Storage.ContractSectors != expectedUsage { - t.Fatalf("expected host to have %d contract sectors, got %v", expectedUsage, metrics.Storage.ContractSectors) - } else if metrics.Storage.TotalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, metrics.Storage.TotalSectors) - } else if metrics.Storage.PhysicalSectors != expectedUsage { - t.Fatalf("expected host to have %d physical sectors, got %v", expectedUsage, metrics.Storage.PhysicalSectors) - } else if metrics.Contracts.Active != 10 { - t.Fatalf("expected host to have 10 active contracts, got %v", metrics.Contracts.Active) - } else if metrics.Contracts.Pending != 0 { - t.Fatalf("expected host to have 0 total contracts, got %v", metrics.Contracts.Pending) - } else if metrics.Contracts.Successful != 0 { - t.Fatalf("expected host to have 0 successful contracts, got %v", metrics.Contracts.Successful) - } - - if hostNode.Store().HostKey().PublicKey() != hostKey { - t.Fatal("host key mismatch") - } - - for _, root := range activeSectors { - data, err := hostNode.Storage().Read(root) - if err != nil { - t.Fatal(err) - } else if rhp2.SectorRoot(data) != root { - t.Fatalf("root mismatch: expected %v, got %v", root, rhp2.SectorRoot(data)) - } - } - - var maxWindowEnd uint64 - for _, contractID := range activeContracts { - fc, err := hostNode.Store().Contract(contractID) - if err != nil { - t.Fatal(err) - } else if fc.Revision.Filesize != 10*rhp2.SectorSize { - t.Fatalf("expected contract to have 10 sectors, got %v", fc.Revision.Filesize/rhp2.SectorSize) - } - - if fc.Revision.WindowEnd > maxWindowEnd { - maxWindowEnd = fc.Revision.WindowEnd - } - } - - waitForSync := func() { - // wait for the host node to sync - for { - if hostNode.Contracts().ScanHeight() == hostNode.ChainManager().TipState().Index.Height { - break - } - time.Sleep(100 * time.Millisecond) - } - } - - waitForSync() - - // mine a block and wait for it to be processed - for { - if err := miner.Mine(types.Address(renter.Wallet().Address()), 1); err != nil { - t.Fatal(err) - } - - waitForSync() - - if hostNode.Contracts().ScanHeight() >= maxWindowEnd+10 { - break - } - } - - time.Sleep(time.Second) - - // check that all contracts are now successful - c, _, err := hostNode.Store().Contracts(contracts.ContractFilter{}) - if err != nil { - t.Fatal(err) - } - - for _, contract := range c { - if contract.Status != contracts.ContractStatusSuccessful { - t.Fatalf("expected contract %v to be successful, got %v", contract.Revision.ParentID, contract.Status) - } - } - - // check that everything was cleaned up - usedSectors, totalSectors, err = hostNode.Storage().Usage() - if err != nil { - t.Fatal(err) - } else if usedSectors != 0 { - t.Fatalf("expected host to have %d used sectors, got %v", 0, usedSectors) - } else if totalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, totalSectors) - } - - // check the host's metrics - metrics, err = hostNode.Store().Metrics(time.Now()) - if err != nil { - t.Fatal(err) - } else if metrics.Storage.ContractSectors != 0 { - t.Fatalf("expected host to have %d contract sectors, got %v", 0, metrics.Storage.ContractSectors) - } else if metrics.Storage.TotalSectors != expectedTotal { - t.Fatalf("expected host to have %d total sectors, got %v", expectedTotal, metrics.Storage.TotalSectors) - } else if metrics.Storage.PhysicalSectors != 0 { - t.Fatalf("expected host to have %d physical sectors, got %v", 0, metrics.Storage.PhysicalSectors) - } else if metrics.Contracts.Active != 0 { - t.Fatalf("expected host to have 0 active contracts, got %v", metrics.Contracts.Active) - } else if metrics.Contracts.Pending != 0 { - t.Fatalf("expected host to have 0 total contracts, got %v", metrics.Contracts.Pending) - } else if metrics.Contracts.Successful != 10 { - t.Fatalf("expected host to have 10 successful contracts, got %v", metrics.Contracts.Successful) - } -} diff --git a/internal/migrate/node_test.go b/internal/migrate/node_test.go deleted file mode 100644 index 20a2055a..00000000 --- a/internal/migrate/node_test.go +++ /dev/null @@ -1,109 +0,0 @@ -//go:build ignore - -// this file is ignored because the tests will not run outside of localhost. -// siad uses a different sector size for testing that is not compatible with -// hostd's sector size. Since the tests require a siad node, they will only run -// on localhost, after modifying siad's sector size to match hostd's. - -package migrate_test - -import ( - "fmt" - "path/filepath" - - "gitlab.com/NebulousLabs/siamux" - "go.sia.tech/siad/crypto" - "go.sia.tech/siad/modules" - sconsensus "go.sia.tech/siad/modules/consensus" - sgateway "go.sia.tech/siad/modules/gateway" - shost "go.sia.tech/siad/modules/host" - stpool "go.sia.tech/siad/modules/transactionpool" - swallet "go.sia.tech/siad/modules/wallet" -) - -type siadNode struct { - gateway modules.Gateway - consensus modules.ConsensusSet - tpool modules.TransactionPool - wallet modules.Wallet - host modules.Host - mux *siamux.SiaMux -} - -func (sn *siadNode) Close() error { - sn.mux.Close() - sn.host.Close() - sn.wallet.Close() - sn.tpool.Close() - sn.consensus.Close() - sn.gateway.Close() - return nil -} - -func (sn *siadNode) Gateway() modules.Gateway { - return sn.gateway -} - -func (sn *siadNode) ConsensusSet() modules.ConsensusSet { - return sn.consensus -} - -func (sn *siadNode) TransactionPool() modules.TransactionPool { - return sn.tpool -} - -func (sn *siadNode) Wallet() modules.Wallet { - return sn.wallet -} - -func (sn *siadNode) Host() modules.Host { - return sn.host -} - -func startSiad(dir string) (*siadNode, error) { - g, err := sgateway.New(":0", false, filepath.Join(dir, "gateway")) - if err != nil { - return nil, fmt.Errorf("failed to create gateway: %v", err) - } - - cs, errCh := sconsensus.New(g, false, filepath.Join(dir, "consensus")) - if err := <-errCh; err != nil { - return nil, fmt.Errorf("failed to create consensus: %v", err) - } - - tp, err := stpool.New(cs, g, filepath.Join(dir, "transactionpool")) - if err != nil { - return nil, fmt.Errorf("failed to create transaction pool: %v", err) - } - - w, err := swallet.New(cs, tp, filepath.Join(dir, "wallet")) - if err != nil { - return nil, fmt.Errorf("failed to create wallet: %v", err) - } - - key := crypto.GenerateSiaKey(crypto.TypeDefaultWallet) - if _, err := w.Encrypt(key); err != nil { - return nil, fmt.Errorf("failed to encrypt wallet: %v", err) - } else if err := w.Unlock(key); err != nil { - return nil, fmt.Errorf("failed to unlock wallet: %v", err) - } - - mux, _, err := modules.NewSiaMux(filepath.Join(dir, "siamux"), dir, ":0", ":0") - if err != nil { - return nil, fmt.Errorf("failed to create siamux: %v", err) - } - - h, err := shost.New(cs, g, tp, w, mux, ":0", filepath.Join(dir, "host")) - if err != nil { - return nil, fmt.Errorf("failed to create host: %v", err) - } - - return &siadNode{ - gateway: g, - consensus: cs, - tpool: tp, - wallet: w, - host: h, - mux: mux, - }, nil -} diff --git a/internal/migrate/storage.go b/internal/migrate/storage.go deleted file mode 100644 index b2dfdb47..00000000 --- a/internal/migrate/storage.go +++ /dev/null @@ -1,205 +0,0 @@ -package migrate - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "sync" - "time" - - rhp2 "go.sia.tech/core/rhp/v2" - "go.sia.tech/core/types" - "go.uber.org/zap" -) - -type ( - storageFolder struct { - Index uint16 - Path string - Usage []uint64 - } - storageSettings struct { - SectorSalt types.Hash256 - StorageFolders []storageFolder - } -) - -// ErrMigrateSectorStored is return if a sector is already stored on the host -var ErrMigrateSectorStored = errors.New("sector already stored") - -func readStorageSettings(ctx context.Context, dir string) (storageSettings, error) { - f, err := os.Open(filepath.Join(dir, "host", "contractmanager", "contractmanager.json")) - if err != nil { - return storageSettings{}, fmt.Errorf("failed to open contractmanager.json: %w", err) - } - defer f.Close() - - dec := json.NewDecoder(f) - - // decode the header, but ignore it - var header string - if err := dec.Decode(&header); err != nil { - return storageSettings{}, fmt.Errorf("failed to decode header: %w", err) - } else if err := dec.Decode(&header); err != nil { - return storageSettings{}, fmt.Errorf("failed to decode header: %w", err) - } - - var settings storageSettings - if err := dec.Decode(&settings); err != nil { - return storageSettings{}, fmt.Errorf("failed to decode settings: %w", err) - } - return settings, nil -} - -func migrateFolderSectors(ctx context.Context, db Store, volumeID int64, maxSectors uint64, newPath, oldPath string, destructive bool, log *zap.Logger) error { - log = log.With(zap.String("newVolume", newPath), zap.String("oldFolder", oldPath)) - - // create the volume file and immediately close it to ensure it exists - dataFile, err := os.OpenFile(newPath, os.O_CREATE|os.O_RDWR, 0600) - if err != nil { - return fmt.Errorf("failed to create storage folder: %w", err) - } - defer dataFile.Close() - - oldDataFile, err := os.OpenFile(oldPath, os.O_RDWR, 0600) - if err != nil { - return fmt.Errorf("failed to open old storage folder: %w", err) - } - defer oldDataFile.Close() - - var sector [rhp2.SectorSize]byte - // read each sector from the file - for i := maxSectors; i > 0; i-- { - select { - case <-ctx.Done(): - return ctx.Err() // abort - default: - } - - startOffset := int64((i - 1) * rhp2.SectorSize) - log.Debug("migrating sector", zap.Uint64("index", i), zap.Uint64("startOffset", uint64(startOffset))) - if _, err := oldDataFile.ReadAt(sector[:], startOffset); err != nil && !errors.Is(err, io.EOF) { - return fmt.Errorf("failed to check sector: %w", err) - } - - root := rhp2.SectorRoot(§or) - log.Debug("read sector", zap.Stringer("root", root)) - - // check if the sector is stored on the host - if ok, err := db.HasSector(root); err != nil { - return fmt.Errorf("failed to check sector: %w", err) - } else if !ok { - // sector is not in use, skip it - // - // note: it would be better to check siad's usage array instead of - // reading everything from disk, but some sectors on our test node - // were marked as free when they were still being used, causing - // contract failures. - log.Debug("skipping unused sector", zap.Stringer("root", root)) - continue - } - - log.Debug("sector is in use", zap.Stringer("root", root)) - - // get the next available volume index - volumeIndex, err := db.SiadMigrateNextVolumeIndex(volumeID) - if err != nil { - return fmt.Errorf("failed to get sector index: %w", err) - } - - log.Debug("new sector index", zap.Int64("volumeIndex", volumeIndex)) - - // write the sector to the volume file - if _, err := dataFile.WriteAt(sector[:], volumeIndex*rhp2.SectorSize); err != nil { - return fmt.Errorf("failed to write sector: %w", err) - } else if err := dataFile.Sync(); err != nil { - return fmt.Errorf("failed to sync volume file: %w", err) - } - - log.Debug("wrote sector to volume file", zap.Stringer("root", root)) - - // update the sector index - err = db.SiadMigrateStoredSector(volumeID, volumeIndex, root) - if err != nil && !errors.Is(err, ErrMigrateSectorStored) { - return fmt.Errorf("failed to update sector index: %w", err) - } - - if destructive { - // shrink the old storage folder - if err := oldDataFile.Truncate(startOffset); err != nil { - return fmt.Errorf("failed to truncate old storage folder to %d: %w", startOffset, err) - } - } - log.Info("migrated sector", zap.Stringer("root", root)) - } - return nil -} - -func migrateStorageFolders(ctx context.Context, db Store, dir string, destructive bool, log *zap.Logger) error { - settings, err := readStorageSettings(ctx, dir) - if err != nil { - return fmt.Errorf("failed to read storage settings: %w", err) - } - - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - var wg sync.WaitGroup - errCh := make(chan error, 1) - - // add all storage folders to the database - for _, folder := range settings.StorageFolders { - newPath := filepath.Join(folder.Path, "data.dat") - - maxSectors := uint64(len(folder.Usage)) * 64 // 1 sector per bit - - volumeID, err := db.SiadMigrateVolume(newPath, maxSectors) - if err != nil { - return fmt.Errorf("failed to add storage folder: %w", err) - } - log.Info("migrated storage folder", zap.String("path", newPath), zap.Uint64("maxSectors", maxSectors)) - - wg.Add(1) - // migrate folder data - go func(ctx context.Context, volumeID int64, newPath, oldPath string) { - defer wg.Done() - - log.Info("migrating storage data", zap.String("newPath", newPath), zap.String("oldPath", oldPath)) - start := time.Now() - err := migrateFolderSectors(ctx, db, volumeID, maxSectors, newPath, oldPath, destructive, log) - if err != nil { - errCh <- fmt.Errorf("failed to migrate storage data %q: %w", oldPath, err) - } - log.Info("finished migrating storage data", zap.String("newPath", newPath), zap.String("oldPath", oldPath), zap.Duration("elapsed", time.Since(start))) - }(ctx, volumeID, newPath, filepath.Join(folder.Path, "siahostdata.dat")) - } - - go func() { - // wait for all migrations to complete, then close the error channel - wg.Wait() - close(errCh) - }() - - // check for errors - for err := range errCh { - return err - } - - if destructive { - for _, folder := range settings.StorageFolders { - oldDataPath := filepath.Join(folder.Path, "siahostdata.dat") - oldMetaPath := filepath.Join(folder.Path, "siahostmetadata.dat") - // delete the old storage folders - if err := os.Remove(oldDataPath); err != nil { - log.Warn("failed to remove old data file", zap.Error(err)) - } else if err := os.Remove(oldMetaPath); err != nil { - log.Warn("failed to remove old metadata file", zap.Error(err)) - } - } - } - return nil -}