diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index de5725ead0..f505acb9f6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,10 @@ name: Tests on: - - push - - pull_request + push: + branches: + - main + - stable-* + pull_request: permissions: contents: read @@ -45,7 +48,7 @@ jobs: - name: Install dependencies run: | - sudo add-apt-repository ppa:ubuntu-lxc/lxc-git-master -y --no-update + sudo add-apt-repository ppa:ubuntu-lxc/daily -y --no-update sudo add-apt-repository ppa:cowsql/stable -y --no-update sudo apt-get update @@ -171,7 +174,7 @@ jobs: - name: Install dependencies run: | set -x - sudo add-apt-repository ppa:ubuntu-lxc/lxc-git-master -y --no-update + sudo add-apt-repository ppa:ubuntu-lxc/daily -y --no-update sudo add-apt-repository ppa:cowsql/stable -y --no-update sudo apt-get update diff --git a/client/incus_operations.go b/client/incus_operations.go index a949ec5403..088b872a49 100644 --- a/client/incus_operations.go +++ b/client/incus_operations.go @@ -89,6 +89,14 @@ func (r *ProtocolIncus) GetOperation(uuid string) (*api.Operation, string, error func (r *ProtocolIncus) GetOperationWait(uuid string, timeout int) (*api.Operation, string, error) { op := api.Operation{} + // Unset the response header timeout so that the request does not time out. + transport, err := r.getUnderlyingHTTPTransport() + if err != nil { + return nil, "", err + } + + transport.ResponseHeaderTimeout = 0 + // Fetch the raw value etag, err := r.queryStruct("GET", fmt.Sprintf("/operations/%s/wait?timeout=%d", url.PathEscape(uuid), timeout), nil, "", &op) if err != nil { diff --git a/cmd/incus-agent/api_1.0.go b/cmd/incus-agent/api_1.0.go index b22e998624..effb82b51f 100644 --- a/cmd/incus-agent/api_1.0.go +++ b/cmd/incus-agent/api_1.0.go @@ -194,8 +194,13 @@ func getClient(CID uint32, port int, serverCertificate string) (*http.Client, er } func startHTTPServer(d *Daemon, debug bool) error { - // Setup the listener on VM's context ID for inbound connections from the host. - l, err := vsock.Listen(ports.HTTPSDefaultPort, nil) + const CIDAny uint32 = 4294967295 // Equivalent to VMADDR_CID_ANY. + + // Setup the listener on wildcard CID for inbound connections from LXD. + // We use the VMADDR_CID_ANY CID so that if the VM's CID changes in the future the listener still works. + // A CID change can occur when restoring a stateful VM that was previously using one CID but is + // subsequently restored using a different one. + l, err := vsock.ListenContextID(CIDAny, ports.HTTPSDefaultPort, nil) if err != nil { return fmt.Errorf("Failed to listen on vsock: %w", err) } diff --git a/cmd/incus-agent/daemon.go b/cmd/incus-agent/daemon.go index 2e160bdcfb..339fd83a82 100644 --- a/cmd/incus-agent/daemon.go +++ b/cmd/incus-agent/daemon.go @@ -4,7 +4,6 @@ import ( "sync" "github.com/lxc/incus/internal/server/events" - "github.com/lxc/incus/internal/server/vsock" ) // A Daemon can respond to requests from a shared client. @@ -17,8 +16,6 @@ type Daemon struct { serverPort uint32 serverCertificate string - localCID uint32 - // The channel which is used to indicate that the agent was able to connect to the host. chConnected chan struct{} @@ -31,11 +28,8 @@ type Daemon struct { func newDaemon(debug, verbose bool) *Daemon { hostEvents := events.NewServer(debug, verbose, nil) - cid, _ := vsock.ContextID() - return &Daemon{ events: hostEvents, chConnected: make(chan struct{}), - localCID: cid, } } diff --git a/cmd/incus-agent/main_agent.go b/cmd/incus-agent/main_agent.go index eccd6c9068..699fdd4a57 100644 --- a/cmd/incus-agent/main_agent.go +++ b/cmd/incus-agent/main_agent.go @@ -17,7 +17,6 @@ import ( "github.com/lxc/incus/internal/linux" "github.com/lxc/incus/internal/server/instance/instancetype" - "github.com/lxc/incus/internal/server/vsock" "github.com/lxc/incus/shared/logger" "github.com/lxc/incus/shared/subprocess" "github.com/lxc/incus/shared/util" @@ -136,31 +135,6 @@ func (c *cmdAgent) Run(cmd *cobra.Command, args []string) error { return fmt.Errorf("Failed to start HTTP server: %w", err) } - // Check context ID periodically, and restart the HTTP server if needed. - go func() { - for range time.Tick(30 * time.Second) { - cid, err := vsock.ContextID() - if err != nil { - continue - } - - if d.localCID == cid { - continue - } - - // Restart server - servers["http"].Close() - - err = startHTTPServer(d, c.global.flagLogDebug) - if err != nil { - errChan <- err - } - - // Update context ID. - d.localCID = cid - } - }() - // Check whether we should start the DevIncus server in the early setup. This way, /dev/incus/sock // will be available for any systemd services starting after the agent. if util.PathExists("agent.conf") { diff --git a/cmd/incus/delete.go b/cmd/incus/delete.go index 93a93ac6e0..7aa393e031 100644 --- a/cmd/incus/delete.go +++ b/cmd/incus/delete.go @@ -116,7 +116,7 @@ func (c *cmdDelete) Run(cmd *cobra.Command, args []string) error { } if ct.Ephemeral { - return nil + continue } } diff --git a/cmd/incus/storage_volume.go b/cmd/incus/storage_volume.go index c97dd8dac1..af37cb28da 100644 --- a/cmd/incus/storage_volume.go +++ b/cmd/incus/storage_volume.go @@ -1644,8 +1644,9 @@ func (c *cmdStorageVolumeMove) Run(cmd *cobra.Command, args []string) error { return fmt.Errorf(i18n.G("No storage pool for target volume specified")) } - // Rename volume if both remotes and pools of source and target are equal. - if srcRemote == dstRemote && srcVolPool == dstVolPool { + // Rename volume if both remotes and pools of source and target are equal + // and no destination cluster member name is set. + if srcRemote == dstRemote && srcVolPool == dstVolPool && c.storageVolume.flagDestinationTarget == "" { var args []string if srcRemote != "" { diff --git a/cmd/incusd/daemon.go b/cmd/incusd/daemon.go index 6f5b9adab5..cac8a25e96 100644 --- a/cmd/incusd/daemon.go +++ b/cmd/incusd/daemon.go @@ -1181,6 +1181,30 @@ func (d *Daemon) init() error { version.UserAgentFeatures([]string{"cluster"}) } + // Load server name and config before patches run (so they can access them from d.State()). + err = d.db.Cluster.Transaction(d.shutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { + config, err := clusterConfig.Load(ctx, tx) + if err != nil { + return err + } + + // Get the local node (will be used if clustered). + serverName, err := tx.GetLocalNodeName(ctx) + if err != nil { + return err + } + + d.globalConfigMu.Lock() + d.serverName = serverName + d.globalConfig = config + d.globalConfigMu.Unlock() + + return nil + }) + if err != nil { + return err + } + // Mount the storage pools. logger.Infof("Initializing storage pools") err = storageStartup(d.State(), false) @@ -1213,13 +1237,7 @@ func (d *Daemon) init() error { return err } - // Get daemon configuration. - bgpAddress := d.localConfig.BGPAddress() - bgpRouterID := d.localConfig.BGPRouterID() - bgpASN := int64(0) - - dnsAddress := d.localConfig.DNSAddress() - + // Load server name and config after patches run (in case its been changed). err = d.db.Cluster.Transaction(d.shutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { config, err := clusterConfig.Load(ctx, tx) if err != nil { @@ -1243,6 +1261,12 @@ func (d *Daemon) init() error { return err } + // Get daemon configuration. + bgpAddress := d.localConfig.BGPAddress() + bgpRouterID := d.localConfig.BGPRouterID() + bgpASN := int64(0) + dnsAddress := d.localConfig.DNSAddress() + // Get specific config keys. d.globalConfigMu.Lock() bgpASN = d.globalConfig.BGPASN() diff --git a/cmd/incusd/patches.go b/cmd/incusd/patches.go index dda796710c..47b4614829 100644 --- a/cmd/incusd/patches.go +++ b/cmd/incusd/patches.go @@ -956,8 +956,18 @@ func patchStorageZfsUnsetInvalidBlockSettings(_ string, d *Daemon) error { continue } - delete(config, "block.filesystem") - delete(config, "block.mount_options") + update := false + for _, k := range []string{"block.filesystem", "block.mount_options"} { + _, found := config[k] + if found { + delete(config, k) + update = true + } + } + + if !update { + continue + } if vol.Type == db.StoragePoolVolumeTypeNameVM { volType = volTypeVM @@ -1056,8 +1066,18 @@ func patchStorageZfsUnsetInvalidBlockSettingsV2(_ string, d *Daemon) error { continue } - delete(config, "block.filesystem") - delete(config, "block.mount_options") + update := false + for _, k := range []string{"block.filesystem", "block.mount_options"} { + _, found := config[k] + if found { + delete(config, k) + update = true + } + } + + if !update { + continue + } if vol.Type == db.StoragePoolVolumeTypeNameVM { volType = volTypeVM diff --git a/internal/server/instance/drivers/driver_lxc.go b/internal/server/instance/drivers/driver_lxc.go index 3caffa402d..9ab887a8a4 100644 --- a/internal/server/instance/drivers/driver_lxc.go +++ b/internal/server/instance/drivers/driver_lxc.go @@ -8125,7 +8125,7 @@ func (d *lxc) UpdateBackupFile() error { return err } - return pool.UpdateInstanceBackupFile(d, nil) + return pool.UpdateInstanceBackupFile(d, true, nil) } // Info returns "lxc" and the currently loaded version of LXC. diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index d7d2d3fb3d..5c00f621ff 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -7724,7 +7724,7 @@ func (d *qemu) UpdateBackupFile() error { return err } - return pool.UpdateInstanceBackupFile(d, nil) + return pool.UpdateInstanceBackupFile(d, true, nil) } type cpuTopology struct { diff --git a/internal/server/network/driver_bridge.go b/internal/server/network/driver_bridge.go index b46d6f288f..107dee2ddb 100644 --- a/internal/server/network/driver_bridge.go +++ b/internal/server/network/driver_bridge.go @@ -1972,7 +1972,7 @@ func (n *bridge) getExternalSubnetInUse() ([]externalSubnetUsage, error) { proxySubnet, err := ParseIPToNet(proxyListenAddr.Address) if err != nil { - return err + continue // If proxy listen isn't a valid IP it can't conflict. } externalSubnets = append(externalSubnets, externalSubnetUsage{ diff --git a/internal/server/storage/backend.go b/internal/server/storage/backend.go index df4e138d15..ce4a006cf2 100644 --- a/internal/server/storage/backend.go +++ b/internal/server/storage/backend.go @@ -2519,7 +2519,7 @@ func (b *backend) BackupInstance(inst instance.Instance, tarWriter *instancewrit } // Ensure the backup file reflects current config. - err = b.UpdateInstanceBackupFile(inst, op) + err = b.UpdateInstanceBackupFile(inst, snapshots, op) if err != nil { return err } @@ -2946,7 +2946,7 @@ func (b *backend) RenameInstanceSnapshot(inst instance.Instance, newName string, }) // Ensure the backup file reflects current config. - err = b.UpdateInstanceBackupFile(inst, op) + err = b.UpdateInstanceBackupFile(inst, true, op) if err != nil { return err } @@ -5937,7 +5937,7 @@ func (b *backend) GenerateInstanceBackupConfig(inst instance.Instance, snapshots } // UpdateInstanceBackupFile writes the instance's config to the backup.yaml file on the storage device. -func (b *backend) UpdateInstanceBackupFile(inst instance.Instance, op *operations.Operation) error { +func (b *backend) UpdateInstanceBackupFile(inst instance.Instance, snapshots bool, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) l.Debug("UpdateInstanceBackupFile started") defer l.Debug("UpdateInstanceBackupFile finished") @@ -5947,7 +5947,7 @@ func (b *backend) UpdateInstanceBackupFile(inst instance.Instance, op *operation return nil } - config, err := b.GenerateInstanceBackupConfig(inst, true, op) + config, err := b.GenerateInstanceBackupConfig(inst, snapshots, op) if err != nil { return err } diff --git a/internal/server/storage/backend_mock.go b/internal/server/storage/backend_mock.go index ea9eb2cc26..fe79dbc786 100644 --- a/internal/server/storage/backend_mock.go +++ b/internal/server/storage/backend_mock.go @@ -148,7 +148,7 @@ func (b *mockBackend) GenerateInstanceBackupConfig(inst instance.Instance, snaps return nil, nil } -func (b *mockBackend) UpdateInstanceBackupFile(inst instance.Instance, op *operations.Operation) error { +func (b *mockBackend) UpdateInstanceBackupFile(inst instance.Instance, snapshot bool, op *operations.Operation) error { return nil } diff --git a/internal/server/storage/pool_interface.go b/internal/server/storage/pool_interface.go index 6cb4ce9374..1acc3fc51b 100644 --- a/internal/server/storage/pool_interface.go +++ b/internal/server/storage/pool_interface.go @@ -71,7 +71,7 @@ type Pool interface { RenameInstance(inst instance.Instance, newName string, op *operations.Operation) error DeleteInstance(inst instance.Instance, op *operations.Operation) error UpdateInstance(inst instance.Instance, newDesc string, newConfig map[string]string, op *operations.Operation) error - UpdateInstanceBackupFile(inst instance.Instance, op *operations.Operation) error + UpdateInstanceBackupFile(inst instance.Instance, snapshots bool, op *operations.Operation) error GenerateInstanceBackupConfig(inst instance.Instance, snapshots bool, op *operations.Operation) (*backupConfig.Config, error) CheckInstanceBackupFileSnapshots(backupConf *backupConfig.Config, projectName string, deleteMissing bool, op *operations.Operation) ([]*api.InstanceSnapshot, error) ImportInstance(inst instance.Instance, poolVol *backupConfig.Config, op *operations.Operation) (revert.Hook, error) diff --git a/internal/server/vsock/vsock.go b/internal/server/vsock/vsock.go index 1c96077035..27df4a408f 100644 --- a/internal/server/vsock/vsock.go +++ b/internal/server/vsock/vsock.go @@ -12,11 +12,6 @@ import ( localtls "github.com/lxc/incus/shared/tls" ) -// ContextID returns the local VM sockets context ID. -func ContextID() (uint32, error) { - return vsock.ContextID() -} - // Dial connects to a remote vsock. func Dial(cid, port uint32) (net.Conn, error) { return vsock.Dial(cid, port, nil) diff --git a/shared/api/url.go b/shared/api/url.go index 16e9a258d7..47ec34633b 100644 --- a/shared/api/url.go +++ b/shared/api/url.go @@ -32,14 +32,20 @@ func (u *URL) Host(host string) *URL { // Path sets the path of the URL from one or more path parts. // It appends each of the pathParts (escaped using url.PathEscape) prefixed with "/" to the URL path. func (u *URL) Path(pathParts ...string) *URL { - var b strings.Builder + var path, rawPath strings.Builder for _, pathPart := range pathParts { - b.WriteString("/") // Build an absolute URL. - b.WriteString(url.PathEscape(pathPart)) + // Generate unencoded path. + path.WriteString("/") // Build an absolute URL. + path.WriteString(pathPart) + + // Generate encoded path hint (this will be used by u.URL.EncodedPath() to decide its methodology). + rawPath.WriteString("/") // Build an absolute URL. + rawPath.WriteString(url.PathEscape(pathPart)) } - u.URL.Path = b.String() + u.URL.Path = path.String() + u.URL.RawPath = rawPath.String() return u } diff --git a/shared/api/url_test.go b/shared/api/url_test.go index 769bd7c73a..3ea4783cf7 100644 --- a/shared/api/url_test.go +++ b/shared/api/url_test.go @@ -14,11 +14,11 @@ func ExampleURL() { fmt.Println(u.Host("example.com")) fmt.Println(u.Scheme("https")) - // Output: /1.0/networks/name-with-%252F-in-it - // /1.0/networks/name-with-%252F-in-it - // /1.0/networks/name-with-%252F-in-it?project=project-with-%25-in-it - // /1.0/networks/name-with-%252F-in-it?project=project-with-%25-in-it - // /1.0/networks/name-with-%252F-in-it?project=project-with-%25-in-it&target=member-with-%25-in-it - // //example.com/1.0/networks/name-with-%252F-in-it?project=project-with-%25-in-it&target=member-with-%25-in-it - // https://example.com/1.0/networks/name-with-%252F-in-it?project=project-with-%25-in-it&target=member-with-%25-in-it + // Output: /1.0/networks/name-with-%2F-in-it + // /1.0/networks/name-with-%2F-in-it + // /1.0/networks/name-with-%2F-in-it?project=project-with-%25-in-it + // /1.0/networks/name-with-%2F-in-it?project=project-with-%25-in-it + // /1.0/networks/name-with-%2F-in-it?project=project-with-%25-in-it&target=member-with-%25-in-it + // //example.com/1.0/networks/name-with-%2F-in-it?project=project-with-%25-in-it&target=member-with-%25-in-it + // https://example.com/1.0/networks/name-with-%2F-in-it?project=project-with-%25-in-it&target=member-with-%25-in-it } diff --git a/test/main.sh b/test/main.sh index 9f84eee521..79aa62dc20 100755 --- a/test/main.sh +++ b/test/main.sh @@ -339,6 +339,7 @@ if [ "${1:-"all"}" != "cluster" ]; then run_test test_backup_export "backup export" run_test test_backup_rename "backup rename" run_test test_backup_volume_export "backup volume export" + run_test test_backup_export_import_instance_only "backup export and import instance only" run_test test_backup_volume_rename_delete "backup volume rename and delete" run_test test_backup_different_instance_uuid "backup instance and check instance UUIDs" run_test test_backup_volume_expiry "backup volume expiry" diff --git a/test/suites/backup.sh b/test/suites/backup.sh index 97cef3f288..b3e3aaa64d 100644 --- a/test/suites/backup.sh +++ b/test/suites/backup.sh @@ -971,7 +971,7 @@ test_backup_export_import_recover() { # Create and export an instance. incus launch testimage c1 incus export c1 "${INCUS_DIR}/c1.tar.gz" - incus rm -f c1 + incus delete -f c1 # Import instance and remove no longer required tarball. incus import "${INCUS_DIR}/c1.tar.gz" c2 @@ -992,3 +992,27 @@ EOF incus rm -f c2 ) } + +test_backup_export_import_instance_only() { + poolName=$(incus profile device get default root pool) + + ensure_import_testimage + ensure_has_localhost_remote "${INCUS_ADDR}" + + # Create an instance with snapshot. + incus init testimage c1 + incus snapshot create c1 + + # Export the instance and remove it. + incus export c1 "${INCUS_DIR}/c1.tar.gz" --instance-only + incus delete -f c1 + + # Import the instance from tarball. + incus import "${INCUS_DIR}/c1.tar.gz" + + # Verify imported instance has no snapshots. + [ "$(incus query "/1.0/storage-pools/${poolName}/volumes/container/c1/snapshots" | jq "length == 0")" = "true" ] + + rm "${INCUS_DIR}/c1.tar.gz" + incus delete -f c1 +} diff --git a/test/suites/basic.sh b/test/suites/basic.sh index 7613b69a00..20814ca44d 100644 --- a/test/suites/basic.sh +++ b/test/suites/basic.sh @@ -655,4 +655,13 @@ test_basic_usage() { ! incus profile assign c1 foo || false incus profile delete foo incus delete -f c1 + + # Multiple ephemeral instances delete + incus launch testimage c1 + incus launch testimage c2 + incus launch testimage c3 + + incus delete -f c1 c2 c3 + remaining_instances="$(incus list --format csv)" + [ -z "${remaining_instances}" ] } diff --git a/test/suites/clustering.sh b/test/suites/clustering.sh index 9a92f048bc..5d6c0362f7 100644 --- a/test/suites/clustering.sh +++ b/test/suites/clustering.sh @@ -658,9 +658,19 @@ test_clustering_storage() { INCUS_DIR="${INCUS_ONE_DIR}" incus storage volume copy pool1/vol1 pool1/vol1 --target=node1 --destination-target=node2 INCUS_DIR="${INCUS_ONE_DIR}" incus storage volume copy pool1/vol1 pool1/vol1 --target=node1 --destination-target=node2 --refresh + # Check renaming storage volume works. + INCUS_DIR="${INCUS_ONE_DIR}" incus storage volume create pool1 vol2 --target=node1 + INCUS_DIR="${INCUS_ONE_DIR}" incus storage volume move pool1/vol2 pool1/vol3 --target=node1 + INCUS_DIR="${INCUS_TWO_DIR}" incus storage volume show pool1 vol3 | grep -q node1 + INCUS_DIR="${INCUS_ONE_DIR}" incus storage volume move pool1/vol3 pool1/vol2 --target=node1 --destination-target=node2 + INCUS_DIR="${INCUS_TWO_DIR}" incus storage volume show pool1 vol2 | grep -q node2 + INCUS_DIR="${INCUS_ONE_DIR}" incus storage volume rename pool1 vol2 vol3 --target=node2 + INCUS_DIR="${INCUS_TWO_DIR}" incus storage volume show pool1 vol3 | grep -q node2 + # Delete pool and check cleaned up. INCUS_DIR="${INCUS_ONE_DIR}" incus storage volume delete pool1 vol1 --target=node1 INCUS_DIR="${INCUS_ONE_DIR}" incus storage volume delete pool1 vol1 --target=node2 + INCUS_DIR="${INCUS_ONE_DIR}" incus storage volume delete pool1 vol3 --target=node2 INCUS_DIR="${INCUS_TWO_DIR}" incus storage delete pool1 ! stat "${INCUS_ONE_SOURCE}/containers" || false ! stat "${INCUS_TWO_SOURCE}/containers" || false