Merge pull request #181 from stgraber/import

Import LXD changes

.github/workflows/tests.yml

@@ -1,7 +1,10 @@
 name: Tests
 on:
-  - push
-  - pull_request
+  push:
+    branches:
+      - main
+      - stable-*
+  pull_request:
 
 permissions:
   contents: read
@@ -45,7 +48,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          sudo add-apt-repository ppa:ubuntu-lxc/lxc-git-master -y --no-update
+          sudo add-apt-repository ppa:ubuntu-lxc/daily -y --no-update
           sudo add-apt-repository ppa:cowsql/stable -y --no-update
           sudo apt-get update
 
@@ -171,7 +174,7 @@ jobs:
       - name: Install dependencies
         run: |
           set -x
-          sudo add-apt-repository ppa:ubuntu-lxc/lxc-git-master -y --no-update
+          sudo add-apt-repository ppa:ubuntu-lxc/daily -y --no-update
           sudo add-apt-repository ppa:cowsql/stable -y --no-update
           sudo apt-get update
 

client/incus_operations.go

@@ -89,6 +89,14 @@ func (r *ProtocolIncus) GetOperation(uuid string) (*api.Operation, string, error
 func (r *ProtocolIncus) GetOperationWait(uuid string, timeout int) (*api.Operation, string, error) {
 	op := api.Operation{}
 
+	// Unset the response header timeout so that the request does not time out.
+	transport, err := r.getUnderlyingHTTPTransport()
+	if err != nil {
+		return nil, "", err
+	}
+
+	transport.ResponseHeaderTimeout = 0
+
 	// Fetch the raw value
 	etag, err := r.queryStruct("GET", fmt.Sprintf("/operations/%s/wait?timeout=%d", url.PathEscape(uuid), timeout), nil, "", &op)
 	if err != nil {

cmd/incus-agent/api_1.0.go

@@ -194,8 +194,13 @@ func getClient(CID uint32, port int, serverCertificate string) (*http.Client, er
 }
 
 func startHTTPServer(d *Daemon, debug bool) error {
-	// Setup the listener on VM's context ID for inbound connections from the host.
-	l, err := vsock.Listen(ports.HTTPSDefaultPort, nil)
+	const CIDAny uint32 = 4294967295 // Equivalent to VMADDR_CID_ANY.
+
+	// Setup the listener on wildcard CID for inbound connections from LXD.
+	// We use the VMADDR_CID_ANY CID so that if the VM's CID changes in the future the listener still works.
+	// A CID change can occur when restoring a stateful VM that was previously using one CID but is
+	// subsequently restored using a different one.
+	l, err := vsock.ListenContextID(CIDAny, ports.HTTPSDefaultPort, nil)
 	if err != nil {
 		return fmt.Errorf("Failed to listen on vsock: %w", err)
 	}

cmd/incus-agent/daemon.go

@@ -4,7 +4,6 @@ import (
 	"sync"
 
 	"github.com/lxc/incus/internal/server/events"
-	"github.com/lxc/incus/internal/server/vsock"
 )
 
 // A Daemon can respond to requests from a shared client.
@@ -17,8 +16,6 @@ type Daemon struct {
 	serverPort        uint32
 	serverCertificate string
 
-	localCID uint32
-
 	// The channel which is used to indicate that the agent was able to connect to the host.
 	chConnected chan struct{}
 
@@ -31,11 +28,8 @@ type Daemon struct {
 func newDaemon(debug, verbose bool) *Daemon {
 	hostEvents := events.NewServer(debug, verbose, nil)
 
-	cid, _ := vsock.ContextID()
-
 	return &Daemon{
 		events:      hostEvents,
 		chConnected: make(chan struct{}),
-		localCID:    cid,
 	}
 }

cmd/incus-agent/main_agent.go

@@ -17,7 +17,6 @@ import (
 
 	"github.com/lxc/incus/internal/linux"
 	"github.com/lxc/incus/internal/server/instance/instancetype"
-	"github.com/lxc/incus/internal/server/vsock"
 	"github.com/lxc/incus/shared/logger"
 	"github.com/lxc/incus/shared/subprocess"
 	"github.com/lxc/incus/shared/util"
@@ -136,31 +135,6 @@ func (c *cmdAgent) Run(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("Failed to start HTTP server: %w", err)
 	}
 
-	// Check context ID periodically, and restart the HTTP server if needed.
-	go func() {
-		for range time.Tick(30 * time.Second) {
-			cid, err := vsock.ContextID()
-			if err != nil {
-				continue
-			}
-
-			if d.localCID == cid {
-				continue
-			}
-
-			// Restart server
-			servers["http"].Close()
-
-			err = startHTTPServer(d, c.global.flagLogDebug)
-			if err != nil {
-				errChan <- err
-			}
-
-			// Update context ID.
-			d.localCID = cid
-		}
-	}()
-
 	// Check whether we should start the DevIncus server in the early setup. This way, /dev/incus/sock
 	// will be available for any systemd services starting after the agent.
 	if util.PathExists("agent.conf") {

cmd/incus/delete.go

@@ -116,7 +116,7 @@ func (c *cmdDelete) Run(cmd *cobra.Command, args []string) error {
 			}
 
 			if ct.Ephemeral {
-				return nil
+				continue
 			}
 		}
 

cmd/incus/storage_volume.go

@@ -1644,8 +1644,9 @@ func (c *cmdStorageVolumeMove) Run(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf(i18n.G("No storage pool for target volume specified"))
 	}
 
-	// Rename volume if both remotes and pools of source and target are equal.
-	if srcRemote == dstRemote && srcVolPool == dstVolPool {
+	// Rename volume if both remotes and pools of source and target are equal
+	// and no destination cluster member name is set.
+	if srcRemote == dstRemote && srcVolPool == dstVolPool && c.storageVolume.flagDestinationTarget == "" {
 		var args []string
 
 		if srcRemote != "" {

cmd/incusd/daemon.go

@@ -1181,6 +1181,30 @@ func (d *Daemon) init() error {
 		version.UserAgentFeatures([]string{"cluster"})
 	}
 
+	// Load server name and config before patches run (so they can access them from d.State()).
+	err = d.db.Cluster.Transaction(d.shutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error {
+		config, err := clusterConfig.Load(ctx, tx)
+		if err != nil {
+			return err
+		}
+
+		// Get the local node (will be used if clustered).
+		serverName, err := tx.GetLocalNodeName(ctx)
+		if err != nil {
+			return err
+		}
+
+		d.globalConfigMu.Lock()
+		d.serverName = serverName
+		d.globalConfig = config
+		d.globalConfigMu.Unlock()
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
 	// Mount the storage pools.
 	logger.Infof("Initializing storage pools")
 	err = storageStartup(d.State(), false)
@@ -1213,13 +1237,7 @@ func (d *Daemon) init() error {
 		return err
 	}
 
-	// Get daemon configuration.
-	bgpAddress := d.localConfig.BGPAddress()
-	bgpRouterID := d.localConfig.BGPRouterID()
-	bgpASN := int64(0)
-
-	dnsAddress := d.localConfig.DNSAddress()
-
+	// Load server name and config after patches run (in case its been changed).
 	err = d.db.Cluster.Transaction(d.shutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error {
 		config, err := clusterConfig.Load(ctx, tx)
 		if err != nil {
@@ -1243,6 +1261,12 @@ func (d *Daemon) init() error {
 		return err
 	}
 
+	// Get daemon configuration.
+	bgpAddress := d.localConfig.BGPAddress()
+	bgpRouterID := d.localConfig.BGPRouterID()
+	bgpASN := int64(0)
+	dnsAddress := d.localConfig.DNSAddress()
+
 	// Get specific config keys.
 	d.globalConfigMu.Lock()
 	bgpASN = d.globalConfig.BGPASN()

cmd/incusd/patches.go

@@ -956,8 +956,18 @@ func patchStorageZfsUnsetInvalidBlockSettings(_ string, d *Daemon) error {
 				continue
 			}
 
-			delete(config, "block.filesystem")
-			delete(config, "block.mount_options")
+			update := false
+			for _, k := range []string{"block.filesystem", "block.mount_options"} {
+				_, found := config[k]
+				if found {
+					delete(config, k)
+					update = true
+				}
+			}
+
+			if !update {
+				continue
+			}
 
 			if vol.Type == db.StoragePoolVolumeTypeNameVM {
 				volType = volTypeVM
@@ -1056,8 +1066,18 @@ func patchStorageZfsUnsetInvalidBlockSettingsV2(_ string, d *Daemon) error {
 				continue
 			}
 
-			delete(config, "block.filesystem")
-			delete(config, "block.mount_options")
+			update := false
+			for _, k := range []string{"block.filesystem", "block.mount_options"} {
+				_, found := config[k]
+				if found {
+					delete(config, k)
+					update = true
+				}
+			}
+
+			if !update {
+				continue
+			}
 
 			if vol.Type == db.StoragePoolVolumeTypeNameVM {
 				volType = volTypeVM

internal/server/instance/drivers/driver_lxc.go

@@ -8125,7 +8125,7 @@ func (d *lxc) UpdateBackupFile() error {
 		return err
 	}
 
-	return pool.UpdateInstanceBackupFile(d, nil)
+	return pool.UpdateInstanceBackupFile(d, true, nil)
 }
 
 // Info returns "lxc" and the currently loaded version of LXC.

internal/server/instance/drivers/driver_qemu.go

@@ -7724,7 +7724,7 @@ func (d *qemu) UpdateBackupFile() error {
 		return err
 	}
 
-	return pool.UpdateInstanceBackupFile(d, nil)
+	return pool.UpdateInstanceBackupFile(d, true, nil)
 }
 
 type cpuTopology struct {

internal/server/network/driver_bridge.go

@@ -1972,7 +1972,7 @@ func (n *bridge) getExternalSubnetInUse() ([]externalSubnetUsage, error) {
 
 			proxySubnet, err := ParseIPToNet(proxyListenAddr.Address)
 			if err != nil {
-				return err
+				continue // If proxy listen isn't a valid IP it can't conflict.
 			}
 
 			externalSubnets = append(externalSubnets, externalSubnetUsage{

internal/server/storage/backend.go

@@ -2519,7 +2519,7 @@ func (b *backend) BackupInstance(inst instance.Instance, tarWriter *instancewrit
 	}
 
 	// Ensure the backup file reflects current config.
-	err = b.UpdateInstanceBackupFile(inst, op)
+	err = b.UpdateInstanceBackupFile(inst, snapshots, op)
 	if err != nil {
 		return err
 	}
@@ -2946,7 +2946,7 @@ func (b *backend) RenameInstanceSnapshot(inst instance.Instance, newName string,
 	})
 
 	// Ensure the backup file reflects current config.
-	err = b.UpdateInstanceBackupFile(inst, op)
+	err = b.UpdateInstanceBackupFile(inst, true, op)
 	if err != nil {
 		return err
 	}
@@ -5937,7 +5937,7 @@ func (b *backend) GenerateInstanceBackupConfig(inst instance.Instance, snapshots
 }
 
 // UpdateInstanceBackupFile writes the instance's config to the backup.yaml file on the storage device.
-func (b *backend) UpdateInstanceBackupFile(inst instance.Instance, op *operations.Operation) error {
+func (b *backend) UpdateInstanceBackupFile(inst instance.Instance, snapshots bool, op *operations.Operation) error {
 	l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()})
 	l.Debug("UpdateInstanceBackupFile started")
 	defer l.Debug("UpdateInstanceBackupFile finished")
@@ -5947,7 +5947,7 @@ func (b *backend) UpdateInstanceBackupFile(inst instance.Instance, op *operation
 		return nil
 	}
 
-	config, err := b.GenerateInstanceBackupConfig(inst, true, op)
+	config, err := b.GenerateInstanceBackupConfig(inst, snapshots, op)
 	if err != nil {
 		return err
 	}

internal/server/storage/backend_mock.go

@@ -148,7 +148,7 @@ func (b *mockBackend) GenerateInstanceBackupConfig(inst instance.Instance, snaps
 	return nil, nil
 }
 
-func (b *mockBackend) UpdateInstanceBackupFile(inst instance.Instance, op *operations.Operation) error {
+func (b *mockBackend) UpdateInstanceBackupFile(inst instance.Instance, snapshot bool, op *operations.Operation) error {
 	return nil
 }
 

internal/server/storage/pool_interface.go

@@ -71,7 +71,7 @@ type Pool interface {
 	RenameInstance(inst instance.Instance, newName string, op *operations.Operation) error
 	DeleteInstance(inst instance.Instance, op *operations.Operation) error
 	UpdateInstance(inst instance.Instance, newDesc string, newConfig map[string]string, op *operations.Operation) error
-	UpdateInstanceBackupFile(inst instance.Instance, op *operations.Operation) error
+	UpdateInstanceBackupFile(inst instance.Instance, snapshots bool, op *operations.Operation) error
 	GenerateInstanceBackupConfig(inst instance.Instance, snapshots bool, op *operations.Operation) (*backupConfig.Config, error)
 	CheckInstanceBackupFileSnapshots(backupConf *backupConfig.Config, projectName string, deleteMissing bool, op *operations.Operation) ([]*api.InstanceSnapshot, error)
 	ImportInstance(inst instance.Instance, poolVol *backupConfig.Config, op *operations.Operation) (revert.Hook, error)

internal/server/vsock/vsock.go

@@ -12,11 +12,6 @@ import (
 	localtls "github.com/lxc/incus/shared/tls"
 )
 
-// ContextID returns the local VM sockets context ID.
-func ContextID() (uint32, error) {
-	return vsock.ContextID()
-}
-
 // Dial connects to a remote vsock.
 func Dial(cid, port uint32) (net.Conn, error) {
 	return vsock.Dial(cid, port, nil)

shared/api/url.go

@@ -32,14 +32,20 @@ func (u *URL) Host(host string) *URL {
 // Path sets the path of the URL from one or more path parts.
 // It appends each of the pathParts (escaped using url.PathEscape) prefixed with "/" to the URL path.
 func (u *URL) Path(pathParts ...string) *URL {
-	var b strings.Builder
+	var path, rawPath strings.Builder
 
 	for _, pathPart := range pathParts {
-		b.WriteString("/") // Build an absolute URL.
-		b.WriteString(url.PathEscape(pathPart))
+		// Generate unencoded path.
+		path.WriteString("/") // Build an absolute URL.
+		path.WriteString(pathPart)
+
+		// Generate encoded path hint (this will be used by u.URL.EncodedPath() to decide its methodology).
+		rawPath.WriteString("/") // Build an absolute URL.
+		rawPath.WriteString(url.PathEscape(pathPart))
 	}
 
-	u.URL.Path = b.String()
+	u.URL.Path = path.String()
+	u.URL.RawPath = rawPath.String()
 
 	return u
 }

shared/api/url_test.go

@@ -14,11 +14,11 @@ func ExampleURL() {
 	fmt.Println(u.Host("example.com"))
 	fmt.Println(u.Scheme("https"))
 
-	// Output: /1.0/networks/name-with-%252F-in-it
-	// /1.0/networks/name-with-%252F-in-it
-	// /1.0/networks/name-with-%252F-in-it?project=project-with-%25-in-it
-	// /1.0/networks/name-with-%252F-in-it?project=project-with-%25-in-it
-	// /1.0/networks/name-with-%252F-in-it?project=project-with-%25-in-it&target=member-with-%25-in-it
-	// //example.com/1.0/networks/name-with-%252F-in-it?project=project-with-%25-in-it&target=member-with-%25-in-it
-	// https://example.com/1.0/networks/name-with-%252F-in-it?project=project-with-%25-in-it&target=member-with-%25-in-it
+	// Output: /1.0/networks/name-with-%2F-in-it
+	// /1.0/networks/name-with-%2F-in-it
+	// /1.0/networks/name-with-%2F-in-it?project=project-with-%25-in-it
+	// /1.0/networks/name-with-%2F-in-it?project=project-with-%25-in-it
+	// /1.0/networks/name-with-%2F-in-it?project=project-with-%25-in-it&target=member-with-%25-in-it
+	// //example.com/1.0/networks/name-with-%2F-in-it?project=project-with-%25-in-it&target=member-with-%25-in-it
+	// https://example.com/1.0/networks/name-with-%2F-in-it?project=project-with-%25-in-it&target=member-with-%25-in-it
 }

test/main.sh

@@ -339,6 +339,7 @@ if [ "${1:-"all"}" != "cluster" ]; then
     run_test test_backup_export "backup export"
     run_test test_backup_rename "backup rename"
     run_test test_backup_volume_export "backup volume export"
+    run_test test_backup_export_import_instance_only "backup export and import instance only"
     run_test test_backup_volume_rename_delete "backup volume rename and delete"
     run_test test_backup_different_instance_uuid "backup instance and check instance UUIDs"
     run_test test_backup_volume_expiry "backup volume expiry"