diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 2c85284b..9d59515d 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -657,9 +657,10 @@ func (s *ApiService) ForkInstance(ctx context.Context, request oapi.ForkInstance } result, err := s.InstanceManager.ForkInstance(ctx, inst.Id, instances.ForkInstanceRequest{ - Name: request.Body.Name, - FromRunning: request.Body.FromRunning != nil && *request.Body.FromRunning, - TargetState: targetState, + Name: request.Body.Name, + FromRunning: request.Body.FromRunning != nil && *request.Body.FromRunning, + TargetState: targetState, + WaitForNetwork: request.Body.WaitForNetwork, }) if err != nil { switch { diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 4e7863aa..896a0632 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -1165,13 +1165,15 @@ func TestForkInstance_Success(t *testing.T) { result: forked, } svc.InstanceManager = mockMgr + waitForNetwork := false resp, err := svc.ForkInstance( mw.WithResolvedInstance(ctx(), source.Id, source), oapi.ForkInstanceRequestObject{ Id: source.Id, Body: &oapi.ForkInstanceRequest{ - Name: "forked-instance", + Name: "forked-instance", + WaitForNetwork: &waitForNetwork, }, }, ) @@ -1185,6 +1187,8 @@ func TestForkInstance_Success(t *testing.T) { assert.Equal(t, "forked-instance", mockMgr.lastReq.Name) assert.False(t, mockMgr.lastReq.FromRunning) assert.Equal(t, instances.State(""), mockMgr.lastReq.TargetState) + require.NotNil(t, mockMgr.lastReq.WaitForNetwork) + assert.False(t, *mockMgr.lastReq.WaitForNetwork) } func TestForkInstance_NotSupported(t *testing.T) { diff --git a/cmd/api/api/snapshots.go b/cmd/api/api/snapshots.go index 620d9574..ffe35474 100644 --- a/cmd/api/api/snapshots.go +++ b/cmd/api/api/snapshots.go @@ -175,7 +175,10 @@ func (s *ApiService) ForkSnapshot(ctx context.Context, request oapi.ForkSnapshot return oapi.ForkSnapshot400JSONResponse{Code: "invalid_request", Message: "request body is required"}, nil } - domainReq := instances.ForkSnapshotRequest{Name: request.Body.Name} + domainReq := instances.ForkSnapshotRequest{ + Name: request.Body.Name, + WaitForNetwork: request.Body.WaitForNetwork, + } if request.Body.TargetState != nil { domainReq.TargetState = instances.State(*request.Body.TargetState) } diff --git a/cmd/api/api/snapshots_test.go b/cmd/api/api/snapshots_test.go index d0730133..f87e9148 100644 --- a/cmd/api/api/snapshots_test.go +++ b/cmd/api/api/snapshots_test.go @@ -1,14 +1,35 @@ package api import ( + "context" "testing" "time" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/oapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +type captureForkSnapshotManager struct { + instances.Manager + lastID string + lastReq *instances.ForkSnapshotRequest + result *instances.Instance + err error +} + +func (m *captureForkSnapshotManager) ForkSnapshot(ctx context.Context, snapshotID string, req instances.ForkSnapshotRequest) (*instances.Instance, error) { + reqCopy := req + m.lastID = snapshotID + m.lastReq = &reqCopy + if m.err != nil { + return nil, m.err + } + return m.result, nil +} + func TestSnapshotScheduleToOAPIPreservesZeroMaxCount(t *testing.T) { t.Parallel() @@ -30,3 +51,43 @@ func TestSnapshotScheduleToOAPIPreservesZeroMaxCount(t *testing.T) { require.NotNil(t, out.Retention.MaxAge) assert.Equal(t, "24h0m0s", *out.Retention.MaxAge) } + +func TestForkSnapshotMapsWaitForNetwork(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + forked := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "forked-instance", + Name: "forked-instance", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeFirecracker, + }, + State: instances.StateRunning, + } + mockMgr := &captureForkSnapshotManager{ + Manager: svc.InstanceManager, + result: forked, + } + svc.InstanceManager = mockMgr + waitForNetwork := false + + resp, err := svc.ForkSnapshot(ctx(), oapi.ForkSnapshotRequestObject{ + SnapshotId: "snap-123", + Body: &oapi.ForkSnapshotRequest{ + Name: "forked-instance", + WaitForNetwork: &waitForNetwork, + }, + }) + require.NoError(t, err) + + created, ok := resp.(oapi.ForkSnapshot201JSONResponse) + require.True(t, ok, "expected 201 response") + assert.Equal(t, "forked-instance", created.Name) + assert.Equal(t, "snap-123", mockMgr.lastID) + require.NotNil(t, mockMgr.lastReq) + assert.Equal(t, "forked-instance", mockMgr.lastReq.Name) + require.NotNil(t, mockMgr.lastReq.WaitForNetwork) + assert.False(t, *mockMgr.lastReq.WaitForNetwork) +} diff --git a/lib/forkvm/README.md b/lib/forkvm/README.md index bb6567c7..88474c72 100644 --- a/lib/forkvm/README.md +++ b/lib/forkvm/README.md @@ -14,6 +14,24 @@ to work across implementations. For networked forks, the fork gets a fresh host/guest identity (IP, MAC, TAP) instead of reusing the source identity. +## Resume network handoff + +Networked standby/running forks need a new host-side allocation, but the guest +memory snapshot still contains the source VM's old interface state. On restore, +Hypeman prepares the fork's TAP/IP/MAC before the VM resumes, then hands the new +guest network config to the guest through a small mailbox embedded in snapshot +memory. After resume, VMGenID tells the guest-agent that this is a restored VM; +the guest-agent reads the mailbox and applies the new MAC, address, route, and +neighbor state with netlink. + +For API calls that return a running fork, `wait_for_network` defaults to true. +In that mode Hypeman waits for a guest UDP "applied" ack before returning, so +the fast path still avoids making host-initiated guest RPC/vsock contact as the +first post-resume dependency. If `wait_for_network=false`, the API returns after +resume once the mailbox has been patched and the guest finishes the network +handoff asynchronously. If the mailbox path is unavailable, restore falls back +to the older host-initiated guest network reconfigure path. + ## Fork data copy behavior - Guest directory copy is **sparse-only** for regular files. diff --git a/lib/guest/client.go b/lib/guest/client.go index cb772dbc..f46b1149 100644 --- a/lib/guest/client.go +++ b/lib/guest/client.go @@ -82,8 +82,12 @@ func GetOrCreateConn(ctx context.Context, dialer hypervisor.VsockDialer) (*grpc. } // Create new connection using the VsockDialer + traceCtx := ctx conn, err := grpc.Dial("passthrough:///vsock", grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + if span := trace.SpanFromContext(traceCtx); span.SpanContext().IsValid() { + ctx = trace.ContextWithSpan(ctx, span) + } netConn, err := dialer.DialVsock(ctx, vsockGuestPort) if err != nil { return nil, &AgentVSockDialError{Err: err} @@ -146,6 +150,109 @@ type ExecOptions struct { ResizeChan <-chan *WindowSize // Optional: channel to receive resize events (pointer to avoid copying mutex) } +type ReconfigureNetworkOptions struct { + InterfaceName string + MAC string + IPv4 string + Prefix uint32 + Gateway string + WaitForAgent time.Duration +} + +func ReconfigureNetworkInInstance(ctx context.Context, dialer hypervisor.VsockDialer, opts ReconfigureNetworkOptions) error { + if opts.WaitForAgent == 0 { + return reconfigureNetworkOnce(ctx, dialer, opts) + } + + ctx, span := otel.Tracer("hypeman/guest").Start(ctx, "guest.reconfigure_network", trace.WithAttributes( + attribute.Bool("wait_for_agent", true), + attribute.Int64("wait_for_agent_ms", opts.WaitForAgent.Milliseconds()), + )) + defer span.End() + + deadline := time.Now().Add(opts.WaitForAgent) + start := time.Now() + attempts := 0 + retryableAttempts := 0 + firstRetryableErrorType := "" + lastRetryableErrorType := "" + lastRetryInterval := time.Duration(0) + + for { + attempts++ + err := reconfigureNetworkOnce(ctx, dialer, opts) + if err == nil { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.SetStatus(otelcodes.Ok, "") + return nil + } + if !isRetryableConnectionError(err) { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(err) + span.SetStatus(otelcodes.Error, err.Error()) + return err + } + + retryableAttempts++ + errType := retryableConnectionErrorType(err) + if firstRetryableErrorType == "" { + firstRetryableErrorType = errType + } + lastRetryableErrorType = errType + CloseConn(dialer.Key()) + + if time.Now().After(deadline) { + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(err) + span.SetStatus(otelcodes.Error, err.Error()) + return err + } + + retryInterval := guestExecRetryInterval(time.Since(start)) + lastRetryInterval = retryInterval + select { + case <-ctx.Done(): + recordGuestExecWait(span, start, attempts, retryableAttempts, firstRetryableErrorType, lastRetryableErrorType, lastRetryInterval) + span.RecordError(ctx.Err()) + span.SetStatus(otelcodes.Error, ctx.Err().Error()) + return ctx.Err() + case <-time.After(retryInterval): + } + } +} + +func reconfigureNetworkOnce(ctx context.Context, dialer hypervisor.VsockDialer, opts ReconfigureNetworkOptions) error { + grpcConn, err := GetOrCreateConn(ctx, dialer) + if err != nil { + return fmt.Errorf("get grpc connection: %w", err) + } + client := NewGuestServiceClient(grpcConn) + + _, span := otel.Tracer("hypeman/guest").Start(ctx, "guest.reconfigure_network.rpc") + _, err = client.ReconfigureNetwork(ctx, &ReconfigureNetworkRequest{ + InterfaceName: opts.InterfaceName, + Mac: opts.MAC, + Ipv4: opts.IPv4, + Prefix: opts.Prefix, + Gateway: opts.Gateway, + }) + finishGuestNetworkStepSpan(span, err) + if err != nil { + return fmt.Errorf("reconfigure network rpc: %w", err) + } + return nil +} + +func finishGuestNetworkStepSpan(span trace.Span, err error) { + if err != nil { + span.RecordError(err) + span.SetStatus(otelcodes.Error, err.Error()) + } else { + span.SetStatus(otelcodes.Ok, "") + } + span.End() +} + // ExecIntoInstance executes command in instance via vsock using gRPC. // The dialer is a hypervisor-specific VsockDialer that knows how to connect to the guest. // If WaitForAgent is set, it will retry on connection errors until the timeout. diff --git a/lib/guest/guest.pb.go b/lib/guest/guest.pb.go index 0de1628e..a239fc97 100644 --- a/lib/guest/guest.pb.go +++ b/lib/guest/guest.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.10 -// protoc v3.21.12 +// protoc v6.33.4 // source: lib/guest/guest.proto package guest @@ -1266,6 +1266,120 @@ func (*ShutdownResponse) Descriptor() ([]byte, []int) { return file_lib_guest_guest_proto_rawDescGZIP(), []int{16} } +// ReconfigureNetworkRequest updates a guest network interface after snapshot restore +type ReconfigureNetworkRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InterfaceName string `protobuf:"bytes,1,opt,name=interface_name,json=interfaceName,proto3" json:"interface_name,omitempty"` // Interface to reconfigure, defaults to eth0 + Mac string `protobuf:"bytes,2,opt,name=mac,proto3" json:"mac,omitempty"` // New MAC address + Ipv4 string `protobuf:"bytes,3,opt,name=ipv4,proto3" json:"ipv4,omitempty"` // New IPv4 address without prefix + Prefix uint32 `protobuf:"varint,4,opt,name=prefix,proto3" json:"prefix,omitempty"` // IPv4 prefix length + Gateway string `protobuf:"bytes,5,opt,name=gateway,proto3" json:"gateway,omitempty"` // Default gateway IPv4 address + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReconfigureNetworkRequest) Reset() { + *x = ReconfigureNetworkRequest{} + mi := &file_lib_guest_guest_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReconfigureNetworkRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReconfigureNetworkRequest) ProtoMessage() {} + +func (x *ReconfigureNetworkRequest) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReconfigureNetworkRequest.ProtoReflect.Descriptor instead. +func (*ReconfigureNetworkRequest) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{17} +} + +func (x *ReconfigureNetworkRequest) GetInterfaceName() string { + if x != nil { + return x.InterfaceName + } + return "" +} + +func (x *ReconfigureNetworkRequest) GetMac() string { + if x != nil { + return x.Mac + } + return "" +} + +func (x *ReconfigureNetworkRequest) GetIpv4() string { + if x != nil { + return x.Ipv4 + } + return "" +} + +func (x *ReconfigureNetworkRequest) GetPrefix() uint32 { + if x != nil { + return x.Prefix + } + return 0 +} + +func (x *ReconfigureNetworkRequest) GetGateway() string { + if x != nil { + return x.Gateway + } + return "" +} + +// ReconfigureNetworkResponse acknowledges the network reconfiguration request +type ReconfigureNetworkResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReconfigureNetworkResponse) Reset() { + *x = ReconfigureNetworkResponse{} + mi := &file_lib_guest_guest_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReconfigureNetworkResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReconfigureNetworkResponse) ProtoMessage() {} + +func (x *ReconfigureNetworkResponse) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReconfigureNetworkResponse.ProtoReflect.Descriptor instead. +func (*ReconfigureNetworkResponse) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{18} +} + var File_lib_guest_guest_proto protoreflect.FileDescriptor const file_lib_guest_guest_proto_rawDesc = "" + @@ -1358,13 +1472,21 @@ const file_lib_guest_guest_proto_rawDesc = "" + "\x05error\x18\b \x01(\tR\x05error\")\n" + "\x0fShutdownRequest\x12\x16\n" + "\x06signal\x18\x01 \x01(\x05R\x06signal\"\x12\n" + - "\x10ShutdownResponse2\xd3\x02\n" + + "\x10ShutdownResponse\"\x9a\x01\n" + + "\x19ReconfigureNetworkRequest\x12%\n" + + "\x0einterface_name\x18\x01 \x01(\tR\rinterfaceName\x12\x10\n" + + "\x03mac\x18\x02 \x01(\tR\x03mac\x12\x12\n" + + "\x04ipv4\x18\x03 \x01(\tR\x04ipv4\x12\x16\n" + + "\x06prefix\x18\x04 \x01(\rR\x06prefix\x12\x18\n" + + "\agateway\x18\x05 \x01(\tR\agateway\"\x1c\n" + + "\x1aReconfigureNetworkResponse2\xae\x03\n" + "\fGuestService\x123\n" + "\x04Exec\x12\x12.guest.ExecRequest\x1a\x13.guest.ExecResponse(\x010\x01\x12F\n" + "\vCopyToGuest\x12\x19.guest.CopyToGuestRequest\x1a\x1a.guest.CopyToGuestResponse(\x01\x12L\n" + "\rCopyFromGuest\x12\x1b.guest.CopyFromGuestRequest\x1a\x1c.guest.CopyFromGuestResponse0\x01\x12;\n" + "\bStatPath\x12\x16.guest.StatPathRequest\x1a\x17.guest.StatPathResponse\x12;\n" + - "\bShutdown\x12\x16.guest.ShutdownRequest\x1a\x17.guest.ShutdownResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" + "\bShutdown\x12\x16.guest.ShutdownRequest\x1a\x17.guest.ShutdownResponse\x12Y\n" + + "\x12ReconfigureNetwork\x12 .guest.ReconfigureNetworkRequest\x1a!.guest.ReconfigureNetworkResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" var ( file_lib_guest_guest_proto_rawDescOnce sync.Once @@ -1378,31 +1500,33 @@ func file_lib_guest_guest_proto_rawDescGZIP() []byte { return file_lib_guest_guest_proto_rawDescData } -var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 20) var file_lib_guest_guest_proto_goTypes = []any{ - (*ExecRequest)(nil), // 0: guest.ExecRequest - (*ExecStart)(nil), // 1: guest.ExecStart - (*WindowSize)(nil), // 2: guest.WindowSize - (*ExecResponse)(nil), // 3: guest.ExecResponse - (*CopyToGuestRequest)(nil), // 4: guest.CopyToGuestRequest - (*CopyToGuestStart)(nil), // 5: guest.CopyToGuestStart - (*CopyToGuestEnd)(nil), // 6: guest.CopyToGuestEnd - (*CopyToGuestResponse)(nil), // 7: guest.CopyToGuestResponse - (*CopyFromGuestRequest)(nil), // 8: guest.CopyFromGuestRequest - (*CopyFromGuestResponse)(nil), // 9: guest.CopyFromGuestResponse - (*CopyFromGuestHeader)(nil), // 10: guest.CopyFromGuestHeader - (*CopyFromGuestEnd)(nil), // 11: guest.CopyFromGuestEnd - (*CopyFromGuestError)(nil), // 12: guest.CopyFromGuestError - (*StatPathRequest)(nil), // 13: guest.StatPathRequest - (*StatPathResponse)(nil), // 14: guest.StatPathResponse - (*ShutdownRequest)(nil), // 15: guest.ShutdownRequest - (*ShutdownResponse)(nil), // 16: guest.ShutdownResponse - nil, // 17: guest.ExecStart.EnvEntry + (*ExecRequest)(nil), // 0: guest.ExecRequest + (*ExecStart)(nil), // 1: guest.ExecStart + (*WindowSize)(nil), // 2: guest.WindowSize + (*ExecResponse)(nil), // 3: guest.ExecResponse + (*CopyToGuestRequest)(nil), // 4: guest.CopyToGuestRequest + (*CopyToGuestStart)(nil), // 5: guest.CopyToGuestStart + (*CopyToGuestEnd)(nil), // 6: guest.CopyToGuestEnd + (*CopyToGuestResponse)(nil), // 7: guest.CopyToGuestResponse + (*CopyFromGuestRequest)(nil), // 8: guest.CopyFromGuestRequest + (*CopyFromGuestResponse)(nil), // 9: guest.CopyFromGuestResponse + (*CopyFromGuestHeader)(nil), // 10: guest.CopyFromGuestHeader + (*CopyFromGuestEnd)(nil), // 11: guest.CopyFromGuestEnd + (*CopyFromGuestError)(nil), // 12: guest.CopyFromGuestError + (*StatPathRequest)(nil), // 13: guest.StatPathRequest + (*StatPathResponse)(nil), // 14: guest.StatPathResponse + (*ShutdownRequest)(nil), // 15: guest.ShutdownRequest + (*ShutdownResponse)(nil), // 16: guest.ShutdownResponse + (*ReconfigureNetworkRequest)(nil), // 17: guest.ReconfigureNetworkRequest + (*ReconfigureNetworkResponse)(nil), // 18: guest.ReconfigureNetworkResponse + nil, // 19: guest.ExecStart.EnvEntry } var file_lib_guest_guest_proto_depIdxs = []int32{ 1, // 0: guest.ExecRequest.start:type_name -> guest.ExecStart 2, // 1: guest.ExecRequest.resize:type_name -> guest.WindowSize - 17, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry + 19, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry 5, // 3: guest.CopyToGuestRequest.start:type_name -> guest.CopyToGuestStart 6, // 4: guest.CopyToGuestRequest.end:type_name -> guest.CopyToGuestEnd 10, // 5: guest.CopyFromGuestResponse.header:type_name -> guest.CopyFromGuestHeader @@ -1413,13 +1537,15 @@ var file_lib_guest_guest_proto_depIdxs = []int32{ 8, // 10: guest.GuestService.CopyFromGuest:input_type -> guest.CopyFromGuestRequest 13, // 11: guest.GuestService.StatPath:input_type -> guest.StatPathRequest 15, // 12: guest.GuestService.Shutdown:input_type -> guest.ShutdownRequest - 3, // 13: guest.GuestService.Exec:output_type -> guest.ExecResponse - 7, // 14: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse - 9, // 15: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse - 14, // 16: guest.GuestService.StatPath:output_type -> guest.StatPathResponse - 16, // 17: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse - 13, // [13:18] is the sub-list for method output_type - 8, // [8:13] is the sub-list for method input_type + 17, // 13: guest.GuestService.ReconfigureNetwork:input_type -> guest.ReconfigureNetworkRequest + 3, // 14: guest.GuestService.Exec:output_type -> guest.ExecResponse + 7, // 15: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse + 9, // 16: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse + 14, // 17: guest.GuestService.StatPath:output_type -> guest.StatPathResponse + 16, // 18: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse + 18, // 19: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse + 14, // [14:20] is the sub-list for method output_type + 8, // [8:14] is the sub-list for method input_type 8, // [8:8] is the sub-list for extension type_name 8, // [8:8] is the sub-list for extension extendee 0, // [0:8] is the sub-list for field type_name @@ -1457,7 +1583,7 @@ func file_lib_guest_guest_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_lib_guest_guest_proto_rawDesc), len(file_lib_guest_guest_proto_rawDesc)), NumEnums: 0, - NumMessages: 18, + NumMessages: 20, NumExtensions: 0, NumServices: 1, }, diff --git a/lib/guest/guest.proto b/lib/guest/guest.proto index c42198a9..317c21b3 100644 --- a/lib/guest/guest.proto +++ b/lib/guest/guest.proto @@ -20,6 +20,9 @@ service GuestService { // Shutdown requests graceful VM shutdown by signaling init (PID 1) rpc Shutdown(ShutdownRequest) returns (ShutdownResponse); + + // ReconfigureNetwork updates the guest network identity without spawning shell commands + rpc ReconfigureNetwork(ReconfigureNetworkRequest) returns (ReconfigureNetworkResponse); } // ExecRequest represents messages from client to server @@ -154,3 +157,15 @@ message ShutdownRequest { // ShutdownResponse acknowledges the shutdown request message ShutdownResponse {} + +// ReconfigureNetworkRequest updates a guest network interface after snapshot restore +message ReconfigureNetworkRequest { + string interface_name = 1; // Interface to reconfigure, defaults to eth0 + string mac = 2; // New MAC address + string ipv4 = 3; // New IPv4 address without prefix + uint32 prefix = 4; // IPv4 prefix length + string gateway = 5; // Default gateway IPv4 address +} + +// ReconfigureNetworkResponse acknowledges the network reconfiguration request +message ReconfigureNetworkResponse {} diff --git a/lib/guest/guest_grpc.pb.go b/lib/guest/guest_grpc.pb.go index 71224327..f93631d9 100644 --- a/lib/guest/guest_grpc.pb.go +++ b/lib/guest/guest_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.0 -// - protoc v3.21.12 +// - protoc v6.33.4 // source: lib/guest/guest.proto package guest @@ -19,11 +19,12 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - GuestService_Exec_FullMethodName = "/guest.GuestService/Exec" - GuestService_CopyToGuest_FullMethodName = "/guest.GuestService/CopyToGuest" - GuestService_CopyFromGuest_FullMethodName = "/guest.GuestService/CopyFromGuest" - GuestService_StatPath_FullMethodName = "/guest.GuestService/StatPath" - GuestService_Shutdown_FullMethodName = "/guest.GuestService/Shutdown" + GuestService_Exec_FullMethodName = "/guest.GuestService/Exec" + GuestService_CopyToGuest_FullMethodName = "/guest.GuestService/CopyToGuest" + GuestService_CopyFromGuest_FullMethodName = "/guest.GuestService/CopyFromGuest" + GuestService_StatPath_FullMethodName = "/guest.GuestService/StatPath" + GuestService_Shutdown_FullMethodName = "/guest.GuestService/Shutdown" + GuestService_ReconfigureNetwork_FullMethodName = "/guest.GuestService/ReconfigureNetwork" ) // GuestServiceClient is the client API for GuestService service. @@ -42,6 +43,8 @@ type GuestServiceClient interface { StatPath(ctx context.Context, in *StatPathRequest, opts ...grpc.CallOption) (*StatPathResponse, error) // Shutdown requests graceful VM shutdown by signaling init (PID 1) Shutdown(ctx context.Context, in *ShutdownRequest, opts ...grpc.CallOption) (*ShutdownResponse, error) + // ReconfigureNetwork updates the guest network identity without spawning shell commands + ReconfigureNetwork(ctx context.Context, in *ReconfigureNetworkRequest, opts ...grpc.CallOption) (*ReconfigureNetworkResponse, error) } type guestServiceClient struct { @@ -117,6 +120,16 @@ func (c *guestServiceClient) Shutdown(ctx context.Context, in *ShutdownRequest, return out, nil } +func (c *guestServiceClient) ReconfigureNetwork(ctx context.Context, in *ReconfigureNetworkRequest, opts ...grpc.CallOption) (*ReconfigureNetworkResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ReconfigureNetworkResponse) + err := c.cc.Invoke(ctx, GuestService_ReconfigureNetwork_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // GuestServiceServer is the server API for GuestService service. // All implementations must embed UnimplementedGuestServiceServer // for forward compatibility. @@ -133,6 +146,8 @@ type GuestServiceServer interface { StatPath(context.Context, *StatPathRequest) (*StatPathResponse, error) // Shutdown requests graceful VM shutdown by signaling init (PID 1) Shutdown(context.Context, *ShutdownRequest) (*ShutdownResponse, error) + // ReconfigureNetwork updates the guest network identity without spawning shell commands + ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) mustEmbedUnimplementedGuestServiceServer() } @@ -158,6 +173,9 @@ func (UnimplementedGuestServiceServer) StatPath(context.Context, *StatPathReques func (UnimplementedGuestServiceServer) Shutdown(context.Context, *ShutdownRequest) (*ShutdownResponse, error) { return nil, status.Error(codes.Unimplemented, "method Shutdown not implemented") } +func (UnimplementedGuestServiceServer) ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ReconfigureNetwork not implemented") +} func (UnimplementedGuestServiceServer) mustEmbedUnimplementedGuestServiceServer() {} func (UnimplementedGuestServiceServer) testEmbeddedByValue() {} @@ -240,6 +258,24 @@ func _GuestService_Shutdown_Handler(srv interface{}, ctx context.Context, dec fu return interceptor(ctx, in, info, handler) } +func _GuestService_ReconfigureNetwork_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReconfigureNetworkRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GuestServiceServer).ReconfigureNetwork(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GuestService_ReconfigureNetwork_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GuestServiceServer).ReconfigureNetwork(ctx, req.(*ReconfigureNetworkRequest)) + } + return interceptor(ctx, in, info, handler) +} + // GuestService_ServiceDesc is the grpc.ServiceDesc for GuestService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -255,6 +291,10 @@ var GuestService_ServiceDesc = grpc.ServiceDesc{ MethodName: "Shutdown", Handler: _GuestService_Shutdown_Handler, }, + { + MethodName: "ReconfigureNetwork", + Handler: _GuestService_ReconfigureNetwork_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 202660e2..d74a75a9 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -81,7 +81,9 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR // the source data directory. Restore the fork while source remains standby and // under lock, then restore the source. if forkErr == nil && targetState == StateRunning { - restoredFork, err := m.applyForkTargetState(ctx, forked.Id, StateRunning) + restoredFork, err := m.applyForkTargetState(ctx, forked.Id, StateRunning, restoreInstanceOptions{ + WaitForGuestNetwork: req.WaitForNetwork, + }) if err != nil { forkErr = fmt.Errorf("restore forked instance before source restore: %w", err) if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { @@ -93,7 +95,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR } log.InfoContext(ctx, "restoring source instance after running fork", "source_instance_id", id) - _, restoreErr := m.restoreInstance(ctx, id) + _, restoreErr := m.restoreInstance(ctx, id, restoreInstanceOptions{}) if restoreErr != nil { if forkErr != nil { @@ -393,7 +395,7 @@ func resolveForkTargetState(requested State, sourceState State) (State, error) { return requested, nil } -func (m *manager) applyForkTargetState(ctx context.Context, forkID string, target State) (*Instance, error) { +func (m *manager) applyForkTargetState(ctx context.Context, forkID string, target State, restoreOpts restoreInstanceOptions) (*Instance, error) { lock := m.getInstanceLock(forkID) lock.Lock() defer lock.Unlock() @@ -403,6 +405,9 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe return nil, err } if inst != nil && (inst.State == StateRunning || inst.State == StateInitializing) { + if guestInitiatedResumeNetworkMailbox(&inst.StoredMetadata) { + return inst, nil + } if err := ensureGuestAgentReadyForForkPhase(ctx, &inst.StoredMetadata, "before returning running fork instance"); err != nil { return nil, fmt.Errorf("wait for forked guest agent readiness: %w", err) } @@ -432,7 +437,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe case StateStandby: switch target { case StateRunning: - return returnWithReadiness(m.restoreInstance(ctx, forkID)) + return returnWithReadiness(m.restoreInstance(ctx, forkID, restoreOpts)) case StateStopped: if err := os.RemoveAll(m.paths.InstanceSnapshotLatest(forkID)); err != nil { return nil, fmt.Errorf("remove fork snapshot: %w", err) diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index eafcd77a..32764063 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -308,7 +308,7 @@ func TestApplyForkTargetStateStoppedRefreshesSnapshotForkCID(t *testing.T) { meta.StoredMetadata.Phases.Record(phasetracking.PhaseStandby, time.Now()) require.NoError(t, manager.saveMetadata(meta)) - inst, err := manager.applyForkTargetState(ctx, forkID, StateStopped) + inst, err := manager.applyForkTargetState(ctx, forkID, StateStopped, restoreInstanceOptions{}) require.NoError(t, err) require.Equal(t, StateStopped, inst.State) require.Equal(t, generateVsockCID(forkID), inst.VsockCID) diff --git a/lib/instances/guest_resume_network.go b/lib/instances/guest_resume_network.go new file mode 100644 index 00000000..354657b0 --- /dev/null +++ b/lib/instances/guest_resume_network.go @@ -0,0 +1,246 @@ +package instances + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "fmt" + stdnet "net" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" + "go.opentelemetry.io/otel/attribute" + "golang.org/x/sys/unix" +) + +const guestResumeNetworkMailboxEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX" +const guestResumeNetworkMailboxTokenEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX_TOKEN" +const firecrackerSnapshotMemoryFile = "memory" + +const guestResumeNetworkMailboxSeqOffset = 64 +const guestResumeNetworkMailboxLengthOffset = 68 +const guestResumeNetworkMailboxPayloadOffset = 72 + +var guestResumeNetworkMailboxMagic = []byte("HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00") +var guestResumeNetworkMailboxOffsets sync.Map + +type guestResumeNetworkPayload struct { + InterfaceName string `json:"interface_name"` + MAC string `json:"mac"` + IPv4 string `json:"ipv4"` + Prefix uint32 `json:"prefix"` + Gateway string `json:"gateway"` + AckPort uint32 `json:"ack_port,omitempty"` +} + +type guestResumeNetworkUDPAck struct { + received time.Time + text string +} + +type guestResumeNetworkUDPWaiter struct { + conn *stdnet.UDPConn + ch chan guestResumeNetworkUDPAck +} + +func guestInitiatedResumeNetworkMailbox(stored *StoredMetadata) bool { + token := guestInitiatedResumeNetworkMailboxToken(stored) + return stored != nil && + stored.HypervisorType == hypervisor.TypeFirecracker && + strings.TrimSpace(stored.Env[guestResumeNetworkMailboxEnv]) == "1" && + token != "" && + len(token) <= guestResumeNetworkMailboxSeqOffset-len(guestResumeNetworkMailboxMagic) +} + +func guestInitiatedResumeNetworkMailboxToken(stored *StoredMetadata) string { + if stored == nil { + return "" + } + return strings.TrimSpace(stored.Env[guestResumeNetworkMailboxTokenEnv]) +} + +func newGuestResumeNetworkPayload(cfg *guestNetworkConfig) guestResumeNetworkPayload { + return guestResumeNetworkPayload{ + InterfaceName: "eth0", + MAC: cfg.mac, + IPv4: cfg.ip, + Prefix: uint32(cfg.prefix), + Gateway: cfg.gateway, + } +} + +func startGuestResumeNetworkUDPWaiter() (*guestResumeNetworkUDPWaiter, error) { + conn, err := stdnet.ListenUDP("udp4", &stdnet.UDPAddr{IP: stdnet.IPv4zero, Port: 0}) + if err != nil { + return nil, fmt.Errorf("listen for guest resume network UDP ack: %w", err) + } + + w := &guestResumeNetworkUDPWaiter{ + conn: conn, + ch: make(chan guestResumeNetworkUDPAck, 128), + } + go w.readLoop() + return w, nil +} + +func (w *guestResumeNetworkUDPWaiter) Port() uint32 { + if w == nil || w.conn == nil { + return 0 + } + return uint32(w.conn.LocalAddr().(*stdnet.UDPAddr).Port) +} + +func (w *guestResumeNetworkUDPWaiter) Close() { + if w == nil || w.conn == nil { + return + } + _ = w.conn.Close() +} + +func (w *guestResumeNetworkUDPWaiter) readLoop() { + buf := make([]byte, 1024) + for { + n, _, err := w.conn.ReadFromUDP(buf) + if err != nil { + return + } + w.ch <- guestResumeNetworkUDPAck{ + received: time.Now(), + text: strings.TrimSpace(string(buf[:n])), + } + } +} + +func (w *guestResumeNetworkUDPWaiter) WaitApplied(ctx context.Context, mac, ip string) (time.Duration, string, error) { + if w == nil { + return 0, "", fmt.Errorf("guest resume network UDP waiter is nil") + } + + start := time.Now() + wantMAC := "mac=" + strings.ToLower(mac) + wantIP := "ip=" + ip + for { + select { + case ack := <-w.ch: + text := strings.ToLower(ack.text) + if strings.Contains(text, "stage=applied") && strings.Contains(text, wantMAC) && strings.Contains(text, wantIP) { + return ack.received.Sub(start), ack.text, nil + } + case <-ctx.Done(): + return 0, "", ctx.Err() + } + } +} + +func (m *manager) waitForGuestResumeNetworkUDPAck(ctx context.Context, waiter *guestResumeNetworkUDPWaiter, stored *StoredMetadata, cfg *guestNetworkConfig) error { + if waiter == nil || cfg == nil { + return nil + } + + log := logger.FromContext(ctx) + waitCtx, waitSpanEnd := m.startLifecycleStep(ctx, "guest.resume_network.udp_ack_wait", + attribute.String("instance_id", stored.Id), + attribute.String("hypervisor", string(stored.HypervisorType)), + attribute.String("operation", "guest_resume_network_udp_ack_wait"), + ) + waitCtx, cancel := context.WithTimeout(waitCtx, 2*time.Second) + defer cancel() + + elapsed, ack, err := waiter.WaitApplied(waitCtx, cfg.mac, cfg.ip) + waitSpanEnd(err) + if err != nil { + return err + } + log.InfoContext(ctx, "guest resume network UDP ack received", "instance_id", stored.Id, "elapsed", elapsed, "ack", ack) + return nil +} + +func patchGuestResumeNetworkMailbox(snapshotDir, token string, payload *guestResumeNetworkPayload) error { + if token == "" { + return fmt.Errorf("resume network mailbox token is empty") + } + if len(token) > guestResumeNetworkMailboxSeqOffset-len(guestResumeNetworkMailboxMagic) { + return fmt.Errorf("resume network mailbox token is too long") + } + if payload == nil { + return fmt.Errorf("resume network mailbox payload is nil") + } + + payloadBytes, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("marshal resume network mailbox payload: %w", err) + } + if len(payloadBytes) > 4096-guestResumeNetworkMailboxPayloadOffset { + return fmt.Errorf("resume network mailbox payload too large: %d bytes", len(payloadBytes)) + } + + file, err := os.OpenFile(filepath.Join(snapshotDir, firecrackerSnapshotMemoryFile), os.O_RDWR, 0) + if err != nil { + return fmt.Errorf("open snapshot memory for resume network mailbox: %w", err) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return fmt.Errorf("stat snapshot memory for resume network mailbox: %w", err) + } + if info.Size() <= 0 { + return fmt.Errorf("resume network mailbox memory file is empty") + } + + marker := make([]byte, 0, len(guestResumeNetworkMailboxMagic)+len(token)) + marker = append(marker, guestResumeNetworkMailboxMagic...) + marker = append(marker, []byte(token)...) + + idx, err := findGuestResumeNetworkMailbox(file, info.Size(), marker, token) + if err != nil { + return err + } + if idx+int64(guestResumeNetworkMailboxPayloadOffset)+int64(len(payloadBytes)) > info.Size() { + return fmt.Errorf("resume network mailbox marker is too close to end of memory file") + } + + if _, err := file.WriteAt(payloadBytes, idx+int64(guestResumeNetworkMailboxPayloadOffset)); err != nil { + return fmt.Errorf("write resume network mailbox payload: %w", err) + } + var u32 [4]byte + binary.LittleEndian.PutUint32(u32[:], uint32(len(payloadBytes))) + if _, err := file.WriteAt(u32[:], idx+int64(guestResumeNetworkMailboxLengthOffset)); err != nil { + return fmt.Errorf("write resume network mailbox payload length: %w", err) + } + binary.LittleEndian.PutUint32(u32[:], 1) + if _, err := file.WriteAt(u32[:], idx+int64(guestResumeNetworkMailboxSeqOffset)); err != nil { + return fmt.Errorf("write resume network mailbox sequence: %w", err) + } + return nil +} + +func findGuestResumeNetworkMailbox(file *os.File, size int64, marker []byte, token string) (int64, error) { + if cached, ok := guestResumeNetworkMailboxOffsets.Load(token); ok { + if offset, ok := cached.(int64); ok && offset >= 0 && offset+int64(len(marker)) <= size { + buf := make([]byte, len(marker)) + if _, err := file.ReadAt(buf, offset); err == nil && bytes.Equal(buf, marker) { + return offset, nil + } + } + } + + data, err := unix.Mmap(int(file.Fd()), 0, int(size), unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + return 0, fmt.Errorf("mmap snapshot memory for resume network mailbox: %w", err) + } + defer unix.Munmap(data) + + idx := bytes.Index(data, marker) + if idx < 0 { + return 0, fmt.Errorf("resume network mailbox marker not found") + } + guestResumeNetworkMailboxOffsets.Store(token, int64(idx)) + return int64(idx), nil +} diff --git a/lib/instances/manager.go b/lib/instances/manager.go index abdd2165..c55ba93a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -389,14 +389,16 @@ func (m *manager) ForkInstance(ctx context.Context, id string, req ForkInstanceR return nil, err } - inst, err := m.applyForkTargetState(ctx, forked.Id, targetState) + inst, err := m.applyForkTargetState(ctx, forked.Id, targetState, restoreInstanceOptions{ + WaitForGuestNetwork: req.WaitForNetwork, + }) if err != nil { if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { return nil, fmt.Errorf("apply fork target state: %w; additionally failed to cleanup forked instance %s: %v", err, forked.Id, cleanupErr) } return nil, fmt.Errorf("apply fork target state: %w", err) } - if inst.State == StateRunning { + if inst.State == StateRunning && !guestInitiatedResumeNetworkMailbox(&inst.StoredMetadata) { if err := ensureGuestAgentReadyForForkPhase(ctx, &inst.StoredMetadata, "before returning running fork instance"); err != nil { if cleanupErr := m.cleanupForkInstanceOnError(ctx, forked.Id); cleanupErr != nil { return nil, fmt.Errorf("wait for fork guest agent readiness: %w; additionally failed to cleanup forked instance %s: %v", err, forked.Id, cleanupErr) @@ -449,7 +451,7 @@ func (m *manager) RestoreInstance(ctx context.Context, id string) (*Instance, er if current.State == StateRunning || current.State == StateInitializing { return current, nil } - inst, err := m.restoreInstance(ctx, id) + inst, err := m.restoreInstance(ctx, id, restoreInstanceOptions{}) if err == nil { m.notifyLifecycleEvent(ctx, LifecycleEventRestore, inst) } diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 9f1a3034..714f319e 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -16,14 +16,24 @@ import ( "github.com/kernel/hypeman/lib/network" snapshotstore "github.com/kernel/hypeman/lib/snapshot" "go.opentelemetry.io/otel/attribute" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -// RestoreInstance restores an instance from standby -// Multi-hop orchestration: Standby → Paused → Running +type restoreInstanceOptions struct { + WaitForGuestNetwork *bool +} + +func (o restoreInstanceOptions) waitForGuestNetwork() bool { + return o.WaitForGuestNetwork == nil || *o.WaitForGuestNetwork +} + +// RestoreInstance restores an instance from standby. +// Multi-hop orchestration: Standby → Paused → Running. func (m *manager) restoreInstance( ctx context.Context, - id string, + opts restoreInstanceOptions, ) (_ *Instance, retErr error) { start := time.Now() log := logger.FromContext(ctx) @@ -44,6 +54,7 @@ func (m *manager) restoreInstance( inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + waitForGuestNetwork := opts.waitForGuestNetwork() ctx = enrichInstancesTrace(ctx, attribute.String("hypervisor", string(stored.HypervisorType))) log.DebugContext(ctx, "loaded instance", "instance_id", id, "state", inst.State, "has_snapshot", inst.HasSnapshot) @@ -227,6 +238,42 @@ func (m *manager) restoreInstance( proxyRegistered = true } + var resumeNetworkAckWaiter *guestResumeNetworkUDPWaiter + var resumeNetworkAckCfg *guestNetworkConfig + resumeNetworkMailboxPatched := false + if allocatedNet != nil && !stored.SkipGuestAgent && guestInitiatedResumeNetworkMailbox(stored) { + resumeNetworkCfg, cfgErr := guestNetworkReconfigureConfig(allocatedNet) + if cfgErr != nil { + log.WarnContext(ctx, "failed to build guest resume network mailbox payload; falling back to host-initiated reconfigure", "instance_id", id, "error", cfgErr) + } else { + payload := newGuestResumeNetworkPayload(resumeNetworkCfg) + if waitForGuestNetwork { + var waitErr error + resumeNetworkAckWaiter, waitErr = startGuestResumeNetworkUDPWaiter() + if waitErr != nil { + log.ErrorContext(ctx, "failed to start guest resume network UDP ack waiter", "instance_id", id, "error", waitErr) + releaseNetwork() + return nil, fmt.Errorf("start guest resume network UDP ack waiter: %w", waitErr) + } + resumeNetworkAckCfg = resumeNetworkCfg + payload.AckPort = resumeNetworkAckWaiter.Port() + } + if patchErr := patchGuestResumeNetworkMailbox(snapshotDir, guestInitiatedResumeNetworkMailboxToken(stored), &payload); patchErr != nil { + if resumeNetworkAckWaiter != nil { + resumeNetworkAckWaiter.Close() + resumeNetworkAckWaiter = nil + resumeNetworkAckCfg = nil + } + log.WarnContext(ctx, "failed to patch guest resume network mailbox; falling back to host-initiated reconfigure", "instance_id", id, "error", patchErr) + } else { + resumeNetworkMailboxPatched = true + } + } + } + if resumeNetworkAckWaiter != nil { + defer resumeNetworkAckWaiter.Close() + } + // 5. Transition: Standby → Paused (start hypervisor + restore) restoreCtx, restoreSpanEnd := m.startLifecycleStep(ctx, "restore_from_snapshot", attribute.String("instance_id", id), @@ -281,15 +328,22 @@ func (m *manager) restoreInstance( attribute.String("hypervisor", string(stored.HypervisorType)), attribute.String("operation", "reconfigure_guest_network"), ) - if err := reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet); err != nil { - reconfigureSpanEnd(err) - log.ErrorContext(ctx, "failed to configure guest network after restore", "instance_id", id, "error", err) + var reconfigureErr error + if resumeNetworkMailboxPatched && waitForGuestNetwork { + reconfigureErr = m.waitForGuestResumeNetworkUDPAck(reconfigureCtx, resumeNetworkAckWaiter, stored, resumeNetworkAckCfg) + } else if resumeNetworkMailboxPatched { + log.InfoContext(ctx, "guest resume network mailbox patched", "instance_id", id) + } else { + reconfigureErr = reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet) + } + reconfigureSpanEnd(reconfigureErr) + if reconfigureErr != nil { + log.ErrorContext(ctx, "failed to configure guest network after restore", "instance_id", id, "error", reconfigureErr) _ = hv.Shutdown(ctx) m.rollbackAdmissionAllocationActive(stored) releaseNetwork() - return nil, fmt.Errorf("configure guest network after restore: %w", err) + return nil, fmt.Errorf("configure guest network after restore: %w", reconfigureErr) } - reconfigureSpanEnd(nil) } // 8. Delete snapshot after successful restore unless the hypervisor is keeping it @@ -379,7 +433,7 @@ func (m *manager) restoreFromSnapshot( } func reconfigureGuestNetwork(ctx context.Context, stored *StoredMetadata, alloc *network.Allocation) error { - cmd, err := guestNetworkReconfigureCommand(alloc) + cfg, err := guestNetworkReconfigureConfig(alloc) if err != nil { return err } @@ -389,6 +443,30 @@ func reconfigureGuestNetwork(ctx context.Context, stored *StoredMetadata, alloc return fmt.Errorf("create vsock dialer: %w", err) } + err = guest.ReconfigureNetworkInInstance(ctx, dialer, guest.ReconfigureNetworkOptions{ + InterfaceName: "eth0", + MAC: cfg.mac, + IPv4: cfg.ip, + Prefix: uint32(cfg.prefix), + Gateway: cfg.gateway, + WaitForAgent: 120 * time.Second, + }) + if err != nil { + if status.Code(err) == codes.Unimplemented { + return reconfigureGuestNetworkWithExec(ctx, dialer, alloc) + } + return fmt.Errorf("reconfigure guest network: %w", err) + } + + return nil +} + +func reconfigureGuestNetworkWithExec(ctx context.Context, dialer hypervisor.VsockDialer, alloc *network.Allocation) error { + cmd, err := guestNetworkReconfigureCommand(alloc) + if err != nil { + return err + } + var stdout, stderr bytes.Buffer exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ Command: []string{"sh", "-c", cmd}, @@ -402,30 +480,44 @@ func reconfigureGuestNetwork(ctx context.Context, stored *StoredMetadata, alloc if exit.Code != 0 { return fmt.Errorf("network reconfiguration command failed (exit=%d, stdout=%q, stderr=%q)", exit.Code, strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String())) } - return nil } -func guestNetworkReconfigureCommand(alloc *network.Allocation) (string, error) { +type guestNetworkConfig struct { + ip string + mac string + gateway string + prefix int +} + +func guestNetworkReconfigureConfig(alloc *network.Allocation) (*guestNetworkConfig, error) { if alloc == nil { - return "", fmt.Errorf("missing network allocation") + return nil, fmt.Errorf("missing network allocation") } ip := strings.TrimSpace(alloc.IP) if ip == "" { - return "", fmt.Errorf("missing network allocation IP") + return nil, fmt.Errorf("missing network allocation IP") } mac := strings.ToLower(strings.TrimSpace(alloc.MAC)) if mac == "" { - return "", fmt.Errorf("missing network allocation MAC") + return nil, fmt.Errorf("missing network allocation MAC") } if _, err := net.ParseMAC(mac); err != nil { - return "", fmt.Errorf("invalid network allocation MAC %q: %w", alloc.MAC, err) + return nil, fmt.Errorf("invalid network allocation MAC %q: %w", alloc.MAC, err) } gateway := strings.TrimSpace(alloc.Gateway) if gateway == "" { - return "", fmt.Errorf("missing network allocation gateway") + return nil, fmt.Errorf("missing network allocation gateway") } prefix, err := netmaskToPrefix(alloc.Netmask) + if err != nil { + return nil, err + } + return &guestNetworkConfig{ip: ip, mac: mac, gateway: gateway, prefix: prefix}, nil +} + +func guestNetworkReconfigureCommand(alloc *network.Allocation) (string, error) { + cfg, err := guestNetworkReconfigureConfig(alloc) if err != nil { return "", err } @@ -445,7 +537,7 @@ func guestNetworkReconfigureCommand(alloc *network.Allocation) (string, error) { "ip route replace default via %s dev eth0 && "+ // Drop snapshotted ARP/neighbor entries so peers are rediscovered. "(ip neigh flush dev eth0 || true)", - mac, ip, prefix, gateway, + cfg.mac, cfg.ip, cfg.prefix, cfg.gateway, ), nil } diff --git a/lib/instances/restore_egress_test.go b/lib/instances/restore_egress_test.go index f65b0eb1..b5fcdb4e 100644 --- a/lib/instances/restore_egress_test.go +++ b/lib/instances/restore_egress_test.go @@ -1,6 +1,9 @@ package instances import ( + "encoding/binary" + "encoding/json" + "os" "testing" "github.com/kernel/hypeman/lib/network" @@ -30,7 +33,25 @@ func TestNetworkConfigFromAllocation_PreservesDNS(t *testing.T) { assert.Equal(t, alloc.TAPDevice, cfg.TAPDevice) } -func TestGuestNetworkReconfigureCommand_AppliesAllocatedMAC(t *testing.T) { +func TestGuestNetworkReconfigureConfig_AppliesAllocatedMAC(t *testing.T) { + t.Parallel() + + alloc := &network.Allocation{ + IP: "10.102.146.62", + MAC: "02:00:00:85:17:c8", + Gateway: "10.102.0.1", + Netmask: "255.255.0.0", + } + + cfg, err := guestNetworkReconfigureConfig(alloc) + require.NoError(t, err) + assert.Equal(t, "10.102.146.62", cfg.ip) + assert.Equal(t, "02:00:00:85:17:c8", cfg.mac) + assert.Equal(t, "10.102.0.1", cfg.gateway) + assert.Equal(t, 16, cfg.prefix) +} + +func TestGuestNetworkReconfigureCommand_FallbackPreservesShellBehavior(t *testing.T) { t.Parallel() alloc := &network.Allocation{ @@ -42,18 +63,15 @@ func TestGuestNetworkReconfigureCommand_AppliesAllocatedMAC(t *testing.T) { cmd, err := guestNetworkReconfigureCommand(alloc) require.NoError(t, err) - assert.Contains(t, cmd, "ip link set dev eth0 down") assert.Contains(t, cmd, "ip link set dev eth0 address 02:00:00:85:17:c8") assert.Contains(t, cmd, "ip addr add 10.102.146.62/16 dev eth0") - assert.Contains(t, cmd, "ip route replace default via 10.102.0.1 dev eth0") assert.Contains(t, cmd, "(ip neigh flush dev eth0 || true)") - assert.NotContains(t, cmd, "cat /sys/class/net/eth0/address") } -func TestGuestNetworkReconfigureCommand_RequiresAllocatedMAC(t *testing.T) { +func TestGuestNetworkReconfigureConfig_RequiresAllocatedMAC(t *testing.T) { t.Parallel() - _, err := guestNetworkReconfigureCommand(&network.Allocation{ + _, err := guestNetworkReconfigureConfig(&network.Allocation{ IP: "10.102.146.62", Gateway: "10.102.0.1", Netmask: "255.255.0.0", @@ -62,6 +80,40 @@ func TestGuestNetworkReconfigureCommand_RequiresAllocatedMAC(t *testing.T) { assert.Contains(t, err.Error(), "missing network allocation MAC") } +func TestPatchGuestResumeNetworkMailbox(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + token := "test-token" + mem := make([]byte, 4096) + copy(mem[512:], guestResumeNetworkMailboxMagic) + copy(mem[512+len(guestResumeNetworkMailboxMagic):], token) + require.NoError(t, os.WriteFile(dir+"/"+firecrackerSnapshotMemoryFile, mem, 0644)) + + payload := &guestResumeNetworkPayload{ + InterfaceName: "eth0", + MAC: "02:00:00:85:17:c8", + IPv4: "10.102.146.62", + Prefix: 16, + Gateway: "10.102.0.1", + AckPort: 43210, + } + require.NoError(t, patchGuestResumeNetworkMailbox(dir, token, payload)) + + patched, err := os.ReadFile(dir + "/" + firecrackerSnapshotMemoryFile) + require.NoError(t, err) + + offset := 512 + require.Equal(t, uint32(1), binary.LittleEndian.Uint32(patched[offset+guestResumeNetworkMailboxSeqOffset:])) + payloadLen := binary.LittleEndian.Uint32(patched[offset+guestResumeNetworkMailboxLengthOffset:]) + require.NotZero(t, payloadLen) + + var decoded guestResumeNetworkPayload + err = json.Unmarshal(patched[offset+guestResumeNetworkMailboxPayloadOffset:offset+guestResumeNetworkMailboxPayloadOffset+int(payloadLen)], &decoded) + require.NoError(t, err) + assert.Equal(t, *payload, decoded) +} + func TestRequiresRestoreConfigDiskRefresh(t *testing.T) { t.Parallel() diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 05d8084e..523917b4 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -127,7 +127,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps } if restoreSource { - _, restoreErr := m.restoreInstance(ctx, id) + _, restoreErr := m.restoreInstance(ctx, id, restoreInstanceOptions{}) if restoreErr != nil { if copyErr != nil { return nil, fmt.Errorf("snapshot copy failed: %v; additionally failed to restore source: %w", copyErr, restoreErr) @@ -337,7 +337,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str } return m.getInstance(ctx, id) case StateRunning: - inst, err := m.restoreInstance(ctx, id) + inst, err := m.restoreInstance(ctx, id, restoreInstanceOptions{}) if err != nil { return nil, err } @@ -486,7 +486,9 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS } cu.Release() - inst, err := m.applyForkTargetState(ctx, forkID, targetState) + inst, err := m.applyForkTargetState(ctx, forkID, targetState, restoreInstanceOptions{ + WaitForGuestNetwork: req.WaitForNetwork, + }) if err != nil { if cleanupErr := m.cleanupForkInstanceOnError(ctx, forkID); cleanupErr != nil { return nil, fmt.Errorf("apply snapshot fork target state: %w; additionally failed to cleanup forked instance %s: %v", err, forkID, cleanupErr) diff --git a/lib/instances/types.go b/lib/instances/types.go index 8c031fc9..e38f5499 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -268,9 +268,10 @@ type UpdateInstanceRequest struct { // ForkInstanceRequest is the domain request for forking an instance. type ForkInstanceRequest struct { - Name string // Required: name for the new forked instance - FromRunning bool // Optional: allow forking from Running by auto standby/fork/restore - TargetState State // Optional: desired final state of forked instance (Stopped, Standby, Running). Empty means inherit source state. + Name string // Required: name for the new forked instance + FromRunning bool // Optional: allow forking from Running by auto standby/fork/restore + TargetState State // Optional: desired final state of forked instance (Stopped, Standby, Running). Empty means inherit source state. + WaitForNetwork *bool // Optional: wait for guest networking before returning a Running fork. Nil defaults to true. } // SnapshotKind determines how snapshot data is captured and restored. @@ -314,6 +315,7 @@ type ForkSnapshotRequest struct { Name string // Required: name for the new instance TargetState State // Optional TargetHypervisor hypervisor.Type // Optional, allowed only for Stopped snapshots + WaitForNetwork *bool // Optional: wait for guest networking before returning a Running fork. Nil defaults to true. } // SnapshotPolicy defines default snapshot behavior for an instance. diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 1c053f62..add4df50 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -732,6 +732,10 @@ type ForkInstanceRequest struct { // TargetState Target state for the forked instance after fork completes TargetState *ForkTargetState `json:"target_state,omitempty"` + + // WaitForNetwork When the fork result is Running, wait for guest networking to be applied before returning. + // Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. + WaitForNetwork *bool `json:"wait_for_network,omitempty"` } // ForkSnapshotRequest defines model for ForkSnapshotRequest. @@ -745,6 +749,10 @@ type ForkSnapshotRequest struct { // TargetState Target state when restoring or forking from a snapshot TargetState *SnapshotTargetState `json:"target_state,omitempty"` + + // WaitForNetwork When the fork result is Running, wait for guest networking to be applied before returning. + // Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. + WaitForNetwork *bool `json:"wait_for_network,omitempty"` } // ForkSnapshotRequestTargetHypervisor Optional hypervisor override. Allowed only when forking from a Stopped snapshot. @@ -15953,213 +15961,215 @@ var swaggerSpec = []string{ "oWvlHKxmPrBPTXxg59CSG37TplMvIgsSFZnAHF0mp1cQlPGJWbTSrChb4IiGY8qS1MsSjaR8lQrQRU2j", "CE94qsw1o00NzzuBeGuwPaZaXLezc19xcbk2clWfxFkG/Fqv0CE40qfWVQOnOEb2a5ccUVD6sutAc2lq", "n0v0znxhPET5z0laxtPpQk/Wk8SQIFJxkKTWYWibaatd+vUWcJa66BHTXy47Hyl0pjc10Qb3a2GLGQHk", - "B7VWY9Gc8h7eP4fXWwfC6w/XOlJa0J2Rq8cgOmQK9DTb9iTDycNQfFUsW+ZryF+CU1jQkPQR7C4IqnGZ", - "iZWddq54kpAw8//0R8xGkmc/SXODoj80dFBzQgXigs5oueOyg+0hg+JuwoqOm27NjsUP6xoqPITwjeZN", - "j6fKoDxcumQtUsycsovQ6XbOM0wMK4nKpHmX4YrUKJIHedaG+Prsw01D2xLBp9SHdASxEPaptcxc0Ndv", - "u4Pz3vD/MQGcmt9ARaPMxE/EPKxAWNj32508r88+nDWNKQOVQMXR1eaURbysgtVyFLGXSvZW0lowjv31", - "wZJ1kuveL3y67FTgmEzS6ZSIcexxrr3Sz5F5wYQ2UYZOfy7rs1pvbms1n5UWB8zmKQ4sJkA76nsccpVp", - "dAvU/ORfrnfEHMNNmYR6qYR9xyYT9tGbDMYDvT77IFEepeTx1JWXtzFS/2y+lDTAkWnRJAZTVnSwAXO2", - "1pDP8g+tK9KjJ/vRX9xGQBuLWZLCNjx/1zt5+3ErDsmiWxoTRBbNeUT0uDcL0mLh8gnztIKSkFg0eToM", - "Y8i2G6hAq2wHtyZSYb96qKO4wtFYRtwXrPFeP0TwEG18fGXyvfQIuigpLaX+vUCFEn/ve3eMlkhN3Z5D", - "h1WXaWmDe23HMg6nca8Uplfq1LdVTJB9XcepQy/xy/JC88v1cD+mkeZ+j1weQMWpbRVJZNIFEKQLuPtn", - "ZD41XmvrGpEkwQIrEi2NZpEdfRGdkmAZRGaPk/pVIrkmwQ0SEV7q17+aPOJUkLGaCyLnPCpfQO9067Bv", - "EoIgF8TCb5g5FRzviqMYi0s4GJ0ijVJmKFBOOthZB2k5Vyq5waR+ef/+zFjXioiFiSgrBvnK2tXqMYnw", - "Ek2IuiKEualgiTB6zTOck2pWjmyAQBBqnBBBeZmGnR1Pv+cmMBPNBA4IMl85FEa7JBJOzbaktL14oLyC", - "gEjZsL7DVetrP52mUbs19g1ruBa0NLjJAr8/OnOp/BkqoSPzdp3KZ0T0zJZz8ISrl3ZbrgaVcF0xzki9", - "M8EnBFAmbOJNMUXIBZYAiob+vJSWUxAOspgPY/uBXWBI1TX7/FMrZa+63X0X6TFmoQ/d0UQ8m9zHWRrr", - "BdFDFincHdHQRJ2YdEyjlhcDtwXBIWVEykriWJCKqNPt9KZ2VgdbWxEPcDTnUh3s7gyfb62O31sZuGnj", - "VMYhXWXfuWgWE+/g8qsMyiJMuswSWzhJWnjADB3XnA8gnuqBYoDoqM+2gobnIocHg1oO3DUOlAPGAZdY", - "6coTF7dtolmyNB9oMIMV3nvxorg/B9476BzV27H/Vo339cxMMJnmEeNuqNDRMPlnr0bFhQ/9hAtlM1wm", - "xEWbZeehi+Wylyqlzp4Pnhdn2Qo8GYRNZZvbfeeZqnm7nPVGCuS2+9c2AHExZZ3DbenVWGmaLmt4SkvE", - "GkutJihPCLsRPfd2d7ZvRs+2EzlxSXkVueTLuT86PTY6UcCZwpQRgWKisEWKLwgZ8CVpKaMN/BCTGPIU", - "pv+xWrQ0xC8Uk+gbb8CPanBrD3L73QAT9M7Eb4YoxoxOtUC2bxZ7lnO8vbd/YMDMQjLd3dvv9/s3TS1+", - "mecSt1qKLZM+Wcgy7sv53dbhATKI28zlS+fs8P0vWpClUphDa0tOKDso/Dv7Z/4A/jD/nFDmzTxuhX9H", - "pzXcu3I8mDb4ze8HBahyp/e0giP2O4MhLBTgDrwwMQrPtG5jOO6ueDC3RozLYUtVASmumE3TAjWOfl59", - "jey8SvCO7TNlikY5oF79AvlWkIhyJWpUDTEqISzDiYoi81fA2ULvCh9oVOkkcs/uFHyxUvX6e13jWr/f", - "nOK1hm39XrZM/rUFy7OQNp6T6JtL/dsEKJV7fzv7zz/+X3n27B/DP377+PG/Fq//8/gN/a+P0dnbO2Wv", - "r0Yz+qaQRPeGQgRROSUoorasdIpV4PFGaUOngcL2ibGtVTDvoyPwmh+MWA/9RhURODpAo04lv2rUQRsE", - "bAL4Sit2uimbJrqpPz4zd2f64y9O4ftabSO0+aDCLkiWRS7TSchjTNnmiI2YbQu5iUjQgPVfIQpwosBx", - "QRnSlt4STQRUY7F3G3nnXfQFJ8nXzRGD6wFyrYSeQYKFyuDXXA/AFHZUJuDSvk5Ch+djrhdGLDuXMjgf", - "c8HVz9RcCGyopqv4ibLaUrE2wvOBD/gIQub1QkZUKqNsZ5yt2SiL5UfPB5t1y2WNNp3x0Ar2g51QL9Pk", - "mLLFXjIMDF0bwT12zrg1gQhaNpk9gsBWUhz+e45cQzktsiU2HnKTQCHNBauKZCF1YrPjBYmH1W05IXPD", - "CJ9FLVKuX5rcmve/nSNFROyyHTcCTc4pDfT8IHaSSplqVqQYHR6dvtzst6gzBbTNxr9iHd9nM6xmxtob", - "x6aL1NywwzHpopNjyG2yOzRX4CAm+RUXKDICJt/XB+iDJBUbEbLXICTSrGS0zK8tzQkw6my6FpOqpDhA", - "7zK9EWdDKdW2Kt+E5vsSmrVRKyZgutZ6t1Y1Rji7yIo2CI/GKsuw0ydusyho76iwFIc9XzGrb7y3izfJ", - "jUZzYe3vG+zu/tWdnZupO65wQTLH0sfd8+JVCLy0oloQrfh3RfO9f6nftQV7su6gBpE++oqf+0qt7PWG", - "w/fD3Zvb/DfFKSsDehSwZDKosvYYYw+B1VW3f6+pGjdGhCL92MZ/Oivv4ymaY8n+ouBhxdYb7jxrhTqv", - "e20bS1mMouRTM6RMSjl0kCwG0OCkXNIoMqG1ks4YjtALtHF+8vrXk99+20Q99PbtaXUpVn3hW58WkGVO", - "VLw++wDXaViOXThScwYOzrPYyDWVStbBU1pF9d0FIs182g5W3E3StJHji6/GWfulhIXmRcPZvEeANBeK", - "WSPjY0Cffcscl+8Pdm0lUNpd0c6s8fJAYGeNwt0HFFaW8+bn+4Ute5DhrC3SVzzrXQLirXHCuh3qSb46", - "lFoEkxCdnOVQ5bmT0TVfmZOteDkcDPrDQRuXa4yDFX2fHh6173ywbVSLAzw5CMIDMr2Dy9cytlHGcXSF", - "lxKNnLk06hj7rGCYFbatNalaXU/X4dhuh75WVWj8ctoodu7eX9ryNk3KTYvsmirYS5xGJoGzWBKnXuVT", - "JgbZzGACZ7rsiMEAuxZhJavMiYNApLk/w5VcM5pvmli+HzFBZMKZNIUY++hXspQopnCHkHUPkUMSZUHc", - "4YhtCBfwn0X2JziVJNQ/QDRt10Vt6qFRBdjH+oMRk/MUKsht9tERZzKNibCuHjSh4IfeRDI1xh2MF6gB", - "9UwlDYkYMf2aBzvtS6aoH+wPBoNBVtmuc7Cj/z3wcdMd8fPc5+1UDvt5rmv4eXgd3F47HL27gpetKodz", - "Xi6E09qiu0O1yFbB5k6ns2Hm9qvxTe7KCAp4GoXaTJjog8F4cUhonU2SqLzGEJwlH9gl0+xcmroNclMc", - "/ZESsUQfT09LF2yCTG0JlRYThw3VsA48udEybK8xrNeO5paQdo8BY1c91AvK1L2D1hU9/C7dzXBoC09/", - "blx5Q5QpM0uj+WTFnCo+2pAsxmnq09n1I5fk/uHDyXGJOTDeHz4fPH/Rez4Z7vd2w8Gwh4c7+73tPTyY", - "7gTPdhqKmLVPUbh91oHXQvNVpXKxh2MXA+ktPNwQgVo5Im1U3RVlIb9qVbI7692GUK3rvh4g2XoI3rBq", - "qHQMLTVIidNCPWMTVlipWdTgedp/Pxiu8Ty1K8PcIH/fi5QFBn0KJHHm0y0WYC4uVr0I683FKQzIhS+v", - "o1ax8/ZEGxzsvTjYuyvRXAjuujFW2ekRF7cp4MBBGFZifF2eScF/UahgDPqGcbPakOBOt5NFLcPfcNBW", - "IuKyx61C8Zs2bNcvRlbJ74aMtJOS2gy3sAbJKDzQWkCWzDRJFcoSHbV6cRTxNEQF348BdoGLkZOCCq2b", - "gXsK6xoyCF0mpFar2gAJCYDElGlBDBdCuhGbvnaAXsO78AjHxrqwg8AsrNyF4HBp7oL1/nJdG11/9ZDP", - "rZoP32idH0FhcT1tTQbrIlzdhNF8DtAbDt9kRgfjVV+jeR20/frrVb/khs0rc/nH0JlV4w7Qq0x1y5Q/", - "q+xtSGL/HFuBlaf9b5aSL+2KdzS35CtXyCvsdgxFO92OIxTkH9YzET/kXF/bf0VW9AVJEBzBXs4zvVJF", - "I4tyCTOhUFrbhgDrxW3SLywgPAnHxjBpCnky6UPWeMk+curLx1O0AXhWf0PWkar/tZmFR5XOuu0Xuy/2", - "n22/2G+FWpEPcL3aeQTJbfXBrdVBgyQdu6q1DVM/OvtgTPDAGLfgmbdzLyQJJ4Jr0aNnnpfBzTt/0X9R", - "BOsIeWpKgtshWWSfr4XC9ytrFjfE+PxBowWdTtkfn4PL7X8IGg+v9+X2xOvQzCvse70/J8Xb3pqrlEx6", - "poiHH08BGErIRsiRd0TCDNA5UQj4p4dwAKZDlpNmWc4Bk1iKexlrd2dn5/mzve1WfGVHV9g4Y/AFeQ5l", - "O4LCFoM30ca783O0VWA406ZL1AWAUGbNSv8+Q7YU2KCskPaHgx0flzQc3DnX2LYXcSPJP1rTzE7KEh1S", - "6zKzrbbLvdTe2Rk82917vtduG1sv5Vhcr5YwLvDckMfi2BZXfgO0yfeHZwjSuqY4KPtNhts7u3v7z57f", - "aFTqRqMCDGaDnXqDgT1/tr+3u7M9bIed44sCsKhQpQ1bll2eTedhCs9qeEhRF73dptPCp04ZBntHggjT", - "+DBwEbyV08dgpI6FeS1fhDYHg3WM1w6uFt+2chxVSk4b1YALlLIMmbu//grwdjd6zWLanAfrxXjdso8w", - "0+SyIA+mFMctaJcIsqA8lffQEFcm1WkacS5u9G2ThfKOyDRS5tqNSvTx9C8gRDRzIalIUjaaLPutgMK4", - "5eRutIFLPOHn6iZitVqNNku/asLdhm3aXZUHXdr+jYgzoRZVKVsffneEoyAF8HmcraeeFWBHQCZnkkRL", - "E6gaRZwzFMwxmxEoI2hKXbAZwmjOo7DvDR7UT8ZT77U9v0IRN1iZl4QkFpfdDEJ/pnUWuiBoo5BJigwr", - "Vcoz7cVGqljk7TI37sX+QkBY+pIfsgxGTU+seAHG0XxS8jFGfCbBClQQgtuvogcnWJjIWsxMnYFFbIzH", - "MvTOtj7tPUOsSG/fEWqOTj61Fq3VMSA/0FASB4JLiUhEZ4Bp//G0kna2IociSz5bH1JXHmwL1jUXaZ6z", - "C8402bocie9A9ASn3+VIBB6GHJQVwWrOGxljlgJSe4GRyXVChWGPdgFpcy7VOIMTueFgpRoDCncqSI45", - "lCVLZg4g9473XHSi7TbkspGft/q6xlX+ppoG2CxTvRT1U6ub8aCPjeuAKisxXHJQmCoCyE0gf3LYZiqh", - "VVpAm0EbjKuSWCpAD2+2Cc7w26i6n5p5aqtM/bY7OG+LxrMafOcMq/kJm3JPyvYNriGd69nGCyZExBRw", - "6FFIGCWhMx6z+0jr24I8v0gSFKbEUs4opAJbgmOzvSHtmjmnGGWziqyvdtjGH2zGsBqkG/q1L7YJs5H+", - "7LD3IgVamcA4iXCeJ9YqypDKsf/+qt6wILM0wgJVEadWDFku44iyyzaty2U84RENkP6gesk85VHEr8b6", - "kfwJ5rLZanb6g3Ge5lC5NDaDs0kuZkEq/eZT+EnPcrOSYgeuly3z/RYk+reJWvLG6r6iEbGgTB8YvS4w", - "ehnFdnd70JR92dBoKe+yDuh1U8ltWda34x3W1mFWNNNzS2mibit3pWVH5NqbPgjrXpVrWnfFoA0XCOVQ", - "gst0LaD1tvKEtIssr4b8udFsSRKUe999vvdsvyVc8p18nSaF/b49m4t4hUezYaVO27jNnu89f/FiZ3fv", - "xfaNHFQuOrRhfZoiRIvrU6mNW3Ga7UEk1eBGgzLxof4hNcSIlgdUqnN76wF9XbF1m4IL8r3ZdMkZFVfS", - "3bOUPaDtfIwrtKXDkspVqAK/QaZTAkbl2NCtlw+mkiHYagwBTnBA1dLjMMFXpmRf9koF7q2NN608WA9J", - "bdsWGkhLLplO8iSKDdc5+qtxrVd44Xlr1HWZTprc+G+rvRonfu4DKl4RtbihyQtD1t0F2XyusCzFmum/", - "A4iezKv8V2NmzRuroaeqgB9wCWiLCxQiKXyQhZXzz35UXP7KchbcviUluUrxVUdo8xa8kQ3tOZE9JnSw", - "PhOmIh/sAXi7r8aTYj2ElQUnSsUT8lP35v22SPapg4VmJ9jN+yukPdzkwyowFvCjHYMled52t8QSDdxU", - "CNP1mCM8Ir0szsHG8CKZGv+q3vMWa9GThxFc8um0DPi01wwQCMh8ELLtesFKkThRXUSuwUwnYQ1dztXR", - "3pOjDuICjTrDeNSpOAG9SRAxvh7bDsq5yoNViH1ZJaDqIKWbwSTiwaWB+leCEtlHAxQTzCRKGWz+io9y", - "OFjta+t2ksLaZPh4xNwQ18QWjGlC5nhBAZbVeqhmpTgWck2VhHgbaOcAhRzs23KdIztD/ZpJUTjIJw2H", - "DmZL27BuUL/HmQsIyt8Fc2kK1ZXYZyJ41ybfaYn99u1p19z/QOSGGVgpPMRN1IxAC8isiwrGaP67P/xq", - "EpExjLuKWRnX6VhMJQM/tSCSKGlB7HJ2qDABCnjKVBXMMm4XwlmOeK8fSSmDWAl7ewaQDbZ3Wx0xJIGt", - "QFt3L5UY/RbMXQm7tJT2xV3u+FgYNgV45vye93fWvV4dgERYkGKJMtNOMSzO+FzHUnGLaZ/t6jG5DggJ", - "q3A8/lfahhraL72hhr9hm/OeVQ+zb0O4WH12/YeLPYexNlG7GBLJOOtBtq1bUpsZa1BDbO51mdFK+HuF", - "DNSxD/zI90KbvClyvZrWb8i1ApDAMI00eZtY14oqexito/itky6aNjQX64ttPkDtBBOud6vqCTbS71EK", - "KNifH6RoQm05zoly755bvmmulVmCNy55BF2ApHulfEVpeKeL7ImOhvFmhed25343iAXoapkvwnBMxokg", - "U3q9glvMC8YSLmcj5zunVE9Voo0YX6PdZyiYYyErY2d0NlfRsnx/uesBA7hTKRFBFGHqBpVn89V0H9aD", - "BexyFlv3acPnhdR9f0VcEo5XwdgdZa+569gEL8Ft0+hjfbazOxjsbA9uhWN3X4V6C+00pSAUvrP3JKWo", - "nmILWcJXvZrTlaCQN5aRSSpBcHwAgcoJDgiKyBRQXrIqemsPi1rXqwdvNSibR57xv1sou27uCqMMFJ11", - "ZSEA3TQ6LgCqDB1QfF4f9goomEzMBDVMGE+Owk5vsP9+uHOwt38wHD4E8F1GpKbo2Gefh1fPom083Y2e", - "L5/9MZw/m23HO17D6wFqQpdzPqslou0cEiKqZbqq5e0kiSgjPZlFlK9P61ghC0yQxtr9fzPHvpnBSmXh", - "vDzJos6AVU6cEmc9Ek6GHf3K24nq8E+OVw/7ViHa1YH4Gaw6FOCndoMBINbhXVE/U9by3PlQeLH1ybMy", - "bWDd2ePL8oSt7V3lBor7+LkkGEs7bNWJXT/VPN7RGRdUzePVx0P2WoYhCHFmn6UKy7gMfXQyY1CUr/hz", - "FlZQNJP0x51uJ/q8W94z9vf2CB0WMy9jQLvURTWgxbU71HxcTQV4JTcthIn809a4HvNPw97wBQS/RZ93", - "fxr0XvTR3wtBeF1DrSL5hu7t0q+DNjQsVrpwCOnDFzeKUHP0XMVBv1JfnYb8ILZoepbH84po7qxwGUml", - "Bc4f19a4AiPQqHHeVbWzp9m4qCWFJMJLXx3rgitWVuzDIpOhCZlRJtt4ZncGmWt2Lx51+ujQYlCCtZoX", - "vCw1D6UOC3xC45iEVCuVxrhvjvjcbultqxoPNwMmdl951LO+Xz97sT6HdF2A+rpjsn+HhKU7mbvtTNxV", - "6c3gOXM2KZRAgRe7iE4RZpU6PLbqrM00hMwRQJg5cIAqOctaGSBzxc95QrpoxhXKcwxbetRS1uz5y8ZP", - "rsGjuiKp2DDE9r1kjGfoJXSV+Do5RongYRrkCTYRDDpPiRZpBcxxhVa/PoTpIR0akLk25QKtd2g0eTDa", - "eSCb1rvifdQM27zUw8H6pX4QL0i3kybhehlmXmonwW6ENbomZcPjkymTvaIJFibzqYVEf1ekYN3INd7i", - "QKtEaeKuUDRP1TnJc6EClwg+2L1jEhF9TNUbQTwK86hSKnMpul6kDvefz5suMeHOqT6QXwlJtK0CABHQ", - "X4zZ0juwapFVtDFwlbOkudLqGaxyS63y4J6t1cQal6p9vdqKV9sklBfLA2egm/dbrNZ+ubaU+EP44b6l", - "kvbWXi5U4NUc+F8GV+r6NxkzAM/Iosp5veu7gPexxXtrGTdBxlWzZopu5sPefxu3Mhr3D7Z++tv/3fv0", - "V697uWI3SyJ6IZlCKNElWfYA7x5pG71fBkwDrF6tTNva/orgGJxGwSUxTqoYXxfHuzfIhMbyDY5rU4AY", - "rJiy7N9rJ/S3f2uOYCqQ8QPIybUse2co64eoE6S4O442YiJmrpKxC7zf7I8YxPJfkqVEhWIEVqVxjPoX", - "mX2iVXRwWuIIXRg1sE/Y4gJNKNR0kSOmrVocBCTR1oQFZaemGB8H6SMIjort2KIILlHOXjmaiAGCPp7W", - "0Pbefnj/89sPb47Hb89evjk8Gf/68r8giOOqZ3oIe5r3dvf2bQm+IiWHniW+A/DvnVD8fOxmsMA8/AVJ", - "KVDX0KMwUwkppS6goPAy2iBxopau2JDLbdm8GTbZYdagN5ztnlHYBy/uo+jMh5VVZhY86mmNugFE1+vA", - "NLTwhmNDUybMvdPk2J55yoCfW2/ijM6wx5ftrUB6H8Vh3IDWgsbV1r+xtIM/OP64ijBspIEhVQURt2KX", - "StVrjp2PtSI1zms9liMyUmbTS2ghYKucSxIztWWLOPlSWkN98q5OKMp3mUMs6sFH6/NkVqryhZkVRtK8", - "NqdOY63o1CsIdKZJczUnghQWAj7IkVtvSDKb7NEiUdqUWkmIyAMhXaYI1D4XFLJHMmeDI0GWEFT3wK5G", - "5j3F11kP4L3HsnbHBfPIMfKHr38edTb76J0rVUqnrgkYRsWe8AOhlrloFU0cV9UXo8hV9Xmb970bz8qq", - "FdKvaW9VmDPvo8SaPn78O6bqFRdggTSnJT84nipYNyERgMtSRUttBTVKYxKOs4LNTfvf1Wg2OclZOey8", - "jJOztjAwsRZy60vtuMTZfAx1SmtykCAVVC3PobCriRAmWBBxmJoN7+rD2p/zjqEq0tev4KecerIQXhNG", - "BA3Q4dkJ7McYM1DS0cfTQiUTU9SmhqEG6uXboxNr4ToYPrBYqALWc8F8h2cnnW5nQYSx8jqD/nZ/AJs5", - "IQwntHPQ2ekP+4OOqegLU9yCIorwp00wzCylk9DqQT+bV/RXAsdEESE7B797EvUgkA1eBn0XzwoWS4Kp", - "sCZLEkH6oGEVqr8FYF13lB6Y89gW5G3toJNqaZMpSPLWLusnUCdh18AUtwcDCzOq7MELqSAm/nzrHzYY", - "Me+3lT4H5PGgzNYsCqdTWpJ/7XZ2B8MbjWfVMGDH+rr9wHCq5lzQzwSGuXdDItyq0xNmMryQwQqzgTbF", - "fQYsVNxhv3/S6yXTOMZi6ciV0yrhskkZJhJhxMiVrQj6Dz7pI3v9AGVj5JynkZYmyKSvOUeDwqI/+4yw", - "COZ0QUbMntNxGimaYAFuhBjp89kYTOWtYbo2q58BC/zMw2WFullzW7q5nnM65wSupiVIMgYc4nFTud/c", - "3UwZ02ISS2JraWR1L+vRPFpcjmXAfflE7wnDTPVkQgI6pQGCl/XutR5tb4OtoPm0wINlIQIQs5yHZnvT", - "n5MKVT/86dzH2TNkyVtWJxhcAwVRGuY6l0uTwmKCo8iL3TSL+ARHY0OfS+JRUV/DG5YoxQIpTrlhPCSm", - "2EWyVHPOzN/pJGUqNX9PBL+SRGgVyBYxs7QmoSlbZlj3CrA+YygkZkqk6j63zBC3vlyS5df+iB2GsSt/", - "K80nOJJcn5q26KDNCzBb2vCuvyxLQ1zJUSoVjy1LZRUY82HyVCWpsnfqkihbeQ1epxIlqZyTcMQUR18E", - "mVGpxPLr1pe8x69guxAcaj4pvGKmtPWFhl+bRi3HWM9+DK96rD8CBBh19Oky6ui/ZwJr2yWVc3CiSHCc", - "zIpLupHh6Wi9cLNK4QAzlPDEYBEBU82xZrlSGxCPjqMIKdhK7lutbcJKNszHphfHk8bcYpMMWtlGlKHT", - "nwubabD73L+fJAkE8Tk4/vP87RsER5VeA/Na7rAyl9pMn6IoTEGTh977I/YSB3Nk9CbAmx11aDjqZNZF", - "uAljTaWN0O71QMX9SQ/tJ9NNl4Y/9fu6KaM9H6Dfv5hWDvReSuKx4peEjTpfu6jwYEbVPJ1kzz75CdqU", - "onleEgRow8j+TVeDGKCi8mPQnBuYhYhbWRstEUa5BCr6USaUYbGygLKH9JaC2pTHM1kkxpcR+G5HnYOR", - "896OOt1Rh7AF/GZdvKPOVz8FrBLdDG5qakg7XTtjov3BYHM9doKlr0eFLr2ot9/Xmva1fW+Kh1W66oqH", - "mZxDZtYraKqBG3XrETSfn3Ho6kv+UPHWqHjWc1FQ3uD74jlg2DcixsCtaGDano2cBrbSOjFsAQUTwOJw", - "SCfG4KBOg8uZt2h+VM35ulmx27TLAhhi5Phv9xH4D/rNShGafl88Vr84ApxxB1z/xNgRFssxYtdvEb8m", - "6nvguMFjiVILiv4t+fep8M9rYvW+nGgVabZFFu6+yY/nBMkm0rZiXta26jmMqXdOmEIv4de+/a+zeCBb", - "+iLis4sDZEgY8RmKKLP3gIXbIn0oWlrCRybfJPvOpp84MM0Nc37+83/+FwZF2eyf//O/Wps2f8F23zKJ", - "kwC+fzEnWKgJweriAP1KSNLDEV0QNxmAxyYLIpZoZwBqZiLgUbGkktVN5IiN2DuiUsEK96UG11LaBsH0", - "YDAfylIibb6OfpFOLeiWcTB7THi3lw0pH3VHdz2JzjCDwgT0qeh4ALJEbfk1a391/N4zM+eS/6zqK695", - "TNfLF0WuleHenhngDQUMkNi37+CBnTTaOD9/udlHYGMYrgBgNdCY82as8tz/IZPWyyQjUcoCBahsZJNJ", - "Ylvt/z2277RzANsW/0weYIu1eQMXsHF5QO66W4EftkILd7Cfbs417PPPHrsszWYH7e3nW+zCxTG1MoTv", - "b50d79Vpbp4USPYtTGC04aLhwY3IBTo7OnE1bTe/GdM/yqmhZ2pr9WVHB+IMsNcezSw74mwa0UChnhsL", - "FGaKSWaqlRnkqYiDd3bUCLt5VSGMi+fbVgmRr/Gky8D58iPv4U+PSqc3OUZymOWc136cJOtY55jKgOtv", - "C9zSC3AChHTqS7ZPi1y0ziFlguuzI2elumTF88mx25CP55qyXaesejY8glA8rgjEbygIK8XjC8DkT4mb", - "P2Sr6BApVniuvi/WHDyeFvTYXiwfmz8lN1ZYIZuWgiaou/EAfU2UCeXuPOBC2x48Ez8nwu1qV0cCZp1N", - "y3xqCquaCcGF9Grb98S80s70Ne39mSxfIM9NNBZL8h8qSgtjN6fVKgP3xJYsfzj7Fnq4kXl7f/e8lsE8", - "RIZgk4nzWJu6u1guWbD5p7rqfZTTzBD7SR5mZ2kUuRuPBREKvT06MTureAZsfYGwpPW6vdttK4+DD+9+", - "6xEWcIhDy2Ko/EqUfXLPGr5ZMDOVH2zSxiY0adHUnWdNGs4d1t+ECyIT4din/N+3X0V0IrBY/vv2Kxwl", - "lJF/3zmMsCJSbT4YswweSzQ/tsb9hJlPK9y0TDQQTQzqva/TULO3Wiqp7v0/lZ5qJn0jTTWj6w9ltY2y", - "WiTXSn3VLsWDaqymj290JZMxm4/a8MjFJ/7JNNXH9fJZjnRA0VSWrz1skT0uwM8LjyhDqSRPMICSZhxX", - "PDZauqvzDbny+HCse3LcBUJCXQRAbbIJIo/kvHbjeHTl1vb7+J7rw3hCZylPZTH3JMYqmBNpk5UiUhbA", - "T03tzo/nRsX7O+bSwWMeHY+uV//g+wfS+KsLaoS3uYFap/O7t9rq/PZ9rfObFGqbu2ahpboOdnCzIajQ", - "JVG3ZeNSrnk92NE3Lp8tgj5oQyU3FxBYEAcj9n+0/fG7Ijj+9JNLkkkHg+19+J2wxaefXJ4MO3WsQhhU", - "PAKU2MM3x3DtN4PscwCSzVPyquMwlSeA9Rx0zr+cgZTffLa3kBwX/rCQWllIBXKttpDsWjysiVSG33p0", - "G8nxm4/gFsTkh5X0GFaSTKdTGlDCVF7ztBYkZksmP8HcMmbvhwrBHaWDtrWVlG3KNQpoXhbg0QN7TnIc", - "xMc2jlwFgqcZI88TC+ltzZH8MGy2R743fhg8rnB+fDvkKbOYUfjrpEu0TukrpQ0Yk3EKZSFRjhACUZ9I", - "2PqPrsU+yitYyzRJuFDS4FSCAmyQ7OdaAfZhWpZhKn24lIDFSInsjhhUKtCPTS7/1iVZGhRKylkGOFmt", - "x+rLvSqjgH7TbXT/OpYf4rSVjvXI29iCVn87HeubiY5H0bROSrUANrKNAQblhGQ7mWfJffQzZbPNJxWB", - "aoRVNrcCnpFH1dqCSn8W13dLZtVkmw7aArSvLT37L3ji1ifp09odFm2BgCikeMa4VDQoVt4twoP+OKFb", - "n9CrKevl5qktke436F9xcdn2iPPUFXsCJ11xht+hL0EPD9DAvr1LAYxtcxpopnn0U7BWLO5bpmDQ6rkY", - "RGkIVentgehUyang8dj+aPBq9a6waKDgoghsq99a2OjeH8Fh9IYrROMkIlqLJyHqGW7Sq2lVfwc3T2Wh", - "tOLNhKHeNsWEGANGJ11pIisi4XLNLdgG3LPXl8srNSM+Ww+CkXXuEB88KBgjZuDwicPOv0CZkIXiXSQi", - "gUJXcxrMAREDCnpBRVcAq8BJcpFBYG0eoNewU4tIYND5hiRCG0IBZ5JHxABdLOL44qCO2Prx9BQ+MmAY", - "Bpv14iAruZ4dEFK/VUS4yGoevbG4HRuakwSPIrOiF9pqLMxv02Jf5BBlI+bDwWDkyjZIp+iiAIlx0YCJ", - "4QTqb3z2zbStbjOwpJmL4kgA4QxvEhZ2mi5iaORHwxgOvPVgWiJzmGE8MDBHbTC/8VkGalliZZwkbdnX", - "DhO4eBHHK3gYbRSKs0oV8lT9TaqQCAEfW+5uYm60gQPzD4UvNaPawkJZeVtgP+91o0GZ85JKC9VCFR3z", - "r0Ucd7odO54COt0NtPc1CCfVBuvXYnplCjAmP/TumwCUlIV9AaGkcnLY8v/NKvc788Kf3j9rCRX+Gbws", - "5fusfBSU5QWgBFRwd1W4nhTSASxkTRczhZF8e8TNsicLxUPbXW/Vyo5+B0bruluvrIZkVuDysa+/6iN4", - "ykkwsjabKRfV9Ph192LfPSPd35LUptqGQ37w5s3dc60YM0lX1BKFUqgS/HxQXxNwnYM557LA9hMyxwvK", - "hUVgt17XjDPBZWGsRxs9d6FZ9cL6by+sen5gfU0IFx/ZPvrwuY2583/hHuVfvCpY25nE7zqVGlAgJcJo", - "IiiZogSnkmhtKY0JMhVGLJA3wcHcVQvvj9j7OUG2PmbBgZCVU6YSXQzjiy6apApFWMzA2jEPTSSdIAGP", - "Y8JCU/N2xOYEL6g21QSKsCIsWPYkgRrIC5IXMNGmu72hNKW2syqrXeSK84KD4aJQevcCJYIAExlzmZXq", - "3I6YSNl/GORK3eyFG+gFIlLhSUTlPKsVEeCQsMALC3n+fYux+3finhNVr077Te4sbyVLv+UlZtGXmdUH", - "/y7uN59YoBYXrrJmCzG/QumVzaZhOfLxPK/I+y+4pc1c3Ry/0c1MRuJVu/j7uJIpleT/cS2j7JYMU9Md", - "KZet/9PeteR1pFNWum6xPtnbXrhklRAyMt9I5m19cX+e3MJH9p1Iwm6jYd+EuZ1P+nsQuZaqt5K538g5", - "aH1JBa/YNxTBdlDfTn3ioiDlvgsxbDZcJo2LMkcJDDYVZz+EcVUY2/CA2wpj53GtXYAXxDNlvSTCTXI5", - "r1rvF8DWIfAvGv1amV1BEH5zwZffCDyasDvJxJsReAleRhz/2e9lAi6ESei05YifDqBYwRdYuGDaAI9b", - "N5MQXZdN8vH0dLNJSgi1UkYI9YQlRLmsaRB7qjW+XRAhaOhKRx6dHtvoVSqRSFkfvY0p1HO8JCSBQjGU", - "pxJBZm5fz8+lttaL4JVyWLsdwpRYJpwytXYU+asPM5ivtyqd98hy0kIq/ukvj8EL//SEFMgOra7YCay2", - "IhVWjcF4LjiNMlPvUmtbeMJT3bqWLK7Q7gzOtimNiFxKRWITmTdNI9hEALprazLZ70xGaRdRJZHeD13I", - "wEuIiKmUlDM5Yrb8e0KE7lt/DsV/8yAjr/Ne4UxqnhnR930EsOnBmJgtrJqoBtACUAe0c9DZwkmyBeWi", - "/UFSdnh3GNIriEhDchlPeEQDFFF2KdFGRC+N0YEWEkX6j82VIW1j+O6+K07dfmdpSp+wKfcW5TA8mzHz", - "nyMJqSzW3CXikxNrr0lxszj5AwvtF2tyrVwTBEc9RWOSJb+jVNGIfjaiTjdCpaKByavJUy+hCLPNvhyx", - "U6KEfgcLggIeRSRQzrmylQgebI3SwWAnSCiglOwQGBwIvObHMfR4dPYB3jOForsjpv8BDb8/PDM3sVNs", - "fQSFgTKirri4RCdbb9cE+Z4Dmf6Fo+TMBFfmQHoX/Mf13c0zmxv3kGzYojxZZQDx5E8fxmk1uB/egqfp", - "LQBoiWw2GzOBA1CK5TxVIb9ifs/AgkdprP9h/jhZB1CicDD/CK9+N9quGc7abtwEn8SmtHMKiSka9E0u", - "KAzBnmp8qSacmwIoMaXIPe8pcKj+jNx9/075Ih2/w6tJS1FXkOu72VuPffLZMTjcrSI9nso2N5zmZqL4", - "au/TFabN3qefIx5cSpQyRaMSqIG22wAHVP+Y4zbaiz9QEyA70pUSR+Q6oQIQbCrwCIjoGUuEkSIipgxH", - "WzBn0wggUDovFl5wCknKQUQhTYyGBCU8igBl52pOGNKzAUeVa6BwTyttBYjiO8UrRsXRhAQ8Jg6Vc9Nn", - "uv0dU/WKizLE5vciF98X6K/no6eq57kGVbS5xzuhjJ7iawhrDlN7TexGtPGa5z8aV1AXwdqMOjsDOep0", - "0aizHY86egWOMLhQsUJ7KKYsVUT20bHxb0Ea6v4ASRJwFkoHDuo8eDsD2ZSUatiyIcNxH757TLXHchWQ", - "8p3txCce9HtIfw8JNmijuOHsngy7sOlCxFMFAdxuX9m3QqLAPbL56DewhT3yw7ZvI8n/brdvSUbBKmtx", - "WVh6I9kz+Mi1XjeXVDHnMkedRAFOcEDVsotwFPEg9x6kMrsd6GVDmQiCL7UN1R+xdxlwpU2EQEdnH7rO", - "aYZCKi9NC9Yv1kdvF0TIdJINDoE0MB48WAwSjpjiKMBRkEaab8l0SgLIYYhoTJVs8KtlQ3nIMoh5J56F", - "dw8z2Jqn5Uzy8wSsXs4WssJxW2aptwQJIkzjolOpShxQfeFKF9y+E90o18fwNLLXW4HgUiLbVI9EdEYn", - "kb2skX30XqscOCYjlkSYMSJQKk3ckR56LxFEytQkxugGoM6s4aguyoFOEsGVdRNHnAtpPLuawz+eIqlI", - "soLN3pmWT2HODwQTbBq3PX0jg6EyhuZjyb6C9IIYTjEE13ykj+lvEOxjBvSt4YSfysZ/L+hsRoTeFdgI", - "WXM1ara1I6fZ9KVMj0aM/PPsrXYY+VmrhWjuQqTzSqCKsXtxDAr0TW5gPZ1f0kYsE/voZtkXv+qPWvZd", - "jvL3D8I+uuMs/yylx84LwdVtkfVzDn9qIPeFkZe2ailBYT0cQeuMhIfMEGiNO/DN4AaeMsoALqUdNMEJ", - "fH+MMHjc7LjHhtl+2rxVQgkoFdZpSJVaD9/5XXDgw+B2fuPs0Fvgdn5X+UqAu/jt8ka/q0ylkh/QFQ/5", - "0yNzPlSCkoHnBBiLpgQlI/VsIMFKQ+mjfaedmWRb/DNp8Pbu+Qb6uyP7D6u/hclQIJbfZWdyox1uC4kT", - "tXSXi3xauQCU9DMkY/iAH7IYgofDW7jF9fr9sYfj08bL9R/1tB7t/j4vOnxy/PSLaBX3XOlg2dKnTg+L", - "YE4XpNnpXt7BlkSJIL2EJ3C5EhqCWXq4s0xh0Z99RrZ5i1Vl/4WogzgmIQqpIIGKlogyxUEimD7+IpHg", - "2hKA51wsfc704s59JXh8aGez5jy0e8o6w/I733jZC7HCvYWTNitcaHe4aXd321rgIcrQ65/RBrlWwiDu", - "oqm2fBCdZiQl1wEhoQSe3CwOeDho8GzSz2Q8m7QZ5Qrs5LcWmxoFqVQ8dmt/cow2oNjCjDC9FlrVn4Im", - "mwi+oKEpRJoTdcEjQ9VhA0Fv6nfVSkVWKcMZF2Zw30SHaXMgzT7TpCwWTOhC56AzoQzD4NaiFJf3lEmo", - "0v1hCmkN+d5xnNP5cYRZy2/DGTuaE7WR44ioODfQeJs/jrmnfMwVA1PdmVY67dqVimwXq9oyhPQhAHOz", - "OObHdVt//H7CK6l8kpGV1nW+yAzSJrf598WCg8c7Hx7bXf7xCYfjvybO+C64yqEB3aKPYX7jAY5QSBYk", - "4glUkTTvdrqdVESdg85cqeRgayvS7825VAfPB88Hna+fvv7/AQAA//8QadjqX44BAA==", + "B7VWY9Gc8h7eP4fXv3Y7V5gqSPwvxeGsjq5gGQ31WqWRKq21bhEIbaJrCy4/xeEywTq2LEqMICoVdnmL", + "oSjg2UTnRFmHo+L2VYSnylp2H0+NkSbTGDxlNCL1XqeUUVg9LJcsmAvOeCqjZStvYttEAU3YtY6mFnzJ", + "yNVjMCVkUvT0tu5JhpOH4chVsX6ZLyZ/CbQUQUPSRyB9IOjIZW5WJNG54klCwsw/1h8xG2mf/STNDRMw", + "KdBBzQkViAs6o+WOyw7IhwwavMlWddz0Y7s+0HYtErZu4cBDCP9pPjTMxKYmRtEk7Rcz7yyTdrqd8wxT", + "xVK9zDrvMlyaGsfkQcK1Ib4++3DT0MhE8Cn1IWVBLI19ai17FzT42+7gvDf8f0wAsN6PoOJTZuJvYh5W", + "IFDs++00l9dnH86axpSBkqDi6GpzyiKmVsGyOYrYS0l7q20tYCcetGKSdZLbbi98ttBU4JhM0umUiHHs", + "cc6+0s+RecGExlGGTn8u20Pa7mrrdTkrLQ64XaY4sJgS7ajvcehWptEtUPOTf7neEaPGNWWi6qUS9h2b", + "jNpHbzIYGPT67INEeZSbx9NbXt7GTI+z+VLSAEemRZNYTlnRQQvM2drCOss/tK5sj53lRw9yGwFtLGZJ", + "Ctvw/F3v5O3HrTgki25pTBCZNucR0ePeLEiLhctHzdNSSkJi0eQpM4wh226gAq2yHdyaSIX96qGO4gpH", + "YxlxX7DPe/0QwUO08fGVyRfUI+iipLSU+vcCFUr8ve/dMVoiNXV7Dh1WXe6lDe71PZRxXI17rjC9Uqe+", + "rWKSNOo6YB26i1+WF5pfroeLMo0093vk8kgquoA1RJBJN0GQbuLiF5D51JzQ1rUmSYIFViRaGs0rO/oi", + "OiXBMojMHif1q2hyTYIbJLK81K9/NXnoqSBjNRdEznlUDmDY6dZhAyUE0S6IhW8xcypc3CiOYiwu4WB0", + "hhhKmaFAOWllZx0k6lyp5AaT+uX9+zPjnVFELExEYjFIXNau5o9JhJdoQtQVIcxNBUuE0Wue4eRUs7pk", + "A4SGUOOECMrLNOzsePo9N4G9aCZwQJD5yil8dkkknJptSWl78UDBBQGRsmF9h6vW1346TaN2a+wb1nAt", + "6G1wkwV+f3TmoCAyVEtH5u06lc+I6Jkt5+AtVy/ttlwNSuK6YpyRemeCTwiglNjErWKKmQtMAhQW/Xkp", + "rasgHGQxn8r2A7vAkKpr9vmnVspedbv7AjFizEIfOqiJmDe5s7M01gsCFkYKd480NFFLxpYwankx8F8Q", + "HFJGpKwkHgapiDrdTm9qZ3WwtRXxAEdzLtXB7s7w+dbq+M+Vgb/WnhmHdJX966weEy/j8vMMSidMuswS", + "WzhJWnhQDR3XnA8gnuqBhoAIqs+2gobnIs8Hg1oO5TUOlANWApdq6cocF7dtolmyNB9oMIOl3nvxorg/", + "B94YhhwV3rH/Vo339cxMMCJYoeCOqdDRMPlnr0bFhQ89hwtlM6QmxEUrZuehiwW0l3Klzp4Pnhdn2Qp8", + "G4RNZZvbfeeZqnm7nDVJCuS2+9c2AHFVZZ3DbenVWHuaLmt4SkvEGkutJihPCLsRPfd2d7ZvRs+2Ezlx", + "SZ0VueTDbDg6PTY6UcCZwpQRgWKisK00UBAy4GvTUkYb+CEmMeS5TP9jtWhpiH8pgjA0RlAc1eD6HiR6", + "ogFm6p2J/w1RjBmdaoFs3yz2LOd4e2//wIDhhWS6u7ff7/dvmpr+Ms9Fb7UUWyb9tpCl3pfzu63DA2Sg", + "t5nLl87Z4ftftCBLpTCH1pacUHZQ+Hf2z/wB/GH+OaHMm7neCj+RTmu4ieV4Qm3wm98PClD3Tu9pBWft", + "d5ZDWDHAZXhhhhSead3GcNxd8YRujTiYw96qAtJgMRurBeog/bw6DMF5leAd22fKFI1yQMZ6AMKtIDXl", + "StSxGuJYQliGMxZF5q+As4XeFT7QsdJJ5J7dKXhnper197rGtX6/OcVrDdv6vWyZ/GsLtmghkTwn0TeX", + "+rcJcCv3/nb2n3/8v/Ls2T+Gf/z28eN/LV7/5/Eb+l8fo7O3d0I/WI2G9U0hre4NxQqiukpQVm1Z6RSr", + "wOON0oZOA4XtE2Nbq2DeR0fgNT8YsR76jSoicHSARp1Kft6ogzYI2ATwlVbsdFM2zXhTf3xm7hb1x1+c", + "wve12kZo84mFXZAMhUCmk5DHmLLNERsx2xZyE5GgAeu/QhTgRIHjgjKkLb0lmgio5mPvNvLOu+gLTpKv", + "myMG1wPkWgk9gwQLlcH3uR6AKeyoTMCufZ2EDg/KXC+MWHYuZXBQ5gKwn6m5EBhTTXfyE2W1pWJthOcD", + "H3AWpFzohYyoVEbZzjhbs1GWC4KeDzbrlssabTrjoRXsBzuhXubLMWWLvWQYGLo2gnvsnHFrAlm0bDJ7", + "BIGtpDj89xy5hnJaZEtsPOTm4lOaC2gVyULqzWbHW2QAVrflhMwNI3wWtUjZf2lys97/do4UEbHLlt0I", + "NDmnNNDzg9hbKmWqWZFidHh0+nKz36JOGdA2G/+KdXyfzbCaWW1vHJsuUnPDDseki06OITfO7tBcgYOY", + "9ldcoMgImHxfH6APklRsRMh+hJBas5LRMr+2NCfAqLPpWkyqkuIAvcv0RpwNpVQbrXwTmu9LaNZGPZmA", + "+1rr3VrVIeHsIivaILweqyxDU5+4zaKgvaPCUhz2fMWsvvHeLt4kNxrNhbW/b7DE+1d3dm6m7rjCF8kc", + "Sx93z4tXIfDSimpTtOLfFc33/qV+1xZ8yrqDGlb66Ct+7ivVs9cbDt8Pd29u898U564MCFPAIsqg7tpj", + "1D0E1lvd/r2matwYUYz0Yxs/7Ky8j6dojiX7i4KHFVtvuPOsVdUC3WvbWNxiFC6fmiFlUsqhy2QxpAZn", + "55JGkQnNlnTGcIReoI3zk9e/nvz22ybqobdvT6tLseoL3/q0gLxzouL12Qe4TsNy7MK1mjO4cJ4FSa6p", + "VLIOvtMqKvQuEHvm03aw9G6Spo0cn341Tt8vJSw9L5rS5j0C7LlQ3hoZHwM671vmSH1/sH0rgfbuipZn", + "jZcHAstrFO4+oLmynDc/3y/s3YMMZ22Rx+JZ7xJYb40z1+1QT/LeodQimITo5CyHus+djK75ypxsxdTh", + "YNAfDtq4XGMcrOj79PCofeeDbaNaHODJQRAekOkdXL6WsY0yjqMrvJRo5MylUcfYZwXDrLBtrUnV6nq6", + "Dud3O/S+qkLjl9NGsXP3/tKWR2pSblpkZ1XBguI0MgnAxZJK9SqxMjHIeAZTOtNlRwwG2LUIPVllVxwE", + "Is39Ga5kn9F808Ty/YgJIhPOpCnk2Ue/kqVEMYU7hKx7iBySKAtyD0dsQ7io5CwzJMGpJKH+AaJpuy5q", + "Uw+NKsDO1h+MmJynUIFws4+OOJNpTIR19aAJBT/0JpKpMe5gvEANqIcraUjEiOnXPNh7XzJF/WB/MBgM", + "ssqInYMd/e+Bj5vuiL/oPm+nctjPc13Dz8Pr4Brb4TDeFfxuVTml83IhpdYW3R2qjbYKxnc6XRaGD1+N", + "b3JXRlDA0yjUZsJEHwzGi0NC62ySROU1quAs+cAumWbn0tRtkJvi6I+UiCX6eHpaumATZGpL8LSYOGyo", + "hnXgyY2WYXuNYb12NLeERHwMGMTqoV5Qpu4d9LDo4XfpkoZDW3j6c+PKG6JMmVkazScr5lTx0YZkMU5T", + "n86uHzmQhA8fTo5LzIHx/vD54PmL3vPJcL+3Gw6GPTzc2e9t7+HBdCd4ttNQBK99isLtsw68FpqvqpmL", + "PRy7GEhv4eqGCNTKEWmj6q4oC/lVq5LvWe82hGpd9/UAydZD8IZVQ6VsaKlBSpwW6mGbsMJKzasGz9P+", + "+8FwjeepXRnvBvn7XqQsMOhlIIkzn26xgHdxsepFfG8uTmFALnx5HbWKnbcn2uBg78XB3l2J5kJw142x", + "yk6PuLhNAQcOArMS4+vyTAr+i0IFbNA3jJvVhgR3up0sahn+hoO2EhGXPW4Vit+0Ybt+MbJKfjdkpJ2U", + "1Ga4hTVIWOGB1gKyZKZJqlCWCKrVi6OIpyEq+H4MMBBcjJwUVOgDl55nXUPF9DytagOkKABaU6YFMVwI", + "6UZs+toBeg3vwiMcG+vCDgKzsHIXgsOluQvW+8t1bXT91UM+t2o+fKN1fgSF6fW0NRmsi3B1E0bzOUBv", + "OHyTGR2MV32N5nXQ9uuvV/2SGzavzOWvQ2dWjTtArzLVLVP+rLK3IYn9c2wFVg4bsVlKTrUr3tHckq9c", + "Ia+w2zEU7XQ7jlCQf1jPRPyQc31t/xVZ0RckQXAEeznP9EoVjSxKKsyEQml2GwKsF7dJv7AFBUg4NoZJ", + "U8iTSR+yxkv2kVNfPp6iDcBD+xuyjlT9r80sPKp01m2/2H2x/2z7xX4r1JN8gOvVziNIbqsPbq0OGiTp", + "2FU9bpj60dkHY4IHxrgFz7ydeyGJOhFcix4987yMct75i/6LIthLyFNTUt4OySJDQc5Km5rXDTE+f9Bo", + "QadT9sfn4HL7H4LGw+t9uT3xOjSzjvzen5PibW/NVUomPVMExo/HAQwlZCNkzTsiYQaQtgz800M4ANMh", + "y0mzLOeAbVwas4+xdnd2dp4/29tuxVd2dIWNMwZfkOdQtiMobDF4E228Oz9HWwWGM226RF0AmGXWrPTv", + "M2RLyQ3KCml/ONjxcUnDwZ1zjW17ETeS/KM1zeykLNEhtS4z22q73EvtnZ3Bs92953vttrH1Uo7F9WoJ", + "4wLPDXksDnJx5TdAm3x/eIYgrWuKg7LfZLi9s7u3/+z5jUalbjQqwPA22Ls3GNjzZ/t7uzvbw3bYS74o", + "AIsqVtqwZdnl2XQepvCshocUddHbbTotfOqUYbB3JIgwjQ8DF8FbOX0MtsFYmNfyRWhzMFjHeO3gavFt", + "K8dRpWS5UQ24QCnLkN37668Ab3ej1yymzXmwXozXLfsIM00uC4JhSrncgnaJIAvKU3kPDXFlUp2mEefi", + "Rt82WSjvDL4GXLtRiT6e/gWEiGYuJBVJykaTZb8VUCG3nNyNNnCJJ/xc3USsVqvRZulXTbjbsE27q/Kg", + "S9u/EZEn1KIqZevD745wFKRQvABn66lnBdgRkMmZJNHSBKpGEecMBXPMZgTKUJpSKWyGMJrzKOx7gwf1", + "k/HUe23Pr1DEDR7LJSGJxfU3g9CfaZ2FLgjaKGSSIsNKlfJee7GRKha5vcyNe7G/kBSWvuSHLINR0xMr", + "XoABNZ+UfIwRn0mwAhWE4Par6NMJFiayFjNTp2IRG+OxDE20rU97zxAr0tt3hJqjk0+tRWt1DMgPNJTE", + "geBSIhLRGdRE+HhaSTtbkUORJZ+tD6krD7YF65qLNM/ZBWeabF3OxncgeoLT73IkAg9DDsqKYDXnjYwx", + "SwHpv8DI5DqhwrBHu4C0OZdqnMGJ3HCwUo0BxT0VJMdkypIlMweQe8d7LjrRdhty2cjPW31d4yp/U00D", + "bJapXor6qdXNeNDHxnVAlZUYLjkoTBUB5CaQPznsN5XQKi2gzaANxlVJLBWgqzfbBGf4bVTdT808tVXK", + "ftsdnLdF41kNvnOG1fyETbknZfsG15DO9WzjBRMiYgp1DFBIGCWhMx6z+0jr24I8v0gSFKbEUs4opAJb", + "gmOzvSHtmjmnGGWziqyvdtjGH2zGsBrkHfq1L7YJs5H+7LD3IgVamcA4iXCeJ9YqypDKsf/+qt6wILM0", + "wgJVEadWDFku44iyyzaty2U84RENkP6gesk85VHEr8b6kfwJ5rLZanb6g3Ge5lC5NDaDs0kuZkEq/eZT", + "+EnPcrOSYgeuly3z/RYk+reJWvLG6r6iEbGgTB8YvS4wehkFeXd70JR92dBoKe+yDuh1U8ltWda34x3W", + "1mFWdNVzS2mibit3pWVH5NqbPgjrXpVrWnfFoA0XCOVQpst0LaA9t/KEtIssr4b8udFsSRKUe999vvds", + "vyXc9p18nSaF/b49m4t4hUezYaVO27jNnu89f/FiZ3fvxfaNHFQuOrRhfZoiRIvrU6mtXHGa7UEk1eBG", + "gzLxof4hNcSIlgdUqpN86wF9XbF1m4IL8r3ZdMkZFVfS3bOUPaDtfIwrtKXDksqV13ZGG2Q6JWBUjg3d", + "evlgKhmCrcYQ4AQHVC09DhN8ZUo+Zq9U4N7aeNPKg/WQ1LZtoYG05JLpJE+i2HCdo78a13qFF563Ru2X", + "6aTJjf+22qtx4uc+oOIVUYsbmrywaN1dkM3nCstSrJn+O4DoSRcnXo+ZNW+shp6qAn7AJaAtTlGIpPBB", + "FlbOP/tRcfkry1lw+5aU5CrFVx2hzVvwRja050T2mNDB+kyYinywB+DtvhpPivU0VhYsKRXfyE/dm/fb", + "ItmnDhaanWA376+Q9nCTD6vAWMCPdgyW5Hnb3RJLNHBTIUzXY47wiPSyOAcbw4tkavyres9brEVPHkZw", + "yafTMuDTXjNAICDzQci26wUrReJEdRG5BjOdhDV0OVeHfU+OOogLNOoM41Gn4gT0JkHE+HpsOyjnKg9W", + "IfZllaSqg5RuBpOIB5emVIQSlMg+GqCYYCZRymDzV3yUw8FqX1u3kxTWJsPHI+aGuCa2YEwTMscLCrCs", + "1kM1K8WxkGuqJMTbQDsHKORg35brZNkZ6tdMisJBPmk4dDBb2oZ1g/o9zlxAUP4umEtTqM7FPhPBuzb5", + "Tkvst29Pu+b+ByI3zMBK4SFuomYEWkBmXVQwRvPf/eFXk4iMYdxVzMq4TsdiKhn4qQWRREkLYpezQ4UJ", + "UMBTpqpglnG7EM5yxHv9SEoZxErY2zOAbLC92+qaIQlsBeO6e6nE6Ldg7krYpaW0L+5yx8fCsCnAM+f3", + "vL+z7vXqACTCghRL3Jl2imFxxuc6lopbzP9sV4/JdUBIWIXj8b/SNtTQfukNNfwN25z3rPqcfRvCxeqz", + "6z9c7DmMtYnaxZBIxlkPsm3dktrMWIMaYnOvy4xWwt8rZKCOfeBHvhfa5E2R69W0fkOuFYAEhmmkydvE", + "ulZU2cNoHcVvnXTRtKG5WF+s9QFqS5hwvVtVl7CRfo9SYML+fP9FJXzLcU6Ue/fc8k1zrdUSvHHJI+gC", + "JN0r5StKwztdZE90NIw3Kzy3O/e7QSxAV8t8EYZjMk4EmdLrFdxiXjCWcDkbOd85pXq8Em3E+BrtPkPB", + "HAtZGTujs7mKluX7y10PGMCdSq0IoghTN6hcnK+m+7AeLGCXs9i6Txs+L6Tu+ysqk3C8CsbuKHvNXccm", + "eAlum0Yf67Od3cFgZ3twKxy7+yr0XGinKQWh8J29JylF9RRbyBK+6tXArgSFvLGMTFIJguMDCFROcEBQ", + "RKaA8pJVYVx7WNS6Xj14q0HZPPKM/91C2XVzVxhloOisKwsB6KbRcQFQZeiA4vP6sFdAwWRiJqhhwnhy", + "FHZ6g/33w52Dvf2D4fAhgO8yIjVFxz77PLx6Fm3j6W70fPnsj+H82Ww73vEaXg9QU7yc81ktMW7nkBBR", + "LfNWLY8oSUQZ6cksonx9WscKWWCCNNbu/5s59s0MVioL5+VJFnUGrHLilDjrkXAy7OhX3k5Uh39yvHrY", + "twrRrg7Ez2DVoQA/tRsMALEO74r6mbKW586HwoutT56VaQPrzh5flidsbe8qN1Dcx88lwVjaYatO7Pqp", + "5vGOzrigah6vPh6y1zIMQYgz+yxVWMZl6KOTGYOijsWfs7CCopmkP+50O9Hn3fKesb+3R+iwmHkZA9ql", + "LqoBLa7doWboairAK7lpIUzkn7bG9Zh/GvaGLyD4Lfq8+9Og96KP/l4IwusaahXJN3Rvl34dtKFhsdKF", + "Q0gfvrhRhJqj5yoO+pX66jTkB7FF07M8nldEc2eFy0gqLXD+uLbGFRiBRo3zrqqdPc3GRS0pJBFe+uqg", + "F1yxsmIfFpkMTciMMtnGM7szyFyze/Go00eHFoMSrNW8YGqpeSgFWeATGsckpFqpNMZ9c8TndktvW9V4", + "uBkwsfvKo571/frZi/U5pOsC1Ncdk/07JCzdydxtZ+KuSm8Gz5mzSaEECrzYRXSKMKvU4bFVi22mIWSO", + "AMLMgQNUyVnWygCZK37OE9JFM65QnmPY0qOWsmbPXzZ+cg0e1RVJxYYhtu8lYzxDL6GrxNfJMUoED9Mg", + "T7CJYNB5SrRIK2COK7T69SFMD+nQgMw1qO+51qHR5MFo54FsWu+K91EzbPNSDwfrl/pBvCDdTpqE62WY", + "eamdBLsR1uialA2PT6ZM9oomWJjMpxYS/V2RgnUj13iLA60SpYm7QtE8Veckz4UKXCL4YPeOSUT0MVVv", + "BPEozKNKqcyl6HqROtx/Pm+6xIQ7p/pAfiUk0bYKAERAfzFmS+/AqkVW0cbAVc6S5kqrZ7DKLbXKg3u2", + "VhNrXKr29WorXm2TUF4sn5yBbt5vsVr75dpS9A/hh/uWStpbe7lQgVdz4H8ZXKnr32TMADwjiyrn9a7v", + "At7HFu+tZdwEGVfNmim6mQ97/23cymjcP9j66W//d+/TX73u5YrdLInohWQKoUSXZNkDvHukbfR+GTAN", + "sHq1Mj2zrEJwDE6j4JIYJ1WMr4vj3RtkQmP5Bse1KUAMVkxZ9u+1E/rbvzVHMBXI+AHk5FqWvTOU9UPU", + "CVLcHUcbMREzV8nYBd5v9kcMYvkvyVKiQjECq9I4Rv2LzD7RKjo4LXGELowa2CdscYEmFGq6yBHTVi0O", + "ApJoa8KCslNTjI+D9BEER8V2bFEElyhnrxxNxABBH09raHtvP7z/+e2HN8fjt2cv3xyejH99+V8QxHHV", + "Mz2EPc17u3v7tgRfkZJDzxLfAfj3Tih+PnYzWGAe/oKkFKhr6FGYqYSUUhdQUHgZbZA4UUtXbMjltmze", + "DJvsMGvQG852zyjsgxf3UXTmw8oqMwse9bRG3QCi63VgGlp4w7GhKRPm3mlybM88ZcDPrTdxRmfY48v2", + "ViC9j+IwbkBrQeNq699Y2sEfHH9cRRg20sCQqoKIW7FLpeo1x87HWpEa57UeyxEZKbPpJbQQsFXOJYmZ", + "2rJFnHwpraE+eVcnFOW7zCEW9eCj9XkyK1X5wswKI2lem1OnsVZ06hUEOtOkuZoTQQoLAR/kyK03JJlN", + "9miRKG1KrSRE5IGQLlMEap8LCtkjmbPBkSBLCKp7YFcj857i66wH8N5jWbvjgnnkGPnD1z+POpt99M6V", + "KqVT1wQMo2JP+IFQy1y0iiaOq+qLUeSq+rzN+96NZ2XVCunXtLcqzJn3UWJNHz/+HVP1iguwQJrTkh8c", + "TxWsm5AIwGWpoqW2ghqlMQnHWcHmpv3vajSbnOSsHHZexslZWxiYWAu59aV2XOJsPoY6pTU5SJAKqpbn", + "UNjVRAgTLIg4TM2Gd/Vh7c95x1AV6etX8FNOPVkIrwkjggbo8OwE9mOMGSjp6ONpoZKJKWpTw1AD9fLt", + "0Ym1cB0MH1gsVAHruWC+w7OTTrezIMJYeZ1Bf7s/gM2cEIYT2jno7PSH/UHHVPSFKW5BEUX40yYYZpbS", + "SWj1oJ/NK/orgWOiiJCdg989iXoQyAYvg76LZwWLJcFUWJMliSB90LAK1d8CsK47Sg/MeWwL8rZ20Em1", + "tMkUJHlrl/UTqJOwa2CK24OBhRlV9uCFVBATf771DxuMmPfbSp8D8nhQZmsWhdMpLcm/dju7g+GNxrNq", + "GLBjfd1+YDhVcy7oZwLD3LshEW7V6QkzGV7IYIXZQJviPgMWKu6w3z/p9ZJpHGOxdOTKaZVw2aQME4kw", + "YuTKVgT9B5/0kb1+gLIxcs7TSEsTZNLXnKNBYdGffUZYBHO6ICNmz+k4jRRNsAA3Qoz0+WwMpvLWMF2b", + "1c+ABX7m4bJC3ay5Ld1czzmdcwJX0xIkGQMO8bip3G/ubqaMaTGJJbG1NLK6l/VoHi0uxzLgvnyi94Rh", + "pnoyIQGd0gDBy3r3Wo+2t8FW0Hxa4MGyEAGIWc5Ds73pz0mFqh/+dO7j7Bmy5C2rEwyugYIoDXOdy6VJ", + "YTHBUeTFbppFfIKjsaHPJfGoqK/hDUuUYoEUp9wwHhJT7CJZqjln5u90kjKVmr8ngl9JIrQKZIuYWVqT", + "0JQtM6x7BVifMRQSMyVSdZ9bZohbXy7J8mt/xA7D2JW/leYTHEmuT01bdNDmBZgtbXjXX5alIa7kKJWK", + "x5alsgqM+TB5qpJU2Tt1SZStvAavU4mSVM5JOGKKoy+CzKhUYvl160ve41ewXQgONZ8UXjFT2vpCw69N", + "o5ZjrGc/hlc91h8BAow6+nQZdfTfM4G17ZLKOThRJDhOZsUl3cjwdLReuFmlcIAZSnhisIiAqeZYs1yp", + "DYhHx1GEFGwl963WNmElG+Zj04vjSWNusUkGrWwjytDpz4XNNNh97t9PkgSC+Bwc/3n+9g2Co0qvgXkt", + "d1iZS22mT1EUpqDJQ+/9EXuJgzkyehPgzY46NBx1Musi3ISxptJGaPd6oOL+pIf2k+mmS8Of+n3dlNGe", + "D9DvX0wrB3ovJfFY8UvCRp2vXVR4MKNqnk6yZ5/8BG1K0TwvCQK0YWT/pqtBDFBR+TFozg3MQsStrI2W", + "CKNcAhX9KBPKsFhZQNlDektBbcrjmSwS48sIfLejzsHIeW9Hne6oQ9gCfrMu3lHnq58CVoluBjc1NaSd", + "rp0x0f5gsLkeO8HS16NCl17U2+9rTfvavjfFwypddcXDTM4hM+sVNNXAjbr1CJrPzzh09SV/qHhrVDzr", + "uSgob/B98Rww7BsRY+BWNDBtz0ZOA1tpnRi2gIIJYHE4pBNjcFCnweXMWzQ/quZ83azYbdplAQwxcvy3", + "+wj8B/1mpQhNvy8eq18cAc64A65/YuwIi+UYseu3iF8T9T1w3OCxRKkFRf+W/PtU+Oc1sXpfTrSKNNsi", + "C3ff5MdzgmQTaVsxL2tb9RzG1DsnTKGX8Gvf/tdZPJAtfRHx2cUBMiSM+AxFlNl7wMJtkT4ULS3hI5Nv", + "kn1n008cmOaGOT//+T//C4OibPbP//lfrU2bv2C7b5nESQDfv5gTLNSEYHVxgH4lJOnhiC6ImwzAY5MF", + "EUu0MwA1MxHwqFhSyeomcsRG7B1RqWCF+1KDayltg2B6MJgPZSmRNl9Hv0inFnTLOJg9Jrzby4aUj7qj", + "u55EZ5hBYQL6VHQ8AFmitvyatb86fu+ZmXPJf1b1ldc8puvliyLXynBvzwzwhgIGSOzbd/DAThptnJ+/", + "3OwjsDEMVwCwGmjMeTNWee7/kEnrZZKRKGWBAlQ2sskksa32/x7bd9o5gG2LfyYPsMXavIEL2Lg8IHfd", + "rcAPW6GFO9hPN+ca9vlnj12WZrOD9vbzLXbh4phaGcL3t86O9+o0N08KJPsWJjDacNHw4EbkAp0dnbia", + "tpvfjOkf5dTQM7W1+rKjA3EG2GuPZpYdcTaNaKBQz40FCjPFJDPVygzyVMTBOztqhN28qhDGxfNtq4TI", + "13jSZeB8+ZH38KdHpdObHCM5zHLOaz9OknWsc0xlwPW3BW7pBTgBQjr1JdunRS5a55AywfXZkbNSXbLi", + "+eTYbcjHc03ZrlNWPRseQSgeVwTiNxSEleLxBWDyp8TNH7JVdIgUKzxX3xdrDh5PC3psL5aPzZ+SGyus", + "kE1LQRPU3XiAvibKhHJ3HnChbQ+eiZ8T4Xa1qyMBs86mZT41hVXNhOBCerXte2JeaWf6mvb+TJYvkOcm", + "Gosl+Q8VpYWxm9NqlYF7YkuWP5x9Cz3cyLy9v3tey2AeIkOwycR5rE3dXSyXLNj8U131PsppZoj9JA+z", + "szSK3I3HggiF3h6dmJ1VPAO2vkBY0nrd3u22lcfBh3e/9QgLOMShZTFUfiXKPrlnDd8smJnKDzZpYxOa", + "tGjqzrMmDecO62/CBZGJcOxT/u/bryI6EVgs/337FY4Sysi/7xxGWBGpNh+MWQaPJZofW+N+wsynFW5a", + "JhqIJgb13tdpqNlbLZVU9/6fSk81k76RpprR9Yey2kZZLZJrpb5ql+JBNVbTxze6ksmYzUdteOTiE/9k", + "murjevksRzqgaCrL1x62yB4X4OeFR5ShVJInGEBJM44rHhst3dX5hlx5fDjWPTnuAiGhLgKgNtkEkUdy", + "XrtxPLpya/t9fM/1YTyhs5Snsph7EmMVzIm0yUoRKQvgp6Z258dzo+L9HXPp4DGPjkfXq3/w/QNp/NUF", + "NcLb3ECt0/ndW211fvu+1vlNCrXNXbPQUl0HO7jZEFTokqjbsnEp17we7Ogbl88WQR+0oZKbCwgsiIMR", + "+z/a/vhdERx/+sklyaSDwfY+/E7Y4tNPLk+GnTpWIQwqHgFK7OGbY7j2m0H2OQDJ5il51XGYyhPAeg46", + "51/OQMpvPttbSI4Lf1hIrSykArlWW0h2LR7WRCrDbz26jeT4zUdwC2Lyw0p6DCtJptMpDShhKq95WgsS", + "syWTn2BuGbP3Q4XgjtJB29pKyjblGgU0Lwvw6IE9JzkO4mMbR64CwdOMkeeJhfS25kh+GDbbI98bPwwe", + "Vzg/vh3ylFnMKPx10iVap/SV0gaMyTiFspAoRwiBqE8kbP1H12If5RWsZZokXChpcCpBATZI9nOtAPsw", + "LcswlT5cSsBipER2RwwqFejHJpd/65IsDQol5SwDnKzWY/XlXpVRQL/pNrp/HcsPcdpKx3rkbWxBq7+d", + "jvXNRMejaFonpVoAG9nGAINyQrKdzLPkPvqZstnmk4pANcIqm1sBz8ijam1BpT+L67sls2qyTQdtAdrX", + "lp79Fzxx65P0ae0Oi7ZAQBRSPGNcKhoUK+8W4UF/nNCtT+jVlPVy89SWSPcb9K+4uGx7xHnqij2Bk644", + "w+/Ql6CHB2hg396lAMa2OQ000zz6KVgrFvctUzBo9VwMojSEqvT2QHSq5FTweGx/NHi1eldYNFBwUQS2", + "1W8tbHTvj+AwesMVonESEa3FkxD1DDfp1bSqv4Obp7JQWvFmwlBvm2JCjAGjk640kRWRcLnmFmwD7tnr", + "y+WVmhGfrQfByDp3iA8eFIwRM3D4xGHnX6BMyELxLhKRQKGrOQ3mgIgBBb2goiuAVeAkucggsDYP0GvY", + "qUUkMOh8QxKhDaGAM8kjYoAuFnF8cVBHbP14egofGTAMg816cZCVXM8OCKnfKiJcZDWP3ljcjg3NSYJH", + "kVnRC201Fua3abEvcoiyEfPhYDByZRukU3RRgMS4aMDEcAL1Nz77ZtpWtxlY0sxFcSSAcIY3CQs7TRcx", + "NPKjYQwH3nowLZE5zDAeGJijNpjf+CwDtSyxMk6StuxrhwlcvIjjFTyMNgrFWaUKear+JlVIhICPLXc3", + "MTfawIH5h8KXmlFtYaGsvC2wn/e60aDMeUmlhWqhio751yKOO92OHU8Bne4G2vsahJNqg/VrMb0yBRiT", + "H3r3TQBKysK+gFBSOTls+f9mlfudeeFP75+1hAr/DF6W8n1WPgrK8gJQAiq4uypcTwrpABaypouZwki+", + "PeJm2ZOF4qHtrrdqZUe/A6N13a1XVkMyK3D52Ndf9RE85SQYWZvNlItqevy6e7HvnpHub0lqU23DIT94", + "8+buuVaMmaQraolCKVQJfj6orwm4zsGcc1lg+wmZ4wXlwiKwW69rxpngsjDWo42eu9CsemH9txdWPT+w", + "viaEi49sH3343Mbc+b9wj/IvXhWs7Uzid51KDSiQEmE0EZRMUYJTSbS2lMYEmQojFsib4GDuqoX3R+z9", + "nCBbH7PgQMjKKVOJLobxRRdNUoUiLGZg7ZiHJpJOkIDHMWGhqXk7YnOCF1SbagJFWBEWLHuSQA3kBckL", + "mGjT3d5QmlLbWZXVLnLFecHBcFEovXuBEkGAiYy5zEp1bkdMpOw/DHKlbvbCDfQCEanwJKJyntWKCHBI", + "WOCFhTz/vsXY/Ttxz4mqV6f9JneWt5Kl3/ISs+jLzOqDfxf3m08sUIsLV1mzhZhfofTKZtOwHPl4nlfk", + "/Rfc0maubo7f6GYmI/GqXfx9XMmUSvL/uJZRdkuGqemOlMvW/2nvWvI60ikrXbdYn+xtL1yySggZmW8k", + "87a+uD9PbuEj+04kYbfRsG/C3M4n/T2IXEvVW8ncb+QctL6kglfsG4pgO6hvpz5xUZBy34UYNhsuk8ZF", + "maMEBpuKsx/CuCqMbXjAbYWx87jWLsAL4pmyXhLhJrmcV633C2DrEPgXjX6tzK4gCL+54MtvBB5N2J1k", + "4s0IvAQvI47/7PcyARfCJHTacsRPB1Cs4AssXDBtgMetm0mIrssm+Xh6utkkJYRaKSOEesISolzWNIg9", + "1RrfLogQNHSlI49Oj230KpVIpKyP3sYU6jleEpJAoRjKU4kgM7ev5+dSW+tF8Eo5rN0OYUosE06ZWjuK", + "/NWHGczXW5XOe2Q5aSEV//SXx+CFf3pCCmSHVlfsBFZbkQqrxmA8F5xGmal3qbUtPOGpbl1LFldodwZn", + "25RGRC6lIrGJzJumEWwiAN21NZnsdyajtIuokkjvhy5k4CVExFRKypkcMVv+PSFC960/h+K/eZCR13mv", + "cCY1z4zo+z4C2PRgTMwWVk1UA2gBqAPaOehs4STZgnLR/iApO7w7DOkVRKQhuYwnPKIBiii7lGgjopfG", + "6EALiSL9x+bKkLYxfHffFaduv7M0pU/YlHuLchiezZj5z5GEVBZr7hLxyYm116S4WZz8gYX2izW5Vq4J", + "gqOeojHJkt9RqmhEPxtRpxuhUtHA5NXkqZdQhNlmX47YKVFCv4MFQQGPIhIo51zZSgQPtkbpYLATJBRQ", + "SnYIDA4EXvPjGHo8OvsA75lC0d0R0/+Aht8fnpmb2Cm2PoLCQBlRV1xcopOtt2uCfM+BTP/CUXJmgitz", + "IL0L/uP67uaZzY17SDZsUZ6sMoB48qcP47Qa3A9vwdP0FgC0RDabjZnAASjFcp6qkF8xv2dgwaM01v8w", + "f5ysAyhROJh/hFe/G23XDGdtN26CT2JT2jmFxBQN+iYXFIZgTzW+VBPOTQGUmFLknvcUOFR/Ru6+f6d8", + "kY7f4dWkpagryPXd7K3HPvnsGBzuVpEeT2WbG05zM1F8tffpCtNm79PPEQ8uJUqZolEJ1EDbbYADqn/M", + "cRvtxR+oCZAd6UqJI3KdUAEINhV4BET0jCXCSBERU4ajLZizaQQQKJ0XCy84hSTlIKKQJkZDghIeRYCy", + "czUnDOnZgKPKNVC4p5W2AkTxneIVo+JoQgIeE4fKuekz3f6OqXrFRRli83uRi+8L9Nfz0VPV81yDKtrc", + "451QRk/xNYQ1h6m9JnYj2njN8x+NK6iLYG1GnZ2BHHW6aNTZjkcdvQJHGFyoWKE9FFOWKiL76Nj4tyAN", + "dX+AJAk4C6UDB3UevJ2BbEpKNWzZkOG4D989ptpjuQpI+c524hMP+j2kv4cEG7RR3HB2T4Zd2HQh4qmC", + "AG63r+xbIVHgHtl89BvYwh75Ydu3keR/t9u3JKNglbW4LCy9kewZfORar5tLqphzmaNOogAnOKBq2UU4", + "iniQew9Smd0O9LKhTATBl9qG6o/Yuwy40iZCoKOzD13nNEMhlZemBesX66O3CyJkOskGh0AaGA8eLAYJ", + "R0xxFOAoSCPNt2Q6JQHkMEQ0pko2+NWyoTxkGcS8E8/Cu4cZbM3Tcib5eQJWL2cLWeG4LbPUW4IEEaZx", + "0alUJQ6ovnClC27fiW6U62N4GtnrrUBwKZFtqkciOqOTyF7WyD56r1UOHJMRSyLMGBEolSbuSA+9lwgi", + "ZWoSY3QDUGfWcFQX5UAnieDKuokjzoU0nl3N4R9PkVQkWcFm70zLpzDnB4IJNo3bnr6RwVAZQ/OxZF9B", + "ekEMpxiCaz7Sx/Q3CPYxA/rWcMJPZeO/F3Q2I0LvCmyErLkaNdvakdNs+lKmRyNG/nn2VjuM/KzVQjR3", + "IdJ5JVDF2L04BgX6Jjewns4vaSOWiX10s+yLX/VHLfsuR/n7B2Ef3XGWf5bSY+eF4Oq2yPo5hz81kPvC", + "yEtbtZSgsB6OoHVGwkNmCLTGHfhmcANPGWUAl9IOmuAEvj9GGDxudtxjw2w/bd4qoQSUCus0pEqth+/8", + "LjjwYXA7v3F26C1wO7+rfCXAXfx2eaPfVaZSyQ/oiof86ZE5HypBycBzAoxFU4KSkXo2kGClofTRvtPO", + "TLIt/pk0eHv3fAP93ZH9h9XfwmQoEMvvsjO50Q63hcSJWrrLRT6tXABK+hmSMXzAD1kMwcPhLdziev3+", + "2MPxaePl+o96Wo92f58XHT45fvpFtIp7rnSwbOlTp4dFMKcL0ux0L+9gS6JEkF7CE7hcCQ3BLD3cWaaw", + "6M8+I9u8xaqy/0LUQRyTEIVUkEBFS0SZ4iARTB9/kUhwbQnAcy6WPmd6cee+Ejw+tLNZcx7aPWWdYfmd", + "b7zshVjh3sJJmxUutDvctLu7bS3wEGXo9c9og1wrYRB30VRbPohOM5KS64CQUAJPbhYHPBw0eDbpZzKe", + "TdqMcgV28luLTY2CVCoeu7U/OUYbUGxhRpheC63qT0GTTQRf0NAUIs2JuuCRoeqwgaA39btqpSKrlOGM", + "CzO4b6LDtDmQZp9pUhYLJnShc9CZUIZhcGtRist7yiRU6f4whbSGfO84zun8OMKs5bfhjB3NidrIcURU", + "nBtovM0fx9xTPuaKganuTCuddu1KRbaLVW0ZQvoQgLlZHPPjuq0/fj/hlVQ+ychK6zpfZAZpk9v8+2LB", + "weOdD4/tLv/4hMPxXxNnfBdc5dCAbtHHML/xAEcoJAsS8QSqSJp3O91OKqLOQWeuVHKwtRXp9+ZcqoPn", + "g+eDztdPX///AAAA//9DeBEen5ABAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/system/guest_agent/main.go b/lib/system/guest_agent/main.go index 46ee3d7a..a968e89a 100644 --- a/lib/system/guest_agent/main.go +++ b/lib/system/guest_agent/main.go @@ -53,8 +53,10 @@ func main() { } // Create gRPC server + guestSvc := &guestServer{} + startResumeNetworkWatcher(guestSvc) grpcServer := grpc.NewServer() - pb.RegisterGuestServiceServer(grpcServer, &guestServer{}) + pb.RegisterGuestServiceServer(grpcServer, guestSvc) // Serve gRPC over vsock if err := grpcServer.Serve(l); err != nil { diff --git a/lib/system/guest_agent/network.go b/lib/system/guest_agent/network.go new file mode 100644 index 00000000..66b71bad --- /dev/null +++ b/lib/system/guest_agent/network.go @@ -0,0 +1,91 @@ +package main + +import ( + "context" + "fmt" + "net" + + pb "github.com/kernel/hypeman/lib/guest" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +func (s *guestServer) ReconfigureNetwork(_ context.Context, req *pb.ReconfigureNetworkRequest) (*pb.ReconfigureNetworkResponse, error) { + iface := req.InterfaceName + if iface == "" { + iface = "eth0" + } + + link, err := netlink.LinkByName(iface) + if err != nil { + return nil, fmt.Errorf("find interface %s: %w", iface, err) + } + + mac, err := net.ParseMAC(req.Mac) + if err != nil { + return nil, fmt.Errorf("parse mac %q: %w", req.Mac, err) + } + ip := net.ParseIP(req.Ipv4).To4() + if ip == nil { + return nil, fmt.Errorf("parse ipv4 %q", req.Ipv4) + } + if req.Prefix > 32 { + return nil, fmt.Errorf("invalid ipv4 prefix %d", req.Prefix) + } + gateway := net.ParseIP(req.Gateway).To4() + if gateway == nil { + return nil, fmt.Errorf("parse gateway %q", req.Gateway) + } + + if err := netlink.LinkSetDown(link); err != nil { + return nil, fmt.Errorf("set %s down: %w", iface, err) + } + if err := netlink.LinkSetHardwareAddr(link, mac); err != nil { + return nil, fmt.Errorf("set %s mac: %w", iface, err) + } + if err := flushIPv4Addrs(link); err != nil { + return nil, err + } + addr := &netlink.Addr{IPNet: &net.IPNet{IP: ip, Mask: net.CIDRMask(int(req.Prefix), 32)}} + if err := netlink.AddrAdd(link, addr); err != nil { + return nil, fmt.Errorf("add ipv4 address: %w", err) + } + if err := netlink.LinkSetUp(link); err != nil { + return nil, fmt.Errorf("set %s up: %w", iface, err) + } + if err := netlink.RouteReplace(&netlink.Route{ + LinkIndex: link.Attrs().Index, + Gw: gateway, + }); err != nil { + return nil, fmt.Errorf("replace default route: %w", err) + } + _ = flushNeighbors(link) + + return &pb.ReconfigureNetworkResponse{}, nil +} + +func flushIPv4Addrs(link netlink.Link) error { + addrs, err := netlink.AddrList(link, unix.AF_INET) + if err != nil { + return fmt.Errorf("list ipv4 addresses: %w", err) + } + for _, addr := range addrs { + if err := netlink.AddrDel(link, &addr); err != nil { + return fmt.Errorf("delete ipv4 address %s: %w", addr.String(), err) + } + } + return nil +} + +func flushNeighbors(link netlink.Link) error { + neighbors, err := netlink.NeighList(link.Attrs().Index, unix.AF_INET) + if err != nil { + return fmt.Errorf("list neighbors: %w", err) + } + for _, neighbor := range neighbors { + if err := netlink.NeighDel(&neighbor); err != nil { + return fmt.Errorf("delete neighbor %s: %w", neighbor.String(), err) + } + } + return nil +} diff --git a/lib/system/guest_agent/resume_network.go b/lib/system/guest_agent/resume_network.go new file mode 100644 index 00000000..060cd687 --- /dev/null +++ b/lib/system/guest_agent/resume_network.go @@ -0,0 +1,201 @@ +//go:build linux + +package main + +import ( + "bufio" + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "log" + "net" + "os" + "strconv" + "strings" + "sync/atomic" + "time" + "unsafe" + + pb "github.com/kernel/hypeman/lib/guest" + "golang.org/x/sys/unix" +) + +const resumeNetworkMailboxEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX" +const resumeNetworkMailboxTokenEnv = "HYPEMAN_RESUME_NETWORK_MAILBOX_TOKEN" +const vmgenIDKmsgSignal = "crng reseeded due to virtual machine fork" +const resumeNetworkMailboxSize = 4096 +const resumeNetworkMailboxSeqOffset = 64 +const resumeNetworkMailboxLengthOffset = 68 +const resumeNetworkMailboxPayloadOffset = 72 + +var resumeNetworkMailboxMagic = []byte("HYPEMAN_RESUME_NETWORK_MAILBOX_V1\x00") + +type resumeNetworkPayload struct { + InterfaceName string `json:"interface_name"` + MAC string `json:"mac"` + IPv4 string `json:"ipv4"` + Prefix uint32 `json:"prefix"` + Gateway string `json:"gateway"` + AckPort uint32 `json:"ack_port,omitempty"` +} + +type vmGenIDResumeWaiter struct { + file *os.File + reader *bufio.Reader +} + +func startResumeNetworkWatcher(s *guestServer) { + if strings.TrimSpace(os.Getenv(resumeNetworkMailboxEnv)) != "1" { + return + } + + mailbox := newResumeNetworkMailbox() + if mailbox == nil { + return + } + + go resumeNetworkMailboxLoop(s, mailbox) +} + +func newResumeNetworkMailbox() []byte { + token := strings.TrimSpace(os.Getenv(resumeNetworkMailboxTokenEnv)) + if token == "" { + log.Printf("[guest-agent] resume network mailbox disabled: missing %s", resumeNetworkMailboxTokenEnv) + return nil + } + if len(token) > resumeNetworkMailboxSeqOffset-len(resumeNetworkMailboxMagic) { + log.Printf("[guest-agent] resume network mailbox disabled: %s is too long", resumeNetworkMailboxTokenEnv) + return nil + } + + buf := make([]byte, resumeNetworkMailboxSize) + copy(buf, resumeNetworkMailboxMagic) + copy(buf[len(resumeNetworkMailboxMagic):resumeNetworkMailboxSeqOffset], token) + if err := unix.Mlock(buf); err != nil { + log.Printf("[guest-agent] resume network mailbox mlock failed: %v", err) + } + log.Printf("[guest-agent] resume network mailbox armed token=%s", token) + return buf +} + +func resumeNetworkMailboxLoop(s *guestServer, mailbox []byte) { + for { + waiter, err := newVMGenIDResumeWaiter() + if err != nil { + log.Printf("[guest-agent] resume network VMGenID prepare failed: %v", err) + time.Sleep(100 * time.Millisecond) + continue + } + + start := time.Now() + if err := waiter.Wait(); err != nil { + waiter.Close() + log.Printf("[guest-agent] resume network VMGenID wait failed: %v", err) + time.Sleep(100 * time.Millisecond) + continue + } + waiter.Close() + + if err := waitAndApplyResumeNetworkMailbox(s, mailbox); err != nil { + log.Printf("[guest-agent] resume network mailbox apply failed: %v", err) + time.Sleep(25 * time.Millisecond) + continue + } + log.Printf("[guest-agent] resume network mailbox applied in %s", time.Since(start)) + } +} + +func waitAndApplyResumeNetworkMailbox(s *guestServer, buf []byte) error { + for { + seq := atomicLoadUint32(buf[resumeNetworkMailboxSeqOffset:]) + if seq == 0 { + time.Sleep(100 * time.Microsecond) + continue + } + + payloadLen := binary.LittleEndian.Uint32(buf[resumeNetworkMailboxLengthOffset:]) + if payloadLen == 0 || int(payloadLen) > len(buf)-resumeNetworkMailboxPayloadOffset { + return fmt.Errorf("invalid mailbox payload length %d", payloadLen) + } + + var payload resumeNetworkPayload + if err := json.Unmarshal(buf[resumeNetworkMailboxPayloadOffset:resumeNetworkMailboxPayloadOffset+int(payloadLen)], &payload); err != nil { + return fmt.Errorf("decode mailbox payload: %w", err) + } + + _, err := s.ReconfigureNetwork(context.Background(), &pb.ReconfigureNetworkRequest{ + InterfaceName: payload.InterfaceName, + Mac: payload.MAC, + Ipv4: payload.IPv4, + Prefix: payload.Prefix, + Gateway: payload.Gateway, + }) + if err != nil { + return err + } + sendResumeNetworkAck(payload, "applied") + atomicStoreUint32(buf[resumeNetworkMailboxSeqOffset:], 0) + return nil + } +} + +func sendResumeNetworkAck(payload resumeNetworkPayload, stage string) { + if payload.AckPort == 0 || payload.Gateway == "" { + return + } + + addr := net.JoinHostPort(payload.Gateway, strconv.FormatUint(uint64(payload.AckPort), 10)) + conn, err := net.DialTimeout("udp4", addr, 100*time.Millisecond) + if err != nil { + log.Printf("[guest-agent] resume network ack dial failed: %v", err) + return + } + defer conn.Close() + + _, _ = fmt.Fprintf(conn, "stage=%s mac=%s ip=%s\n", stage, payload.MAC, payload.IPv4) +} + +func atomicLoadUint32(buf []byte) uint32 { + return atomic.LoadUint32((*uint32)(unsafe.Pointer(&buf[0]))) +} + +func atomicStoreUint32(buf []byte, value uint32) { + atomic.StoreUint32((*uint32)(unsafe.Pointer(&buf[0])), value) +} + +func newVMGenIDResumeWaiter() (*vmGenIDResumeWaiter, error) { + f, err := os.Open("/dev/kmsg") + if err != nil { + return nil, fmt.Errorf("open /dev/kmsg: %w", err) + } + + if _, err := f.Seek(0, io.SeekEnd); err != nil { + log.Printf("[guest-agent] warning: failed to seek /dev/kmsg to end: %v", err) + } + + return &vmGenIDResumeWaiter{ + file: f, + reader: bufio.NewReader(f), + }, nil +} + +func (w *vmGenIDResumeWaiter) Close() { + if w == nil || w.file == nil { + return + } + _ = w.file.Close() +} + +func (w *vmGenIDResumeWaiter) Wait() error { + for { + line, err := w.reader.ReadString('\n') + if err != nil { + return fmt.Errorf("read /dev/kmsg: %w", err) + } + if strings.Contains(line, vmgenIDKmsgSignal) { + return nil + } + } +} diff --git a/lib/system/guest_agent/resume_network_other.go b/lib/system/guest_agent/resume_network_other.go new file mode 100644 index 00000000..dfa74e07 --- /dev/null +++ b/lib/system/guest_agent/resume_network_other.go @@ -0,0 +1,7 @@ +//go:build !linux + +package main + +type resumeNetworkController struct{} + +func startResumeNetworkWatcher(_ *guestServer) {} diff --git a/lib/system/versions.go b/lib/system/versions.go index aa07108d..7743add0 100644 --- a/lib/system/versions.go +++ b/lib/system/versions.go @@ -20,14 +20,18 @@ const ( // Kernel_202603301 is the current kernel version with expanded nftables/raw support for Docker bridge networking Kernel_202603301 KernelVersion = "ch-6.12.8-kernel-1.6-202603301" + + // Kernel_202605291 is the current kernel version with VMGenID support for snapshot resume detection + Kernel_202605291 KernelVersion = "ch-6.12.8-kernel-3.0-202605291" ) var ( // DefaultKernelVersion is the kernel version used for new instances - DefaultKernelVersion = Kernel_202603301 + DefaultKernelVersion = Kernel_202605291 // SupportedKernelVersions lists all supported kernel versions SupportedKernelVersions = []KernelVersion{ + Kernel_202605291, Kernel_202603301, Kernel_202603091, Kernel_202602101, @@ -37,6 +41,10 @@ var ( // KernelDownloadURLs maps kernel versions and architectures to download URLs var KernelDownloadURLs = map[KernelVersion]map[string]string{ + Kernel_202605291: { + "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-3.0-202605291/vmlinux-x86_64", + "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-3.0-202605291/Image-arm64", + }, Kernel_202603301: { "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.6-202603301/vmlinux-x86_64", "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.6-202603301/Image-arm64", @@ -58,6 +66,10 @@ var KernelDownloadURLs = map[KernelVersion]map[string]string{ // KernelHeaderURLs maps kernel versions and architectures to kernel header tarball URLs // These tarballs contain kernel headers needed for DKMS to build out-of-tree modules (e.g., NVIDIA vGPU drivers) var KernelHeaderURLs = map[KernelVersion]map[string]string{ + Kernel_202605291: { + "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-3.0-202605291/kernel-headers-x86_64.tar.gz", + "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-3.0-202605291/kernel-headers-aarch64.tar.gz", + }, Kernel_202603301: { "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.6-202603301/kernel-headers-x86_64.tar.gz", "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.6-202603301/kernel-headers-aarch64.tar.gz", diff --git a/openapi.yaml b/openapi.yaml index c6bce872..e6d03230 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -690,6 +690,13 @@ components: Optional final state for the forked instance. Default is the source instance state at fork time. For example, forking from Running defaults the fork result to Running. + wait_for_network: + type: boolean + description: | + When the fork result is Running, wait for guest networking to be applied before returning. + Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. + default: true + example: true ForkTargetState: type: string @@ -875,6 +882,13 @@ components: Optional hypervisor override. Allowed only when forking from a Stopped snapshot. Standby snapshots must fork with their original hypervisor. example: cloud-hypervisor + wait_for_network: + type: boolean + description: | + When the fork result is Running, wait for guest networking to be applied before returning. + Defaults to true. Set false to return after the VM is resumed while guest networking finishes asynchronously. + default: true + example: true SnapshotScheduleRetention: type: object