Revert "Remove task serialization and use host resource manager for t…

…ask resources (#3723)"

This reverts commit 0a4673a.

agent/acs/handler/acs_handler_test.go

@@ -1418,6 +1418,7 @@ func validateAddedTask(expectedTask apitask.Task, addedTask apitask.Task) error
 		Family:              addedTask.Family,
 		Version:             addedTask.Version,
 		DesiredStatusUnsafe: addedTask.GetDesiredStatus(),
+		StartSequenceNumber: addedTask.StartSequenceNumber,
 	}
 
 	if !reflect.DeepEqual(expectedTask, taskToCompareFromAdded) {

agent/api/ecsclient/client.go

@@ -315,10 +315,6 @@ func (client *APIECSClient) GetHostResources() (map[string]*ecs.Resource, error)
 	}
 	resourceMap := make(map[string]*ecs.Resource)
 	for _, resource := range resources {
-		if *resource.Name == "PORTS" {
-			// Except for RCI, TCP Ports are named as PORTS_TCP in agent for Host Resources purpose
-			resource.Name = utils.Strptr("PORTS_TCP")
-		}
 		resourceMap[*resource.Name] = resource
 	}
 	return resourceMap, nil

agent/api/task/task.go

@@ -33,6 +33,7 @@ import (
 	"github.com/aws/amazon-ecs-agent/agent/config"
 	"github.com/aws/amazon-ecs-agent/agent/dockerclient"
 	"github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi"
+	"github.com/aws/amazon-ecs-agent/agent/ecs_client/model/ecs"
 	"github.com/aws/amazon-ecs-agent/agent/taskresource"
 	"github.com/aws/amazon-ecs-agent/agent/taskresource/asmauth"
 	"github.com/aws/amazon-ecs-agent/agent/taskresource/asmsecret"
@@ -48,7 +49,6 @@ import (
 	apieni "github.com/aws/amazon-ecs-agent/ecs-agent/api/eni"
 	apierrors "github.com/aws/amazon-ecs-agent/ecs-agent/api/errors"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/credentials"
-	"github.com/aws/amazon-ecs-agent/ecs-agent/ecs_client/model/ecs"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/logger"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/logger/field"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/utils/arn"
@@ -235,6 +235,9 @@ type Task struct {
 	// is handled properly so that the state storage continues to work.
 	SentStatusUnsafe apitaskstatus.TaskStatus `json:"SentStatus"`
 
+	StartSequenceNumber int64
+	StopSequenceNumber  int64
+
 	// ExecutionCredentialsID is the ID of credentials that are used by agent to
 	// perform some action at the task level, such as pulling image from ECR
 	ExecutionCredentialsID string `json:"executionCredentialsID"`
@@ -310,6 +313,11 @@ func TaskFromACS(acsTask *ecsacs.Task, envelope *ecsacs.PayloadMessage) (*Task,
 	if err := json.Unmarshal(data, task); err != nil {
 		return nil, err
 	}
+	if task.GetDesiredStatus() == apitaskstatus.TaskRunning && envelope.SeqNum != nil {
+		task.StartSequenceNumber = *envelope.SeqNum
+	} else if task.GetDesiredStatus() == apitaskstatus.TaskStopped && envelope.SeqNum != nil {
+		task.StopSequenceNumber = *envelope.SeqNum
+	}
 
 	// Overrides the container command if it's set
 	for _, container := range task.Containers {
@@ -2818,6 +2826,22 @@ func (task *Task) GetAppMesh() *apiappmesh.AppMesh {
 	return task.AppMesh
 }
 
+// GetStopSequenceNumber returns the stop sequence number of a task
+func (task *Task) GetStopSequenceNumber() int64 {
+	task.lock.RLock()
+	defer task.lock.RUnlock()
+
+	return task.StopSequenceNumber
+}
+
+// SetStopSequenceNumber sets the stop seqence number of a task
+func (task *Task) SetStopSequenceNumber(seqnum int64) {
+	task.lock.Lock()
+	defer task.lock.Unlock()
+
+	task.StopSequenceNumber = seqnum
+}
+
 // SetPullStartedAt sets the task pullstartedat timestamp and returns whether
 // this field was updated or not
 func (task *Task) SetPullStartedAt(timestamp time.Time) bool {
@@ -3521,6 +3545,10 @@ func (task *Task) IsServiceConnectConnectionDraining() bool {
 //
 // * GPU
 //   - Return num of gpus requested (len of GPUIDs field)
+//
+// TODO remove this once ToHostResources is used
+//
+//lint:file-ignore U1000 Ignore all unused code
 func (task *Task) ToHostResources() map[string]*ecs.Resource {
 	resources := make(map[string]*ecs.Resource)
 	// CPU
@@ -3634,13 +3662,3 @@ func (task *Task) ToHostResources() map[string]*ecs.Resource {
 	})
 	return resources
 }
-
-func (task *Task) HasActiveContainers() bool {
-	for _, container := range task.Containers {
-		containerStatus := container.GetKnownStatus()
-		if containerStatus >= apicontainerstatus.ContainerPulled && containerStatus <= apicontainerstatus.ContainerResourcesProvisioned {
-			return true
-		}
-	}
-	return false
-}

agent/api/task/task_test.go

@@ -42,6 +42,7 @@ import (
 	"github.com/aws/amazon-ecs-agent/agent/dockerclient"
 	"github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi"
 	mock_dockerapi "github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi/mocks"
+	"github.com/aws/amazon-ecs-agent/agent/ecs_client/model/ecs"
 	mock_s3_factory "github.com/aws/amazon-ecs-agent/agent/s3/factory/mocks"
 	mock_ssm_factory "github.com/aws/amazon-ecs-agent/agent/ssm/factory/mocks"
 	"github.com/aws/amazon-ecs-agent/agent/taskresource"
@@ -53,7 +54,6 @@ import (
 	apieni "github.com/aws/amazon-ecs-agent/ecs-agent/api/eni"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/credentials"
 	mock_credentials "github.com/aws/amazon-ecs-agent/ecs-agent/credentials/mocks"
-	"github.com/aws/amazon-ecs-agent/ecs-agent/ecs_client/model/ecs"
 	"github.com/aws/aws-sdk-go/service/secretsmanager"
 
 	"github.com/aws/amazon-ecs-agent/agent/taskresource/asmsecret"
@@ -1861,9 +1861,10 @@ func TestTaskFromACS(t *testing.T) {
 				Type: "elastic-inference",
 			},
 		},
-		CPU:                2.0,
-		Memory:             512,
-		ResourcesMapUnsafe: make(map[string][]taskresource.TaskResource),
+		StartSequenceNumber: 42,
+		CPU:                 2.0,
+		Memory:              512,
+		ResourcesMapUnsafe:  make(map[string][]taskresource.TaskResource),
 	}
 
 	seqNum := int64(42)

agent/api/task/taskvolume_test.go

@@ -119,6 +119,8 @@ func TestMarshalTaskVolumesEFS(t *testing.T) {
 		"PullStoppedAt": "0001-01-01T00:00:00Z",
 		"ExecutionStoppedAt": "0001-01-01T00:00:00Z",
 		"SentStatus": "NONE",
+		"StartSequenceNumber": 0,
+		"StopSequenceNumber": 0,
 		"executionCredentialsID": "",
 		"ENI": null,
 		"AppMesh": null,
@@ -166,6 +168,8 @@ func TestUnmarshalTaskVolumesEFS(t *testing.T) {
 		"PullStoppedAt": "0001-01-01T00:00:00Z",
 		"ExecutionStoppedAt": "0001-01-01T00:00:00Z",
 		"SentStatus": "NONE",
+		"StartSequenceNumber": 0,
+		"StopSequenceNumber": 0,
 		"executionCredentialsID": "",
 		"ENI": null,
 		"AppMesh": null,

agent/api/task/taskvolume_windows_test.go

@@ -77,6 +77,8 @@ func TestMarshalTaskVolumeFSxWindowsFileServer(t *testing.T) {
 		"PullStoppedAt": "0001-01-01T00:00:00Z",
 		"ExecutionStoppedAt": "0001-01-01T00:00:00Z",
 		"SentStatus": "NONE",
+		"StartSequenceNumber": 0,
+		"StopSequenceNumber": 0,
 		"executionCredentialsID": "",
 		"ENI": null,
 		"AppMesh": null,
@@ -116,6 +118,8 @@ func TestUnmarshalTaskVolumeFSxWindowsFileServer(t *testing.T) {
 		"PullStoppedAt": "0001-01-01T00:00:00Z",
 		"ExecutionStoppedAt": "0001-01-01T00:00:00Z",
 		"SentStatus": "NONE",
+		"StartSequenceNumber": 0,
+		"StopSequenceNumber": 0,
 		"executionCredentialsID": "",
 		"ENI": null,
 		"AppMesh": null,

agent/app/agent_unix_test.go

@@ -478,7 +478,6 @@ func TestDoStartCgroupInitHappyPath(t *testing.T) {
 		state.EXPECT().AllImageStates().Return(nil),
 		state.EXPECT().AllENIAttachments().Return(nil),
 		state.EXPECT().AllTasks().Return(nil),
-		state.EXPECT().AllTasks().Return(nil),
 		client.EXPECT().DiscoverPollEndpoint(gomock.Any()).Do(func(x interface{}) {
 			// Ensures that the test waits until acs session has bee started
 			discoverEndpointsInvoked.Done()
@@ -647,7 +646,6 @@ func TestDoStartGPUManagerHappyPath(t *testing.T) {
 		state.EXPECT().AllImageStates().Return(nil),
 		state.EXPECT().AllENIAttachments().Return(nil),
 		state.EXPECT().AllTasks().Return(nil),
-		state.EXPECT().AllTasks().Return(nil),
 		client.EXPECT().DiscoverPollEndpoint(gomock.Any()).Do(func(x interface{}) {
 			// Ensures that the test waits until acs session has been started
 			discoverEndpointsInvoked.Done()

agent/app/data.go

@@ -18,12 +18,12 @@ import (
 	"strings"
 
 	"github.com/aws/amazon-ecs-agent/agent/data"
+	"github.com/aws/amazon-ecs-agent/agent/ecs_client/model/ecs"
 	"github.com/aws/amazon-ecs-agent/agent/engine"
 	"github.com/aws/amazon-ecs-agent/agent/engine/dockerstate"
 	"github.com/aws/amazon-ecs-agent/agent/engine/execcmd"
 	"github.com/aws/amazon-ecs-agent/agent/engine/serviceconnect"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/credentials"
-	"github.com/aws/amazon-ecs-agent/ecs-agent/ecs_client/model/ecs"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/eventstream"
 
 	"github.com/pkg/errors"

agent/engine/common_integ_test.go

@@ -225,16 +225,14 @@ func skipIntegTestIfApplicable(t *testing.T) {
 	}
 }
 
-// Values in host resources from getTestHostResources() should be looked at and CPU/Memory assigned
-// accordingly
 func createTestContainerWithImageAndName(image string, name string) *apicontainer.Container {
 	return &apicontainer.Container{
 		Name:                name,
 		Image:               image,
 		Command:             []string{},
 		Essential:           true,
 		DesiredStatusUnsafe: apicontainerstatus.ContainerRunning,
-		CPU:                 256,
+		CPU:                 1024,
 		Memory:              128,
 	}
 }

agent/engine/common_test.go

@@ -32,13 +32,13 @@ import (
 	"github.com/aws/amazon-ecs-agent/agent/dockerclient"
 	"github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi"
 	mock_dockerapi "github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi/mocks"
+	"github.com/aws/amazon-ecs-agent/agent/ecs_client/model/ecs"
 	"github.com/aws/amazon-ecs-agent/agent/engine/execcmd"
 	mock_engine "github.com/aws/amazon-ecs-agent/agent/engine/mocks"
 	"github.com/aws/amazon-ecs-agent/agent/statechange"
-	"github.com/aws/amazon-ecs-agent/agent/utils"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/credentials"
-	"github.com/aws/amazon-ecs-agent/ecs-agent/ecs_client/model/ecs"
 	mock_ttime "github.com/aws/amazon-ecs-agent/ecs-agent/utils/ttime/mocks"
+	"github.com/aws/amazon-ecs-agent/agent/utils"
 	"github.com/cihub/seelog"
 	dockercontainer "github.com/docker/docker/api/types/container"
 	"github.com/golang/mock/gomock"

agent/engine/default.go

@@ -19,12 +19,12 @@ import (
 	"github.com/aws/amazon-ecs-agent/agent/config"
 	"github.com/aws/amazon-ecs-agent/agent/containermetadata"
 	"github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi"
+	"github.com/aws/amazon-ecs-agent/agent/ecs_client/model/ecs"
 	"github.com/aws/amazon-ecs-agent/agent/engine/dockerstate"
 	"github.com/aws/amazon-ecs-agent/agent/engine/execcmd"
 	"github.com/aws/amazon-ecs-agent/agent/engine/serviceconnect"
 	"github.com/aws/amazon-ecs-agent/agent/taskresource"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/credentials"
-	"github.com/aws/amazon-ecs-agent/ecs-agent/ecs_client/model/ecs"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/eventstream"
 )
 

agent/engine/docker_image_manager_integ_test.go

@@ -568,23 +568,23 @@ func createImageCleanupHappyTestTask(taskName string) *apitask.Task {
 				Image:               test1Image1Name,
 				Essential:           false,
 				DesiredStatusUnsafe: apicontainerstatus.ContainerRunning,
-				CPU:                 256,
+				CPU:                 512,
 				Memory:              256,
 			},
 			{
 				Name:                "test2",
 				Image:               test1Image2Name,
 				Essential:           false,
 				DesiredStatusUnsafe: apicontainerstatus.ContainerRunning,
-				CPU:                 256,
+				CPU:                 512,
 				Memory:              256,
 			},
 			{
 				Name:                "test3",
 				Image:               test1Image3Name,
 				Essential:           false,
 				DesiredStatusUnsafe: apicontainerstatus.ContainerRunning,
-				CPU:                 256,
+				CPU:                 512,
 				Memory:              256,
 			},
 		},
@@ -603,23 +603,23 @@ func createImageCleanupThresholdTestTask(taskName string) *apitask.Task {
 				Image:               test2Image1Name,
 				Essential:           false,
 				DesiredStatusUnsafe: apicontainerstatus.ContainerRunning,
-				CPU:                 256,
+				CPU:                 512,
 				Memory:              256,
 			},
 			{
 				Name:                "test2",
 				Image:               test2Image2Name,
 				Essential:           false,
 				DesiredStatusUnsafe: apicontainerstatus.ContainerRunning,
-				CPU:                 256,
+				CPU:                 512,
 				Memory:              256,
 			},
 			{
 				Name:                "test3",
 				Image:               test2Image3Name,
 				Essential:           false,
 				DesiredStatusUnsafe: apicontainerstatus.ContainerRunning,
-				CPU:                 256,
+				CPU:                 512,
 				Memory:              256,
 			},
 		},