Try to fix cluster, container, and taks ARNs in AWS ECS detector when…

… some of them are not valid
open-telemetry · Apr 27, 2023 · a7b3bc9 · a7b3bc9
1 parent 830138c
commit a7b3bc9
Show file tree

Hide file tree

Showing 5 changed files with 282 additions and 10 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,12 +17,14 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 - Update `go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc` to align gRPC server span status with the changes in the OpenTelemetry specification. (#3685)
 - Adding the `db.statement` tag to spans in `go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver/mongo/otelmongo` is now disabled by default. (#3519)
+- Update `go.opentelemetry.io/contrib/detectors/aws/ecs` to fix the task ARN when it's not valid. (#3583)
 
 ### Fixed
 
 - The error received by `otelecho` middleware is then passed back to upstream middleware instead of being swallowed. (#3656)
 - Prevent taking from reservoir in AWS XRay Remote Sampler when there is zero capacity in `go.opentelemetry.io/contrib/samplers/aws/xray`. (#3684)
 - Fix `otelhttp.Handler` in `go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp` to propagate multiple `WriteHeader` calls while persisting the initial `statusCode`. (#3580)
+- Do not panic in `go.opentelemetry.io/contrib/detectors/aws/ecs` when the container ARN is not valid. (#3583)
 
 ## [1.16.0-rc.2/0.41.0-rc.2/0.10.0-rc.2] - 2023-03-23
 
@@ -32,7 +34,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 ### Fixed
 
-- AWS SDK rename attributes `aws.operation`, `aws.service` to `rpc.method`,`rpc.service` in `go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws`. (#3582, #3617) 
+- AWS SDK rename attributes `aws.operation`, `aws.service` to `rpc.method`,`rpc.service` in `go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws`. (#3582, #3617)
 - AWS SDK span name to be of the format `Service.Operation` in `go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws`. (#3582, #3521)
 - Prevent sampler configuration reset from erroneously sampling first span in `go.opentelemetry.io/contrib/samplers/jaegerremote`. (#3603, #3604)
 

diff --git a/detectors/aws/ecs/ecs.go b/detectors/aws/ecs/ecs.go
@@ -101,20 +101,28 @@ func (detector *resourceDetector) Detect(ctx context.Context) (*resource.Resourc
 		if err != nil {
 			return empty, err
 		}
-		attributes = append(
-			attributes,
-			semconv.AWSECSContainerARN(containerMetadata.ContainerARN),
-		)
 
 		taskMetadata, err := ecsmetadata.GetTaskV4(ctx, &http.Client{})
 		if err != nil {
 			return empty, err
 		}
 
-		clusterArn := taskMetadata.Cluster
-		if !strings.HasPrefix(clusterArn, "arn:") {
-			baseArn := containerMetadata.ContainerARN[:strings.LastIndex(containerMetadata.ContainerARN, ":")]
-			clusterArn = fmt.Sprintf("%s:cluster/%s", baseArn, clusterArn)
+		baseArn := detector.getBaseArn(
+			taskMetadata.TaskARN,
+			containerMetadata.ContainerARN,
+			taskMetadata.Cluster,
+		)
+
+		if baseArn != "" {
+			if !strings.HasPrefix(taskMetadata.Cluster, "arn:") {
+				taskMetadata.Cluster = fmt.Sprintf("%s:cluster/%s", baseArn, taskMetadata.Cluster)
+			}
+			if !strings.HasPrefix(containerMetadata.ContainerARN, "arn:") {
+				containerMetadata.ContainerARN = fmt.Sprintf("%s:container/%s", baseArn, containerMetadata.ContainerARN)
+			}
+			if !strings.HasPrefix(taskMetadata.TaskARN, "arn:") {
+				taskMetadata.TaskARN = fmt.Sprintf("%s:task/%s", baseArn, taskMetadata.TaskARN)
+			}
 		}
 
 		logAttributes, err := detector.getLogsAttributes(containerMetadata)
@@ -128,7 +136,8 @@ func (detector *resourceDetector) Detect(ctx context.Context) (*resource.Resourc
 
 		attributes = append(
 			attributes,
-			semconv.AWSECSClusterARN(clusterArn),
+			semconv.AWSECSContainerARN(containerMetadata.ContainerARN),
+			semconv.AWSECSClusterARN(taskMetadata.Cluster),
 			semconv.AWSECSLaunchtypeKey.String(strings.ToLower(taskMetadata.LaunchType)),
 			semconv.AWSECSTaskARN(taskMetadata.TaskARN),
 			semconv.AWSECSTaskFamily(taskMetadata.Family),
@@ -139,6 +148,15 @@ func (detector *resourceDetector) Detect(ctx context.Context) (*resource.Resourc
 	return resource.NewWithAttributes(semconv.SchemaURL, attributes...), nil
 }
 
+func (detector *resourceDetector) getBaseArn(arns ...string) string {
+	for _, arn := range arns {
+		if i := strings.LastIndex(arn, ":"); i >= 0 {
+			return arn[:i]
+		}
+	}
+	return ""
+}
+
 func (detector *resourceDetector) getLogsAttributes(metadata *ecsmetadata.ContainerMetadataV4) ([]attribute.KeyValue, error) {
 	if metadata.LogDriver != "awslogs" {
 		return []attribute.KeyValue{}, nil

diff --git a/detectors/aws/ecs/test/ecs_test.go b/detectors/aws/ecs/test/ecs_test.go
@@ -91,6 +91,120 @@ func TestDetectV4LaunchTypeEc2(t *testing.T) {
 	assert.Equal(t, expectedResource, res, "Resource returned is incorrect")
 }
 
+// successfully returns resource when process is running on Amazon ECS environment
+// with Metadata v4 with the EC2 Launch type and bad ContainerARN.
+func TestDetectV4LaunchTypeEc2BadContainerArn(t *testing.T) {
+	testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
+		if strings.HasSuffix(req.URL.String(), "/task") {
+			content, err := os.ReadFile("metadatav4-response-task-ec2.json")
+			if err == nil {
+				_, err = res.Write(content)
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+		} else {
+			content, err := os.ReadFile("metadatav4-response-container-ec2-bad-container-arn.json")
+			if err == nil {
+				_, err = res.Write(content)
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+		}
+	}))
+	defer testServer.Close()
+
+	os.Clearenv()
+	_ = os.Setenv(metadataV4EnvVar, testServer.URL)
+
+	hostname, err := os.Hostname()
+	assert.NoError(t, err, "Error")
+
+	attributes := []attribute.KeyValue{
+		semconv.CloudProviderAWS,
+		semconv.CloudPlatformAWSECS,
+		semconv.ContainerName(hostname),
+		// We are not running the test in an actual container,
+		// the container id is tested with mocks of the cgroup
+		// file in the unit tests
+		semconv.ContainerID(""),
+		semconv.AWSECSContainerARN("arn:aws:ecs:us-west-2:111122223333:container/0206b271-b33f-47ab-86c6-a0ba208a70a9"),
+		semconv.AWSECSClusterARN("arn:aws:ecs:us-west-2:111122223333:cluster/default"),
+		semconv.AWSECSLaunchtypeKey.String("ec2"),
+		semconv.AWSECSTaskARN("arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c"),
+		semconv.AWSECSTaskFamily("curltest"),
+		semconv.AWSECSTaskRevision("26"),
+		semconv.AWSLogGroupNames("/ecs/metadata"),
+		semconv.AWSLogGroupARNs("arn:aws:logs:us-west-2:111122223333:log-group:/ecs/metadata:*"),
+		semconv.AWSLogStreamNames("ecs/curl/8f03e41243824aea923aca126495f665"),
+		semconv.AWSLogStreamARNs("arn:aws:logs:us-west-2:111122223333:log-group:/ecs/metadata:log-stream:ecs/curl/8f03e41243824aea923aca126495f665"),
+	}
+	expectedResource := resource.NewWithAttributes(semconv.SchemaURL, attributes...)
+	detector := ecs.NewResourceDetector()
+	res, err := detector.Detect(context.Background())
+
+	assert.Equal(t, nil, err, "Detector should not fail")
+	assert.Equal(t, expectedResource, res, "Resource returned is incorrect")
+}
+
+// successfully returns resource when process is running on Amazon ECS environment
+// with Metadata v4 with the EC2 Launch type and bad TaskARN.
+func TestDetectV4LaunchTypeEc2BadTaskArn(t *testing.T) {
+	testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
+		if strings.HasSuffix(req.URL.String(), "/task") {
+			content, err := os.ReadFile("metadatav4-response-task-ec2-bad-task-arn.json")
+			if err == nil {
+				_, err = res.Write(content)
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+		} else {
+			content, err := os.ReadFile("metadatav4-response-container-ec2.json")
+			if err == nil {
+				_, err = res.Write(content)
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+		}
+	}))
+	defer testServer.Close()
+
+	os.Clearenv()
+	_ = os.Setenv(metadataV4EnvVar, testServer.URL)
+
+	hostname, err := os.Hostname()
+	assert.NoError(t, err, "Error")
+
+	attributes := []attribute.KeyValue{
+		semconv.CloudProviderAWS,
+		semconv.CloudPlatformAWSECS,
+		semconv.ContainerName(hostname),
+		// We are not running the test in an actual container,
+		// the container id is tested with mocks of the cgroup
+		// file in the unit tests
+		semconv.ContainerID(""),
+		semconv.AWSECSContainerARN("arn:aws:ecs:us-west-2:111122223333:container/0206b271-b33f-47ab-86c6-a0ba208a70a9"),
+		semconv.AWSECSClusterARN("arn:aws:ecs:us-west-2:111122223333:cluster/default"),
+		semconv.AWSECSLaunchtypeKey.String("ec2"),
+		semconv.AWSECSTaskARN("arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c"),
+		semconv.AWSECSTaskFamily("curltest"),
+		semconv.AWSECSTaskRevision("26"),
+		semconv.AWSLogGroupNames("/ecs/metadata"),
+		semconv.AWSLogGroupARNs("arn:aws:logs:us-west-2:111122223333:log-group:/ecs/metadata:*"),
+		semconv.AWSLogStreamNames("ecs/curl/8f03e41243824aea923aca126495f665"),
+		semconv.AWSLogStreamARNs("arn:aws:logs:us-west-2:111122223333:log-group:/ecs/metadata:log-stream:ecs/curl/8f03e41243824aea923aca126495f665"),
+	}
+	expectedResource := resource.NewWithAttributes(semconv.SchemaURL, attributes...)
+	detector := ecs.NewResourceDetector()
+	res, err := detector.Detect(context.Background())
+
+	assert.Equal(t, nil, err, "Detector should not fail")
+	assert.Equal(t, expectedResource, res, "Resource returned is incorrect")
+}
+
 // successfully returns resource when process is running on Amazon ECS environment
 // with Metadata v4 with the Fargate Launch type.
 func TestDetectV4LaunchTypeFargate(t *testing.T) {

diff --git a/detectors/aws/ecs/test/metadatav4-response-container-ec2-bad-container-arn.json b/detectors/aws/ecs/test/metadatav4-response-container-ec2-bad-container-arn.json
@@ -0,0 +1,44 @@
+{
+    "DockerId": "ea32192c8553fbff06c9340478a2ff089b2bb5646fb718b4ee206641c9086d66",
+    "Name": "curl",
+    "DockerName": "ecs-curltest-24-curl-cca48e8dcadd97805600",
+    "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
+    "ImageID": "sha256:d691691e9652791a60114e67b365688d20d19940dde7c4736ea30e660d8d3553",
+    "Labels": {
+        "com.amazonaws.ecs.cluster": "default",
+        "com.amazonaws.ecs.container-name": "curl",
+        "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/8f03e41243824aea923aca126495f665",
+        "com.amazonaws.ecs.task-definition-family": "curltest",
+        "com.amazonaws.ecs.task-definition-version": "24"
+    },
+    "DesiredStatus": "RUNNING",
+    "KnownStatus": "RUNNING",
+    "Limits": {
+        "CPU": 10,
+        "Memory": 128
+    },
+    "CreatedAt": "2020-10-02T00:15:07.620912337Z",
+    "StartedAt": "2020-10-02T00:15:08.062559351Z",
+    "Type": "NORMAL",
+    "LogDriver": "awslogs",
+    "LogOptions": {
+        "awslogs-create-group": "true",
+        "awslogs-group": "/ecs/metadata",
+        "awslogs-region": "us-west-2",
+        "awslogs-stream": "ecs/curl/8f03e41243824aea923aca126495f665"
+    },
+    "ContainerARN": "0206b271-b33f-47ab-86c6-a0ba208a70a9",
+    "Networks": [
+        {
+            "NetworkMode": "awsvpc",
+            "IPv4Addresses": [
+                "10.0.2.100"
+            ],
+            "AttachmentIndex": 0,
+            "MACAddress": "0e:9e:32:c7:48:85",
+            "IPv4SubnetCIDRBlock": "10.0.2.0/24",
+            "PrivateDNSName": "ip-10-0-2-100.us-west-2.compute.internal",
+            "SubnetGatewayIpv4Address": "10.0.2.1/24"
+        }
+    ]
+}
diff --git a/detectors/aws/ecs/test/metadatav4-response-task-ec2-bad-task-arn.json b/detectors/aws/ecs/test/metadatav4-response-task-ec2-bad-task-arn.json
@@ -0,0 +1,94 @@
+{
+    "Cluster": "default",
+    "TaskARN": "default/158d1c8083dd49d6b527399fd6414f5c",
+    "Family": "curltest",
+    "Revision": "26",
+    "DesiredStatus": "RUNNING",
+    "KnownStatus": "RUNNING",
+    "PullStartedAt": "2020-10-02T00:43:06.202617438Z",
+    "PullStoppedAt": "2020-10-02T00:43:06.31288465Z",
+    "AvailabilityZone": "us-west-2d",
+    "LaunchType": "EC2",
+    "Containers": [
+        {
+            "DockerId": "598cba581fe3f939459eaba1e071d5c93bb2c49b7d1ba7db6bb19deeb70d8e38",
+            "Name": "~internal~ecs~pause",
+            "DockerName": "ecs-curltest-26-internalecspause-e292d586b6f9dade4a00",
+            "Image": "amazon/amazon-ecs-pause:0.1.0",
+            "ImageID": "",
+            "Labels": {
+                "com.amazonaws.ecs.cluster": "default",
+                "com.amazonaws.ecs.container-name": "~internal~ecs~pause",
+                "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c",
+                "com.amazonaws.ecs.task-definition-family": "curltest",
+                "com.amazonaws.ecs.task-definition-version": "26"
+            },
+            "DesiredStatus": "RESOURCES_PROVISIONED",
+            "KnownStatus": "RESOURCES_PROVISIONED",
+            "Limits": {
+                "CPU": 0,
+                "Memory": 0
+            },
+            "CreatedAt": "2020-10-02T00:43:05.602352471Z",
+            "StartedAt": "2020-10-02T00:43:06.076707576Z",
+            "Type": "CNI_PAUSE",
+            "Networks": [
+                {
+                    "NetworkMode": "awsvpc",
+                    "IPv4Addresses": [
+                        "10.0.2.61"
+                    ],
+                    "AttachmentIndex": 0,
+                    "MACAddress": "0e:10:e2:01:bd:91",
+                    "IPv4SubnetCIDRBlock": "10.0.2.0/24",
+                    "PrivateDNSName": "ip-10-0-2-61.us-west-2.compute.internal",
+                    "SubnetGatewayIpv4Address": "10.0.2.1/24"
+                }
+            ]
+        },
+        {
+            "DockerId": "ee08638adaaf009d78c248913f629e38299471d45fe7dc944d1039077e3424ca",
+            "Name": "curl",
+            "DockerName": "ecs-curltest-26-curl-a0e7dba5aca6d8cb2e00",
+            "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
+            "ImageID": "sha256:d691691e9652791a60114e67b365688d20d19940dde7c4736ea30e660d8d3553",
+            "Labels": {
+                "com.amazonaws.ecs.cluster": "default",
+                "com.amazonaws.ecs.container-name": "curl",
+                "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c",
+                "com.amazonaws.ecs.task-definition-family": "curltest",
+                "com.amazonaws.ecs.task-definition-version": "26"
+            },
+            "DesiredStatus": "RUNNING",
+            "KnownStatus": "RUNNING",
+            "Limits": {
+                "CPU": 10,
+                "Memory": 128
+            },
+            "CreatedAt": "2020-10-02T00:43:06.326590752Z",
+            "StartedAt": "2020-10-02T00:43:06.767535449Z",
+            "Type": "NORMAL",
+            "LogDriver": "awslogs",
+            "LogOptions": {
+                "awslogs-create-group": "true",
+                "awslogs-group": "/ecs/metadata",
+                "awslogs-region": "us-west-2",
+                "awslogs-stream": "ecs/curl/158d1c8083dd49d6b527399fd6414f5c"
+            },
+            "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/abb51bdd-11b4-467f-8f6c-adcfe1fe059d",
+            "Networks": [
+                {
+                    "NetworkMode": "awsvpc",
+                    "IPv4Addresses": [
+                        "10.0.2.61"
+                    ],
+                    "AttachmentIndex": 0,
+                    "MACAddress": "0e:10:e2:01:bd:91",
+                    "IPv4SubnetCIDRBlock": "10.0.2.0/24",
+                    "PrivateDNSName": "ip-10-0-2-61.us-west-2.compute.internal",
+                    "SubnetGatewayIpv4Address": "10.0.2.1/24"
+                }
+            ]
+        }
+    ]
+}