Skip to content

Commit

Permalink
feat(rule): Adding a general tag rule (#233)
Browse files Browse the repository at this point in the history
* feat(rule): Adding a general tag rule

- will match defined tag values in config to the ones on the resource
- if matched and calculated age is older than today, the resource is marked

* - A bit of needed cleanup w/r to tags
- make tags a distinct object on the resource for convenience
- temporal tags are managed internally
- fix a bug on ttl calculation

* - keep it simple, remove the need for a basic rule
- renamed TemporalThresholdRule to ExpiredResourceRule
- Updated to use a clock to determine if resource is expired
- clean up test

* - s/prefix/suffix/ in docs

* - cleanup

* - fix amazonTagExclusion based on testing

* - cleanup
  • Loading branch information
jeyrschabu authored and mergify[bot] committed Oct 29, 2019
1 parent 061707a commit 70f12bd
Show file tree
Hide file tree
Showing 10 changed files with 258 additions and 150 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Copyright 2019 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.spinnaker.swabbie.aws

import com.netflix.spinnaker.swabbie.aws.model.AmazonResource
import com.netflix.spinnaker.swabbie.model.Result
import com.netflix.spinnaker.swabbie.model.Rule
import com.netflix.spinnaker.swabbie.model.Summary
import org.springframework.stereotype.Component
import java.time.Clock

/**
* This rule applies if this amazon resource has expired.
* A resource is expired if it's tagged with the following keys: ("expiration_time", "expires", "ttl")
* Acceptable tag value: a number followed by a suffix such as d (days), w (weeks), m (month), y (year)
* @see com.netflix.spinnaker.swabbie.tagging.TemporalTags.supportedTemporalTagValues
*/

@Component
class ExpiredResourceRule<T : AmazonResource>(
private val clock: Clock
) : Rule<T> {
override fun apply(resource: T): Result {
if (resource.expired(clock)) {
return Result(
Summary(
description = "$${resource.resourceId} has expired. tags: ${resource.tags()}",
ruleName = javaClass.simpleName
)
)
}

return Result(null)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,50 +21,35 @@ import com.netflix.spinnaker.config.ExclusionType
import com.netflix.spinnaker.swabbie.aws.model.AmazonResource
import com.netflix.spinnaker.swabbie.exclusions.Excludable
import com.netflix.spinnaker.swabbie.exclusions.ResourceExclusionPolicy
import com.netflix.spinnaker.swabbie.model.BasicTag
import org.springframework.stereotype.Component
import java.time.Clock

@Component
class AmazonTagExclusionPolicy : ResourceExclusionPolicy {
private val tagsField = "tags"
class AmazonTagExclusionPolicy(
val clock: Clock
) : ResourceExclusionPolicy {
override fun getType(): ExclusionType = ExclusionType.Tag
override fun apply(excludable: Excludable, exclusions: List<Exclusion>): String? {
if (excludable is AmazonResource) {
keysAndValues(exclusions, ExclusionType.Tag)
.let { excludingTags ->
if (tagsField in excludable.details) {
(excludable.details[tagsField] as List<Map<*, *>>).map { tag ->
tag.keys.find { key ->
excludingTags[key] != null
}?.let { key ->
if (key in TemporalTagExclusionSupplier.temporalTags) {
excludingTags[key]!!.map { target ->
TemporalTagExclusionSupplier
.computeAndCompareAge(
excludable = excludable,
tagValue = tag[key] as String,
target = target
).let {
when {
it.age == Age.OLDER || it.age == Age.INFINITE ->
return patternMatchMessage(tag[key] as String, excludingTags[key]!!.toSet())
it.age == Age.YOUNGER ->
return null
else -> {
// no need to check age here.
log.debug("Resource age comparison with {}. Result: {}", excludable.createTs, it)
}
}
}
}
}
if (excludable !is AmazonResource || excludable.tags().isNullOrEmpty()) {
return null
}

val tags = excludable.tags()!!
// Exclude this resource if it's tagged with a ttl but has not yet expired
val temporalTags = tags.filter(BasicTag::isTemporal)
if (temporalTags.isNotEmpty() && !excludable.expired(clock)) {
val keysAsString = temporalTags.map { it.key }.joinToString { "," }
val valuesAsString = temporalTags.map { it.value }.toString()
return patternMatchMessage(keysAsString, setOf(valuesAsString))
}

if (excludingTags[key]!!.contains(tag[key] as? String)) {
return patternMatchMessage(tag[key] as String, excludingTags[key]!!.toSet())
}
}
}
}
}
val configuredKeysAndTargetValues = keysAndValues(exclusions, ExclusionType.Tag)
tags.forEach { tag ->
val target = configuredKeysAndTargetValues[tag.key] ?: emptyList()
if (target.contains(tag.value as String)) {
return patternMatchMessage(tag.key, setOf(tag.value as String))
}
}

return null
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,98 +19,25 @@ package com.netflix.spinnaker.swabbie.aws.exclusions
import com.netflix.spinnaker.config.Attribute
import com.netflix.spinnaker.config.Exclusion
import com.netflix.spinnaker.config.ExclusionType
import com.netflix.spinnaker.swabbie.aws.model.AmazonResource
import com.netflix.spinnaker.swabbie.exclusions.ExclusionsSupplier
import com.netflix.spinnaker.swabbie.tagging.TemporalTags
import org.springframework.stereotype.Component
import java.time.Instant
import java.time.LocalDate
import java.time.ZoneId
import java.time.temporal.ChronoUnit

@Component
class TemporalTagExclusionSupplier : ExclusionsSupplier {
companion object {
fun computeAndCompareAge(excludable: AmazonResource, tagValue: String, target: String): AgeCompareResult {
if (target.startsWith("pattern:")) {
target.split(":").last().toRegex().find(tagValue)?.groupValues?.let {
val unit = it[1]
val suppliedAmountWithoutUnit = it[0].replace(unit, "").toLong()
supportedTemporalUnits[unit]?.between(
Instant.ofEpochMilli(excludable.createTs),
Instant.now()
)?.let { elapsedSinceCreation ->
val computedTargetStamp = LocalDate.now()
.plus(suppliedAmountWithoutUnit, supportedTemporalUnits[unit])
.atStartOfDay(ZoneId.systemDefault())
.toInstant()
.toEpochMilli()

return if (suppliedAmountWithoutUnit >= elapsedSinceCreation) {
AgeCompareResult(
age = Age.OLDER,
suppliedStamp = computedTargetStamp,
comparedStamp = excludable.createTs
)
} else {
AgeCompareResult(
age = Age.YOUNGER,
suppliedStamp = computedTargetStamp,
comparedStamp = excludable.createTs
)
}
}
}
}

return if (tagValue == "never") {
AgeCompareResult(
age = Age.INFINITE,
suppliedStamp = null,
comparedStamp = excludable.createTs
)
} else {
AgeCompareResult(
age = Age.UNKNOWN,
suppliedStamp = null,
comparedStamp = excludable.createTs
)
}
}

val temporalTags = listOf("expiration_time", "expires", "ttl")
private val supportedTemporalUnits = mapOf(
"d" to ChronoUnit.DAYS,
"m" to ChronoUnit.MONTHS,
"y" to ChronoUnit.YEARS,
"w" to ChronoUnit.WEEKS
)
}

private val supportedTemporalTagValues = listOf("pattern:^\\d+(d|m|y|w)$", "never")

override fun get(): List<Exclusion> {
return listOf(
Exclusion()
.withType(ExclusionType.Tag.toString())
.withAttributes(
temporalTags.map { key ->
TemporalTags.temporalTags.map { key ->
Attribute()
.withKey(key)
.withValue(
supportedTemporalTagValues
TemporalTags.supportedTemporalTagValues
)
}.toSet()
)
)
}
}

enum class Age {
INFINITE, OLDER, EQUAL, YOUNGER, UNKNOWN
}

data class AgeCompareResult(
val age: Age,
val suppliedStamp: Long?,
val comparedStamp: Long
)
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ import java.time.ZoneId
data class AmazonInstance(
val instanceId: String,
val imageId: String,
val tags: List<Map<String, String>>,
private val launchTime: Long,
override val resourceId: String = instanceId,
override val resourceType: String = INSTANCE,
Expand All @@ -38,9 +37,7 @@ data class AmazonInstance(
LocalDateTime.ofInstant(Instant.ofEpochMilli(launchTime), ZoneId.systemDefault()).toString()
) : AmazonResource(creationDate) {
fun getAutoscalingGroup(): String? {
return tags
.find { it.containsKey("aws:autoscaling:groupName") }
?.get("aws:autoscaling:groupName")
return tags()?.find { it.key == "aws:autoscaling:groupName" }?.value as String
}

override fun equals(other: Any?): Boolean {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ abstract class AmazonResource(
get() {
if (resourceType.contains("image", ignoreCase = true) || resourceType.contains("snapshot", ignoreCase = true)) {
// Images and snapshots have only packageName, not app, to group by
getTagValue("appversion")?.let { AppVersion.parseName(it)?.packageName }?.let { packageName ->
getTagValue("appversion")?.let { AppVersion.parseName(it as String)?.packageName }?.let { packageName ->
return Grouping(packageName, GroupingType.PACKAGE_NAME)
}
return null
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
* Copyright 2019 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.spinnaker.swabbie.aws

import com.netflix.spinnaker.kork.test.time.MutableClock
import com.netflix.spinnaker.swabbie.aws.autoscalinggroups.AmazonAutoScalingGroup
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import strikt.api.expectThat
import strikt.assertions.isNotNull
import strikt.assertions.isNull
import java.time.Duration
import java.time.Instant

object ExpiredResourceRuleTest {
private val clock = MutableClock()
private val subject = ExpiredResourceRule<AmazonAutoScalingGroup>(clock)
private val now = Instant.now(clock).toEpochMilli()
private val asg = AmazonAutoScalingGroup(
autoScalingGroupName = "testapp-v001",
instances = listOf(
mapOf("instanceId" to "i-01234")
),
loadBalancerNames = listOf(),
createdTime = now
)

@BeforeEach
fun setup() {
asg.set("tags", null)
}

@Test
fun `should not apply if resource is not tagged with a ttl`() {
expectThat(
subject.apply(asg).summary
).isNull()
}

@Test
fun `should not apply if resource is not expired`() {
val tags = listOf(
mapOf("ttl" to "4d")
)

asg.set("tags", tags)
expectThat(
subject.apply(asg).summary
).isNull()
}

@Test
fun `should apply if resource is expired`() {
val tags = listOf(
mapOf("ttl" to "2d")
)

asg.set("tags", tags)

clock.incrementBy(Duration.ofDays(3))
expectThat(
subject.apply(asg).summary
).isNotNull()
}
}
Loading

0 comments on commit 70f12bd

Please sign in to comment.