Merge pull request #237 from meshuga/filters

Allowed filtering by all fields

README.md

@@ -82,20 +82,21 @@ Read-only permissions
 
 #### Filtering
 
-Filters are a way to choose which resources `terraformer` imports.
-
-For example:
-```
-terraformer import aws --resources=vpc,subnet --filter=aws_vpc=myvpcid --regions=eu-west-1
-```
-will only import the VPC with id `myvpcid`.
+Filters are a way to choose which resources `terraformer` imports. It's possible to filter resources by its identifiers or attributes. Multiple filtering values are separated by `:`. If an identifier contains this symbol, value should be wrapped in `'` e.g. `--filter=resource=id1:'project:dataset_id'`. Identifier based filters will be executed before Terraformer will try to refresh remote state.
 
 ##### Resource ID
 
 Filtering is based on Terraform resource ID patterns. To find valid ID patterns for your resource, check the import part of the [Terraform documentation][terraform-providers].
 
 [terraform-providers]: https://www.terraform.io/docs/providers/
 
+Example usage:
+
+```
+terraformer import aws --resources=vpc,subnet --filter=aws_vpc=myvpcid --regions=eu-west-1
+```
+Will only import the vpc with id `myvpcid`. This form of filters can help when it's necessary to select resources by its identifiers.
+
 #### Planning
 
 The `plan` command generates a planfile that contains all the resources set to be imported. By modifying the planfile before running the `import` command, you can rename or filter the resources you'd like to import.
@@ -321,6 +322,8 @@ Example:
  terraformer import aws --resources=vpc,subnet --filter=aws_vpc=vpc_id1:vpc_id2:vpc_id3 --regions=eu-west-1
 ```
 
+#### Profiles support
+
 To load profiles from the shared AWS configuration file (typically `~/.aws/config`), set the `AWS_SDK_LOAD_CONFIG` to `true`:
 
 ```
@@ -333,7 +336,7 @@ terraformer import aws --resources=cloudfront --profile=prod
 ```
 In that case terraformer will not know with which region resources are associated with and will not assume any region. That scenario is useful in case of global resources (e.g. CloudFront distributions or Route 53 records) and when region is passed implicitly through environmental variables or metadata service.
 
-List of supported AWS services:
+#### Supported services
 
 *   `acm`
     * `aws_acm_certificate`
@@ -439,6 +442,8 @@ List of supported AWS services:
 *   `vpn_gateway`
     * `aws_vpn_gateway`
 
+#### Global services
+
 AWS services that are global will be imported without specified region even if several regions will be passed. It is to ensure only one representation of an AWS resource is imported.
 
 List of global AWS services:
@@ -447,6 +452,19 @@ List of global AWS services:
 *   `organization`
 *   `route53`
 
+#### Attribute filters
+
+Attribute filters allow filtering across different resource types by its attributes.
+
+```
+terraformer import aws --resources=ec2_instance,ebs --filter=Name=tags.costCenter;Value=20000:'20001:1' --regions=eu-west-1
+```
+Will only import AWS EC2 instances along with EBS volumes annotated with tag `costCenter` with values `20000` or `20001:1`. Attribute filters are by default applicable to all resource types although it's possible to specify to what resource type a given filter should be applicable to by providing `Type=<type>` parameter. For example:
+```
+terraformer import aws --resources=ec2_instance,ebs --filter=Type=ec2_instance;Name=tags.costCenter;Value=20000:'20001:1' --regions=eu-west-1
+```
+Will work as same as example above with a change the filter will be applicable only to `ec2_instance` resources.
+
 ### Use with Azure
 
 Example:

cmd/import.go

@@ -88,16 +88,13 @@ func Import(provider terraform_utils.ProviderGenerator, options ImportOptions, a
 		if err != nil {
 			return err
 		}
+		provider.GetService().ParseFilters(options.Filter)
 		err = provider.GetService().InitResources()
 		provider.GetService().PopulateIgnoreKeys(provider.GetBasicConfig())
 		if err != nil {
 			return err
 		}
-
-		if len(options.Filter) != 0 {
-			provider.GetService().ParseFilter(options.Filter)
-			provider.GetService().CleanupWithFilter()
-		}
+		provider.GetService().InitialCleanup()
 
 		providerWrapper, err := provider_wrapper.NewProviderWrapper(provider.GetName(), provider.GetConfig())
 		if err != nil {
@@ -119,6 +116,8 @@ func Import(provider terraform_utils.ProviderGenerator, options ImportOptions, a
 
 		providerWrapper.Kill()
 
+		provider.GetService().PostRefreshCleanup()
+
 		// change structs with additional data for each resource
 		err = provider.GetService().PostConvertHook()
 		if err != nil {

providers/aws/ebs.go

@@ -16,6 +16,7 @@ package aws
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/GoogleCloudPlatform/terraformer/terraform_utils"
 	"github.com/aws/aws-sdk-go/aws"
@@ -37,8 +38,19 @@ func (g EbsGenerator) volumeAttachmentId(device, volumeID, instanceID string) st
 func (g *EbsGenerator) InitResources() error {
 	sess := g.generateSession()
 	svc := ec2.New(sess)
-
-	err := svc.DescribeVolumesPages(&ec2.DescribeVolumesInput{}, func(volumes *ec2.DescribeVolumesOutput, lastPage bool) bool {
+	var filters []*ec2.Filter
+	for _, filter := range g.Filter {
+		if strings.HasPrefix(filter.FieldPath, "tags.") && filter.IsApplicable("aws_ebs_volume") {
+			filters = append(filters, &ec2.Filter{
+				Name:   aws.String("tag:" + strings.TrimPrefix(filter.FieldPath, "tags.")),
+				Values: aws.StringSlice(filter.AcceptableValues),
+			})
+		}
+	}
+	input := ec2.DescribeVolumesInput{
+		Filters:filters,
+	}
+	err := svc.DescribeVolumesPages(&input, func(volumes *ec2.DescribeVolumesOutput, lastPage bool) bool {
 		for _, volume := range volumes.Volumes {
 
 			isRootDevice := false // Let's leave root device configuration to be done in ec2_instance resources

providers/aws/ec2.go

@@ -32,7 +32,19 @@ type Ec2Generator struct {
 func (g *Ec2Generator) InitResources() error {
 	sess := g.generateSession()
 	svc := ec2.New(sess)
-	err := svc.DescribeInstancesPages(&ec2.DescribeInstancesInput{}, func(instances *ec2.DescribeInstancesOutput, lastPage bool) bool {
+	var filters []*ec2.Filter
+	for _, filter := range g.Filter {
+		if strings.HasPrefix(filter.FieldPath, "tags.") && filter.IsApplicable("aws_instance") {
+			filters = append(filters, &ec2.Filter{
+				Name:   aws.String("tag:" + strings.TrimPrefix(filter.FieldPath, "tags.")),
+				Values: aws.StringSlice(filter.AcceptableValues),
+			})
+		}
+	}
+	input := ec2.DescribeInstancesInput{
+		Filters: filters,
+	}
+	err := svc.DescribeInstancesPages(&input, func(instances *ec2.DescribeInstancesOutput, lastPage bool) bool {
 		for _, reservation := range instances.Reservations {
 			for _, instance := range reservation.Instances {
 				name := ""

providers/aws/s3.go

@@ -16,10 +16,8 @@ package aws
 
 import (
 	"fmt"
-	"log"
-	"strings"
-
 	"github.com/GoogleCloudPlatform/terraformer/terraform_utils"
+	"log"
 
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -116,18 +114,19 @@ POLICY`, policy)
 	return nil
 }
 
-func (g *S3Generator) ParseFilter(rawFilter []string) {
-	g.Filter = map[string][]string{}
-	for _, resource := range rawFilter {
-		t := strings.Split(resource, "=")
-		if len(t) != 2 {
-			log.Println("Pattern for filter must be resource_type=id1:id2:id4")
-			continue
-		}
-		resourceName, resourcesID := t[0], t[1]
-		g.Filter[resourceName] = strings.Split(resourcesID, ":")
-		if resourceName == "aws_s3_bucket" {
-			g.Filter["aws_s3_bucket_policy"] = strings.Split(resourcesID, ":")
+func (g *S3Generator) ParseFilters(rawFilters []string) {
+	g.Filter = []terraform_utils.ResourceFilter{}
+	for _, rawFilter := range rawFilters {
+		filters := g.ParseFilter(rawFilter)
+		for _, resourceFilter := range filters {
+			g.Filter = append(g.Filter, resourceFilter)
+			if resourceFilter.ResourceName == "aws_s3_bucket" {
+				g.Filter = append(g.Filter, terraform_utils.ResourceFilter{
+					ResourceName:     "aws_s3_bucket_policy",
+					FieldPath:        resourceFilter.FieldPath,
+					AcceptableValues: resourceFilter.AcceptableValues,
+				})
+			}
 		}
 	}
 }

terraform_utils/resource.go

@@ -36,6 +36,48 @@ type Resource struct {
 	AdditionalFields map[string]interface{} `json:",omitempty"`
 }
 
+type ApplicableFilter interface {
+	IsApplicable(resourceName string) bool
+}
+
+type ResourceFilter struct {
+	ApplicableFilter
+	ResourceName     string
+	FieldPath        string
+	AcceptableValues []string
+}
+
+func (rf *ResourceFilter) Filter(resource Resource) bool {
+	if !rf.IsApplicable(resource.InstanceInfo.Type) {
+		return true
+	}
+	var vals []interface{}
+	if rf.FieldPath == "id" {
+		vals = []interface{}{resource.InstanceState.ID}
+	} else {
+		vals = WalkAndGet(rf.FieldPath, resource.InstanceState.Attributes)
+		if len(vals) == 0 {
+			vals = WalkAndGet(rf.FieldPath, resource.Item)
+		}
+	}
+	for _, val := range vals {
+		for _, acceptableValue := range rf.AcceptableValues {
+			if val == acceptableValue {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func (rf *ResourceFilter) IsApplicable(resourceName string) bool {
+	return rf.ResourceName == "" || rf.ResourceName == resourceName
+}
+
+func (rf *ResourceFilter) isInitial() bool {
+	return rf.FieldPath == "id"
+}
+
 func NewResource(ID, resourceName, resourceType, provider string,
 	attributes map[string]string,
 	allowEmptyValues []string,

terraform_utils/service.go

@@ -25,64 +25,94 @@ type ServiceGenerator interface {
 	InitResources() error
 	GetResources() []Resource
 	SetResources(resources []Resource)
-	ParseFilter(rawFilter []string)
+	ParseFilter(rawFilter string) []ResourceFilter
+	ParseFilters(rawFilters []string)
 	PostConvertHook() error
 	GetArgs() map[string]interface{}
 	SetArgs(args map[string]interface{})
 	SetName(name string)
 	SetProviderName(name string)
 	GetName() string
-	CleanupWithFilter()
+	InitialCleanup()
 	PopulateIgnoreKeys(cty.Value)
+	PostRefreshCleanup()
 }
 
 type Service struct {
 	Name         string
 	Resources    []Resource
 	ProviderName string
 	Args         map[string]interface{}
-	Filter       map[string][]string
+	Filter       []ResourceFilter
 }
 
 func (s *Service) SetProviderName(providerName string) {
 	s.ProviderName = providerName
 }
 
-func (s *Service) ParseFilter(rawFilter []string) {
-	s.Filter = map[string][]string{}
-	for _, resource := range rawFilter {
-		t := strings.Split(resource, "=")
-		if len(t) != 2 {
-			log.Println("Pattern for filter must be resource_type=id1:id2:id4")
-			continue
+func (s *Service) ParseFilters(rawFilters []string) {
+	s.Filter = []ResourceFilter{}
+	for _, rawFilter := range rawFilters {
+		filters := s.ParseFilter(rawFilter)
+		for _, resourceFilter := range filters {
+			s.Filter = append(s.Filter, resourceFilter)
 		}
-		resourceName, resourcesID := t[0], t[1]
-		s.Filter[resourceName] = strings.Split(resourcesID, ":")
 	}
 }
 
+func (s *Service) ParseFilter(rawFilter string) []ResourceFilter {
+	var filters []ResourceFilter
+	if len(strings.Split(rawFilter, "=")) == 2 {
+		parts := strings.Split(rawFilter, "=")
+		resourceName, resourcesID := parts[0], parts[1]
+		filters = append(filters, ResourceFilter{
+			ResourceName:     resourceName,
+			FieldPath:        "id",
+			AcceptableValues: ParseFilterValues(resourcesID),
+		})
+	} else {
+		parts := strings.Split(rawFilter, ";")
+		if len(parts) != 2 && len(parts) != 3 {
+			log.Print("Invalid filter: " + rawFilter)
+			return filters
+		}
+		var ResourceNamePart string
+		var FieldPathPart string
+		var AcceptableValuesPart string
+		if len(parts) == 2 {
+			ResourceNamePart = ""
+			FieldPathPart = parts[0]
+			AcceptableValuesPart = parts[1]
+		} else {
+			ResourceNamePart = strings.TrimPrefix(parts[0], "Type=")
+			FieldPathPart = parts[1]
+			AcceptableValuesPart = parts[2]
+		}
+
+		filters = append(filters, ResourceFilter{
+			ResourceName:     ResourceNamePart,
+			FieldPath:        strings.TrimPrefix(FieldPathPart, "Name="),
+			AcceptableValues: ParseFilterValues(strings.TrimPrefix(AcceptableValuesPart, "Value=")),
+		})
+	}
+	return filters
+}
+
 func (s *Service) SetName(name string) {
 	s.Name = name
 }
 func (s *Service) GetName() string {
 	return s.Name
 }
 
-func (s *Service) CleanupWithFilter() {
-	if len(s.Filter) == 0 {
-		return
-	}
-	newListOfResources := []Resource{}
-	for _, v := range s.Resources {
-		if _, exist := s.Filter[v.InstanceInfo.Type]; exist {
-			for _, r := range s.Filter[v.InstanceInfo.Type] {
-				if v.InstanceState.ID == r {
-					newListOfResources = append(newListOfResources, v)
-				}
-			}
-		}
+func (s *Service) InitialCleanup() {
+	FilterCleanup(s, true)
+}
+
+func (s *Service) PostRefreshCleanup() {
+	if len(s.Filter) != 0 {
+		FilterCleanup(s, false)
 	}
-	s.Resources = newListOfResources
 }
 
 func (s *Service) GetArgs() map[string]interface{} {