UPSTREAM 2811: Handle the migration to the new TXT format - create mi…

…ssing records

controller/controller.go

@@ -167,6 +167,9 @@ func (c *Controller) RunOnce(ctx context.Context) error {
 		deprecatedRegistryErrors.Inc()
 		return err
 	}
+
+	missingRecords := c.Registry.MissingRecords()
+
 	registryEndpointsTotal.Set(float64(len(records)))
 	regARecords := filterARecords(records)
 	registryARecords.Set(float64(len(regARecords)))
@@ -189,6 +192,7 @@ func (c *Controller) RunOnce(ctx context.Context) error {
 		Policies:           []plan.Policy{c.Policy},
 		Current:            records,
 		Desired:            endpoints,
+		Missing:            missingRecords,
 		DomainFilter:       endpoint.MatchAllDomainFilters{c.DomainFilter, c.Registry.GetDomainFilter()},
 		PropertyComparator: c.Registry.PropertyValuesEqual,
 		ManagedRecords:     c.ManagedRecordTypes,

docs/registry.md

@@ -7,7 +7,8 @@ It contains record type it manages, e.g.:
 
 Prefix and suffix are extended with %{record_type} template where the user can control how prefixed/suffixed records should look like.
 
-In order to maintain compatibility, both records will be maintained for some time, in order to have downgrade possibility.
+In order to maintain compatibility, both records will be maintained for some time, in order to have downgrade possibility.  
+The controller will try to create the "new format" TXT records if they are not present to ease the migration from the versions < 0.12.0.
 
 Later on, the old format will be dropped and only the new format will be kept (<record_type>-<endpoint_name>).
 

main.go

@@ -341,7 +341,7 @@ func main() {
 	case "noop":
 		r, err = registry.NewNoopRegistry(p)
 	case "txt":
-		r, err = registry.NewTXTRegistry(p, cfg.TXTPrefix, cfg.TXTSuffix, cfg.TXTOwnerID, cfg.TXTCacheInterval, cfg.TXTWildcardReplacement)
+		r, err = registry.NewTXTRegistry(p, cfg.TXTPrefix, cfg.TXTSuffix, cfg.TXTOwnerID, cfg.TXTCacheInterval, cfg.TXTWildcardReplacement, cfg.ManagedDNSRecordTypes)
 	case "aws-sd":
 		r, err = registry.NewAWSSDRegistry(p.(*awssd.AWSSDProvider), cfg.TXTOwnerID)
 	default:

plan/plan.go

@@ -37,6 +37,8 @@ type Plan struct {
 	Current []*endpoint.Endpoint
 	// List of desired records
 	Desired []*endpoint.Endpoint
+	// List of missing records to be created, use for the migrations (e.g. old-new TXT format)
+	Missing []*endpoint.Endpoint
 	// Policies under which the desired changes are calculated
 	Policies []Policy
 	// List of changes necessary to move towards desired state
@@ -170,6 +172,11 @@ func (p *Plan) Calculate() *Plan {
 		changes = pol.Apply(changes)
 	}
 
+	// Handle the migration of the TXT records created before the new format (introduced in v0.12.0)
+	if len(p.Missing) > 0 {
+		changes.Create = append(changes.Create, filterRecordsForPlan(p.Missing, p.DomainFilter, append(p.ManagedRecords, endpoint.RecordTypeTXT))...)
+	}
+
 	plan := &Plan{
 		Current:        p.Current,
 		Desired:        p.Desired,

plan/plan_test.go

@@ -48,6 +48,9 @@ type PlanTestSuite struct {
 	domainFilterFiltered2            *endpoint.Endpoint
 	domainFilterFiltered3            *endpoint.Endpoint
 	domainFilterExcluded             *endpoint.Endpoint
+	domainFilterFilteredTXT1         *endpoint.Endpoint
+	domainFilterFilteredTXT2         *endpoint.Endpoint
+	domainFilterExcludedTXT          *endpoint.Endpoint
 }
 
 func (suite *PlanTestSuite) SetupTest() {
@@ -203,6 +206,21 @@ func (suite *PlanTestSuite) SetupTest() {
 		Targets:    endpoint.Targets{"1.1.1.1"},
 		RecordType: "A",
 	}
+	suite.domainFilterFilteredTXT1 = &endpoint.Endpoint{
+		DNSName:    "a-foo.domain.tld",
+		Targets:    endpoint.Targets{"\"heritage=external-dns,external-dns/owner=owner\""},
+		RecordType: "TXT",
+	}
+	suite.domainFilterFilteredTXT2 = &endpoint.Endpoint{
+		DNSName:    "cname-bar.domain.tld",
+		Targets:    endpoint.Targets{"\"heritage=external-dns,external-dns/owner=owner\""},
+		RecordType: "TXT",
+	}
+	suite.domainFilterExcludedTXT = &endpoint.Endpoint{
+		DNSName:    "cname-bar.otherdomain.tld",
+		Targets:    endpoint.Targets{"\"heritage=external-dns,external-dns/owner=owner\""},
+		RecordType: "TXT",
+	}
 }
 
 func (suite *PlanTestSuite) TestSyncFirstRound() {
@@ -667,6 +685,22 @@ func (suite *PlanTestSuite) TestDomainFiltersUpdate() {
 	validateEntries(suite.T(), changes.Delete, expectedDelete)
 }
 
+func (suite *PlanTestSuite) TestMissing() {
+
+	missing := []*endpoint.Endpoint{suite.domainFilterFilteredTXT1, suite.domainFilterFilteredTXT2, suite.domainFilterExcludedTXT}
+	expectedCreate := []*endpoint.Endpoint{suite.domainFilterFilteredTXT1, suite.domainFilterFilteredTXT2}
+
+	p := &Plan{
+		Policies:       []Policy{&SyncPolicy{}},
+		Missing:        missing,
+		DomainFilter:   endpoint.NewDomainFilter([]string{"domain.tld"}),
+		ManagedRecords: []string{endpoint.RecordTypeA, endpoint.RecordTypeCNAME},
+	}
+
+	changes := p.Calculate().Changes
+	validateEntries(suite.T(), changes.Create, expectedCreate)
+}
+
 func TestPlan(t *testing.T) {
 	suite.Run(t, new(PlanTestSuite))
 }

registry/aws_sd_registry.go

@@ -67,6 +67,11 @@ func (sdr *AWSSDRegistry) Records(ctx context.Context) ([]*endpoint.Endpoint, er
 	return records, nil
 }
 
+// MissingRecords returns nil because there is no missing records for AWSSD registry
+func (sdr *AWSSDRegistry) MissingRecords() []*endpoint.Endpoint {
+	return nil
+}
+
 // ApplyChanges filters out records not owned the External-DNS, additionally it adds the required label
 // inserted in the AWS SD instance as a CreateID field
 func (sdr *AWSSDRegistry) ApplyChanges(ctx context.Context, changes *plan.Changes) error {

registry/noop.go

@@ -45,6 +45,11 @@ func (im *NoopRegistry) Records(ctx context.Context) ([]*endpoint.Endpoint, erro
 	return im.provider.Records(ctx)
 }
 
+// MissingRecords returns nil because there is no missing records for Noop registry
+func (im *NoopRegistry) MissingRecords() []*endpoint.Endpoint {
+	return nil
+}
+
 // ApplyChanges propagates changes to the dns provider
 func (im *NoopRegistry) ApplyChanges(ctx context.Context, changes *plan.Changes) error {
 	return im.provider.ApplyChanges(ctx, changes)

registry/registry.go

@@ -35,6 +35,7 @@ type Registry interface {
 	PropertyValuesEqual(attribute string, previous string, current string) bool
 	AdjustEndpoints(endpoints []*endpoint.Endpoint) []*endpoint.Endpoint
 	GetDomainFilter() endpoint.DomainFilterInterface
+	MissingRecords() []*endpoint.Endpoint
 }
 
 //TODO(ideahitme): consider moving this to Plan

registry/txt.go

@@ -47,10 +47,15 @@ type TXTRegistry struct {
 	// registry TXT records corresponding to wildcard records will be invalid (and rejected by most providers), due to
 	// having a '*' appear (not as the first character) - see https://tools.ietf.org/html/rfc1034#section-4.3.3
 	wildcardReplacement string
+
+	managedRecordTypes []string
+
+	// missingTXTRecords stores TXT records which are missing after the migration to the new format
+	missingTXTRecords []*endpoint.Endpoint
 }
 
 // NewTXTRegistry returns new TXTRegistry object
-func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID string, cacheInterval time.Duration, txtWildcardReplacement string) (*TXTRegistry, error) {
+func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID string, cacheInterval time.Duration, txtWildcardReplacement string, managedRecordTypes []string) (*TXTRegistry, error) {
 	if ownerID == "" {
 		return nil, errors.New("owner id cannot be empty")
 	}
@@ -67,6 +72,7 @@ func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID st
 		mapper:              mapper,
 		cacheInterval:       cacheInterval,
 		wildcardReplacement: txtWildcardReplacement,
+		managedRecordTypes:  managedRecordTypes,
 	}, nil
 }
 
@@ -95,8 +101,10 @@ func (im *TXTRegistry) Records(ctx context.Context) ([]*endpoint.Endpoint, error
 	}
 
 	endpoints := []*endpoint.Endpoint{}
+	missingEndpoints := []*endpoint.Endpoint{}
 
 	labelMap := map[string]endpoint.Labels{}
+	txtRecordsMap := map[string]struct{}{}
 
 	for _, record := range records {
 		if record.RecordType != endpoint.RecordTypeTXT {
@@ -117,6 +125,7 @@ func (im *TXTRegistry) Records(ctx context.Context) ([]*endpoint.Endpoint, error
 		}
 		key := fmt.Sprintf("%s::%s", im.mapper.toEndpointName(record.DNSName), record.SetIdentifier)
 		labelMap[key] = labels
+		txtRecordsMap[record.DNSName] = struct{}{}
 	}
 
 	for _, ep := range endpoints {
@@ -135,6 +144,18 @@ func (im *TXTRegistry) Records(ctx context.Context) ([]*endpoint.Endpoint, error
 				ep.Labels[k] = v
 			}
 		}
+		// Handle the migration of TXT records created before the new format (introduced in v0.12.0)
+		if len(txtRecordsMap) > 0 {
+			if isManagedRecord(ep, im.managedRecordTypes) {
+				// get desired TXT records and detect the missing ones
+				for _, desiredTXT := range im.generateTXTRecord(ep) {
+					if _, exists := txtRecordsMap[desiredTXT.DNSName]; !exists {
+						// add missing TXT record
+						missingEndpoints = append(missingEndpoints, desiredTXT)
+					}
+				}
+			}
+		}
 	}
 
 	// Update the cache.
@@ -143,12 +164,25 @@ func (im *TXTRegistry) Records(ctx context.Context) ([]*endpoint.Endpoint, error
 		im.recordsCacheRefreshTime = time.Now()
 	}
 
+	im.missingTXTRecords = missingEndpoints
+
 	return endpoints, nil
 }
 
+// MissingRecords returns the TXT record to be created.
+// The missing records are collected during the run of Records method.
+func (im *TXTRegistry) MissingRecords() []*endpoint.Endpoint {
+	return im.missingTXTRecords
+}
+
 // generateTXTRecord generates both "old" and "new" TXT records.
 // Once we decide to drop old format we need to drop toTXTName() and rename toNewTXTName
 func (im *TXTRegistry) generateTXTRecord(r *endpoint.Endpoint) []*endpoint.Endpoint {
+	// missing TXT records are added to the set of changes,
+	// obviously we don't need any other TXT record for them
+	if r.RecordType == endpoint.RecordTypeTXT {
+		return nil
+	}
 	// old TXT record format
 	txt := endpoint.NewEndpoint(im.mapper.toTXTName(r.DNSName), endpoint.RecordTypeTXT, r.Labels.Serialize(true)).WithSetIdentifier(r.SetIdentifier)
 	txt.ProviderSpecific = r.ProviderSpecific
@@ -397,3 +431,12 @@ func (im *TXTRegistry) removeFromCache(ep *endpoint.Endpoint) {
 		}
 	}
 }
+
+func isManagedRecord(record *endpoint.Endpoint, managedTypes []string) bool {
+	for _, mt := range managedTypes {
+		if record.RecordType == mt {
+			return true
+		}
+	}
+	return false
+}