From 43a6b587902e41d388f95db12996b7fa70504868 Mon Sep 17 00:00:00 2001 From: rodman10 <1181591811hzr@gmail.com> Date: Fri, 21 Jul 2023 16:27:39 +0800 Subject: [PATCH 1/9] feat: support expression route for tcproute. --- internal/dataplane/parser/atc/field.go | 4 ++- .../dataplane/parser/translate_tcproute.go | 9 +++---- internal/dataplane/parser/translate_utils.go | 8 ++++++ .../parser/translators/tcproute_atc.go | 25 +++++++++++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 internal/dataplane/parser/translators/tcproute_atc.go diff --git a/internal/dataplane/parser/atc/field.go b/internal/dataplane/parser/atc/field.go index b48db3380e..a3cdb503ea 100644 --- a/internal/dataplane/parser/atc/field.go +++ b/internal/dataplane/parser/atc/field.go @@ -47,6 +47,7 @@ const ( FieldHTTPMethod StringField = "http.method" FieldHTTPHost StringField = "http.host" FieldHTTPPath StringField = "http.path" + FieldNetDstIP StringField = "net.dst.ip" ) // IntField is defined for fields with constant name and having integer type. @@ -64,7 +65,8 @@ func (f IntField) String() string { // https://docs.konghq.com/gateway/latest/reference/router-expressions-language/#available-fields const ( - FieldNetPort IntField = "net.port" + FieldNetPort IntField = "net.port" + FieldNetDstPort IntField = "net.dst.port" ) // HTTPHeaderField extracts the value of an HTTP header from the request. diff --git a/internal/dataplane/parser/translate_tcproute.go b/internal/dataplane/parser/translate_tcproute.go index cabec220fb..311d7947c1 100644 --- a/internal/dataplane/parser/translate_tcproute.go +++ b/internal/dataplane/parser/translate_tcproute.go @@ -25,11 +25,6 @@ func (p *Parser) ingressRulesFromTCPRoutes() ingressRules { var errs []error for _, tcproute := range tcpRouteList { - if p.featureFlags.ExpressionRoutes { - p.registerResourceFailureNotSupportedForExpressionRoutes(tcproute) - continue - } - if err := p.ingressRulesFromTCPRoute(&result, tcproute); err != nil { err = fmt.Errorf("TCPRoute %s/%s can't be routed: %w", tcproute.Namespace, tcproute.Name, err) errs = append(errs, err) @@ -40,6 +35,10 @@ func (p *Parser) ingressRulesFromTCPRoutes() ingressRules { } } + if p.featureFlags.ExpressionRoutes { + applyExpressionToIngressRules(&result) + } + if len(errs) > 0 { for _, err := range errs { p.logger.Errorf(err.Error()) diff --git a/internal/dataplane/parser/translate_utils.go b/internal/dataplane/parser/translate_utils.go index ca7e864347..3c64bfe8e4 100644 --- a/internal/dataplane/parser/translate_utils.go +++ b/internal/dataplane/parser/translate_utils.go @@ -181,3 +181,11 @@ func maybePrependRegexPrefix(path, controllerPrefix string, applyLegacyHeuristic } return path } + +func applyExpressionToIngressRules(result *ingressRules) { + for _, svc := range result.ServiceNameToServices { + for i := range svc.Routes { + translators.ApplyExpressionToKongRoute(&svc.Routes[i]) + } + } +} diff --git a/internal/dataplane/parser/translators/tcproute_atc.go b/internal/dataplane/parser/translators/tcproute_atc.go new file mode 100644 index 0000000000..45517a0212 --- /dev/null +++ b/internal/dataplane/parser/translators/tcproute_atc.go @@ -0,0 +1,25 @@ +package translators + +import ( + "github.com/samber/lo" + + "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane/kongstate" + "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane/parser/atc" +) + +func ApplyExpressionToKongRoute(r *kongstate.Route) { + matchers := []atc.Matcher{} + + sniMatcher := sniMatcherFromSNIs(lo.Map(r.Route.SNIs, func(item *string, _ int) string { return *item })) + matchers = append(matchers, sniMatcher) + + // TODO(rodman10): replace with helper function. + portMatchers := make([]atc.Matcher, 0, len(r.Destinations)) + for _, dst := range r.Destinations { + portMatchers = append(portMatchers, atc.NewPredicate(atc.FieldNetDstPort, atc.OpEqual, atc.IntLiteral(*dst.Port))) + } + matchers = append(matchers, atc.Or(portMatchers...)) + + r.ExpressionRoutes = true + atc.ApplyExpression(&r.Route, atc.And(matchers...), 1) +} From f99caf133ba9ddca1186cd0cd521d300a111b9b2 Mon Sep 17 00:00:00 2001 From: rodman10 <1181591811hzr@gmail.com> Date: Fri, 21 Jul 2023 16:28:20 +0800 Subject: [PATCH 2/9] tests: add unittest. --- .../parser/translate_tcproute_test.go | 261 ++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 internal/dataplane/parser/translate_tcproute_test.go diff --git a/internal/dataplane/parser/translate_tcproute_test.go b/internal/dataplane/parser/translate_tcproute_test.go new file mode 100644 index 0000000000..3905d0ed9c --- /dev/null +++ b/internal/dataplane/parser/translate_tcproute_test.go @@ -0,0 +1,261 @@ +package parser + +import ( + "strings" + "testing" + + "github.com/kong/go-kong/kong" + "github.com/samber/lo" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane/failures" + "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane/kongstate" + "github.com/kong/kubernetes-ingress-controller/v2/internal/store" + "github.com/kong/kubernetes-ingress-controller/v2/internal/util/builder" +) + +func TestIngressRulesFromTCPRoutesUsingExpressionRoutes(t *testing.T) { + tcpRouteTypeMeta := metav1.TypeMeta{Kind: "TCPRoute", APIVersion: gatewayv1alpha2.SchemeGroupVersion.String()} + + testCases := []struct { + name string + tcpRoutes []*gatewayv1alpha2.TCPRoute + expectedKongServices []kongstate.Service + expectedKongRoutes map[string][]kongstate.Route + expectedFailures []failures.ResourceFailure + }{ + { + name: "tcproute with single rule and single backendref", + tcpRoutes: []*gatewayv1alpha2.TCPRoute{ + { + TypeMeta: tcpRouteTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "tcproute-1", + }, + Spec: gatewayv1alpha2.TCPRouteSpec{ + Rules: []gatewayv1alpha2.TCPRouteRule{ + { + BackendRefs: []gatewayv1alpha2.BackendRef{ + builder.NewBackendRef("service1").WithPort(80).Build(), + }, + }, + }, + }, + }, + }, + expectedKongServices: []kongstate.Service{ + { + Service: kong.Service{ + Name: kong.String("tcproute.default.tcproute-1.0"), + }, + Backends: []kongstate.ServiceBackend{ + { + Name: "service1", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(80)}, + }, + }, + }, + }, + expectedKongRoutes: map[string][]kongstate.Route{ + "tcproute.default.tcproute-1.0": { + { + Route: kong.Route{ + Name: kong.String("tcproute.default.tcproute-1.0.0"), + Expression: kong.String(`net.dst.port == 80`), + PreserveHost: kong.Bool(true), + Protocols: kong.StringSlice("tcp"), + }, + ExpressionRoutes: true, + }, + }, + }, + }, + { + name: "tcproute with single rule and multiple backendrefs", + tcpRoutes: []*gatewayv1alpha2.TCPRoute{ + { + TypeMeta: tcpRouteTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "tcproute-1", + }, + Spec: gatewayv1alpha2.TCPRouteSpec{ + Rules: []gatewayv1alpha2.TCPRouteRule{ + { + BackendRefs: []gatewayv1alpha2.BackendRef{ + builder.NewBackendRef("service1").WithPort(80).Build(), + builder.NewBackendRef("service2").WithPort(443).Build(), + }, + }, + }, + }, + }, + }, + expectedKongServices: []kongstate.Service{ + { + Service: kong.Service{ + Name: kong.String("tcproute.default.tcproute-1.0"), + }, + Backends: []kongstate.ServiceBackend{ + { + Name: "service1", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(80)}, + }, + { + Name: "service2", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(443)}, + }, + }, + }, + }, + expectedKongRoutes: map[string][]kongstate.Route{ + "tcproute.default.tcproute-1.0": { + { + Route: kong.Route{ + Name: kong.String("tcproute.default.tcproute-1.0.0"), + Expression: kong.String(`(net.dst.port == 80) || (net.dst.port == 443)`), + PreserveHost: kong.Bool(true), + Protocols: kong.StringSlice("tcp"), + }, + ExpressionRoutes: true, + }, + }, + }, + }, + { + name: "tcproute with multiple rules", + tcpRoutes: []*gatewayv1alpha2.TCPRoute{ + { + TypeMeta: tcpRouteTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "tcproute-1", + }, + Spec: gatewayv1alpha2.TCPRouteSpec{ + Rules: []gatewayv1alpha2.TCPRouteRule{ + { + BackendRefs: []gatewayv1alpha2.BackendRef{ + builder.NewBackendRef("service1").WithPort(80).Build(), + builder.NewBackendRef("service2").WithPort(443).Build(), + }, + }, + { + BackendRefs: []gatewayv1alpha2.BackendRef{ + builder.NewBackendRef("service3").WithPort(8080).Build(), + builder.NewBackendRef("service4").WithPort(8443).Build(), + }, + }, + }, + }, + }, + }, + expectedKongServices: []kongstate.Service{ + { + Service: kong.Service{ + Name: kong.String("tcproute.default.tcproute-1.0"), + }, + Backends: []kongstate.ServiceBackend{ + { + Name: "service1", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(80)}, + }, + { + Name: "service2", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(443)}, + }, + }, + }, + { + Service: kong.Service{ + Name: kong.String("tcproute.default.tcproute-1.1"), + }, + Backends: []kongstate.ServiceBackend{ + { + Name: "service3", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(8080)}, + }, + { + Name: "service4", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(8443)}, + }, + }, + }, + }, + expectedKongRoutes: map[string][]kongstate.Route{ + "tcproute.default.tcproute-1.0": { + { + Route: kong.Route{ + Name: kong.String("tcproute.default.tcproute-1.0.0"), + Expression: kong.String(`(net.dst.port == 80) || (net.dst.port == 443)`), + PreserveHost: kong.Bool(true), + Protocols: kong.StringSlice("tcp"), + }, + ExpressionRoutes: true, + }, + }, + "tcproute.default.tcproute-1.1": { + { + Route: kong.Route{ + Name: kong.String("tcproute.default.tcproute-1.1.0"), + Expression: kong.String(`(net.dst.port == 8080) || (net.dst.port == 8443)`), + PreserveHost: kong.Bool(true), + Protocols: kong.StringSlice("tcp"), + }, + ExpressionRoutes: true, + }, + }, + }, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + fakestore, err := store.NewFakeStore(store.FakeObjects{TCPRoutes: tc.tcpRoutes}) + require.NoError(t, err) + parser := mustNewParser(t, fakestore) + parser.featureFlags.ExpressionRoutes = true + + failureCollector, err := failures.NewResourceFailuresCollector(logrus.New()) + require.NoError(t, err) + parser.failuresCollector = failureCollector + + result := parser.ingressRulesFromTCPRoutes() + // check services + require.Equal(t, len(tc.expectedKongServices), len(result.ServiceNameToServices), + "should have expected number of services") + for _, expectedKongService := range tc.expectedKongServices { + kongService, ok := result.ServiceNameToServices[*expectedKongService.Name] + require.Truef(t, ok, "should find service %s", expectedKongService.Name) + require.Equal(t, expectedKongService.Backends, kongService.Backends) + // check routes + expectedKongRoutes := tc.expectedKongRoutes[*kongService.Name] + require.Equal(t, len(expectedKongRoutes), len(kongService.Routes)) + + kongRouteNameToRoute := lo.SliceToMap(kongService.Routes, func(r kongstate.Route) (string, kongstate.Route) { + return *r.Name, r + }) + for _, expectedRoute := range expectedKongRoutes { + routeName := expectedRoute.Name + r, ok := kongRouteNameToRoute[*routeName] + require.Truef(t, ok, "should find route %s", *routeName) + require.Equal(t, expectedRoute.Expression, r.Expression) + require.Equal(t, expectedRoute.Protocols, r.Protocols) + } + } + // check translation failures + translationFailures := failureCollector.PopResourceFailures() + require.Equal(t, len(tc.expectedFailures), len(translationFailures)) + for _, expectedTranslationFailure := range tc.expectedFailures { + expectedFailureMessage := expectedTranslationFailure.Message() + require.True(t, lo.ContainsBy(translationFailures, func(failure failures.ResourceFailure) bool { + return strings.Contains(failure.Message(), expectedFailureMessage) + })) + } + }) + } +} From d29468cbd81b6431a162f03c6684feb8db8dc9b8 Mon Sep 17 00:00:00 2001 From: rodman10 <1181591811hzr@gmail.com> Date: Mon, 31 Jul 2023 12:14:23 +0800 Subject: [PATCH 3/9] refactor: rename source file and function. --- internal/dataplane/parser/translate_utils.go | 2 +- .../parser/translators/{tcproute_atc.go => l4route_atc.go} | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) rename internal/dataplane/parser/translators/{tcproute_atc.go => l4route_atc.go} (81%) diff --git a/internal/dataplane/parser/translate_utils.go b/internal/dataplane/parser/translate_utils.go index 3c64bfe8e4..f219d06d52 100644 --- a/internal/dataplane/parser/translate_utils.go +++ b/internal/dataplane/parser/translate_utils.go @@ -185,7 +185,7 @@ func maybePrependRegexPrefix(path, controllerPrefix string, applyLegacyHeuristic func applyExpressionToIngressRules(result *ingressRules) { for _, svc := range result.ServiceNameToServices { for i := range svc.Routes { - translators.ApplyExpressionToKongRoute(&svc.Routes[i]) + translators.ApplyExpressionToL4KongRoute(&svc.Routes[i]) } } } diff --git a/internal/dataplane/parser/translators/tcproute_atc.go b/internal/dataplane/parser/translators/l4route_atc.go similarity index 81% rename from internal/dataplane/parser/translators/tcproute_atc.go rename to internal/dataplane/parser/translators/l4route_atc.go index 45517a0212..fd73ca5f9d 100644 --- a/internal/dataplane/parser/translators/tcproute_atc.go +++ b/internal/dataplane/parser/translators/l4route_atc.go @@ -7,7 +7,9 @@ import ( "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane/parser/atc" ) -func ApplyExpressionToKongRoute(r *kongstate.Route) { +// ApplyExpressionToL4KongRoute convert route flavor from traditional to expressions +// against protocols, snis and dest ports. +func ApplyExpressionToL4KongRoute(r *kongstate.Route) { matchers := []atc.Matcher{} sniMatcher := sniMatcherFromSNIs(lo.Map(r.Route.SNIs, func(item *string, _ int) string { return *item })) From 1f88a1e87f2c514c2c7635ed5b82a284da2a60fa Mon Sep 17 00:00:00 2001 From: rodman10 <1181591811hzr@gmail.com> Date: Sun, 13 Aug 2023 22:45:21 +0800 Subject: [PATCH 4/9] bugfix: clear destinations field. --- internal/dataplane/parser/translate_utils.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/dataplane/parser/translate_utils.go b/internal/dataplane/parser/translate_utils.go index f219d06d52..9dd311debf 100644 --- a/internal/dataplane/parser/translate_utils.go +++ b/internal/dataplane/parser/translate_utils.go @@ -186,6 +186,8 @@ func applyExpressionToIngressRules(result *ingressRules) { for _, svc := range result.ServiceNameToServices { for i := range svc.Routes { translators.ApplyExpressionToL4KongRoute(&svc.Routes[i]) + svc.Routes[i].Destinations = nil + svc.Routes[i].SNIs = nil } } } From a5bd0d3a8671174eba55890adafd0199cb14fa2f Mon Sep 17 00:00:00 2001 From: rodman10 <1181591811hzr@gmail.com> Date: Sun, 13 Aug 2023 22:46:47 +0800 Subject: [PATCH 5/9] tests: enable expression flavor of tcproute. --- test/integration/tcproute_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/integration/tcproute_test.go b/test/integration/tcproute_test.go index e08a2d3624..b0d705bbd4 100644 --- a/test/integration/tcproute_test.go +++ b/test/integration/tcproute_test.go @@ -29,7 +29,6 @@ import ( ) func TestTCPRouteEssentials(t *testing.T) { - skipTestForExpressionRouter(t) ctx := context.Background() t.Log("locking TCP port") @@ -412,7 +411,6 @@ func TestTCPRouteEssentials(t *testing.T) { } func TestTCPRouteReferenceGrant(t *testing.T) { - skipTestForExpressionRouter(t) ctx := context.Background() t.Log("locking TCP port") From 4a5e028099aa93eada22b26fb7b25e4c9deae353 Mon Sep 17 00:00:00 2001 From: rodman10 <1181591811hzr@gmail.com> Date: Wed, 16 Aug 2023 21:20:15 +0800 Subject: [PATCH 6/9] tests: fix tests. --- .../parser/translate_failures_test.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/internal/dataplane/parser/translate_failures_test.go b/internal/dataplane/parser/translate_failures_test.go index 0e1706175c..56e8b6cecb 100644 --- a/internal/dataplane/parser/translate_failures_test.go +++ b/internal/dataplane/parser/translate_failures_test.go @@ -106,18 +106,6 @@ func TestTranslationFailureUnsupportedObjectsExpressionRoutes(t *testing.T) { { name: "TCPRoutes, UDPRoutes and TLSRoutes in gateway APIs are not supported", objects: store.FakeObjects{ - TCPRoutes: []*gatewayv1alpha2.TCPRoute{ - { - TypeMeta: metav1.TypeMeta{ - Kind: "TCPRoute", - APIVersion: gatewayv1alpha2.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "tcproute-1", - Namespace: "default", - }, - }, - }, UDPRoutes: []*gatewayv1alpha2.UDPRoute{ { TypeMeta: metav1.TypeMeta{ @@ -144,12 +132,6 @@ func TestTranslationFailureUnsupportedObjectsExpressionRoutes(t *testing.T) { }, }, causingObjects: []client.Object{ - &gatewayv1alpha2.TCPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tcproute-1", - Namespace: "default", - }, - }, &gatewayv1alpha2.UDPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "udproute-1", From 03beefb3cf33be9dadacbb21b5370c73281442e0 Mon Sep 17 00:00:00 2001 From: rodman10 <1181591811hzr@gmail.com> Date: Sun, 20 Aug 2023 21:46:53 +0800 Subject: [PATCH 7/9] style: change assert style. --- internal/dataplane/parser/translate_tcproute_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/dataplane/parser/translate_tcproute_test.go b/internal/dataplane/parser/translate_tcproute_test.go index 3905d0ed9c..e7cddd1183 100644 --- a/internal/dataplane/parser/translate_tcproute_test.go +++ b/internal/dataplane/parser/translate_tcproute_test.go @@ -226,7 +226,7 @@ func TestIngressRulesFromTCPRoutesUsingExpressionRoutes(t *testing.T) { result := parser.ingressRulesFromTCPRoutes() // check services - require.Equal(t, len(tc.expectedKongServices), len(result.ServiceNameToServices), + require.Len(t, result.ServiceNameToServices, len(tc.expectedKongServices), "should have expected number of services") for _, expectedKongService := range tc.expectedKongServices { kongService, ok := result.ServiceNameToServices[*expectedKongService.Name] @@ -234,7 +234,7 @@ func TestIngressRulesFromTCPRoutesUsingExpressionRoutes(t *testing.T) { require.Equal(t, expectedKongService.Backends, kongService.Backends) // check routes expectedKongRoutes := tc.expectedKongRoutes[*kongService.Name] - require.Equal(t, len(expectedKongRoutes), len(kongService.Routes)) + require.Len(t, kongService.Routes, len(expectedKongRoutes)) kongRouteNameToRoute := lo.SliceToMap(kongService.Routes, func(r kongstate.Route) (string, kongstate.Route) { return *r.Name, r @@ -249,7 +249,7 @@ func TestIngressRulesFromTCPRoutesUsingExpressionRoutes(t *testing.T) { } // check translation failures translationFailures := failureCollector.PopResourceFailures() - require.Equal(t, len(tc.expectedFailures), len(translationFailures)) + require.Len(t, translationFailures, len(tc.expectedFailures)) for _, expectedTranslationFailure := range tc.expectedFailures { expectedFailureMessage := expectedTranslationFailure.Message() require.True(t, lo.ContainsBy(translationFailures, func(failure failures.ResourceFailure) bool { From 741dcd15c8b000e6eb33b4ddbfea627c70e6ed79 Mon Sep 17 00:00:00 2001 From: rodman10 <1181591811hzr@gmail.com> Date: Sun, 20 Aug 2023 21:48:54 +0800 Subject: [PATCH 8/9] style: remove unused const. --- internal/dataplane/parser/atc/field.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/dataplane/parser/atc/field.go b/internal/dataplane/parser/atc/field.go index a3cdb503ea..afe1008cd8 100644 --- a/internal/dataplane/parser/atc/field.go +++ b/internal/dataplane/parser/atc/field.go @@ -47,7 +47,6 @@ const ( FieldHTTPMethod StringField = "http.method" FieldHTTPHost StringField = "http.host" FieldHTTPPath StringField = "http.path" - FieldNetDstIP StringField = "net.dst.ip" ) // IntField is defined for fields with constant name and having integer type. From 3ffaf12d1a264f0b826327cb71b48193fea69a14 Mon Sep 17 00:00:00 2001 From: rodman10 <1181591811hzr@gmail.com> Date: Sun, 20 Aug 2023 21:55:11 +0800 Subject: [PATCH 9/9] docs: update CHANGELOG. --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd30975fa2..f3122175d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,13 @@ Adding a new version? You'll need three changes: ## Unreleased +### Added + +- Added translator to translate `TCPRoute` in gateway APIs to + expression based kong routes. Similar to ingresses, this translator is only + enabled when feature gate `ExpressionRoutes` is turned on and the managed + Kong gateway runs in router flavor `expressions`. + [#4385](https://github.com/Kong/kubernetes-ingress-controller/pull/4385) ### Changes