-
Notifications
You must be signed in to change notification settings - Fork 141
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[ISSUE-48][FEATURE][FOLLOW UP] Add controller component (#214)
### What changes were proposed in this pull request? For issue #48 I add some codes about controller module, and I'll add more unit tests in next PR. ### Why are the changes needed? Support K8S ### Does this PR introduce _any_ user-facing change? Yes, we will add the doc later ### How was this patch tested? Manual test
- Loading branch information
1 parent
e5c3412
commit e6b4260
Showing
20 changed files
with
3,157 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one or more | ||
* contributor license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright ownership. | ||
* The ASF licenses this file to You under the Apache License, Version 2.0 | ||
* (the "License"); you may not use this file except in compliance with | ||
* the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package main | ||
|
||
import ( | ||
"flag" | ||
|
||
"k8s.io/klog/v2" | ||
ctrl "sigs.k8s.io/controller-runtime" | ||
|
||
"github.com/apache/incubator-uniffle/deploy/kubernetes/operator/pkg/controller/config" | ||
"github.com/apache/incubator-uniffle/deploy/kubernetes/operator/pkg/controller/controller" | ||
) | ||
|
||
func main() { | ||
klog.InitFlags(nil) | ||
cfg := &config.Config{} | ||
cfg.AddFlags() | ||
flag.Parse() | ||
|
||
cfg.Complete() | ||
klog.Infof("run config: %+v", cfg) | ||
|
||
// create a manager for leader election. | ||
mgr, err := ctrl.NewManager(cfg.RESTConfig, ctrl.Options{ | ||
LeaderElection: true, | ||
LeaderElectionID: cfg.LeaderElectionID(), | ||
}) | ||
if err != nil { | ||
klog.Fatal(err) | ||
} | ||
// create a rss controller. | ||
rc := controller.NewRSSController(cfg) | ||
if err = mgr.Add(rc); err != nil { | ||
klog.Fatal(err) | ||
} | ||
// start the rss controller. | ||
if err = mgr.Start(cfg.RunCtx); err != nil { | ||
klog.Fatal(err) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
51 changes: 51 additions & 0 deletions
51
deploy/kubernetes/operator/pkg/controller/config/config.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one or more | ||
* contributor license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright ownership. | ||
* The ASF licenses this file to You under the Apache License, Version 2.0 | ||
* (the "License"); you may not use this file except in compliance with | ||
* the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package config | ||
|
||
import ( | ||
"flag" | ||
|
||
"github.com/apache/incubator-uniffle/deploy/kubernetes/operator/pkg/constants" | ||
"github.com/apache/incubator-uniffle/deploy/kubernetes/operator/pkg/utils" | ||
) | ||
|
||
const ( | ||
flagWorkers = "workers" | ||
) | ||
|
||
// Config contains all configurations. | ||
type Config struct { | ||
Workers int | ||
utils.GenericConfig | ||
} | ||
|
||
// LeaderElectionID returns leader election ID. | ||
func (c *Config) LeaderElectionID() string { | ||
return "rss-controller-" + constants.LeaderIDSuffix | ||
} | ||
|
||
// AddFlags adds all configurations to the global flags. | ||
func (c *Config) AddFlags() { | ||
flag.IntVar(&c.Workers, flagWorkers, 1, "Concurrency of the rss controller.") | ||
c.GenericConfig.AddFlags() | ||
} | ||
|
||
// Complete is called before rss-controller runs. | ||
func (c *Config) Complete() { | ||
c.GenericConfig.Complete() | ||
} |
83 changes: 83 additions & 0 deletions
83
deploy/kubernetes/operator/pkg/controller/constants/constants.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,83 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one or more | ||
* contributor license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright ownership. | ||
* The ASF licenses this file to You under the Apache License, Version 2.0 | ||
* (the "License"); you may not use this file except in compliance with | ||
* the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package constants | ||
|
||
const ( | ||
// ContainerShuffleServerRPCPort indicates rpc port used in shuffle server containers. | ||
ContainerShuffleServerRPCPort int32 = 19997 | ||
// ContainerShuffleServerHTTPPort indicates http port used in shuffle server containers. | ||
ContainerShuffleServerHTTPPort int32 = 19996 | ||
// ContainerCoordinatorRPCPort indicates rpc port used in coordinator containers. | ||
ContainerCoordinatorRPCPort int32 = 19997 | ||
// ContainerCoordinatorHTTPPort indicates http port used in coordinator containers. | ||
ContainerCoordinatorHTTPPort int32 = 19996 | ||
|
||
// ShuffleServerRPCPortEnv indicates environment name of rpc port used by shuffle servers. | ||
ShuffleServerRPCPortEnv = "SERVER_RPC_PORT" | ||
// ShuffleServerHTTPPortEnv indicates environment name of http port used by shuffle servers. | ||
ShuffleServerHTTPPortEnv = "SERVER_HTTP_PORT" | ||
// CoordinatorRPCPortEnv indicates environment name of rpc port used by coordinators. | ||
CoordinatorRPCPortEnv = "COORDINATOR_RPC_PORT" | ||
// CoordinatorHTTPPortEnv indicates environment name of http port used by coordinators. | ||
CoordinatorHTTPPortEnv = "COORDINATOR_HTTP_PORT" | ||
// RSSCoordinatorQuorumEnv indicates environment name of rss coordinator quorum used by shuffle servers. | ||
RSSCoordinatorQuorumEnv = "RSS_COORDINATOR_QUORUM" | ||
// XmxSizeEnv indicates environment name of xmx size used by coordinators or shuffle servers. | ||
XmxSizeEnv = "XMX_SIZE" | ||
// ServiceNameEnv indicates environment name of service name used by coordinators or shuffle servers. | ||
ServiceNameEnv = "SERVICE_NAME" | ||
// NodeNameEnv indicates environment name of physical node name used by coordinators or shuffle servers. | ||
NodeNameEnv = "NODE_NAME" | ||
// RssIPEnv indicates environment name of shuffle servers' ip addresses. | ||
RssIPEnv = "RSS_IP" | ||
|
||
// CoordinatorServiceName defines environment variable value of "SERVICE_NAME" used by coordinators. | ||
CoordinatorServiceName = "coordinator" | ||
// ShuffleServerServiceName defines environment variable value of "SERVICE_NAME" used by shuffle servers. | ||
ShuffleServerServiceName = "server" | ||
|
||
// ExcludeNodesFile indicates volume mounting name of exclude nodes file | ||
ExcludeNodesFile = "exclude-nodes-file" | ||
|
||
// UpdateStatusError means reason of updating status of rss error | ||
UpdateStatusError = "UpdateStatusError" | ||
|
||
// OwnerLabel is the label of configMap's owner. | ||
OwnerLabel = "uniffle.apache.org/owner-label" | ||
|
||
// ConfigurationVolumeName is the name of configMap volume records configuration of coordinators or shuffle servers. | ||
ConfigurationVolumeName = "configuration" | ||
) | ||
|
||
// PropertyKey defines property key in configuration of coordinators or shuffle servers. | ||
type PropertyKey string | ||
|
||
const ( | ||
// RPCServerPort represent rss port property in configuration of coordinators or shuffle servers. | ||
RPCServerPort PropertyKey = "rss.rpc.server.port" | ||
// JettyHTTPPort represent http port property in configuration of coordinators or shuffle servers. | ||
JettyHTTPPort PropertyKey = "rss.jetty.http.port" | ||
|
||
// CoordinatorQuorum represent coordinator quorum property in configuration of shuffle servers. | ||
CoordinatorQuorum PropertyKey = "rss.coordinator.quorum" | ||
// StorageBasePath represent storage base path property in configuration of shuffle servers. | ||
StorageBasePath PropertyKey = "rss.storage.basePath" | ||
|
||
// CoordinatorExcludeNodesPath represent exclude nodes path property in configuration of coordinators. | ||
CoordinatorExcludeNodesPath PropertyKey = "rss.coordinator.exclude.nodes.file.path" | ||
) |
99 changes: 99 additions & 0 deletions
99
deploy/kubernetes/operator/pkg/controller/controller/process_rss_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,99 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one or more | ||
* contributor license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright ownership. | ||
* The ASF licenses this file to You under the Apache License, Version 2.0 | ||
* (the "License"); you may not use this file except in compliance with | ||
* the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package controller | ||
|
||
import ( | ||
"context" | ||
"reflect" | ||
"testing" | ||
|
||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
kubefake "k8s.io/client-go/kubernetes/fake" | ||
|
||
unifflev1alpha1 "github.com/apache/incubator-uniffle/deploy/kubernetes/operator/api/uniffle/v1alpha1" | ||
"github.com/apache/incubator-uniffle/deploy/kubernetes/operator/pkg/controller/config" | ||
"github.com/apache/incubator-uniffle/deploy/kubernetes/operator/pkg/generated/clientset/versioned/fake" | ||
"github.com/apache/incubator-uniffle/deploy/kubernetes/operator/pkg/utils" | ||
) | ||
|
||
func buildEmptyPhaseRssObj() *unifflev1alpha1.RemoteShuffleService { | ||
return &unifflev1alpha1.RemoteShuffleService{ | ||
ObjectMeta: metav1.ObjectMeta{ | ||
Name: testRssName, | ||
Namespace: testNamespace, | ||
ResourceVersion: "test", | ||
}, | ||
Spec: unifflev1alpha1.RemoteShuffleServiceSpec{ | ||
Coordinator: &unifflev1alpha1.CoordinatorConfig{ | ||
ExcludeNodesFilePath: "/exclude_nodes", | ||
}, | ||
}, | ||
Status: unifflev1alpha1.RemoteShuffleServiceStatus{}, | ||
} | ||
} | ||
|
||
func TestProcessEmptyPhaseRss(t *testing.T) { | ||
rss := buildEmptyPhaseRssObj() | ||
|
||
rssClient := fake.NewSimpleClientset(rss) | ||
kubeClient := kubefake.NewSimpleClientset() | ||
|
||
rc := newRSSController(&config.Config{ | ||
GenericConfig: utils.GenericConfig{ | ||
KubeClient: kubeClient, | ||
RSSClient: rssClient, | ||
}, | ||
}) | ||
|
||
for _, tt := range []struct { | ||
name string | ||
expectedRssStatus unifflev1alpha1.RemoteShuffleServiceStatus | ||
expectedNeedRetry bool | ||
expectedError error | ||
}{ | ||
{ | ||
name: "process rss object which has just been created, and whose status phase is empty", | ||
expectedRssStatus: unifflev1alpha1.RemoteShuffleServiceStatus{ | ||
Phase: unifflev1alpha1.RSSPending, | ||
}, | ||
expectedNeedRetry: false, | ||
}, | ||
} { | ||
needRetry, err := rc.processNormal(rss) | ||
if err != nil { | ||
t.Errorf("process rss object failed: %v", err) | ||
return | ||
} | ||
if needRetry != tt.expectedNeedRetry { | ||
t.Errorf("unexpected result indicates whether to retrys: %v, expected: %v", | ||
needRetry, tt.expectedNeedRetry) | ||
return | ||
} | ||
updatedRss, getErr := rssClient.UniffleV1alpha1().RemoteShuffleServices(rss.Namespace). | ||
Get(context.TODO(), rss.Name, metav1.GetOptions{}) | ||
if getErr != nil { | ||
t.Errorf("get updated rss object failed: %v", err) | ||
return | ||
} | ||
if !reflect.DeepEqual(updatedRss.Status, tt.expectedRssStatus) { | ||
t.Errorf("unexpected status of updated rss object: %+v, expected: %+v", | ||
updatedRss.Status, tt.expectedRssStatus) | ||
return | ||
} | ||
} | ||
} |
Oops, something went wrong.