feat(caicloud#346): use cache volume to speed up pipeline

api/conversion/conversion.go

@@ -70,10 +70,26 @@ func ConvertPipelineToService(projectName string, pipeline *newapi.Pipeline) (*a
 		service.Dockerfile = pipeline.Build.Stages.ImageBuild.BuildInfos[0].Dockerfile
 		service.ImageName = pipeline.Build.Stages.ImageBuild.BuildInfos[0].ImageName
 	}
+	service.BuildInfo = convertBuildInfo(pipeline.Build.BuildInfo)
 
 	return service, nil
 }
 
+// convertBuildInfo converts the build info from new API to old API.
+func convertBuildInfo(buildInfo *newapi.BuildInfo) *api.BuildInfo {
+	if buildInfo == nil {
+		return nil
+	}
+
+	return &api.BuildInfo{
+		&api.BuildTool{
+			buildInfo.BuildTool.Name,
+			buildInfo.BuildTool.Version,
+		},
+		buildInfo.UseDependencyCache,
+	}
+}
+
 // convertBuildStagesToCaicloudYaml converts the config of build stages in pipeline to caicloud.yml.
 func convertBuildStagesToCaicloudYaml(pipeline *newapi.Pipeline) (string, error) {
 	if pipeline.Build == nil || pipeline.Build.BuilderImage == nil || pipeline.Build.Stages == nil {
@@ -114,11 +130,11 @@ func convertBuildStagesToCaicloudYaml(pipeline *newapi.Pipeline) (string, error)
 
 		// Now only support one build info.
 		buildInfo := imageBuildConfig.BuildInfos[0]
-		if (len(buildInfo.ContextDir) != 0) {
+		if len(buildInfo.ContextDir) != 0 {
 			build.ContextDir = buildInfo.ContextDir
 		}
 
-		if (len(buildInfo.DockerfilePath) != 0) {
+		if len(buildInfo.DockerfilePath) != 0 {
 			build.DockerfileName = buildInfo.DockerfilePath
 		}
 

api/types.go

@@ -87,6 +87,20 @@ type Service struct {
 	CaicloudYaml string `bson:"caicloudYaml,omitempty" json:"caicloudYaml,omitempty"`
 
 	ImageName string `bson:"image_name,omitempty" json:"image_name,omitempty"`
+
+	BuildInfo *BuildInfo `bson:"build_info,omitempty" json:"build_info,omitempty"`
+}
+
+// BuildInfo represents the basic build information of the pipeline.
+type BuildInfo struct {
+	BuildTool          *BuildTool `bson:"buildTool,omitempty" json:"buildTool,omitempty" description:"tool to build package"`
+	UseDependencyCache bool       `bson:"useDependencyCache,omitempty" json:"useDependencyCache,omitempty" description:"whether use dependency cache to speedup"`
+}
+
+// BuildTool represents the build tool for CI.
+type BuildTool struct {
+	Name    string `bson:"name,omitempty" json:"name,omitempty" description:"name of build tool"`
+	Version string `bson:"version,omitempty" json:"version,omitempty" description:"version of build tool"`
 }
 
 // Hook is a callback hook

cloud/k8s.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/zoumo/logdog"
@@ -235,6 +236,33 @@ func (cloud *K8SCloud) Provision(id string, wopts WorkerOptions) (Worker, error)
 		},
 	}
 
+	// Mount the cache volume to the worker.
+	cacheVolume := wopts.CacheVolume
+	if len(cacheVolume) != 0 {
+		mountPath := "/tmp"
+		volumeName := "cache-dependency"
+		if strings.Contains(cacheVolume, "maven") {
+			mountPath = "/root/.m2"
+		}
+
+		pod.Spec.Containers[0].VolumeMounts = []apiv1.VolumeMount{
+			apiv1.VolumeMount{
+				Name:      volumeName,
+				MountPath: mountPath,
+			},
+		}
+
+		pod.Spec.Volumes = []apiv1.Volume{
+			apiv1.Volume{
+				Name: volumeName,
+			},
+		}
+
+		pod.Spec.Volumes[0].PersistentVolumeClaim = &apiv1.PersistentVolumeClaimVolumeSource{
+			ClaimName: cacheVolume,
+		}
+	}
+
 	// pod, err = cloud.Client.CoreV1().Pods(cloud.namespace).Create(pod)
 	// if err != nil {
 	// 	return nil, err

cloud/worker.go

@@ -221,6 +221,9 @@ type WorkerOptions struct {
 
 	// Namespace represents the k8s namespace where to create worker, only works for k8s cloud provider.
 	Namespace string
+
+	// CacheVolume represents the volume to cache dependency for worker.
+	CacheVolume string
 }
 
 // NewWorkerOptions creates a new WorkerOptions with default value

event/event.go

@@ -147,6 +147,7 @@ func createServiceHandler(event *api.Event) error {
 	logdog.Infof("create service handler")
 	opts := workerOptions.DeepCopy()
 	opts.Namespace = getNamespaceFromEvent(event)
+	opts.CacheVolume = getCacheVolumeFromEvent(event)
 	worker, err := CloudController.Provision(string(event.EventID), opts)
 	if err != nil {
 		return err
@@ -240,6 +241,7 @@ func createVersionHandler(event *api.Event) error {
 	opts := workerOptions.DeepCopy()
 	opts.Quota = Resource2Quota(event.Version.BuildResource, opts.Quota)
 	opts.Namespace = getNamespaceFromEvent(event)
+	opts.CacheVolume = getCacheVolumeFromEvent(event)
 	worker, err := CloudController.Provision(string(event.EventID), opts)
 	if err != nil {
 		return err
@@ -421,3 +423,19 @@ func getNamespaceFromEvent(event *api.Event) string {
 
 	return ""
 }
+
+// getCacheVolumeFromEvent gets the cache volume from event data for worker. Will return empty string if can not get it.
+func getCacheVolumeFromEvent(event *api.Event) string {
+	if event == nil {
+		log.Error("Can not get cache volume from data as event is nil")
+		return ""
+	}
+
+	if v, e := event.Data["cachevolume"]; e {
+		if sv, ok := v.(string); ok {
+			return sv
+		}
+	}
+
+	return ""
+}

event/operations.go

@@ -49,9 +49,20 @@ func SendCreateServiceEvent(service *api.Service) error {
 		Data:      map[string]interface{}{"Token": project.SCM.Token},
 	}
 
-	// Set the namespace for worker in k8s cloud.
-	if project.Worker != nil && len(project.Worker.Namespace) != 0 {
-		event.Data["namespace"] = project.Worker.Namespace
+	// Set the config for worker in k8s cloud.
+	worker := project.Worker
+	if worker != nil {
+		if len(worker.Namespace) != 0 {
+			event.Data["namespace"] = worker.Namespace
+		}
+
+		// Use cache dependency.
+		buildInfo := service.BuildInfo
+		if buildInfo != nil && buildInfo.UseDependencyCache && buildInfo.BuildTool != nil {
+			if cache, ok := worker.DependencyCaches[buildInfo.BuildTool.Name]; ok {
+				event.Data["cachevolume"] = cache
+			}
+		}
 	}
 
 	log.Infof("send create service event: %v", event)
@@ -104,9 +115,20 @@ func SendCreateVersionEvent(service *api.Service, version *api.Version) error {
 		Status: api.EventStatusPending,
 	}
 
-	// Set the namespace for worker in k8s cloud.
-	if project.Worker != nil && len(project.Worker.Namespace) != 0 {
-		event.Data["namespace"] = project.Worker.Namespace
+	// Set the config for worker in k8s cloud.
+	worker := project.Worker
+	if worker != nil {
+		if len(worker.Namespace) != 0 {
+			event.Data["namespace"] = worker.Namespace
+		}
+
+		// Use cache dependency.
+		buildInfo := service.BuildInfo
+		if buildInfo != nil && buildInfo.UseDependencyCache && buildInfo.BuildTool != nil {
+			if cache, ok := worker.DependencyCaches[buildInfo.BuildTool.Name]; ok {
+				event.Data["cachevolume"] = cache
+			}
+		}
 	}
 
 	log.Infof("send create version event: %v", event)

pkg/api/types.go

@@ -36,7 +36,13 @@ type Project struct {
 
 // Worker represents the config of worker for the pipelines of the project.
 type Worker struct {
-	Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty" description:"k8s namespace to create the worker"`
+	Namespace        string                     `bson:"namespace,omitempty" json:"namespace,omitempty" description:"k8s namespace to create the worker"`
+	DependencyCaches map[string]DependencyCache `bson:"dependencyCaches,omitempty" json:"dependencyCaches,omitempty" description:"dependency caches for worker to speed up"`
+}
+
+// DependencyCache represents the cache volume of dependency for CI.
+type DependencyCache struct {
+	Name string `bson:"name,omitempty" json:"name,omitempty" description:"name of the dependency cache"`
 }
 
 // Pipeline represents a set of configs to describe the workflow of CI/CD.
@@ -60,10 +66,23 @@ type Pipeline struct {
 
 // Build represents the build config and stages of CI.
 type Build struct {
+	BuildInfo    *BuildInfo    `bson:"buildInfo,omitempty" json:"buildInfo,omitempty" description:"information to build package"`
 	BuilderImage *BuilderImage `bson:"builderImage,omitempty" json:"builderImage,omitempty" description:"image information of the builder"`
 	Stages       *BuildStages  `bson:"stages,omitempty" json:"stages,omitempty" description:"stages of CI"`
 }
 
+// BuildInfo represents the basic build information of the pipeline.
+type BuildInfo struct {
+	BuildTool          *BuildTool `bson:"buildTool,omitempty" json:"buildTool,omitempty" description:"tool to build package"`
+	UseDependencyCache bool       `bson:"useDependencyCache,omitempty" json:"useDependencyCache,omitempty" description:"whether use dependency cache to speedup"`
+}
+
+// BuildTool represents the build tool for CI.
+type BuildTool struct {
+	Name    string `bson:"name,omitempty" json:"name,omitempty" description:"name of build tool"`
+	Version string `bson:"version,omitempty" json:"version,omitempty" description:"version of build tool"`
+}
+
 // EnvVar represents the environment variables with name and value.
 type EnvVar struct {
 	Name  string `bson:"name,omitempty" json:"name,omitempty" description:"name of the environment variable"`