Import & Export from nuctl (#1657)

docs/tasks/exporting-and-importing.md

@@ -0,0 +1,88 @@
+# Exporting and Importing in Nuclio
+
+This tutorial guides you through the process of exporting and importing functions and projects.
+
+### In this document
+- [Exporting a deployed function](#exporting-a-deployed-function)
+- [Importing a function](#importing-a-function)
+- [Redeploying an imported function](#redeploying-an-imported-function)
+- [Exporting a project](#exporting-a-project)
+- [Importing a project](#importing-a-project)
+
+## Exporting a deployed function
+
+Once you have some functions deployed in nuclio (see [Deploying functions](/docs/tasks/deploying-functions.md)), you'll be able to export them for later import in this nuclio system or any other nuclio system. After which, it's recommended that you save the output to a file with a redirection (e.g. `command --with output > file.yaml` ).
+
+In order to export the function, run the command (remember to replace `function-name` with the relevant function name):
+```sh
+nuctl export function --namespace nuclio function-name
+```
+The command will print the exported function config to the stdout.
+
+> **Note:** This command by default will export the function in yaml format. However, you can supply the flag `--output json` if you prefer a json output.
+
+## Importing a function
+
+Once you have [exported a function](#exporting-a-deployed-function), you can use the exported function config you saved in a file to import said function.
+
+Either run this command with the file path:
+```sh
+nuctl import function --namespace nuclio path-to-exported-function-file
+```
+
+Or pipe the function config to the command:
+```sh
+cat path-to-exported-function-file | nuctl import function --namespace nuclio
+```
+
+It is also possible to POST the exported file to the dashboard API. For example with the `http` command-line tool:
+```sh
+cat path-to-exported-function-file | http post 'http://<nuclio-system-url>/api/functions/?import=true'
+```
+
+## Redeploying an imported function
+
+Once you import a function, you'll notice it's state is `imported` and it is not deployed.
+To deploy an exported function, use the deploy command and supply the command with exported function config file:
+```sh
+nuctl deploy --namespace nuclio --file path-to-exported-function-file
+```
+However, if you already imported the function, you can deploy the imported function with this command:
+```sh
+nuctl deploy --namespace nuclio imported-function-name
+```
+
+## Exporting a project
+
+Exporting a project is done similarly to [exporting a function](#exporting-a-deployed-function):
+```sh
+nuctl export function --namespace nuclio project-name
+```
+
+The output of this command will contain the config for the project, the config for each of the project's functions and the configs for all their function events.  Again, similarly to exporting functions, it's recommended that you save the output to a file with a redirection (e.g. `command --with output > file.yaml` ).
+> **Note:** Again similarly to [exporting a function](#exporting-a-deployed-function), this command by default will export the function in yaml format. However, you can supply the flag `--output json` if you prefer a json output.
+
+## Importing a project
+
+Importing a project is done similarly to [importing a function](#importing-a-function):
+```sh
+nuctl import project --namespace nuclio path-to-exported-project-file
+
+# or
+
+cat path-to-exported-project-file | nuctl import project --namespace nuclio
+```
+
+Similarly to importing a function, it is also possible to POST the exported file to the dashboard API. For example with the `http` command-line tool:
+```sh
+cat path-to-exported-project-file | http post 'http://<nuclio-system-url>/api/projects/?import=true'
+```
+
+> **Important Note**: As mentioned, importing a project is a high-level flow with many sub-flows:
+> - Importing the project config
+> - Importing all the project's functions
+> - Importing all the project functions' function events
+>
+> In order for this flow to run smoothly, if one of the resources fails to import, an error will be printed to the stderr, but the command will still continue to run and try to import as many of the resources as possible.
+>
+> For example: If the namespace contains a function named `my-func`, but a function named `my-func` already exists in the system (function names are unique namespace-wide, not only project-wide), this function will not be imported(and neither will its function events), but the project as a whole will still be imported as well as any other function and function event present in the config.

pkg/dashboard/resource/function.go

@@ -22,7 +22,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"runtime/debug"
-	"strconv"
 	"strings"
 	"time"
 
@@ -214,46 +213,21 @@ func (fr *functionResource) GetCustomRoutes() ([]restful.CustomRoute, error) {
 }
 
 func (fr *functionResource) export(function platform.Function) restful.Attributes {
-	functionSpec := fr.cleanFunctionSpec(function.GetConfig().Spec)
-	functionMeta := function.GetConfig().Meta
+	functionConfig := function.GetConfig()
 
-	fr.Logger.DebugWith("Exporting function", "functionName", functionMeta.Name)
+	fr.Logger.DebugWith("Preparing function for export", "functionName", functionConfig.Meta.Name)
+	functionConfig.PrepareFunctionForExport(false)
 
-	fr.prepareFunctionForExport(&functionMeta, &functionSpec)
+	fr.Logger.DebugWith("Exporting function", "functionName", functionConfig.Meta.Name)
 
 	attributes := restful.Attributes{
-		"metadata": functionMeta,
-		"spec":     functionSpec,
+		"metadata": functionConfig.Meta,
+		"spec":     functionConfig.Spec,
 	}
 
 	return attributes
 }
 
-func (fr *functionResource) prepareFunctionForExport(functionMeta *functionconfig.Meta, functionSpec *functionconfig.Spec) {
-
-	fr.Logger.DebugWith("Preparing function for export", "functionName", functionMeta.Name)
-
-	if functionMeta.Annotations == nil {
-		functionMeta.Annotations = map[string]string{}
-	}
-
-	// add annotations for not deploying or building on import
-	functionMeta.Annotations[functionconfig.FunctionAnnotationSkipBuild] = strconv.FormatBool(true)
-	functionMeta.Annotations[functionconfig.FunctionAnnotationSkipDeploy] = strconv.FormatBool(true)
-
-	// scrub namespace from function meta
-	functionMeta.Namespace = ""
-
-	// remove secrets and passwords from triggers
-	newTriggers := functionSpec.Triggers
-	for triggerName, trigger := range newTriggers {
-		trigger.Password = ""
-		trigger.Secret = ""
-		newTriggers[triggerName] = trigger
-	}
-	functionSpec.Triggers = newTriggers
-}
-
 func (fr *functionResource) storeAndDeployFunction(functionInfo *functionInfo, authConfig *platform.AuthConfig, waitForFunction bool) error {
 
 	creationStateUpdatedTimeout := 45 * time.Second
@@ -382,24 +356,13 @@ func (fr *functionResource) deleteFunction(request *http.Request) (*restful.Cust
 	}, err
 }
 
-func (fr *functionResource) cleanFunctionSpec(functionSpec functionconfig.Spec) functionconfig.Spec {
-
-	// artifacts are created unique to the cluster not needed to be returned to any client of nuclio REST API
-	functionSpec.RunRegistry = ""
-	functionSpec.Build.Registry = ""
-	if functionSpec.Build.FunctionSourceCode != "" {
-		functionSpec.Image = ""
-	}
-
-	return functionSpec
-}
-
 func (fr *functionResource) functionToAttributes(function platform.Function) restful.Attributes {
-	functionSpec := fr.cleanFunctionSpec(function.GetConfig().Spec)
+	functionConfig := function.GetConfig()
+	functionConfig.CleanFunctionSpec()
 
 	attributes := restful.Attributes{
-		"metadata": function.GetConfig().Meta,
-		"spec":     functionSpec,
+		"metadata": functionConfig.Meta,
+		"spec":     functionConfig.Spec,
 	}
 
 	status := function.GetStatus()

pkg/dashboard/resource/functionevent.go

@@ -173,11 +173,11 @@ func (fer *functionEventResource) storeAndDeployFunctionEvent(functionEvent *fun
 	return newFunctionEvent, nil
 }
 
-func (fer *functionEventResource) getFunctionEvents(function platform.Function) []platform.FunctionEvent {
+func (fer *functionEventResource) getFunctionEvents(function platform.Function, namespace string) []platform.FunctionEvent {
 	getFunctionEventOptions := platform.GetFunctionEventsOptions{
 		Meta: platform.FunctionEventMeta{
 			Name:      "",
-			Namespace: function.GetConfig().Meta.Namespace,
+			Namespace: namespace,
 			Labels: map[string]string{
 				"nuclio.io/function-name": function.GetConfig().Meta.Name,
 			},

pkg/dashboard/resource/project.go

@@ -23,9 +23,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
-	"time"
 
-	"github.com/nuclio/nuclio/pkg/common"
 	"github.com/nuclio/nuclio/pkg/dashboard"
 	"github.com/nuclio/nuclio/pkg/platform"
 	"github.com/nuclio/nuclio/pkg/restful"
@@ -50,11 +48,6 @@ type projectImportInfo struct {
 	FunctionEvents map[string]*functionEventInfo
 }
 
-const (
-	ProjectGetUponCreationTimeout       = 30 * time.Second
-	ProjectGetUponCreationRetryInterval = 1 * time.Second
-)
-
 // GetAll returns all projects
 func (pr *projectResource) GetAll(request *http.Request) (map[string]restful.Attributes, error) {
 	response := map[string]restful.Attributes{}
@@ -201,11 +194,13 @@ func (pr *projectResource) export(project platform.Project) restful.Attributes {
 		return attributes
 	}
 
+	namespace := project.GetConfig().Meta.Namespace
+
 	// create a map of attributes keyed by the function id (name)
 	for _, function := range functions {
 		functionsMap[function.GetConfig().Meta.Name] = functionResourceInstance.export(function)
 
-		functionEvents := functionEventResourceInstance.getFunctionEvents(function)
+		functionEvents := functionEventResourceInstance.getFunctionEvents(function, namespace)
 		for _, functionEvent := range functionEvents {
 			functionEventsMap[functionEvent.GetConfig().Meta.Name] =
 				functionEventResourceInstance.functionEventToAttributes(functionEvent)
@@ -263,10 +258,22 @@ func (pr *projectResource) importProject(projectImportInfoInstance *projectImpor
 	})
 	if err != nil || len(projects) == 0 {
 		pr.Logger.InfoWith("Project doesn't exist, creating it", "project", projectImportInfoInstance.Project.Meta.Name)
-		err = pr.createAndWaitForProjectCreation(projectImportInfoInstance.Project)
+
+		// create a project config
+		projectConfig := platform.ProjectConfig{
+			Meta: *projectImportInfoInstance.Project.Meta,
+			Spec: *projectImportInfoInstance.Project.Spec,
+		}
+
+		// create a project
+		newProject, err := platform.NewAbstractProject(pr.Logger, pr.getPlatform(), projectConfig)
 		if err != nil {
 			return "", nil, nuclio.WrapErrInternalServerError(err)
 		}
+
+		if err = newProject.CreateAndWait(); err != nil {
+			return "", nil, nuclio.WrapErrInternalServerError(err)
+		}
 	}
 
 	failedFunctions := pr.importProjectFunctions(projectImportInfoInstance, authConfig)
@@ -400,47 +407,6 @@ func (pr *projectResource) importProjectFunctionEvents(projectImportInfoInstance
 	return failedFunctionEvents
 }
 
-func (pr *projectResource) createAndWaitForProjectCreation(projectInfoInstance *projectInfo) error {
-
-	// create a project config
-	projectConfig := platform.ProjectConfig{
-		Meta: *projectInfoInstance.Meta,
-		Spec: *projectInfoInstance.Spec,
-	}
-
-	// create a project
-	newProject, err := platform.NewAbstractProject(pr.Logger, pr.getPlatform(), projectConfig)
-	if err != nil {
-		return nuclio.WrapErrInternalServerError(err)
-	}
-
-	// just deploy. the status is async through polling
-	err = pr.getPlatform().CreateProject(&platform.CreateProjectOptions{
-		ProjectConfig: *newProject.GetConfig(),
-	})
-	if err != nil {
-		return nuclio.WrapErrInternalServerError(err)
-	}
-
-	err = common.RetryUntilSuccessful(ProjectGetUponCreationTimeout, ProjectGetUponCreationRetryInterval, func() bool {
-		pr.Logger.DebugWith("Trying to get created project",
-			"projectName", projectConfig.Meta.Name,
-			"timeout", ProjectGetUponCreationTimeout,
-			"retryInterval", ProjectGetUponCreationRetryInterval)
-		projects, err := pr.getPlatform().GetProjects(&platform.GetProjectsOptions{
-			Meta: *projectInfoInstance.Meta,
-		})
-		return err == nil && len(projects) > 0
-	})
-	if err != nil {
-		return nuclio.WrapErrInternalServerError(errors.Wrapf(err,
-			"Failed to wait for a created project %s",
-			projectConfig.Meta.Name))
-	}
-
-	return nil
-}
-
 func (pr *projectResource) deleteProject(request *http.Request) (*restful.CustomRouteFuncResponse, error) {
 
 	// get project config and status from body

pkg/functionconfig/types.go

@@ -334,6 +334,14 @@ func (m *Meta) GetUniqueID() string {
 	return m.Namespace + ":" + m.Name
 }
 
+func (m *Meta) AddSkipDeployAnnotation() {
+	m.Annotations[FunctionAnnotationSkipDeploy] = strconv.FormatBool(true)
+}
+
+func (m *Meta) AddSkipBuildAnnotation() {
+	m.Annotations[FunctionAnnotationSkipBuild] = strconv.FormatBool(true)
+}
+
 func (m *Meta) RemoveSkipDeployAnnotation() {
 	delete(m.Annotations, FunctionAnnotationSkipDeploy)
 }
@@ -373,6 +381,50 @@ func NewConfig() *Config {
 	}
 }
 
+func (c *Config) CleanFunctionSpec() {
+
+	// artifacts are created unique to the cluster not needed to be returned to any client of nuclio REST API
+	c.Spec.RunRegistry = ""
+	c.Spec.Build.Registry = ""
+	if c.Spec.Build.FunctionSourceCode != "" {
+		c.Spec.Image = ""
+	}
+}
+
+func (c *Config) PrepareFunctionForExport(noScrub bool) {
+	if !noScrub {
+		c.scrubFunctionData()
+	}
+	c.AddSkipAnnotations()
+}
+
+func (c *Config) AddSkipAnnotations() {
+
+	if c.Meta.Annotations == nil {
+		c.Meta.Annotations = map[string]string{}
+	}
+
+	// add annotations for not deploying or building on import
+	c.Meta.AddSkipBuildAnnotation()
+	c.Meta.AddSkipDeployAnnotation()
+}
+
+func (c *Config) scrubFunctionData() {
+	c.CleanFunctionSpec()
+
+	// scrub namespace from function meta
+	c.Meta.Namespace = ""
+
+	// remove secrets and passwords from triggers
+	newTriggers := c.Spec.Triggers
+	for triggerName, trigger := range newTriggers {
+		trigger.Password = ""
+		trigger.Secret = ""
+		newTriggers[triggerName] = trigger
+	}
+	c.Spec.Triggers = newTriggers
+}
+
 // FunctionState is state of function
 type FunctionState string