Merge pull request #387 from digitalrebar/v451-pullback

V451 pullback

README.rst

@@ -124,7 +124,6 @@ Table of Contents
    doc/content-packages
    doc/rackn/license
    Trademark
-   LICENSE
    doc/fun-facts
 
 .. _rs_license:
@@ -150,3 +149,5 @@ Digital Rebar Provision documentation is available from multiple authors under t
 
     In the case of works protected by multiple Creative Common licenses,
     the user may choose either.
+
+.. Release v4.6.0 Start

cli/catalog.go

@@ -57,47 +57,121 @@ func getLocalCatalog() (res *models.Content, err error) {
 	return
 }
 
-func catalogCommands() *cobra.Command {
-
-	type catItem struct {
-		Type     string
-		Versions []string
+func fetchCatalog() (res *models.Content, err error) {
+	buf := []byte{}
+	buf, err = bufOrFile(catalog)
+	if err == nil {
+		err = json.Unmarshal(buf, &res)
+	}
+	if err != nil {
+		err = fmt.Errorf("Error fetching catalog: %v", err)
 	}
+	return
+}
 
-	fetchCatalog := func() (res *models.Content, err error) {
-		buf := []byte{}
-		buf, err = bufOrFile(catalog)
-		if err == nil {
-			err = json.Unmarshal(buf, &res)
+func itemsFromCatalog(cat *models.Content, name string) map[string]*models.CatalogItem {
+	res := map[string]*models.CatalogItem{}
+	for k, v := range cat.Sections["catalog_items"] {
+		item := &models.CatalogItem{}
+		if err := models.Remarshal(v, &item); err != nil {
+			continue
 		}
-		if err != nil {
-			err = fmt.Errorf("Error fetching catalog: %v", err)
+		if name == "" || name == item.Name {
+			res[k] = item
+		}
+	}
+	return res
+}
+
+func oneItem(cat *models.Content, name, version string) *models.CatalogItem {
+	items := itemsFromCatalog(cat, name)
+	for _, v := range items {
+		if v.Version == version {
+			return v
 		}
-		return
 	}
+	return nil
+}
 
-	itemsFromCatalog := func(cat *models.Content, name string) map[string]*models.CatalogItem {
-		res := map[string]*models.CatalogItem{}
-		for k, v := range cat.Sections["catalog_items"] {
-			item := &models.CatalogItem{}
-			if err := models.Remarshal(v, &item); err != nil {
+func installItem(catalog *models.Content, name, version, arch, tgtos string, replaceWritable bool, inflight map[string]struct{}) error {
+	inflight[name] = struct{}{}
+	if name == "BasicStore" {
+		return nil
+	}
+	item := oneItem(catalog, name, version)
+	if item == nil {
+		return fmt.Errorf("%s version %s not in catalog", name, version)
+	}
+	src, err := urlOrFileAsReadCloser(item.DownloadUrl(arch, tgtos))
+	if src != nil {
+		defer src.Close()
+	}
+	if err != nil {
+		return fmt.Errorf("Unable to contact source URL for %s: %v", item.Name, err)
+	}
+	switch item.ContentType {
+	case "ContentPackage":
+		content := &models.Content{}
+		if err := json.NewDecoder(src).Decode(&content); err != nil {
+			return fmt.Errorf("Error downloading content bundle %s: %v", item.Name, err)
+		}
+
+		prereqs := strings.Split(content.Meta.Prerequisites, ",")
+		for _, p := range prereqs {
+			p = strings.TrimSpace(p)
+			if p == "" {
 				continue
 			}
-			if name == "" || name == item.Name {
-				res[k] = item
+			pversion := version
+			if pversion != "tip" && pversion != "stable" {
+				pversion = "stable"
 			}
-		}
-		return res
-	}
+			parts := strings.Split(p, ":")
+			pname := strings.TrimSpace(parts[0])
 
-	oneItem := func(cat *models.Content, name, version string) *models.CatalogItem {
-		items := itemsFromCatalog(cat, name)
-		for _, v := range items {
-			if v.Version == version {
-				return v
+			if _, err := Session.GetContentItem(pname); err == nil {
+				inflight[pname] = struct{}{}
+				continue
+			}
+
+			if err := installItem(catalog, pname, pversion, arch, tgtos, replaceWritable, inflight); err != nil {
+				return err
 			}
 		}
-		return nil
+		return doReplaceContent(content, "", replaceWritable)
+	case "PluginProvider":
+		res := &models.PluginProviderUploadInfo{}
+		req := Session.Req().Post(src).UrlFor("plugin_providers", item.Name)
+		if replaceWritable {
+			req = req.Params("replaceWritable", "true")
+		}
+		// TODO: One day handle prereqs.  Save to local file, mark executable, get contents, check prereqs
+		if err := req.Do(res); err != nil {
+			return err
+		}
+		return prettyPrint(res)
+	case "DRP":
+		if info, err := Session.PostBlob(src, "system", "upgrade"); err != nil {
+			return generateError(err, "Failed to post upgrade of DRP")
+		} else {
+			return prettyPrint(info)
+		}
+	case "DRPUX":
+		if info, err := Session.PostBlobExplode(src, true, "files", "ux", "drp-ux.zip"); err != nil {
+			return generateError(err, "Failed to post upgrade of DRP")
+		} else {
+			return prettyPrint(info)
+		}
+	default:
+		return fmt.Errorf("Don't know how to install %s of type %s yet", item.Name, item.ContentType)
+	}
+}
+
+func catalogCommands() *cobra.Command {
+
+	type catItem struct {
+		Type     string
+		Versions []string
 	}
 
 	cmd := &cobra.Command{
@@ -220,39 +294,13 @@ and wind up in a file with the same name as the item + the default file extensio
 			if err != nil {
 				return fmt.Errorf("Unable to fetch session information to determine endpoint arch and OS")
 			}
-			item := oneItem(catalog, args[0], version)
-			if item == nil {
-				return fmt.Errorf("%s version %s not in catalog", args[0], version)
-			}
 			arch = info.Arch
 			tgtos = info.Os
-			src, err := urlOrFileAsReadCloser(item.DownloadUrl(arch, tgtos))
-			if src != nil {
-				defer src.Close()
-			}
+			err = installItem(catalog, args[0], version, arch, tgtos, replaceWritable, map[string]struct{}{})
 			if err != nil {
-				return fmt.Errorf("Unable to contact source URL for %s: %v", item.Name, err)
-			}
-			switch item.ContentType {
-			case "ContentPackage":
-				content := &models.Content{}
-				if err := json.NewDecoder(src).Decode(&content); err != nil {
-					return fmt.Errorf("Error downloading content bundle %s: %v", item.Name, err)
-				}
-				return doReplaceContent(content, "", replaceWritable)
-			case "PluginProvider":
-				res := &models.PluginProviderUploadInfo{}
-				req := Session.Req().Post(src).UrlFor("plugin_providers", item.Name)
-				if replaceWritable{
-					req = req.Params("replaceWritable","true")
-				}
-				if err := req.Do(res); err != nil {
-					return err
-				}
-				return prettyPrint(res)
-			default:
-				return fmt.Errorf("Don't know how to install %s of type %s yet", item.Name, item.ContentType)
+				return err
 			}
+			return nil
 		},
 	}
 	install.Flags().BoolVar(&replaceWritable, "replaceWritable", false, "Replace identically named writable objects")
@@ -389,3 +437,7 @@ and wind up in a file with the same name as the item + the default file extensio
 func init() {
 	addRegistrar(func(c *cobra.Command) { c.AddCommand(catalogCommands()) })
 }
+
+func installOne() {
+
+}

conf.py

@@ -361,7 +361,7 @@ def fetch_urls(url):
     "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/chef-bootstrap.rst",
     "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/classify.rst",
     "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/cloud-wrappers.rst",
-    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/coheisity.rst",
+    "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/cohesity.rst",
     "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/coreos.rst",
     "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/dell-support.rst",
     "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/dev-library.rst",
@@ -401,7 +401,6 @@ def fetch_urls(url):
     "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/vmware.rst",
     "https://s3-us-west-2.amazonaws.com/rebar-catalog/docs/vmware-lib.rst",
 ]
-
 fetch_n_save(urls, path="doc/content-packages")
 
 urls = [

doc/api.rst

@@ -52,6 +52,26 @@ The API includes specialized filter behavior for Params that allows deep searchi
 
 To filter Machines or Profiles by Param values, pass the Param name and value using the normal Field filter specification.  When the Field is not found, the backend will search model's Params keys and evalute the filter against the Param value.
 
+.. _rs_api_proxy:
+
+Leveraging Multi-Site Proxy Forwarding
+--------------------------------------
+
+In the :ref:`rs_manager_arch`, API calls to a DRP manager for remote objects are automatically forwarded to the correct attached endpoint.  This allows operators to make API calls to remote endpoints from a centralized manager without knowing the actual owner of the object.  The owning endpoint can be determined for any object by inspecting its `Endpoint` property.
+
+For most cases, no additional information or API notation is required to leverage this functionality. The exception is creating objects on attached endpoints.  The create (aka POST) case requires API callers to specify the target remote endpoint (the endpoint must be registered or the request will be rejected) to the manager.
+
+To make an explicit API proxy call, prefix the path of the request with the endpoint identity.  For example:
+
+  ::
+
+    /[target endpoint id]/api/v3/machines
+
+
+This pattern is also invoked by with the `-u` flag in the DRPCLI.
+
+NOTE: Multi-site is a licensed feature available in DRP v4.5+ and must be enabled for the endpoint.
+
 .. _rs_api_slim:
 
 Payload Reduction (slim)

doc/arch.rst

@@ -27,4 +27,5 @@ dr-provision for developers and power users.
    arch/cluster
    arch/pooling
    arch/manager
+   arch/Swagger
 

doc/arch/manager.rst

@@ -68,8 +68,9 @@ The two main functions of manager are discussed here.
 Single Pane of Glass
 ====================
 
-When making API requests, the manager will provide results from all the attached endpoints.  Additionally, all objects
-have a field, **Endpoint**, that is populated with the Endpoint Id (the High Availability Id) of the owning endpoint.
+When making API requests, the manager will provide results from all the attached endpoints.  Additionally, all objects have a field, **Endpoint**, that is populated with the Endpoint Id (the High Availability Id) of the owning endpoint.  API requests made to objects from attached endpoints are
+automatically forward to the correct endpoint.  See :ref:`rs_api_proxy` for more information about
+using this automatic API behavior explicity.
 
 Only the local objects are replicated up to the manager, objects provided by content packages and plugins are not
 replicated to the manager.  It is assumed that the manager will have all the content packages and plugins loaded to

doc/kb/kb-00042.rst

@@ -39,6 +39,7 @@ JQ Raw Mode
 ===========
 
 Raw JSON output is usefull when passing the results of one ``jq`` command in to another for scripted interaction.  Be sure to specify "Raw" mode in this case - to prevent colorization and extraneous quotes being wrapped around Key/Value data output.
+
   ::
 
       <some command> | jq -r ...
@@ -52,11 +53,13 @@ Filter Out gohai-inventory
 The ``gohai-inventory`` module is extremely useful for providing Machine classification information for use by other stages or tasks.  However, it is very long and causes a lot of content to be output to the console when listing Machine information.  Using a simple ``jq`` filter, you can delete the ``gohai-inventory`` content from the output display.
 
 Note that since the Param name is ``gohai-inventory``, we have to provide some quoting of the Param name, since the dash (``-``) has special meaning in JSON parsing.
+
   ::
 
     drpcli machines list | jq 'del(.[].Params."gohai-inventory")'
 
 Subsequently, if you are listing an individual Machine, then you can also filter it's ``gohai-inventory`` output as well, with:
+
   ::
 
     drpcli machines show <UUID> | jq 'del(.Params."gohai-inventory")'
@@ -68,6 +71,7 @@ List BootEnv Names
 ==================
 
 Get list of bootenvs available in the installed content, by name:
+
   ::
 
     drpcli bootenvs list | jq '.[].Name'
@@ -79,11 +83,13 @@ Reformat Output With Specific Keys
 ==================================
 
 Get list of machines, output "Name:Uuid" pairs from the the JSON output:
+
   ::
 
     drpcli machines list | jq -r '.[] | "\(.Name):\(.Uuid)"'
 
 Output is printed as follows:
+
   ::
 
     machine1:05abe5dc-637a-4952-a1be-5ec85ba00686
@@ -98,6 +104,7 @@ Extract Specific Key From Output
 ================================
 
 ``jq`` can also pull out only specific Keys from the JSON input.  Here is an example to get ISO File name for a bootenv:
+
   ::
 
     drpcli contents show os-discovery | jq '.sections.bootenvs.discovery.OS.IsoFile'
@@ -109,6 +116,7 @@ Display Job Logs for Specific Machine
 =====================================
 
 The Job Logs provide a lot of information about the provisioning process of your DRP Endpoint.  However, you often only want to see Job Logs for a specific Machine to evaluate provisioning status.  To get specific Jobs from the job list - based on Machine UUID, do:
+
   ::
 
     export UUID=`abcd-efgh-ijkl-mnop-qrps"
@@ -120,7 +128,7 @@ The Job Logs provide a lot of information about the provisioning process of your
 Using --args deal with pipes and quotes (NOT A DRPCLI JQ feature)
 =================================================================
 
-.. note:: This is NOT supported by the DROCLI JQ_
+.. note:: This is NOT supported by the DROCLI JQ
 
 In scripts, it can become very difficult to correctly pass variables inside of pipes.  For this reason, operators may want to use the `jq --args` instead of attempting to inject values into jq queries.
 
@@ -133,7 +141,7 @@ In the example below, we needed to match a value inside an array of JSON objects
 
 This example looks for a value in an array.
 
- ::
+  ::
 
     licensed=$(jq --args m "$mc" -r 'contains(["$m"])' <<< $endpoints)
 

doc/operation.rst

@@ -20,7 +20,6 @@ Some of these operations are in the :ref:`rs_ui`, but not all.  This will focus
 
    operations/airgap
    operations/bios
-   operations/commands
    operations/contexts
    operations/drpcli
    operations/esxi-getting-started

doc/operations/drpcli.rst

@@ -775,4 +775,4 @@ to assist them with troubleshooting your issue. drpcli can be used to create the
     drpcli support machine-bundle [id]
 
 
-How to update drpcli to obtain the support commands: :ref:`_rs_kb_00049`
+How to update drpcli to obtain the support commands: :ref:`rs_kb_00049`

doc/operations/manager.rst

@@ -80,6 +80,12 @@ or content packages.  The **files** api can be used to update the catalog.
 
 The UX has a helper button for this action.
 
+Proxy Creating an Object on Managed Endpoint
+____________________________________________
+
+It is possible to create objects on managed endpoints by using proxy pass-through from the manager.  Details are available in :ref:`rs_api_proxy`.
+
+
 Manager Common Methods
 ----------------------