Merge pull request #603 from ystia/feature/GH-205-hp-consumable-resou…

…rces

Feature/gh 205 hp consumable resources

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## UNRELEASED
 
+### FEATURES
+
+* Add a consumable resources concept to the Hosts Pool ([GH-205](https://github.com/ystia/yorc/issues/205))
+
 ### BUG FIXES
 
 * Yorc Bootstrap doesn't uninstall yorc binary ([GH-605](https://github.com/ystia/yorc/issues/605))

commands/hostspool/hosts_pool.go

@@ -107,25 +107,6 @@ func getColoredHostStatus(colorize bool, status string) string {
 	}
 }
 
-// padSlices is padding as necessary slices in arguments so that both slices
-// have the same number of elements
-func padSlices(slice1 []string, slice2 []string) ([]string, []string) {
-	slice1Size := len(slice1)
-	slice2Size := len(slice2)
-
-	if slice1Size > slice2Size {
-		for i := 0; i < slice1Size-slice2Size; i++ {
-			slice2 = append(slice2, "")
-		}
-	} else {
-		for i := 0; i < slice2Size-slice1Size; i++ {
-			slice1 = append(slice1, "")
-		}
-	}
-
-	return slice1, slice2
-}
-
 // AddRow adds a row to a table, with text colored according to the operation
 // longTable specifies table with all headers
 func addRow(table tabutil.Table, colorize bool, operation int, host *rest.Host, fullTable bool) {
@@ -141,13 +122,13 @@ func addRow(table tabutil.Table, colorize bool, operation int, host *rest.Host,
 
 	allocationsSubRows := make([]string, 0)
 	for _, alloc := range host.Allocations {
-		allocationsSubRows = append(allocationsSubRows, strings.Split(alloc.String(), ",")...)
+		allocationsSubRows = append(allocationsSubRows, strings.Split(" - "+alloc.String(), "|")...)
 	}
 
 	connectionSubRows := strings.Split(host.Connection.String(), ",")
 	var labelSubRows []string
 	if host.Labels != nil {
-		labelSubRows = strings.Split(toPrintableLabels(host.Labels), ",")
+		labelSubRows = strings.Split(toPrintableLabels(host.Labels), "|")
 		sort.Strings(labelSubRows)
 	}
 
@@ -204,7 +185,7 @@ func addHostInErrorRow(table tabutil.Table, colorize bool, operation int, host *
 	connectionSubRows := strings.Split(host.Connection.String(), ",")
 	var labelSubRows []string
 	if host.Labels != nil {
-		labelSubRows = strings.Split(toPrintableLabels(host.Labels), ",")
+		labelSubRows = strings.Split(toPrintableLabels(host.Labels), "|")
 		sort.Strings(labelSubRows)
 	}
 

commands/hostspool/hosts_pool_apply.go

@@ -323,7 +323,12 @@ func toPrintableLabels(labels map[string]string) string {
 	var labelsList string
 	for k, v := range labels {
 		if labelsList != "" {
-			labelsList += ","
+			labelsList += "|"
+		}
+
+		// Specific display for generic resources
+		if strings.HasPrefix(k, "host.resource.") {
+			v = fmt.Sprintf("\"%s\"", v)
 		}
 		labelsList += fmt.Sprintf("%s: %s", k, v)
 	}
@@ -357,13 +362,13 @@ func addUpdateRows(table tabutil.Table, colorize bool, oldHost *rest.Host, newHo
 		allocationsSubRows = append(allocationsSubRows, strings.Split(alloc.String(), ",")...)
 	}
 	oldConnectionSubRows := strings.Split(toPrintableConnection(oldConnection), ",")
-	oldLabelSubRows := strings.Split(toPrintableLabels(oldLabels), ",")
+	oldLabelSubRows := strings.Split(toPrintableLabels(oldLabels), "|")
 	// Sorting labels for an easier comparison between old and new labels
 	sort.Strings(oldLabelSubRows)
 	sliceutil.PadSlices("", &allocationsSubRows, &oldConnectionSubRows, &oldLabelSubRows)
 
 	newConnectionSubRows := strings.Split(toPrintableConnection(newConnection), ",")
-	newLabelSubRows := strings.Split(toPrintableLabels(newLabels), ",")
+	newLabelSubRows := strings.Split(toPrintableLabels(newLabels), "|")
 	// Sorting labels for an easier comparison between old and new labels
 	sort.Strings(newLabelSubRows)
 	sliceutil.PadSlices("", &allocationsSubRows, &newConnectionSubRows, &newLabelSubRows)

data/tosca/yorc-hostspool-types.yml

@@ -33,6 +33,8 @@ node_types:
         properties:
           credentials:
             user: "not significant, will be set by yorc itself"
+      host:
+        type: yorc.capabilities.hostspool.Container
 
 policy_types:
   yorc.policies.hostspool.Placement:
@@ -53,3 +55,40 @@ policy_types:
       The yorc hostpool TOSCA Policy placement which allows to allocate a host with a bin packing algorithm.
       It means the host the more allocated will be elect preferentially.
     targets: [ tosca.nodes.Compute ]
+
+capability_types:
+  yorc.capabilities.hostspool.Container:
+    derived_from: tosca.capabilities.Container
+    properties:
+      resources:
+        type: list
+        description: >
+          A list of generic resources that the container must provide.
+        entry_schema:
+          type: yorc.datatypes.hostspool.GenericResource
+        required: false
+
+data_types:
+  yorc.datatypes.hostspool.GenericResource:
+    derived_from: tosca.datatypes.Root
+    properties:
+      name:
+        type: string
+        required: true
+        description: >
+          The name of the generic resource. Can be "gpu" and must be bound to host labels as: host.resource.gpu by instance.
+      ids:
+        type: list
+        required: false
+        description: >
+          Each list entry corresponds to the required generic resource ID's for each instance.
+          Each list entry is a comma-separated list of ID's.
+          Either ids or number must be filled to define the resource need.
+          An ID must only contains the following characters: a-zA-Z0-9_:./-
+        entry_schema:
+          type: string
+      number:
+        type: integer
+        required: false
+        description: >
+          The number of generic resource required.  Either ids or number must be filled to define the resource need.

doc/infrastructures.rst

@@ -151,7 +151,135 @@ The resources host pool labels (``host.num_cpus``, ``host.disk_size``, ``host.me
 only if you specify any of these Tosca ``host`` resources capabilities Compute in its Alien4Cloud applications.
 If you apply a new configuration on allocated hosts with new host resources labels, they will be recalculated depending on existing allocations resources.
 
-
+
+Hosts Pool Generic Resources
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you want to require specific resources as GPU by instance for your application's computes, you can declare in the hosts pool configuration the available list of GPUs for each host.
+To do that, you will use a ``host.resource.gpu`` label, a ``comma-separated list`` of GPU resources as in the example below:
+
+.. code-block:: YAML
+
+  hosts:
+  - name: host1
+    connection:
+      user: centos
+      host: 1.2.3.4
+      private_key: /home/user/.ssh/yorc.pem
+      port: 22
+    labels:
+      os.architecture: x86_64
+      os.distribution: centos
+      os.type: linux
+      host.resource.gpu: "gpu2"
+  - name: hostspool-ci-1
+    connection:
+      user: centos
+      host: 6.7.8.9
+      private_key: /home/user/.ssh/yorc.pem
+      port: 22
+    labels:
+      os.architecture: x86_64
+      os.distribution: centos
+      os.type: linux
+      host.resource.gpu: "gpu0,gpu1"
+
+
+In this example:
+  * ``host1`` provides a list of GPUs with a single GPU ID: ``gpu2``.
+  * ``host2`` provides a list of GPUs with 2 ids: ``gpu0`` and ``gpu1``.
+
+To require these specific resources in your application, an implicit matching will be done between the ``host.resource.<name>`` labels and the Tosca ``host`` capability.
+
+The ``host`` capability has been extended with ``yorc.capabilities.hostspool.Container`` to provide a ``resources`` property.
+
+The ``resource`` property is a list of ``yorc.datatypes.hostspool.GenericResource``
+
+A ``Generic Resource`` is defined with the following properties:
+
+  * ``name``: The name of the generic resource. Can be "gpu" by instance and must be bound to host labels as: ``host.resource.<name>``.
+  * ``ids``: List of required generic resource ID's by node instance. Each list entry corresponds to a comma-separated list of required generic resource ID's for each node instance. An ID must only contains the following characters:  ``-_0-9a-zA-Z_:./-``
+  * ``number``: The number of generic resource required.  Either ``ids`` or ``number`` must be filled to define the resource need.
+
+Here is an example of an application which requires some GPUs:
+
+.. code-block:: YAML
+
+  topology_template:
+    node_templates:
+      ComputeA:
+        type: yorc.nodes.hostspool.Compute
+        properties:
+          shareable: true
+        capabilities:
+          host:
+            properties:
+              resources:
+              - name: gpu
+                ids:
+                  - gpu2
+      ComputeB:
+        type: yorc.nodes.hostspool.Compute
+        properties:
+          shareable: true
+        capabilities:
+          host:
+            properties:
+              resources:
+              - name: gpu
+                number: 2
+
+
+The ``ComputeA`` node requires a specific GPU's ID: ``gpu2``.
+
+The ``ComputeB`` node requires 2 GPUs without specifying any ID's requirement.
+
+
+If you deploy the application on the hosts pool location previously defined, you will get the following allocations:
+
+.. code-block:: bash
+
+  $ yorc hp list -l hp
+  +----------------+--------------------------------------------+-----------+--------------------------------+---------+-----------------------------------+
+  | Name           | Connection                                 | Status    | Allocations                    | Message | Labels                            |
+  +----------------+--------------------------------------------+-----------+--------------------------------+---------+-----------------------------------+
+  | host1          | user: centos                               | allocated | deployment: testApp            |         | host.resource.gpu: ""             |
+  |                | private key: /home/user/.ssh/yorc.pem      |           | node-instance: ComputeA        |         | os.architecture: x86_64           |
+  |                | host: 1.2.3.4                              |           | shareable: true                |         | os.distribution: centos           |
+  |                | port: 22                                   |           | host.resource.gpu: "gpu2"      |         | os.type: linux                    |
+  |                |                                            |           |                                |         |                                   |
+  |                |                                            |           |                                |         |                                   |
+  |                |                                            |           |                                |         |                                   |
+  |                |                                            |           |                                |         |                                   |
+  | host2          | user: centos                               | allocated | deployment: testApp            |         | host.resource.gpu: ""             |
+  |                | private key: /home/user/.ssh/yorc.pem      |           | node-instance: ComputeB        |         | os.architecture: x86_64           |
+  |                | host: 6.7.8.9                              |           | shareable: true                |         | os.distribution: centos           |
+  |                | port: 22                                   |           | host.resource.gpu: "gpu0,gpu1" |         | os.type: linux                    |
+  +----------------+--------------------------------------------+-----------+--------------------------------+---------+-----------------------------------+
+
+
+The ``ComputeA`` GPU requirement on a ``gpu2`` ID has been done by ``host1``.
+
+The ``ComputeB`` GPU requirement of ``2`` GPUs ID has been done by ``host2``.
+
+Both ``host1`` and ``host2`` are no longer providing GPUs resources as these resources are defined as ``consumable``.
+
+By default, a generic resource is consumable. It means a resource can be only used by a single compute. If you want to share a generic resource among many computes, you have to specify the following label
+``host.resource.gpu.no_consume: true`` as below in the host declaration:
+
+.. code-block:: YAML
+
+  hosts:
+  - name: hostspool-ci-1
+    labels:
+      host.resource.gpu: "gpu0,gpu1"
+      host.resource.gpu.no_consume: true
+
+
+
+A Tosca instance attribute "gpu" will be exposed with the allocated resources for each node instance once the application is deployed.
+
+Note: If you apply a new configuration on allocated hosts with new host generic resources labels, they will be recalculated depending on existing allocations resources.
 
 .. _yorc_infras_slurm_section:
 

helper/labelsutil/internal/filters_test.go

@@ -243,6 +243,14 @@ func TestFiltersRegexMatching(t *testing.T) {
 		want    bool
 		wantErr bool
 	}{
+		{"TestRegexpContainsOnEmptyString", `lr1 ~= "^(.+)$"`, args{map[string]string{"lr1": ""}}, false, false},
+		{"TestRegexpContainsOnEmptyString", `lr1 ~= "^(.+)$"`, args{map[string]string{"lr1": "a"}}, true, false},
+		{"TestRegexpContainsOnCommaSeparatedListCountHere", `lr1 ~= "^/gpu:0,|,/gpu:0,|,/gpu:0$|^/gpu:0$"`, args{map[string]string{"lr1": "/gpu:0,/gpu:1"}}, true, false},
+		{"TestRegexpContainsOnCommaSeparatedListCount", `lr1 ~= "^([^,]*,){3}.*$"`, args{map[string]string{"lr1": "a1,bn,c2,d"}}, true, false},
+		{"TestRegexpContainsOnCommaSeparatedListBegin", `lr1 ~= "^a,|,a,|,a$|^a$"`, args{map[string]string{"lr1": "a,b,c,d"}}, true, false},
+		{"TestRegexpContainsOnCommaSeparatedListMiddle", `lr1 ~= "^b,|,b,|,b$|^b$"`, args{map[string]string{"lr1": "a,b,c,d"}}, true, false},
+		{"TestRegexpNotContainsOnCommaSeparatedList", `lr1 ~= "^ab,|,ab,|,ab$|^ab$"`, args{map[string]string{"lr1": "a,b,c,d"}}, false, false},
+		{"TestRegexpContainsOnCommaSeparatedListEnd", `lr1 ~= "^d,|,d,|,d$|^d$"`, args{map[string]string{"lr1": "a,b,c,d"}}, true, false},
 		{"TestRegexpContainsOnMatchingString", `lr1 ~= "vv"`, args{map[string]string{"lr1": "vv", "m2": "v2"}}, true, false},
 		{"TestRegexpContainsOnContainingString", `lr1 ~= "vv"`, args{map[string]string{"lr1": "vvvvv", "m2": "v2"}}, true, false},
 		{"TestRegexpExcludingOnContainingString", `lr1 !~ "vv"`, args{map[string]string{"lr1": "vvvvv", "m2": "v2"}}, false, false},