Merge branch 'fix_wiki_gen_exotic' into 'master'

jonglezb · jonglezb · commit 7a21cf4c8e00 · 2020-12-17T10:58:22.000+01:00
[gen/wiki] Fix wiki links to exotic clusters by simplifying hardware page headings

See merge request grid5000/reference-repository!156
diff --git a/lib/refrepo/gen/wiki/generators/hardware.rb b/lib/refrepo/gen/wiki/generators/hardware.rb
@@ -381,9 +381,7 @@ def generate_storage
               ((!d['reservation'].nil? && d['reservation']) ? '[[Disk_reservation|*]]' : '')
             }.join(', ') + ")"
           end
-          queues = cluster_hash['queues'] - ['admin', 'default']
-          queue_t = (queues.nil? || (queues.empty? ? '' : "_.28" + queues[0].gsub(' ', '_') + ' queue.29'))
-          nodes_data << { 'uid' => node_uid, 'data' => { 'main' => maindisk_t, 'hdd' => hdd_t, 'ssd' => ssd_t, 'reservation' => reservable_disks, 'queue' => queue_t } }
+          nodes_data << { 'uid' => node_uid, 'data' => { 'main' => maindisk_t, 'hdd' => hdd_t, 'ssd' => ssd_t, 'reservation' => reservable_disks } }
         end
         nd = nodes_data.group_by { |d| d['data'] }
         nd.each do |data, nodes|
@@ -396,7 +394,7 @@ def generate_storage
           end
           table_data << [
             "[[#{site_uid.capitalize}:Hardware|#{site_uid.capitalize}]]",
-              "[[#{site_uid.capitalize}:Hardware##{cluster_uid}#{data['queue']}|#{nodesetname}]]",
+              "[[#{site_uid.capitalize}:Hardware##{cluster_uid}|#{nodesetname}]]",
               nodes.length,
               data['main'],
               data['hdd'],
@@ -445,7 +443,7 @@ def generate_interfaces
         network_interfaces.sort.to_h.each { |num, interfaces|
           table_data << [
             "[[#{site_uid.capitalize}:Network|#{site_uid.capitalize}]]",
-            "[[#{site_uid.capitalize}:Hardware##{cluster_uid}" + (interfaces['queues'] == '' ? '' : "_.28#{queues.gsub(' ', '_')}.29") + "|#{cluster_uid}" + (network_interfaces.size==1 ? '' : '-' + G5K.nodeset(num)) + "]]",
+            "[[#{site_uid.capitalize}:Hardware##{cluster_uid}" + "|#{cluster_uid}" + (network_interfaces.size==1 ? '' : '-' + G5K.nodeset(num)) + "]]",
             num.count,
             interfaces['25g_count'].zero? ? '' : interfaces['25g_count'],
             interfaces['10g_count'].zero? ? '' : interfaces['10g_count'],
diff --git a/lib/refrepo/gen/wiki/generators/site_hardware.rb b/lib/refrepo/gen/wiki/generators/site_hardware.rb
@@ -77,14 +77,17 @@ def self.generate_summary_data(site, with_sites)
     hardware[site].sort.to_h.each { |cluster_uid, cluster_hash|
       cluster_nodes = cluster_hash.keys.flatten.count
       queue = cluster_hash.map { |k, v| v['queue']}.first
-      queue_str = cluster_hash.map { |k, v| v['queue_str']}.first
       access_conditions = []
-      access_conditions << "<b>#{queue}</b>&nbsp;queue" if queue != ''
-      access_conditions << '<b>exotic</b>&nbsp;job&nbsp;type' if cluster_hash.map { |k, v| v['exotic']}.first
+      if queue == 'production'
+        access_conditions << "<b>[[Grid5000:UsagePolicy#Rules_for_the_production_queue|#{queue}]]</b>&nbsp;queue"
+      elsif queue != ''
+        access_conditions << "<b>#{queue}</b>&nbsp;queue"
+      end
+      access_conditions << '<b>[[Getting_Started#Selecting_specific_resources|exotic]]</b>&nbsp;job&nbsp;type' if cluster_hash.map { |k, v| v['exotic']}.first
       table_columns = (with_sites == true ? ['Site'] : []) + ['Cluster',  'Access Condition', 'Date of arrival', { attributes: 'data-sort-type="number"', text: 'Nodes' }, 'CPU', { attributes: 'data-sort-type="number"', text: 'Cores' }, { attributes: 'data-sort-type="number"', text: 'Memory' }, { attributes: 'data-sort-type="number"', text: 'Storage' }, { attributes: 'data-sort-type="number"', text: 'Network' }] + ((site_accelerators.zero? && with_sites == false) ? [] : ['Accelerators'])
       data = partition(cluster_hash)
       table_data <<  (with_sites == true ? ["[[#{site.capitalize}:Hardware|#{site.capitalize}]]"] : []) + [
-        (with_sites == true ? "[[#{site.capitalize}:Hardware##{cluster_uid}" + (queue_str == '' ? '' : "_.28#{queue_str.gsub(' ', '_')}.29") + "|#{cluster_uid}]]" : "[[##{cluster_uid}" + (queue_str == '' ? '' : "_.28#{queue_str.gsub(' ', '_')}.29") + "|#{cluster_uid}]]"),
+        (with_sites == true ? "[[#{site.capitalize}:Hardware##{cluster_uid}" + "|#{cluster_uid}]]" : "[[##{cluster_uid}" + "|#{cluster_uid}]]"),
         access_conditions.join(",<br/>"),
         cell_data(data, 'date'),
         cluster_nodes,
@@ -109,53 +112,69 @@ def self.generate_description(site)
       site_accelerators += cluster_hash.select { |k, v| v['accelerators'] != '' }.count
     }
 
-    hardware[site].sort.to_h.each { |cluster_uid, cluster_hash|
-      subclusters = cluster_hash.keys.count != 1
-      cluster_nodes = cluster_hash.keys.flatten.count
-      cluster_cpus = cluster_hash.map { |k, v| k.count * v['cpus_per_node'] }.reduce(:+)
-      cluster_cores = cluster_hash.map { |k, v| k.count * v['cpus_per_node'] * v['cores_per_cpu'] }.reduce(:+)
-      queue_str = cluster_hash.map { |k, v| v['queue_str']}.first
-      access_conditions = []
-      access_conditions << queue_str if queue_str != ''
-      access_conditions << "exotic job type" if cluster_hash.map { |k, v| v['exotic']}.first
-      table_columns = ['Cluster',  'Queue', 'Date of arrival', { attributes: 'data-sort-type="number"', text: 'Nodes' }, 'CPU', { attributes: 'data-sort-type="number"', text: 'Cores' }, { attributes: 'data-sort-type="number"', text: 'Memory' }, { attributes: 'data-sort-type="number"', text: 'Storage' }, { attributes: 'data-sort-type="number"', text: 'Network' }] + (site_accelerators.zero? ? [] : ['Accelerators'])
-
-      text_data <<  ["\n== #{cluster_uid}" + (access_conditions.empty? ? '' : " (#{access_conditions.join(", ")})") + " ==\n"]
-      text_data << ["'''#{cluster_nodes} #{G5K.pluralize(cluster_nodes, 'node')}, #{cluster_cpus} #{G5K.pluralize(cluster_cpus, 'cpu')}, #{cluster_cores} #{G5K.pluralize(cluster_cores, 'core')}" + (subclusters == true ? ",''' split as follows due to differences between nodes " : "''' ") + "([https://public-api.grid5000.fr/stable/sites/#{site}/clusters/#{cluster_uid}/nodes.json?pretty=1 json])"]
-
-      cluster_hash.sort.to_h.each_with_index { |(num, h), i|
-        if subclusters
-          subcluster_nodes = num.count
-          subcluster_cpus = subcluster_nodes * h['cpus_per_node']
-          subcluster_cores = subcluster_nodes * h['cpus_per_node'] * h['cores_per_cpu']
-          text_data << "<hr style=\"height:10pt; visibility:hidden;\" />\n" if i != 0 # smaller vertical <br />
-          text_data << ["; #{cluster_uid}-#{G5K.nodeset(num)} (#{subcluster_nodes} #{G5K.pluralize(subcluster_nodes, 'node')}, #{subcluster_cpus} #{G5K.pluralize(subcluster_cpus, 'cpu')}, #{subcluster_cores} #{G5K.pluralize(subcluster_cores, 'core')})"]
-        end
+    # Group by queue
+    # Alphabetic ordering of queue names matches what we want: "default" < "production" < "testing"
+    hardware[site].group_by { |cluster_uid, cluster_hash| cluster_hash.map { |k, v| v['queue']}.first }.sort.each { |queue, clusters|
+      queue = (queue.nil? || queue.empty?) ? 'default' : queue
+      text_data << "\n= Clusters in #{queue} queue ="
+      clusters.sort.to_h.each { |cluster_uid, cluster_hash|
+        subclusters = cluster_hash.keys.count != 1
+        cluster_nodes = cluster_hash.keys.flatten.count
+        cluster_cpus = cluster_hash.map { |k, v| k.count * v['cpus_per_node'] }.reduce(:+)
+        cluster_cores = cluster_hash.map { |k, v| k.count * v['cpus_per_node'] * v['cores_per_cpu'] }.reduce(:+)
+        queue_str = cluster_hash.map { |k, v| v['queue_str']}.first
+        access_conditions = []
+        access_conditions << queue_str if queue_str != ''
+        access_conditions << "exotic job type" if cluster_hash.map { |k, v| v['exotic']}.first
+        table_columns = ['Cluster',  'Queue', 'Date of arrival', { attributes: 'data-sort-type="number"', text: 'Nodes' }, 'CPU', { attributes: 'data-sort-type="number"', text: 'Cores' }, { attributes: 'data-sort-type="number"', text: 'Memory' }, { attributes: 'data-sort-type="number"', text: 'Storage' }, { attributes: 'data-sort-type="number"', text: 'Network' }] + (site_accelerators.zero? ? [] : ['Accelerators'])
+
+        text_data <<  ["\n== #{cluster_uid} ==\n"]
+        text_data << ["'''#{cluster_nodes} #{G5K.pluralize(cluster_nodes, 'node')}, #{cluster_cpus} #{G5K.pluralize(cluster_cpus, 'cpu')}, #{cluster_cores} #{G5K.pluralize(cluster_cores, 'core')}" + (subclusters == true ? ",''' split as follows due to differences between nodes " : "''' ") + "([https://public-api.grid5000.fr/stable/sites/#{site}/clusters/#{cluster_uid}/nodes.json?pretty=1 json])"]
+
+        reservation_cmd = "\n{{Term|location=f#{site}|cmd="
+        reservation_cmd += "<code class=\"command\">oarsub</code> "
+        reservation_cmd += "<code class=\"replace\">-q #{queue}</code> " if queue != 'default'
+        reservation_cmd += "<code class=\"replace\">-t exotic</code> " if cluster_hash.map { |k, v| v['exotic']}.first
+        reservation_cmd += "<code class=\"env\">-p \"cluster='#{cluster_uid}'\"</code> "
+        reservation_cmd += "<code>-I</code>"
+        reservation_cmd += "}}\n"
+        text_data << "\n'''Reservation example:'''"
+        text_data << reservation_cmd
+
+        cluster_hash.sort.to_h.each_with_index { |(num, h), i|
+          if subclusters
+            subcluster_nodes = num.count
+            subcluster_cpus = subcluster_nodes * h['cpus_per_node']
+            subcluster_cores = subcluster_nodes * h['cpus_per_node'] * h['cores_per_cpu']
+            text_data << "<hr style=\"height:10pt; visibility:hidden;\" />\n" if i != 0 # smaller vertical <br />
+            text_data << ["; #{cluster_uid}-#{G5K.nodeset(num)} (#{subcluster_nodes} #{G5K.pluralize(subcluster_nodes, 'node')}, #{subcluster_cpus} #{G5K.pluralize(subcluster_cpus, 'cpu')}, #{subcluster_cores} #{G5K.pluralize(subcluster_cores, 'core')})"]
+          end
 
-        accelerators = nil
-        if h['gpu_str'] != '' && h['mic_str'] != ''
-          accelerators = 'GPU/Xeon Phi'
-        elsif h['gpu_str'] != ''
-          accelerators = 'GPU'
-        elsif h['mic_str'] != ''
-          accelerators = 'Xeon Phi'
-        end
-        hash = {
-          'Model' => h['model'],
-          'Date of arrival' => h['date'],
-          'CPU' => h['processor_description'],
-          'Memory' => h['ram_size'] + (!h['pmem_size'].nil? ? " + #{h['pmem_size']} [[PMEM]]" : ''),
-          'Storage' => h['storage_description'],
-          'Network' => h['network_description'],
+          accelerators = nil
+          if h['gpu_str'] != '' && h['mic_str'] != ''
+            accelerators = 'GPU/Xeon Phi'
+          elsif h['gpu_str'] != ''
+            accelerators = 'GPU'
+          elsif h['mic_str'] != ''
+            accelerators = 'Xeon Phi'
+          end
+          hash = {}
+          hash['Access condition'] = access_conditions.join(", ") if not access_conditions.empty?
+          hash.merge!({
+            'Model' => h['model'],
+            'Date of arrival' => h['date'],
+            'CPU' => h['processor_description'],
+            'Memory' => h['ram_size'] + (!h['pmem_size'].nil? ? " + #{h['pmem_size']} [[PMEM]]" : ''),
+            'Storage' => h['storage_description'],
+            'Network' => h['network_description'],
+          })
+          hash[accelerators] = h['accelerators_long'] if accelerators
+          text_data << MW::generate_hash_table(hash)
         }
-        hash[accelerators] = h['accelerators_long'] if accelerators
-        text_data << MW::generate_hash_table(hash)
       }
     }
 
-    generated_content = "\n= Cluster details =\n"
-    generated_content += text_data.join("\n")
-    return generated_content
+    return text_data.join("\n")
   end
 end
 
