From ea2abead3a914ab6ca909b05801a6753a653725f Mon Sep 17 00:00:00 2001 From: AriettaZ Date: Mon, 24 Sep 2018 17:00:27 -0400 Subject: [PATCH 01/11] add config validation --- Gemfile | 2 + Gemfile.lock | 3 + lib/tasks/config_schema.json | 571 +++++++++++++++++++++++++++++ lib/tasks/quick_config_schema.json | 242 ++++++++++++ lib/tasks/validate.rake | 91 +++++ 5 files changed, 909 insertions(+) create mode 100644 lib/tasks/config_schema.json create mode 100644 lib/tasks/quick_config_schema.json create mode 100644 lib/tasks/validate.rake diff --git a/Gemfile b/Gemfile index 41c170ee..d6a92a3a 100644 --- a/Gemfile +++ b/Gemfile @@ -59,9 +59,11 @@ gem 'rails_12factor', group: :production gem 'mocha', '~> 1.1', group: :test gem "spring", group: :test gem 'autoprefixer-rails', '~> 8.4' +gem 'json-schema', '~> 2.8' # OOD specific gems gem 'ood_support', '~> 0.0.2' gem 'ood_appkit', '~> 1.0' gem 'ood_core', '~> 0.1' gem 'pbs', '~> 2.2.1' + diff --git a/Gemfile.lock b/Gemfile.lock index 282365e5..1e80db58 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -84,6 +84,8 @@ GEM railties (>= 4.2.0) thor (>= 0.14, < 2.0) json (1.8.6) + json-schema (2.8.0) + addressable (>= 2.4) lograge (0.10.0) actionpack (>= 4) activesupport (>= 4) @@ -203,6 +205,7 @@ DEPENDENCIES jbuilder (~> 2.0) jquery-datatables-rails (~> 3.4) jquery-rails + json-schema (~> 2.8) mocha (~> 1.1) ood_appkit (~> 1.0) ood_core (~> 0.1) diff --git a/lib/tasks/config_schema.json b/lib/tasks/config_schema.json new file mode 100644 index 00000000..77ce68ea --- /dev/null +++ b/lib/tasks/config_schema.json @@ -0,0 +1,571 @@ +{ + "description": "Config schema for OSC clusters", + "type": "object", + "title": "The Config Schema", + "required": [ + "v2" + ], + "properties": { + "v2": { + "$id": "#/properties/v2", + "type": "object", + "title": "The V2 Schema", + "required": [ + "metadata", + "login", + "job", + "custom", + "batch_connect" + ], + "properties": { + "metadata": { + "$id": "#/properties/v2/properties/metadata", + "type": "object", + "title": "The Metadata Schema", + "required": [ + "title", + "url", + "hidden" + ], + "properties": { + "title": { + "$id": "#/properties/v2/properties/metadata/properties/title", + "type": "string", + "title": "The Title Schema", + "default": "", + "examples": [ + "Owens" + ], + "pattern": "^(.*)$" + }, + "url": { + "$id": "#/properties/v2/properties/metadata/properties/url", + "type": "string", + "title": "The Url Schema", + "default": "", + "examples": [ + "https://www.osc.edu/supercomputing/computing/owens" + ], + "pattern": "^(.*)$" + }, + "hidden": { + "$id": "#/properties/v2/properties/metadata/properties/hidden", + "type": "boolean", + "title": "The Hidden Schema", + "default": false, + "examples": [ + false + ] + } + } + }, + "login": { + "$id": "#/properties/v2/properties/login", + "type": "object", + "title": "The Login Schema", + "required": [ + "host" + ], + "properties": { + "host": { + "$id": "#/properties/v2/properties/login/properties/host", + "type": "string", + "title": "The Host Schema", + "default": "", + "examples": [ + "owens.osc.edu" + ], + "pattern": "^(.*)$" + } + } + }, + "job": { + "$id": "#/properties/v2/properties/job", + "type": "object", + "title": "The Job Schema", + "required": [ + "adapter", + "host", + "lib", + "bin", + "version" + ], + "properties": { + "adapter": { + "$id": "#/properties/v2/properties/job/properties/adapter", + "type": "string", + "title": "The Adapter Schema", + "default": "", + "examples": [ + "torque" + ], + "pattern": "^(.*)$" + }, + "host": { + "$id": "#/properties/v2/properties/job/properties/host", + "type": "string", + "title": "The Host Schema", + "default": "", + "examples": [ + "owens-batch.ten.osc.edu" + ], + "pattern": "^(.*)$" + }, + "lib": { + "$id": "#/properties/v2/properties/job/properties/lib", + "type": "string", + "title": "The Lib Schema", + "default": "", + "examples": [ + "/opt/torque/lib64" + ], + "pattern": "^(.*)$" + }, + "bin": { + "$id": "#/properties/v2/properties/job/properties/bin", + "type": "string", + "title": "The Bin Schema", + "default": "", + "examples": [ + "/opt/torque/bin" + ], + "pattern": "^(.*)$" + }, + "version": { + "$id": "#/properties/v2/properties/job/properties/version", + "type": "string", + "title": "The Version Schema", + "default": "", + "examples": [ + "6.0.1" + ], + "pattern": "^(.*)$" + } + } + }, + "custom": { + "$id": "#/properties/v2/properties/custom", + "type": "object", + "title": "The Custom Schema", + "required": [ + "pbs", + "moab", + "rsv_query", + "ganglia" + ], + "properties": { + "pbs": { + "$id": "#/properties/v2/properties/custom/properties/pbs", + "type": "object", + "title": "The Pbs Schema", + "required": [ + "host", + "lib", + "bin", + "version" + ], + "properties": { + "host": { + "$id": "#/properties/v2/properties/custom/properties/pbs/properties/host", + "type": "string", + "title": "The Host Schema", + "default": "", + "examples": [ + "owens-batch.ten.osc.edu" + ], + "pattern": "^(.*)$" + }, + "lib": { + "$id": "#/properties/v2/properties/custom/properties/pbs/properties/lib", + "type": "string", + "title": "The Lib Schema", + "default": "", + "examples": [ + "/opt/torque/lib64" + ], + "pattern": "^(.*)$" + }, + "bin": { + "$id": "#/properties/v2/properties/custom/properties/pbs/properties/bin", + "type": "string", + "title": "The Bin Schema", + "default": "", + "examples": [ + "/opt/torque/bin" + ], + "pattern": "^(.*)$" + }, + "version": { + "$id": "#/properties/v2/properties/custom/properties/pbs/properties/version", + "type": "string", + "title": "The Version Schema", + "default": "", + "examples": [ + "6.0.1" + ], + "pattern": "^(.*)$" + } + } + }, + "moab": { + "$id": "#/properties/v2/properties/custom/properties/moab", + "type": "object", + "title": "The Moab Schema", + "required": [ + "host", + "bin", + "version", + "homedir" + ], + "properties": { + "host": { + "$id": "#/properties/v2/properties/custom/properties/moab/properties/host", + "type": "string", + "title": "The Host Schema", + "default": "", + "examples": [ + "owens-batch.ten.osc.edu" + ], + "pattern": "^(.*)$" + }, + "bin": { + "$id": "#/properties/v2/properties/custom/properties/moab/properties/bin", + "type": "string", + "title": "The Bin Schema", + "default": "", + "examples": [ + "/opt/moab/bin" + ], + "pattern": "^(.*)$" + }, + "version": { + "$id": "#/properties/v2/properties/custom/properties/moab/properties/version", + "type": "string", + "title": "The Version Schema", + "default": "", + "examples": [ + "9.0.1" + ], + "pattern": "^(.*)$" + }, + "homedir": { + "$id": "#/properties/v2/properties/custom/properties/moab/properties/homedir", + "type": "string", + "title": "The Homedir Schema", + "default": "", + "examples": [ + "/var/spool/moab" + ], + "pattern": "^(.*)$" + } + } + }, + "rsv_query": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query", + "type": "object", + "title": "The Rsv_query Schema", + "required": [ + "torque_host", + "torque_lib", + "torque_bin", + "torque_version", + "moab_host", + "moab_bin", + "moab_version", + "moab_homedir", + "acls" + ], + "properties": { + "torque_host": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/torque_host", + "type": "string", + "title": "The Torque_host Schema", + "default": "", + "examples": [ + "owens-batch.ten.osc.edu" + ], + "pattern": "^(.*)$" + }, + "torque_lib": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/torque_lib", + "type": "string", + "title": "The Torque_lib Schema", + "default": "", + "examples": [ + "/opt/torque/lib64" + ], + "pattern": "^(.*)$" + }, + "torque_bin": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/torque_bin", + "type": "string", + "title": "The Torque_bin Schema", + "default": "", + "examples": [ + "/opt/torque/bin" + ], + "pattern": "^(.*)$" + }, + "torque_version": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/torque_version", + "type": "string", + "title": "The Torque_version Schema", + "default": "", + "examples": [ + "6.0.1" + ], + "pattern": "^(.*)$" + }, + "moab_host": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/moab_host", + "type": "string", + "title": "The Moab_host Schema", + "default": "", + "examples": [ + "owens-batch.ten.osc.edu" + ], + "pattern": "^(.*)$" + }, + "moab_bin": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/moab_bin", + "type": "string", + "title": "The Moab_bin Schema", + "default": "", + "examples": [ + "/opt/moab/bin" + ], + "pattern": "^(.*)$" + }, + "moab_version": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/moab_version", + "type": "string", + "title": "The Moab_version Schema", + "default": "", + "examples": [ + "9.0.1" + ], + "pattern": "^(.*)$" + }, + "moab_homedir": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/moab_homedir", + "type": "string", + "title": "The Moab_homedir Schema", + "default": "", + "examples": [ + "/var/spool/moab" + ], + "pattern": "^(.*)$" + }, + "acls": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/acls", + "type": "array", + "title": "The Acls Schema", + "items": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/acls/items", + "type": "object", + "title": "The Items Schema", + "required": [ + "adapter", + "groups", + "type" + ], + "properties": { + "adapter": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/acls/items/properties/adapter", + "type": "string", + "title": "The Adapter Schema", + "default": "", + "examples": [ + "group" + ], + "pattern": "^(.*)$" + }, + "groups": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/acls/items/properties/groups", + "type": "array", + "title": "The Groups Schema", + "items": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/acls/items/properties/groups/items", + "type": "string", + "title": "The Items Schema", + "default": "", + "examples": [ + "sysall", + "sappall", + "clntall" + ], + "pattern": "^(.*)$" + } + }, + "type": { + "$id": "#/properties/v2/properties/custom/properties/rsv_query/properties/acls/items/properties/type", + "type": "string", + "title": "The Type Schema", + "default": "", + "examples": [ + "blacklist" + ], + "pattern": "^(.*)$" + } + } + } + } + } + }, + "ganglia": { + "$id": "#/properties/v2/properties/custom/properties/ganglia", + "type": "object", + "title": "The Ganglia Schema", + "required": [ + "host", + "scheme", + "segments", + "req_query", + "opt_query", + "version" + ], + "properties": { + "host": { + "$id": "#/properties/v2/properties/custom/properties/ganglia/properties/host", + "type": "string", + "title": "The Host Schema", + "default": "", + "examples": [ + "ganglia.osc.edu" + ], + "pattern": "^(.*)$" + }, + "scheme": { + "$id": "#/properties/v2/properties/custom/properties/ganglia/properties/scheme", + "type": "string", + "title": "The Scheme Schema", + "default": "", + "examples": [ + "https://" + ], + "pattern": "^(.*)$" + }, + "segments": { + "$id": "#/properties/v2/properties/custom/properties/ganglia/properties/segments", + "type": "array", + "title": "The Segments Schema", + "items": { + "$id": "#/properties/v2/properties/custom/properties/ganglia/properties/segments/items", + "type": "string", + "title": "The Items Schema", + "default": "", + "examples": [ + "graph.php" + ], + "pattern": "^(.*)$" + } + }, + "req_query": { + "$id": "#/properties/v2/properties/custom/properties/ganglia/properties/req_query", + "type": "object", + "title": "The Req_query Schema", + "required": [ + "c" + ], + "properties": { + "c": { + "$id": "#/properties/v2/properties/custom/properties/ganglia/properties/req_query/properties/c", + "type": "string", + "title": "The C Schema", + "default": "", + "examples": [ + "Owens" + ], + "pattern": "^(.*)$" + } + } + }, + "opt_query": { + "$id": "#/properties/v2/properties/custom/properties/ganglia/properties/opt_query", + "type": "object", + "title": "The Opt_query Schema", + "required": [ + "h" + ], + "properties": { + "h": { + "$id": "#/properties/v2/properties/custom/properties/ganglia/properties/opt_query/properties/h", + "type": "string", + "title": "The H Schema", + "default": "", + "examples": [ + "%{h}.ten.osc.edu" + ], + "pattern": "^(.*)$" + } + } + }, + "version": { + "$id": "#/properties/v2/properties/custom/properties/ganglia/properties/version", + "type": "string", + "title": "The Version Schema", + "default": "", + "examples": [ + "3" + ], + "pattern": "^(.*)$" + } + } + } + } + }, + "batch_connect": { + "$id": "#/properties/v2/properties/batch_connect", + "type": "object", + "title": "The Batch_connect Schema", + "required": [ + "basic", + "vnc" + ], + "properties": { + "basic": { + "$id": "#/properties/v2/properties/batch_connect/properties/basic", + "type": "object", + "title": "The Basic Schema", + "required": [ + "script_wrapper" + ], + "properties": { + "script_wrapper": { + "$id": "#/properties/v2/properties/batch_connect/properties/basic/properties/script_wrapper", + "type": "string", + "title": "The Script_wrapper Schema", + "default": "", + "examples": [ + "module restore\n%s" + ], + "pattern": "^(.*)$" + } + } + }, + "vnc": { + "$id": "#/properties/v2/properties/batch_connect/properties/vnc", + "type": "object", + "title": "The Vnc Schema", + "required": [ + "script_wrapper" + ], + "properties": { + "script_wrapper": { + "$id": "#/properties/v2/properties/batch_connect/properties/vnc/properties/script_wrapper", + "type": "string", + "title": "The Script_wrapper Schema", + "default": "", + "examples": [ + "module restore\nmodule load ondemand-vnc\n%s" + ], + "pattern": "^(.*)$" + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/lib/tasks/quick_config_schema.json b/lib/tasks/quick_config_schema.json new file mode 100644 index 00000000..8ad37f09 --- /dev/null +++ b/lib/tasks/quick_config_schema.json @@ -0,0 +1,242 @@ +{ + "description": "Config schema for OSC quick clusters", + "type": "object", + "title": "The Config Schema", + "required": [ + "v2" + ], + "properties": { + "v2": { + "$id": "#/properties/v2", + "type": "object", + "title": "The V2 Schema", + "required": [ + "metadata", + "job", + "custom", + "batch_connect" + ], + "properties": { + "metadata": { + "$id": "#/properties/v2/properties/metadata", + "type": "object", + "title": "The Metadata Schema", + "required": [ + "title", + "url", + "hidden" + ], + "properties": { + "title": { + "$id": "#/properties/v2/properties/metadata/properties/title", + "type": "string", + "title": "The Title Schema", + "default": "", + "examples": [ + "Quick" + ], + "pattern": "^(.*)$" + }, + "url": { + "$id": "#/properties/v2/properties/metadata/properties/url", + "type": "string", + "title": "The Url Schema", + "default": "", + "examples": [ + "" + ], + "pattern": "^(.*)$" + }, + "hidden": { + "$id": "#/properties/v2/properties/metadata/properties/hidden", + "type": "boolean", + "title": "The Hidden Schema", + "default": false, + "examples": [ + true + ] + } + } + }, + "job": { + "$id": "#/properties/v2/properties/job", + "type": "object", + "title": "The Job Schema", + "required": [ + "adapter", + "host", + "lib", + "bin", + "version" + ], + "properties": { + "adapter": { + "$id": "#/properties/v2/properties/job/properties/adapter", + "type": "string", + "title": "The Adapter Schema", + "default": "", + "examples": [ + "torque" + ], + "pattern": "^(.*)$" + }, + "host": { + "$id": "#/properties/v2/properties/job/properties/host", + "type": "string", + "title": "The Host Schema", + "default": "", + "examples": [ + "quick-batch.ten.osc.edu" + ], + "pattern": "^(.*)$" + }, + "lib": { + "$id": "#/properties/v2/properties/job/properties/lib", + "type": "string", + "title": "The Lib Schema", + "default": "", + "examples": [ + "/opt/torque/lib64" + ], + "pattern": "^(.*)$" + }, + "bin": { + "$id": "#/properties/v2/properties/job/properties/bin", + "type": "string", + "title": "The Bin Schema", + "default": "", + "examples": [ + "/opt/torque/bin" + ], + "pattern": "^(.*)$" + }, + "version": { + "$id": "#/properties/v2/properties/job/properties/version", + "type": "string", + "title": "The Version Schema", + "default": "", + "examples": [ + "6.0.2" + ], + "pattern": "^(.*)$" + } + } + }, + "custom": { + "$id": "#/properties/v2/properties/custom", + "type": "object", + "title": "The Custom Schema", + "required": [ + "pbs" + ], + "properties": { + "pbs": { + "$id": "#/properties/v2/properties/custom/properties/pbs", + "type": "object", + "title": "The Pbs Schema", + "required": [ + "host", + "lib", + "bin", + "version" + ], + "properties": { + "host": { + "$id": "#/properties/v2/properties/custom/properties/pbs/properties/host", + "type": "string", + "title": "The Host Schema", + "default": "", + "examples": [ + "quick-batch.ten.osc.edu" + ], + "pattern": "^(.*)$" + }, + "lib": { + "$id": "#/properties/v2/properties/custom/properties/pbs/properties/lib", + "type": "string", + "title": "The Lib Schema", + "default": "", + "examples": [ + "/opt/torque/lib64" + ], + "pattern": "^(.*)$" + }, + "bin": { + "$id": "#/properties/v2/properties/custom/properties/pbs/properties/bin", + "type": "string", + "title": "The Bin Schema", + "default": "", + "examples": [ + "/opt/torque/bin" + ], + "pattern": "^(.*)$" + }, + "version": { + "$id": "#/properties/v2/properties/custom/properties/pbs/properties/version", + "type": "string", + "title": "The Version Schema", + "default": "", + "examples": [ + "6.0.2" + ], + "pattern": "^(.*)$" + } + } + } + } + }, + "batch_connect": { + "$id": "#/properties/v2/properties/batch_connect", + "type": "object", + "title": "The Batch_connect Schema", + "required": [ + "basic", + "vnc" + ], + "properties": { + "basic": { + "$id": "#/properties/v2/properties/batch_connect/properties/basic", + "type": "object", + "title": "The Basic Schema", + "required": [ + "script_wrapper" + ], + "properties": { + "script_wrapper": { + "$id": "#/properties/v2/properties/batch_connect/properties/basic/properties/script_wrapper", + "type": "string", + "title": "The Script_wrapper Schema", + "default": "", + "examples": [ + "module restore\n%s" + ], + "pattern": "^(.*)$" + } + } + }, + "vnc": { + "$id": "#/properties/v2/properties/batch_connect/properties/vnc", + "type": "object", + "title": "The Vnc Schema", + "required": [ + "script_wrapper" + ], + "properties": { + "script_wrapper": { + "$id": "#/properties/v2/properties/batch_connect/properties/vnc/properties/script_wrapper", + "type": "string", + "title": "The Script_wrapper Schema", + "default": "", + "examples": [ + "module restore\nmodule load ondemand-vnc\n%s" + ], + "pattern": "^(.*)$" + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/lib/tasks/validate.rake b/lib/tasks/validate.rake new file mode 100644 index 00000000..a4abe2ed --- /dev/null +++ b/lib/tasks/validate.rake @@ -0,0 +1,91 @@ +namespace :validate do + namespace :clusters do + task :all => Pathname.glob((Pathname.new(ENV['OOD_CLUSTERS'] || '/etc/ood/config/clusters.d').expand_path).join("*.yml")).map{|c| (c.to_s).to_sym } + + Pathname.glob((Pathname.new(ENV['OOD_CLUSTERS'] || '/etc/ood/config/clusters.d').expand_path).join("*.yml")).each do |config| + desc "Validate the cluster config: #{config}" + task (config.to_s).to_sym do + config_json = YAML.safe_load(config.read).to_json + puts "Validating cluster config: #{config}..." + if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') + schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) + else + schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) + end + + if !JSON::Validator.validate(schema, config_json) + puts "Test for '#{config}' FAILED!" + puts JSON::Validator.fully_validate(schema, config_json) + end + + puts "Finished validating cluster config: #{config}" + end + end + + # task :all, [:filepath] do |t, args| + # puts args[:filepath] + # if args[:filepath] + # config = Pathname.new(args[:filepath].to_s).expand_path + # if config.file? + # desc "Validate the cluster config: #{config}" + # config_json = YAML.safe_load(config.read).to_json + # puts "Validating cluster config: #{config}..." + # if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') + # schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) + # else + # schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) + # end + + # if !JSON::Validator.validate(schema, config_json) + # puts "Test for '#{config}' FAILED!" + # puts JSON::Validator.fully_validate(schema, config_json) + # end + + # puts "Finished validating cluster config: #{config}" + + # elsif config.directory? + # Pathname.glob(config.join("*.yml")).each do |p| + # desc "Validate the cluster config: #{config}" + # config_json = YAML.safe_load(p.read).to_json + # puts "Validating cluster config: #{config}..." + # if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') + # schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) + # else + # schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) + # end + + # if !JSON::Validator.validate(schema, config_json) + # puts "Test for '#{config}' FAILED!" + # puts JSON::Validator.fully_validate(schema, config_json) + # end + + # puts "Finished validating cluster config: #{config}" + # end + # else + # puts "configuration file '#{config}' does not exist" + # end + # else + # desc "Test all clusters" + # Pathname.glob((Pathname.new(ENV['OOD_CLUSTERS'] || '/etc/ood/config/clusters.d').expand_path).join("*.yml")).each do |config| + # desc "Validate the cluster config: #{config}" + # config_json = YAML.safe_load(config.read).to_json + # puts "Validating cluster config: #{config}..." + # if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') + # schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) + # else + # schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) + # end + + # if !JSON::Validator.validate(schema, config_json) + # puts "Test for '#{config}' FAILED!" + # puts JSON::Validator.fully_validate(schema, config_json) + # end + + # puts "Finished validating cluster config: #{config}" + # end + # end + # end + end + desc "Validate all cluster configs" + task :clusters => "clusters:all" +end From 98eaca102b63c064fb5c47d45eb705846eed952f Mon Sep 17 00:00:00 2001 From: AriettaZ Date: Tue, 25 Sep 2018 16:30:48 -0400 Subject: [PATCH 02/11] raise error and add option argument --- lib/tasks/validate.rake | 157 +++++++++++++++++++--------------------- 1 file changed, 75 insertions(+), 82 deletions(-) diff --git a/lib/tasks/validate.rake b/lib/tasks/validate.rake index a4abe2ed..2f159425 100644 --- a/lib/tasks/validate.rake +++ b/lib/tasks/validate.rake @@ -1,91 +1,84 @@ namespace :validate do - namespace :clusters do - task :all => Pathname.glob((Pathname.new(ENV['OOD_CLUSTERS'] || '/etc/ood/config/clusters.d').expand_path).join("*.yml")).map{|c| (c.to_s).to_sym } - - Pathname.glob((Pathname.new(ENV['OOD_CLUSTERS'] || '/etc/ood/config/clusters.d').expand_path).join("*.yml")).each do |config| - desc "Validate the cluster config: #{config}" - task (config.to_s).to_sym do - config_json = YAML.safe_load(config.read).to_json - puts "Validating cluster config: #{config}..." - if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') - schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) + namespace :clusters do + task :selector , [:filepath] do |task, args| + is_failed = false + if !args.filepath + Pathname.glob((Pathname.new(ENV['OOD_CLUSTERS'] || '/etc/ood/config/clusters.d').expand_path).join("*.yml")).map{|c| c.to_s }.each do |file| + begin + Rake::Task["validate:clusters:#{file}"].invoke + rescue + is_failed = true + next + end + end else - schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) + config = Pathname.new(args[:filepath].to_s).expand_path + if config.file? + desc "Validate the cluster config: #{config}" + config_json = YAML.safe_load(config.read).to_json + puts "Validating cluster config: #{config}..." + if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') + schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) + else + schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) + end + + if !JSON::Validator.validate(schema, config_json) + puts "Test for '#{config}' FAILED!" + puts JSON::Validator.fully_validate(schema, config_json) + is_failed = true + end + + puts "Finished validating cluster config: #{config}" + + elsif config.directory? + Pathname.glob(config.join("*.yml")).each do |p| + desc "Validate the cluster config: #{config}" + config_json = YAML.safe_load(p.read).to_json + puts "Validating cluster config: #{config}..." + if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') + schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) + else + schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) + end + + if !JSON::Validator.validate(schema, config_json) + puts "Test for '#{config}' FAILED!" + puts JSON::Validator.fully_validate(schema, config_json) + is_failed = true + end + + puts "Finished validating cluster config: #{config}" + end + else + puts "configuration file '#{config}' does not exist" + end end - - if !JSON::Validator.validate(schema, config_json) - puts "Test for '#{config}' FAILED!" - puts JSON::Validator.fully_validate(schema, config_json) - end - - puts "Finished validating cluster config: #{config}" + raise 'One or more cluster configurations are invalid.' if is_failed end - end - - # task :all, [:filepath] do |t, args| - # puts args[:filepath] - # if args[:filepath] - # config = Pathname.new(args[:filepath].to_s).expand_path - # if config.file? - # desc "Validate the cluster config: #{config}" - # config_json = YAML.safe_load(config.read).to_json - # puts "Validating cluster config: #{config}..." - # if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') - # schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) - # else - # schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) - # end - - # if !JSON::Validator.validate(schema, config_json) - # puts "Test for '#{config}' FAILED!" - # puts JSON::Validator.fully_validate(schema, config_json) - # end - - # puts "Finished validating cluster config: #{config}" - - # elsif config.directory? - # Pathname.glob(config.join("*.yml")).each do |p| - # desc "Validate the cluster config: #{config}" - # config_json = YAML.safe_load(p.read).to_json - # puts "Validating cluster config: #{config}..." - # if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') - # schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) - # else - # schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) - # end + + Pathname.glob((Pathname.new(ENV['OOD_CLUSTERS'] || '/etc/ood/config/clusters.d').expand_path).join("*.yml")).each do |config| + desc "Validate the cluster config: #{config}" + task (config.to_s).to_sym do + config_json = YAML.safe_load(config.read).to_json + puts "Validating cluster config: #{config}..." + if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') + schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) + else + schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) + end - # if !JSON::Validator.validate(schema, config_json) - # puts "Test for '#{config}' FAILED!" - # puts JSON::Validator.fully_validate(schema, config_json) - # end + if !JSON::Validator.validate(schema, config_json) + puts "Test for '#{config}' FAILED!" + puts JSON::Validator.fully_validate(schema, config_json) + raise 'One or more cluster configurations are invalid.' + end - # puts "Finished validating cluster config: #{config}" - # end - # else - # puts "configuration file '#{config}' does not exist" - # end - # else - # desc "Test all clusters" - # Pathname.glob((Pathname.new(ENV['OOD_CLUSTERS'] || '/etc/ood/config/clusters.d').expand_path).join("*.yml")).each do |config| - # desc "Validate the cluster config: #{config}" - # config_json = YAML.safe_load(config.read).to_json - # puts "Validating cluster config: #{config}..." - # if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') - # schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) - # else - # schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) - # end - - # if !JSON::Validator.validate(schema, config_json) - # puts "Test for '#{config}' FAILED!" - # puts JSON::Validator.fully_validate(schema, config_json) - # end - - # puts "Finished validating cluster config: #{config}" - # end - # end - # end + puts "Finished validating cluster config: #{config}" + end + end + end desc "Validate all cluster configs" - task :clusters => "clusters:all" + task :clusters, [:filepath] => "clusters:selector" end From a9691848697b04b111cefa75eb36cd940afbf904 Mon Sep 17 00:00:00 2001 From: AriettaZ Date: Thu, 27 Sep 2018 16:29:50 -0400 Subject: [PATCH 03/11] add validate before testing jobs --- .env | 1 + lib/tasks/test.rake | 3 +- lib/tasks/test.yml | 115 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 .env create mode 100644 lib/tasks/test.yml diff --git a/.env b/.env new file mode 100644 index 00000000..3b24097a --- /dev/null +++ b/.env @@ -0,0 +1 @@ +OOD_BC_APP_CONFIG_ROOT="/users/PZS0002/azhu/ondemand/dev/osc-ood-config/ondemand.osc.edu/apps" diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index 67bc20c3..70ed2092 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -8,7 +8,7 @@ namespace :test do OodAppkit.clusters.each do |cluster| desc "Test the cluster: #{cluster.id}" - task cluster.id => [:environment, WORKDIR] do + task cluster.id => ["validate:clusters:/etc/ood/config/clusters.d/#{cluster.id}.yml", :environment, WORKDIR] do unless cluster.job_allow? puts "Skipping '#{cluster.id}' as it doesn't allow job submission." next @@ -59,3 +59,4 @@ namespace :test do desc "Test all clusters" task :jobs => "jobs:all" end + diff --git a/lib/tasks/test.yml b/lib/tasks/test.yml new file mode 100644 index 00000000..4628fa37 --- /dev/null +++ b/lib/tasks/test.yml @@ -0,0 +1,115 @@ +--- +v1: + title: "Pitzer" + url: "https://www.osc.edu/supercomputing/computing/pitzer" + validators: + cluster: + - type: "OodAppkit::Validators::Groups" + data: + groups: + - "pitzerc" + allow: true + rsv_query: + - type: "OodAppkit::Validators::Groups" + data: + groups: + - "sysall" + - "sappall" + - "clntall" + allow: false + cluster: + type: "OodCluster::Cluster" + data: + hpc_cluster: true + servers: + login: + type: "OodCluster::Servers::Ssh" + data: + host: "pitzer.osc.edu" + resource_mgr: + type: "OodCluster::Servers::Torque" + data: + host: "pitzer-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + scheduler: + type: "OodCluster::Servers::Moab" + data: + host: "pitzer-batch.ten.osc.edu" + bin: "/opt/moab/bin" + version: "9.0.1" + moabhomedir: "/var/spool/moab" + ganglia: + type: "OodCluster::Servers::Ganglia" + data: + host: "ganglia.osc.edu" + scheme: "https://" + segments: + - "graph.php" + req_query: + c: "Pitzer" + opt_query: + h: "%{h}.ten.osc.edu" + version: "3" + +v2: + metadata: + title: "Pitzer" + url: "https://www.osc.edu/supercomputing/computing/pitzer" + hidden: false + acls: + - adapter: "group" + groups: + - "pitzerc" + type: "whitelist" + login: + host: "pitzer.osc.edu" + job: + adapter: "torque" + host: "pitzer-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + custom: + pbs: + host: "pitzer-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + moab: + host: "pitzer-batch.ten.osc.edu" + bin: "/opt/moab/bin" + version: "9.0.1" + homedir: "/var/spool/moab" + rsv_query: + torque_host: "pitzer-batch.ten.osc.edu" + torque_lib: "/opt/torque/lib64" + torque_bin: "/opt/torque/bin" + torque_version: "6.0.1" + moab_host: "pitzer-batch.ten.osc.edu" + moab_bin: "/opt/moab/bin" + moab_version: "9.0.1" + moab_homedir: "/var/spool/moab" + acls: + - adapter: "group" + groups: + - "sysall" + - "sappall" + - "clntall" + type: "blacklist" + ganga: + host: "ganglia.osc.edu" + scheme: "https://" + segments: + - "graph.php" + req_query: + c: "Pitzer" + opt_query: + h: "%{h}.ten.osc.edu" + version: "3" + batch_connect: + basic: + script_wrapper: "module restore\n%s" + vnc: + script_wrapper: "module restore\nmodule load ondemand-vnc\n%s" \ No newline at end of file From de089daf087bfcca3ea67827c7bbd45c1dc6ee6d Mon Sep 17 00:00:00 2001 From: AriettaZ Date: Thu, 27 Sep 2018 16:37:52 -0400 Subject: [PATCH 04/11] remove env file --- .env | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .env diff --git a/.env b/.env deleted file mode 100644 index 3b24097a..00000000 --- a/.env +++ /dev/null @@ -1 +0,0 @@ -OOD_BC_APP_CONFIG_ROOT="/users/PZS0002/azhu/ondemand/dev/osc-ood-config/ondemand.osc.edu/apps" From aabee7c9c0ed5aafeedf3c6ffe8148a9de859f7b Mon Sep 17 00:00:00 2001 From: AriettaZ Date: Fri, 28 Sep 2018 11:42:32 -0400 Subject: [PATCH 05/11] handle environment variable --- lib/tasks/test.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index 70ed2092..72de898e 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -8,7 +8,7 @@ namespace :test do OodAppkit.clusters.each do |cluster| desc "Test the cluster: #{cluster.id}" - task cluster.id => ["validate:clusters:/etc/ood/config/clusters.d/#{cluster.id}.yml", :environment, WORKDIR] do + task cluster.id => ["validate:clusters:#{(Pathname.new(ENV['OOD_CLUSTERS'] || "/etc/ood/config/clusters.d/").expand_path).join("#{cluster.id}.yml")}", :environment, WORKDIR] do unless cluster.job_allow? puts "Skipping '#{cluster.id}' as it doesn't allow job submission." next From 070be3fe7b0d1e02c62aac953024a69f7b4944f6 Mon Sep 17 00:00:00 2001 From: AriettaZ Date: Fri, 28 Sep 2018 11:47:11 -0400 Subject: [PATCH 06/11] remove test.yml --- lib/tasks/test.yml | 115 --------------------------------------------- 1 file changed, 115 deletions(-) delete mode 100644 lib/tasks/test.yml diff --git a/lib/tasks/test.yml b/lib/tasks/test.yml deleted file mode 100644 index 4628fa37..00000000 --- a/lib/tasks/test.yml +++ /dev/null @@ -1,115 +0,0 @@ ---- -v1: - title: "Pitzer" - url: "https://www.osc.edu/supercomputing/computing/pitzer" - validators: - cluster: - - type: "OodAppkit::Validators::Groups" - data: - groups: - - "pitzerc" - allow: true - rsv_query: - - type: "OodAppkit::Validators::Groups" - data: - groups: - - "sysall" - - "sappall" - - "clntall" - allow: false - cluster: - type: "OodCluster::Cluster" - data: - hpc_cluster: true - servers: - login: - type: "OodCluster::Servers::Ssh" - data: - host: "pitzer.osc.edu" - resource_mgr: - type: "OodCluster::Servers::Torque" - data: - host: "pitzer-batch.ten.osc.edu" - lib: "/opt/torque/lib64" - bin: "/opt/torque/bin" - version: "6.0.1" - scheduler: - type: "OodCluster::Servers::Moab" - data: - host: "pitzer-batch.ten.osc.edu" - bin: "/opt/moab/bin" - version: "9.0.1" - moabhomedir: "/var/spool/moab" - ganglia: - type: "OodCluster::Servers::Ganglia" - data: - host: "ganglia.osc.edu" - scheme: "https://" - segments: - - "graph.php" - req_query: - c: "Pitzer" - opt_query: - h: "%{h}.ten.osc.edu" - version: "3" - -v2: - metadata: - title: "Pitzer" - url: "https://www.osc.edu/supercomputing/computing/pitzer" - hidden: false - acls: - - adapter: "group" - groups: - - "pitzerc" - type: "whitelist" - login: - host: "pitzer.osc.edu" - job: - adapter: "torque" - host: "pitzer-batch.ten.osc.edu" - lib: "/opt/torque/lib64" - bin: "/opt/torque/bin" - version: "6.0.1" - custom: - pbs: - host: "pitzer-batch.ten.osc.edu" - lib: "/opt/torque/lib64" - bin: "/opt/torque/bin" - version: "6.0.1" - moab: - host: "pitzer-batch.ten.osc.edu" - bin: "/opt/moab/bin" - version: "9.0.1" - homedir: "/var/spool/moab" - rsv_query: - torque_host: "pitzer-batch.ten.osc.edu" - torque_lib: "/opt/torque/lib64" - torque_bin: "/opt/torque/bin" - torque_version: "6.0.1" - moab_host: "pitzer-batch.ten.osc.edu" - moab_bin: "/opt/moab/bin" - moab_version: "9.0.1" - moab_homedir: "/var/spool/moab" - acls: - - adapter: "group" - groups: - - "sysall" - - "sappall" - - "clntall" - type: "blacklist" - ganga: - host: "ganglia.osc.edu" - scheme: "https://" - segments: - - "graph.php" - req_query: - c: "Pitzer" - opt_query: - h: "%{h}.ten.osc.edu" - version: "3" - batch_connect: - basic: - script_wrapper: "module restore\n%s" - vnc: - script_wrapper: "module restore\nmodule load ondemand-vnc\n%s" \ No newline at end of file From 48b05876b0290763248fd2774ded23e438102f74 Mon Sep 17 00:00:00 2001 From: AriettaZ Date: Tue, 27 Nov 2018 16:48:48 -0500 Subject: [PATCH 07/11] handle syntax error --- lib/tasks/validate.rake | 111 ++++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 44 deletions(-) diff --git a/lib/tasks/validate.rake b/lib/tasks/validate.rake index 2f159425..3b24b524 100644 --- a/lib/tasks/validate.rake +++ b/lib/tasks/validate.rake @@ -15,27 +15,74 @@ namespace :validate do config = Pathname.new(args[:filepath].to_s).expand_path if config.file? desc "Validate the cluster config: #{config}" - config_json = YAML.safe_load(config.read).to_json - puts "Validating cluster config: #{config}..." - if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') - schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) + begin + puts "Validating cluster config: #{config}..." + config_json = YAML.safe_load(config.read).to_json + rescue Psych::SyntaxError => e + puts "Test for '#{config}' FAILED!" + puts "Psych::SyntaxError: #{e}" + is_failed = true else - schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) - end - - if !JSON::Validator.validate(schema, config_json) - puts "Test for '#{config}' FAILED!" - puts JSON::Validator.fully_validate(schema, config_json) - is_failed = true - end - - puts "Finished validating cluster config: #{config}" - + if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') + schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) + else + schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) + end + + if !JSON::Validator.validate(schema, config_json) + puts "Test for '#{config}' FAILED!" + puts JSON::Validator.fully_validate(schema, config_json) + is_failed = true + end + ensure + puts "Finished validating cluster config: #{config}" + puts + end elsif config.directory? - Pathname.glob(config.join("*.yml")).each do |p| - desc "Validate the cluster config: #{config}" - config_json = YAML.safe_load(p.read).to_json + Pathname.glob(config.join("*.yml")).each do |f| + begin + puts "Validating cluster config: #{f}..." + config_json = YAML.safe_load(f.read).to_json + rescue Psych::SyntaxError => e + puts "Test for '#{f}' FAILED!" + puts "Psych::SyntaxError: #{e}" + is_failed = true + else + if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') + schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) + else + schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) + end + + if !JSON::Validator.validate(schema, config_json) + puts "Test for '#{f}' FAILED!" + puts JSON::Validator.fully_validate(schema, config_json) + is_failed = true + end + ensure + puts "Finished validating cluster config: #{f}" + puts + end + end + else + puts "configuration file '#{config}' does not exist" + puts + end + end + raise 'One or more cluster configurations are invalid.' if is_failed + end + + Pathname.glob((Pathname.new(ENV['OOD_CLUSTERS'] || '/etc/ood/config/clusters.d').expand_path).join("*.yml")).each do |config| + desc "Validate the cluster config: #{config}" + task (config.to_s).to_sym do + begin puts "Validating cluster config: #{config}..." + config_json = YAML.safe_load(config.read).to_json + rescue Psych::SyntaxError => e + puts "Test for '#{config}' FAILED!" + puts "Psych::SyntaxError: #{e}" + is_failed = true + else if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) else @@ -47,35 +94,11 @@ namespace :validate do puts JSON::Validator.fully_validate(schema, config_json) is_failed = true end - + ensure puts "Finished validating cluster config: #{config}" - end - else - puts "configuration file '#{config}' does not exist" + puts end end - raise 'One or more cluster configurations are invalid.' if is_failed - end - - Pathname.glob((Pathname.new(ENV['OOD_CLUSTERS'] || '/etc/ood/config/clusters.d').expand_path).join("*.yml")).each do |config| - desc "Validate the cluster config: #{config}" - task (config.to_s).to_sym do - config_json = YAML.safe_load(config.read).to_json - puts "Validating cluster config: #{config}..." - if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') - schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) - else - schema = File.read((File.dirname(__FILE__) +'/config_schema.json')) - end - - if !JSON::Validator.validate(schema, config_json) - puts "Test for '#{config}' FAILED!" - puts JSON::Validator.fully_validate(schema, config_json) - raise 'One or more cluster configurations are invalid.' - end - - puts "Finished validating cluster config: #{config}" - end end end From 293032b344dc4103edf86b8d393c1465fd3c096a Mon Sep 17 00:00:00 2001 From: AriettaZ Date: Tue, 27 Nov 2018 16:59:30 -0500 Subject: [PATCH 08/11] catch other error when loading yaml --- lib/tasks/validate.rake | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/tasks/validate.rake b/lib/tasks/validate.rake index 3b24b524..bd87f037 100644 --- a/lib/tasks/validate.rake +++ b/lib/tasks/validate.rake @@ -22,6 +22,10 @@ namespace :validate do puts "Test for '#{config}' FAILED!" puts "Psych::SyntaxError: #{e}" is_failed = true + rescue + puts "Test for '#{config}' FAILED!" + puts "Failed to load Yaml file" + is_failed = true else if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) @@ -47,6 +51,10 @@ namespace :validate do puts "Test for '#{f}' FAILED!" puts "Psych::SyntaxError: #{e}" is_failed = true + rescue + puts "Test for '#{config}' FAILED!" + puts "Failed to load Yaml file" + is_failed = true else if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) @@ -82,6 +90,10 @@ namespace :validate do puts "Test for '#{config}' FAILED!" puts "Psych::SyntaxError: #{e}" is_failed = true + rescue + puts "Test for '#{config}' FAILED!" + puts "Failed to load Yaml file" + is_failed = true else if (JSON.parse(config_json)["v2"]["metadata"]["title"]).downcase.include?('quick') schema = File.read((File.dirname(__FILE__) +'/quick_config_schema.json')) From 717167923ed5688e8a9a6e0e72b8ab78466c96c1 Mon Sep 17 00:00:00 2001 From: AriettaZ Date: Tue, 27 Nov 2018 17:12:33 -0500 Subject: [PATCH 09/11] update gems --- lib/tasks/clusters.d/oakley.yml | 104 +++++++++++++++++++++++ lib/tasks/clusters.d/owens.yml | 104 +++++++++++++++++++++++ lib/tasks/clusters.d/pitzer.yml | 115 ++++++++++++++++++++++++++ lib/tasks/clusters.d/quick.yml | 28 +++++++ lib/tasks/clusters.d/quick_pitzer.yml | 51 ++++++++++++ lib/tasks/clusters.d/quick_ruby.yml | 51 ++++++++++++ lib/tasks/clusters.d/ruby.yml | 115 ++++++++++++++++++++++++++ 7 files changed, 568 insertions(+) create mode 100644 lib/tasks/clusters.d/oakley.yml create mode 100644 lib/tasks/clusters.d/owens.yml create mode 100644 lib/tasks/clusters.d/pitzer.yml create mode 100644 lib/tasks/clusters.d/quick.yml create mode 100644 lib/tasks/clusters.d/quick_pitzer.yml create mode 100644 lib/tasks/clusters.d/quick_ruby.yml create mode 100644 lib/tasks/clusters.d/ruby.yml diff --git a/lib/tasks/clusters.d/oakley.yml b/lib/tasks/clusters.d/oakley.yml new file mode 100644 index 00000000..cf5d9da6 --- /dev/null +++ b/lib/tasks/clusters.d/oakley.yml @@ -0,0 +1,104 @@ +--- +v1: + title: "Oakley" + url: "https://www.osc.edu/supercomputing/computing/oakley" + validators: + rsv_query: + - type: "OodAppkit::Validators::Groups" + data: + groups: + - "sysall" + - "sappall" + - "clntall" + allow: false + cluster: + type: "OodCluster::Cluster" + data: + hpc_cluster: true + servers: + login: + type: "OodCluster::Servers::Ssh" + data: + host: "oakley.osc.edu" + resource_mgr: + type: "OodCluster::Servers::Torque" + data: + host: "oak-batch.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + scheduler: + type: "OodCluster::Servers::Moab" + data: + host: "oak-batch.osc.edu" + bin: "/opt/moab/bin" + version: "9.0.1" + moabhomedir: "/var/spool/moab" + ganglia: + type: "OodCluster::Servers::Ganglia" + data: + host: "ganglia.osc.edu" + scheme: "https://" + segments: + - "graph.php" + req_query: + c: "Oakley" + opt_query: + h: "%{h}.ten.osc.edu" + version: "3" + +v2: + metadata: + title: "Oakley" + url: "https://www.osc.edu/supercomputing/computing/oakley" + hidden: false + login: + host: "oakley.osc.edu" + job: + adapter: "torque" + host: "oak-batch.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + custom: + pbs: + host: "oak-batch.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + moab: + host: "oak-batch.osc.edu" + bin: "/opt/moab/bin" + version: "9.0.1" + homedir: "/var/spool/moab" + rsv_query: + torque_host: "oak-batch.osc.edu" + torque_lib: "/opt/torque/lib64" + torque_bin: "/opt/torque/bin" + torque_version: "6.0.1" + moab_host: "oak-batch.osc.edu" + moab_bin: "/opt/moab/bin" + moab_version: "9.0.1" + moab_homedir: "/var/spool/moab" + acls: + - adapter: "group" + groups: + - "sysall" + - "sappall" + - "clntall" + type: "blacklist" + ganglia: + host: "ganglia.osc.edu" + scheme: "https://" + segments: + - "graph.php" + req_query: + c: "Oakley" + opt_query: + h: "%{h}.ten.osc.edu" + version: "3" + batch_connect: + basic: + script_wrapper: "module restore\n%s" + vnc: + script_wrapper: "module restore\nmodule load ondemand-vnc\n%s" diff --git a/lib/tasks/clusters.d/owens.yml b/lib/tasks/clusters.d/owens.yml new file mode 100644 index 00000000..f33c24b8 --- /dev/null +++ b/lib/tasks/clusters.d/owens.yml @@ -0,0 +1,104 @@ +--- +v1: + title: "Owens" + url: "https://www.osc.edu/supercomputing/computing/owens" + validators: + rsv_query: + - type: "OodAppkit::Validators::Groups" + data: + groups: + - "sysall" + - "sappall" + - "clntall" + allow: false + cluster: + type: "OodCluster::Cluster" + data: + hpc_cluster: true + servers: + login: + type: "OodCluster::Servers::Ssh" + data: + host: "owens.osc.edu" + resource_mgr: + type: "OodCluster::Servers::Torque" + data: + host: "owens-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + scheduler: + type: "OodCluster::Servers::Moab" + data: + host: "owens-batch.ten.osc.edu" + bin: "/opt/moab/bin" + version: "9.0.1" + moabhomedir: "/var/spool/moab" + ganglia: + type: "OodCluster::Servers::Ganglia" + data: + host: "ganglia.osc.edu" + scheme: "https://" + segments: + - "graph.php" + req_query: + c: "Owens" + opt_query: + h: "%{h}.ten.osc.edu" + version: "3" + +v2: + metadata: + title: "Owens" + url: "https://www.osc.edu/supercomputing/computing/owens" + hidden: false + login: + host: "owens.osc.edu" + job: + adapter: "torque" + host: "owens-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + custom: + pbs: + host: "owens-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + moab: + host: "owens-batch.ten.osc.edu" + bin: "/opt/moab/bin" + version: "9.0.1" + homedir: "/var/spool/moab" + rsv_query: + torque_host: "owens-batch.ten.osc.edu" + torque_lib: "/opt/torque/lib64" + torque_bin: "/opt/torque/bin" + torque_version: "6.0.1" + moab_host: "owens-batch.ten.osc.edu" + moab_bin: "/opt/moab/bin" + moab_version: "9.0.1" + moab_homedir: "/var/spool/moab" + acls: + - adapter: "group" + groups: + - "sysall" + - "sappall" + - "clntall" + type: "blacklist" + ganglia: + host: "ganglia.osc.edu" + scheme: "https://" + segments: + - "graph.php" + req_query: + c: "Owens" + opt_query: + h: "%{h}.ten.osc.edu" + version: "3" + batch_connect: + basic: + script_wrapper: "module restore\n%s" + vnc: + script_wrapper: "module restore\nmodule load ondemand-vnc\n%s" diff --git a/lib/tasks/clusters.d/pitzer.yml b/lib/tasks/clusters.d/pitzer.yml new file mode 100644 index 00000000..f18df0be --- /dev/null +++ b/lib/tasks/clusters.d/pitzer.yml @@ -0,0 +1,115 @@ +--- +v1: + title: "Pitzer" + url: "https://www.osc.edu/supercomputing/computing/pitzer" + validators: + cluster: + - type: "OodAppkit::Validators::Groups" + data: + groups: + - "pitzerc" + allow: true + rsv_query: + - type: "OodAppkit::Validators::Groups" + data: + groups: + - "sysall" + - "sappall" + - "clntall" + allow: false + cluster: + type: "OodCluster::Cluster" + data: + hpc_cluster: true + servers: + login: + type: "OodCluster::Servers::Ssh" + data: + host: "pitzer.osc.edu" + resource_mgr: + type: "OodCluster::Servers::Torque" + data: + host: "pitzer-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + scheduler: + type: "OodCluster::Servers::Moab" + data: + host: "pitzer-batch.ten.osc.edu" + bin: "/opt/moab/bin" + version: "9.0.1" + moabhomedir: "/var/spool/moab" + ganglia: + type: "OodCluster::Servers::Ganglia" + data: + host: "ganglia.osc.edu" + scheme: "https://" + segments: + - "graph.php" + req_query: + c: "Pitzer" + opt_query: + h: "%{h}.ten.osc.edu" + version: "3" + +v2: + metadata: + title: "Pitzer" + url: "https://www.osc.edu/supercomputing/computing/pitzer" + hidden: false + acls: + - adapter: "group" + groups: + - "pitzerc" + type: "whitelist" + login: + host: "pitzer.osc.edu" + job: + adapter: "torque" + host: "pitzer-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + custom: + pbs: + host: "pitzer-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + moab: + host: "pitzer-batch.ten.osc.edu" + bin: "/opt/moab/bin" + version: "9.0.1" + homedir: "/var/spool/moab" + rsv_query: + torque_host: "pitzer-batch.ten.osc.edu" + torque_lib: "/opt/torque/lib64" + torque_bin: "/opt/torque/bin" + torque_version: "6.0.1" + moab_host: "pitzer-batch.ten.osc.edu" + moab_bin: "/opt/moab/bin" + moab_version: "9.0.1" + moab_homedir: "/var/spool/moab" + acls: + - adapter: "group" + groups: + - "sysall" + - "sappall" + - "clntall" + type: "blacklist" + ganglia: + host: "ganglia.osc.edu" + scheme: "https://" + segments: + - "graph.php" + req_query: + c: "Pitzer" + opt_query: + h: "%{h}.ten.osc.edu" + version: "3" + batch_connect: + basic: + script_wrapper: "module restore\n%s" + vnc: + script_wrapper: "module restore\nmodule load ondemand-vnc\n%s" diff --git a/lib/tasks/clusters.d/quick.yml b/lib/tasks/clusters.d/quick.yml new file mode 100644 index 00000000..749ba014 --- /dev/null +++ b/lib/tasks/clusters.d/quick.yml @@ -0,0 +1,28 @@ +--- +v1: + title: "Quick" + url: "" + cluster: + type: "OodCluster::Cluster" + data: + hpc_cluster: false + servers: + resource_mgr: + type: "OodCluster::Servers::Torque" + data: + host: "quick-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.2" + +v2: + metadata: + title: "Quick" + url: "" + hidden: true + job: + adapter: "torque" + host: "quick-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.2" diff --git a/lib/tasks/clusters.d/quick_pitzer.yml b/lib/tasks/clusters.d/quick_pitzer.yml new file mode 100644 index 00000000..0d1746c2 --- /dev/null +++ b/lib/tasks/clusters.d/quick_pitzer.yml @@ -0,0 +1,51 @@ +--- +v1: + title: "Quick Pitzer" + url: "" + validators: + cluster: + - type: "OodAppkit::Validators::Groups" + data: + groups: + - "pitzerc" + allow: true + cluster: + type: "OodCluster::Cluster" + data: + hpc_cluster: false + servers: + resource_mgr: + type: "OodCluster::Servers::Torque" + data: + host: "quick-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.2" + +v2: + metadata: + title: "Quick Pitzer" + url: "" + hidden: true + acls: + - adapter: "group" + groups: + - "pitzerc" + type: "whitelist" + job: + adapter: "torque" + host: "quick-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.2" + custom: + pbs: + host: "quick-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.2" + batch_connect: + basic: + script_wrapper: "module restore\n%s" + vnc: + script_wrapper: "module restore\nmodule load ondemand-vnc\n%s" diff --git a/lib/tasks/clusters.d/quick_ruby.yml b/lib/tasks/clusters.d/quick_ruby.yml new file mode 100644 index 00000000..a3f417a3 --- /dev/null +++ b/lib/tasks/clusters.d/quick_ruby.yml @@ -0,0 +1,51 @@ +--- +v1: + title: "Quick Ruby" + url: "" + validators: + cluster: + - type: "OodAppkit::Validators::Groups" + data: + groups: + - "ruby" + allow: true + cluster: + type: "OodCluster::Cluster" + data: + hpc_cluster: false + servers: + resource_mgr: + type: "OodCluster::Servers::Torque" + data: + host: "quick-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.2" + +v2: + metadata: + title: "Quick Ruby" + url: "" + hidden: true + acls: + - adapter: "group" + groups: + - "ruby" + type: "whitelist" + job: + adapter: "torque" + host: "quick-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.2" + custom: + pbs: + host: "quick-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.2" + batch_connect: + basic: + script_wrapper: "module restore\n%s" + vnc: + script_wrapper: "module restore\nmodule load ondemand-vnc\n%s" diff --git a/lib/tasks/clusters.d/ruby.yml b/lib/tasks/clusters.d/ruby.yml new file mode 100644 index 00000000..7b894c62 --- /dev/null +++ b/lib/tasks/clusters.d/ruby.yml @@ -0,0 +1,115 @@ +--- +v1: + title: "Ruby" + url: "https://www.osc.edu/supercomputing/computing/ruby" + validators: + cluster: + - type: "OodAppkit::Validators::Groups" + data: + groups: + - "ruby" + allow: true + rsv_query: + - type: "OodAppkit::Validators::Groups" + data: + groups: + - "sysall" + - "sappall" + - "clntall" + allow: false + cluster: + type: "OodCluster::Cluster" + data: + hpc_cluster: true + servers: + login: + type: "OodCluster::Servers::Ssh" + data: + host: "ruby.osc.edu" + resource_mgr: + type: "OodCluster::Servers::Torque" + data: + host: "ruby-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + scheduler: + type: "OodCluster::Servers::Moab" + data: + host: "ruby-batch.ten.osc.edu" + bin: "/opt/moab/bin" + version: "9.0.1" + moabhomedir: "/var/spool/moab" + ganglia: + type: "OodCluster::Servers::Ganglia" + data: + host: "ganglia.osc.edu" + scheme: "https://" + segments: + - "graph.php" + req_query: + c: "Ruby" + opt_query: + h: "%{h}.ten.osc.edu" + version: "3" + +v2: + metadata: + title: "Ruby" + url: "https://www.osc.edu/supercomputing/computing/ruby" + hidden: false + acls: + - adapter: "group" + groups: + - "ruby" + type: "whitelist" + login: + host: "ruby.osc.edu" + job: + adapter: "torque" + host: "ruby-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + custom: + pbs: + host: "ruby-batch.ten.osc.edu" + lib: "/opt/torque/lib64" + bin: "/opt/torque/bin" + version: "6.0.1" + moab: + host: "ruby-batch.ten.osc.edu" + bin: "/opt/moab/bin" + version: "9.0.1" + homedir: "/var/spool/moab" + rsv_query: + torque_host: "ruby-batch.ten.osc.edu" + torque_lib: "/opt/torque/lib64" + torque_bin: "/opt/torque/bin" + torque_version: "6.0.1" + moab_host: "ruby-batch.ten.osc.edu" + moab_bin: "/opt/moab/bin" + moab_version: "9.0.1" + moab_homedir: "/var/spool/moab" + acls: + - adapter: "group" + groups: + - "sysall" + - "sappall" + - "clntall" + type: "blacklist" + ganglia: + host: "ganglia.osc.edu" + scheme: "https://" + segments: + - "graph.php" + req_query: + c: "Ruby" + opt_query: + h: "%{h}.ten.osc.edu" + version: "3" + batch_connect: + basic: + script_wrapper: "module restore\n%s" + vnc: + script_wrapper: "module restore\nmodule load ondemand-vnc\n%s" From 94570d3a9658b5ec654456e5a5f83f6d40d1f4d7 Mon Sep 17 00:00:00 2001 From: AriettaZ Date: Tue, 27 Nov 2018 17:13:10 -0500 Subject: [PATCH 10/11] update gems --- Gemfile.lock | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 1e80db58..9f0916c9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -54,7 +54,7 @@ GEM coffee-script-source execjs coffee-script-source (1.12.2) - concurrent-ruby (1.0.5) + concurrent-ruby (1.1.3) crass (1.0.4) data-confirm-modal (1.6.2) railties (>= 3.0) @@ -71,7 +71,7 @@ GEM activesupport (>= 4.2.0) i18n (0.9.5) concurrent-ruby (~> 1.0) - jbuilder (2.7.0) + jbuilder (2.8.0) activesupport (>= 4.2.0) multi_json (>= 1.2) jquery-datatables-rails (3.4.0) @@ -84,17 +84,17 @@ GEM railties (>= 4.2.0) thor (>= 0.14, < 2.0) json (1.8.6) - json-schema (2.8.0) + json-schema (2.8.1) addressable (>= 2.4) lograge (0.10.0) actionpack (>= 4) activesupport (>= 4) railties (>= 4) request_store (~> 1.0) - loofah (2.2.2) + loofah (2.2.3) crass (~> 1.0.2) nokogiri (>= 1.5.9) - mail (2.7.0) + mail (2.7.1) mini_mime (>= 0.1.1) metaclass (0.0.4) mini_mime (1.0.1) @@ -103,7 +103,7 @@ GEM mocha (1.7.0) metaclass (~> 0.0.1) multi_json (1.13.1) - nokogiri (1.8.4) + nokogiri (1.8.5) mini_portile2 (~> 2.3.0) ood_appkit (1.0.3) addressable (~> 2.4) @@ -116,8 +116,8 @@ GEM ood_support (0.0.3) pbs (2.2.1) ffi (~> 1.9, >= 1.9.6) - public_suffix (3.0.2) - rack (1.6.10) + public_suffix (3.0.3) + rack (1.6.11) rack-test (0.6.3) rack (>= 1.0) rails (4.2.10) @@ -157,7 +157,7 @@ GEM redcarpet (3.4.0) request_store (1.4.1) rack (>= 1.4) - sass (3.5.7) + sass (3.7.2) sass-listen (~> 4.0.0) sass-listen (4.0.0) rb-fsevent (~> 0.9, >= 0.9.4) @@ -185,7 +185,7 @@ GEM tilt (2.0.8) tzinfo (1.2.5) thread_safe (~> 0.1) - uglifier (4.1.18) + uglifier (4.1.20) execjs (>= 0.3.0, < 3) PLATFORMS @@ -219,3 +219,6 @@ DEPENDENCIES spring thor (= 0.19.1) uglifier (>= 1.3.0) + +BUNDLED WITH + 1.13.7 From 1c2975b361b051a3aa99a114f7632737f6b614cc Mon Sep 17 00:00:00 2001 From: AriettaZ Date: Thu, 29 Nov 2018 15:07:24 -0500 Subject: [PATCH 11/11] try solve travis error --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 1eb6acd6..2fe21fca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -138,7 +138,7 @@ GEM nokogiri (~> 1.6) rails-deprecated_sanitizer (>= 1.0.1) rails-html-sanitizer (1.0.4) - loofah (~> 2.2, >= 2.2.3) + loofah (~> 2.2, >= 2.2.2) rails_12factor (0.0.3) rails_serve_static_assets rails_stdout_logging @@ -182,7 +182,7 @@ GEM sprockets (>= 3.0.0) thor (0.19.1) thread_safe (0.3.6) - tilt (2.0.8) + tilt (2.0.9) tzinfo (1.2.5) thread_safe (~> 0.1) uglifier (4.1.20)