Merge branch 'dev' of https://github.com/marklogic/roxy into cluster-…

…verify-issue

* 'dev' of https://github.com/marklogic/roxy:
  implemented based on review points
  implement review points
  fix comments
  add feature to load multiple custom configuration files
  marklogic-community#733 corrected invalid-values default; addressed missed case
  issue marklogic-community#740: make -EXT work from modules and filesystem
  removed unneeded FLWOR statements, fixed indent
  fixed mistake in validate-roles/users
  retrieval of existing roles/users outside loop
  issue marklogic-community#736: fix CTRL-EXT from a filesystem modules DB
  Update version

deploy/default.properties

@@ -262,7 +262,11 @@ http.read-timeout=300
 http.retry-delay=15
 
 #
+# Application configuration files
+#
+application-conf-file=src/app/config/config.xqy
+
 # Verified restart config
 #
 verify_retry_max=5
-verify_retry_interval=10
+verify_retry_interval=10

deploy/lib/server_config.rb

@@ -186,22 +186,26 @@ def self.init
       sample_properties = "roxy/sample/build.sample.properties"
       sample_options = "roxy/sample/all.sample.xml"
       sample_rest_properties = "roxy/sample/properties.sample.xml"
+      sample_app_config = "roxy/deploy/sample/custom-config.xqy"
     else
-    sample_config = ServerConfig.expand_path("#{@@path}/sample/ml-config.sample.xml")
-    sample_properties = ServerConfig.expand_path("#{@@path}/sample/build.sample.properties")
+      sample_config = ServerConfig.expand_path("#{@@path}/sample/ml-config.sample.xml")
+      sample_properties = ServerConfig.expand_path("#{@@path}/sample/build.sample.properties")
       sample_options = ServerConfig.expand_path("#{@@path}/sample/all.sample.xml")
       sample_rest_properties = ServerConfig.expand_path("#{@@path}/sample/properties.sample.xml")
+      sample_app_config = ServerConfig.expand_path("#{@@path}/sample/custom-config.xqy")
     end
 
     # output files
     build_properties = ServerConfig.expand_path("#{@@path}/build.properties")
     options_file = ServerConfig.expand_path("#{@@path}/../rest-api/config/options/all.xml")
     rest_properties = ServerConfig.expand_path("#{@@path}/../rest-api/config/properties.xml")
+    app_config = ServerConfig.expand_path("#{@@path}/../src/config/config.xqy")
 
     # dirs to create
     rest_ext_dir = ServerConfig.expand_path("#{@@path}/../rest-api/ext")
     rest_transforms_dir = ServerConfig.expand_path("#{@@path}/../rest-api/transforms")
     options_dir = ServerConfig.expand_path("#{@@path}/../rest-api/config/options")
+    config_dir = ServerConfig.expand_path("#{@@path}/../src/config")
 
     # get supplied options
     force = find_arg(['--force']).present?
@@ -258,6 +262,10 @@ def self.init
       # Update properties file to set server-version to value specified on command-line
       properties_file.gsub!(/server-version=6/, "server-version=#{server_version}")
 
+      if ["rest", "bare"].include? app_type
+        properties_file.gsub!(/application-conf-file=src\/app\/config\/config.xqy/, 'application-conf-file=src/config/config.xqy')
+      end
+
       # save the replacements
       open(build_properties, 'w') {|f| f.write(properties_file) }
     end
@@ -271,6 +279,11 @@ def self.init
       copy_file sample_rest_properties, rest_properties
     end
 
+    if ["rest", "bare"].include? app_type
+      FileUtils.mkdir_p config_dir
+      copy_file sample_app_config, app_config
+    end
+
     target_config = ServerConfig.expand_path(ServerConfig.properties["ml.config.file"])
 
     if !force && !force_config && File.exists?(target_config)
@@ -1943,8 +1956,7 @@ def deploy_modules
   def deploy_src
     test_dir = @properties['ml.xquery-test.dir']
     xquery_dir = @properties['ml.xquery.dir']
-    # modules_db = @properties['ml.modules-db']
-    app_config_file = File.join xquery_dir, "/app/config/config.xqy"
+    app_configs = @properties['ml.application-conf-file']
     test_config_file = File.join test_dir, "/test-config.xqy"
     load_html_as_xml = @properties['ml.load-html-as-xml']
     load_js_as_binary = @properties['ml.load-js-as-binary']
@@ -1971,6 +1983,7 @@ def deploy_src
 
     end
 
+    total_count = 0
     modules_databases.each do |dest_db|
       if dest_db == "filesystem"
         logger.info "Skipping deployment of src to #{dest_db}.."
@@ -1979,7 +1992,6 @@ def deploy_src
 
       ignore_us = []
       ignore_us << "^#{test_dir}.*$" unless test_dir.blank? || deploy_tests?(dest_db)
-      ignore_us << "^#{app_config_file}$"
       ignore_us << "^#{test_config_file}$"
       ignore_us << "^#{folders_to_ignore}$" unless folders_to_ignore.blank?
 
@@ -1994,7 +2006,34 @@ def deploy_src
         src_permissions.flatten!
       end
 
-      @logger.debug("source permissions: #{src_permissions}")
+      @logger.debug "source permissions: #{src_permissions}"
+      if app_configs.present?
+        logger.debug "Deploying application configurations"
+
+        app_configs.split(',').each do |item|
+          buffer = File.read item
+          replace_properties(buffer, File.basename(item))
+
+          item_name = item
+          prefix = '/'
+          if item_name === 'src/app/config/config.xqy'
+            item_name = '/config.xqy'
+            ignore_us << '/app/config/config.xqy'
+            prefix = 'app/config/'
+          elsif item.start_with?("src/")
+            item_name = '/' + item[4, item.length]
+            ignore_us << item_name
+          end
+
+          logger.debug "deploying application configuration #{item} with name #{item_name} on #{dest_db}"
+          total_count += xcc.load_buffer item_name,
+                                         buffer,
+                                         :db => dest_db,
+                                         :add_prefix => File.join(@properties["ml.modules-root"], prefix),
+                                         :permissions => src_permissions
+        end
+        logger.debug "Done deploying application configurations"
+      end
 
       total_count = load_data xquery_dir,
                               :add_prefix => @properties["ml.modules-prefix"],
@@ -2006,16 +2045,6 @@ def deploy_src
                               :load_css_as_binary => load_css_as_binary,
                               :permissions => src_permissions
 
-      if File.exist? app_config_file
-        buffer = File.read app_config_file
-        replace_properties(buffer, File.basename(app_config_file))
-
-        total_count += xcc.load_buffer "/config.xqy",
-                                       buffer,
-                                       :db => dest_db,
-                                       :add_prefix => File.join(@properties["ml.modules-root"], "app/config"),
-                                       :permissions => src_permissions
-      end
 
       if deploy_tests?(dest_db) && File.exist?(test_config_file)
         buffer = File.read test_config_file

deploy/lib/xquery/setup.xqy

@@ -72,6 +72,11 @@ declare variable $replicating-map-file := "/roxy/status/cleanup/replicating-map.
 declare variable $replicating-map-file-internal := "/roxy/status/cleanup/replicating-map-internal.xml";
 declare variable $replicating-map := map:map();
 
+(: Several functions take an optional invalid-values parameter. Use this as the
+ : default value when it's not provided.
+ :)
+declare variable $default-invalid-values := "reject";
+
 declare variable $group-settings :=
   <settings>
     <setting>list-cache-size</setting>
@@ -2258,7 +2263,7 @@ declare function setup:validated-range-element-indexes(
             $index-config/db:localname/fn:string(.),
             fn:string($index-config/db:collation[../db:scalar-type = 'string']),
             ($index-config/db:range-value-positions/xs:boolean(.), false())[1],
-            ($index-config/db:invalid-values, "reject")[1]
+            ($index-config/db:invalid-values, $default-invalid-values)[1]
           )
         else
           xdmp:apply(
@@ -2320,7 +2325,7 @@ declare function setup:validated-range-element-attribute-indexes(
             $index-config/db:localname/fn:string(.),
             fn:string($index-config/db:collation[../db:scalar-type = 'string']),
             ($index-config/db:range-value-positions/xs:boolean(.), false())[1],
-            ($index-config/db:invalid-values, "reject")[1]
+            ($index-config/db:invalid-values, $default-invalid-values)[1]
           )
         else
           xdmp:apply(
@@ -2449,7 +2454,7 @@ declare function setup:add-range-path-indexes(
                $index/db:path-expression,
                $index/db:collation,
                $index/db:range-value-positions,
-               $index/db:invalid-values
+               ($index/db:invalid-values, $default-invalid-values)[1]
              )
          )"
       )
@@ -2478,16 +2483,18 @@ declare function setup:validate-range-path-indexes(
       declare namespace db="http://marklogic.com/xdmp/database";
       declare variable $database external;
       declare variable $x external;
+      declare variable $default-invalid-values external;
 
       admin:database-range-path-index(
        $database,
        $x/db:scalar-type,
        $x/db:path-expression,
        fn:string($x/db:collation[../db:scalar-type = "string"]),
        $x/db:range-value-positions,
-       $x/db:invalid-values)',
+       ($x/db:invalid-values, $default-invalid-values)[1])',
       (xs:QName("database"), $database,
-       xs:QName("x"), $expected))
+       xs:QName("x"), $expected,
+       xs:QName("default-invalid-values"), $default-invalid-values))
   return
     if ($existing[fn:deep-equal(., $expected)]) then ()
     else
@@ -2689,7 +2696,7 @@ declare function setup:add-range-field-indexes-helper(
             $index/db:field-name,
             ($index/db:collation/fn:string(), "")[1], (: ML6 requires xs:string; later requires xs:string? :)
             $index/db:range-value-positions,
-            $index/db:invalid-values
+            ($index/db:invalid-values, $default-invalid-values)[1]
           )
         else
           admin:database-range-field-index(
@@ -2738,7 +2745,7 @@ declare function setup:add-geospatial-element-indexes(
           $index/db:coordinate-system,
           $index/db:range-value-positions,
           ($index/db:point-format, "point")[1],
-          ($index/db:invalid-values, "ignore")[1]
+          ($index/db:invalid-values, $default-invalid-values)[1]
         )
       else
         admin:database-geospatial-element-index(
@@ -2792,7 +2799,7 @@ declare function setup:add-geospatial-element-attribute-pair-indexes(
           $index/db:longitude-localname,
           $index/db:coordinate-system,
           $index/db:range-value-positions,
-          ($index/db:invalid-values, "ignore")[1]
+          ($index/db:invalid-values, $default-invalid-values)[1]
         )
       else
         admin:database-geospatial-element-attribute-pair-index(
@@ -2849,7 +2856,7 @@ declare function setup:add-geospatial-element-pair-indexes(
           $index/db:longitude-localname,
           $index/db:coordinate-system,
           $index/db:range-value-positions,
-          ($index/db:invalid-values, "ignore")[1]
+          ($index/db:invalid-values, $default-invalid-values)[1]
         )
       else
         admin:database-geospatial-element-pair-index(
@@ -2905,7 +2912,7 @@ declare function setup:add-geospatial-element-child-indexes(
           $index/db:coordinate-system,
           $index/db:range-value-positions,
           ($index/db:point-format, "point")[1],
-          ($index/db:invalid-values, "ignore")[1]
+          ($index/db:invalid-values, $default-invalid-values)[1]
         )
       else
         admin:database-geospatial-element-child-index(
@@ -4742,6 +4749,9 @@ declare function setup:validate-external-security(
 declare function setup:create-roles(
   $import-config as element(configuration))
 {
+  (: get the existing role names from the default security DB :)
+  let $existing-role-names := setup:get-existing-role-names()
+
   (: Create all missing roles :)
   for $role in $import-config/sec:roles/sec:role
   let $role-name as xs:string := $role/sec:role-name
@@ -4754,7 +4764,7 @@ declare function setup:create-roles(
     </options>
   return
     (: if the role exists, then don't create it :)
-    if (setup:get-roles(())/sec:role[sec:role-name = $role-name]) then ()
+    if ($existing-role-names[. = $role-name]) then ()
     else
     (
       xdmp:eval(
@@ -4911,6 +4921,9 @@ declare function setup:create-roles(
 declare function setup:validate-roles(
   $import-config as element(configuration))
 {
+  (: get the existing role names from the default security DB :)
+  let $existing-roles := setup:get-roles(())
+
   for $role in $import-config/sec:roles/sec:role
   let $role-name as xs:string := $role/sec:role-name
   let $description as xs:string? := $role/sec:description
@@ -4921,7 +4934,7 @@ declare function setup:validate-roles(
   let $privileges as element(sec:privilege)* := $role/sec:privileges/sec:privilege
   let $amps as element(sec:amp)* := $role/sec:amps/*
   let $external-names as xs:string* := $role/sec:external-names/sec:external-name
-  let $match := setup:get-roles(())/sec:role[sec:role-name = $role-name]
+  let $match := $existing-roles/sec:role[sec:role-name = $role-name]
   return
     if ($match) then
       if ($match/sec:role-name != $role-name or
@@ -4964,6 +4977,10 @@ declare function setup:associate-users-with-roles($import-config as element(conf
 
 declare function setup:create-users($import-config as element(configuration))
 {
+  (: get the existing user names from the default security DB :)
+  let $existing-user-names := setup:get-existing-user-names()
+
+  (: Create all missing users :)
   for $user in $import-config/sec:users/sec:user
   let $user-name as xs:string := $user/sec:user-name
   let $description as xs:string? := $user/sec:description
@@ -4978,7 +4995,7 @@ declare function setup:create-users($import-config as element(configuration))
       <isolation>different-transaction</isolation>
     </options>
   return
-    if (setup:get-users(())/sec:user[sec:user-name = $user-name]) then
+    if ($existing-user-names[. = $user-name]) then
     (
       xdmp:eval(
         'import module namespace sec="http://marklogic.com/xdmp/security" at "/MarkLogic/security.xqy";
@@ -5088,6 +5105,9 @@ declare function setup:create-users($import-config as element(configuration))
 
 declare function setup:validate-users($import-config as element(configuration))
 {
+  (: get the existing users from the default security DB :)
+  let $existing-users := setup:get-users(())
+
   for $user in $import-config/sec:users/sec:user
   let $user-name as xs:string := $user/sec:user-name
   let $description as xs:string? := $user/sec:description
@@ -5096,7 +5116,7 @@ declare function setup:validate-users($import-config as element(configuration))
   let $permissions as element(sec:permission)* := $user/sec:permissions/*
   let $collections as xs:string* := $user/sec:collections/*
   let $external-names as xs:string* := $user/sec:external-names/sec:external-name
-  let $match := setup:get-users(())/sec:user[sec:user-name = $user-name]
+  let $match := $existing-users/sec:user[sec:user-name = $user-name]
   return
     if ($match) then
       if ($match/sec:description != $description or
@@ -5451,6 +5471,19 @@ declare function setup:get-privilege-by-name($name as xs:string) as element(sec:
     </options>)
 };
 
+(: Gets the user names from the default security database :)
+declare function setup:get-existing-user-names() as element(sec:user-name)* {
+  let $user-names :=
+    xdmp:eval(
+      'import module namespace sec="http://marklogic.com/xdmp/security" at "/MarkLogic/security.xqy";
+       /sec:user',
+       (),
+       <options xmlns="xdmp:eval">
+        <database>{$default-security}</database>
+       </options>)/sec:user-name
+  return $user-names
+};
+
 declare function setup:get-users-by-name($names as xs:string*) as element(sec:users)? {
   let $ids :=
     for $name in $names
@@ -5530,6 +5563,19 @@ declare function setup:get-user-id($user-name as xs:string) as xs:unsignedLong?
      </options>)
 };
 
+(: Gets the role names from the default security database :)
+declare function setup:get-existing-role-names() as element(sec:role-name)* {
+  let $role-names :=
+    xdmp:eval(
+      'import module namespace sec="http://marklogic.com/xdmp/security" at "/MarkLogic/security.xqy";
+        /sec:role',
+      (),
+      <options xmlns="xdmp:eval">
+        <database>{$default-security}</database>
+      </options>)/sec:role-name
+  return $role-names
+};
+
 declare function setup:get-roles-by-name($roles as xs:string*) as element(sec:roles)? {
   let $ids :=
     for $role in $roles

deploy/sample/build.sample.properties

@@ -208,3 +208,11 @@ mlcp-vmargs=-Xmx512m
 
 # enable the following to save and upload svn/git commit info upon deploy modules
 save-commit-info=false
+
+#
+# Application configuration files
+#
+# Assign a comma separated custom configuration files relative to root folder.
+#
+# Contents of the files will be replaced with data from the properties files.
+application-conf-file=src/app/config/config.xqy