Dispatch 990 Use patterns for policy vhost hostnames

doc/book/policy.adoc

@@ -90,6 +90,64 @@ xref:example2[Example 2] illustrates how the default vhost feature can
 be used to apply a single vhost policy set of restrictions to any
 number of vhost connections.
 
+=== Vhost Patterns
+
+Policy vhost names may be interpreted as literal host names or 
+as host name patterns. Vhost name patterns are a convenience
+for letting a single policy rule cover a wide range of vhosts.
+
+Host name patterns consist of a series of host and domain name
+labels and one or more tokens all concatenated with periods or dots.
+A token can be one of the following:
+
+[options="header"]
+|====
+| Token character | Match rule
+| asterisk *      | matches a single hostname label
+| hash #          | matches zero or more hostname labels
+|====
+
+Some simple examples show how given policy name patterns match
+incoming connection vhost names.
+
+[options="header"]
+|====
+| Policy pattern         | Connection vhost           | Policy match
+| *.example.com          | example.com                | no
+| *.example.com          | www.example.com            | yes
+| *.example.com          | srv2.www.example.com       | no
+| #.example.com          | example.com                | yes
+| #.example.com          | www.example.com            | yes
+| #.example.com          | a.b.c.d.example.com        | yes
+| #.example.com          | bighost.com                | no
+| www.*.test.example.com | www.test.example.com       | no
+| www.*.test.example.com | www.a.test.example.com     | yes
+| www.*.test.example.com | www.a.b.c.test.example.com | no
+| www.#.test.example.com | www.test.example.com       | yes
+| www.#.test.example.com | www.a.test.example.com     | yes
+| www.#.test.example.com | www.a.b.c.test.example.com | yes
+|====
+
+Pattern matching applies the following precedence rules.
+
+[options="header"]
+|====
+| Policy pattern      | Precedence
+| exact match         | high
+| asterisk *          | medium
+| hash #              | low
+|====
+
+Policy vhost name patterns are optimised before they are used 
+in connection vhost name matching. As a result of this
+optimisation the names stored for pattern match lookups are
+not necessarily the same as the patterns specified in the 
+vhost policy hostname. The policy agent disallows vhost
+name patterns that reduce to the same pattern as an existing name 
+pattern. For instance, name pattern _pass:[#.#.#.#.com]_ is reduced to _pass:[#.com]_.
+Attempts to create a vhost name pattern whose optimised
+name conflicts with an existing optimised name will be denied.
+
 == Policy Schema
 
 Policy configuration is specified in two schema objects.
@@ -123,6 +181,7 @@ created as needed.
 | enableVhostPolicy   | false      | Enable vhost policy connection denial, and resource limit enforcement.
 | policyDir           | ""         | Absolute path to a directory that holds vhost definition .json files. All vhost definitions in all .json files in this directory are processed.
 | defaultVhost        | "$default" | Vhost rule set name to use for connections with a vhost that is otherwise not defined. Default vhost processing may be disabled either by erasing the definition of _defaultVhost_ or by not defining a _vhost_ object named _$default_.
+| enableVhostNamePatterns | false  | Enable vhost name patterns. When false vhost hostnames are treated as literal strings. When true vhost hostnames are treated as match patterns.
 |====
 
 === Vhost Policy

python/qpid_dispatch/management/qdrouter.json

@@ -1678,6 +1678,13 @@
                     "required": false,
                     "create": true
                 },
+                "enableVhostNamePatterns": {
+                    "type": "boolean",
+                    "default": false,
+                    "description": "Enable vhost name patterns. When false vhost hostnames are treated as literal strings. When true vhost hostnames are treated as match patterns.",
+                    "required": false,
+                    "create": true
+                },
                 "policyDir": {
                     "type": "path",
                     "default": "",

python/qpid_dispatch_internal/dispatch.py

@@ -78,6 +78,10 @@ def __init__(self, handle):
         self._prototype(self.qd_dispatch_policy_c_counts_alloc, c_long, [], check=False)
         self._prototype(self.qd_dispatch_policy_c_counts_free, None, [c_long], check=False)
         self._prototype(self.qd_dispatch_policy_c_counts_refresh, None, [c_long, py_object])
+        self._prototype(self.qd_dispatch_policy_host_pattern_add, ctypes.c_bool, [self.qd_dispatch_p, c_char_p])
+        self._prototype(self.qd_dispatch_policy_host_pattern_remove, None, [self.qd_dispatch_p, c_char_p])
+        self._prototype(self.qd_dispatch_policy_host_pattern_lookup, c_char_p, [self.qd_dispatch_p, c_char_p])
+
 
         self._prototype(self.qd_dispatch_register_display_name_service, None, [self.qd_dispatch_p, py_object])
 

python/qpid_dispatch_internal/management/config.py

@@ -174,13 +174,21 @@ def configure(attributes):
     from qpid_dispatch_internal.display_name.display_name import DisplayNameService
     displayname_service = DisplayNameService()
     qd.qd_dispatch_register_display_name_service(dispatch, displayname_service)
-    policyDir = config.by_type('policy')[0]['policyDir']
-    policyDefaultVhost = config.by_type('policy')[0]['defaultVhost']
+
+    # Configure policy and policy manager before vhosts
+    policyDir           = config.by_type('policy')[0]['policyDir']
+    policyDefaultVhost  = config.by_type('policy')[0]['defaultVhost']
+    useHostnamePatterns = config.by_type('policy')[0]['enableVhostNamePatterns']
+    for a in config.by_type("policy"):
+        configure(a)
+    agent.policy.set_default_vhost(policyDefaultVhost)
+    agent.policy.set_use_hostname_patterns(useHostnamePatterns)
+
     # Remaining configuration
     for t in "sslProfile", "authServicePlugin", "listener", "connector", \
              "router.config.address", "router.config.linkRoute", "router.config.autoLink", \
              "router.config.exchange", "router.config.binding", \
-             "policy", "vhost":
+             "vhost":
         for a in config.by_type(t):
             configure(a)
             if t == "sslProfile":
@@ -201,6 +209,3 @@ def configure(attributes):
                 pconfig = PolicyConfig(os.path.join(apath, i))
                 for a in pconfig.by_type("vhost"):
                     agent.configure(a)
-
-    # Set policy default application after all rulesets loaded
-    agent.policy.set_default_vhost(policyDefaultVhost)

python/qpid_dispatch_internal/policy/policy_local.py

@@ -508,6 +508,11 @@ def __init__(self, manager):
         #  open.hostname is not found in the rulesetdb
         self._default_vhost = ""
 
+        # _use_hostname_patterns
+        #  holds policy setting.
+        #  When true policy ruleset definitions are propagated to C code
+        self.use_hostname_patterns = False
+
     #
     # Service interfaces
     #
@@ -526,6 +531,11 @@ def create_ruleset(self, attributes):
         if len(warnings) > 0:
             for warning in warnings:
                 self._manager.log_warning(warning)
+        # Reject if parse tree optimized name collision
+        if self.use_hostname_patterns:
+            agent = self._manager.get_agent()
+            if not agent.qd.qd_dispatch_policy_host_pattern_add(agent.dispatch, name):
+                raise PolicyError("Policy '%s' optimized pattern conflicts with existing pattern" % name)
         if name not in self.rulesetdb:
             if name not in self.statsdb:
                 self.statsdb[name] = AppStats(name, self._manager, candidate)
@@ -545,6 +555,9 @@ def policy_delete(self, name):
         if name not in self.rulesetdb:
             raise PolicyError("Policy '%s' does not exist" % name)
         # TODO: ruleset lock
+        if self.use_hostname_patterns:
+            agent = self._manager.get_agent()
+            agent.qd.qd_dispatch_policy_host_pattern_remove(agent.dispatch, name)
         del self.rulesetdb[name]
 
     #
@@ -592,15 +605,24 @@ def lookup_user(self, user, rhost, vhost_in, conn_name, conn_id):
         """
         try:
             # choose rule set based on incoming vhost or default vhost
+            # or potential vhost found by pattern matching
             vhost = vhost_in
-            if vhost_in not in self.rulesetdb:
+            if self.use_hostname_patterns:
+                agent = self._manager.get_agent()
+                vhost = agent.qd.qd_dispatch_policy_host_pattern_lookup(agent.dispatch, vhost)
+            if vhost not in self.rulesetdb:
                 if self.default_vhost_enabled():
                     vhost = self._default_vhost
                 else:
                     self._manager.log_info(
                         "DENY AMQP Open for user '%s', rhost '%s', vhost '%s': "
-                        "No policy defined for vhost" % (user, rhost, vhost))
+                        "No policy defined for vhost" % (user, rhost, vhost_in))
                     return ""
+            if vhost != vhost_in:
+                self._manager.log_debug(
+                    "AMQP Open for user '%s', rhost '%s', vhost '%s': "
+                    "proceeds using vhost '%s' ruleset" % (user, rhost, vhost_in, vhost))
+
             ruleset = self.rulesetdb[vhost]
 
             # look up the stats
@@ -681,9 +703,16 @@ def lookup_settings(self, vhost_in, groupname, upolicy):
         """
         try:
             vhost = vhost_in
+            if self.use_hostname_patterns:
+                agent = self._manager.get_agent()
+                vhost = agent.qd.qd_dispatch_policy_host_pattern_lookup(agent.dispatch, vhost)
             if vhost not in self.rulesetdb:
                 if self.default_vhost_enabled():
                     vhost = self._default_vhost
+            if vhost != vhost_in:
+                self._manager.log_debug(
+                    "AMQP Open lookup settings for vhost '%s': "
+                    "proceeds using vhost '%s' ruleset" % (vhost_in, vhost))
 
             if vhost not in self.rulesetdb:
                 self._manager.log_info(

python/qpid_dispatch_internal/policy/policy_manager.py

@@ -43,6 +43,7 @@ def __init__(self, agent):
         self._agent = agent
         self._policy_local = PolicyLocal(self)
         self.log_adapter = LogAdapter("POLICY")
+        self._use_hostname_patterns = False
 
     def log(self, level, text):
         info = traceback.extract_stack(limit=2)[0] # Caller frame info
@@ -70,6 +71,13 @@ def log_warning(self, text):
     def get_agent(self):
         return self._agent
 
+    def get_use_hostname_patterns(self):
+        return self._use_hostname_patterns
+
+    def set_use_hostname_patterns(self, v):
+        self._use_hostname_patterns = v
+        self._policy_local.use_hostname_patterns = v
+
     #
     # Management interface to create a ruleset
     #

src/dispatch.c

@@ -270,6 +270,21 @@ void qd_dispatch_policy_c_counts_refresh(long ccounts, qd_entity_t *entity)
     qd_policy_c_counts_refresh(ccounts, entity);
 }
 
+bool qd_dispatch_policy_host_pattern_add(qd_dispatch_t *qd, char *hostPattern)
+{
+    return qd_policy_host_pattern_add(qd->policy, hostPattern);
+}
+
+void qd_dispatch_policy_host_pattern_remove(qd_dispatch_t *qd, char *hostPattern)
+{
+    qd_policy_host_pattern_remove(qd->policy, hostPattern);
+}
+
+char * qd_dispatch_policy_host_pattern_lookup(qd_dispatch_t *qd, char *hostPattern)
+{
+    return qd_policy_host_pattern_lookup(qd->policy, hostPattern);
+}
+
 qd_error_t qd_dispatch_prepare(qd_dispatch_t *qd)
 {
     qd->server             = qd_server(qd, qd->thread_count, qd->router_id, qd->sasl_config_path, qd->sasl_config_name);

src/policy.c

@@ -61,10 +61,13 @@ struct qd_policy_t {
     qd_dispatch_t        *qd;
     qd_log_source_t      *log_source;
     void                 *py_policy_manager;
+    sys_mutex_t          *tree_lock;
+    qd_parse_tree_t      *hostname_tree;
                           // configured settings
     int                   max_connection_limit;
     char                 *policyDir;
     bool                  enableVhostPolicy;
+    bool                  enableVhostNamePatterns;
                           // live statistics
     int                   connections_processed;
     int                   connections_denied;
@@ -77,15 +80,12 @@ struct qd_policy_t {
 qd_policy_t *qd_policy(qd_dispatch_t *qd)
 {
     qd_policy_t *policy = NEW(qd_policy_t);
+    ZERO(policy);
     policy->qd                   = qd;
     policy->log_source           = qd_log_source("POLICY");
     policy->max_connection_limit = 65535;
-    policy->py_policy_manager    = 0;
-    policy->policyDir            = 0;
-    policy->enableVhostPolicy    = false;
-    policy->connections_processed= 0;
-    policy->connections_denied   = 0;
-    policy->connections_current  = 0;
+    policy->tree_lock            = sys_mutex();
+    policy->hostname_tree        = qd_parse_tree_new(QD_PARSE_TREE_ADDRESS);
 
     qd_log(policy->log_source, QD_LOG_TRACE, "Policy Initialized");
     return policy;
@@ -99,6 +99,7 @@ void qd_policy_free(qd_policy_t *policy)
 {
     if (policy->policyDir)
         free(policy->policyDir);
+    qd_parse_tree_free(policy->hostname_tree);
     free(policy);
 }
 
@@ -114,8 +115,16 @@ qd_error_t qd_entity_configure_policy(qd_policy_t *policy, qd_entity_t *entity)
     policy->policyDir =
         qd_entity_opt_string(entity, "policyDir", 0); CHECK();
     policy->enableVhostPolicy = qd_entity_opt_bool(entity, "enableVhostPolicy", false); CHECK();
-    qd_log(policy->log_source, QD_LOG_INFO, "Policy configured maxConnections: %d, policyDir: '%s', access rules enabled: '%s'",
-           policy->max_connection_limit, policy->policyDir, (policy->enableVhostPolicy ? "true" : "false"));
+    policy->enableVhostNamePatterns = qd_entity_opt_bool(entity, "enableVhostNamePatterns", false); CHECK();
+    qd_log(policy->log_source, QD_LOG_INFO,
+           "Policy configured maxConnections: %d, "
+           "policyDir: '%s',"
+           "access rules enabled: '%s', "
+           "use hostname patterns: '%s'",
+           policy->max_connection_limit,
+           policy->policyDir,
+           (policy->enableVhostPolicy ? "true" : "false"),
+           (policy->enableVhostNamePatterns ? "true" : "false"));
     return QD_ERROR_NONE;
 
 error:
@@ -822,3 +831,50 @@ bool qd_policy_approve_link_name(const char *username,
     }
     return false;
 }
+
+
+// Add a hostname to the lookup parse_tree
+bool qd_policy_host_pattern_add(qd_policy_t *policy, char *hostPattern)
+{
+    sys_mutex_lock(policy->tree_lock);
+    void *oldp = qd_parse_tree_add_pattern_str(policy->hostname_tree, hostPattern, hostPattern);
+    if (oldp) {
+        void *recovered = qd_parse_tree_add_pattern_str(policy->hostname_tree, (char *)oldp, oldp);
+        assert (recovered);
+    }
+    sys_mutex_unlock(policy->tree_lock);
+    if (oldp)
+        qd_log(policy->log_source,
+            QD_LOG_WARNING,
+            "vhost hostname pattern '%s' failed to replace optimized pattern '%s'",
+            hostPattern, oldp);
+    return oldp == 0;
+}
+
+
+// Remove a hostname from the lookup parse_tree
+void qd_policy_host_pattern_remove(qd_policy_t *policy, char *hostPattern)
+{
+    sys_mutex_lock(policy->tree_lock);
+    void *oldp = qd_parse_tree_remove_pattern_str(policy->hostname_tree, hostPattern);
+    sys_mutex_unlock(policy->tree_lock);
+    if (!oldp) {
+        qd_log(policy->log_source, QD_LOG_WARNING, "vhost hostname pattern '%s' for removal not found", hostPattern);
+    }
+}
+
+
+// Look up a hostname in the lookup parse_tree
+char * qd_policy_host_pattern_lookup(qd_policy_t *policy, char *hostPattern)
+{
+    void *payload = 0;
+    sys_mutex_lock(policy->tree_lock);
+    bool matched = qd_parse_tree_retrieve_match_str(policy->hostname_tree, hostPattern, &payload);
+    sys_mutex_unlock(policy->tree_lock);
+    if (!matched) {
+        payload = 0;
+    }
+    qd_log(policy->log_source, QD_LOG_TRACE, "vhost hostname pattern '%s' lookup returned '%s'", 
+           hostPattern, (payload ? (char *)payload : "null"));
+    return payload;
+}

src/policy.h

@@ -177,4 +177,29 @@ bool qd_policy_approve_link_name(const char *username,
                                   const char *proposed,
                                   bool isReceiver
                                 );
+
+/** Add a hostname to the lookup parse_tree
+ * Note that the parse_tree may store an 'optimised' pattern for a given
+ * pattern. Thus the patterns a user puts in may collide with existing
+ * patterns even though the text of the host patterns is different.
+ * This function does not allow new patterns with thier optimizations 
+ * to overwrite existing patterns that may have been optimised.
+ * @param[in] policy qd_policy_t
+ * @param[in] hostPattern the hostname pattern with possible parse_tree wildcards
+ * @return True if the possibly optimised pattern was added to the lookup parse tree
+ */
+bool qd_policy_host_pattern_add(qd_policy_t *policy, char *hostPattern);
+
+/** Remove a hostname from the lookup parse_tree
+ * @param[in] policy qd_policy_t
+ * @param[in] hostPattern the hostname pattern with possible parse_tree wildcards
+ */
+void qd_policy_host_pattern_remove(qd_policy_t *policy, char *hostPattern);
+
+/** Look up a hostname in the lookup parse_tree
+ * @param[in] policy qd_policy_t
+ * @param[in] hostname a concrete vhost name
+ * @return the name of the ruleset whose hostname pattern matched this actual hostname
+ */
+char * qd_policy_host_pattern_lookup(qd_policy_t *policy, char *hostPattern);
 #endif