httpd: Introduce topic for discovering services to monitor

Motivation:

httpd has a configurable list of cells to monitor. In particular when
using replicated services, this system is fragile as there may be
more than a single consumer on monitored named queue.

Modification:

Introduces a topic, WatchedServiceTopic, for httpd to discover services
to monitor.

Introduced a new dcache.topic main branch for topic names analogous to
dcache.queue. Other topics ought to eventually be migrated to this
scheme.

One could have considered alternative designs, such as watched services
broadcasting on a topic or using ZooKeeper. The justification for the
chosen design is because it is an easy evolution of what we already got.

Result:

Introduced a new topic for httpd to discover services to monitor. One
side effect is that space manager becomes monitored automatically when
discovered.

Target: trunk
Require-notes: yes
Require-book: no
Acked-by: Paul Millar <paul.millar@desy.de>
Acked-by: Albert Rossir <arossi@fnal.gov>

Reviewed at https://rb.dcache.org/r/9629/

skel/share/defaults/billing.properties

@@ -18,7 +18,7 @@ billing.cell.name=billing
 #
 billing.cell.consume=${billing.cell.name}
 
-#  ---- Topics to subscribe to
+#  ---- Topics to which the service subscribes
 #
 #   A service can subscribe to messages published on a topic. Other service
 #   can send messages to such a topic. A topic an unqualified cell address,

skel/share/defaults/cleaner.properties

@@ -36,7 +36,7 @@ cleaner.cell.name=cleaner
 cleaner.cell.consume = ${cleaner.cell.name}
 
 # Message topics to subscribe to.
-cleaner.cell.subscribe=PoolUpTopic
+cleaner.cell.subscribe = ${dcache.topic.pool-heartbeat}
 
 # Whether space reservations are enabled
 (one-of?true|false|${dcache.enable.space-reservation})cleaner.enable.space-reservation=${dcache.enable.space-reservation}

skel/share/defaults/dcache.properties

@@ -457,6 +457,30 @@ dcache.queue.pinmanager = PinManager
 dcache.queue.spacemanager = SpaceManager
 dcache.queue.poolmanager = PoolManager
 
+#  -----------------------------------------------------------------------
+#          Topics
+#  -----------------------------------------------------------------------
+#
+#   Topics are used by message publishers and message subscribers. One or
+#   more publishers may write messages to a topic, and one or more
+#   subscribers may read messages from a topic. In contrast to named
+#   queues, a message on a topic is received by all subscribers.
+#
+#   The publish-subscribe paradigm realized by topics is analogous to
+#   multicasting in networking.
+#
+#   The topics here must be simple names - fully qualified addresses are
+#   not allowed.
+#
+#   Note: Currently not all topics are defined like this.
+#
+
+# Monitoring services ping on this topic to discover services to monitor
+dcache.topic.watched = WatchedServiceTopic
+
+# Pools announce their presence, mode and load on this topic
+dcache.topic.pool-heartbeat = PoolUpTopic
+
 #  ---- Topic for loginbroker update
 #
 # Doors periodically publish information about themselves on this topic, and other

skel/share/defaults/gplazma.properties

@@ -49,7 +49,7 @@ gplazma.cell.max-message-threads = 30
 #
 gplazma.cell.max-messages-queued=
 
-#  ---- Named queues to subscribe to
+#  ---- Named queues to consume from
 #
 #   A service can consume messages from named queues. Other service can
 #   write messages to such queues. A named queue has an unqualified cell
@@ -60,6 +60,18 @@ gplazma.cell.max-messages-queued=
 #
 gplazma.cell.consume = ${gplazma.cell.service}
 
+#  ---- Topics to which the service subscribes
+#
+#   A service can subscribe to messages on topics. Other services can
+#   write messages to such topics and all subscribers receive such
+#   messages. A topic has an unqualified cell address, that is, an
+#   address without a domain name.
+#
+#   This property contains a comma separated list of topics to
+#   subscribe to.
+#
+gplazma.cell.subscribe = ${dcache.topic.watched}
+
 #  ---- Location of the configuration file
 #
 #  The location of the gPlazma configuration file.  This controls

skel/share/defaults/httpd.properties

@@ -37,7 +37,7 @@ httpd.net.port = 2288
 # Cell addresses to monitor. Only supported by the old httpd pages. Simple names
 # are expected to be named queues or topics and httpd adds recipients of these
 # addresses to the list of cells to monitor.
-httpd.watch=${httpd.service.pnfsmanager} ${httpd.service.poolmanager} ${httpd.service.gplazma} ${httpd.service.srmmanager}
+httpd.watch = ${dcache.topic.watched}
 
 #
 #     For authenticated pages (all admin pages in the webadmin servlet are

skel/share/defaults/pnfsmanager.properties

@@ -28,7 +28,7 @@ pnfsmanager.cell.name = ${dcache.queue.pnfsmanager}
 #
 (immutable)pnfsmanager.cell.replicable = true
 
-#  ---- Named queues to subscribe to
+#  ---- Named queues to consume from
 #
 #   A service can consume messages from named queues. Other service can
 #   write messages to such queues. A named queue has an unqualified cell
@@ -39,6 +39,18 @@ pnfsmanager.cell.name = ${dcache.queue.pnfsmanager}
 #
 pnfsmanager.cell.consume = ${pnfsmanager.cell.service}
 
+#  ---- Topics to which the service subscribes
+#
+#   A service can subscribe to messages on topics. Other services can
+#   write messages to such topics and all subscribers receive such
+#   messages. A topic has an unqualified cell address, that is, an
+#   address without a domain name.
+#
+#   This property contains a comma separated list of topics to
+#   subscribe to.
+#
+pnfsmanager.cell.subscribe = ${dcache.topic.watched}
+
 #   -- replace with org.dcache.chimera.namespace.ChimeraEnstoreStorageInfoExtractor
 #      if you are running an enstore HSM backend.
 #

skel/share/defaults/pool.properties

@@ -425,7 +425,7 @@ pool.service.door.timeout=3
 (one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)pool.service.door.timeout.unit=SECONDS
 
 # Cell address to which to send poolup messages
-pool.destination.heartbeat = PoolUpTopic
+pool.destination.heartbeat = ${dcache.topic.pool-heartbeat}
 
 # Address of cell to notify with a replication request on arrival of new files
 #

skel/share/defaults/poolmanager.properties

@@ -36,7 +36,7 @@ poolmanager.cell.service = ${dcache.queue.poolmanager}
 poolmanager.cell.consume = ${poolmanager.cell.service}
 
 # Message topics to subscribe to.
-poolmanager.cell.subscribe=PoolUpTopic
+poolmanager.cell.subscribe=${dcache.topic.pool-heartbeat},${dcache.topic.watched}
 
 # Pool manager plugins
 poolmanager.plugins.quota-manager = none

skel/share/defaults/spacemanager.properties

@@ -43,7 +43,7 @@ spacemanager.cell.name = ${dcache.queue.spacemanager}
 #   consume from.
 #
 spacemanager.cell.consume = ${spacemanager.cell.service}
-spacemanager.cell.subscribe=${spacemanager.pool-monitor.topic}
+spacemanager.cell.subscribe=${spacemanager.pool-monitor.topic},${dcache.topic.watched}
 
 # ---- Allow uploads to link groups outside of space reservation
 #

skel/share/defaults/srmmanager.properties

@@ -50,7 +50,7 @@ srmmanager.cell.name = ${dcache.queue.srmmanager}
 #
 (immutable)srmmanager.cell.replicable = true
 
-#  ---- Named queues to subscribe to
+#  ---- Named queues to consume from
 #
 #   A service can consume messages from named queues. Other service can
 #   write messages to such queues. A named queue has an unqualified cell
@@ -61,7 +61,7 @@ srmmanager.cell.name = ${dcache.queue.srmmanager}
 #
 srmmanager.cell.consume=${srmmanager.cell.service}
 
-#  ---- Topics to subscribe to
+#  ---- Topics to which the service subscribes
 #
 #   A service can subscribe to messages on topics. Other services can
 #   write messages to such topics and all subscribers receive such
@@ -71,7 +71,7 @@ srmmanager.cell.consume=${srmmanager.cell.service}
 #   This property contains a comma separated list of topics to
 #   subscribe to.
 #
-srmmanager.cell.subscribe = ${srmmanager.loginbroker.update-topic},${srmmanager.pool-monitor.topic}
+srmmanager.cell.subscribe = ${srmmanager.loginbroker.update-topic},${srmmanager.pool-monitor.topic},${dcache.topic.watched}
 
 # Cell message processing limits
 srmmanager.cell.max-message-threads = 500

skel/share/services/gplazma.batch

@@ -8,7 +8,9 @@ check -strong gplazma.configuration.file
 check -strong gplazma.service.pnfsmanager
 check gplazma.authz.upload-directory
 check gplazma.cell.consume
+check gplazma.cell.subscribe
 
 create org.dcache.cells.UniversalSpringCell "${gplazma.cell.name}" \
    "classpath:org/dcache/services/login/gplazma.xml \
-     -consume=${gplazma.cell.consume} -monitor -cell.max-message-threads=${gplazma.cell.max-message-threads} -cell.max-messages-queued=${gplazma.cell.max-messages-queued}"
+     -consume=${gplazma.cell.consume} -subscribe=${gplazma.cell.subscribe} \
+     -monitor -cell.max-message-threads=${gplazma.cell.max-message-threads} -cell.max-messages-queued=${gplazma.cell.max-messages-queued}"

skel/share/services/pnfsmanager.batch

@@ -19,6 +19,7 @@ check -strong pnfsmanager.limits.log-slow-threshold
 check -strong pnfsmanager.limits.queue-length
 check -strong pnfsmanager.cell.name
 check pnfsmanager.cell.consume
+check pnfsmanager.cell.subscribe
 check -strong pnfsmanager.db.user
 check -strong pnfsmanager.db.url
 check -strong pnfsmanager.db.connections.idle
@@ -28,6 +29,6 @@ check pnfsmanager.db.password.file
 
 create org.dcache.cells.UniversalSpringCell ${pnfsmanager.cell.name} \
    "classpath:diskCacheV111/namespace/pnfsmanager-chimera.xml \
-        -consume=${pnfsmanager.cell.consume} \
+        -consume=${pnfsmanager.cell.consume} -subscribe=${pnfsmanager.cell.subscribe} \
         -namespace-provider=org.dcache.chimera.namespace.ChimeraNameSpaceProviderFactory \
        "