billing: Make service replicable

Motivation:

In a replicated billing setup, one would expect all billing records to
reach all instances, while queries from the statistics service would
reach a single of those.

Modification:

Introduce a billing topic. All services announce billing records on
this topic. Introduce a billing queue. The statistics service queries
on this queue.

Extend the statistics service to not hard-code cell names.

Backwards compatibility is achieved by using 'billing' as the name
of the billing queue. Thus a replicated setup for billing only works
out of the box when all components have upgraded to 2.17.

Result:

Billing service is replicable in all components run 2.17 or newer.  Introduced
statistics.service.poolmanager, statistics.service.pool, and
statistics.service.billing property trees. Introduced dcache.queue.billing and
dcache.topic.billing. Obsoleted dcache.service.billing.

Target: trunk
Require-notes: yes
Require-book: yes
Acked-by: Paul Millar <paul.millar@desy.de>

Reviewed at https://rb.dcache.org/r/9637/

modules/dcache/src/main/java/diskCacheV111/services/PoolStatisticsV0.java

@@ -30,6 +30,7 @@
 import java.util.Map;
 import java.util.StringTokenizer;
 import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
 
 import diskCacheV111.poolManager.PoolManagerCellInfo;
@@ -267,9 +268,14 @@ protected void startUp() throws Exception
                     "[-htmlBase=<htmlBase>|none] [-create] [-images=<images>]");
         }
 
-        _poolManager = new CellStub(this, new CellPath("PoolManager"), 20000);
-        _billing = new CellStub(this, new CellPath("billing"), 20000);
-        _poolStub = new CellStub(this, null, 20000);
+        _poolManager = new CellStub(this, new CellPath(args.getOption("poolManager")),
+                                    TimeUnit.MILLISECONDS.convert(args.getLongOption("poolManagerTimeout"),
+                                                                  TimeUnit.valueOf(args.getOption("poolManagerTimeoutUnit"))));
+        _billing = new CellStub(this, new CellPath(args.getOption("billing")),
+                                TimeUnit.MILLISECONDS.convert(args.getLongOption("billingTimeout"),
+                                                              TimeUnit.valueOf(args.getOption("billingTimeoutUnit"))));
+        _poolStub = new CellStub(this, null, TimeUnit.MILLISECONDS.convert(args.getLongOption("poolTimeout"),
+                                                                           TimeUnit.valueOf(args.getOption("poolTimeoutUnit"))));
 
         _htmlBase = _dbBase = new File(args.argv(0));
 

skel/share/defaults/billing.properties

@@ -26,7 +26,22 @@ billing.cell.consume=${billing.cell.name}
 #
 #   This property contains a comma separated list of topics to subscribe to.
 #
-billing.cell.subscribe=
+billing.cell.subscribe=${dcache.topic.billing}
+
+#  ----- Whether the service is replicable
+#
+#   Any service in dCache can have several instances as long as these
+#   represent separate logical services. Some services can have several
+#   instances representing the same logical service, providing some
+#   degree of fault tolerance and load balancing. Such services are said
+#   to be replicable.
+#
+#   Instances of a logical service share the same service name, and it is
+#   important that the configuration for such instances is synchronized.
+#
+#   This property indicates if this service is replicable.
+#
+(immutable)billing.cell.replicable = true
 
 #  ---- Disable billing to plain text file
 #

skel/share/defaults/dcache.properties

@@ -418,7 +418,6 @@ dcache.service.alarms = ${alarms.cell.name}
 dcache.service.acm = acm
 dcache.service.pnfsmanager = ${dcache.queue.pnfsmanager}
 dcache.service.poolmanager = ${dcache.queue.poolmanager}
-dcache.service.billing = ${billing.cell.name}
 dcache.service.gplazma = ${dcache.queue.gplazma}
 dcache.service.spacemanager = ${dcache.queue.spacemanager}
 dcache.service.pinmanager = ${dcache.queue.pinmanager}
@@ -456,6 +455,7 @@ dcache.queue.srmmanager = SrmManager
 dcache.queue.pinmanager = PinManager
 dcache.queue.spacemanager = SpaceManager
 dcache.queue.poolmanager = PoolManager
+dcache.queue.billing = billing
 
 #  -----------------------------------------------------------------------
 #          Topics
@@ -481,6 +481,9 @@ dcache.topic.watched = WatchedServiceTopic
 # Pools announce their presence, mode and load on this topic
 dcache.topic.pool-heartbeat = PoolUpTopic
 
+# Services publish billing records on this topic
+dcache.topic.billing = BillingTopic
+
 #  ---- Topic for loginbroker update
 #
 # Doors periodically publish information about themselves on this topic, and other
@@ -1018,3 +1021,4 @@ dcache.description =
 
 (obsolete)dcache.broker.host = See dcache.zookeeper.connection
 (obsolete)dcache.broker.domain = See dcache.broker.scheme
+(obsolete)dcache.service.billing = See dcache.queue.billing and dcache.topic.billing

skel/share/defaults/dcap.properties

@@ -109,7 +109,7 @@ dcap.service.pnfsmanager=${dcache.service.pnfsmanager}
 dcap.service.gplazma=${dcache.service.gplazma}
 
 # Cell address of billing service
-dcap.service.billing=${dcache.service.billing}
+dcap.service.billing=${dcache.topic.billing}
 
 # Cell address of spacemanager service
 dcap.service.spacemanager=${dcache.service.spacemanager}

skel/share/defaults/ftp.properties

@@ -240,7 +240,7 @@ ftp.service.pnfsmanager.timeout=300
 ftp.service.gplazma=${dcache.service.gplazma}
 
 #  Cell address of billing service
-ftp.service.billing=${dcache.service.billing}
+ftp.service.billing=${dcache.topic.billing}
 
 #  Topic to publish login broker updates to
 ftp.loginbroker.update-topic=${dcache.loginbroker.update-topic}

skel/share/defaults/httpd.properties

@@ -19,7 +19,7 @@ httpd.setup=${dcache.paths.etc}/httpd.conf
 #
 httpd.service.info = ${dcache.queue.info}
 httpd.service.alarms=${dcache.service.alarms}
-httpd.service.billing=${dcache.service.billing}
+httpd.service.billing=${dcache.queue.billing}
 httpd.service.gplazma=${dcache.service.gplazma}
 httpd.service.pnfsmanager=${dcache.service.pnfsmanager}
 httpd.service.poolmanager=${dcache.service.poolmanager}

skel/share/defaults/nfs.properties

@@ -116,7 +116,7 @@ nfs.service.gplazma.timeout = 3000
 (one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)nfs.service.gplazma.timeout.unit=MILLISECONDS
 
 nfs.service.pnfsmanager=${dcache.service.pnfsmanager}
-nfs.service.billing=${dcache.service.billing}
+nfs.service.billing=${dcache.topic.billing}
 
 #
 # Database related settings reserved for internal use.

skel/share/defaults/pool.properties

@@ -397,7 +397,7 @@ pool.service.pnfsmanager.timeout=300
 (one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)pool.service.pnfsmanager.timeout.unit=SECONDS
 
 # Cell address of billing service
-pool.service.billing=${dcache.service.billing}
+pool.service.billing=${dcache.topic.billing}
 
 # Timeout for requests to other pools
 pool.service.pool.timeout=60

skel/share/defaults/poolmanager.properties

@@ -45,7 +45,7 @@ poolmanager.plugins.quota-manager = none
 poolmanager.service.pnfsmanager=${dcache.service.pnfsmanager}
 
 # Cell address of billing service
-poolmanager.service.billing=${dcache.service.billing}
+poolmanager.service.billing=${dcache.topic.billing}
 
 # Cell address to which to send pool status changes
 poolmanager.destination.pool-status=PoolStatusTopic

skel/share/defaults/srmmanager.properties

@@ -894,7 +894,7 @@ srmmanager.service.transfermanager.timeout=24
 srmmanager.service.transfermanager.timeout.unit=HOURS
 
 # Cell address of billing service
-srmmanager.service.billing = ${dcache.service.billing}
+srmmanager.service.billing = ${dcache.topic.billing}
 
 # Cell address of pinmanager service
 srmmanager.service.pinmanager=${dcache.service.pinmanager}

skel/share/defaults/statistics.properties

@@ -12,3 +12,13 @@ statistics.cell.name=PoolStatistics
 #
 statistics.location = ${dcache.paths.statistics}
 
+statistics.service.billing=${dcache.queue.billing}
+statistics.service.billing.timeout=20
+(one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)statistics.service.billing.timeout.unit=SECONDS
+
+statistics.service.poolmanager=${dcache.service.poolmanager}
+statistics.service.poolmanager.timeout=20
+(one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)statistics.service.poolmanager.timeout.unit=SECONDS
+
+statistics.service.pool.timeout=20
+(one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)statistics.service.pool.timeout.unit=SECONDS

skel/share/defaults/transfermanagers.properties

@@ -40,7 +40,7 @@ transfermanagers.service.poolmanager.timeout = 300
 transfermanagers.service.spacemanager=${dcache.service.spacemanager}
 
 # Cell address of billing service
-transfermanagers.service.billing=${dcache.service.billing}
+transfermanagers.service.billing=${dcache.topic.billing}
 
 # Timeout for billing requests
 transfermanagers.service.billing.timeout = 300

skel/share/defaults/webdav.properties

@@ -62,7 +62,7 @@ webdav.service.gplazma.timeout = 180000
 (one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)webdav.service.gplazma.timeout.unit=MILLISECONDS
 
 # Cell address of billing service
-webdav.service.billing=${dcache.service.billing}
+webdav.service.billing=${dcache.topic.billing}
 
 # Cell address of missing-files service
 webdav.service.missing-files=${dcache.service.missing-files}

skel/share/defaults/xrootd.properties

@@ -63,7 +63,7 @@ xrootd.service.gplazma.timeout=30
 (one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)xrootd.service.gplazma.timeout.unit=SECONDS
 
 # Cell address of billing service
-xrootd.service.billing=${dcache.service.billing}
+xrootd.service.billing=${dcache.topic.billing}
 
 # Timeout for pool communication
 xrootd.service.pool.timeout = 15000

skel/share/services/statistics.batch

@@ -1,10 +1,26 @@
 onerror shutdown
 check -strong statistics.cell.name
 check -strong statistics.location
+check -strong statistics.service.billing
+check -strong statistics.service.billing.timeout
+check -strong statistics.service.billing.timeout.unit
+check -strong statistics.service.poolmanager
+check -strong statistics.service.poolmanager.timeout
+check -strong statistics.service.poolmanager.timeout.unit
+check -strong statistics.service.pool.timeout
+check -strong statistics.service.pool.timeout.unit
 
 create diskCacheV111.services.PoolStatisticsV0 ${statistics.cell.name}  \
      "${statistics.location}  \
 #      -create  \
 #      -htmlBase=${statistics.location}  \
+      -poolManager=${statistics.service.poolmanager} \
+      -poolManagerTimeout=${statistics.service.poolmanager.timeout} \
+      -poolManagerTimeoutUnit=${statistics.service.poolmanager.timeout.unit} \
+      -billing=${statistics.service.billing} \
+      -billingTimeout=${statistics.service.billing.timeout} \
+      -billingTimeoutUnit=${statistics.service.billing.timeout.unit} \
+      -poolTimeout=${statistics.service.pool.timeout} \
+      -poolTimeoutUnit=${statistics.service.pool.timeout.unit} \
       -domain=${thisFqHostname}"