-
Notifications
You must be signed in to change notification settings - Fork 131
/
bulk.xml
338 lines (305 loc) · 16.6 KB
/
bulk.xml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context.xsd">
<context:property-placeholder/>
<context:annotation-config/>
<bean id="pnfs-manager-stub" class="org.dcache.cells.CellStub">
<description>PnfsManager communication stub</description>
<property name="destination" value="${bulk.service.pnfsmanager}"/>
<property name="timeout" value="${bulk.service.pnfsmanager.timeout}"/>
<property name="timeoutUnit" value="${bulk.service.pnfsmanager.timeout.unit}"/>
</bean>
<bean id="pin-manager-stub" class="org.dcache.cells.CellStub">
<description>PinManager communication stub</description>
<property name="destination" value="${bulk.service.pinmanager}"/>
<property name="timeout" value="${bulk.service.pinmanager.timeout}"/>
<property name="timeoutUnit" value="${bulk.service.pinmanager.timeout.unit}"/>
</bean>
<bean id="pool-manager-stub" class="org.dcache.cells.CellStub">
<description>PoolManager communication stub</description>
<property name="destination" value="${bulk.service.poolmanager}"/>
<property name="timeout" value="${bulk.service.poolmanager.timeout}"/>
<property name="timeoutUnit" value="${bulk.service.poolmanager.timeout.unit}"/>
</bean>
<bean id="qos-engine-stub" class="org.dcache.cells.CellStub">
<description>QoS Engine communication stub</description>
<property name="destination" value="${bulk.service.qos}"/>
<property name="timeout" value="${bulk.service.qos.timeout}"/>
<property name="timeoutUnit" value="${bulk.service.qos.timeout.unit}"/>
</bean>
<bean id="ha-leader-stub" class="org.dcache.cells.CellStub">
<description>Endpoint communication stub</description>
<property name="timeout" value="${bulk.service.ha-leader.timeout}"/>
<property name="timeoutUnit" value="${bulk.service.ha-leader.timeout.unit}"/>
</bean>
<bean id="pool-monitor" class="org.dcache.poolmanager.RemotePoolMonitor">
<description>Maintains runtime information about all pools</description>
<property name="poolManagerStub" ref="pool-manager-stub"/>
</bean>
<bean id="list-handler" class="org.dcache.util.list.ListDirectoryHandler">
<description>Client stub for directory listing</description>
<constructor-arg>
<bean class="diskCacheV111.util.PnfsHandler">
<constructor-arg ref="pnfs-manager-stub"/>
</bean>
</constructor-arg>
</bean>
<bean id="incoming-thread-executor" class="org.dcache.util.CDCExecutorServiceDecorator">
<description>Used to execute request message handling</description>
<constructor-arg>
<bean class="org.dcache.util.BoundedCachedExecutor">
<constructor-arg value="${bulk.limits.incoming-request-threads}"/>
</bean>
</constructor-arg>
</bean>
<bean id="container-job-executor" class="org.dcache.util.CDCExecutorServiceDecorator">
<description>Used to execute jobs that are executed by a batch container.</description>
<constructor-arg>
<bean class="org.dcache.util.BoundedCachedExecutor">
<constructor-arg value="${bulk.limits.container-processing-threads}"/>
</bean>
</constructor-arg>
</bean>
<bean id="cancellation-executor" class="org.dcache.util.CDCScheduledExecutorServiceDecorator">
<description>Used to cancel requests.</description>
<constructor-arg>
<bean class="java.util.concurrent.ScheduledThreadPoolExecutor">
<constructor-arg value="${bulk.limits.cancellation-threads}"/>
</bean>
</constructor-arg>
</bean>
<bean id="bulk-data-source" class="com.zaxxer.hikari.HikariDataSource">
<description>Encapsulates the bulk database connection pool and properties.</description>
<constructor-arg>
<bean class="com.zaxxer.hikari.HikariConfig">
<constructor-arg>
<bean class="org.dcache.util.configuration.ConfigurationPropertiesFactoryBean">
<property name="prefix" value="qos.db.verifier.hikari-properties"/>
<property name="staticEnvironment">
<map>
<entry key="jdbcUrl" value="${bulk.db.url}"/>
<entry key="username" value="${bulk.db.user}"/>
<entry key="password"
value="#{ T(diskCacheV111.util.Pgpass).getPassword('${bulk.db.password.file}', '${bulk.db.url}', '${bulk.db.user}', '${bulk.db.password}') }"/>
<entry key="autoCommit" value="true"/>
<entry key="transactionIsolation" value="TRANSACTION_READ_COMMITTED"/>
<entry key="minimumIdle" value="${bulk.db.connections.idle}"/>
<entry key="maximumPoolSize" value="${bulk.db.connections.max}"/>
<entry key="connectionTimeout" value="60000"/>
</map>
</property>
</bean>
</constructor-arg>
</bean>
</constructor-arg>
</bean>
<bean id="bulk-liquibase" class="org.dcache.util.SpringLiquibase">
<description>Bulk database schema manager.</description>
<property name="dataSource" ref="bulk-data-source"/>
<property name="changeLog" value="classpath:${bulk.db.schema.changelog}"/>
<property name="shouldUpdate" value="${bulk.db.schema.auto}"/>
</bean>
<bean id="bulk-jdbc-dao-utils" class="org.dcache.services.bulk.store.jdbc.JdbcBulkDaoUtils">
<description>Bulk data access shared methods.</description>
<property name="fetchSize" value="${bulk.db.fetch-size}"/>
</bean>
<bean id="bulk-request-dao"
class="org.dcache.services.bulk.store.jdbc.request.JdbcBulkRequestDao">
<description>Bulk request data access object handler.</description>
<property name="dataSource" ref="bulk-data-source"/>
<property name="utils" ref="bulk-jdbc-dao-utils"/>
</bean>
<bean id="bulk-request-permissions-dao"
class="org.dcache.services.bulk.store.jdbc.request.JdbcBulkRequestPermissionsDao">
<description>Bulk request permissions access object handler.</description>
<property name="dataSource" ref="bulk-data-source"/>
<property name="utils" ref="bulk-jdbc-dao-utils"/>
</bean>
<bean id="bulk-request-target-dao"
class="org.dcache.services.bulk.store.jdbc.rtarget.JdbcRequestTargetDao">
<description>Bulk request target data access object handler.</description>
<property name="dataSource" ref="bulk-data-source"/>
<property name="utils" ref="bulk-jdbc-dao-utils"/>
<property name="statistics" ref="statistics"/>
</bean>
<bean id="bulk-archive-dao"
class="org.dcache.services.bulk.store.jdbc.request.JdbcBulkArchiveDao">
<description>Archived request data access object handler.</description>
<property name="dataSource" ref="bulk-data-source"/>
<property name="utils" ref="bulk-jdbc-dao-utils"/>
</bean>
<bean id="request-store" class="org.dcache.services.bulk.store.jdbc.request.JdbcBulkRequestStore"
init-method="initialize">
<description>RDBMS implementation of request storage interface</description>
<property name="requestDao" ref="bulk-request-dao"/>
<property name="requestTargetDao" ref="bulk-request-target-dao"/>
<property name="requestPermissionsDao" ref="bulk-request-permissions-dao"/>
<property name="archiveDao" ref="bulk-archive-dao"/>
<property name="targetStore" ref="target-store"/>
<property name="expiry" value="${bulk.limits.request-cache-expiration}"/>
<property name="expiryUnit" value="${bulk.limits.request-cache-expiration.unit}"/>
<property name="capacity" value="${bulk.limits.container-processing-threads}"/>
<property name="statistics" ref="statistics"/>
<property name="pnfsManager" ref="pnfs-manager-stub"/>
</bean>
<bean id="target-store" class="org.dcache.services.bulk.store.jdbc.rtarget.JdbcBulkTargetStore">
<description>RDBMS implementation of target storage interface</description>
<property name="targetDao" ref="bulk-request-target-dao"/>
</bean>
<bean id="qos-response-receiver" class="org.dcache.services.bulk.activity.plugin.qos.QoSResponseReceiver">
<description>Relays messages received from the qos-engine concerning update completion.</description>
</bean>
<bean id="activity-factory" class="org.dcache.services.bulk.activity.BulkActivityFactory">
<description>Factory for creating various types of bulk activities.</description>
<property name="pnfsManager" ref="pnfs-manager-stub"/>
<property name="pinManager" ref="pin-manager-stub"/>
<property name="poolManager" ref="pool-manager-stub"/>
<property name="qosEngine" ref="qos-engine-stub"/>
<property name="poolMonitor" ref="pool-monitor"/>
<property name="qoSResponseReceiver" ref="qos-response-receiver"/>
<!-- while it is possible to specify these separately, for now,
all activities share the same two executors -->
<property name="retryPolicies">
<map>
<entry key="DELETE"><bean class="${bulk.plugin!delete.retry-policy}"/></entry>
<entry key="PIN"><bean class="${bulk.plugin!pin.retry-policy}"/></entry>
<entry key="STAGE"><bean class="${bulk.plugin!stage.retry-policy}"/></entry>
<entry key="UNPIN"><bean class="${bulk.plugin!unpin.retry-policy}"/></entry>
<entry key="RELEASE"><bean class="${bulk.plugin!release.retry-policy}"/></entry>
<entry key="UPDATE_QOS"><bean class="${bulk.plugin!update-qos.retry-policy}"/></entry>
<entry key="LOG_TARGET"><bean class="${bulk.plugin!log-target.retry-policy}"/></entry>
</map>
</property>
</bean>
<bean id="job-factory" class="org.dcache.services.bulk.job.RequestContainerJobFactory">
<description>Constructs jobs from activities and targets</description>
<property name="activityFactory" ref="activity-factory"/>
<property name="listHandler" ref="list-handler"/>
<property name="pnfsManager" ref="pnfs-manager-stub"/>
<property name="requestStore" ref="request-store"/>
<property name="targetStore" ref="target-store"/>
<property name="statistics" ref="statistics"/>
<property name="dirListSemaphore" value="${bulk.limits.dir-list-semaphore}"/>
<property name="inFlightSemaphore" value="${bulk.limits.in-flight-semaphore}"/>
</bean>
<bean id="statistics" class="org.dcache.services.bulk.util.BulkServiceStatistics">
<description>Tracks request and target states (counts), sweeper state, etc.</description>
<property name="requestContainerJobFactory" ref="job-factory"/>
</bean>
<bean id="request-handler" class="org.dcache.services.bulk.handler.BulkRequestHandler">
<description>Handles submission, cancellation and completion of requests.</description>
<property name="requestStore" ref="request-store"/>
<property name="targetStore" ref="target-store"/>
<property name="statistics" ref="statistics"/>
<property name="jobFactory" ref="job-factory"/>
<property name="requestManager" ref="request-manager"/>
<property name="cancelExecutor" ref="cancellation-executor"/>
</bean>
<bean id="scheduler-provider" class="org.dcache.services.bulk.manager.scheduler.BulkSchedulerProvider">
<description>Furnishes the request and job schedulers to the job processor.</description>
<property name="requestScheduler">
<bean class="${bulk.request-scheduler}"/>
</property>
<property name="requestStore" ref="request-store"/>
</bean>
<bean id="request-manager" class="org.dcache.services.bulk.manager.ConcurrentRequestManager">
<description>Core of the service which manages the request and target job lifecycle.</description>
<property name="statistics" ref="statistics"/>
<property name="targetStore" ref="target-store"/>
<property name="requestStore" ref="request-store"/>
<property name="completionHandler" ref="request-handler"/>
<property name="submissionHandler" ref="request-handler"/>
<property name="schedulerProvider" ref="scheduler-provider"/>
<property name="maxActiveRequests" value="${bulk.limits.container-processing-threads}"/>
<property name="executor">
<bean class="org.dcache.util.BoundedCachedExecutor">
<constructor-arg value="${bulk.limits.in-flight-semaphore}"/>
</bean>
</property>
<property name="callbackExecutor">
<bean class="org.dcache.util.BoundedCachedExecutor">
<constructor-arg value="${bulk.db.connections.max}"/>
</bean>
</property>
<property name="timeout" value="${bulk.limits.sweep-interval}"/>
<property name="timeoutUnit" value="${bulk.limits.sweep-interval.unit}"/>
</bean>
<bean id="bulk-service" class="org.dcache.services.bulk.BulkService">
<description>Initializes and starts the request manager; handles incoming messages.</description>
<property name="incomingExecutorService" ref="incoming-thread-executor"/>
<property name="maxRequestsPerUser" value="${bulk.limits.max-requests-per-user}"/>
<property name="maxFlatTargets" value="${bulk.limits.max.targets-per-flat-request}"/>
<property name="maxShallowTargets" value="${bulk.limits.max.targets-per-shallow-request}"/>
<property name="maxRecursiveTargets" value="${bulk.limits.max.targets-per-recursive-request}"/>
<property name="allowedDepth" value="${bulk.allowed-directory-expansion}"/>
<property name="requestStore" ref="request-store"/>
<property name="activityFactory" ref="activity-factory"/>
<property name="statistics" ref="statistics"/>
<property name="namespace" ref="pnfs-manager-stub"/>
<property name="submissionHandler" ref="request-handler"/>
<property name="requestManager" ref="request-manager"/>
<property name="targetStore" ref="target-store"/>
<property name="leadershipManager" ref="ha-service-leadership-manager"/>
<property name="endpointStub" ref="ha-leader-stub"/>
</bean>
<bean id="admin-commands" class="org.dcache.services.bulk.BulkServiceCommands">
<description>Holds the various commands for administrative purposes.</description>
<property name="service" ref="bulk-service"/>
<property name="activityFactory" ref="activity-factory"/>
<property name="requestStore" ref="request-store"/>
<property name="targetStore" ref="target-store"/>
<property name="requestManager" ref="request-manager"/>
<property name="submissionHandler" ref="request-handler"/>
<property name="statistics" ref="statistics"/>
<property name="archiver" ref="request-archiver"/>
<property name="archiveDao" ref="bulk-archive-dao"/>
</bean>
<bean id="shutdownManager" class="org.dcache.services.bulk.util.BulkShutdownManager" destroy-method="shutdown">
<property name="executorList">
<list>
<ref bean="incoming-thread-executor"/>
<ref bean="container-job-executor"/>
<ref bean="cancellation-executor"/>
</list>
</property>
<property name="dataSource" ref="bulk-data-source"/>
<property name="await" value="${bulk.limits.shutdown-manager-wait}"/>
<property name="awaitUnit" value="${bulk.limits.shutdown-manager-wait.unit}"/>
</bean>
<bean id="request-archiver" class="org.dcache.services.bulk.store.jdbc.request.JdbcBulkRequestArchiver"
destroy-method="shutdown">
<property name="requestDao" ref="bulk-request-dao"/>
<property name="requestStore" ref="request-store"/>
<property name="archiverScheduler">
<bean class="java.util.concurrent.ScheduledThreadPoolExecutor">
<constructor-arg value="1"/>
</bean>
</property>
<property name="archiverPeriod" value="${bulk.limits.archiver-period}"/>
<property name="archiverPeriodUnit" value="${bulk.limits.archiver-period.unit}"/>
<property name="archiverWindow" value="${bulk.limits.archiver-window}"/>
<property name="archiverWindowUnit" value="${bulk.limits.archiver-window.unit}"/>
</bean>
<bean id="ha-service-leadership-manager" class="org.dcache.cells.HAServiceLeadershipManager"
init-method="initZkLeaderListener" destroy-method="shutdown">
<description>Coordinates which bulk service handles the requests</description>
<constructor-arg value="bulk"/>
<property name="leadershipListener">
<ref bean="leaderlistener-group"/>
</property>
</bean>
<bean id="leaderlistener-group" class="org.dcache.cells.LeadershipListenerGroup">
<description>Propagates leadership change notifications to managed listeners</description>
<property name="leaderElectionAwareComponents">
<set>
<ref bean="bulk-service"/>
<ref bean="request-archiver"/>
</set>
</property>
</bean>
</beans>