Permalink
Browse files

Manual squashed merge of the services-r10 branch.

Not done using 'git merge' to work around the gerrit/merge issue.

Also bumped cf-release/src/{services,vblob_src,tools}, and made
debian_nfs_server no_root_squash configurable from the manifest.

Change-Id: Ib689f169a3798f4421bdcabe74ff0b012819ecbf
  • Loading branch information...
1 parent 9ae8a90 commit 7a5d592bda032e0eaec0409e8dd189a8cd340039 @kushmerick kushmerick committed May 4, 2012
Showing with 432 additions and 689 deletions.
  1. +0 −8 config/blobs.yml
  2. +1 −0 jobs/backup_manager/spec
  3. +2 −0 jobs/backup_manager/templates/backup_manager.yml.erb
  4. +0 −4 jobs/cloud_controller/templates/cloud_controller.yml.erb
  5. +2 −1 jobs/debian_nfs_server/templates/exports.erb
  6. +0 −5 jobs/micro/monit.yml
  7. +0 −2 jobs/micro/prepare
  8. +8 −7 jobs/mongodb_gateway/templates/mongodb_gateway.yml.erb
  9. +7 −8 jobs/mongodb_gateway/templates/mongodb_gateway_ctl
  10. +19 −6 jobs/mongodb_node/monit
  11. +1 −1 jobs/mongodb_node/templates/mongodb_backup.cron.erb
  12. +1 −1 jobs/mongodb_node/templates/mongodb_backup.erb
  13. +1 −1 jobs/mongodb_node/templates/mongodb_backup.yml.erb
  14. +1 −1 jobs/mongodb_node/templates/mongodb_node.yml.erb
  15. +14 −10 jobs/mongodb_node/templates/mongodb_node_ctl
  16. +12 −11 jobs/mongodb_node/templates/mongodb_worker.yml.erb
  17. +5 −23 jobs/mongodb_node/templates/mongodb_worker_ctl
  18. +8 −7 jobs/mysql_gateway/templates/mysql_gateway.yml.erb
  19. +7 −7 jobs/mysql_gateway/templates/mysql_gateway_ctl
  20. +19 −6 jobs/mysql_node/monit
  21. +1 −1 jobs/mysql_node/templates/mysql_backup.cron.erb
  22. +1 −1 jobs/mysql_node/templates/mysql_backup.erb
  23. +1 −1 jobs/mysql_node/templates/mysql_backup.yml.erb
  24. +1 −1 jobs/mysql_node/templates/mysql_node.yml.erb
  25. +14 −10 jobs/mysql_node/templates/mysql_node_ctl
  26. +12 −11 jobs/mysql_node/templates/mysql_worker.yml.erb
  27. +5 −23 jobs/mysql_node/templates/mysql_worker_ctl
  28. +7 −0 jobs/opentsdb/templates/opentsdb_ctl
  29. +1 −0 jobs/postgresql_gateway/spec
  30. +14 −1 jobs/postgresql_gateway/templates/postgresql_gateway.yml.erb
  31. +24 −0 jobs/postgresql_gateway/templates/postgresql_gateway_ctl
  32. +22 −0 jobs/postgresql_node/monit
  33. +3 −2 jobs/postgresql_node/spec
  34. +1 −1 jobs/postgresql_node/templates/postgresql_backup.cron.erb
  35. +1 −1 jobs/postgresql_node/templates/postgresql_backup.erb
  36. +2 −1 jobs/postgresql_node/templates/postgresql_backup.yml.erb
  37. +2 −1 jobs/postgresql_node/templates/postgresql_node.yml.erb
  38. +12 −5 jobs/postgresql_node/templates/postgresql_node_ctl
  39. +39 −0 jobs/postgresql_node/templates/postgresql_worker.yml.erb
  40. +35 −0 jobs/postgresql_node/templates/postgresql_worker_ctl
  41. +1 −1 jobs/rabbit_gateway/templates/rabbit_gateway.yml.erb
  42. +1 −1 jobs/rabbit_node/templates/rabbit_node.yml.erb
  43. +0 −5 jobs/rabbitmq_srs_gateway/monit
  44. +0 −10 jobs/rabbitmq_srs_gateway/spec
  45. +0 −54 jobs/rabbitmq_srs_gateway/templates/rabbitmq_srs_vcap_web_ctl
  46. +0 −17 jobs/rabbitmq_srs_gateway/templates/srs-config.json.erb
  47. +0 −5 jobs/rabbitmq_srs_node/monit
  48. +0 −11 jobs/rabbitmq_srs_node/spec
  49. +0 −56 jobs/rabbitmq_srs_node/templates/rabbitmq_srs_agent_ctl
  50. +0 −8 jobs/rabbitmq_srs_node/templates/srs-config.json.erb
  51. +0 −58 jobs/rabbitmq_srs_service/README
  52. +0 −12 jobs/rabbitmq_srs_service/monit
  53. +0 −16 jobs/rabbitmq_srs_service/spec
  54. +0 −10 jobs/rabbitmq_srs_service/templates/rabbitmq.config
  55. +0 −7 jobs/rabbitmq_srs_service/templates/rabbitmq.env.erb
  56. +0 −44 jobs/rabbitmq_srs_service/templates/rabbitmq_ctl.erb
  57. +0 −53 jobs/rabbitmq_srs_service/templates/rabbitmq_srs_controller_ctl
  58. +0 −8 jobs/rabbitmq_srs_service/templates/srs-config.json.erb
  59. +8 −7 jobs/redis_gateway/templates/redis_gateway.yml.erb
  60. +7 −8 jobs/redis_gateway/templates/redis_gateway_ctl
  61. +19 −6 jobs/redis_node/monit
  62. +1 −1 jobs/redis_node/templates/redis_backup.cron.erb
  63. +1 −1 jobs/redis_node/templates/redis_backup.erb
  64. +1 −1 jobs/redis_node/templates/redis_backup.yml.erb
  65. +2 −1 jobs/redis_node/templates/redis_node.yml.erb
  66. +14 −10 jobs/redis_node/templates/redis_node_ctl
  67. +12 −11 jobs/redis_node/templates/redis_worker.yml.erb
  68. +5 −23 jobs/redis_node/templates/redis_worker_ctl
  69. +10 −7 jobs/serialization_data_server/templates/serialization_data_server.yml.erb
  70. +8 −9 jobs/serialization_data_server/templates/serialization_data_server_ctl
  71. +1 −1 jobs/vblob_gateway/templates/vblob_gateway.yml.erb
  72. +4 −5 jobs/vblob_node/templates/vblob_node.yml.erb
  73. +17 −9 jobs/vcap_redis/templates/vcap_redis.conf.erb
  74. +8 −0 jobs/vcap_redis/templates/vcap_redis_ctl
  75. +1 −1 packages/backup_manager/packaging
  76. +1 −0 packages/backup_manager/spec
  77. +1 −0 packages/postgresql_node/spec
  78. +0 −11 packages/rabbitmq_srs/packaging
  79. +0 −12 packages/rabbitmq_srs/pre_packaging
  80. +0 −7 packages/rabbitmq_srs/spec
  81. +0 −6 packages/rabbitmq_srs_builds/packaging
  82. +0 −4 packages/rabbitmq_srs_builds/spec
  83. +1 −1 src/services
  84. +1 −1 src/tools
  85. +1 −1 src/vblob_src
View
@@ -454,14 +454,6 @@ ruby/ruby-1.8.7-p357.tar.bz2:
object_id: eyJvaWQiOiI0ZTRlNzhiY2ExMWUxMjEyMDRlNGU4NmVlMTUyOTQwNGYzMDcx%0ANDUzZGFmZiIsInNpZyI6IkFhZCt6VlRiMDd1dVBMa24vZFh2SnVHVUtldz0i%0AfQ==%0A
sha: ba5ec0ab663976f00fb405be6cfef546bcf0e4e9
size: 4208157
-rabbitmq-srs/rabbitmq-srs-builds-c908e85ec005.tar.bz2:
- object_id: eyJvaWQiOiI0ZTRlNzhiY2E1MWUxMjIyMDRlNGU5ODYzZjI4ZjMwNGYzMDcx%0ANTUxMmRkOSIsInNpZyI6IlJiUjV3Rm80QTZHQ2ZsSmFlTDl2SExpMjBGWT0i%0AfQ==%0A
- sha: adcd301df8d92516a19d27f6f58c2c77bad4f772
- size: 3309874
-rabbitmq-srs/rabbitmq-srs-ab18c8ca0059.tar.bz2:
- object_id: eyJvaWQiOiI0ZTRlNzhiY2E2MWUxMjEyMDRlNGU4NmVlYmU1OTEwNGYzMDcx%0ANjM3NWU2MyIsInNpZyI6ImtORHZvbS9CdkUxS1pEVU14dEhJbDB2RU9MUT0i%0AfQ==%0A
- sha: bb912925826abc234652e3b1215d36cc3eec09ff
- size: 290143
rabbitmq/rabbitmq-server-with-plugins-generic-unix-2.4.1.tar.gz:
object_id: eyJvaWQiOiI0ZTRlNzhiY2ExMWUxMjEwMDRlNGU3ZDUxMWY4MjEwNGY0NGYx%0AYmRjYmY0NSIsInNpZyI6IkZXWUQwYi9oQmtRdW9adjNJb0d6K1lzd0NuND0i%0AfQ==%0A
sha: dfdfcf988a780af034c1568cae96e6a84e0c3668
View
@@ -10,5 +10,6 @@ packages:
- common
- backup_manager
- ruby
+ - sqlite
- syslog_aggregator
@@ -1,4 +1,6 @@
---
+mbus: nats://<%= properties.nats.user %>:<%= properties.nats.password %>@<%= properties.nats.address %>:<%= properties.nats.port %>
+z_interval: <%= properties.service_backup_manager && properties.service_backup_manager.z_interval || 30 %>
logging:
level: debug
file: /var/vcap/sys/log/backup_manager/backup_manager.log
@@ -164,10 +164,6 @@ builtin_services:
mongodb:
token: <%= properties.mongodb_gateway.token %>
<% end %>
-<% if properties.rabbitmq_srs %>
- rabbitmq:
- token: <%= properties.rabbitmq_srs.token %>
-<% end %>
<% if properties.rabbit_gateway %>
rabbitmq:
token: <%= properties.rabbit_gateway.token %>
@@ -1 +1,2 @@
-/var/vcap/store/shared <%= properties.nfs_server.network %>(rw,fsid=0,no_subtree_check,sync)
+<% no_root_squash = (properties.debian_nfs_server && properties.debian_nfs_server.no_root_squash) ? ',no_root_squash' : '' %>
+/var/vcap/store/shared <%= properties.nfs_server.network %>(rw,fsid=0,no_subtree_check,sync<%= no_root_squash %>)
View
@@ -15,11 +15,6 @@ mysql:
postgres:
- postgresql_node
- postgresql_gateway
-rabbitmq:
- - rabbitmq_srs_agent
- - rabbitmq_srs_vcap_web
- - rabbitmq_srs_controller
- - rabbitmq
redis:
- redis_node
- redis_gateway
View
@@ -13,8 +13,6 @@ def job_dir(job)
end
JOBS = YAML.load_file("monit.yml")
-# HACK: since rabbitmq_srs doesn't follow the convention we can't just load it :(
-JOBS['rabbitmq'] = %w[rabbitmq_srs_node rabbitmq_srs_gateway rabbitmq_srs_service]
spec = {
"name" => "micro",
@@ -2,10 +2,11 @@
<%
service = "mongodb"
acls = properties.mongodb_gateway.acls
-plan_enabled = properties.service_plans
+plan_enabled = properties.service_plans && properties.service_plans.send(service.to_sym)
plan_mgmt = plan_enabled && properties.service_plans.send(service.to_sym)
plan_mgmt.instance_eval("def fields; @table.keys; end;") if plan_mgmt
nats = "nats://#{properties.nats.user}:#{properties.nats.password}@#{properties.nats.address}:#{properties.nats.port}"
+lifecycle = properties.service_lifecycle
%>
mbus: <%= nats %>
index: <%= spec.index %>
@@ -55,17 +56,17 @@ check_orphan_interval: <%= properties.mongodb_gateway.check_orphan_interval || -
double_check_orphan_interval: <%= properties.mongodb_gateway.double_check_orphan_interval || 300 %>
max_nats_payload: <%= properties.nats.max_payload || 1048576 %>
-<% if properties.service_snapshot and properties.service_snapshot.resque %>
+<% if lifecycle and lifecycle.resque %>
api_extensions:
- snapshots
- serialization
- jobs
resque:
- host: <%= properties.service_snapshot.resque.host %>
- port: <%= properties.service_snapshot.resque.port %>
- password: <%= properties.service_snapshot.resque.password %>
- expire: <%= properties.service_snapshot.resque.expire %>
-upload_temp_dir: <%= properties.service_snapshot.tmp_dir || "/tmp" %>
+ host: <%= lifecycle.resque.host %>
+ port: <%= lifecycle.resque.port %>
+ password: <%= lifecycle.resque.password %>
+ expire: <%= lifecycle.resque.expire %>
+upload_temp_dir: <%= lifecycle.tmp_dir || "/tmp" %>
<% end %>
<% if plan_enabled %>
@@ -9,11 +9,11 @@ PIDFILE=$RUN_DIR/mongodb_gateway.pid
JOB_DIR=/var/vcap/jobs/mongodb_gateway
<%
-snapshot = properties.service_snapshot
+lifecycle = properties.service_lifecycle
%>
-<% if snapshot %>
-SNAPSHOT_DIR=<%= snapshot.mount_point || "/var/vcap/services_snapshot" %>
+<% if lifecycle %>
+SNAPSHOT_DIR=<%= lifecycle.mount_point || "/var/vcap/services_snapshot" %>
<% end %>
source /var/vcap/packages/common/utils.sh
@@ -32,14 +32,14 @@ case $1 in
/var/vcap/packages/syslog_aggregator/setup_syslog_forwarder.sh $JOB_DIR/config
<% end %>
- <% if snapshot %>
+ <% if lifecycle %>
if [ ! -d $SNAPSHOT_DIR ]; then
mkdir -p $SNAPSHOT_DIR
fi
<% end %>
- <% if snapshot && snapshot.nfs_server %>
- mount -t nfs <%= snapshot.nfs_server.address %>:<%= snapshot.nfs_server.export_dir %> $SNAPSHOT_DIR
+ <% if lifecycle && lifecycle.nfs_server %>
+ mount -t nfs <%= lifecycle.nfs_server.address %>:<%= lifecycle.nfs_server.export_dir %> $SNAPSHOT_DIR
<% end %>
exec /var/vcap/packages/mongodb_gateway/bin/services/mongodb_gateway \
@@ -51,8 +51,7 @@ case $1 in
stop)
kill_and_wait $PIDFILE
-
- <% if snapshot && snapshot.nfs_server %>
+ <% if lifecycle && lifecycle.nfs_server %>
umount $SNAPSHOT_DIR
<% end %>
View
@@ -4,10 +4,23 @@ check process mongodb_node
stop program "/var/vcap/jobs/mongodb_node/bin/mongodb_node_ctl stop"
group vcap
-<% if properties.service_snapshot %>
-check process mongodb_worker
- with pidfile /var/vcap/sys/run/mongodb_node/mongodb_worker.pid
- start program "/var/vcap/jobs/mongodb_node/bin/mongodb_worker_ctl start"
- stop program "/var/vcap/jobs/mongodb_node/bin/mongodb_worker_ctl stop"
+<%
+service = "mongodb"
+plan_enabled = properties.service_plans && properties.service_plans.send(service.to_sym)
+plan = properties.plan || "free"
+plan_conf = plan_enabled && properties.service_plans.send(service.to_sym).send(plan.to_sym).configuration
+if properties.service_lifecycle and plan_enabled and plan_conf.lifecycle and plan_conf.lifecycle.enable
+ worker_count = plan_conf.lifecycle.worker_count || 1
+ worker_count.times do |ind|
+%>
+check process mongodb_worker_<%= ind %>
+ with pidfile /var/vcap/sys/run/mongodb_node/mongodb_worker_<%= ind %>.pid
+ start program "/var/vcap/jobs/mongodb_node/bin/mongodb_worker_ctl start <%= ind %>"
+ stop program "/var/vcap/jobs/mongodb_node/bin/mongodb_worker_ctl stop <%= ind %>"
+ group worker
group vcap
-<% end %>
+ depends on mongodb_node
+<%
+ end
+end
+%>
@@ -3,7 +3,7 @@ service = "mongodb"
cron_time = "0 5 * * *"
interval = 5
-plan_enabled = properties.service_plans
+plan_enabled = properties.service_plans && properties.service_plans.send(service.to_sym)
plan = properties.plan || "free"
plan_conf = plan_enabled && properties.service_plans.send(service.to_sym).send(plan.to_sym).configuration
backup_enabled = plan_enabled && plan_conf.backup && plan_conf.enable
@@ -1,7 +1,7 @@
#!/bin/bash
<%
service = "mongodb"
-plan_enabled = properties.service_plans
+plan_enabled = properties.service_plans && properties.service_plans.send(service.to_sym)
plan = properties.plan || "free"
plan_conf = plan_enabled && properties.service_plans.send(service.to_sym).send(plan.to_sym).configuration
@@ -1,7 +1,7 @@
---
<%
service = "mongodb"
-plan_enabled = properties.service_plans
+plan_enabled = properties.service_plans && properties.service_plans.send(service.to_sym)
plan = properties.plan || "free"
plan_conf = plan_enabled && properties.service_plans.send(service.to_sym).send(plan.to_sym).configuration
backup_enabled = plan_enabled && plan_conf.backup && plan_conf.backup.enable
@@ -1,7 +1,7 @@
---
<%
service = "mongodb"
-plan_enabled = properties.service_plans
+plan_enabled = properties.service_plans && properties.service_plans.send(service.to_sym)
plan = properties.plan || "free"
plan_conf = plan_enabled && properties.service_plans.send(service.to_sym).send(plan.to_sym).configuration
%>
@@ -1,18 +1,17 @@
#!/bin/bash
-
+<%
+migration = properties.service_migration
+lifecycle = properties.service_lifecycle
+%>
export PATH=/var/vcap/packages/ruby/bin:$PATH
RUN_DIR=/var/vcap/sys/run/mongodb_node
LOG_DIR=/var/vcap/sys/log/mongodb_node
SERV_LOG_DIR=/var/vcap/sys/service-log/mongodb
PIDFILE=$RUN_DIR/mongodb_node.pid
JOB_DIR=/var/vcap/jobs/mongodb_node
-
-<%
-migration = properties.service_migration
-%>
-
-MIG_DIR=<%= (migration && migration.mount_point) ? migration.mount_point : "/var/vcap/services_migration" %>
+SNAPSHOT_DIR=<%= lifecycle && lifecycle.mount_point || "/var/vcap/services_snapshot" %>
+MIG_DIR=<%= migration && migration.mount_point || "/var/vcap/services_migration" %>
source /var/vcap/packages/common/utils.sh
@@ -33,9 +32,11 @@ case $1 in
cp $JOB_DIR/config/logrotate /etc/logrotate.d/vcap-services
- if [ ! -d $MIG_DIR ]; then
- mkdir -p $MIG_DIR
- fi
+ mkdir -p $MIG_DIR
+ <% if lifecycle %>
+ mkdir -p $SNAPSHOT_DIR
+ mount -o noac -t nfs <%= lifecycle.nfs_server.address %>:<%= lifecycle.nfs_server.export_dir %> $SNAPSHOT_DIR
+ <% end %>
chmod +x $JOB_DIR/config/mongodb_backup
$JOB_DIR/config/mongodb_backup start
@@ -52,6 +53,9 @@ case $1 in
chmod +x $JOB_DIR/config/mongodb_backup
$JOB_DIR/config/mongodb_backup stop
+ <% if lifecycle %>
+ umount -l $SNAPSHOT_DIR
+ <% end %>
rm /etc/logrotate.d/vcap-services
;;
@@ -1,22 +1,23 @@
---
<%
service = "mongodb"
-plan_enabled = properties.service_plans
+plan_enabled = properties.service_plans && properties.service_plans.send(service.to_sym)
plan = properties.plan || "free"
+plan_conf = plan_enabled && properties.service_plans.send(service.to_sym).send(plan.to_sym).configuration
+lifecycle = properties.service_lifecycle
%>
-<% if properties.service_snapshot and properties.service_snapshot.resque %>
-<% snapshot_dir = properties.service_snapshot.mount_point || "/var/vcap/services_snapshot" %>
+<% if lifecycle and lifecycle.resque %>
+<% snapshot_dir = lifecycle.mount_point || "/var/vcap/services_snapshot" %>
service_name: mongodb
snapshots_base_dir: <%= snapshot_dir %>
serialization_base_dir: <%= snapshot_dir %>
local_db: sqlite3:/var/vcap/store/mongodb_node.db
-pid: /var/vcap/sys/run/mongodb_node/mongodb_worker.pid
-tmp_dir: <%= properties.service_snapshot.tmp_dir || "/tmp" %>
+tmp_dir: <%= lifecycle.tmp_dir || "/tmp" %>
resque:
- host: <%= properties.service_snapshot.resque.host %>
- port: <%= properties.service_snapshot.resque.port %>
- password: <%= properties.service_snapshot.resque.password %>
- expire: <%= properties.service_snapshot.resque.expire %>
+ host: <%= lifecycle.resque.host %>
+ port: <%= lifecycle.resque.port %>
+ password: <%= lifecycle.resque.password %>
+ expire: <%= lifecycle.resque.expire %>
logging:
level: debug
file: /var/vcap/sys/log/mongodb_node/mongodb_worker.log
@@ -25,8 +26,8 @@ logging:
<% end %>
index: <%= spec.index %>
node_id: <%= "#{service}_node#{'_' + plan if plan != 'free'}_#{spec.index.to_i + 1}" %>
-download_url_template: "http://<%= properties.service_snapshot.download_url %>/serialized/#{service}/#{name}?token=#{token}"
-worker_count: 5
+download_url_template: "http://<%= lifecycle.download_url %>/serialized/#{service}/#{name}?token=#{token}"
+snapshot_quota: <%= plan_enabled && plan_conf.lifecycle && plan_conf.lifecycle.snapshot_quota || 0 %>
mongodump_path: /var/vcap/packages/mongodb/bin/mongodump
mongorestore_path: /var/vcap/packages/mongodb/bin/mongorestore
tar_path: /bin/tar
@@ -1,19 +1,12 @@
#!/bin/bash
-
-<%
- snapshot = properties.service_snapshot
-%>
-
-<% if snapshot && snapshot.nfs_server %>
export PATH=/var/vcap/packages/ruby/bin:$PATH
+INDEX=$2
RUN_DIR=/var/vcap/sys/run/mongodb_node
LOG_DIR=/var/vcap/sys/log/mongodb_node
-PIDFILE=$RUN_DIR/mongodb_worker.pid
+PIDFILE=$RUN_DIR/mongodb_worker_$INDEX.pid
JOB_DIR=/var/vcap/jobs/mongodb_node
-SNAPSHOT_DIR=<%= snapshot.mount_point || "/var/vcap/services_snapshot" %>
-
source /var/vcap/packages/common/utils.sh
case $1 in
@@ -24,26 +17,18 @@ case $1 in
mkdir -p $RUN_DIR
mkdir -p $LOG_DIR
- if [ ! -d $SNAPSHOT_DIR ]; then
- mkdir -p $SNAPSHOT_DIR
- fi
-
- mount -o noac -t nfs <%= snapshot.nfs_server.address %>:<%= snapshot.nfs_server.export_dir %> $SNAPSHOT_DIR
-
- echo $$ > $PIDFILE
export CLOUD_FOUNDRY_CONFIG_PATH=$JOB_DIR/config/
+ export PIDFILE=$PIDFILE
exec /var/vcap/packages/mongodb_node/bin/services/mongodb_worker \
- >>$LOG_DIR/mongodb_worker.stdout.log \
- 2>>$LOG_DIR/mongodb_worker.stderr.log
+ >>$LOG_DIR/mongodb_worker_$INDEX.stdout.log \
+ 2>>$LOG_DIR/mongodb_worker_$INDEX.stderr.log
;;
stop)
kill_and_wait $PIDFILE
- umount $SNAPSHOT_DIR
-
;;
*)
@@ -52,6 +37,3 @@ case $1 in
;;
esac
-<% else %>
-echo "Service snapshots are disabled"
-<% end %>
Oops, something went wrong.

0 comments on commit 7a5d592

Please sign in to comment.