Permalink
Browse files

[dev_setup] refactor import_from_data api for lifecycle feature

Change-Id: Iafe1123d430c3aa343ac9a26081793f8707f0cc1
  • Loading branch information...
1 parent 4bf16e2 commit a4d83fad5329024f6f3c0ea8cb45419e6f08ff3e Frank Lu committed Jul 31, 2012
@@ -171,7 +171,17 @@ runtimes:
python2:
version: 2.[67]
-<% if node[:service_lifecycle][:enable] == true %>
+<% if node[:service_lifecycle][:enable] == true
+ if node[:serialization_data_server][:use_nginx] === true
+ sds_port = node[:serialization_data_server][:nginx_port] || 4997
+ else
+ sds_port = node[:serialization_data_server][:port] || 4998
+ end
+%>
service_lifecycle:
max_upload_size: <%= node[:service_lifecycle][:max_upload_size] || 1 %>
+ upload_token: <%= node[:serialization_data_server][:upload_token]%>
+ upload_timeout: <%= node[:serialization_data_server][:upload_timeout]%>
+ serialization_data_server:
+ - <%= node[:serialization_data_server][:host]%>:<%= sds_port %>
<% end %>
@@ -11,6 +11,7 @@
node[:postgresql][:host] ||= cf_local_ip
node[:redis_resque][:host] ||= cf_local_ip
node[:vcap_redis][:host] ||= cf_local_ip
+node[:serialization_data_server][:host] ||= cf_local_ip
[
node[:deployment][:home], File.join(node[:deployment][:home], "deploy"),
@@ -35,7 +35,6 @@ resque:
password: <%= node[:redis_resque][:password] %>
expire: <%= node[:redis_resque][:expire]%>
download_url_template: "http://service-serialization.<%= node[:deployment][:domain] %>/serialized/%{service}/%{name}/snapshots/%{snapshot_id}?token=%{token}"
-upload_temp_dir: /tmp
plan_management:
plans:
free:
@@ -4,7 +4,7 @@ base_dir: /var/vcap/services/mongodb/instances
snapshots_base_dir: <%= node[:snapshot][:dir]%>
serialization_base_dir: <%= node[:snapshot][:dir] %>
local_db: sqlite3:/var/vcap/services/mongodb/mongodb_node.db
-tmp_dir: <%= node[:service_lifecycle][:tmp_dir]|| "/tmp"%>
+tmp_dir: <%= node[:service_lifecycle][:tmp_dir]|| "/var/vcap/snapshot/tmp"%>
resque:
host: <%= node[:redis_resque][:host]%>
port: <%= node[:redis_resque][:port]%>
@@ -19,7 +19,9 @@ snapshot_quota: <%= node[:mongodb_node][:snapshot_quota] || "5" %>
<%else%>
snapshot_quota: 0
<%end%>
-
+serialization:
+ max_download_size_mb: <%= node[:service_lifecycle][:serialization_max_download_size]|| 10 %>
+default_version: <%= node[:mongodb][:default_version] %>
mongodump_path:
<%
node[:mongodb][:supported_versions].each_key do |version|
@@ -23,7 +23,6 @@ resque:
password: <%= node[:redis_resque][:password] %>
expire: <%= node[:redis_resque][:expire]%>
download_url_template: "http://service-serialization.<%= node[:deployment][:domain] %>/serialized/%{service}/%{name}/snapshots/%{snapshot_id}?token=%{token}"
-upload_temp_dir: /tmp
plan_management:
plans:
free:
@@ -3,7 +3,7 @@ service_name: mysql
snapshots_base_dir: <%= node[:snapshot][:dir]%>
serialization_base_dir: <%= node[:snapshot][:dir] %>
local_db: sqlite3:/var/vcap/services/mysql/mysql_node.db
-tmp_dir: <%= node[:service_lifecycle][:tmp_dir]|| "/tmp"%>
+tmp_dir: <%= node[:service_lifecycle][:tmp_dir]|| "/var/vcap/snapshot/tmp"%>
resque:
host: <%= node[:redis_resque][:host]%>
port: <%= node[:redis_resque][:port]%>
@@ -21,6 +21,8 @@ snapshot_quota: <%= node[:mysql_node][:snapshot_quota] || "5" %>
<%else%>
snapshot_quota: 0
<%end%>
+serialization:
+ max_download_size_mb: <%= node[:service_lifecycle][:serialization_max_download_size]|| 10 %>
mysql:
host: <%= node[:mysql][:host] %>
port: 3306
@@ -203,6 +203,20 @@
mode 0755
end
+ template "nginx_sds.conf" do
+ path File.join(nginx_path, "conf", "nginx_sds.conf")
+ source "sds-nginx.conf.erb"
+ owner node[:deployment][:user]
+ mode 0644
+ end
+
+ template "nginx_sds" do
+ path File.join("", "etc", "init.d", "nginx_sds")
+ source "sds-nginx.erb"
+ owner node[:deployment][:user]
+ mode 0755
+ end
+
bash "Stop running nginx" do
code <<-EOH
pid=`ps -ef | grep nginx | grep -v grep | awk '{print $2}'`
@@ -220,6 +234,11 @@
action [ :enable, :restart ]
end
+ service "nginx_sds" do
+ supports :status => true, :restart => true, :reload => true
+ action [ :enable, :restart ]
+ end
+
else
Chef::Log.error("Installation of nginx packages not supported on this platform.")
end
@@ -60,7 +60,7 @@ http {
root /var/vcap/shared;
}
- location ~ /apps/.*/application {
+ location ~ (/apps/.*/application|/services/v\d+/configurations/.*/serialized/data) {
# Pass altered request body to this location
upload_pass @cc_uploads;
upload_pass_args on;
@@ -0,0 +1,80 @@
+#deployment cloudcontroller nginx.conf
+user <%= node[:deployment][:user] %> root;
+worker_processes 1;
+
+error_log <%= node[:nginx][:log_home] %>/nginx_sds_error.log debug;
+pid /var/run/nginx_sds.pid;
+
+events {
+ worker_connections <%= node[:nginx][:worker_connections] %>;
+ use epoll;
+}
+
+http {
+ include mime.types;
+ default_type text/html;
+ server_tokens off;
+
+ log_format main '$host - [$time_local] '
+ '"$request" $status $bytes_sent '
+ '"$http_referer" "$http_user_agent" '
+ '$remote_addr response_time:$upstream_response_time';
+
+ access_log <%= node[:nginx][:log_home] %>/nginx_sds_main.log main;
+
+ sendfile on; #enable use of sendfile()
+ tcp_nopush on;
+ tcp_nodelay on; #disable nagel's algorithm
+
+ keepalive_timeout 75 20; #inherited from router
+
+ client_max_body_size 256M; #already enforced upstream/but doesn't hurt.
+
+ upstream serialization_data_server {
+ server 127.0.0.1:<%= node[:serialization_data_server][:upstream_port] || "20000"%>;
+ }
+
+ server {
+ listen <%= node[:serialization_data_server][:nginx_port] || "4997"%>;
+ server_name _;
+ server_name_in_redirect off;
+
+ #proxy and log all SDS traffic
+ location / {
+ access_log <%= node[:nginx][:log_home] %>/nginx_sds_access.log main;
+ proxy_pass http://serialization_data_server;
+ }
+
+ location /protected {
+ internal;
+ alias <%= node[:snapshot][:dir] %>;
+ }
+
+ location ~ /serialized/.*/.*/serialized/data {
+ upload_pass @sds_uploads;
+ upload_pass_args on;
+
+ # Storee files to this directory
+ upload_store /var/vcap/data/serialization_data_server/tmp/uploads;
+
+ # No limit for output body forwarded to SDS
+ upload_max_output_body_len 0;
+
+ # Allow uploaded files to be read only by user
+ # upload_store_accesscess user:r;
+
+ # Set specified fields in request body
+ upload_set_form_field "${upload_field_name}_name" $upload_file_name;
+ upload_set_form_field "${upload_field_name}_path" $upload_tmp_path;
+
+ upload_pass_form_field "^_method$";
+
+ #only any error, delete uploaded files.
+ upload_cleanup 400-505;
+ }
+
+ location @sds_uploads {
+ proxy_pass http://serialization_data_server;
+ }
+ }
+}
@@ -0,0 +1,53 @@
+#! /bin/sh
+
+### BEGIN INIT INFO
+# Provides: nginx
+# Required-Start: $local_fs $remote_fs $network $syslog
+# Required-Stop: $local_fs $remote_fs $network $syslog
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: init script for sds nginx
+# Description: init script for sds nginx
+### END INIT INFO
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+PROG=<%= node[:nginx][:path] %>/sbin/nginx
+ARGS=-cconf/nginx_sds.conf
+NAME=nginx_sds
+UPLOADS_DIR=/var/vcap/data/serialization_data_server/tmp/uploads
+
+test -x $PROG || exit 0
+
+set -e
+
+. /lib/lsb/init-functions
+
+case "$1" in
+ start)
+
+ if [ ! -d "$UPLOADS_DIR" ]; then
+ mkdir -p $UPLOADS_DIR
+ chown -R <%= node[:deployment][:user]%>:root $UPLOADS_DIR
+ fi
+ log_action_begin_msg "Starting: $NAME"
+ start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid --exec $PROG -- $ARGS
+ log_action_end_msg $?
+ ;;
+ stop)
+ log_action_begin_msg "Stopping: $NAME"
+ start-stop-daemon --stop --quiet --pidfile /var/run/$NAME.pid --exec $PROG || true
+ log_action_end_msg $?
+ ;;
+ restart)
+ $0 stop; sleep 1; $0 start
+ ;;
+ status)
+ status_of_proc -p /var/run/$NAME.pid "$PROG" $NAME && exit 0 || exit $?
+ ;;
+ *)
+ echo "Usage: $NAME {start|stop|restart|status}" >&2
+ exit 1
+ ;;
+esac
+
+exit 0
@@ -23,7 +23,6 @@ resque:
password: <%= node[:redis_resque][:password] %>
expire: <%= node[:redis_resque][:expire]%>
download_url_template: "http://service-serialization.<%= node[:deployment][:domain] %>/serialized/%{service}/%{name}/snapshots/%{snapshot_id}?token=%{token}"
-upload_temp_dir: /tmp
plan_management:
plans:
free:
@@ -3,7 +3,7 @@ service_name: postgresql
snapshots_base_dir: <%= node[:snapshot][:dir]%>
serialization_base_dir: <%= node[:snapshot][:dir] %>
local_db: sqlite3:/var/vcap/services/postgresql/postgresql_node.db
-tmp_dir: <%= node[:service_lifecycle][:tmp_dir]|| "/tmp"%>
+tmp_dir: <%= node[:service_lifecycle][:tmp_dir]|| "/var/vcap/snapshot/tmp"%>
resque:
host: <%= node[:redis_resque][:host]%>
port: <%= node[:redis_resque][:port]%>
@@ -20,6 +20,8 @@ snapshot_quota: <%= node[:postgresql_node][:snapshot_quota] || "5" %>
<%else%>
snapshot_quota: 0
<%end%>
+serialization:
+ max_download_size_mb: <%= node[:service_lifecycle][:serialization_max_download_size]|| 10 %>
postgresql:
host: <%= node[:postgresql_node][:host] %>
port: <%= node[:postgresql][:service_port]%>
@@ -23,7 +23,6 @@ resque:
password: <%= node[:redis_resque][:password] %>
expire: <%= node[:redis_resque][:expire]%>
download_url_template: "http://service-serialization.<%= node[:deployment][:domain] %>/serialized/%{service}/%{name}/snapshots/%{snapshot_id}?token=%{token}"
-upload_temp_dir: /tmp
plan_management:
plans:
free:
@@ -4,7 +4,7 @@ base_dir: /var/vcap/services/redis/instances
snapshots_base_dir: <%= node[:snapshot][:dir]%>
serialization_base_dir: <%= node[:snapshot][:dir] %>
local_db: sqlite3:/var/vcap/services/redis/redis_node.db
-tmp_dir: <%= node[:service_lifecycle][:tmp_dir]|| "/tmp"%>
+tmp_dir: <%= node[:service_lifecycle][:tmp_dir]|| "/var/vcap/snapshot/tmp"%>
resque:
host: <%= node[:redis_resque][:host]%>
port: <%= node[:redis_resque][:port]%>
@@ -19,6 +19,8 @@ snapshot_quota: <%= node[:mysql_node][:snapshot_quota] || "5" %>
<%else%>
snapshot_quota: 0
<%end%>
+serialization:
+ max_download_size_mb: <%= node[:service_lifecycle][:serialization_max_download_size]|| 10 %>
logging:
level: debug
command_rename_prefix: protect-command-prefix
@@ -1,15 +1,25 @@
default[:service_lifecycle][:enable] = false
default[:service_lifecycle][:max_upload_size] = 5
-default[:service_lifecycle][:tmp_dir] = "/tmp"
+default[:service_lifecycle][:serialization_max_download_size] = 5
+default[:service_lifecycle][:tmp_dir] = "/var/vcap/snapshot/tmp"
default[:snapshot][:dir]="/var/vcap/snapshot"
default[:snapshot_manager][:config_file]="snapshot_manager.yml"
default[:snapshot_manager][:cleanup_max_days] = "3"
default[:snapshot_manager][:greedy_mark] = false
default[:snapshot_manager][:wakeup_interval_in_sec] = "3600"
-default[:redis_resque][:host]= "localhost"
+default[:redis_resque][:host] = "localhost"
default[:redis_resque][:port] = 4999
default[:redis_resque][:password] = "redis"
default[:redis_resque][:expire] = 60
-default[:serialization_data_server][:config_file]="serialization_data_server.yml"
+default[:serialization_data_server][:config_file] = "serialization_data_server.yml"
+default[:serialization_data_server][:expire_time] = "600"
+default[:serialization_data_server][:purge_expired_interval] = "30"
+default[:serialization_data_server][:purge_num] = "1000"
+default[:serialization_data_server][:upload_token] = "sdsuploadtokenPxYWj"
+default[:serialization_data_server][:upload_timeout] = 10
+default[:serialization_data_server][:port] = "4998"
+default[:serialization_data_server][:nginx_port] = "4997"
+default[:serialization_data_server][:upstream_port] = "20000"
+default[:serialization_data_server][:use_nginx] = true
@@ -2,8 +2,19 @@
serialization_base_dir: <%= node[:snapshot][:dir]%>
pid: /var/vcap/sys/run/serialization_data_server.pid
mbus: nats://<%= node[:nats_server][:user] %>:<%= node[:nats_server][:password] %>@<%= node[:nats_server][:host] %>:<%= node[:nats_server][:port] %>
-ip_route: localhost
external_uri: service-serialization.<%= node[:deployment][:domain]%>
+upload_token: <%= node[:serialization_data_server][:upload_token] %>
+expire_time: <%= node[:serialization_data_server][:expire_time]%>
+purge_expired_interval: <%= node[:serialization_data_server][:purge_expired_interval]%>
+purge_num: <%= node[:serialization_data_server][:purge_num]%>
+<% if node[:serialization_data_server][:use_nginx] === true %>
+port: <%= node[:serialization_data_server][:upstream_port]%>
+nginx:
+ nginx_path: /protected
+ nginx_port: <%= node[:serialization_data_server][:nginx_port]%>
+<% else %>
+port: <%= node[:serialization_data_server][:port]%>
+<% end %>
logging:
level: debug
redis:
@@ -12,4 +23,3 @@ redis:
password: <%= node[:redis_resque][:password] %>
expire: <%= node[:redis_resque][:expire]%>
index: 0
-

0 comments on commit a4d83fa

Please sign in to comment.