From fb91d4cb54da9e53ff6d6279663864993221ef0a Mon Sep 17 00:00:00 2001 From: Dmitrii Zagorodnov Date: Fri, 2 Dec 2011 10:54:11 -0800 Subject: [PATCH 1/7] NC config parameter cleanup; proper disk size setting and reporting; removed old storage.? stuff --- cluster/Makefile | 4 +- node/Makefile | 3 - node/handlers.c | 218 ++++++-- node/handlers.h | 6 +- node/handlers_kvm.c | 1 - node/handlers_xen.c | 1 - storage/Makefile | 8 +- storage/backing.c | 28 +- storage/backing.h | 2 + storage/blobstore.c | 164 ++++-- storage/blobstore.h | 14 + storage/imager.c | 4 +- storage/storage.c | 1264 ------------------------------------------- storage/storage.h | 94 ---- storage/vbr.c | 2 +- util/eucalyptus.h | 6 +- 16 files changed, 336 insertions(+), 1483 deletions(-) delete mode 100644 storage/storage.c delete mode 100644 storage/storage.h diff --git a/cluster/Makefile b/cluster/Makefile index 46c16499f85..c757def324b 100644 --- a/cluster/Makefile +++ b/cluster/Makefile @@ -18,7 +18,7 @@ SHUTDOWNCC=shutdownCC #WSDL2C=${AXIS2C_HOME}/bin/tools/wsdl2c/WSDL2C.sh NCLIBS=../util/data.o ../node/client-marshal-adb.o ../util/ipc.o NC_FAKE_LIBS=../util/data.o ../node/client-marshal-fake.o ../util/ipc.o -SCLIBS=../storage/storage.o ../storage/storage-windows.o ../storage/walrus.o ../storage/http.o +SCLIBS=../storage/storage-windows.o ../storage/walrus.o ../storage/http.o VNLIBS=../net/vnetwork.o ../util/misc.o ../storage/diskutil.o WSSECLIBS=../util/euca_axis.o ../util/euca_auth.o CC_LIBS = ${LIBS} ${LDFLAGS} -lcurl -lssl -lcrypto -lrampart @@ -69,7 +69,7 @@ server: $(NCLIBS) $(VNLIBS) $(SERVICE_SO) fake: all $(NC_FAKE_LIBS) $(SERVICE_SO_FAKE) -$(SERVICE_SO): generated/stubs ../storage/storage.o server-marshal.o handlers.o handlers-state.o server-marshal-state.o ../util/windows-bundle.o +$(SERVICE_SO): generated/stubs server-marshal.o handlers.o handlers-state.o server-marshal-state.o ../util/windows-bundle.o $(CC) -shared generated/*.o server-marshal.o handlers.o handlers-state.o server-marshal-state.o ../util/windows-bundle.o $(SCLIBS) $(NCLIBS) $(VNLIBS) $(WSSECLIBS) $(CC_LIBS) -o $(SERVICE_SO) $(SERVICE_SO_FAKE): diff --git a/node/Makefile b/node/Makefile index 86a01e313c2..6c67abac155 100644 --- a/node/Makefile +++ b/node/Makefile @@ -59,9 +59,6 @@ server: $(SERVICE_SO) ../storage/blobstore.o: ../storage/blobstore.c ../util/misc.o ../util/data.o make -C ../storage -../storage/storage.o: ../storage/storage.c ../util/misc.o ../util/data.o - make -C ../storage - ../storage/walrus.o: ../storage/walrus.c ../util/misc.o ../util/data.o make -C ../storage diff --git a/node/handlers.c b/node/handlers.c index 2536be30043..3bb72912055 100644 --- a/node/handlers.c +++ b/node/handlers.c @@ -127,6 +127,10 @@ const int bundling_cleanup_threshold = 60 * 60; /* after this many seconds any B const int createImage_cleanup_threshold = 60 * 60; /* after this many seconds any CREATEIMAGE domains will be cleaned up */ const int teardown_state_duration = 180; /* after this many seconds in TEARDOWN state (no resources), we'll forget about the instance */ +#define MIN_BLOBSTORE_SIZE_MB 10 // even with boot-from-EBS one will need work space for kernel and ramdisk +#define FS_BUFFER_MB 100 // leave 100MB extra when deciding on blobstore sizes automatically +#define WORK_BS_PERCENT 0.33 // give a third of available space to work, the rest to cache + // a NULL-terminated array of available handlers static struct handlers * available_handlers [] = { &default_libvirt_handlers, @@ -601,7 +605,6 @@ void *startup_thread (void * arg) logprintfl(EUCAWARN, "[%s] can't determine the host's bitness (assuming 64)\n", instance->instanceId); instance->hypervisorBitness = 64; } - instance->combinePartitions = nc_state.convert_to_disk; instance->do_inject_key = nc_state.do_inject_key; @@ -832,48 +835,44 @@ static int init (void) *tmp=NULL, *pubinterface=NULL; struct stat mystat; - struct statfs fs; struct handlers ** h; - long long fs_free_blocks = 0; - long long fs_block_size = 0; - long long instances_bytes = 0; // TODO: get this value from instance backing code - pthread_t tcb; - if (initialized>0) /* 0 => hasn't run, -1 => failed, 1 => ok */ + if (initialized>0) // 0 => hasn't run, -1 => failed, 1 => ok return 0; else if (initialized<0) return 1; bzero (&nc_state, sizeof(struct nc_state_t)); // ensure that MAXes are zeroed out - /* read in configuration - this should be first! */ + // read in configuration - this should be first! + + // determine home ($EUCALYPTUS) tmp = getenv(EUCALYPTUS_ENV_VAR_NAME); if (!tmp) { - nc_state.home[0] = '\0'; + nc_state.home[0] = '\0'; // empty string means '/' do_warn = 1; } else { strncpy(nc_state.home, tmp, MAX_PATH - 1); } - /* set the minimum log for now */ + // set the minimum log for now snprintf(log, MAX_PATH, "%s/var/log/eucalyptus/nc.log", nc_state.home); - logfile(log, EUCADEBUG, 4); + logfile(log, EUCAINFO, 4); logprintfl (EUCAINFO, "Eucalyptus node controller initializing %s\n", compile_timestamp_str); if (do_warn) logprintfl (EUCAWARN, "env variable %s not set, using /\n", EUCALYPTUS_ENV_VAR_NAME); - /* search for the config file */ + // search for the config file snprintf(configFiles[1], MAX_PATH, EUCALYPTUS_CONF_LOCATION, nc_state.home); if (stat(configFiles[1], &mystat)) { logprintfl (EUCAFATAL, "could not open configuration file %s\n", configFiles[1]); return 1; } snprintf(configFiles[0], MAX_PATH, EUCALYPTUS_CONF_OVERRIDE_LOCATION, nc_state.home); - logprintfl (EUCAINFO, "NC is looking for configuration in %s,%s\n", configFiles[1], configFiles[0]); - /* reset the log to the right value */ + // reset the log level to the requested value tmp = getConfString(configFiles, 2, "LOGLEVEL"); i = EUCADEBUG; if (tmp) { @@ -898,6 +897,11 @@ static int init (void) // if 'dir' does not exist, init_hooks() will silently fail, // and all future call_hooks() will silently succeed init_hooks (nc_state.home, dir); + + if (call_hooks (NC_EVENT_PRE_INIT, nc_state.home)) { + logprintfl (EUCAFATAL, "hooks prevented initialization\n"); + return ERROR_FATAL; + } } #define GET_VAR_INT(var,name) \ @@ -907,24 +911,23 @@ static int init (void) free (s);\ } GET_VAR_INT(nc_state.config_max_mem, CONFIG_MAX_MEM); - GET_VAR_INT(nc_state.config_max_disk, CONFIG_MAX_DISK); GET_VAR_INT(nc_state.config_max_cores, CONFIG_MAX_CORES); GET_VAR_INT(nc_state.save_instance_files, CONFIG_SAVE_INSTANCES); int disable_injection = 0; GET_VAR_INT(disable_injection, CONFIG_DISABLE_KEY_INJECTION); nc_state.do_inject_key = !disable_injection; - nc_state.config_network_port = NC_NET_PORT_DEFAULT; strcpy(nc_state.admin_user_id, EUCALYPTUS_ADMIN); - - add_euca_to_path (nc_state.home); // add three eucalyptus directories with executables to PATH of this process - + + // add three eucalyptus directories with executables to PATH of this process + add_euca_to_path (nc_state.home); + + // read in .pem files if (euca_init_cert ()) { logprintfl (EUCAERROR, "init(): failed to find cryptographic certificates\n"); return ERROR_FATAL; } - /* from now on we have unrecoverable failure, so no point in - * retrying to re-init */ + //// from now on we have unrecoverable failure, so no point in retrying to re-init //// initialized = -1; hyp_sem = sem_alloc (1, "mutex"); @@ -944,40 +947,139 @@ static int init (void) // set default in the paths. the driver will override nc_state.config_network_path[0] = '\0'; - nc_state.gen_libvirt_cmd_path[0] = '\0'; nc_state.xm_cmd_path[0] = '\0'; nc_state.virsh_cmd_path[0] = '\0'; nc_state.get_info_cmd_path[0] = '\0'; snprintf (nc_state.libvirt_xslt_path, MAX_PATH, EUCALYPTUS_LIBVIRT_XSLT, nc_state.home); snprintf (nc_state.rootwrap_cmd_path, MAX_PATH, EUCALYPTUS_ROOTWRAP, nc_state.home); - // backing store configuration - int cache_size_mb = DEFAULT_NC_CACHE_SIZE; - int work_size_mb = DEFAULT_NC_WORK_SIZE; - GET_VAR_INT(cache_size_mb, CONFIG_NC_CACHE_SIZE); - GET_VAR_INT(work_size_mb, CONFIG_NC_WORK_SIZE); - char * instance_path = getConfString(configFiles, 2, INSTANCE_PATH); + { // backing store configuration + char * instance_path = getConfString(configFiles, 2, INSTANCE_PATH); - if (instance_path == NULL || init_backing_store (instance_path, work_size_mb, cache_size_mb)) { - logprintfl (EUCAFATAL, "error: failed to initialize backing store\n"); - if(instance_path) free(instance_path); - return ERROR_FATAL; - } - if (statfs (instance_path, &fs) == -1) { // TODO: get the values from instance backing code - logprintfl(EUCAWARN, "Failed to stat %s\n", instance_path); - } else { - nc_state.disk_max = (long long)fs.f_bsize * (long long)fs.f_bavail + instances_bytes; /* max for Euca, not total */ - nc_state.disk_max /= BYTES_PER_DISK_UNIT; - if (nc_state.config_max_disk && nc_state.config_max_disk < nc_state.disk_max) - nc_state.disk_max = nc_state.config_max_disk; - - logprintfl (EUCAINFO, "Maximum disk available: %lld (under %s)\n", nc_state.disk_max, instance_path); - } - if (instance_path) free (instance_path); + // determine bytes available on the file system to which instance path belongs + if (instance_path == NULL) { + logprintfl (EUCAFATAL, "error: %s is not set\n", INSTANCE_PATH); + return ERROR_FATAL; + } + struct statfs fs; + if (statfs (instance_path, &fs) == -1) { + logprintfl (EUCAERROR, "error: failed to stat %s (%s): %s\n", INSTANCE_PATH, instance_path, strerror(errno)); + free (instance_path); + return ERROR_FATAL; + } + long long fs_avail_mb = (long long)fs.f_bsize * (long long)(fs.f_bavail/MEGABYTE); // TODO: is 2 petabytes enough? + long long fs_size_mb = (long long)fs.f_bsize * (long long)(fs.f_blocks/MEGABYTE); + if (fs_avail_mb < MIN_BLOBSTORE_SIZE_MB) { + logprintfl (EUCAERROR, "error: insufficient available disk space (%d) under %s\n", fs_avail_mb, instance_path); + free (instance_path); + return ERROR_FATAL; + } + + // determine how much is used/available in work and cache areas on the backing store + blobstore_meta work_meta, cache_meta; + stat_backing_store (instance_path, &work_meta, &cache_meta); // will zero-out work_ and cache_meta + long long work_bs_size_mb = work_meta.blocks_limit ? (work_meta.blocks_limit / 2048) : -1; // convert sectors->MB + long long work_bs_used_mb = work_meta.blocks_limit ? ((work_meta.blocks_allocated + work_meta.blocks_used) / 2048) : 0; + long long cache_bs_size_mb = cache_meta.blocks_limit ? (cache_meta.blocks_limit / 2048) : -1; + long long cache_bs_used_mb = cache_meta.blocks_limit ? ((cache_meta.blocks_allocated + cache_meta.blocks_used) / 2048) : 0; + + // look up configuration file settings for work and cache size + int conf_work_size_mb = -1; GET_VAR_INT(conf_work_size_mb, CONFIG_NC_WORK_SIZE); + int conf_cache_size_mb = -1; GET_VAR_INT(conf_cache_size_mb, CONFIG_NC_CACHE_SIZE); + { // accommodate legacy MAX_DISK setting by converting it + int max_disk_gb = -1; GET_VAR_INT(max_disk_gb, CONFIG_MAX_DISK); + if (max_disk_gb != -1) { + if (conf_work_size_mb == -1) { + logprintfl (EUCAWARN, "warning: using deprecated setting %s for the new setting %s\n", CONFIG_MAX_DISK, CONFIG_NC_WORK_SIZE); + conf_work_size_mb = max_disk_gb * 1024; + } else { + logprintfl (EUCAWARN, "warning: ignoring deprecated setting %s in favor of the new setting %s\n", CONFIG_MAX_DISK, CONFIG_NC_WORK_SIZE); + } + } + } + + // decide what work and cache sizes should be, based on all the inputs + int work_size_mb = -1; + int cache_size_mb = -1; + + // above all, try to respect user-specified limits for work and cache + if (conf_work_size_mb != -1) { + if (conf_work_size_mb < MIN_BLOBSTORE_SIZE_MB) { + logprintfl (EUCAWARN, "warning: ignoring specified work size (%s=%d) that is below acceptable minimum (%d)\n", + CONFIG_NC_WORK_SIZE, conf_work_size_mb, MIN_BLOBSTORE_SIZE_MB); + } else { + if (work_bs_size_mb != -1 && work_bs_size_mb != conf_work_size_mb) { + logprintfl (EUCAWARN, "warning: specified work size (%s=%d) differs from existing work limits (%d), will try resizing\n", + CONFIG_NC_WORK_SIZE, conf_work_size_mb, work_bs_size_mb); + } + work_size_mb = conf_work_size_mb; + } + } + if (conf_cache_size_mb != -1) { // respect user-specified limit + if (conf_cache_size_mb < MIN_BLOBSTORE_SIZE_MB) { + cache_size_mb = 0; // so it won't be used + } else { + if (cache_bs_size_mb != -1 && cache_bs_size_mb != conf_cache_size_mb) { + logprintfl (EUCAWARN, "warning: specified cache size (%s=%d) differs from existing cache limits (%d), will try resizing\n", + CONFIG_NC_CACHE_SIZE, conf_cache_size_mb, cache_bs_size_mb); + } + cache_size_mb = conf_cache_size_mb; + } + } + + // if the user did not specify sizes, try existing blobstores, + // if any, whose limits would have been chosen earlier + if (work_size_mb == -1 && work_bs_size_mb != -1) + work_size_mb = work_bs_size_mb; + if (cache_size_mb == -1 && cache_bs_size_mb != -1) + cache_size_mb = cache_bs_size_mb; + + // if the user did not specify either or both of the sizes, + // and blobstores do not exist yet, make reasonable choices + if (work_size_mb == -1 && cache_size_mb == -1) { + work_size_mb = (long long)((double)(fs_avail_mb - FS_BUFFER_MB) * WORK_BS_PERCENT); + cache_size_mb = fs_avail_mb - FS_BUFFER_MB - work_size_mb; + } else if (work_size_mb == -1) { + work_size_mb = fs_avail_mb - FS_BUFFER_MB - cache_size_mb + cache_bs_used_mb; + } else if (cache_size_mb == -1) { + cache_size_mb = fs_avail_mb - FS_BUFFER_MB - work_size_mb + work_bs_used_mb; + } + + // sanity-check final results + if (cache_size_mb < MIN_BLOBSTORE_SIZE_MB) + cache_size_mb = 0; + if (work_size_mb < MIN_BLOBSTORE_SIZE_MB) { + logprintfl (EUCAERROR, "error: insufficient disk space (%d) for virtual machines\n", work_size_mb); + free (instance_path); + return ERROR_FATAL; + } + if ((cache_size_mb + work_size_mb - cache_bs_used_mb - work_bs_used_mb) > fs_avail_mb) { + logprintfl (EUCAWARN, "warning: sum of work and cache sizes exceeds available disk space\n"); + } + + if (init_backing_store (instance_path, work_size_mb, cache_size_mb)) { + logprintfl (EUCAFATAL, "error: failed to initialize backing store\n"); + free (instance_path); + return ERROR_FATAL; + } + + // record the work-space limit for max_disk + nc_state.disk_max = (long long)(work_size_mb / MB_PER_DISK_UNIT); + + logprintfl (EUCAINFO, "disk space for instances: %s/work\n", instance_path); + logprintfl (EUCAINFO, " %06lldMB allocated (%03.1f% of the file system)\n", work_size_mb, (work_size_mb/fs_size_mb)*100.0 ); + logprintfl (EUCAINFO, " %06lldMB in use (%03.1% of allocated)\n", work_bs_used_mb, (work_bs_used_mb/work_size_mb)*100.0 ); + if (cache_size_mb) { + logprintfl (EUCAINFO, " disk space for cache: %s/cache\n", instance_path); + logprintfl (EUCAINFO, " %06lldMB allocated (%03.1f% of the file system)\n", cache_size_mb, (cache_size_mb/fs_size_mb)*100.0 ); + logprintfl (EUCAINFO, " %06lldMB in use (%03.1% of allocated)\n", cache_bs_used_mb, (cache_bs_used_mb/cache_size_mb)*100.0 ); + } else { + logprintfl (EUCAWARN, "warning: disk cache will not be used\n"); + } + free (instance_path); + } // determine the hypervisor to use - - //if (get_conf_var(config, CONFIG_HYPERVISOR, &hypervisor)<1) { hypervisor = getConfString(configFiles, 2, CONFIG_HYPERVISOR); if (!hypervisor) { logprintfl (EUCAFATAL, "value %s is not set in the config file\n", CONFIG_HYPERVISOR); @@ -1085,7 +1187,7 @@ static int init (void) if (bridge) free(bridge); if (tmp) free(tmp); - if(initFail) + if (initFail) return ERROR_FATAL; // set NC helper path @@ -1132,18 +1234,20 @@ static int init (void) } } - // start the monitoring thread - if (pthread_create(&tcb, NULL, monitoring_thread, &nc_state)) { - logprintfl (EUCAFATAL, "failed to spawn a monitoring thread\n"); - return ERROR_FATAL; - } - - if (pthread_detach(tcb)) { - logprintfl (EUCAFATAL, "failed to detach the monitoring thread\n"); - return ERROR_FATAL; + { // start the monitoring thread + pthread_t tcb; + if (pthread_create(&tcb, NULL, monitoring_thread, &nc_state)) { + logprintfl (EUCAFATAL, "failed to spawn a monitoring thread\n"); + return ERROR_FATAL; + } + if (pthread_detach(tcb)) { + logprintfl (EUCAFATAL, "failed to detach the monitoring thread\n"); + return ERROR_FATAL; + } } - if (call_hooks (NC_EVENT_INIT, NULL)) { + // post-init hook + if (call_hooks (NC_EVENT_POST_INIT, nc_state.home)) { logprintfl (EUCAFATAL, "hooks prevented initialization\n"); return ERROR_FATAL; } diff --git a/node/handlers.h b/node/handlers.h index 64ddd194196..033727b7fde 100644 --- a/node/handlers.h +++ b/node/handlers.h @@ -70,7 +70,8 @@ permission notice: #define LIBVIRT_QUERY_RETRIES 5 #define MAXDOMS 1024 -#define BYTES_PER_DISK_UNIT 1073741824 /* disk stats are in Gigs */ +#define BYTES_PER_DISK_UNIT 1073741824 // describeResource disk units are GBs +#define MB_PER_DISK_UNIT 1024 // describeResource disk units are GBs /* NC state */ struct nc_state_t { @@ -79,7 +80,6 @@ struct nc_state_t { hypervisorCapabilityType capability; vnetConfig *vnetconfig; // network config // globals - int config_network_port; char admin_user_id[CHAR_BUFFER_SIZE]; int save_instance_files; char uri[CHAR_BUFFER_SIZE]; @@ -88,7 +88,6 @@ struct nc_state_t { boolean convert_to_disk; boolean do_inject_key; // defined max - long long config_max_disk; long long config_max_mem; long long config_max_cores; // current max @@ -98,7 +97,6 @@ struct nc_state_t { // paths char home[MAX_PATH]; char config_network_path [MAX_PATH]; - char gen_libvirt_cmd_path[MAX_PATH]; char libvirt_xslt_path[MAX_PATH]; char get_info_cmd_path[MAX_PATH]; char rootwrap_cmd_path[MAX_PATH]; diff --git a/node/handlers_kvm.c b/node/handlers_kvm.c index 211f77aab69..6df880c1520 100644 --- a/node/handlers_kvm.c +++ b/node/handlers_kvm.c @@ -100,7 +100,6 @@ static int doInitialize (struct nc_state_t *nc) logprintfl(EUCADEBUG, "doInitialized() invoked\n"); /* set up paths of Eucalyptus commands NC relies on */ - snprintf (nc->gen_libvirt_cmd_path, MAX_PATH, EUCALYPTUS_GEN_KVM_LIBVIRT_XML, nc->home, nc->home); snprintf (nc->get_info_cmd_path, MAX_PATH, EUCALYPTUS_GET_KVM_INFO, nc->home, nc->home); strcpy(nc->uri, HYPERVISOR_URI); nc->convert_to_disk = 1; diff --git a/node/handlers_xen.c b/node/handlers_xen.c index 4ce2efac048..d7c765061b6 100644 --- a/node/handlers_xen.c +++ b/node/handlers_xen.c @@ -98,7 +98,6 @@ static int doInitialize (struct nc_state_t *nc) logprintfl(EUCADEBUG, "doInitialized() invoked\n"); /* set up paths of Eucalyptus commands NC relies on */ - snprintf (nc->gen_libvirt_cmd_path, MAX_PATH, EUCALYPTUS_GEN_LIBVIRT_XML, nc->home, nc->home); snprintf (nc->get_info_cmd_path, MAX_PATH, EUCALYPTUS_GET_XEN_INFO, nc->home, nc->home); snprintf (nc->virsh_cmd_path, MAX_PATH, EUCALYPTUS_VIRSH, nc->home); snprintf (nc->xm_cmd_path, MAX_PATH, EUCALYPTUS_XM); diff --git a/storage/Makefile b/storage/Makefile index 0623784fab9..c0bf98ab1f7 100644 --- a/storage/Makefile +++ b/storage/Makefile @@ -14,8 +14,8 @@ TESTS=test_vbr test_blobstore test IMAGER=_euca_imager #EFENCE=-lefence -all: vbr.o backing.o storage.o storage-windows.o walrus.o diskutil.o Wclient $(IMAGER) test_blobstore test_vbr -#all: storage.o storage-windows.o walrus.o test_blobstore +all: vbr.o backing.o storage-windows.o walrus.o diskutil.o Wclient $(IMAGER) test_blobstore test_vbr +#all: storage-windows.o walrus.o test_blobstore build: all @@ -28,8 +28,8 @@ Wclient: Makefile Wclient.c ../util/euca_auth.o ../util/misc.o walrus.o http.o walrus.o: walrus.c walrus.h $(CC) $(CFLAGS) $(INCLUDES) -c walrus.c -test: test.c storage.h storage.o storage-windows.o ../util/misc.o ../util/data.o ../util/ipc.o ../util/euca_auth.o walrus.o - $(CC) $(CFLAGS) $(INCLUDES) test.c storage.o storage-windows.o ../util/misc.o diskutil.o ../util/data.o ../util/ipc.o ../util/euca_auth.o walrus.o $(STORAGE_LIBS) -o test +test: test.c blobstore.o storage-windows.o ../util/misc.o ../util/data.o ../util/ipc.o ../util/euca_auth.o walrus.o + $(CC) $(CFLAGS) $(INCLUDES) test.c blobstore.o storage-windows.o ../util/misc.o diskutil.o ../util/data.o ../util/ipc.o ../util/euca_auth.o walrus.o $(STORAGE_LIBS) -o test test_blobstore: blobstore.o blobstore.c blobstore.h map.o diskutil.o ../util/misc.o $(CC) -g -rdynamic $(CFLAGS) $(INCLUDES) -D_UNIT_TEST blobstore.c -o test_blobstore map.o diskutil.o ../util/misc.o ../util/ipc.o $(STORAGE_LIBS) $(EFENCE) ../util/euca_auth.o diff --git a/storage/backing.c b/storage/backing.c index a137d0c2dec..ae74e9b79f7 100644 --- a/storage/backing.c +++ b/storage/backing.c @@ -100,6 +100,30 @@ static void bs_errors (const char * msg) { logprintfl (EUCADEBUG2, "{%u} blobstore: %s", (unsigned int)pthread_self(), msg); } +static void stat_blobstore (const char * conf_instances_path, const char * name, blobstore_meta * meta) +{ + bzero (meta, sizeof (meta)); + char path [MAX_PATH]; + snprintf (path, sizeof (path), "%s/%s", conf_instances_path, name); + blobstore * bs = blobstore_open (path, + 0, // any size + 0, // no flags = do not create it + BLOBSTORE_FORMAT_ANY, + BLOBSTORE_REVOCATION_ANY, + BLOBSTORE_SNAPSHOT_ANY); + if (bs == NULL) + return; + blobstore_stat (bs, meta); + blobstore_close (bs); +} + +void stat_backing_store (const char * conf_instances_path, blobstore_meta * work_meta, blobstore_meta * cache_meta) +{ + assert (conf_instances_path); + stat_blobstore (conf_instances_path, "work", work_meta); + stat_blobstore (conf_instances_path, "cache", cache_meta); +} + int init_backing_store (const char * conf_instances_path, unsigned int conf_work_size_mb, unsigned int conf_cache_size_mb) { logprintfl (EUCAINFO, "initializing backing store...\n"); @@ -125,7 +149,7 @@ int init_backing_store (const char * conf_instances_path, unsigned int conf_work blobstore_set_error_function ( &bs_errors ); if (cache_limit_blocks) { - cache_bs = blobstore_open (cache_path, cache_limit_blocks, BLOBSTORE_FORMAT_DIRECTORY, BLOBSTORE_REVOCATION_LRU, BLOBSTORE_SNAPSHOT_ANY); + cache_bs = blobstore_open (cache_path, cache_limit_blocks, BLOBSTORE_FLAG_CREAT, BLOBSTORE_FORMAT_DIRECTORY, BLOBSTORE_REVOCATION_LRU, BLOBSTORE_SNAPSHOT_ANY); if (cache_bs==NULL) { logprintfl (EUCAERROR, "ERROR: %s\n", blobstore_get_error_str(blobstore_get_error())); return ERROR; @@ -135,7 +159,7 @@ int init_backing_store (const char * conf_instances_path, unsigned int conf_work return ERROR; } } - work_bs = blobstore_open (work_path, work_limit_blocks, BLOBSTORE_FORMAT_FILES, BLOBSTORE_REVOCATION_NONE, BLOBSTORE_SNAPSHOT_ANY); + work_bs = blobstore_open (work_path, work_limit_blocks, BLOBSTORE_FLAG_CREAT, BLOBSTORE_FORMAT_FILES, BLOBSTORE_REVOCATION_NONE, BLOBSTORE_SNAPSHOT_ANY); if (work_bs==NULL) { logprintfl (EUCAERROR, "ERROR: %s\n", blobstore_get_error_str(blobstore_get_error())); blobstore_close (cache_bs); diff --git a/storage/backing.h b/storage/backing.h index c49a0c826d9..0cbf1f9cc01 100644 --- a/storage/backing.h +++ b/storage/backing.h @@ -62,10 +62,12 @@ */ #include "data.h" // ncInstance +#include "blobstore.h" // blobstore_meta #define BACKING_FILE_PERM 0660 #define BACKING_DIRECTORY_PERM 0771 +void stat_backing_store (const char * conf_instances_path, blobstore_meta * work_meta, blobstore_meta * cache_meta); int init_backing_store (const char * conf_instances_path, unsigned int conf_work_size_mb, unsigned int conf_cache_size_mb); int create_instance_backing (ncInstance * instance); int clone_bundling_backing (ncInstance * instance, const char * filePrefix, char * blockPath); diff --git a/storage/blobstore.c b/storage/blobstore.c index d342ba7dbbb..87c5a48a153 100644 --- a/storage/blobstore.c +++ b/storage/blobstore.c @@ -828,6 +828,7 @@ int blobstore_cleanup (void) blobstore * blobstore_open ( const char * path, unsigned long long limit_blocks, + unsigned int flags, // BLOBSTORE_FLAG_CREAT - same semantcs as for open() flags blobstore_format_t format, blobstore_revocation_t revocation_policy, blobstore_snapshot_t snapshot_policy) @@ -844,29 +845,51 @@ blobstore * blobstore_open ( const char * path, char meta_path [PATH_MAX]; snprintf (meta_path, sizeof(meta_path), "%s/%s", bs->path, BLOBSTORE_METADATA_FILE); - _blobstore_errno = BLOBSTORE_ERROR_OK; - _err_off(); - bs->fd = open_and_lock (meta_path, BLOBSTORE_FLAG_CREAT | BLOBSTORE_FLAG_EXCL, 0, BLOBSTORE_FILE_PERM); - _err_on(); - if (bs->fd != -1) { // managed to create blobstore metadata file and got exclusive lock - - gen_id (bs->id, sizeof(bs->id)); - bs->limit_blocks = limit_blocks; - bs->revocation_policy = (revocation_policy==BLOBSTORE_REVOCATION_ANY) ? BLOBSTORE_REVOCATION_NONE : revocation_policy; - bs->snapshot_policy = (snapshot_policy==BLOBSTORE_SNAPSHOT_ANY) ? BLOBSTORE_SNAPSHOT_DM : snapshot_policy; // TODO: verify that DM is available? - bs->format = (format==BLOBSTORE_FORMAT_ANY) ? BLOBSTORE_FORMAT_FILES : format; - - // write metadata to disk - write_store_metadata (bs); - close_and_unlock (bs->fd); // try to close, thus giving up the exclusive lock - } - if (_blobstore_errno != BLOBSTORE_ERROR_OK && // either open or write failed - _blobstore_errno != BLOBSTORE_ERROR_EXIST && // it is OK if file already exists - _blobstore_errno != BLOBSTORE_ERROR_AGAIN ) { // it is OK if we lost the race for the write lock - ERR (_blobstore_errno, "failed to open or create blobstore"); - goto free; - } + int write_flags = 0; + if (flags & BLOBSTORE_FLAG_CREAT) { + write_flags = BLOBSTORE_FLAG_CREAT | BLOBSTORE_FLAG_EXCL; + }; + + write_metadata: + if (write_flags) { + _blobstore_errno = BLOBSTORE_ERROR_OK; + _err_off(); + bs->fd = open_and_lock (meta_path, write_flags, 0, BLOBSTORE_FILE_PERM); + _err_on(); + if (bs->fd != -1) { // managed to create or open blobstore metadata file and got exclusive lock + + // the intention is to create the blobstore for the first time + if (write_flags & BLOBSTORE_FLAG_CREAT) { + gen_id (bs->id, sizeof(bs->id)); + bs->limit_blocks = limit_blocks; + bs->revocation_policy = (revocation_policy==BLOBSTORE_REVOCATION_ANY) ? BLOBSTORE_REVOCATION_NONE : revocation_policy; + bs->snapshot_policy = (snapshot_policy==BLOBSTORE_SNAPSHOT_ANY) ? BLOBSTORE_SNAPSHOT_DM : snapshot_policy; // TODO: verify that DM is available? + bs->format = (format==BLOBSTORE_FORMAT_ANY) ? BLOBSTORE_FORMAT_FILES : format; + + // write metadata to disk + write_store_metadata (bs); + + } else if (write_flags & BLOBSTORE_FLAG_RDWR) { // the intention is to adjust metadata + if (read_store_metadata (bs)) + goto free; + assert (bs->id); + if (limit_blocks) + bs->limit_blocks = limit_blocks; + if (revocation_policy != BLOBSTORE_REVOCATION_ANY) + bs->revocation_policy = revocation_policy; + write_store_metadata (bs); + } + close_and_unlock (bs->fd); // try to close, thus giving up the exclusive lock + } + if (_blobstore_errno != BLOBSTORE_ERROR_OK && // either open or write failed + _blobstore_errno != BLOBSTORE_ERROR_EXIST && // it is OK if file already exists + _blobstore_errno != BLOBSTORE_ERROR_AGAIN ) { // it is OK if we lost the race for the write lock + ERR (_blobstore_errno, "failed to open or create blobstore"); + goto free; + } + } + // now (re)open, with a shared read lock bs->fd = open_and_lock (meta_path, BLOBSTORE_FLAG_RDONLY, BLOBSTORE_METADATA_TIMEOUT_USEC, BLOBSTORE_FILE_PERM); if (bs->fd == -1) { @@ -879,8 +902,13 @@ blobstore * blobstore_open ( const char * path, // verify that parameters are not being changed if (limit_blocks && limit_blocks != bs->limit_blocks) { - ERR (BLOBSTORE_ERROR_INVAL, "'limit_blocks' does not match existing blobstore"); - goto free; + if (flags & BLOBSTORE_FLAG_STRICT ) { + ERR (BLOBSTORE_ERROR_INVAL, "'limit_blocks' does not match existing blobstore"); + goto free; + } else { + write_flags = BLOBSTORE_FLAG_RDWR; + goto write_metadata; + } } if (snapshot_policy != BLOBSTORE_SNAPSHOT_ANY && snapshot_policy != bs->snapshot_policy) { @@ -894,8 +922,13 @@ blobstore * blobstore_open ( const char * path, } if (revocation_policy != BLOBSTORE_REVOCATION_ANY && revocation_policy != bs->revocation_policy) { - ERR (BLOBSTORE_ERROR_INVAL, "'revocation_policy' does not match existing blobstore"); // TODO: maybe make revocation_policy changeable after creation - goto free; + if (flags & BLOBSTORE_FLAG_STRICT) { + ERR (BLOBSTORE_ERROR_INVAL, "'revocation_policy' does not match existing blobstore"); // TODO: maybe make revocation_policy changeable after creation + goto free; + } else { + write_flags = BLOBSTORE_FLAG_RDWR; + goto write_metadata; + } } int fd = bs->fd; bs->fd = -1; @@ -1482,6 +1515,52 @@ static long long purge_blockblobs_lru ( blobstore * bs, blockblob * bb_list, lon return purged; } +int blobstore_stat (blobstore * bs, blobstore_meta * meta) +{ + int ret = 0; + + if (blobstore_lock(bs, BLOBSTORE_LOCK_TIMEOUT_USEC)==-1) { // lock it so we can traverse blobstore safely + return ERROR; + } + + // put existing items in the blobstore into a LL + _blobstore_errno = BLOBSTORE_ERROR_OK; + blockblob * bbs = scan_blobstore (bs, NULL); + if (bbs==NULL) { + if (_blobstore_errno != BLOBSTORE_ERROR_OK) { + goto unlock; + } + } + + // analyze the LL, calculating sizes + meta->blocks_allocated = 0; + meta->blocks_used = 0; + meta->num_blobs = 0; + for (blockblob * abb = bbs; abb; abb=abb->next) { + long long abb_size_blocks = round_up_sec (abb->size_bytes) / 512; + if (abb->in_use & ~BLOCKBLOB_STATUS_BACKED) { + meta->blocks_used += abb_size_blocks; // these can't be purged if we need space (TODO: look into recursive purging of unused references?) + } else { + meta->blocks_allocated += abb_size_blocks; // these can be purged + } + meta->num_blobs++; + } + + unlock: + + if (blobstore_unlock(bs)==-1) { + ERR (BLOBSTORE_ERROR_UNKNOWN, "failed to unlock the blobstore"); + } + + safe_strncpy (meta->id, bs->id, sizeof (meta->id)); + meta->revocation_policy = bs->revocation_policy; + meta->snapshot_policy = bs->snapshot_policy; + meta->format = bs->format; + meta->blocks_limit = bs->limit_blocks; + + return ret; +} + // integrity check of the blobstore // // with a non-NULL examiner(), each found blob is passed to it @@ -1513,10 +1592,10 @@ int blobstore_fsck (blobstore * bs, int (* examiner) (const blockblob * bb)) } // run through LL, examining each blockblob - unsigned int blobs_total = 0; + unsigned int num_blobs = 0; unsigned int blobs_deleted = 0; for (blockblob * abb = bbs; abb; abb=abb->next) { - blobs_total++; + num_blobs++; // examiner(), if specified, tell us whether to delete the blob if (examiner && examiner (abb)) { @@ -1543,7 +1622,7 @@ int blobstore_fsck (blobstore * bs, int (* examiner) (const blockblob * bb)) set_blockblob_metadata_path (BLOCKBLOB_PATH_LOOPBACK, bs, abb->id, path, sizeof(path)); // load path of .../loopback file unlink (path); } - logprintfl (EUCAINFO, "fsck deleted %d of %d blobs\n", blobs_deleted, blobs_total); + logprintfl (EUCAINFO, "fsck deleted %d of %d blobs\n", blobs_deleted, num_blobs); free: if (bbs) @@ -1580,11 +1659,11 @@ int blobstore_search ( blobstore * bs, const char * regex, blockblob_meta ** res } // run through LL, looking for matches - unsigned int blobs_total = 0; + unsigned int num_blobs = 0; unsigned int blobs_matched = 0; blockblob_meta * prev = NULL; for (blockblob * abb = bbs; abb; abb=abb->next) { - blobs_total++; + num_blobs++; if (regexec (&re, abb->id, 0, NULL, 0) != 0) continue; blobs_matched++; @@ -1788,19 +1867,19 @@ blockblob * blockblob_open ( blobstore * bs, // analyze the LL, calculating sizes long long blocks_allocated = 0; - long long blocks_inuse = 0; - unsigned int blobs_total = 0; + long long blocks_used = 0; + unsigned int num_blobs = 0; for (blockblob * abb = bbs; abb; abb=abb->next) { long long abb_size_blocks = round_up_sec (abb->size_bytes) / 512; if (abb->in_use & ~BLOCKBLOB_STATUS_BACKED) { - blocks_inuse += abb_size_blocks; // these can't be purged if we need space (TODO: look into recursive purging of unused references?) + blocks_used += abb_size_blocks; // these can't be purged if we need space (TODO: look into recursive purging of unused references?) } else { blocks_allocated += abb_size_blocks; // these can be purged } - blobs_total++; + num_blobs++; } - long long blocks_free = bs->limit_blocks - (blocks_allocated + blocks_inuse); + long long blocks_free = bs->limit_blocks - (blocks_allocated + blocks_used); if (blocks_free < size_blocks) { if (!(bs->revocation_policy==BLOBSTORE_REVOCATION_LRU) // not allowed to purge || @@ -2251,7 +2330,7 @@ int blockblob_delete ( blockblob * bb, long long timeout_usec ) blobstore * dep_bs = bs; if (strcmp (bs->path, store_path)) { // if deleting reference in a different blobstore // need to open it - dep_bs = blobstore_open (store_path, 0, BLOBSTORE_FORMAT_ANY, BLOBSTORE_REVOCATION_ANY, BLOBSTORE_SNAPSHOT_ANY); + dep_bs = blobstore_open (store_path, 0, BLOBSTORE_FLAG_CREAT, BLOBSTORE_FORMAT_ANY, BLOBSTORE_REVOCATION_ANY, BLOBSTORE_SNAPSHOT_ANY); if (dep_bs == NULL) continue; // TODO: print a warning about store/blob corruption? if (blobstore_lock (dep_bs, timeout_usec)==-1) { // lock this (different) blobstore, too, so .refs are updated atomically @@ -2867,7 +2946,7 @@ static blobstore * create_teststore (int size_blocks, const char * base, const c return NULL; } printf ("created %s\n", bs_path); - blobstore * bs = blobstore_open (bs_path, size_blocks, format, revocation, snapshot); + blobstore * bs = blobstore_open (bs_path, size_blocks, BLOBSTORE_FLAG_CREAT, format, revocation, snapshot); if (bs==NULL) { printf ("ERROR: %s\n", blobstore_get_error_str(blobstore_get_error())); return NULL; @@ -3075,13 +3154,10 @@ static int do_clone_stresstest (const char * base, const char * name, blobstore_ _DELWARN(bbs2[i+STRESS_BLOBS]); } - // blobstore_close (bs1); - // blobstore_close (bs2); - printf ("completed cloning stress-test\n"); done: - if(bs1 != NULL) blobstore_close (bs1); - if(bs2 != NULL) blobstore_close (bs2); + if (bs1 != NULL) blobstore_close (bs1); + if (bs2 != NULL) blobstore_close (bs2); return errors; } @@ -3703,7 +3779,7 @@ int main (int argc, char ** argv) // this allows two simultaneous invocations of test_blobstore to compete // for the same blob so as to test the inter-process locks manually if (argc > 1) { - blobstore * bs = blobstore_open (".", 1000, BLOBSTORE_FORMAT_FILES, BLOBSTORE_REVOCATION_ANY, BLOBSTORE_SNAPSHOT_ANY); + blobstore * bs = blobstore_open (".", 1000, BLOBSTORE_FLAG_CREAT, BLOBSTORE_FORMAT_FILES, BLOBSTORE_REVOCATION_ANY, BLOBSTORE_SNAPSHOT_ANY); if (bs==NULL) { printf ("ERROR: when opening blobstore: %s\n", blobstore_get_error_str(blobstore_get_error())); return 1; diff --git a/storage/blobstore.h b/storage/blobstore.h index f17a18ad89f..6940beb1a21 100644 --- a/storage/blobstore.h +++ b/storage/blobstore.h @@ -83,6 +83,7 @@ #define BLOBSTORE_FLAG_RDONLY 00002 #define BLOBSTORE_FLAG_CREAT 00004 #define BLOBSTORE_FLAG_EXCL 00010 +#define BLOBSTORE_FLAG_STRICT 01000 // in-use flags for blockblobs #define BLOCKBLOB_STATUS_OPENED 00002 // currently opened by someone (read or write) @@ -201,13 +202,26 @@ typedef struct _blockblob_meta { struct _blockblob_meta * prev; } blockblob_meta; +typedef struct _blobstore_meta { + char id [BLOBSTORE_MAX_PATH]; // ID of the blobstore, to handle directory moving + unsigned long long blocks_limit; // max size of the blobstore, in blocks + unsigned long long blocks_allocated; // number of blocks in blobstore allocated to a blob that is not in use and is not mapped + unsigned long long blocks_used; // number of blocks in blobstore allocated to a blob that is in use or is mapped (a dependency) + unsigned int num_blobs; // count of blobs in the blobstore + blobstore_revocation_t revocation_policy; + blobstore_snapshot_t snapshot_policy; + blobstore_format_t format; +} blobstore_meta; + // blobstore operations blobstore * blobstore_open ( const char * path, unsigned long long limit_blocks, // on create: 0 is not valid; on open: 0 = any size + unsigned int flags, blobstore_format_t format, blobstore_revocation_t revocation_policy, blobstore_snapshot_t snapshot_policy); +int blobstore_stat (blobstore * bs, blobstore_meta * meta); int blobstore_fsck (blobstore * bs, int (* examiner) (const blockblob * bb)); int blobstore_search ( blobstore * bs, const char * regex, blockblob_meta ** results ); // returns a list of blockblobs matching an expression int blobstore_delete_regex (blobstore * bs, const char * regex); // delete all blobs in blobstore that match regex, return number deleted or -1 if error diff --git a/storage/imager.c b/storage/imager.c index c2acd3cc170..f86a831b63e 100644 --- a/storage/imager.c +++ b/storage/imager.c @@ -275,7 +275,7 @@ int main (int argc, char * argv[]) if (ensure_directories_exist (get_work_dir(), 0, NULL, NULL, BLOBSTORE_DIRECTORY_PERM) == -1) err ("failed to open or create work directory"); - work_bs = blobstore_open (get_work_dir(), get_work_limit()/512, BLOBSTORE_FORMAT_FILES, BLOBSTORE_REVOCATION_NONE, BLOBSTORE_SNAPSHOT_ANY); + work_bs = blobstore_open (get_work_dir(), get_work_limit()/512, BLOBSTORE_FLAG_CREAT, BLOBSTORE_FORMAT_FILES, BLOBSTORE_REVOCATION_NONE, BLOBSTORE_SNAPSHOT_ANY); if (work_bs==NULL) { logprintfl (EUCAERROR, "ERROR: %s\n", blobstore_get_error_str(blobstore_get_error())); err ("failed to open work blobstore"); @@ -287,7 +287,7 @@ int main (int argc, char * argv[]) if (root && tree_uses_cache (root)) { if (ensure_directories_exist (get_cache_dir(), 0, NULL, NULL, BLOBSTORE_DIRECTORY_PERM) == -1) err ("failed to open or create cache directory"); - cache_bs = blobstore_open (get_cache_dir(), get_cache_limit()/512, BLOBSTORE_FORMAT_DIRECTORY, BLOBSTORE_REVOCATION_LRU, BLOBSTORE_SNAPSHOT_ANY); + cache_bs = blobstore_open (get_cache_dir(), get_cache_limit()/512, BLOBSTORE_FLAG_CREAT, BLOBSTORE_FORMAT_DIRECTORY, BLOBSTORE_REVOCATION_LRU, BLOBSTORE_SNAPSHOT_ANY); if (cache_bs==NULL) { logprintfl (EUCAERROR, "ERROR: %s\n", blobstore_get_error_str(blobstore_get_error())); blobstore_close (work_bs); diff --git a/storage/storage.c b/storage/storage.c deleted file mode 100644 index 8ca59d2a67c..00000000000 --- a/storage/storage.c +++ /dev/null @@ -1,1264 +0,0 @@ -// -*- mode: C; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- -// vim: set softtabstop=4 shiftwidth=4 tabstop=4 expandtab: - -/* - Copyright (c) 2009 Eucalyptus Systems, Inc. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, only version 3 of the License. - - This file is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License along - with this program. If not, see . - - Please contact Eucalyptus Systems, Inc., 130 Castilian - Dr., Goleta, CA 93101 USA or visit - if you need additional information or have any questions. - - This file may incorporate work covered under the following copyright and - permission notice: - - Software License Agreement (BSD License) - - Copyright (c) 2008, Regents of the University of California - - - Redistribution and use of this software in source and binary forms, with - or without modification, are permitted provided that the following - conditions are met: - - Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. USERS OF - THIS SOFTWARE ACKNOWLEDGE THE POSSIBLE PRESENCE OF OTHER OPEN SOURCE - LICENSED MATERIAL, COPYRIGHTED MATERIAL OR PATENTED MATERIAL IN THIS - SOFTWARE, AND IF ANY SUCH MATERIAL IS DISCOVERED THE PARTY DISCOVERING - IT MAY INFORM DR. RICH WOLSKI AT THE UNIVERSITY OF CALIFORNIA, SANTA - BARBARA WHO WILL THEN ASCERTAIN THE MOST APPROPRIATE REMEDY, WHICH IN - THE REGENTS' DISCRETION MAY INCLUDE, WITHOUT LIMITATION, REPLACEMENT - OF THE CODE SO IDENTIFIED, LICENSING OF THE CODE SO IDENTIFIED, OR - WITHDRAWAL OF THE CODE CAPABILITY TO THE EXTENT NEEDED TO COMPLY WITH - ANY SUCH LICENSES OR RIGHTS. -*/ -#define _FILE_OFFSET_BITS 64 // so large-file support works on 32-bit systems -#include -#include -#define __USE_GNU /* strnlen */ -#include -#include -#include -#include -#include -#include -#include -#include -#include /* open|read|close dir */ -#include /* time() */ - -#include "eucalyptus.h" -#include "ipc.h" -#include "walrus.h" -#include "euca_auth.h" -#include -#include -#include -#include -#include -#include - -#define BUFSIZE 512 /* random buffer size used all over the place */ - -/* default paths(may be overriden from config file) */ -static char add_key_command_path [BUFSIZE] = ""; -static long long swap_size_mb = DEFAULT_SWAP_SIZE; /* default swap in MB, if not specified in config file */ -static long long cache_size_mb = DEFAULT_NC_CACHE_SIZE; /* in MB */ -static long long cache_free_mb = DEFAULT_NC_CACHE_SIZE; -static long long work_size_mb = DEFAULT_NC_WORK_SIZE; -static long long work_free_mb = DEFAULT_NC_WORK_SIZE; - -static char *sc_instance_path = ""; -static char disk_convert_command_path [BUFSIZE] = ""; -static int scConfigInit=0; -static sem * sc_sem; -static sem * disk_sem; - -int scInitConfig (void) -{ - struct stat mystat; - char configFiles[2][MAX_PATH]; - char * s; - int concurrent_disk_ops; - - if (scConfigInit) { - return 0; - } - - if ((sc_sem = sem_alloc (1, "mutex")) == NULL) { /* TODO: use this semaphore to fix the race */ - logprintfl (EUCAERROR, "failed to create and initialize storage semaphore\n"); - return 1; - } - - /* read in configuration */ - char *home, *tmp; - tmp = getenv (EUCALYPTUS_ENV_VAR_NAME); - if (tmp) { - home = strdup(tmp); - } else { - home = strdup(""); /* root by default */ - } - if (!home) { - logprintfl (EUCAERROR, "out of memory\n"); - return 1; - } - - snprintf(configFiles[0], BUFSIZE, EUCALYPTUS_CONF_OVERRIDE_LOCATION, home); - snprintf(configFiles[1], BUFSIZE, EUCALYPTUS_CONF_LOCATION, home); - if (stat(configFiles[1], &mystat)==0 || stat(configFiles[0], &mystat) == 0) { - logprintfl (EUCAINFO, "SC is looking for configuration in files (%s,%s)\n", configFiles[1], configFiles[0]); - s = getConfString(configFiles, 2, INSTANCE_PATH); - if (s) { - sc_instance_path = s; - } - - s = getConfString(configFiles, 2, CONFIG_NC_CACHE_SIZE); - if (s) { - cache_size_mb = atoll (s); - cache_free_mb = cache_size_mb; - free (s); - } - - s = getConfString(configFiles, 2, CONFIG_NC_WORK_SIZE); - if (s) { - work_size_mb = atoll (s); - work_free_mb = work_size_mb; - free (s); - } - - - s = getConfString(configFiles, 2, CONFIG_NC_SWAP_SIZE); - if (s){ - swap_size_mb = atoll (s); - free (s); - } - - concurrent_disk_ops = 1; - s = getConfString(configFiles, 2, CONFIG_CONCURRENT_DISK_OPS); - if (s) { - concurrent_disk_ops = atoi(s); - free(s); - } - /* set the initial value of semaphore to number of 'disk intensive' operations that can run in parallel on this node */ - if ((disk_sem = sem_alloc (concurrent_disk_ops, "mutex")) == NULL) { - logprintfl (EUCAERROR, "failed to create and initialize disk semaphore\n"); - if (home) free(home); - return(1); - } - } - snprintf(add_key_command_path, BUFSIZE, EUCALYPTUS_ADD_KEY, home, home, home); - - /* we need to have valid path */ - if (check_directory(sc_instance_path)) { - logprintfl (EUCAERROR, "ERROR: INSTANCE_PATH (%s) does not exist!\n", sc_instance_path); - if (home) free(home); - return(1); - } - - if (euca_init_cert ()) { - logprintfl (EUCAFATAL, "failed to find cryptographic certificates\n"); - if (home) free(home); - return 1; - } - - snprintf (disk_convert_command_path, BUFSIZE, EUCALYPTUS_DISK_CONVERT, home, home); - if (home) free(home); - - scConfigInit=1; - return(0); -} - -void scSaveInstanceInfo (const ncInstance * instance) -{ - char file_path [BUFSIZE]; - int fd; - - if (instance==NULL) return; - snprintf(file_path, BUFSIZE, "%s/%s/%s/instance-checkpoint", sc_instance_path, instance->userId, instance->instanceId); - if ((fd=open(file_path, O_CREAT | O_WRONLY, 0600))<0) return; - write (fd, (char *)instance, sizeof(struct ncInstance_t)); - close (fd); -} - -ncInstance * scRecoverInstanceInfo (const char *instanceId) -{ - const int file_size = sizeof(struct ncInstance_t); - ncInstance * instance = malloc(file_size); - char file_path [BUFSIZE]; - struct dirent * dir_entry; - DIR * insts_dir; - char * userId = NULL; - int fd; - - if (instance==NULL) { - logprintfl(EUCADEBUG, "scRecoverInstanceInfo: NULL instance!\n"); - return NULL; - } - - /* we don't know userId, so we'll look for instanceId in every user's - * directory (we're assuming that instanceIds are unique in the system) */ - if ((insts_dir=opendir(sc_instance_path))==NULL) { - logprintfl(EUCADEBUG, "scRecoverInstanceInfo: failed to open %s!\n", sc_instance_path); - free(instance); - return NULL; - } - while ((dir_entry=readdir(insts_dir))!=NULL) { - char tmp_path [BUFSIZE]; - struct stat mystat; - - snprintf(tmp_path, BUFSIZE, "%s/%s/%s", sc_instance_path, dir_entry->d_name, instanceId); - if (stat(tmp_path, &mystat)==0) { - userId = strdup (dir_entry->d_name); - break; /* we got it! */ - } - } - closedir(insts_dir); - if (userId==NULL) { - logprintfl(EUCADEBUG, "scRecoverInstanceInfo: didn't find instance %s!\n", instanceId); - free(instance); - return NULL; - } - - snprintf(file_path, BUFSIZE, "%s/%s/%s/instance-checkpoint", sc_instance_path, userId, instanceId); - free(userId); - if ((fd=open(file_path, O_RDONLY))<0 || - read(fd, instance, file_size)stateCode = NO_STATE; - return instance; -} - -typedef struct cache_entry_t { - char path [BUFSIZE]; - long long size_mb; - struct cache_entry_t * next; - struct cache_entry_t * prev; -} cache_entry; - -static cache_entry * cache_head = NULL; - -static void add_to_cache (const char * cached_path, const long long file_size_bytes) -{ - long long file_size_mb = file_size_bytes/MEGABYTE; - - cache_entry * e = malloc (sizeof(cache_entry)); - if (e==NULL) { - logprintfl (EUCAFATAL, "error: out of memory in add_to_cache()\n"); - return; - } - - safe_strncpy (e->path, cached_path, BUFSIZE); - e->size_mb = file_size_mb; - e->next = NULL; - e->prev = NULL; - - // add at the end - cache_entry ** pp; - cache_entry * p = NULL; - for ( pp = & cache_head; * pp != NULL; pp = & ((* pp)->next)) p = * pp; - if ( p ) { - e->prev = p; - } - * pp = e; - - cache_free_mb -= file_size_mb; -} - -void LogprintfCache (void) -{ - struct stat mystat; - cache_entry * e; - if (cache_head) { - logprintfl (EUCAINFO, "cached images (free=%lld of %lldMB):\n", cache_free_mb, cache_size_mb); - } else { - logprintfl (EUCAINFO, "cached images (free=%lld of %lldMB): none\n", cache_free_mb, cache_size_mb); - } - for ( e = cache_head; e; e=e->next) { - bzero (&mystat, sizeof (mystat)); - if (stat (e->path, &mystat) == 0) { - logprintfl (EUCAINFO, "\t%5dMB %8dsec %s\n", e->size_mb, mystat.st_mtime, e->path); - } - } -} - -/* Returns 1 if there is space in the cache for the image, - * purging the cache in LRU fashion, if necessary. - * The function assumes the image is not in the cache already */ -static int ok_to_cache (const char * cached_path, const long long file_size_bytes) -{ - long long file_size_mb = file_size_bytes/MEGABYTE; - - if (file_size_mb > cache_size_mb) return 0; - - while (file_size_mb > cache_free_mb) { - time_t oldest_mtime = time (NULL) + 1; - off_t oldest_size = 0; - cache_entry * oldest_entry = NULL; - struct stat mystat; - cache_entry * e; - for ( e = cache_head; e; e=e->next) { - if (stat (e->path, &mystat)<0) { - logprintfl (EUCAERROR, "error: ok_to_cache() can't stat %s\n", cached_path); - return 0; - } - if (mystat.st_mtimesize_mb; /* (mystat.st_size doesn't include digest) */ - oldest_entry = e; - } else { - if (mystat.st_mtime==oldest_mtime) { - /* smaller ones get purged first */ - if (oldest_size > e->size_mb) { - oldest_size = e->size_mb; - oldest_entry = e; - } - } - } - } - - if ( oldest_entry ) { // remove it - logprintfl (EUCAINFO, "purging from cache image %s\n", oldest_entry->path); - if ( oldest_entry->next ) { - oldest_entry->next->prev = oldest_entry->prev; - } - if ( oldest_entry->prev ) { - oldest_entry->prev->next = oldest_entry->next; - } else { - cache_head = oldest_entry->next; - } - if ( unlink (oldest_entry->path) != 0 ) { // should allow open descriptors to complete - logprintfl (EUCAERROR, "error: failed to unlink file %s (%s)\n", oldest_entry->path, strerror (errno)); - } - char digest_path [BUFSIZE]; - snprintf (digest_path, BUFSIZE, "%s-digest", oldest_entry->path); - unlink (digest_path); - cache_free_mb += oldest_entry->size_mb; - free (oldest_entry); - } else { - logprintfl (EUCAERROR, "error: cannot find oldest entry in cache\n"); - return 0; - } - } - add_to_cache (cached_path, file_size_bytes); - return 1; -} - -static long long init_cache (const char * cache_path) -{ - long long total_size = 0; - - logprintfl (EUCAINFO, "checking the integrity of the cache directory (%s)\n", cache_path); - - if (cache_path==NULL) { - logprintfl (EUCAINFO, "no cache directory yet\n"); - return total_size; - } - - struct stat mystat; - if (stat (cache_path, &mystat) < 0) { - logprintfl (EUCAFATAL, "error: could not stat %s\n", cache_path); - return -1; - } - total_size += mystat.st_size; - - DIR * cache_dir; - if ((cache_dir=opendir(cache_path))==NULL) { - logprintfl (EUCAFATAL, "errror: could not open cache directory %s\n", cache_path); - return -1; - } - - struct dirent * cache_dir_entry; - while ((cache_dir_entry=readdir(cache_dir))!=NULL) { - char * image_name = cache_dir_entry->d_name; - char image_path [BUFSIZE]; - int image_size = 0; - int image_files = 0; - - if (!strcmp(".", image_name) || - !strcmp("..", image_name)) - continue; - - DIR * image_dir; - snprintf (image_path, BUFSIZE, "%s/%s", cache_path, image_name); - if ((image_dir=opendir(image_path))==NULL) { - logprintfl (EUCAWARN, "warning: unopeneable directory %s\n", image_path); - continue; - } - - if (stat (image_path, &mystat) < 0) { - logprintfl (EUCAWARN, "warning: could not stat %s\n", image_path); - closedir(image_dir); - continue; - } - image_size += mystat.st_size; - - /* make sure that image directory contains only two files: one - * named X and another X-digest, also add up their sizes */ - char X [BUFSIZE] = ""; - char X_digest [BUFSIZE] = ""; - struct dirent * image_dir_entry; - while ((image_dir_entry=readdir(image_dir))!=NULL) { - char name [BUFSIZE]; - strncpy (name, image_dir_entry->d_name, BUFSIZE); - - if (!strcmp(".", name) || - !strcmp("..", name)) - continue; - - image_files++; - - char filepath [BUFSIZE]; - snprintf (filepath, BUFSIZE, "%s/%s", image_path, name); - if (stat (filepath, &mystat) < 0 ) { - logprintfl (EUCAERROR, "error: could not stat file %s\n", filepath); - break; - } - if (mystat.st_size < 1) { - logprintfl (EUCAERROR, "error: empty file among cached images in %s\n", filepath); - break; - } - image_size += mystat.st_size; - - char * suffix; - if ((suffix=strstr (name, "-digest"))==NULL) { - if (strlen (X)) - break; /* already saw X => fail */ - safe_strncpy (X, name, BUFSIZE); - } else { - if (strlen (X_digest)) - break; /* already saw X-digest => fail */ - * suffix = '\0'; - safe_strncpy (X_digest, name, BUFSIZE); - } - } - closedir(image_dir); - - if (image_files > 0) { /* ignore empty directories */ - if (image_files != 2 || strncmp (X, X_digest, BUFSIZE) != 0 ) { - logprintfl (EUCAERROR, "error: inconsistent state of cached image %s, deleting it\n", image_name); - if (vrun ("rm -rf %s", image_path)) { - logprintfl (EUCAWARN, "warning: failed to remove %s\n", image_path); - } - } else { - char filepath [BUFSIZE]; - snprintf (filepath, BUFSIZE, "%s/%s", image_path, X); - if (image_size>0) { - logprintfl (EUCAINFO, "- cached image %s directory, size=%d\n", image_name, image_size); - total_size += image_size; - add_to_cache (filepath, image_size); - } else { - logprintfl (EUCAWARN, "warning: empty cached image directory %s\n", image_path); - } - } - } - } - closedir (cache_dir); - - return total_size; -} - -#define F1 "/tmp/improbable-cache-file-1" -#define F2 "/tmp/improbable-cache-file-2" -#define F3 "/tmp/improbable-cache-file-3" -#define F4 "/tmp/improbable-cache-file-4" -#define F5 "/tmp/improbable-cache-file-5" -#define RM_CMD "rm -rf /tmp/improbable-cache-file-?" - -int test_cache (void) -{ - int error = 0; - - /* save the current values */ - long long saved_size = cache_size_mb; - long long saved_free = cache_free_mb; - cache_entry * saved_head = cache_head; - - cache_size_mb = 10; - cache_free_mb = 10; - cache_head = NULL; - - touch (F1); - if (ok_to_cache (F1, 3*MEGABYTE)!=1) { error= 1; goto out; } - LogprintfCache(); - sleep (1); - - touch (F2); - add_to_cache (F2, 3*MEGABYTE); - LogprintfCache(); - sleep (1); - - touch (F3); - if (ok_to_cache (F3, 11*MEGABYTE)!=0) { error = 2; goto out; } - if (ok_to_cache (F3, 7*MEGABYTE)!=1) { error = 3; goto out; } - LogprintfCache(); - - touch (F4); - if (ok_to_cache (F4, 4*MEGABYTE)!=1) { error = 4; goto out; } - touch (F5); - if (ok_to_cache (F5, 6*MEGABYTE)!=1) { error = 5; goto out; } - LogprintfCache(); - - touch (F3); - add_to_cache (F3, 3*MEGABYTE); - touch (F2); - add_to_cache (F2, 5*MEGABYTE); - LogprintfCache(); - - touch (F1); - if (ok_to_cache (F1, 1*MEGABYTE)!=1) { error = 6; goto out; } - LogprintfCache(); - - out: - cache_size_mb = saved_size; - cache_free_mb = saved_free; - cache_head = saved_head; - system (RM_CMD); - return error; -} - -/* perform integrity check on instances directory, including the cache: - * remove any files from non-running Eucalyptus instances, delete files - * from cache that are not complete, return the amount of bytes used up by - * everything - */ -long long scFSCK (bunchOfInstances ** instances) -{ - long long total_size = 0; - struct stat mystat; - - if (instances==NULL) return -1; - - logprintfl (EUCAINFO, "checking the integrity of instances directory (%s)\n", sc_instance_path); - - /* let us not 'rm -rf /' accidentally */ - if (strlen(sc_instance_path)<2 || - sc_instance_path[0]!='/' ) { - logprintfl (EUCAFATAL, "error: instances directory cannot be /, sorry\n"); - return -1; - } - - if (stat (sc_instance_path, &mystat) < 0) { - logprintfl (EUCAFATAL, "error: could not stat %s\n", sc_instance_path); - return -1; - } - total_size += mystat.st_size; - - DIR * insts_dir; - if ((insts_dir=opendir(sc_instance_path))==NULL) { - logprintfl (EUCAFATAL, "error: could not open instances directory %s\n", sc_instance_path); - return -1; - } - - /*** run through all users ***/ - - char * cache_path = NULL; - char * work_path = NULL; - struct dirent * inst_dir_entry; - while ((inst_dir_entry=readdir(insts_dir))!=NULL) { - char * uname = inst_dir_entry->d_name; - char user_path [BUFSIZE]; - struct dirent * user_dir_entry; - DIR * user_dir; - - if (!strcmp(".", uname) || - !strcmp("..", uname)) - continue; - - snprintf (user_path, BUFSIZE, "%s/%s", sc_instance_path, uname); - if ((user_dir=opendir(user_path))==NULL) { - logprintfl (EUCAWARN, "warning: unopeneable directory %s\n", user_path); - continue; - } - - /*** run through all instances of a user ***/ - while ((user_dir_entry=readdir(user_dir))!=NULL) { - char * iname = user_dir_entry->d_name; - - if (!strcmp(".", iname) || - !strcmp("..", iname)) - continue; - - char instance_path [BUFSIZE]; - snprintf (instance_path, BUFSIZE, "%s/%s", user_path, iname); - - if (!strcmp("cache", iname) && - !strcmp(EUCALYPTUS_ADMIN, uname)) { /* cache is in admin's dir */ - if (cache_path) { - logprintfl (EUCADEBUG, "Found a second cache_path?\n"); - free(cache_path); - } - cache_path = strdup (instance_path); - continue; - } - - if (!strcmp("work", iname) && - !strcmp(EUCALYPTUS_ADMIN, uname)) { /* work is in admin's dir */ - if (work_path) { - logprintfl (EUCADEBUG, "Found a second work_path?\n"); - free(work_path); - } - work_path = strdup (instance_path); - continue; - } - - /* spare directories of running instances, but count their usage */ - if (find_instance (instances, iname)) { - long long bytes = dir_size (instance_path); - if (bytes>0) { - logprintfl (EUCAINFO, "- running instance %s directory, size=%d\n", iname, bytes); - total_size += bytes; - } else if (bytes==0) { - logprintfl (EUCAWARN, "warning: empty instance directory %s\n", instance_path); - } else { - logprintfl (EUCAWARN, "warning: non-standard instance directory %s\n", instance_path); - } - continue; - } - - /* looks good - destroy it */ - if (vrun ("rm -rf %s", instance_path)) { - logprintfl (EUCAWARN, "warning: failed to remove %s\n", instance_path); - } - } - closedir (user_dir); - } - closedir (insts_dir); - - /*** scan the cache ***/ - long long cache_bytes = init_cache (cache_path); - free (cache_path); - if (cache_bytes < 0) { - if (work_path) free(work_path); - return -1; - } - - // clean up work directory - if (work_path) { - if (vrun ("rm -rf %s", work_path)) { - logprintfl (EUCAWARN, "warning: failed to clean work directory %s\n", work_path); - } - free (work_path); - } - - return total_size + cache_bytes; -} - -const char * scGetInstancePath(void) -{ - return sc_instance_path; -} - -int scSetInstancePath(char *path) -{ - sc_instance_path = strdup(path); - return(0); -} - -int scCleanupInstanceImage (char *user, char *instId) -{ - return vrun ("rm -rf %s/%s/%s/", sc_instance_path, user, instId); -} - -/* if path=A/B/C but only A exists, this will try to create B and C */ -int ensure_path_exists (const char * path) -{ - mode_t mode = 0777; - int len = strlen(path); - char * path_copy = strdup(path); - int i; - - if (path_copy==NULL) return errno; - - for (i=0; i0) { - path_copy[i] = '\0'; - try_it = 1; - } else if (path[i]!='/' && i+1==len) { /* last one */ - try_it = 1; - } - - if ( try_it ) { - if ( stat (path_copy, &buf) == -1 ) { - printf ("trying to create path %s\n", path_copy); - if ( mkdir (path_copy, mode) == -1) { - printf ("error: failed to create path %s\n", path_copy); - if (path_copy) free(path_copy); - return errno; - } - } - path_copy[i] = '/'; /* restore the slash */ - } - } - - free (path_copy); - return 0; -} - -/* if path=A/B/C/D but only A exists, this will try to create B and C, but not D */ -int ensure_subdirectory_exists (const char * path) -{ - int len = strlen(path); - char * path_copy = strdup(path); - int i; - - if (path_copy==NULL) return errno; - - for (i=len-1; i>0; i--) { - if (path[i]=='/') { - path_copy[i] = '\0'; - ensure_path_exists (path_copy); - break; - } - } - - free (path_copy); - return 0; -} - -/* wait for file 'appear' to appear or for file 'disappear' to disappear */ -static int wait_for_file (const char * appear, const char * disappear, const int iterations, const char * name) -{ - int done, i; - if (!appear && !disappear) return 1; - - for ( i=0, done=0; i", ""); - free (xml_file); - } - if (file_size_b > 0) { - long long full_size_b = file_size_b+digest_size_b; - if (convert_to_disk) { - full_size_b += swap_size_mb*MEGABYTE + MEGABYTE; /* TODO: take into account extra padding required for disks (over partitions) */ - } - if ( full_size_b/MEGABYTE + 1 > limit_mb ) { - logprintfl (EUCAFATAL, "error: insufficient disk capacity remaining (%lldMB) in VM Type of instance %s for component %s\n", limit_mb, instance_id, file_name); - action = ABORT; - - } else if ( ok_to_cache (cached_path, full_size_b) ) { /* will invalidate the cache, if needed */ - ensure_path_exists (cached_dir); /* creates missing directories */ - should_cache = 1; - if ( touch (staging_path) ) { /* indicate that we'll be caching it */ - logprintfl (EUCAERROR, "error: failed to create staging file %s\n", staging_path); - action = ABORT; - } - } - } else { - logprintfl (EUCAERROR, "error: failed to obtain file size from digest %s\n", url); - action = ABORT; - } - } else { - logprintfl (EUCAERROR, "error: failed to obtain digest from %s\n", url); - action = ABORT; - } - } - sem_v (sc_sem); /***** release lock *****/ - - switch (action) { - case STAGE: - logprintfl (EUCAINFO, "downloading and preparing image into %s...\n", file_path); - if (strstr(url, "services/Walrus")) { - e = walrus_image_by_manifest_url (url, file_path, 1); - } else { - e = http_get_timeout(url, file_path, 10, 4, 0, 0); - } - - /* for KVM, convert partition into disk */ - if (e==OK && convert_to_disk) { - sem_p (s); - sem_p (disk_sem); - /* for the cached disk swap==0 and ephemeral==0 as we'll append them below */ - if ((e=vrun("%s %s %d %d", disk_convert_command_path, file_path, 0, 0))!=0) { - logprintfl (EUCAERROR, "error: partition-to-disk image conversion command failed\n"); - } - sem_v(disk_sem); - sem_v (s); - - /* recalculate file size now that it was converted */ - if ( stat (file_path, &mystat ) != 0 ) { - logprintfl (EUCAERROR, "error: file %s not found\n", file_path); - } else if (mystat.st_size < 1) { - logprintfl (EUCAERROR, "error: file %s has the size of 0\n", file_path); - } else { - file_size_b = (long long)mystat.st_size; - } - } - - sem_p (disk_sem); - /* cache the partition or disk, if possible */ - if ( e==OK && should_cache ) { - if ( (e=vrun ("cp -a %s %s", file_path, cached_path)) != 0) { - logprintfl (EUCAERROR, "failed to copy file %s into cache at %s\n", file_path, cached_path); - } - if ( e==OK && (e=vrun ("cp -a %s %s", tmp_digest_path, digest_path)) != 0) { - logprintfl (EUCAERROR, "failed to copy digest file %s into cache at %s\n", tmp_digest_path, digest_path); - } - } - sem_v (disk_sem); - - sem_p (sc_sem); - if (should_cache) { - unlink (staging_path); - } - if ( e ) { - logprintfl (EUCAERROR, "error: failed to download or prepare into %s\n", file_path); - unlink (file_path); - unlink (tmp_digest_path); - if (should_cache) { - unlink (cached_path); - unlink (digest_path); - if ( rmdir(cached_dir) ) { - logprintfl (EUCAWARN, "warning: failed to remove cache directory %s\n", cached_dir); - } - } - } - sem_v (sc_sem); - break; - - case WAIT: - logprintfl (EUCAINFO, "waiting for disappearance of %s...\n", staging_path); - /* wait for staging_path to disappear, which means both either the - * download succeeded or it failed */ - if ( (e=wait_for_file (NULL, staging_path, 180, "cached image")) ) - return 0L; - /* yes, it is OK to fall through */ - - case VERIFY: - logprintfl (EUCAINFO, "verifying cached file in %s...\n", cached_path); - sem_p (sc_sem); /***** acquire lock *****/ - e = ERROR; - if ( stat (cached_path, &mystat ) != 0 ) { - logprintfl (EUCAERROR, "error: file %s not found\n", cached_path); - } else if (mystat.st_size < 1) { - logprintfl (EUCAERROR, "error: file %s has the size of 0\n", cached_path); - } else if ((e=walrus_verify_digest (url, digest_path))<0) { - /* negative status => digest changed */ - unlink (cached_path); - unlink (staging_path); /* TODO: needed? */ - unlink (digest_path); - if ( rmdir (cached_dir) ) { - logprintfl (EUCAWARN, "warning: failed to remove cache directory %s\n", cached_dir); - } else { - logprintfl (EUCAINFO, "due to failure, removed cache directory %s\n", cached_dir); - } - } else { - file_size_b = mystat.st_size; - - /* touch the digest so cache can use mtime for invalidation */ - if ( touch (digest_path) ) { - logprintfl (EUCAERROR, "error: failed to touch digest file %s\n", digest_path); - } else if ( stat (digest_path, &mystat) ) { - logprintfl (EUCAERROR, "error: digest file %s not found\n", digest_path); - } else { - digest_size_b = (long long)mystat.st_size; - } - } - sem_v (sc_sem); /***** release lock *****/ - - if (e<0) { /* digest changed */ - if (action==VERIFY) { /* i.e. we did not download/waited for this file */ - /* try downloading anew */ - goto retry; - } else { - logprintfl (EUCAERROR, "error: digest mismatch, giving up\n"); - return 0L; - } - } else if (e>0) { /* problem with file or digest */ - return 0L; - - } else { /* all good - copy it, finally */ - ensure_subdirectory_exists (file_path); /* creates missing directories */ - sem_p (disk_sem); - if ( (e=vrun ("cp -a %s %s", cached_path, file_path)) != 0) { - logprintfl (EUCAERROR, "failed to copy file %s from cache at %s\n", file_path, cached_path); - sem_v (disk_sem); - return 0L; - } - sem_v (disk_sem); - } - break; - - case ABORT: - logprintfl (EUCAERROR, "get_cached_file() failed (errno=%d)\n", e); - e = ERROR; - } - - if (e==OK && file_size_b > 0 && convert_to_disk ) { // if all went well above - long long ephemeral_mb = limit_mb - swap_size_mb - (file_size_b+digest_size_b)/MEGABYTE; - if ( swap_size_mb>0L || ephemeral_mb>0L ) { - sem_p (s); - sem_p (disk_sem); - if ((e=vrun("%s %s %lld %lld", disk_convert_command_path, file_path, swap_size_mb, ephemeral_mb))!=0) { - logprintfl (EUCAERROR, "error: failed to add swap or ephemeral to the disk image\n"); - } - sem_v (disk_sem); - sem_v (s); - - /* recalculate file size (again!) now that it was converted */ - if ( stat (file_path, &mystat ) != 0 ) { - logprintfl (EUCAERROR, "error: file %s not found\n", file_path); - } else if (mystat.st_size < 1) { - logprintfl (EUCAERROR, "error: file %s has the size of 0\n", file_path); - } else { - file_size_b = (long long)mystat.st_size; - } - } - } - - if (e==OK && action!=ABORT) - return file_size_b + digest_size_b; - return 0L; -} - -// DEPRECATED -char * get_disk_path ( - const char * instanceId, - const char * userId) -{ - char file_path [MAX_PATH_SIZE]; - struct stat mystat; - - snprintf (file_path, MAX_PATH_SIZE, "%s/%s/%s/disk", sc_instance_path, userId, instanceId); - if (stat (file_path, &mystat)!=0) { - snprintf (file_path, MAX_PATH_SIZE, "%s/%s/%s/root", sc_instance_path, userId, instanceId); - if (stat (file_path, &mystat) !=0) { - logprintfl (EUCAERROR, "failed to stat disk %s\n", file_path); - return NULL; - } - } - return strdup (file_path); -} - -long long get_bundling_size ( - const char * instanceId, - const char * userId) -{ - char file_path [MAX_PATH_SIZE]; - struct stat mystat; - - snprintf (file_path, MAX_PATH_SIZE, "%s/%s/%s/disk", sc_instance_path, userId, instanceId); - if (stat (file_path, &mystat)!=0) { - snprintf (file_path, MAX_PATH_SIZE, "%s/%s/%s/root", sc_instance_path, userId, instanceId); - if (stat (file_path, &mystat) !=0) { - logprintfl (EUCAERROR, "failed to stat disk %s\n", file_path); - return -1L; - } - } - - return ((long long)mystat.st_size)*2L; // bundling requires twice the size of disk -} - -char * alloc_work_path ( - const char * instanceId, // IN: id of instance that needs work space - const char * userId, // IN: id of owner of the instance - const long long sizeMb) // IN: size needed under work path -{ - char file_path [MAX_PATH_SIZE]; - if (sizeMb < 0L) - return NULL; - - long long left = work_free_mb - sizeMb; - if (left>0) { - sem_p (sc_sem); - work_free_mb -= sizeMb; - sem_v (sc_sem); - if (snprintf (file_path, MAX_PATH_SIZE, "%s/%s/work/%s", sc_instance_path, EUCALYPTUS_ADMIN, instanceId)<1) { // work is in admin's directory - return NULL; - } - } else { - logprintfl (EUCAERROR, "work disk space limit exceeded (free=%lld size=%lld)\n", work_free_mb, sizeMb); - return NULL; - } - ensure_path_exists (file_path); - - return strdup (file_path); -} - -int free_work_path ( - const char * instanceId, // IN: id of instance giving up work space - const char * userId, // IN: id of owner of the instance - const long long sizeMb) // IN: size needed under work path -{ - if (sizeMb < 0L) - return ERROR; - - char workPath [MAX_PATH_SIZE]; - if (snprintf (workPath, MAX_PATH_SIZE, "%s/%s/work/%s", sc_instance_path, EUCALYPTUS_ADMIN, instanceId)<1) { // work is in admin's directory - return ERROR; - } - if (vrun ("rm -rf %s", workPath)) { - logprintfl (EUCAWARN, "warning: failed to clean work directory %s\n", workPath); - } else { - sem_p (sc_sem); - work_free_mb += sizeMb; - sem_v (sc_sem); - } - return OK; -} - -int scMakeInstanceImage (char *euca_home, char *userId, char *imageId, char *imageURL, char *kernelId, char *kernelURL, char *ramdiskId, char *ramdiskURL, char *instanceId, char *keyName, char *platform, char **instance_path, sem * s, int convert_to_disk, long long total_disk_limit_mb) -{ - char image_path [BUFSIZE]; long long image_size_b = 0L; - char kernel_path [BUFSIZE]; long long kernel_size_b = 0L; - char ramdisk_path [BUFSIZE]; long long ramdisk_size_b = 0L; - char config_path [BUFSIZE]; - char rundir_path [BUFSIZE]; - int e = ERROR; - - logprintfl (EUCAINFO, "retrieving images for instance %s (disk limit=%lldMB)...\n", instanceId, total_disk_limit_mb); - - /* get the necessary files from Walrus, caching them if possible */ - char * image_name; - int mount_offset = 0; - long long limit_mb = total_disk_limit_mb; - if (convert_to_disk) { - image_name = "disk"; - mount_offset = 32256; /* 1st partition offset in the disk image */ - } else { - image_name = "root"; - limit_mb -= swap_size_mb; /* account for swap, which will be a separate file */ - } - -#define CHECK_LIMIT(WHAT) \ - if (limit_mb < 1L) { \ - logprintfl (EUCAFATAL, "error: insufficient disk capacity remaining (%lldMB) in VM Type of instance %s for component %s\n", limit_mb, instanceId, WHAT); \ - return e; \ - } - CHECK_LIMIT("swap"); - - /* do kernel & ramdisk first, since either the disk or the ephemeral partition will take up the rest */ - if (kernelId && strnlen(kernelId, CHAR_BUFFER_SIZE) ) { - if ((kernel_size_b=get_cached_file (userId, kernelURL, kernelId, instanceId, "kernel", kernel_path, s, 0, limit_mb))<1L) return e; - limit_mb -= kernel_size_b/MEGABYTE; - CHECK_LIMIT("kernel"); - } - if (ramdiskId && strnlen (ramdiskId, CHAR_BUFFER_SIZE) ) { - if ((ramdisk_size_b=get_cached_file (userId, ramdiskURL, ramdiskId, instanceId, "ramdisk", ramdisk_path, s, 0, limit_mb))<1L) return e; - limit_mb -= ramdisk_size_b/MEGABYTE; - CHECK_LIMIT("ramdisk") - } - - if ((image_size_b=get_cached_file (userId, imageURL, imageId, instanceId, image_name, image_path, s, convert_to_disk, limit_mb))<1L) return e; - limit_mb -= image_size_b/MEGABYTE; - - snprintf (rundir_path, BUFSIZE, "%s/%s/%s", sc_instance_path, userId, instanceId); - - logprintfl (EUCAINFO, "preparing images for instance %s...\n", instanceId); - - if (strstr(platform, "windows")) { - e = makeWindowsFloppy(euca_home, rundir_path, keyName, instanceId); - if (e) { - logprintfl(EUCAERROR, "could not create windows bootup script floppy\n"); - } - } else { - - /* embed the key, which is contained in keyName */ - char *key_template = NULL; - if (keyName && strlen(keyName)) { - int key_len = strlen(keyName); - int fd = -1; - int ret; - - key_template = strdup("/tmp/sckey.XXXXXX"); - - if (((fd = safe_mkstemp(key_template)) < 0)) { - logprintfl (EUCAERROR, "failed to create a temporary key file\n"); - } else if ((ret = write (fd, keyName, key_len))0) { - sem_p (disk_sem); - if ((e=vrun ("dd bs=1M count=%lld if=/dev/zero of=%s/swap 2>/dev/null", swap_size_mb, rundir_path)) != 0) { - logprintfl (EUCAINFO, "creation of swap (dd) at %s/swap failed\n", rundir_path); - sem_v (disk_sem); - return e; - } - if ((e=vrun ("mkswap %s/swap >/dev/null", rundir_path)) != 0) { - logprintfl (EUCAINFO, "initialization of swap (mkswap) at %s/swap failed\n", rundir_path); - sem_v (disk_sem); - return e; - } - sem_v (disk_sem); - } - /* create ephemeral partition */ - if (limit_mb>0) { - sem_p (disk_sem); - if ((e=vrun ("dd bs=1M count=%lld if=/dev/zero of=%s/ephemeral 2>/dev/null", limit_mb, rundir_path )) != 0) { - logprintfl (EUCAINFO, "creation of ephemeral disk (dd) at %s/ephemeral failed\n", rundir_path); - sem_v (disk_sem); - return e; - } - if ((e=vrun ("mkfs.ext3 -F %s/ephemeral >/dev/null 2>&1", rundir_path)) != 0) { - logprintfl (EUCAINFO, "initialization of ephemeral disk (mkfs.ext3) at %s/ephemeral failed\n", rundir_path); - sem_v (disk_sem); - return e; - } - sem_v (disk_sem); - } - } - } - - * instance_path = strdup (rundir_path); - if (*instance_path==NULL) return errno; - return 0; -} - -int scStoreStringToInstanceFile (const char *userId, const char *instanceId, const char * file, const char * data) -{ - FILE * fp; - int ret = ERROR; - char path [BUFSIZE]; - snprintf (path, BUFSIZE, "%s/%s/%s/%s", sc_instance_path, userId, instanceId, file); - if ( (fp = fopen (path, "w")) != NULL ) { - if ( fputs (data, fp) != EOF ) { - ret = OK; - } - fclose (fp); - } - return ret; -} diff --git a/storage/storage.h b/storage/storage.h deleted file mode 100644 index 9d7c95786d0..00000000000 --- a/storage/storage.h +++ /dev/null @@ -1,94 +0,0 @@ -// -*- mode: C; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- -// vim: set softtabstop=4 shiftwidth=4 tabstop=4 expandtab: - -/* - Copyright (c) 2009 Eucalyptus Systems, Inc. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, only version 3 of the License. - - This file is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License along - with this program. If not, see . - - Please contact Eucalyptus Systems, Inc., 130 Castilian - Dr., Goleta, CA 93101 USA or visit - if you need additional information or have any questions. - - This file may incorporate work covered under the following copyright and - permission notice: - - Software License Agreement (BSD License) - - Copyright (c) 2008, Regents of the University of California - - - Redistribution and use of this software in source and binary forms, with - or without modification, are permitted provided that the following - conditions are met: - - Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER - OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. USERS OF - THIS SOFTWARE ACKNOWLEDGE THE POSSIBLE PRESENCE OF OTHER OPEN SOURCE - LICENSED MATERIAL, COPYRIGHTED MATERIAL OR PATENTED MATERIAL IN THIS - SOFTWARE, AND IF ANY SUCH MATERIAL IS DISCOVERED THE PARTY DISCOVERING - IT MAY INFORM DR. RICH WOLSKI AT THE UNIVERSITY OF CALIFORNIA, SANTA - BARBARA WHO WILL THEN ASCERTAIN THE MOST APPROPRIATE REMEDY, WHICH IN - THE REGENTS' DISCRETION MAY INCLUDE, WITHOUT LIMITATION, REPLACEMENT - OF THE CODE SO IDENTIFIED, LICENSING OF THE CODE SO IDENTIFIED, OR - WITHDRAWAL OF THE CODE CAPABILITY TO THE EXTENT NEEDED TO COMPLY WITH - ANY SUCH LICENSES OR RIGHTS. -*/ -#ifndef INCLUDE_STORAGE_H -#define INCLUDE_STORAGE_H - -#include "data.h" -#include "ipc.h" -#include -#include -#ifndef MAX_PATH -#define MAX_PATH 4096 -#endif - -int scInitConfig (void); -int scSetInstancePath(char *path); -const char * scGetInstancePath(void); -void scSaveInstanceInfo (const ncInstance * instance); -ncInstance * scRecoverInstanceInfo (const char *instanceId); -void LogprintfCache (void); -long long scFSCK (bunchOfInstances ** instances); -int scGetConfigXML(char *user, char *amiId, char **out); -int scMakeInstanceImage(char *euca_home, char *user, char *imageId, char *imageURL, char *kernelId, char *kernelURL, char *ramdiskId, char *ramdiskURL, char *instId, char *keyName, char *platform, char **instance_path, sem * s, int convert_to_disk, long long total_disk_limit_mb); -int scCleanupInstanceImage(char *user, char *instId); -int scStoreStringToInstanceFile (const char *userId, const char *instanceId, const char * file, const char * data); -char * get_disk_path (const char * instanceId, const char * userId); -long long get_bundling_size (const char * instanceId, const char * userId); -char * alloc_work_path (const char * instanceId, const char * userId, const long long sizeMb); -int free_work_path (const char * instanceId, const char * userId, const long long sizeMb); - -/* utility function (exported for now so it can be tested by test.c) */ -int ensure_path_exists (const char * path); -int test_cache (void); - -#endif diff --git a/storage/vbr.c b/storage/vbr.c index 2a127bf806d..a0226e62579 100644 --- a/storage/vbr.c +++ b/storage/vbr.c @@ -1851,7 +1851,7 @@ static blobstore * create_teststore (int size_blocks, const char * base, const c return NULL; } printf ("created %s\n", bs_path); - blobstore * bs = blobstore_open (bs_path, size_blocks, format, revocation, snapshot); + blobstore * bs = blobstore_open (bs_path, size_blocks, BLOBSTORE_FLAG_CREAT, format, revocation, snapshot); if (bs==NULL) { printf ("ERROR: %s\n", blobstore_get_error_str(blobstore_get_error())); return NULL; diff --git a/util/eucalyptus.h b/util/eucalyptus.h index f71c286502c..fe4c03c09ad 100644 --- a/util/eucalyptus.h +++ b/util/eucalyptus.h @@ -134,13 +134,11 @@ permission notice: #define EUCA_MAX_PATH 4096 #define EUCA_MAX_PARTITIONS 32 // partitions per disk #define EUCA_MAX_DISKS 26 // disks per bus: sd[a-z] -#define DEFAULT_NC_CACHE_SIZE 999999 // in MB -#define DEFAULT_NC_WORK_SIZE 999999 // in MB -#define DEFAULT_SWAP_SIZE 512 /* in MB */ #define MAX_PATH_SIZE 4096 // TODO: remove // NC hook events -#define NC_EVENT_INIT "euca-nc-post-init" // p1: NULL +#define NC_EVENT_PRE_INIT "euca-nc-pre-init" // p1: eucalyptusHome +#define NC_EVENT_POST_INIT "euca-nc-post-init" // p1: eucalyptusHome #define NC_EVENT_PRE_BOOT "euca-nc-pre-boot" // p1: instancePath #define NC_EVENT_ADOPTING "euca-nc-pre-adopt" // p1: instancePath #define NC_EVENT_PRE_CLEAN "euca-nc-pre-clean" // p1: instancePath From 889542a6d266ff8f607e2454b07c4a5cb48ed12b Mon Sep 17 00:00:00 2001 From: root Date: Fri, 2 Dec 2011 21:47:03 -0800 Subject: [PATCH 2/7] .blobstore rewriting bug fix; more thorough disk stats; corrected allocated blocks calculation; code style --- node/handlers.c | 65 ++++++++++++++++++++++++++++++++------------- node/handlers_kvm.c | 15 ++--------- node/handlers_xen.c | 19 +++++-------- storage/backing.c | 7 ++--- storage/blobstore.c | 39 ++++++++++++++++++--------- storage/blobstore.h | 6 +++-- 6 files changed, 90 insertions(+), 61 deletions(-) diff --git a/node/handlers.c b/node/handlers.c index 3bb72912055..71e0f74b61f 100644 --- a/node/handlers.c +++ b/node/handlers.c @@ -978,14 +978,16 @@ static int init (void) // determine how much is used/available in work and cache areas on the backing store blobstore_meta work_meta, cache_meta; stat_backing_store (instance_path, &work_meta, &cache_meta); // will zero-out work_ and cache_meta - long long work_bs_size_mb = work_meta.blocks_limit ? (work_meta.blocks_limit / 2048) : -1; // convert sectors->MB - long long work_bs_used_mb = work_meta.blocks_limit ? ((work_meta.blocks_allocated + work_meta.blocks_used) / 2048) : 0; - long long cache_bs_size_mb = cache_meta.blocks_limit ? (cache_meta.blocks_limit / 2048) : -1; - long long cache_bs_used_mb = cache_meta.blocks_limit ? ((cache_meta.blocks_allocated + cache_meta.blocks_used) / 2048) : 0; + long long work_bs_size_mb = work_meta.blocks_limit ? (work_meta.blocks_limit / 2048) : (-1L); // convert sectors->MB + long long work_bs_allocated_mb = work_meta.blocks_limit ? (work_meta.blocks_allocated / 2048) : 0; + long long work_bs_reserved_mb = work_meta.blocks_limit ? ((work_meta.blocks_locked + work_meta.blocks_unlocked) / 2048) : 0; + long long cache_bs_size_mb = cache_meta.blocks_limit ? (cache_meta.blocks_limit / 2048) : (-1L); + long long cache_bs_allocated_mb = cache_meta.blocks_limit ? (cache_meta.blocks_allocated / 2048) : 0; + long long cache_bs_reserved_mb = cache_meta.blocks_limit ? ((cache_meta.blocks_locked + cache_meta.blocks_unlocked) / 2048) : 0; // look up configuration file settings for work and cache size - int conf_work_size_mb = -1; GET_VAR_INT(conf_work_size_mb, CONFIG_NC_WORK_SIZE); - int conf_cache_size_mb = -1; GET_VAR_INT(conf_cache_size_mb, CONFIG_NC_CACHE_SIZE); + long long conf_work_size_mb = -1; GET_VAR_INT(conf_work_size_mb, CONFIG_NC_WORK_SIZE); + long long conf_cache_size_mb = -1; GET_VAR_INT(conf_cache_size_mb, CONFIG_NC_CACHE_SIZE); { // accommodate legacy MAX_DISK setting by converting it int max_disk_gb = -1; GET_VAR_INT(max_disk_gb, CONFIG_MAX_DISK); if (max_disk_gb != -1) { @@ -999,8 +1001,8 @@ static int init (void) } // decide what work and cache sizes should be, based on all the inputs - int work_size_mb = -1; - int cache_size_mb = -1; + long long work_size_mb = -1; + long long cache_size_mb = -1; // above all, try to respect user-specified limits for work and cache if (conf_work_size_mb != -1) { @@ -1009,7 +1011,7 @@ static int init (void) CONFIG_NC_WORK_SIZE, conf_work_size_mb, MIN_BLOBSTORE_SIZE_MB); } else { if (work_bs_size_mb != -1 && work_bs_size_mb != conf_work_size_mb) { - logprintfl (EUCAWARN, "warning: specified work size (%s=%d) differs from existing work limits (%d), will try resizing\n", + logprintfl (EUCAWARN, "warning: specified work size (%s=%d) differs from existing work size (%d), will try resizing\n", CONFIG_NC_WORK_SIZE, conf_work_size_mb, work_bs_size_mb); } work_size_mb = conf_work_size_mb; @@ -1020,7 +1022,7 @@ static int init (void) cache_size_mb = 0; // so it won't be used } else { if (cache_bs_size_mb != -1 && cache_bs_size_mb != conf_cache_size_mb) { - logprintfl (EUCAWARN, "warning: specified cache size (%s=%d) differs from existing cache limits (%d), will try resizing\n", + logprintfl (EUCAWARN, "warning: specified cache size (%s=%d) differs from existing cache size (%d), will try resizing\n", CONFIG_NC_CACHE_SIZE, conf_cache_size_mb, cache_bs_size_mb); } cache_size_mb = conf_cache_size_mb; @@ -1040,20 +1042,21 @@ static int init (void) work_size_mb = (long long)((double)(fs_avail_mb - FS_BUFFER_MB) * WORK_BS_PERCENT); cache_size_mb = fs_avail_mb - FS_BUFFER_MB - work_size_mb; } else if (work_size_mb == -1) { - work_size_mb = fs_avail_mb - FS_BUFFER_MB - cache_size_mb + cache_bs_used_mb; + work_size_mb = fs_avail_mb - FS_BUFFER_MB - cache_size_mb + cache_bs_allocated_mb; } else if (cache_size_mb == -1) { - cache_size_mb = fs_avail_mb - FS_BUFFER_MB - work_size_mb + work_bs_used_mb; + cache_size_mb = fs_avail_mb - FS_BUFFER_MB - work_size_mb + work_bs_allocated_mb; } // sanity-check final results if (cache_size_mb < MIN_BLOBSTORE_SIZE_MB) cache_size_mb = 0; if (work_size_mb < MIN_BLOBSTORE_SIZE_MB) { - logprintfl (EUCAERROR, "error: insufficient disk space (%d) for virtual machines\n", work_size_mb); + logprintfl (EUCAERROR, "error: insufficient disk space for virtual machines (free space: %dMB, reserved for cache: %dMB)\n", + work_size_mb, (fs_avail_mb - FS_BUFFER_MB), cache_size_mb); free (instance_path); return ERROR_FATAL; } - if ((cache_size_mb + work_size_mb - cache_bs_used_mb - work_bs_used_mb) > fs_avail_mb) { + if ((cache_size_mb + work_size_mb - cache_bs_allocated_mb - work_bs_allocated_mb) > fs_avail_mb) { logprintfl (EUCAWARN, "warning: sum of work and cache sizes exceeds available disk space\n"); } @@ -1067,12 +1070,28 @@ static int init (void) nc_state.disk_max = (long long)(work_size_mb / MB_PER_DISK_UNIT); logprintfl (EUCAINFO, "disk space for instances: %s/work\n", instance_path); - logprintfl (EUCAINFO, " %06lldMB allocated (%03.1f% of the file system)\n", work_size_mb, (work_size_mb/fs_size_mb)*100.0 ); - logprintfl (EUCAINFO, " %06lldMB in use (%03.1% of allocated)\n", work_bs_used_mb, (work_bs_used_mb/work_size_mb)*100.0 ); + logprintfl (EUCAINFO, " %06lldMB limit (%.1f%% of the file system)\n", + work_size_mb, + ((double)work_size_mb/(double)fs_size_mb)*100.0 ); + logprintfl (EUCAINFO, " %06lldMB reserved for use (%.1f%% of limit)\n", + work_bs_reserved_mb, + ((double)work_bs_reserved_mb/(double)work_size_mb)*100.0 ); + logprintfl (EUCAINFO, " %06lldMB allocated for use (%.1f%% of limit, %.1f%% of file system)\n", + work_bs_allocated_mb, + ((double)work_bs_allocated_mb/(double)work_size_mb)*100.0, + ((double)work_bs_allocated_mb/(double)fs_size_mb)*100.0 ); if (cache_size_mb) { logprintfl (EUCAINFO, " disk space for cache: %s/cache\n", instance_path); - logprintfl (EUCAINFO, " %06lldMB allocated (%03.1f% of the file system)\n", cache_size_mb, (cache_size_mb/fs_size_mb)*100.0 ); - logprintfl (EUCAINFO, " %06lldMB in use (%03.1% of allocated)\n", cache_bs_used_mb, (cache_bs_used_mb/cache_size_mb)*100.0 ); + logprintfl (EUCAINFO, " %06lldMB limit (%.1f%% of the file system)\n", + cache_size_mb, + ((double)cache_size_mb/(double)fs_size_mb)*100.0 ); + logprintfl (EUCAINFO, " %06lldMB reserved for use (%.1f%% of limit)\n", + cache_bs_reserved_mb, + ((double)cache_bs_reserved_mb/(double)cache_size_mb)*100.0 ); + logprintfl (EUCAINFO, " %06lldMB allocated for use (%.1f%% of limit, %.1f%% of file system)\n", + cache_bs_allocated_mb, + ((double)cache_bs_allocated_mb/(double)cache_size_mb)*100.0, + ((double)cache_bs_allocated_mb/(double)fs_size_mb)*100.0 ); } else { logprintfl (EUCAWARN, "warning: disk cache will not be used\n"); } @@ -1119,6 +1138,16 @@ static int init (void) return ERROR_FATAL; } + // now that hypervisor-specific initializers have discovered mem_max and cores_max, + // adjust the values based on configuration parameters, if any + if (nc_state.config_max_mem && nc_state.config_max_mem < nc_state.mem_max) + nc_state.mem_max = nc_state.config_max_mem; + if (nc_state.config_max_cores) + nc_state.cores_max = nc_state.config_max_cores; + logprintfl(EUCAINFO, "physical memory available for instances: %lldMB\n", nc_state.mem_max); + logprintfl(EUCAINFO, "virtual cpu cores available for instances: %lld\n", nc_state.cores_max); + + // adopt running instances adopt_instances(); diff --git a/node/handlers_kvm.c b/node/handlers_kvm.c index 6df880c1520..6a2ae6b4fc6 100644 --- a/node/handlers_kvm.c +++ b/node/handlers_kvm.c @@ -97,9 +97,7 @@ static int doInitialize (struct nc_state_t *nc) { char *s = NULL; - logprintfl(EUCADEBUG, "doInitialized() invoked\n"); - - /* set up paths of Eucalyptus commands NC relies on */ + // set up paths of Eucalyptus commands NC relies on snprintf (nc->get_info_cmd_path, MAX_PATH, EUCALYPTUS_GET_KVM_INFO, nc->home, nc->home); strcpy(nc->uri, HYPERVISOR_URI); nc->convert_to_disk = 1; @@ -117,18 +115,9 @@ static int doInitialize (struct nc_state_t *nc) GET_VALUE("total_memory", nc->mem_max); if (s) free(s); - /* we leave 256M to the host */ + // we leave 256M to the host nc->mem_max -= 256; - /* let's adjust the values based on the config values */ - if (nc->config_max_mem && nc->config_max_mem < nc->mem_max) - nc->mem_max = nc->config_max_mem; - if (nc->config_max_cores) - nc->cores_max = nc->config_max_cores; - - logprintfl(EUCAINFO, "Using %lld cores\n", nc->cores_max); - logprintfl(EUCAINFO, "Using %lld memory\n", nc->mem_max); - return OK; } diff --git a/node/handlers_xen.c b/node/handlers_xen.c index d7c765061b6..75f479c165d 100644 --- a/node/handlers_xen.c +++ b/node/handlers_xen.c @@ -95,9 +95,7 @@ static int doInitialize (struct nc_state_t *nc) virNodeInfo ni; long long dom0_min_mem; - logprintfl(EUCADEBUG, "doInitialized() invoked\n"); - - /* set up paths of Eucalyptus commands NC relies on */ + // set up paths of Eucalyptus commands NC relies on snprintf (nc->get_info_cmd_path, MAX_PATH, EUCALYPTUS_GET_XEN_INFO, nc->home, nc->home); snprintf (nc->virsh_cmd_path, MAX_PATH, EUCALYPTUS_VIRSH, nc->home); snprintf (nc->xm_cmd_path, MAX_PATH, EUCALYPTUS_XM); @@ -106,18 +104,18 @@ static int doInitialize (struct nc_state_t *nc) nc->convert_to_disk = 0; nc->capability = HYPERVISOR_XEN_AND_HARDWARE; // TODO: set to XEN_PARAVIRTUALIZED if on older Xen kernel - /* check connection is fresh */ + // check connection is fresh if (!check_hypervisor_conn()) { return ERROR_FATAL; } - /* get resources */ + // get resources if (virNodeGetInfo(nc->conn, &ni)) { logprintfl (EUCAFATAL, "error: failed to discover resources\n"); return ERROR_FATAL; } - /* dom0-min-mem has to come from xend config file */ + // dom0-min-mem has to come from xend config file s = system_output (nc->get_info_cmd_path); if (get_value (s, "dom0-min-mem", &dom0_min_mem)) { logprintfl (EUCAFATAL, "error: did not find dom0-min-mem in output from %s\n", nc->get_info_cmd_path); @@ -126,21 +124,18 @@ static int doInitialize (struct nc_state_t *nc) } free (s); - /* calculate the available memory */ + // calculate the available memory nc->mem_max = ni.memory/1024 - 32 - dom0_min_mem; - /* calculate the available cores */ + // calculate the available cores nc->cores_max = ni.cpus; - /* let's adjust the values based on the config values */ + // let's adjust the values based on the config values if (nc->config_max_mem && nc->config_max_mem < nc->mem_max) nc->mem_max = nc->config_max_mem; if (nc->config_max_cores) nc->cores_max = nc->config_max_cores; - logprintfl(EUCAINFO, "Using %lld cores\n", nc->cores_max); - logprintfl(EUCAINFO, "Using %lld memory\n", nc->mem_max); - return OK; } diff --git a/storage/backing.c b/storage/backing.c index ae74e9b79f7..e531b6ebcdb 100644 --- a/storage/backing.c +++ b/storage/backing.c @@ -102,7 +102,7 @@ static void bs_errors (const char * msg) { static void stat_blobstore (const char * conf_instances_path, const char * name, blobstore_meta * meta) { - bzero (meta, sizeof (meta)); + bzero (meta, sizeof (blobstore_meta)); char path [MAX_PATH]; snprintf (path, sizeof (path), "%s/%s", conf_instances_path, name); blobstore * bs = blobstore_open (path, @@ -151,7 +151,7 @@ int init_backing_store (const char * conf_instances_path, unsigned int conf_work if (cache_limit_blocks) { cache_bs = blobstore_open (cache_path, cache_limit_blocks, BLOBSTORE_FLAG_CREAT, BLOBSTORE_FORMAT_DIRECTORY, BLOBSTORE_REVOCATION_LRU, BLOBSTORE_SNAPSHOT_ANY); if (cache_bs==NULL) { - logprintfl (EUCAERROR, "ERROR: %s\n", blobstore_get_error_str(blobstore_get_error())); + logprintfl (EUCAERROR, "ERROR: failed to open/create cache blobstore: %s\n", blobstore_get_error_str(blobstore_get_error())); return ERROR; } if (blobstore_fsck (cache_bs, NULL)) { // TODO: verify checksums? @@ -161,7 +161,8 @@ int init_backing_store (const char * conf_instances_path, unsigned int conf_work } work_bs = blobstore_open (work_path, work_limit_blocks, BLOBSTORE_FLAG_CREAT, BLOBSTORE_FORMAT_FILES, BLOBSTORE_REVOCATION_NONE, BLOBSTORE_SNAPSHOT_ANY); if (work_bs==NULL) { - logprintfl (EUCAERROR, "ERROR: %s\n", blobstore_get_error_str(blobstore_get_error())); + logprintfl (EUCAERROR, "ERROR: failed to open/create work blobstore: %s\n", blobstore_get_error_str(blobstore_get_error())); + logprintfl (EUCAERROR, "ERROR: %s\n", blobstore_get_last_trace()); blobstore_close (cache_bs); return ERROR; } diff --git a/storage/blobstore.c b/storage/blobstore.c index 87c5a48a153..c32e20026ad 100644 --- a/storage/blobstore.c +++ b/storage/blobstore.c @@ -736,12 +736,11 @@ static int read_store_metadata (blobstore * bs) if (size == -1) return -1; - if (size<30) { ERR (BLOBSTORE_ERROR_NOENT, "metadata size is too small"); return -1; } - + char * val; if ((val = get_val (buf, "id"))==NULL) return -1; @@ -766,6 +765,8 @@ static int write_store_metadata (blobstore * bs) { if (ftruncate (bs->fd, 0)==-1) { ERR (BLOBSTORE_ERROR_NOENT, "failed to truncate the metadata file"); return -1; } + if (lseek (bs->fd, 0, SEEK_SET)==-1) + { ERR (BLOBSTORE_ERROR_ACCES, "failed to seek in metadata file"); return -1; } char buf [1024]; snprintf (buf, sizeof (buf), "id: %s\n" \ @@ -778,8 +779,9 @@ static int write_store_metadata (blobstore * bs) bs->revocation_policy, bs->snapshot_policy, bs->format); - int len = write (bs->fd, buf, strlen (buf)); - if (len != strlen (buf)) + int slen = strlen (buf); + int len = write (bs->fd, buf, slen); + if (len != slen) { ERR (BLOBSTORE_ERROR_NOENT, "failed to write to the metadata file"); return -1; } return 0; @@ -833,6 +835,8 @@ blobstore * blobstore_open ( const char * path, blobstore_revocation_t revocation_policy, blobstore_snapshot_t snapshot_policy) { + int saved_errno; + if (blobstore_init()) return NULL; @@ -906,7 +910,9 @@ blobstore * blobstore_open ( const char * path, ERR (BLOBSTORE_ERROR_INVAL, "'limit_blocks' does not match existing blobstore"); goto free; } else { + logprintfl (EUCAINFO, "adjusting blobstore limit from %d to %d\n", bs->limit_blocks, limit_blocks); write_flags = BLOBSTORE_FLAG_RDWR; + close_and_unlock (bs->fd); goto write_metadata; } } @@ -927,6 +933,7 @@ blobstore * blobstore_open ( const char * path, goto free; } else { write_flags = BLOBSTORE_FLAG_RDWR; + close_and_unlock (bs->fd); goto write_metadata; } } @@ -936,11 +943,14 @@ blobstore * blobstore_open ( const char * path, goto out; free: + saved_errno = _blobstore_errno; close_and_unlock (bs->fd); if (bs) { free (bs); bs = NULL; } + _blobstore_errno = saved_errno; + out: return bs; } @@ -1445,6 +1455,7 @@ static blockblob ** walk_bs (blobstore * bs, const char * dir_path, blockblob ** safe_strncpy (bb->blocks_path, entry_path, sizeof(bb->blocks_path)); set_device_path (bb); // read .dm and .loopback and set bb->device_path accordingly bb->size_bytes = sb.st_size; + bb->blocks_allocated = sb.st_blocks; bb->last_accessed = sb.st_atime; bb->last_modified = sb.st_mtime; bb->snapshot_type = BLOBSTORE_FORMAT_ANY; // it is not necessary to know whether this is a snapshot @@ -1534,15 +1545,17 @@ int blobstore_stat (blobstore * bs, blobstore_meta * meta) // analyze the LL, calculating sizes meta->blocks_allocated = 0; - meta->blocks_used = 0; + meta->blocks_unlocked = 0; + meta->blocks_locked = 0; meta->num_blobs = 0; for (blockblob * abb = bbs; abb; abb=abb->next) { long long abb_size_blocks = round_up_sec (abb->size_bytes) / 512; if (abb->in_use & ~BLOCKBLOB_STATUS_BACKED) { - meta->blocks_used += abb_size_blocks; // these can't be purged if we need space (TODO: look into recursive purging of unused references?) + meta->blocks_locked += abb_size_blocks; // these can't be purged if we need space (TODO: look into recursive purging of unused references?) } else { - meta->blocks_allocated += abb_size_blocks; // these can be purged + meta->blocks_unlocked += abb_size_blocks; // these can be purged } + meta->blocks_allocated += abb->blocks_allocated; meta->num_blobs++; } @@ -1866,24 +1879,24 @@ blockblob * blockblob_open ( blobstore * bs, } // analyze the LL, calculating sizes - long long blocks_allocated = 0; - long long blocks_used = 0; + long long blocks_unlocked = 0; + long long blocks_locked = 0; unsigned int num_blobs = 0; for (blockblob * abb = bbs; abb; abb=abb->next) { long long abb_size_blocks = round_up_sec (abb->size_bytes) / 512; if (abb->in_use & ~BLOCKBLOB_STATUS_BACKED) { - blocks_used += abb_size_blocks; // these can't be purged if we need space (TODO: look into recursive purging of unused references?) + blocks_locked += abb_size_blocks; // these can't be purged if we need space (TODO: look into recursive purging of unused references?) } else { - blocks_allocated += abb_size_blocks; // these can be purged + blocks_unlocked += abb_size_blocks; // these can be purged } num_blobs++; } - long long blocks_free = bs->limit_blocks - (blocks_allocated + blocks_used); + long long blocks_free = bs->limit_blocks - (blocks_unlocked + blocks_locked); if (blocks_free < size_blocks) { if (!(bs->revocation_policy==BLOBSTORE_REVOCATION_LRU) // not allowed to purge || - (blocks_free+blocks_allocated) < size_blocks) { // not enough purgeable material + (blocks_free+blocks_unlocked) < size_blocks) { // not enough purgeable material ERR (BLOBSTORE_ERROR_NOSPC, NULL); goto clean; } diff --git a/storage/blobstore.h b/storage/blobstore.h index 6940beb1a21..4b4d44cf72d 100644 --- a/storage/blobstore.h +++ b/storage/blobstore.h @@ -166,6 +166,7 @@ typedef struct _blockblob { char device_path [BLOBSTORE_MAX_PATH]; // full path of a block device on which blob can be accessed char dm_name [MAX_DM_NAME]; // name of the main device mapper device if this is a clone unsigned long long size_bytes; // size of the blob in bytes + unsigned long long blocks_allocated; // actual number of blocks on disk taken by the blob blobstore_snapshot_t snapshot_type; // ANY = not initialized/known, NONE = not a snapshot, DM = DM-based snapshot unsigned int in_use; // flags showing how the blockblob is being used (OPENED, LOCKED, LINKED) time_t last_accessed; // timestamp of last access @@ -205,8 +206,9 @@ typedef struct _blockblob_meta { typedef struct _blobstore_meta { char id [BLOBSTORE_MAX_PATH]; // ID of the blobstore, to handle directory moving unsigned long long blocks_limit; // max size of the blobstore, in blocks - unsigned long long blocks_allocated; // number of blocks in blobstore allocated to a blob that is not in use and is not mapped - unsigned long long blocks_used; // number of blocks in blobstore allocated to a blob that is in use or is mapped (a dependency) + unsigned long long blocks_unlocked; // number of blocks in blobstore allocated to blobs that are not in use and is not mapped + unsigned long long blocks_locked; // number of blocks in blobstore allocated to blobs that are in use or is mapped (a dependency) + unsigned long long blocks_allocated; // number of blocks in blobstore that have been allocated on disk unsigned int num_blobs; // count of blobs in the blobstore blobstore_revocation_t revocation_policy; blobstore_snapshot_t snapshot_policy; From a34b5f9d23f3e24d602f719178447df6ee0c1a04 Mon Sep 17 00:00:00 2001 From: Dmitrii Zagorodnov Date: Sat, 3 Dec 2011 10:39:35 -0800 Subject: [PATCH 3/7] added CONFIG_CONCURRENT_DISK_OPS support back --- node/handlers.c | 29 ++++++++++++++++------------- node/handlers.h | 1 + storage/backing.c | 25 ++++++++++++++++++++++--- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/node/handlers.c b/node/handlers.c index cb8adb9f45f..d1b47a6f0ec 100644 --- a/node/handlers.c +++ b/node/handlers.c @@ -904,17 +904,19 @@ static int init (void) } } -#define GET_VAR_INT(var,name) \ +#define GET_VAR_INT(var,name,def) \ s = getConfString(configFiles, 2, name); \ if (s){ \ var = atoi(s);\ free (s);\ - } - GET_VAR_INT(nc_state.config_max_mem, CONFIG_MAX_MEM); - GET_VAR_INT(nc_state.config_max_cores, CONFIG_MAX_CORES); - GET_VAR_INT(nc_state.save_instance_files, CONFIG_SAVE_INSTANCES); - int disable_injection = 0; - GET_VAR_INT(disable_injection, CONFIG_DISABLE_KEY_INJECTION); + } else { \ + var = def; \ + } + GET_VAR_INT(nc_state.config_max_mem, CONFIG_MAX_MEM, 0); + GET_VAR_INT(nc_state.config_max_cores, CONFIG_MAX_CORES, 0); + GET_VAR_INT(nc_state.save_instance_files, CONFIG_SAVE_INSTANCES, 0); + GET_VAR_INT(nc_state.concurrent_disk_ops, CONFIG_CONCURRENT_DISK_OPS, 1); + int disable_injection; GET_VAR_INT(disable_injection, CONFIG_DISABLE_KEY_INJECTION, 0); nc_state.do_inject_key = !disable_injection; strcpy(nc_state.admin_user_id, EUCALYPTUS_ADMIN); @@ -986,10 +988,10 @@ static int init (void) long long cache_bs_reserved_mb = cache_meta.blocks_limit ? ((cache_meta.blocks_locked + cache_meta.blocks_unlocked) / 2048) : 0; // look up configuration file settings for work and cache size - long long conf_work_size_mb = -1; GET_VAR_INT(conf_work_size_mb, CONFIG_NC_WORK_SIZE); - long long conf_cache_size_mb = -1; GET_VAR_INT(conf_cache_size_mb, CONFIG_NC_CACHE_SIZE); + long long conf_work_size_mb; GET_VAR_INT(conf_work_size_mb, CONFIG_NC_WORK_SIZE, -1); + long long conf_cache_size_mb; GET_VAR_INT(conf_cache_size_mb, CONFIG_NC_CACHE_SIZE, -1); { // accommodate legacy MAX_DISK setting by converting it - int max_disk_gb = -1; GET_VAR_INT(max_disk_gb, CONFIG_MAX_DISK); + int max_disk_gb; GET_VAR_INT(max_disk_gb, CONFIG_MAX_DISK, -1); if (max_disk_gb != -1) { if (conf_work_size_mb == -1) { logprintfl (EUCAWARN, "warning: using deprecated setting %s for the new setting %s\n", CONFIG_MAX_DISK, CONFIG_NC_WORK_SIZE); @@ -1121,9 +1123,9 @@ static int init (void) // only load virtio config for kvm if (!strncmp("kvm", hypervisor, CHAR_BUFFER_SIZE) || !strncmp("KVM", hypervisor, CHAR_BUFFER_SIZE)) { - GET_VAR_INT(nc_state.config_use_virtio_net, CONFIG_USE_VIRTIO_NET); - GET_VAR_INT(nc_state.config_use_virtio_disk, CONFIG_USE_VIRTIO_DISK); - GET_VAR_INT(nc_state.config_use_virtio_root, CONFIG_USE_VIRTIO_ROOT); + GET_VAR_INT(nc_state.config_use_virtio_net, CONFIG_USE_VIRTIO_NET, 0); + GET_VAR_INT(nc_state.config_use_virtio_disk, CONFIG_USE_VIRTIO_DISK, 0); + GET_VAR_INT(nc_state.config_use_virtio_root, CONFIG_USE_VIRTIO_ROOT, 0); } free (hypervisor); @@ -1147,6 +1149,7 @@ static int init (void) logprintfl(EUCAINFO, "physical memory available for instances: %lldMB\n", nc_state.mem_max); logprintfl(EUCAINFO, "virtual cpu cores available for instances: %lld\n", nc_state.cores_max); + // adopt running instances adopt_instances(); diff --git a/node/handlers.h b/node/handlers.h index 033727b7fde..6682c36fb1a 100644 --- a/node/handlers.h +++ b/node/handlers.h @@ -87,6 +87,7 @@ struct nc_state_t { virConnectPtr conn; boolean convert_to_disk; boolean do_inject_key; + int concurrent_disk_ops; // defined max long long config_max_mem; long long config_max_cores; diff --git a/storage/backing.c b/storage/backing.c index e531b6ebcdb..a57686cdceb 100644 --- a/storage/backing.c +++ b/storage/backing.c @@ -84,6 +84,7 @@ #include "backing.h" #include "iscsi.h" #include "vbr.h" +#include "ipc.h" // sem #define CACHE_TIMEOUT_USEC 1000000LL*60*60*2 #define STORE_TIMEOUT_USEC 1000000LL*60*2 @@ -93,6 +94,8 @@ static char instances_path [MAX_PATH]; static blobstore * cache_bs = NULL; static blobstore * work_bs; +static sem * disk_sem = NULL; + extern struct nc_state_t nc_state; static void bs_errors (const char * msg) { @@ -172,6 +175,13 @@ int init_backing_store (const char * conf_instances_path, unsigned int conf_work return ERROR; } + // set the initial value of the semaphore to the number of + // disk-intensive operations that can run in parallel on this node + if (nc_state.concurrent_disk_ops && (disk_sem = sem_alloc (nc_state.concurrent_disk_ops, "mutex")) == NULL) { + logprintfl (EUCAERROR, "failed to create and initialize disk semaphore\n"); + return ERROR; + } + return OK; } @@ -347,12 +357,21 @@ int create_instance_backing (ncInstance * instance) TRUE, // make working copy of runtime-modifiable files (instance->do_inject_key)?(instance->keyName):(NULL), // the SSH key instance->instanceId); // ID is for logging - if (sentinel == NULL || - art_implement_tree (sentinel, work_bs, cache_bs, work_prefix, INSTANCE_PREP_TIMEOUT_USEC) != OK) { // download/create/combine the dependencies + if (sentinel == NULL) { logprintfl (EUCAERROR, "[%s] error: failed to prepare backing for instance\n", instance->instanceId); goto out; } - + + sem_p (disk_sem); + // download/create/combine the dependencies + int rc = art_implement_tree (sentinel, work_bs, cache_bs, work_prefix, INSTANCE_PREP_TIMEOUT_USEC); + sem_v (disk_sem); + + if (rc != OK) { + logprintfl (EUCAERROR, "[%s] error: failed to implement backing for instance\n", instance->instanceId); + goto out; + } + if (save_instance_struct (instance)) // update instance checkpoint now that the struct got updated goto out; From 73da38baa07a3c9e4271395c9719848418a778fb Mon Sep 17 00:00:00 2001 From: Dmitrii Zagorodnov Date: Sat, 3 Dec 2011 10:42:22 -0800 Subject: [PATCH 4/7] bumped up default for CONFIG_CONCURRENT_DISK_OPS to 4 --- node/handlers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/handlers.c b/node/handlers.c index d1b47a6f0ec..58f9f54ff19 100644 --- a/node/handlers.c +++ b/node/handlers.c @@ -915,7 +915,7 @@ static int init (void) GET_VAR_INT(nc_state.config_max_mem, CONFIG_MAX_MEM, 0); GET_VAR_INT(nc_state.config_max_cores, CONFIG_MAX_CORES, 0); GET_VAR_INT(nc_state.save_instance_files, CONFIG_SAVE_INSTANCES, 0); - GET_VAR_INT(nc_state.concurrent_disk_ops, CONFIG_CONCURRENT_DISK_OPS, 1); + GET_VAR_INT(nc_state.concurrent_disk_ops, CONFIG_CONCURRENT_DISK_OPS, 4); int disable_injection; GET_VAR_INT(disable_injection, CONFIG_DISABLE_KEY_INJECTION, 0); nc_state.do_inject_key = !disable_injection; strcpy(nc_state.admin_user_id, EUCALYPTUS_ADMIN); From 3c8e4c8a9331673177c2fd3f5ad88b992d7c8df5 Mon Sep 17 00:00:00 2001 From: Dmitrii Zagorodnov Date: Sat, 3 Dec 2011 13:18:03 -0800 Subject: [PATCH 5/7] preserving errno in blockblob_clone cleanup code --- storage/blobstore.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/storage/blobstore.c b/storage/blobstore.c index c32e20026ad..48e29dd9988 100644 --- a/storage/blobstore.c +++ b/storage/blobstore.c @@ -2388,7 +2388,7 @@ int blockblob_delete ( blockblob * bb, long long timeout_usec ) int saved_errno = 0; unlock: - saved_errno = _blobstore_errno; // save it because + saved_errno = _blobstore_errno; // save it because blobstore_unlock may overwrite it if (blobstore_unlock (bs)==-1) { ERR (BLOBSTORE_ERROR_UNKNOWN, "failed to unlock the blobstore"); } @@ -2745,7 +2745,12 @@ int blockblob_clone ( blockblob * bb, // destination blob, which blocks may be u goto free; - cleanup: + int saved_errno; + cleanup: // this is failure cleanup code path + + saved_errno = _blobstore_errno; // save it because dm_delete_devices may overwrite it + logprintfl (EUCAERROR, "error: blockblob_clone: %s (%d)\n", blobstore_get_last_msg(), _blobstore_errno); + // remove dm devices that may have been created if (dm_delete_devices (dev_names, devices)==0) { @@ -2755,7 +2760,8 @@ int blockblob_clone ( blockblob * bb, // destination blob, which blocks may be u set_blockblob_metadata_path (BLOCKBLOB_PATH_DM, bb->store, bb->id, path, sizeof (path)); unlink (path); } - + _blobstore_errno = saved_errno; + free: for (int i=0; i Date: Sat, 3 Dec 2011 13:30:51 -0800 Subject: [PATCH 6/7] handling MAX_DISK=0 correctly --- node/handlers.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node/handlers.c b/node/handlers.c index 58f9f54ff19..f76d6aff162 100644 --- a/node/handlers.c +++ b/node/handlers.c @@ -995,7 +995,11 @@ static int init (void) if (max_disk_gb != -1) { if (conf_work_size_mb == -1) { logprintfl (EUCAWARN, "warning: using deprecated setting %s for the new setting %s\n", CONFIG_MAX_DISK, CONFIG_NC_WORK_SIZE); - conf_work_size_mb = max_disk_gb * 1024; + if (max_disk_gb == 0) { + conf_work_size_mb = -1; // change in semantics: 0 used to mean 'unlimited', now 'unset' or -1 means that + } else { + conf_work_size_mb = max_disk_gb * 1024; + } } else { logprintfl (EUCAWARN, "warning: ignoring deprecated setting %s in favor of the new setting %s\n", CONFIG_MAX_DISK, CONFIG_NC_WORK_SIZE); } From 168ee95a10b00802c87d00bb370417bea8d79f62 Mon Sep 17 00:00:00 2001 From: Neil Soman Date: Sat, 3 Dec 2011 15:28:51 -0800 Subject: [PATCH 7/7] reduce ttl for front end responses. --- .../dns/src/main/java/com/eucalyptus/dns/TransientZone.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clc/modules/dns/src/main/java/com/eucalyptus/dns/TransientZone.java b/clc/modules/dns/src/main/java/com/eucalyptus/dns/TransientZone.java index ad9ddac151a..04a0534eb34 100644 --- a/clc/modules/dns/src/main/java/com/eucalyptus/dns/TransientZone.java +++ b/clc/modules/dns/src/main/java/com/eucalyptus/dns/TransientZone.java @@ -195,7 +195,7 @@ public SetResponse findRecords( Name name, int type ) { try { InetAddress cloudIp = Topology.lookup( Eucalyptus.class ).getInetAddress( ); if (cloudIp != null) { - resp.addRRset( new RRset( new ARecord( name, 1, ttl, cloudIp ) ) ); + resp.addRRset( new RRset( new ARecord( name, 1, 20/*ttl*/, cloudIp ) ) ); } return resp; } catch (Exception e) { @@ -256,7 +256,7 @@ public SetResponse findRecords( Name name, int type ) { LOG.error(e); return super.findRecords( name, type ); } - resp.addRRset( new RRset( new ARecord( name, 1, ttl, walrusIp ) ) ); + resp.addRRset( new RRset( new ARecord( name, 1, 20/*ttl*/, walrusIp ) ) ); return resp; } else { return super.findRecords( name, type );