Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

.blobstore rewriting bug fix; more thorough disk stats; corrected all…

…ocated blocks calculation; code style
  • Loading branch information...
commit 889542a6d266ff8f607e2454b07c4a5cb48ed12b 1 parent fb91d4c
root authored
View
65 node/handlers.c
@@ -978,14 +978,16 @@ static int init (void)
// determine how much is used/available in work and cache areas on the backing store
blobstore_meta work_meta, cache_meta;
stat_backing_store (instance_path, &work_meta, &cache_meta); // will zero-out work_ and cache_meta
- long long work_bs_size_mb = work_meta.blocks_limit ? (work_meta.blocks_limit / 2048) : -1; // convert sectors->MB
- long long work_bs_used_mb = work_meta.blocks_limit ? ((work_meta.blocks_allocated + work_meta.blocks_used) / 2048) : 0;
- long long cache_bs_size_mb = cache_meta.blocks_limit ? (cache_meta.blocks_limit / 2048) : -1;
- long long cache_bs_used_mb = cache_meta.blocks_limit ? ((cache_meta.blocks_allocated + cache_meta.blocks_used) / 2048) : 0;
+ long long work_bs_size_mb = work_meta.blocks_limit ? (work_meta.blocks_limit / 2048) : (-1L); // convert sectors->MB
+ long long work_bs_allocated_mb = work_meta.blocks_limit ? (work_meta.blocks_allocated / 2048) : 0;
+ long long work_bs_reserved_mb = work_meta.blocks_limit ? ((work_meta.blocks_locked + work_meta.blocks_unlocked) / 2048) : 0;
+ long long cache_bs_size_mb = cache_meta.blocks_limit ? (cache_meta.blocks_limit / 2048) : (-1L);
+ long long cache_bs_allocated_mb = cache_meta.blocks_limit ? (cache_meta.blocks_allocated / 2048) : 0;
+ long long cache_bs_reserved_mb = cache_meta.blocks_limit ? ((cache_meta.blocks_locked + cache_meta.blocks_unlocked) / 2048) : 0;
// look up configuration file settings for work and cache size
- int conf_work_size_mb = -1; GET_VAR_INT(conf_work_size_mb, CONFIG_NC_WORK_SIZE);
- int conf_cache_size_mb = -1; GET_VAR_INT(conf_cache_size_mb, CONFIG_NC_CACHE_SIZE);
+ long long conf_work_size_mb = -1; GET_VAR_INT(conf_work_size_mb, CONFIG_NC_WORK_SIZE);
+ long long conf_cache_size_mb = -1; GET_VAR_INT(conf_cache_size_mb, CONFIG_NC_CACHE_SIZE);
{ // accommodate legacy MAX_DISK setting by converting it
int max_disk_gb = -1; GET_VAR_INT(max_disk_gb, CONFIG_MAX_DISK);
if (max_disk_gb != -1) {
@@ -999,8 +1001,8 @@ static int init (void)
}
// decide what work and cache sizes should be, based on all the inputs
- int work_size_mb = -1;
- int cache_size_mb = -1;
+ long long work_size_mb = -1;
+ long long cache_size_mb = -1;
// above all, try to respect user-specified limits for work and cache
if (conf_work_size_mb != -1) {
@@ -1009,7 +1011,7 @@ static int init (void)
CONFIG_NC_WORK_SIZE, conf_work_size_mb, MIN_BLOBSTORE_SIZE_MB);
} else {
if (work_bs_size_mb != -1 && work_bs_size_mb != conf_work_size_mb) {
- logprintfl (EUCAWARN, "warning: specified work size (%s=%d) differs from existing work limits (%d), will try resizing\n",
+ logprintfl (EUCAWARN, "warning: specified work size (%s=%d) differs from existing work size (%d), will try resizing\n",
CONFIG_NC_WORK_SIZE, conf_work_size_mb, work_bs_size_mb);
}
work_size_mb = conf_work_size_mb;
@@ -1020,7 +1022,7 @@ static int init (void)
cache_size_mb = 0; // so it won't be used
} else {
if (cache_bs_size_mb != -1 && cache_bs_size_mb != conf_cache_size_mb) {
- logprintfl (EUCAWARN, "warning: specified cache size (%s=%d) differs from existing cache limits (%d), will try resizing\n",
+ logprintfl (EUCAWARN, "warning: specified cache size (%s=%d) differs from existing cache size (%d), will try resizing\n",
CONFIG_NC_CACHE_SIZE, conf_cache_size_mb, cache_bs_size_mb);
}
cache_size_mb = conf_cache_size_mb;
@@ -1040,20 +1042,21 @@ static int init (void)
work_size_mb = (long long)((double)(fs_avail_mb - FS_BUFFER_MB) * WORK_BS_PERCENT);
cache_size_mb = fs_avail_mb - FS_BUFFER_MB - work_size_mb;
} else if (work_size_mb == -1) {
- work_size_mb = fs_avail_mb - FS_BUFFER_MB - cache_size_mb + cache_bs_used_mb;
+ work_size_mb = fs_avail_mb - FS_BUFFER_MB - cache_size_mb + cache_bs_allocated_mb;
} else if (cache_size_mb == -1) {
- cache_size_mb = fs_avail_mb - FS_BUFFER_MB - work_size_mb + work_bs_used_mb;
+ cache_size_mb = fs_avail_mb - FS_BUFFER_MB - work_size_mb + work_bs_allocated_mb;
}
// sanity-check final results
if (cache_size_mb < MIN_BLOBSTORE_SIZE_MB)
cache_size_mb = 0;
if (work_size_mb < MIN_BLOBSTORE_SIZE_MB) {
- logprintfl (EUCAERROR, "error: insufficient disk space (%d) for virtual machines\n", work_size_mb);
+ logprintfl (EUCAERROR, "error: insufficient disk space for virtual machines (free space: %dMB, reserved for cache: %dMB)\n",
+ work_size_mb, (fs_avail_mb - FS_BUFFER_MB), cache_size_mb);
free (instance_path);
return ERROR_FATAL;
}
- if ((cache_size_mb + work_size_mb - cache_bs_used_mb - work_bs_used_mb) > fs_avail_mb) {
+ if ((cache_size_mb + work_size_mb - cache_bs_allocated_mb - work_bs_allocated_mb) > fs_avail_mb) {
logprintfl (EUCAWARN, "warning: sum of work and cache sizes exceeds available disk space\n");
}
@@ -1067,12 +1070,28 @@ static int init (void)
nc_state.disk_max = (long long)(work_size_mb / MB_PER_DISK_UNIT);
logprintfl (EUCAINFO, "disk space for instances: %s/work\n", instance_path);
- logprintfl (EUCAINFO, " %06lldMB allocated (%03.1f% of the file system)\n", work_size_mb, (work_size_mb/fs_size_mb)*100.0 );
- logprintfl (EUCAINFO, " %06lldMB in use (%03.1% of allocated)\n", work_bs_used_mb, (work_bs_used_mb/work_size_mb)*100.0 );
+ logprintfl (EUCAINFO, " %06lldMB limit (%.1f%% of the file system)\n",
+ work_size_mb,
+ ((double)work_size_mb/(double)fs_size_mb)*100.0 );
+ logprintfl (EUCAINFO, " %06lldMB reserved for use (%.1f%% of limit)\n",
+ work_bs_reserved_mb,
+ ((double)work_bs_reserved_mb/(double)work_size_mb)*100.0 );
+ logprintfl (EUCAINFO, " %06lldMB allocated for use (%.1f%% of limit, %.1f%% of file system)\n",
+ work_bs_allocated_mb,
+ ((double)work_bs_allocated_mb/(double)work_size_mb)*100.0,
+ ((double)work_bs_allocated_mb/(double)fs_size_mb)*100.0 );
if (cache_size_mb) {
logprintfl (EUCAINFO, " disk space for cache: %s/cache\n", instance_path);
- logprintfl (EUCAINFO, " %06lldMB allocated (%03.1f% of the file system)\n", cache_size_mb, (cache_size_mb/fs_size_mb)*100.0 );
- logprintfl (EUCAINFO, " %06lldMB in use (%03.1% of allocated)\n", cache_bs_used_mb, (cache_bs_used_mb/cache_size_mb)*100.0 );
+ logprintfl (EUCAINFO, " %06lldMB limit (%.1f%% of the file system)\n",
+ cache_size_mb,
+ ((double)cache_size_mb/(double)fs_size_mb)*100.0 );
+ logprintfl (EUCAINFO, " %06lldMB reserved for use (%.1f%% of limit)\n",
+ cache_bs_reserved_mb,
+ ((double)cache_bs_reserved_mb/(double)cache_size_mb)*100.0 );
+ logprintfl (EUCAINFO, " %06lldMB allocated for use (%.1f%% of limit, %.1f%% of file system)\n",
+ cache_bs_allocated_mb,
+ ((double)cache_bs_allocated_mb/(double)cache_size_mb)*100.0,
+ ((double)cache_bs_allocated_mb/(double)fs_size_mb)*100.0 );
} else {
logprintfl (EUCAWARN, "warning: disk cache will not be used\n");
}
@@ -1119,6 +1138,16 @@ static int init (void)
return ERROR_FATAL;
}
+ // now that hypervisor-specific initializers have discovered mem_max and cores_max,
+ // adjust the values based on configuration parameters, if any
+ if (nc_state.config_max_mem && nc_state.config_max_mem < nc_state.mem_max)
+ nc_state.mem_max = nc_state.config_max_mem;
+ if (nc_state.config_max_cores)
+ nc_state.cores_max = nc_state.config_max_cores;
+ logprintfl(EUCAINFO, "physical memory available for instances: %lldMB\n", nc_state.mem_max);
+ logprintfl(EUCAINFO, "virtual cpu cores available for instances: %lld\n", nc_state.cores_max);
+
+
// adopt running instances
adopt_instances();
View
15 node/handlers_kvm.c
@@ -97,9 +97,7 @@ static int doInitialize (struct nc_state_t *nc)
{
char *s = NULL;
- logprintfl(EUCADEBUG, "doInitialized() invoked\n");
-
- /* set up paths of Eucalyptus commands NC relies on */
+ // set up paths of Eucalyptus commands NC relies on
snprintf (nc->get_info_cmd_path, MAX_PATH, EUCALYPTUS_GET_KVM_INFO, nc->home, nc->home);
strcpy(nc->uri, HYPERVISOR_URI);
nc->convert_to_disk = 1;
@@ -117,18 +115,9 @@ static int doInitialize (struct nc_state_t *nc)
GET_VALUE("total_memory", nc->mem_max);
if (s) free(s);
- /* we leave 256M to the host */
+ // we leave 256M to the host
nc->mem_max -= 256;
- /* let's adjust the values based on the config values */
- if (nc->config_max_mem && nc->config_max_mem < nc->mem_max)
- nc->mem_max = nc->config_max_mem;
- if (nc->config_max_cores)
- nc->cores_max = nc->config_max_cores;
-
- logprintfl(EUCAINFO, "Using %lld cores\n", nc->cores_max);
- logprintfl(EUCAINFO, "Using %lld memory\n", nc->mem_max);
-
return OK;
}
View
19 node/handlers_xen.c
@@ -95,9 +95,7 @@ static int doInitialize (struct nc_state_t *nc)
virNodeInfo ni;
long long dom0_min_mem;
- logprintfl(EUCADEBUG, "doInitialized() invoked\n");
-
- /* set up paths of Eucalyptus commands NC relies on */
+ // set up paths of Eucalyptus commands NC relies on
snprintf (nc->get_info_cmd_path, MAX_PATH, EUCALYPTUS_GET_XEN_INFO, nc->home, nc->home);
snprintf (nc->virsh_cmd_path, MAX_PATH, EUCALYPTUS_VIRSH, nc->home);
snprintf (nc->xm_cmd_path, MAX_PATH, EUCALYPTUS_XM);
@@ -106,18 +104,18 @@ static int doInitialize (struct nc_state_t *nc)
nc->convert_to_disk = 0;
nc->capability = HYPERVISOR_XEN_AND_HARDWARE; // TODO: set to XEN_PARAVIRTUALIZED if on older Xen kernel
- /* check connection is fresh */
+ // check connection is fresh
if (!check_hypervisor_conn()) {
return ERROR_FATAL;
}
- /* get resources */
+ // get resources
if (virNodeGetInfo(nc->conn, &ni)) {
logprintfl (EUCAFATAL, "error: failed to discover resources\n");
return ERROR_FATAL;
}
- /* dom0-min-mem has to come from xend config file */
+ // dom0-min-mem has to come from xend config file
s = system_output (nc->get_info_cmd_path);
if (get_value (s, "dom0-min-mem", &dom0_min_mem)) {
logprintfl (EUCAFATAL, "error: did not find dom0-min-mem in output from %s\n", nc->get_info_cmd_path);
@@ -126,21 +124,18 @@ static int doInitialize (struct nc_state_t *nc)
}
free (s);
- /* calculate the available memory */
+ // calculate the available memory
nc->mem_max = ni.memory/1024 - 32 - dom0_min_mem;
- /* calculate the available cores */
+ // calculate the available cores
nc->cores_max = ni.cpus;
- /* let's adjust the values based on the config values */
+ // let's adjust the values based on the config values
if (nc->config_max_mem && nc->config_max_mem < nc->mem_max)
nc->mem_max = nc->config_max_mem;
if (nc->config_max_cores)
nc->cores_max = nc->config_max_cores;
- logprintfl(EUCAINFO, "Using %lld cores\n", nc->cores_max);
- logprintfl(EUCAINFO, "Using %lld memory\n", nc->mem_max);
-
return OK;
}
View
7 storage/backing.c
@@ -102,7 +102,7 @@ static void bs_errors (const char * msg) {
static void stat_blobstore (const char * conf_instances_path, const char * name, blobstore_meta * meta)
{
- bzero (meta, sizeof (meta));
+ bzero (meta, sizeof (blobstore_meta));
char path [MAX_PATH];
snprintf (path, sizeof (path), "%s/%s", conf_instances_path, name);
blobstore * bs = blobstore_open (path,
@@ -151,7 +151,7 @@ int init_backing_store (const char * conf_instances_path, unsigned int conf_work
if (cache_limit_blocks) {
cache_bs = blobstore_open (cache_path, cache_limit_blocks, BLOBSTORE_FLAG_CREAT, BLOBSTORE_FORMAT_DIRECTORY, BLOBSTORE_REVOCATION_LRU, BLOBSTORE_SNAPSHOT_ANY);
if (cache_bs==NULL) {
- logprintfl (EUCAERROR, "ERROR: %s\n", blobstore_get_error_str(blobstore_get_error()));
+ logprintfl (EUCAERROR, "ERROR: failed to open/create cache blobstore: %s\n", blobstore_get_error_str(blobstore_get_error()));
return ERROR;
}
if (blobstore_fsck (cache_bs, NULL)) { // TODO: verify checksums?
@@ -161,7 +161,8 @@ int init_backing_store (const char * conf_instances_path, unsigned int conf_work
}
work_bs = blobstore_open (work_path, work_limit_blocks, BLOBSTORE_FLAG_CREAT, BLOBSTORE_FORMAT_FILES, BLOBSTORE_REVOCATION_NONE, BLOBSTORE_SNAPSHOT_ANY);
if (work_bs==NULL) {
- logprintfl (EUCAERROR, "ERROR: %s\n", blobstore_get_error_str(blobstore_get_error()));
+ logprintfl (EUCAERROR, "ERROR: failed to open/create work blobstore: %s\n", blobstore_get_error_str(blobstore_get_error()));
+ logprintfl (EUCAERROR, "ERROR: %s\n", blobstore_get_last_trace());
blobstore_close (cache_bs);
return ERROR;
}
View
39 storage/blobstore.c
@@ -736,12 +736,11 @@ static int read_store_metadata (blobstore * bs)
if (size == -1)
return -1;
-
if (size<30) {
ERR (BLOBSTORE_ERROR_NOENT, "metadata size is too small");
return -1;
}
-
+
char * val;
if ((val = get_val (buf, "id"))==NULL)
return -1;
@@ -766,6 +765,8 @@ static int write_store_metadata (blobstore * bs)
{
if (ftruncate (bs->fd, 0)==-1)
{ ERR (BLOBSTORE_ERROR_NOENT, "failed to truncate the metadata file"); return -1; }
+ if (lseek (bs->fd, 0, SEEK_SET)==-1)
+ { ERR (BLOBSTORE_ERROR_ACCES, "failed to seek in metadata file"); return -1; }
char buf [1024];
snprintf (buf, sizeof (buf),
"id: %s\n" \
@@ -778,8 +779,9 @@ static int write_store_metadata (blobstore * bs)
bs->revocation_policy,
bs->snapshot_policy,
bs->format);
- int len = write (bs->fd, buf, strlen (buf));
- if (len != strlen (buf))
+ int slen = strlen (buf);
+ int len = write (bs->fd, buf, slen);
+ if (len != slen)
{ ERR (BLOBSTORE_ERROR_NOENT, "failed to write to the metadata file"); return -1; }
return 0;
@@ -833,6 +835,8 @@ blobstore * blobstore_open ( const char * path,
blobstore_revocation_t revocation_policy,
blobstore_snapshot_t snapshot_policy)
{
+ int saved_errno;
+
if (blobstore_init())
return NULL;
@@ -906,7 +910,9 @@ blobstore * blobstore_open ( const char * path,
ERR (BLOBSTORE_ERROR_INVAL, "'limit_blocks' does not match existing blobstore");
goto free;
} else {
+ logprintfl (EUCAINFO, "adjusting blobstore limit from %d to %d\n", bs->limit_blocks, limit_blocks);
write_flags = BLOBSTORE_FLAG_RDWR;
+ close_and_unlock (bs->fd);
goto write_metadata;
}
}
@@ -927,6 +933,7 @@ blobstore * blobstore_open ( const char * path,
goto free;
} else {
write_flags = BLOBSTORE_FLAG_RDWR;
+ close_and_unlock (bs->fd);
goto write_metadata;
}
}
@@ -936,11 +943,14 @@ blobstore * blobstore_open ( const char * path,
goto out;
free:
+ saved_errno = _blobstore_errno;
close_and_unlock (bs->fd);
if (bs) {
free (bs);
bs = NULL;
}
+ _blobstore_errno = saved_errno;
+
out:
return bs;
}
@@ -1445,6 +1455,7 @@ static blockblob ** walk_bs (blobstore * bs, const char * dir_path, blockblob **
safe_strncpy (bb->blocks_path, entry_path, sizeof(bb->blocks_path));
set_device_path (bb); // read .dm and .loopback and set bb->device_path accordingly
bb->size_bytes = sb.st_size;
+ bb->blocks_allocated = sb.st_blocks;
bb->last_accessed = sb.st_atime;
bb->last_modified = sb.st_mtime;
bb->snapshot_type = BLOBSTORE_FORMAT_ANY; // it is not necessary to know whether this is a snapshot
@@ -1534,15 +1545,17 @@ int blobstore_stat (blobstore * bs, blobstore_meta * meta)
// analyze the LL, calculating sizes
meta->blocks_allocated = 0;
- meta->blocks_used = 0;
+ meta->blocks_unlocked = 0;
+ meta->blocks_locked = 0;
meta->num_blobs = 0;
for (blockblob * abb = bbs; abb; abb=abb->next) {
long long abb_size_blocks = round_up_sec (abb->size_bytes) / 512;
if (abb->in_use & ~BLOCKBLOB_STATUS_BACKED) {
- meta->blocks_used += abb_size_blocks; // these can't be purged if we need space (TODO: look into recursive purging of unused references?)
+ meta->blocks_locked += abb_size_blocks; // these can't be purged if we need space (TODO: look into recursive purging of unused references?)
} else {
- meta->blocks_allocated += abb_size_blocks; // these can be purged
+ meta->blocks_unlocked += abb_size_blocks; // these can be purged
}
+ meta->blocks_allocated += abb->blocks_allocated;
meta->num_blobs++;
}
@@ -1866,24 +1879,24 @@ blockblob * blockblob_open ( blobstore * bs,
}
// analyze the LL, calculating sizes
- long long blocks_allocated = 0;
- long long blocks_used = 0;
+ long long blocks_unlocked = 0;
+ long long blocks_locked = 0;
unsigned int num_blobs = 0;
for (blockblob * abb = bbs; abb; abb=abb->next) {
long long abb_size_blocks = round_up_sec (abb->size_bytes) / 512;
if (abb->in_use & ~BLOCKBLOB_STATUS_BACKED) {
- blocks_used += abb_size_blocks; // these can't be purged if we need space (TODO: look into recursive purging of unused references?)
+ blocks_locked += abb_size_blocks; // these can't be purged if we need space (TODO: look into recursive purging of unused references?)
} else {
- blocks_allocated += abb_size_blocks; // these can be purged
+ blocks_unlocked += abb_size_blocks; // these can be purged
}
num_blobs++;
}
- long long blocks_free = bs->limit_blocks - (blocks_allocated + blocks_used);
+ long long blocks_free = bs->limit_blocks - (blocks_unlocked + blocks_locked);
if (blocks_free < size_blocks) {
if (!(bs->revocation_policy==BLOBSTORE_REVOCATION_LRU) // not allowed to purge
||
- (blocks_free+blocks_allocated) < size_blocks) { // not enough purgeable material
+ (blocks_free+blocks_unlocked) < size_blocks) { // not enough purgeable material
ERR (BLOBSTORE_ERROR_NOSPC, NULL);
goto clean;
}
View
6 storage/blobstore.h
@@ -166,6 +166,7 @@ typedef struct _blockblob {
char device_path [BLOBSTORE_MAX_PATH]; // full path of a block device on which blob can be accessed
char dm_name [MAX_DM_NAME]; // name of the main device mapper device if this is a clone
unsigned long long size_bytes; // size of the blob in bytes
+ unsigned long long blocks_allocated; // actual number of blocks on disk taken by the blob
blobstore_snapshot_t snapshot_type; // ANY = not initialized/known, NONE = not a snapshot, DM = DM-based snapshot
unsigned int in_use; // flags showing how the blockblob is being used (OPENED, LOCKED, LINKED)
time_t last_accessed; // timestamp of last access
@@ -205,8 +206,9 @@ typedef struct _blockblob_meta {
typedef struct _blobstore_meta {
char id [BLOBSTORE_MAX_PATH]; // ID of the blobstore, to handle directory moving
unsigned long long blocks_limit; // max size of the blobstore, in blocks
- unsigned long long blocks_allocated; // number of blocks in blobstore allocated to a blob that is not in use and is not mapped
- unsigned long long blocks_used; // number of blocks in blobstore allocated to a blob that is in use or is mapped (a dependency)
+ unsigned long long blocks_unlocked; // number of blocks in blobstore allocated to blobs that are not in use and is not mapped
+ unsigned long long blocks_locked; // number of blocks in blobstore allocated to blobs that are in use or is mapped (a dependency)
+ unsigned long long blocks_allocated; // number of blocks in blobstore that have been allocated on disk
unsigned int num_blobs; // count of blobs in the blobstore
blobstore_revocation_t revocation_policy;
blobstore_snapshot_t snapshot_policy;
Please sign in to comment.
Something went wrong with that request. Please try again.