Skip to content

Commit

Permalink
LU-1282 misc: Use present cpu numbers to save memory.
Browse files Browse the repository at this point in the history 
lprocfs stats data should allocated by the number of present cpus in
stead of by possible cpu numbers which wastes a lot of memory.

OSS minimum thread number also better be decided by present cpu
numbers.

Signed-off-by: Bobi Jam <bobijam@whamcloud.com>
Change-Id: Id1690f185f4f83fae75be7eddb756e413cbc4fba
  • Loading branch information
@morrone
Bobi Jam authored and morrone committed Apr 7, 2012
1 parent 3223515 commit 36211fd
Show file tree
Hide file tree
Showing 7 changed files with 93 additions and 83 deletions.
10 changes: 8 additions & 2 deletions libcfs/include/libcfs/linux/kp30.h
View file
Expand Up @@ -160,9 +160,15 @@ do { \
# define time(a) CURRENT_TIME

#ifndef num_possible_cpus
#define cfs_num_possible_cpus() NR_CPUS
# define cfs_num_possible_cpus() NR_CPUS
#else
#define cfs_num_possible_cpus() num_possible_cpus()
# define cfs_num_possible_cpus() num_possible_cpus()
#endif

#ifndef num_present_cpus
# define cfs_num_present_cpus() NR_CPUS
#else
# define cfs_num_present_cpus() num_present_cpus()
#endif

/******************************************************************************/
Expand Down
18 changes: 11 additions & 7 deletions libcfs/include/libcfs/user-prim.h
View file
Expand Up @@ -59,21 +59,25 @@ typedef struct proc_dir_entry cfs_proc_dir_entry_t;
* Just present a single processor until will add thread support.
*/
#ifndef smp_processor_id
#define cfs_smp_processor_id() 0
# define cfs_smp_processor_id() 0
#else
#define cfs_smp_processor_id() smp_processor_id()
# define cfs_smp_processor_id() smp_processor_id()
#endif
#ifndef num_online_cpus
#define cfs_num_online_cpus() 1
# define cfs_num_online_cpus() 1
#else
#define cfs_num_online_cpus() num_online_cpus()
# define cfs_num_online_cpus() num_online_cpus()
#endif
#ifndef num_possible_cpus
#define cfs_num_possible_cpus() 1
# define cfs_num_possible_cpus() 1
#else
#define cfs_num_possible_cpus() num_possible_cpus()
# define cfs_num_possible_cpus() num_possible_cpus()
#endif
#ifndef num_present_cpus
# define cfs_num_present_cpus() 1
#else
# define cfs_num_present_cpus() num_present_cpus()
#endif

/*
* Wait Queue.
*/
Expand Down
1 change: 1 addition & 0 deletions libcfs/include/libcfs/winnt/winnt-prim.h
View file
Expand Up @@ -862,6 +862,7 @@ libcfs_arch_cleanup(void);
#define CFS_NR_CPUS (32)
#define smp_num_cpus ((CCHAR)KeNumberProcessors)
#define cfs_num_possible_cpus() smp_num_cpus
#define cfs_num_present_cpus() smp_num_cpus
#define cfs_num_online_cpus() smp_num_cpus
#define cfs_smp_processor_id() ((USHORT)KeGetCurrentProcessorNumber())
#define smp_call_function(f, a, n, w) do {} while(0)
Expand Down
137 changes: 68 additions & 69 deletions libcfs/libcfs/linux/linux-tracefile.c
View file
Expand Up @@ -42,9 +42,9 @@

/* percents to share the total debug memory for each type */
static unsigned int pages_factor[CFS_TCD_TYPE_MAX] = {
80, /* 80% pages for CFS_TCD_TYPE_PROC */
10, /* 10% pages for CFS_TCD_TYPE_SOFTIRQ */
10 /* 10% pages for CFS_TCD_TYPE_IRQ */
80, /* 80% pages for CFS_TCD_TYPE_PROC */
10, /* 10% pages for CFS_TCD_TYPE_SOFTIRQ */
10 /* 10% pages for CFS_TCD_TYPE_IRQ */
};

char *cfs_trace_console_buffers[NR_CPUS][CFS_TCD_TYPE_MAX];
Expand All @@ -53,115 +53,114 @@ cfs_rw_semaphore_t cfs_tracefile_sem;

int cfs_tracefile_init_arch()
{
int i;
int j;
struct cfs_trace_cpu_data *tcd;
int i;
int j;
struct cfs_trace_cpu_data *tcd;

cfs_init_rwsem(&cfs_tracefile_sem);
cfs_init_rwsem(&cfs_tracefile_sem);

/* initialize trace_data */
memset(cfs_trace_data, 0, sizeof(cfs_trace_data));
for (i = 0; i < CFS_TCD_TYPE_MAX; i++) {
cfs_trace_data[i] =
/* initialize trace_data */
memset(cfs_trace_data, 0, sizeof(cfs_trace_data));
for (i = 0; i < CFS_TCD_TYPE_MAX; i++) {
cfs_trace_data[i] =
kmalloc(sizeof(union cfs_trace_data_union) * NR_CPUS,
GFP_KERNEL);
if (cfs_trace_data[i] == NULL)
goto out;
if (cfs_trace_data[i] == NULL)
goto out;

}
}

/* arch related info initialized */
cfs_tcd_for_each(tcd, i, j) {
cfs_spin_lock_init(&tcd->tcd_lock);
tcd->tcd_pages_factor = pages_factor[i];
tcd->tcd_type = i;
tcd->tcd_cpu = j;
}
/* arch related info initialized */
cfs_tcd_for_each(tcd, i, j) {
cfs_spin_lock_init(&tcd->tcd_lock);
tcd->tcd_pages_factor = pages_factor[i];
tcd->tcd_type = i;
tcd->tcd_cpu = j;
}

for (i = 0; i < num_possible_cpus(); i++)
for (j = 0; j < 3; j++) {
for (i = 0; i < cfs_num_possible_cpus(); i++)
for (j = 0; j < 3; j++) {
cfs_trace_console_buffers[i][j] =
kmalloc(CFS_TRACE_CONSOLE_BUFFER_SIZE,
GFP_KERNEL);

if (cfs_trace_console_buffers[i][j] == NULL)
goto out;
}
if (cfs_trace_console_buffers[i][j] == NULL)
goto out;
}

return 0;
return 0;

out:
cfs_tracefile_fini_arch();
printk(KERN_ERR "lnet: Not enough memory\n");
return -ENOMEM;

cfs_tracefile_fini_arch();
printk(KERN_ERR "lnet: Not enough memory\n");
return -ENOMEM;
}

void cfs_tracefile_fini_arch()
{
int i;
int j;

for (i = 0; i < num_possible_cpus(); i++)
for (j = 0; j < 3; j++)
if (cfs_trace_console_buffers[i][j] != NULL) {
kfree(cfs_trace_console_buffers[i][j]);
cfs_trace_console_buffers[i][j] = NULL;
}

for (i = 0; cfs_trace_data[i] != NULL; i++) {
kfree(cfs_trace_data[i]);
cfs_trace_data[i] = NULL;
}

cfs_fini_rwsem(&cfs_tracefile_sem);
int i;
int j;

for (i = 0; i < cfs_num_possible_cpus(); i++)
for (j = 0; j < 3; j++)
if (cfs_trace_console_buffers[i][j] != NULL) {
kfree(cfs_trace_console_buffers[i][j]);
cfs_trace_console_buffers[i][j] = NULL;
}

for (i = 0; cfs_trace_data[i] != NULL; i++) {
kfree(cfs_trace_data[i]);
cfs_trace_data[i] = NULL;
}

cfs_fini_rwsem(&cfs_tracefile_sem);
}

void cfs_tracefile_read_lock()
{
cfs_down_read(&cfs_tracefile_sem);
cfs_down_read(&cfs_tracefile_sem);
}

void cfs_tracefile_read_unlock()
{
cfs_up_read(&cfs_tracefile_sem);
cfs_up_read(&cfs_tracefile_sem);
}

void cfs_tracefile_write_lock()
{
cfs_down_write(&cfs_tracefile_sem);
cfs_down_write(&cfs_tracefile_sem);
}

void cfs_tracefile_write_unlock()
{
cfs_up_write(&cfs_tracefile_sem);
cfs_up_write(&cfs_tracefile_sem);
}

cfs_trace_buf_type_t cfs_trace_buf_idx_get()
{
if (in_irq())
return CFS_TCD_TYPE_IRQ;
else if (in_softirq())
return CFS_TCD_TYPE_SOFTIRQ;
else
return CFS_TCD_TYPE_PROC;
if (in_irq())
return CFS_TCD_TYPE_IRQ;
else if (in_softirq())
return CFS_TCD_TYPE_SOFTIRQ;
else
return CFS_TCD_TYPE_PROC;
}

int cfs_trace_lock_tcd(struct cfs_trace_cpu_data *tcd)
{
__LASSERT(tcd->tcd_type < CFS_TCD_TYPE_MAX);
__LASSERT(tcd->tcd_type < CFS_TCD_TYPE_MAX);
if (tcd->tcd_type == CFS_TCD_TYPE_IRQ)
cfs_spin_lock_irqsave(&tcd->tcd_lock, tcd->tcd_lock_flags);
else if (tcd->tcd_type == CFS_TCD_TYPE_SOFTIRQ)
cfs_spin_lock_bh(&tcd->tcd_lock);
else
cfs_spin_lock(&tcd->tcd_lock);
return 1;
return 1;
}

void cfs_trace_unlock_tcd(struct cfs_trace_cpu_data *tcd)
{
__LASSERT(tcd->tcd_type < CFS_TCD_TYPE_MAX);
__LASSERT(tcd->tcd_type < CFS_TCD_TYPE_MAX);
if (tcd->tcd_type == CFS_TCD_TYPE_IRQ)
cfs_spin_unlock_irqrestore(&tcd->tcd_lock, tcd->tcd_lock_flags);
else if (tcd->tcd_type == CFS_TCD_TYPE_SOFTIRQ)
Expand All @@ -173,11 +172,11 @@ void cfs_trace_unlock_tcd(struct cfs_trace_cpu_data *tcd)
int cfs_tcd_owns_tage(struct cfs_trace_cpu_data *tcd,
struct cfs_trace_page *tage)
{
/*
* XXX nikita: do NOT call portals_debug_msg() (CDEBUG/ENTRY/EXIT)
* from here: this will lead to infinite recursion.
*/
return tcd->tcd_cpu == tage->cpu;
/*
* XXX nikita: do NOT call portals_debug_msg() (CDEBUG/ENTRY/EXIT)
* from here: this will lead to infinite recursion.
*/
return tcd->tcd_cpu == tage->cpu;
}

void
Expand Down Expand Up @@ -228,13 +227,13 @@ void cfs_print_to_console(struct ptldebug_header *hdr, int mask,
printk("%s%s: %d:%d:(%s:%d:%s()) %.*s", ptype, prefix,
hdr->ph_pid, hdr->ph_extern_pid, file, hdr->ph_line_num,
fn, len, buf);
}
return;
}
return;
}

int cfs_trace_max_debug_mb(void)
{
int total_mb = (cfs_num_physpages >> (20 - PAGE_SHIFT));
int total_mb = (cfs_num_physpages >> (20 - PAGE_SHIFT));

return MAX(512, (total_mb * 80)/100);
return MAX(512, (total_mb * 80)/100);
}
4 changes: 2 additions & 2 deletions lustre/include/lprocfs_status.h
View file
Expand Up @@ -365,7 +365,7 @@ static inline int lprocfs_stats_lock(struct lprocfs_stats *stats, int opc)
cfs_spin_lock(&stats->ls_lock);
return 1;
} else {
return cfs_num_possible_cpus();
return cfs_num_present_cpus();
}
}
}
Expand Down Expand Up @@ -417,7 +417,7 @@ static inline __u64 lprocfs_stats_collector(struct lprocfs_stats *stats,
int i;

LASSERT(stats != NULL);
for (i = 0; i < cfs_num_possible_cpus(); i++)
for (i = 0; i < cfs_num_present_cpus(); i++)
ret += lprocfs_read_helper(&(stats->ls_percpu[i]->lp_cntr[idx]),
field);
return ret;
Expand Down
4 changes: 2 additions & 2 deletions lustre/obdclass/lprocfs_status.c
View file
Expand Up @@ -1192,7 +1192,7 @@ struct lprocfs_stats *lprocfs_alloc_stats(unsigned int num,
if (flags & LPROCFS_STATS_FLAG_NOPERCPU)
num_cpu = 1;
else
num_cpu = cfs_num_possible_cpus();
num_cpu = cfs_num_present_cpus();

OBD_ALLOC(stats, offsetof(typeof(*stats), ls_percpu[num_cpu]));
if (stats == NULL)
Expand Down Expand Up @@ -1244,7 +1244,7 @@ void lprocfs_free_stats(struct lprocfs_stats **statsh)
if (stats->ls_flags & LPROCFS_STATS_FLAG_NOPERCPU)
num_cpu = 1;
else
num_cpu = cfs_num_possible_cpus();
num_cpu = cfs_num_present_cpus();

percpusize = offsetof(struct lprocfs_percpu, lp_cntr[stats->ls_num]);
if (num_cpu > 1)
Expand Down
2 changes: 1 addition & 1 deletion lustre/ost/ost_handler.c
View file
Expand Up @@ -2390,7 +2390,7 @@ static int ost_setup(struct obd_device *obd, struct lustre_cfg* lcfg)
} else {
/* Base min threads on memory and cpus */
oss_min_threads =
cfs_num_possible_cpus() * CFS_NUM_CACHEPAGES >>
cfs_num_present_cpus() * CFS_NUM_CACHEPAGES >>
(27 - CFS_PAGE_SHIFT);
if (oss_min_threads < OSS_THREADS_MIN)
oss_min_threads = OSS_THREADS_MIN;
Expand Down

0 comments on commit 36211fd

Please sign in to comment.