Skip to content

Commit

Permalink
Fix cppcheck failure in dlcp.
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhouhaiy @CaoZhongZ
zhouhaiy authored and CaoZhongZ committed May 3, 2018
1 parent 44c7042 commit adf5bfb
Show file tree
Hide file tree
Showing 5 changed files with 109 additions and 51 deletions.
52 changes: 25 additions & 27 deletions dlcp/src/dl_compression_impl.cpp
View file
Expand Up @@ -66,7 +66,7 @@ dl_comp_return_t DLCompressDFP::compress_block(float *src, int8_t *dst, float *d
// only handle float buffer as src and int8_t as dst
float max_abs = 0.;
float max_abs_log2 = 0.;
float round_value, d_value;
float d_value;
int8_t decomp_value = 0;

if (NULL != diff) {
Expand Down Expand Up @@ -97,7 +97,8 @@ dl_comp_return_t DLCompressDFP::compress_block(float *src, int8_t *dst, float *d
// It's corner case that the result value of src[i]*pow2_scale will be
// bigger than 127.5f. The value will rounded up to 128. This is out of range
// of int8_t. (-128 - 127) So we set it as 127.
round_value = std::round(src[i]*pow2_scale);

float round_value = std::round(src[i]*pow2_scale);
if (round_value <= 127.0f) {
decomp_value = (int8_t)round_value;
} else {
Expand Down Expand Up @@ -184,8 +185,7 @@ dl_comp_return_t DLCompressDFP::compress_buffer(float *src, int8_t *dst, float *
dl_comp_return_t ret = DL_COMP_FAIL;
dl_comp_head *compHead = NULL;
int scale = 0;
size_t comp_block = 0;
for (size_t i = 0; i < count; i += DL_COMP_BLOCK_NUM) {
for (size_t i = 0, comp_block = 0; i < count; i += DL_COMP_BLOCK_NUM) {
comp_block = (i + DL_COMP_BLOCK_NUM) < count ? DL_COMP_BLOCK_NUM : (count - i);
compHead = (dl_comp_head *)dst;
if (!inPlace) {
Expand Down Expand Up @@ -226,8 +226,6 @@ dl_comp_return_t DLCompressDFP::decompress_buffer(const int8_t *src, float *dst,
{
dl_comp_head *compHead = NULL;
dl_comp_return_t ret;
size_t count;
int scale;
const int8_t *origSrc = src;
float *origDst = dst;
int8_t decomp_block[DL_COMP_BLOCK_NUM];
Expand All @@ -244,8 +242,8 @@ dl_comp_return_t DLCompressDFP::decompress_buffer(const int8_t *src, float *dst,
if (compHead->magic != DL_COMP_HEAD_MAGIC) {
return DL_COMP_FAIL_INVALID_COMPRESSED_FORMAT;
}
count = compHead->payloadLen;
scale = compHead->exponent;
size_t count = compHead->payloadLen;
int scale = compHead->exponent;
if (blockCount == 1) {
memcpy(decomp_block, src + sizeof(dl_comp_head), count);
}
Expand Down Expand Up @@ -284,7 +282,7 @@ dl_comp_return_t DLCompressDFP::avx512_decompress_block(const int8_t *src, float
// Do de-quantization
float pow2_scale_inv = 1.0f / std::pow(2, scale);
size_t group_size = 16;
size_t num_group = count / group_size;
//size_t num_group = count / group_size;
__m512 scale_factor = _mm512_set1_ps(pow2_scale_inv);

#ifdef _OPENMP
Expand Down Expand Up @@ -321,8 +319,8 @@ dl_comp_return_t DLCompressDFP::decompress_block(const int8_t *src, float *dst,
return DL_COMP_OK;
}

#if 0
size_t DLCompressDFP::get_dataCount_in_compressed_buffer(const int8_t *src, size_t blockCount) {
size_t count = 0;
size_t sum = 0;
dl_comp_head *compHead = NULL;

Expand All @@ -333,7 +331,7 @@ size_t DLCompressDFP::get_dataCount_in_compressed_buffer(const int8_t *src, size
do {
compHead = (dl_comp_head *)src;
DLCP_ASSERT(compHead->magic == DL_COMP_HEAD_MAGIC, "Invalid compHead!!!\n");
count = compHead->payloadLen;
size_t count = compHead->payloadLen;
src += sizeof(dl_comp_head);
src += count;
sum += count;
Expand All @@ -342,7 +340,9 @@ size_t DLCompressDFP::get_dataCount_in_compressed_buffer(const int8_t *src, size

return sum;
}
#endif

#if 0
dl_comp_return_t DLCompressDFP::compress_sum(const int8_t *invec, int8_t *inoutvec, size_t blockCount)
{
dl_comp_return_t ret = DL_COMP_OK;
Expand Down Expand Up @@ -372,6 +372,7 @@ dl_comp_return_t DLCompressDFP::compress_sum(const int8_t *invec, int8_t *inoutv

return ret;
}
#endif

dl_comp_return_t DLCompressDFP::compress_sum2(const int8_t *invec, int8_t *inoutvec, size_t blockCount)
{
Expand Down Expand Up @@ -436,7 +437,7 @@ dl_comp_return_t DLCompressDFP::compress_block_sum(const int8_t *invec, int8_t *
int minScale = std::min(inScale, outScale);
int inScaleGap = inScale - minScale;
int outScaleGap = outScale - minScale;
int8_t left, right;
int8_t left;
int max_abs = 0;


Expand All @@ -448,7 +449,7 @@ dl_comp_return_t DLCompressDFP::compress_block_sum(const int8_t *invec, int8_t *
#endif
for (size_t i = 0; i < count; i++) {
left = invec[i] >> inScaleGap;
right = inoutvec[i] >> outScaleGap;
int8_t right = inoutvec[i] >> outScaleGap;
resvec[i] = left + right;
// This is for compensation of final right shift
// To make it an unbiased estimator, we only
Expand Down Expand Up @@ -520,7 +521,6 @@ dl_comp_return_t DLCompressDFP::compress_block_sum2(const int8_t *invec, int8_t
int minScale = std::min(inScale, outScale);
int inScaleGap = inScale - minScale;
int outScaleGap = outScale - minScale;
int8_t left, right;
int max_abs = 0;
size_t group_size = 16;
__mmask16 mask = _mm512_int2mask(0xFFFF);
Expand Down Expand Up @@ -575,12 +575,10 @@ dl_comp_return_t DLCompressDFP::compress_block_sum2(const int8_t *invec, int8_t
return DL_COMP_OK;
}

#if 0
void DLCompressDFP::dump_compressed_buffer(const int8_t *src, size_t blockCount)
{
size_t count = 0;
dl_comp_head *compHead = NULL;
int scale = 0;
float pow2_scale = .0;

if (blockCount == 0) return;

Expand All @@ -591,10 +589,10 @@ void DLCompressDFP::dump_compressed_buffer(const int8_t *src, size_t blockCount)
DLCP_LOG(INFO, "Invalid compHead!!!\n");
return;
}
count = compHead->payloadLen;
scale = compHead->exponent;
DLCP_LOG(INFO, "count = %lu Scale = %d\n", count, scale);
pow2_scale = std::pow(2, scale);
size_t count = compHead->payloadLen;
int scale = compHead->exponent;
DLCP_LOG(INFO, "count = %lu Scale = %d\n", (unsigned long)count, scale);
float pow2_scale = std::pow(2, scale);
src += sizeof(dl_comp_head);
for (size_t i = 0; i < count; i++) {
float d_value = ((float)src[i])/pow2_scale;
Expand All @@ -610,20 +608,17 @@ bool DLCompressDFP::check_compressed_buffer(const float *comp1, const int8_t *co
{
float epislon = 1e-9;
dl_comp_head *compHead = NULL;
int scale = 0;
float pow2_scale = .0;
size_t count = 0;

do {
compHead = (dl_comp_head *)comp2;
if (compHead->magic != DL_COMP_HEAD_MAGIC) {
DLCP_LOG(ERROR, "Invalid compHead!!!\n");
return false;
}
count = compHead->payloadLen;
scale = compHead->exponent;
size_t count = compHead->payloadLen;
int scale = compHead->exponent;
comp2 += sizeof(dl_comp_head);
pow2_scale = std::pow(2, scale);
float pow2_scale = std::pow(2, scale);
for (size_t i = 0; i < count; i++) {
float d_value = ((float)comp2[i])/pow2_scale;
if (d_value * comp1[i] < 0.0f) {
Expand All @@ -647,12 +642,15 @@ dl_comp_return_t compress_helper(float *src, int8_t *dst, float *diff, dl_comp_m
dl_comp_return_t ret = compInst->compress_buffer(src, dst, diff, count);
return ret;
}
#endif

#if 0
dl_comp_return_t decompress_helper(const int8_t *src, float *dst, dl_comp_method_t method)
{
DLCompressBase *compInst = DLCompressBase::get_compression_instance(method);
return compInst->decompress_buffer(src, dst, 0);
}
#endif

void dl_comp_float_vector_add(const float* invec, float *inoutvec, size_t count)
{
Expand Down
32 changes: 16 additions & 16 deletions dlcp/src/dl_compression_impl.hpp
View file
Expand Up @@ -73,11 +73,11 @@ class DLCompressBase {
// Compress without error feedback
virtual dl_comp_return_t compress_buffer(float *src, int8_t *dst, size_t count, bool inPlace = false) = 0;
virtual dl_comp_return_t decompress_buffer(const int8_t *src, float *dst, size_t blockCount) = 0;
virtual size_t get_dataCount_in_compressed_buffer(const int8_t *src, size_t blockCount) = 0;
virtual dl_comp_return_t compress_sum(const int8_t *invec, int8_t *inoutvec, size_t blockCount) = 0;
// virtual size_t get_dataCount_in_compressed_buffer(const int8_t *src, size_t blockCount) = 0;
// virtual dl_comp_return_t compress_sum(const int8_t *invec, int8_t *inoutvec, size_t blockCount) = 0;
virtual dl_comp_return_t compress_sum2(const int8_t *invec, int8_t *inoutvec, size_t blockCount) = 0;
virtual void dump_compressed_buffer(const int8_t *src, size_t blockCount) = 0;
virtual bool check_compressed_buffer(const float *comp1, const int8_t *comp2, const float *diff, size_t blockCount) = 0;
// virtual void dump_compressed_buffer(const int8_t *src, size_t blockCount) = 0;
// virtual bool check_compressed_buffer(const float *comp1, const int8_t *comp2, const float *diff, size_t blockCount) = 0;
virtual ~DLCompressBase(void) {};

public:
Expand All @@ -95,22 +95,22 @@ class DLCompressDFP : public DLCompressBase {
virtual dl_comp_return_t compress_buffer(float *src, int8_t *dst, float *diff, size_t count, bool inPlace = false);
virtual dl_comp_return_t compress_buffer(float *src, int8_t *dst, size_t count, bool inPlace = false);
virtual dl_comp_return_t decompress_buffer(const int8_t *src, float *dst, size_t blockCount);
virtual size_t get_dataCount_in_compressed_buffer(const int8_t *src, size_t blockCount);
virtual dl_comp_return_t compress_sum(const int8_t *invec, int8_t *inoutvec, size_t blockCount);
// virtual size_t get_dataCount_in_compressed_buffer(const int8_t *src, size_t blockCount);
// virtual dl_comp_return_t compress_sum(const int8_t *invec, int8_t *inoutvec, size_t blockCount);
virtual dl_comp_return_t compress_sum2(const int8_t *invec, int8_t *inoutvec, size_t blockCount);
virtual void dump_compressed_buffer(const int8_t *src, size_t blockCount);
virtual bool check_compressed_buffer(const float *comp1, const int8_t *comp2, const float *diff, size_t blockCount);
// virtual void dump_compressed_buffer(const int8_t *src, size_t blockCount);
// virtual bool check_compressed_buffer(const float *comp1, const int8_t *comp2, const float *diff, size_t blockCount);

private:
DLCompressDFP(): avx512_enabled_(dl_comp_check_avx512_supported()) {};
public:
static dl_comp_return_t compress_block(float *src, int8_t *dst, float *diff, size_t count, int *scale);
static dl_comp_return_t decompress_block(const int8_t *src, float *dst, size_t count, int scale);
static dl_comp_return_t compress_block_sum(const int8_t *invec, int8_t *inoutvec);
static dl_comp_return_t avx512_decompress_block(const int8_t *src, float *dst, size_t count, int scale);
static dl_comp_return_t avx512_compress_block(float *src, int8_t *dst, float *diff, size_t count, int *scale);
static dl_comp_return_t compress_block_sum2(const int8_t *invec, int8_t *inoutvec);

private:
dl_comp_return_t compress_block(float *src, int8_t *dst, float *diff, size_t count, int *scale);
dl_comp_return_t decompress_block(const int8_t *src, float *dst, size_t count, int scale);
dl_comp_return_t avx512_decompress_block(const int8_t *src, float *dst, size_t count, int scale);
dl_comp_return_t avx512_compress_block(float *src, int8_t *dst, float *diff, size_t count, int *scale);
dl_comp_return_t compress_block_sum(const int8_t *invec, int8_t *inoutvec);
dl_comp_return_t compress_block_sum2(const int8_t *invec, int8_t *inoutvec);
DLCompressDFP(): avx512_enabled_(dl_comp_check_avx512_supported()) {};

private:
bool avx512_enabled_;
Expand Down
10 changes: 5 additions & 5 deletions dlcp/src/dl_compression_util.cpp
View file
Expand Up @@ -26,11 +26,11 @@

int g_log_th = 0;

void dl_comp_get_time(char* buf, size_t bufSize)
void dl_comp_get_time(char* buf, size_t buf_size)
{
time_t timer;
struct tm* timeInfo = 0;
time(&timer);
timeInfo = localtime(&timer);
strftime(buf, bufSize, "%Y:%m:%d %H:%M:%S", timeInfo);
struct tm timeInfo;
timer = time(&timer);
localtime_r(&timer, &timeInfo);
strftime(buf, buf_size, "%Y:%m:%d %H:%M:%S", &timeInfo);
}
4 changes: 2 additions & 2 deletions dlcp/src/dl_compression_util.hpp
View file
Expand Up @@ -50,7 +50,7 @@ do { \
{ \
case ERROR: \
{ \
printf("%s: ERROR: (%ld): %s:%u " fmt "\n", time_buf, GET_TID(), \
printf("%s: ERROR: (%ld): %s:%d " fmt "\n", time_buf, GET_TID(), \
__FUNCTION__, __LINE__, ##__VA_ARGS__); \
break; \
} \
Expand All @@ -62,7 +62,7 @@ do { \
case DEBUG: \
case TRACE: \
{ \
printf("%s: (%ld): %s:%u " fmt "\n", time_buf, GET_TID(), \
printf("%s: (%ld): %s:%d " fmt "\n", time_buf, GET_TID(), \
__FUNCTION__, __LINE__, ##__VA_ARGS__); \
break; \
} \
Expand Down
62 changes: 61 additions & 1 deletion dlcp/test/main.cpp
View file
Expand Up @@ -25,6 +25,9 @@ bool test_decompress_buffer();

bool test_compressed_buffer_reduce_sum();

bool test_compressed_buffer_sum();

#if 0
void addVec(const float *vec1, const float *vec2, float *vec3, int count) {
for (int i = 0; i < count; i++) {
vec3[i] = vec1[i] + vec2[i];
Expand All @@ -38,7 +41,7 @@ void cmpVec(const float *vec1, const float *vec2, int count) {
}
}
}

#endif
float getSum(const float *src, int count) {
float sum = 0.0f;
for (int i = 0; i < count; i++) {
Expand All @@ -47,11 +50,13 @@ float getSum(const float *src, int count) {
return sum;
}

#if 0
void dumpVec(const float *vec, int count) {
for (int i = 0; i < count; i++) {
printf("vec[%d] = %lf\n", i, vec[i]);
}
}
#endif

float sumVec(const float *vec1, const float *vec2, int count) {
float sum = 0.0f;
Expand All @@ -62,6 +67,7 @@ float sumVec(const float *vec1, const float *vec2, int count) {
return sum;
}

#if 0
float sumVec2(const float *vec1, const float *vec2, int count) {
float sum = 0.0f;
for (int i = 0; i < count; i++) {
Expand All @@ -70,6 +76,7 @@ float sumVec2(const float *vec1, const float *vec2, int count) {
}
return sum;
}
#endif

int main(int argc, char *argv[])
{
Expand All @@ -93,6 +100,12 @@ int main(int argc, char *argv[])
printf("test_compressed_buffer_reduce_sum successful!\n");
}

if (!test_compressed_buffer_sum()) {
printf("test_compressed_buffer_sum failure!\n");
} else {
printf("test_compressed_buffer_sum successful!\n");
}

return 0;
}

Expand Down Expand Up @@ -291,3 +304,50 @@ bool test_compressed_buffer_reduce_sum()
free(sum3);
return true;
}

bool test_compressed_buffer_sum()
{
float *tempData1 = (float *)malloc(sizeof(float) * DATA_LEN);
float *tempData2 = (float *)malloc(sizeof(float) * DATA_LEN);
float *sum1 = (float *)malloc(sizeof(float) * DATA_LEN);

memcpy(tempData1, data1, sizeof(float) * DATA_LEN);
memcpy(tempData2, data2, sizeof(float) * DATA_LEN);

dl_comp_get_sizeof_block(DL_COMP_FLOAT32,
4,
DL_COMP_DFP);

dl_comp_return_t ret = dl_comp_compress_buffer((const void *)tempData1,
tempData1,
DATA_LEN,
NULL,
DL_COMP_FLOAT32,
4,
DL_COMP_DFP);

if (ret != DL_COMP_OK) {
printf("compress failed error = %d!\n", ret);
free(tempData1);
free(tempData2);
free(sum1);
return false;
}

dl_comp_compress_buffer_FLOAT32ToINT8((const void *)tempData2,
tempData2,
NULL,
DATA_LEN);

dl_comp_compressed_buffer_sum(tempData1,
tempData2,
DATA_LEN,
sum1);

dl_comp_get_elem_num_in_block();

dl_comp_decompress_buffer_INT8ToFLOAT32(sum1, sum1, DATA_LEN);

return true;

}

0 comments on commit adf5bfb

Please sign in to comment.