Skip to content

Commit

Permalink
more tests and a few fixes for mem motion
Browse files Browse the repository at this point in the history
  • Loading branch information
@cyrush
cyrush committed Jun 22, 2021
1 parent abce893 commit db8a965
Show file tree
Hide file tree
Showing 6 changed files with 204 additions and 50 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s

## Unreleased

### Added

#### General
- Added support to register custom memory allocators and a custom data movement handler. This allows conduit to move trees of data between heterogenous memory spaces (e.g. CPU and GPU memory). See conduit_utils.hpp for API details.


### Fixed

#### General
Expand Down
18 changes: 8 additions & 10 deletions src/libs/conduit/conduit_data_array.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1625,16 +1625,14 @@ void
DataArray<T>::compact_elements_to(uint8 *data) const
{
// copy all elements
index_t num_ele = m_dtype.number_of_elements();
index_t ele_bytes = DataType::default_bytes(m_dtype.id());
uint8 *data_ptr = data;
for(index_t i=0;i<num_ele;i++)
{
memcpy(data_ptr,
element_ptr(i),
(size_t)ele_bytes);
data_ptr+=ele_bytes;
}
index_t ele_bytes = DataType::default_bytes(dtype().id());

utils::conduit_memcpy_strided_elements(data, // dest
(size_t)dtype().number_of_elements(), // num eles to copy
ele_bytes, // bytes per element
ele_bytes, // dest stride per ele
element_ptr(0), // src
(size_t)dtype().stride()); // src stride per ele
}


Expand Down
44 changes: 27 additions & 17 deletions src/libs/conduit/conduit_node.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8564,14 +8564,19 @@ Node::update(const Node &n_src)
(this->dtype().number_of_elements() >=
n_src.dtype().number_of_elements()))
{
for(index_t idx = 0;
idx < n_src.dtype().number_of_elements();
idx++)
{
memcpy(element_ptr(idx),
n_src.element_ptr(idx),
(size_t)this->dtype().element_bytes());
}
size_t ele_bytes = (size_t) dtype().element_bytes();
size_t stride = (size_t) dtype().stride();
size_t num_ele = (size_t) n_src.dtype().number_of_elements();
size_t src_stride = (size_t) n_src.dtype().stride();
//
// Note: conduit_memcpy_strided_elements will use a single
// memcpy when src and dest are compactly strided
utils::conduit_memcpy_strided_elements(element_ptr(0), // dest ptr
num_ele, // num ele
ele_bytes, // dest bytes per ele
stride, // dest stride
n_src.element_ptr(0), // src ptr
src_stride); // src stride
}
else // not compatible
{
Expand Down Expand Up @@ -8628,14 +8633,19 @@ Node::update_compatible(const Node &n_src)
(this->dtype().number_of_elements() >=
n_src.dtype().number_of_elements()))
{
for(index_t idx = 0;
idx < n_src.dtype().number_of_elements();
idx++)
{
memcpy(element_ptr(idx),
n_src.element_ptr(idx),
(size_t)this->dtype().element_bytes());
}
size_t ele_bytes = (size_t) dtype().element_bytes();
size_t stride = (size_t) dtype().stride();
size_t num_ele = (size_t) n_src.dtype().number_of_elements();
size_t src_stride = (size_t) n_src.dtype().stride();
//
// Note: conduit_memcpy_strided_elements will use a single
// memcpy when src and dest are compactly strided
utils::conduit_memcpy_strided_elements(element_ptr(0), // dest ptr
num_ele, // num ele
ele_bytes, // dest bytes per ele
stride, // dest stride
n_src.element_ptr(0), // src ptr
src_stride); // src stride
}
}
}
Expand Down Expand Up @@ -12990,7 +13000,7 @@ Node::to_base64_json(std::ostream &os,
// and on the host, so we can use it directly in utils::base64_encode
const char *src_ptr = (const char*)n.data_ptr();
char *dest_ptr = (char*)bb64_data.data_ptr();
memset(dest_ptr,0,(size_t)enc_buff_size);
utils::conduit_memset(dest_ptr,0,(size_t)enc_buff_size);

utils::base64_encode(src_ptr,nbytes,dest_ptr);

Expand Down
16 changes: 8 additions & 8 deletions src/libs/conduit/conduit_utils.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,10 +269,10 @@ namespace utils
//-----------------------------------------------------------------------------

// conduit uses a single pair of memset and memcpy functions to
// manage data movement.
// manage data movement.

// this strategy allows downstream users to support complex cases
// like moving between memory spaces not accessible on the host.
// like moving between memory spaces not accessible on the host.
//
// These methods aren't bound to allocators b/c allocators
// won't be tied into all of the places where source and dest pointers
Expand All @@ -294,8 +294,8 @@ namespace utils
size_t num);

// general memcpy interface used by conduit
void CONDUIT_API conduit_memcpy(void * destination,
const void * source,
void CONDUIT_API conduit_memcpy(void *destination,
const void *source,
size_t num);

void CONDUIT_API conduit_memcpy_strided_elements(void *dest,
Expand All @@ -305,10 +305,10 @@ namespace utils
const void *src,
size_t src_stride);

// general memset interface used by conduit
// I note that the default memset returns the orig pointer, but
// other allocators like cuda do not
// CYRUS TODO: GIVEN THIS DO WE NEED TO PASS AN ALLOC_ID?
// general memset interface used by conduit
// NOTE (cyrush): The default memset returns the orig pointer, but
// other allocators like cuda do not.
// TODO: GIVEN THIS DO WE NEED TO PASS AN ALLOC_ID?
void CONDUIT_API conduit_memset(void * ptr,
int value,
size_t num);
Expand Down
61 changes: 46 additions & 15 deletions src/tests/conduit/t_conduit_array.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ TEST(conduit_array, basic_construction)
da_3[0] = -16;

std::cout << da_3.to_string() << std::endl;

// test other variants of to_string and to stream, etc
da_3.to_string_stream(std::cout);
da_3.to_json_stream(std::cout);
Expand Down Expand Up @@ -90,7 +90,7 @@ TEST(conduit_array, array_stride_int8)
std::cout << (int64) data[i] << " ";
}
std::cout << std::endl;

DataType arr_t(DataType::INT8_ID,
10,
0,
Expand All @@ -115,7 +115,7 @@ TEST(conduit_array, array_stride_int8)

arr[1] = 100;
EXPECT_EQ(data[2],100);

std::cout << "Full Data" << std::endl;

for(int i=0;i<20;i++)
Expand All @@ -130,18 +130,18 @@ TEST(conduit_array, array_stride_int8)
true); /// true for external

int8_array arr_2 = n2.as_int8_array();

for(int i=0;i<10;i++)
{
// note: the cast is for proper printing to std::out
std::cout << "value[" << i << "] = " << ((int64)arr_2[i] ) << std::endl;
}
std::cout << std::endl;

EXPECT_EQ(arr_2[0],0);
EXPECT_EQ(arr_2[9],-9);
EXPECT_EQ(arr_2[9],-9);

}
}

//-----------------------------------------------------------------------------
TEST(conduit_array, array_stride_int8_external)
Expand All @@ -164,7 +164,7 @@ TEST(conduit_array, array_stride_int8_external)
std::cout << (int64) data[i] << " ";
}
std::cout << std::endl;

Node n;
n["value"].set_external(data);

Expand Down Expand Up @@ -208,7 +208,7 @@ TEST(conduit_array, set_using_ptrs)


Node n;

// int8_array
n["vint8"].set(DataType::int8(10));
n["vint8"].as_int8_array().set(&v_int8[0],10);
Expand Down Expand Up @@ -334,7 +334,7 @@ TEST(conduit_array, set_using_data_array)


Node n;

// int8_array
n["vint8"].set(DataType::int8(10));
n["vint8"].as_int8_array().set(va_int8);
Expand Down Expand Up @@ -448,7 +448,7 @@ TEST(conduit_array, set_using_std_vectors)


Node n;

// int8_array
n["vint8"].set(DataType::int8(10));
n["vint8"].as_int8_array().set(v_int8);
Expand Down Expand Up @@ -702,7 +702,33 @@ TEST(conduit_array, fill)

}

//-----------------------------------------------------------------------------
TEST(conduit_array, compact_to_bytes)
{
std::vector<int64> vals(8,0);

vals[0] = 3;
vals[2] = 7;
vals[4] = 9;
vals[6] = 11;

// stride every 16 bytes (2 int64s)
int64_array varray(&vals[0],
DataType::int64(4,0,16));

std::cout << varray.to_string() << std::endl;

uint8 buff[64*4];

varray.compact_elements_to(buff);

int64 *vals_cpt = (int64*)buff;

EXPECT_EQ(vals_cpt[0],3);
EXPECT_EQ(vals_cpt[1],7);
EXPECT_EQ(vals_cpt[2],9);
EXPECT_EQ(vals_cpt[3],11);
}

//-----------------------------------------------------------------------------
#ifdef CONDUIT_USE_CXX11
Expand All @@ -718,7 +744,7 @@ TEST(conduit_array, summary_stats)
int64_array va_int64(&v_int64[0],DataType::int64(3));
uint64_array va_uint64(&v_uint64[0],DataType::uint64(3));
float64_array va_float64(&v_float64[0],DataType::float64(3));

va_int64.set({-1,0,1});
va_uint64.set({1,2,3});
va_float64.set({-1.0,0.0,1.0});
Expand All @@ -739,6 +765,8 @@ TEST(conduit_array, summary_stats)
EXPECT_EQ(va_float64.sum(),0.0);

}


//-----------------------------------------------------------------------------
TEST(conduit_array, summary_print)
{
Expand Down Expand Up @@ -884,7 +912,7 @@ TEST(conduit_array, cxx_11_init_lists)
EXPECT_EQ(va_int8[1],2);
EXPECT_EQ(va_int8[2],3);
va_int8.print();

va_int8 = {1ul,2ul,3ul};
va_int8.print();
EXPECT_EQ(va_int8[0],1);
Expand Down Expand Up @@ -1148,7 +1176,7 @@ TEST(conduit_array, cxx_11_init_lists)
va_uint8 = {1.0,2.0,3.0};
va_uint8.print();
}

// uint 16
{
va_uint16.set({1,2,3});
Expand Down Expand Up @@ -1373,7 +1401,7 @@ TEST(conduit_array, cxx_11_init_lists)
va_float32 = {1.0,2.0,3.0};
va_float32.print();
}

// float 64
{
va_float64.set({-1,-2,-3});
Expand Down Expand Up @@ -1424,3 +1452,6 @@ TEST(conduit_array, cxx_11_init_lists)






Loading

0 comments on commit db8a965

Please sign in to comment.