Skip to content

CTF parser and CTF chunkdataset #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 11 commits into from
Closed

CTF parser and CTF chunkdataset #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
302d0e6
Adding chunk dataset API and tests
jaliyae Dec 27, 2018
e845c84
Add CTFParser and CTFChunkDataset support with unit tests
Nov 28, 2018
4ec4123
Add template to CTF parser and ChunkDataset implementation
Dec 14, 2018
0319cf2
Refactor CTFParser and CTFChunkDataset to improve performance
Dec 16, 2018
1b5f23b
Improve CTF parsing performance by reserving capacity for vectors
Dec 17, 2018
bcf14f3
Implement buffering for ctf parsing
Jan 2, 2019
9b0853f
Improve CTF Reader performance
Jan 5, 2019
f7d280f
Improve CTF Parser performance
Jan 5, 2019
6ce6a6e
Refactor to improve CTF Parser performance
Jan 7, 2019
7a586e4
Improve CTF Reader as per jaliyae comment
Jan 11, 2019
0646b4c
Change parser to store data per stream instead of per sample
Jan 25, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions test/cpp/api/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,16 @@ set(TORCH_API_TEST_SOURCES
${TORCH_ROOT}/test/cpp/common/main.cpp
${TORCH_API_TEST_DIR}/any.cpp
${TORCH_API_TEST_DIR}/dataloader.cpp
${TORCH_API_TEST_DIR}/data/ctf/ctf_sample_classification.cpp
${TORCH_API_TEST_DIR}/data/ctf/ctf_sample_comments.cpp
${TORCH_API_TEST_DIR}/data/ctf/ctf_sample_dssm.cpp
${TORCH_API_TEST_DIR}/data/ctf/ctf_sample_empty_values.cpp
${TORCH_API_TEST_DIR}/data/ctf/ctf_sample_exponent_values.cpp
${TORCH_API_TEST_DIR}/data/ctf/ctf_sample_generic.cpp
${TORCH_API_TEST_DIR}/data/ctf/ctf_sample_learning_to_rank.cpp
${TORCH_API_TEST_DIR}/data/ctf/ctf_sample_part_of_speech_tagging.cpp
${TORCH_API_TEST_DIR}/data/ctf/ctf_sample_sequence_classification.cpp
${TORCH_API_TEST_DIR}/data/ctf/ctf_sample_sequence_to_sequence.cpp
${TORCH_API_TEST_DIR}/expanding-array.cpp
${TORCH_API_TEST_DIR}/integration.cpp
${TORCH_API_TEST_DIR}/jit.cpp
Expand Down
126 changes: 126 additions & 0 deletions test/cpp/api/data/ctf/ctf_sample_classification.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
#include <gtest/gtest.h>

#include <test/cpp/api/data/ctf/samples/ctf_samples.h>
#include <torch/data/ctf/ctf_parser.h>

/// Tests must be executed from root directory of the repo
/// Order of CTFValue<int16_t>s inside CTFSample are important

namespace torch {
namespace data {
namespace ctf {

TEST(DataTest, CTF_SAMPLE_CLASSIFICATION_SUCCESS) {
/// Actual data
std::vector<CTFInputStreamInformation> input_streams;
input_streams.emplace_back(
0,
"features",
"features",
5,
CTFInputStreamType::Feature,
CTFDataStorage::Dense);
input_streams.emplace_back(
1,
"class",
"class",
0,
CTFInputStreamType::Label,
CTFDataStorage::Sparse);
CTFConfiguration config(
std::string(CTF_SAMPLE_DIR + "/ctf_sample_classification.ctf"),
input_streams,
CTFDataType(CTFDataType::Int16));

CTFParser<int16_t> ctf_parser(config);
ctf_parser.read_from_file();

/// Expected data
CTFDataset<int16_t> dataset(CTFDataType::Int16, input_streams);
#ifdef CTF_DEBUG
size_t sequence_id = 0;
#endif
size_t input_stream_id = 0;

{
// 0 (implicit)
#ifdef CTF_DEBUG
sequence_id = 0;
#endif
CTFSequenceData sequence;
input_stream_id = 0;
sequence.emplace_back(std::make_shared<CTFDenseInputStreamData<int16_t>>(
input_stream_id, input_streams[input_stream_id].dimension));
input_stream_id = 1;
sequence.emplace_back(std::make_shared<CTFSparseInputStreamData<int16_t>>(
input_stream_id, input_streams[input_stream_id].dimension));
{
input_stream_id = 1;
auto sparse_stream_ptr =
static_cast<CTFSparseInputStreamData<int16_t>*>(
sequence[input_stream_id].get());
// |class 23:1
sparse_stream_ptr->data.push_back(1);
sparse_stream_ptr->indices.push_back(23);
}
{
input_stream_id = 0;
auto dense_stream_ptr = static_cast<CTFDenseInputStreamData<int16_t>*>(
sequence[input_stream_id].get());
// |features 2 3 4 5 6
dense_stream_ptr->data.push_back(2);
dense_stream_ptr->data.push_back(3);
dense_stream_ptr->data.push_back(4);
dense_stream_ptr->data.push_back(5);
dense_stream_ptr->data.push_back(6);
}
dataset.sequences.push_back(sequence);
#ifdef CTF_DEBUG
dataset.sequences_id.push_back(sequence_id);
#endif
}

{
// 1 (implicit)
#ifdef CTF_DEBUG
sequence_id = 1;
#endif
CTFSequenceData sequence;
input_stream_id = 0;
sequence.emplace_back(std::make_shared<CTFDenseInputStreamData<int16_t>>(
input_stream_id, input_streams[input_stream_id].dimension));
input_stream_id = 1;
sequence.emplace_back(std::make_shared<CTFSparseInputStreamData<int16_t>>(
input_stream_id, input_streams[input_stream_id].dimension));

{
input_stream_id = 1;
auto sparse_stream_ptr =
static_cast<CTFSparseInputStreamData<int16_t>*>(
sequence[input_stream_id].get());
// |class 13:1
sparse_stream_ptr->data.push_back(1);
sparse_stream_ptr->indices.push_back(13);
}
{
input_stream_id = 0;
auto dense_stream_ptr = static_cast<CTFDenseInputStreamData<int16_t>*>(
sequence[input_stream_id].get());
// |features 1 2 0 2 3
dense_stream_ptr->data.push_back(1);
dense_stream_ptr->data.push_back(2);
dense_stream_ptr->data.push_back(0);
dense_stream_ptr->data.push_back(2);
dense_stream_ptr->data.push_back(3);
}
dataset.sequences.push_back(sequence);
#ifdef CTF_DEBUG
dataset.sequences_id.push_back(sequence_id);
#endif
}

EXPECT_TRUE(*ctf_parser.get_dataset() == dataset);
}
} // namespace ctf
} // namespace data
} // namespace torch
189 changes: 189 additions & 0 deletions test/cpp/api/data/ctf/ctf_sample_comments.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,189 @@
#include <gtest/gtest.h>

#include <test/cpp/api/data/ctf/samples/ctf_samples.h>
#include <torch/data/ctf/ctf_parser.h>

/// Tests must be executed from root directory of the repo
/// Order of CTFValue<float>s inside CTFSample are important

namespace torch {
namespace data {
namespace ctf {

TEST(DataTest, CTF_SAMPLE_COMMENTS_SUCCESS) {
/// Actual data
std::vector<CTFInputStreamInformation> input_streams;
input_streams.emplace_back(
0, "A", "A", 5, CTFInputStreamType::Feature, CTFDataStorage::Dense);
input_streams.emplace_back(
1, "B", "B", 0, CTFInputStreamType::Feature, CTFDataStorage::Sparse);
input_streams.emplace_back(
2, "C", "C", 1, CTFInputStreamType::Label, CTFDataStorage::Dense);
CTFConfiguration config(
std::string(CTF_SAMPLE_DIR + "/ctf_sample_comments.ctf"),
input_streams,
CTFDataType(CTFDataType::Float));

CTFParser<float> ctf_parser(config);
ctf_parser.read_from_file();

/// Expected data
CTFDataset<float> dataset(CTFDataType::Float, input_streams);
#ifdef CTF_DEBUG
size_t sequence_id = 0;
#endif
size_t input_stream_id = 0;

{
// 0 (implicit)
#ifdef CTF_DEBUG
sequence_id = 0;
#endif
CTFSequenceData sequence;
input_stream_id = 0;
sequence.emplace_back(std::make_shared<CTFDenseInputStreamData<float>>(
input_stream_id, input_streams[input_stream_id].dimension));
input_stream_id = 1;
sequence.emplace_back(std::make_shared<CTFSparseInputStreamData<float>>(
input_stream_id, input_streams[input_stream_id].dimension));
input_stream_id = 2;
sequence.emplace_back(std::make_shared<CTFDenseInputStreamData<float>>(
input_stream_id, input_streams[input_stream_id].dimension));
{
input_stream_id = 1;
auto sparse_stream_ptr = static_cast<CTFSparseInputStreamData<float>*>(
sequence[input_stream_id].get());
// |B 100:3 123:4
sparse_stream_ptr->data.push_back(3);
sparse_stream_ptr->indices.push_back(100);
sparse_stream_ptr->data.push_back(4);
sparse_stream_ptr->indices.push_back(123);
}
{
input_stream_id = 2;
auto dense_stream_ptr = static_cast<CTFDenseInputStreamData<float>*>(
sequence[input_stream_id].get());
// |C 8
dense_stream_ptr->data.push_back(8);
}
{
input_stream_id = 0;
auto dense_stream_ptr = static_cast<CTFDenseInputStreamData<float>*>(
sequence[input_stream_id].get());
// |A 0 1 2 3 4
dense_stream_ptr->data.push_back(0);
dense_stream_ptr->data.push_back(1);
dense_stream_ptr->data.push_back(2);
dense_stream_ptr->data.push_back(3);
dense_stream_ptr->data.push_back(4);
}
dataset.sequences.push_back(sequence);
#ifdef CTF_DEBUG
dataset.sequences_id.push_back(sequence_id);
#endif
}

{
// 1 (implicit)
#ifdef CTF_DEBUG
sequence_id = 1;
#endif
CTFSequenceData sequence;
input_stream_id = 0;
sequence.emplace_back(std::make_shared<CTFDenseInputStreamData<float>>(
input_stream_id, input_streams[input_stream_id].dimension));
input_stream_id = 1;
sequence.emplace_back(std::make_shared<CTFSparseInputStreamData<float>>(
input_stream_id, input_streams[input_stream_id].dimension));
input_stream_id = 2;
sequence.emplace_back(std::make_shared<CTFDenseInputStreamData<float>>(
input_stream_id, input_streams[input_stream_id].dimension));
{
input_stream_id = 0;
auto dense_stream_ptr = static_cast<CTFDenseInputStreamData<float>*>(
sequence[input_stream_id].get());
// |A 0 1.1 22 0.3 54
dense_stream_ptr->data.push_back(0);
dense_stream_ptr->data.push_back(1.1);
dense_stream_ptr->data.push_back(22);
dense_stream_ptr->data.push_back(0.3);
dense_stream_ptr->data.push_back(54);
}
{
input_stream_id = 2;
auto dense_stream_ptr = static_cast<CTFDenseInputStreamData<float>*>(
sequence[input_stream_id].get());
// |C 123917
dense_stream_ptr->data.push_back(123917);
}
{
input_stream_id = 1;
auto sparse_stream_ptr = static_cast<CTFSparseInputStreamData<float>*>(
sequence[input_stream_id].get());
// |B 1134:1.911 13331:0.014
sparse_stream_ptr->data.push_back(1.911);
sparse_stream_ptr->indices.push_back(1134);
sparse_stream_ptr->data.push_back(0.014);
sparse_stream_ptr->indices.push_back(13331);
}
dataset.sequences.push_back(sequence);
#ifdef CTF_DEBUG
dataset.sequences_id.push_back(sequence_id);
#endif
}

{
// 2 (implicit)
#ifdef CTF_DEBUG
sequence_id = 2;
#endif
CTFSequenceData sequence;
input_stream_id = 0;
sequence.emplace_back(std::make_shared<CTFDenseInputStreamData<float>>(
input_stream_id, input_streams[input_stream_id].dimension));
input_stream_id = 1;
sequence.emplace_back(std::make_shared<CTFSparseInputStreamData<float>>(
input_stream_id, input_streams[input_stream_id].dimension));
input_stream_id = 2;
sequence.emplace_back(std::make_shared<CTFDenseInputStreamData<float>>(
input_stream_id, input_streams[input_stream_id].dimension));
{
input_stream_id = 2;
auto dense_stream_ptr = static_cast<CTFDenseInputStreamData<float>*>(
sequence[input_stream_id].get());
// |C -0.001
dense_stream_ptr->data.push_back(-0.001);
}
{
input_stream_id = 0;
auto dense_stream_ptr = static_cast<CTFDenseInputStreamData<float>*>(
sequence[input_stream_id].get());
// |A 3.9 1.11 121.2 99.13 0.04
dense_stream_ptr->data.push_back(3.9);
dense_stream_ptr->data.push_back(1.11);
dense_stream_ptr->data.push_back(121.2);
dense_stream_ptr->data.push_back(99.13);
dense_stream_ptr->data.push_back(0.04);
}
{
input_stream_id = 1;
auto sparse_stream_ptr = static_cast<CTFSparseInputStreamData<float>*>(
sequence[input_stream_id].get());
// |B 999:0.001 918918:-9.19
sparse_stream_ptr->data.push_back(0.001);
sparse_stream_ptr->indices.push_back(999);
sparse_stream_ptr->data.push_back(-9.19);
sparse_stream_ptr->indices.push_back(918918);
}
dataset.sequences.push_back(sequence);
#ifdef CTF_DEBUG
dataset.sequences_id.push_back(sequence_id);
#endif
}

EXPECT_TRUE(*ctf_parser.get_dataset() == dataset);
}

} // namespace ctf
} // namespace data
} // namespace torch
Loading