Skip to content

F backend batch support #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
LinusJungemann merged 11 commits into from
Mar 14, 2024
Merged

F backend batch support #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
6d0f8db
Remove hardcoded batch size from kernel execution
LinusJungemann Feb 23, 2024
ddb376f
Implement setBatchSize for complete Stack
LinusJungemann Feb 23, 2024
aec895b
Remove RingBuffer from Synchronous Inference and add full batch mapping
LinusJungemann Feb 28, 2024
2e67f97
Deduplicate batchsize in basedriver & fix unittests
LinusJungemann Feb 29, 2024
407234a
Fix integrationtests
LinusJungemann Feb 29, 2024
5aba44b
Change input kernel code to run concurrrently to output kernel code
LinusJungemann Mar 1, 2024
c072653
Optimize inference of lower batch sizes
LinusJungemann Mar 6, 2024
9996054
Increase packing performance
LinusJungemann Mar 7, 2024
5dafa7b
Further optimize OpenMP
LinusJungemann Mar 7, 2024
47d5eda
Optimize Utils
LinusJungemann Mar 8, 2024
1c95f78
Some small changes
LinusJungemann Mar 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions benchmarks/DeviceBufferBenchmark.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ static void BM_VecCopyBase(benchmark::State& state) {
auto idb = Finn::SyncDeviceInputBuffer<uint8_t>("Tester", device, kernel, myShapePacked, benchmarkBufferSize);
auto dat1 = std::vector<uint8_t>();
auto dat2 = std::vector<uint8_t>();
dat1.resize(idb.size(SIZE_SPECIFIER::ELEMENTS_PER_PART));
dat2.resize(idb.size(SIZE_SPECIFIER::ELEMENTS_PER_PART));
dat1.resize(idb.size(SIZE_SPECIFIER::FEATUREMAP_SIZE));
dat2.resize(idb.size(SIZE_SPECIFIER::FEATUREMAP_SIZE));
filler.fillRandom(dat1);
filler.fillRandom(dat2);
for (auto _ : state) {
Expand All @@ -60,7 +60,7 @@ static void BM_StoreCVRP(benchmark::State& state) {
auto kernel = xrt::kernel();
auto idb = Finn::SyncDeviceInputBuffer<uint8_t>("Tester", device, kernel, myShapePacked, benchmarkBufferSize);
Finn::vector<uint8_t> data;
data.resize(idb.size(SIZE_SPECIFIER::ELEMENTS_PER_PART));
data.resize(idb.size(SIZE_SPECIFIER::FEATUREMAP_SIZE));
for (auto _ : state) {
for (unsigned int i = 0; i < benchmarkBufferSize; i++) {
filler.fillRandom(data.begin(), data.end());
Expand All @@ -77,7 +77,7 @@ static void BM_StoreSVRP(benchmark::State& state) {
auto kernel = xrt::kernel();
auto idb = Finn::SyncDeviceInputBuffer<uint8_t>("Tester", device, kernel, myShapePacked, benchmarkBufferSize);
std::vector<uint8_t> data;
data.resize(idb.size(SIZE_SPECIFIER::ELEMENTS_PER_PART));
data.resize(idb.size(SIZE_SPECIFIER::FEATUREMAP_SIZE));
for (auto _ : state) {
for (unsigned int i = 0; i < benchmarkBufferSize; i++) {
filler.fillRandom(data);
Expand All @@ -94,7 +94,7 @@ static void BM_StoreCVIP(benchmark::State& state) {
auto kernel = xrt::kernel();
auto idb = Finn::SyncDeviceInputBuffer<uint8_t>("Tester", device, kernel, myShapePacked, benchmarkBufferSize);
Finn::vector<uint8_t> data;
data.resize(idb.size(SIZE_SPECIFIER::ELEMENTS_PER_PART));
data.resize(idb.size(SIZE_SPECIFIER::FEATUREMAP_SIZE));
for (auto _ : state) {
for (unsigned int i = 0; i < benchmarkBufferSize; i++) {
idb.store(data);
Expand All @@ -110,7 +110,7 @@ static void BM_StoreSVIP(benchmark::State& state) {
auto kernel = xrt::kernel();
auto idb = Finn::SyncDeviceInputBuffer<uint8_t>("Tester", device, kernel, myShapePacked, benchmarkBufferSize);
Finn::vector<uint8_t> data;
data.resize(idb.size(SIZE_SPECIFIER::ELEMENTS_PER_PART));
data.resize(idb.size(SIZE_SPECIFIER::FEATUREMAP_SIZE));
for (auto _ : state) {
for (unsigned int i = 0; i < benchmarkBufferSize; i++) {
idb.store(data);
Expand All @@ -125,9 +125,9 @@ static void BM_StoreFCVRP(benchmark::State& state) {
auto device = xrt::device();
auto kernel = xrt::kernel();
auto idb = Finn::DeviceInputBuffer<uint8_t>("Tester", device, kernel, myShapePacked, benchmarkBufferSize);
const auto bytesPerSample = idb.size(SIZE_SPECIFIER::ELEMENTS_PER_PART);
const auto bytesPerSample = idb.size(SIZE_SPECIFIER::FEATUREMAP_SIZE);
std::vector<uint8_t> data;
data.resize(idb.size(SIZE_SPECIFIER::ELEMENTS_PER_PART));
data.resize(idb.size(SIZE_SPECIFIER::FEATUREMAP_SIZE));
for (auto _ : state) {
for (unsigned int i = 0; i < benchmarkBufferSize; i++) {
idb.storeFast(data);
Expand All @@ -142,9 +142,9 @@ static void BM_StoreFCVIP(benchmark::State& state) {
auto device = xrt::device();
auto kernel = xrt::kernel();
auto idb = Finn::DeviceInputBuffer<uint8_t>("Tester", device, kernel, myShapePacked, benchmarkBufferSize);
const auto bytesPerSample = idb.size(SIZE_SPECIFIER::ELEMENTS_PER_PART);
const auto bytesPerSample = idb.size(SIZE_SPECIFIER::FEATUREMAP_SIZE);
std::vector<uint8_t> data;
data.resize(idb.size(SIZE_SPECIFIER::ELEMENTS_PER_PART));
data.resize(idb.size(SIZE_SPECIFIER::FEATUREMAP_SIZE));
for (auto _ : state) {
for (unsigned int i = 0; i < benchmarkBufferSize; i++) {
idb.storeFast(data.begin(), data.end());
Expand Down Expand Up @@ -177,7 +177,7 @@ static void BM_StoreMultithreaded_CVRP(benchmark::State& state) {
datas[i].resize(countSamplesPerThread);
for (unsigned int j = 0; j < countSamplesPerThread; j++) {
datas[i][j] = Sample();
datas[i][j].resize(idb.size(SIZE_SPECIFIER::ELEMENTS_PER_PART));
datas[i][j].resize(idb.size(SIZE_SPECIFIER::FEATUREMAP_SIZE));
filler.fillRandom(datas[i][j].begin(), datas[i][j].end());
}
}
Expand Down
8 changes: 4 additions & 4 deletions integrationtest/AsyncInference.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ using namespace std::literals::chrono_literals;

// auto driver = Finn::Driver<false>(conf, 10, 0, conf.deviceWrappers[0].idmas[0]->kernelName, 0, conf.deviceWrappers[0].odmas[0]->kernelName, 1, true);

// Finn::vector<int8_t> data(driver.size(SIZE_SPECIFIER::ELEMENTS_PER_PART, 0, conf.deviceWrappers[0].idmas[0]->kernelName), 1);
// Finn::vector<int8_t> data(driver.size(SIZE_SPECIFIER::FEATUREMAP_SIZE, 0, conf.deviceWrappers[0].idmas[0]->kernelName), 1);

// std::iota(data.begin(), data.end(), -127);

Expand All @@ -53,11 +53,11 @@ using namespace std::literals::chrono_literals;

// auto driver = Finn::Driver<false>(conf, static_cast<uint>(batchLength), 0, conf.deviceWrappers[0].idmas[0]->kernelName, 0, conf.deviceWrappers[0].odmas[0]->kernelName, static_cast<uint>(batchLength), true);

// Finn::vector<int8_t> data(driver.size(SIZE_SPECIFIER::ELEMENTS_PER_PART, 0, conf.deviceWrappers[0].idmas[0]->kernelName) * batchLength, 1);
// Finn::vector<int8_t> data(driver.size(SIZE_SPECIFIER::FEATUREMAP_SIZE, 0, conf.deviceWrappers[0].idmas[0]->kernelName) * batchLength, 1);

// for (std::size_t i = 0; i < batchLength; ++i) {
// std::iota(data.begin() + static_cast<decltype(data)::difference_type>(i * driver.size(SIZE_SPECIFIER::ELEMENTS_PER_PART, 0, conf.deviceWrappers[0].idmas[0]->kernelName)),
// data.begin() + static_cast<decltype(data)::difference_type>((i + 1) * driver.size(SIZE_SPECIFIER::ELEMENTS_PER_PART, 0, conf.deviceWrappers[0].idmas[0]->kernelName)), -127);
// std::iota(data.begin() + static_cast<decltype(data)::difference_type>(i * driver.size(SIZE_SPECIFIER::FEATUREMAP_SIZE, 0, conf.deviceWrappers[0].idmas[0]->kernelName)),
// data.begin() + static_cast<decltype(data)::difference_type>((i + 1) * driver.size(SIZE_SPECIFIER::FEATUREMAP_SIZE, 0, conf.deviceWrappers[0].idmas[0]->kernelName)), -127);
// }

// // Run inference
Expand Down
12 changes: 6 additions & 6 deletions integrationtest/SyncInference.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ TEST(SyncInference, syncInferenceTest) {
std::string exampleNetworkConfig = "config.json";
Finn::Config conf = Finn::createConfigFromPath(exampleNetworkConfig);

auto driver = Finn::Driver<true>(conf, 10, 0, conf.deviceWrappers[0].idmas[0]->kernelName, 0, conf.deviceWrappers[0].odmas[0]->kernelName, 1, true);
auto driver = Finn::Driver<true>(conf, 0, conf.deviceWrappers[0].idmas[0]->kernelName, 0, conf.deviceWrappers[0].odmas[0]->kernelName, 1, true);

Finn::vector<int8_t> data(driver.size(SIZE_SPECIFIER::ELEMENTS_PER_PART, 0, conf.deviceWrappers[0].idmas[0]->kernelName), 1);
Finn::vector<int8_t> data(driver.size(SIZE_SPECIFIER::FEATUREMAP_SIZE, 0, conf.deviceWrappers[0].idmas[0]->kernelName), 1);

std::iota(data.begin(), data.end(), -127);

Expand All @@ -46,13 +46,13 @@ TEST(SyncInference, syncBatchInferenceTest) {
Finn::Config conf = Finn::createConfigFromPath(exampleNetworkConfig);
std::size_t batchLength = 10;

auto driver = Finn::Driver<true>(conf, static_cast<uint>(batchLength), 0, conf.deviceWrappers[0].idmas[0]->kernelName, 0, conf.deviceWrappers[0].odmas[0]->kernelName, static_cast<uint>(batchLength), true);
auto driver = Finn::Driver<true>(conf, 0, conf.deviceWrappers[0].idmas[0]->kernelName, 0, conf.deviceWrappers[0].odmas[0]->kernelName, static_cast<uint>(batchLength), true);

Finn::vector<int8_t> data(driver.size(SIZE_SPECIFIER::ELEMENTS_PER_PART, 0, conf.deviceWrappers[0].idmas[0]->kernelName) * batchLength, 1);
Finn::vector<int8_t> data(driver.size(SIZE_SPECIFIER::FEATUREMAP_SIZE, 0, conf.deviceWrappers[0].idmas[0]->kernelName) * batchLength, 1);

for (std::size_t i = 0; i < batchLength; ++i) {
std::iota(data.begin() + static_cast<decltype(data)::difference_type>(i * driver.size(SIZE_SPECIFIER::ELEMENTS_PER_PART, 0, conf.deviceWrappers[0].idmas[0]->kernelName)),
data.begin() + static_cast<decltype(data)::difference_type>((i + 1) * driver.size(SIZE_SPECIFIER::ELEMENTS_PER_PART, 0, conf.deviceWrappers[0].idmas[0]->kernelName)), -127);
std::iota(data.begin() + static_cast<decltype(data)::difference_type>(i * driver.size(SIZE_SPECIFIER::FEATUREMAP_SIZE, 0, conf.deviceWrappers[0].idmas[0]->kernelName)),
data.begin() + static_cast<decltype(data)::difference_type>((i + 1) * driver.size(SIZE_SPECIFIER::FEATUREMAP_SIZE, 0, conf.deviceWrappers[0].idmas[0]->kernelName)), -127);
}

// Run inference
Expand Down
Loading