Skip to content

Commit

Permalink
Shugeo release fix1 (#61)
Browse files Browse the repository at this point in the history 
* Added a pair of tests for failed ops.

* Fixed cpu helper for draw_bounding_boxes op.

* Refactored implementation of draw_bounding_boxes op to full conform with TF.

* Improved multithreading with draw_bounding_boxes op cuda helper.

* Eliminated log messages.

* Changed logging with draw_bounding_boxes op helper and tests.

* Resize_biliear with 3D input allowed.

* Refactored 3D input acception with resize_bilinear op.

* And another improvement.

* Refactored reshape of input/output for resize_bilinear.

* Improvements final.

* Finished with 3D replication for image.resize_bilinear/_nearest_neighbor.

* Added copyrights for TF code.

* Using new form of multithreading for cpu implementation.

* Fixed shape error.

* Added multithreaded with batches on crop_and_resize functor.

* Refactored multithreading with crop_and_resize and draw_bounding_boxes.
  • Loading branch information
@shugeo
shugeo committed Nov 20, 2019
1 parent 59e955c commit 13e5c0a
Show file tree
Hide file tree
Showing 9 changed files with 560 additions and 101 deletions.
3 changes: 2 additions & 1 deletion libnd4j/include/ops/declarable/generic/parity_ops/crop_and_resize.cpp
View file
Expand Up @@ -57,6 +57,7 @@ namespace nd4j {

DECLARE_SHAPE_FN(crop_and_resize) {
auto in = inputShape->at(0);
auto boxShape = inputShape->at(1);

Nd4jLong outputShape[4];

Expand All @@ -68,7 +69,7 @@ namespace nd4j {
width = newImageSize->e<int>(0);
height = newImageSize->e<int>(1);

outputShape[0] = in[1];
outputShape[0] = boxShape[1];
outputShape[1] = width;
outputShape[2] = height;
outputShape[3] = in[4];
Expand Down
24 changes: 23 additions & 1 deletion libnd4j/include/ops/declarable/generic/parity_ops/draw_bounding_boxes.cpp
View file
Expand Up @@ -29,9 +29,31 @@ namespace nd4j {

auto images = INPUT_VARIABLE(0);
auto boxes = INPUT_VARIABLE(1);
auto colors = INPUT_VARIABLE(2);

auto colors = (NDArray*) nullptr;
if (block.width() > 2) // TF v.1.x ommits color set for boxes, and use color 1.0 for fill up
colors = INPUT_VARIABLE(2); // but v.2.y require color set

auto output = OUTPUT_VARIABLE(0);
REQUIRE_TRUE(images->dataType() == output->dataType(), 0, "draw_bounding_boxes: Input and Output types "
"should be equals, but %d and %d occured.",
(int)images->dataType(), (int)output->dataType());
REQUIRE_TRUE(images->rankOf() == 4, 0, "draw_bounding_boxes: Images input should be 4D tensor, but %i occured.",
images->rankOf());
REQUIRE_TRUE(boxes->rankOf() == 3, 0, "draw_bounding_boxes: Boxes should be 3D tensor, but %i occured.",
boxes->rankOf());
if (colors) {

REQUIRE_TRUE(colors->rankOf() == 2, 0, "draw_bounding_boxes: Color set should be 2D matrix, but %i occured.",
colors->rankOf());
REQUIRE_TRUE(colors->sizeAt(1) >= images->sizeAt(3), 0, "draw_bounding_boxes: Color set last dim "
"should be not less than images depth, but "
"%lld and %lld occured.",
colors->sizeAt(1), images->sizeAt(3));
}
REQUIRE_TRUE(boxes->sizeAt(0) == images->sizeAt(0), 0, "draw_bounding_boxes: Batches for images and boxes "
"should be the same, but %lld and %lld occured.",
images->sizeAt(0), boxes->sizeAt(0));
helpers::drawBoundingBoxesFunctor(block.launchContext(), images, boxes, colors, output);
return ND4J_STATUS_OK;
}
Expand Down
35 changes: 28 additions & 7 deletions libnd4j/include/ops/declarable/generic/parity_ops/resize_linear.cpp
View file
Expand Up @@ -33,6 +33,15 @@ namespace nd4j {
int width;
int height;
bool center = false; // - default value
auto inRank = image->rankOf();
REQUIRE_TRUE( inRank == 4 || inRank == 3, 0, "resize_bilinear: input image should be 4D "
"tensor, but input has rank %i",
image->rankOf());
REQUIRE_TRUE(inRank == output->rankOf(), 0, "resize_bilinear: Input and output ranks should be equals, but %i and %i occured.", inRank, output->rankOf());

auto source = inRank == 4?image->reshape(image->ordering(), {image->sizeAt(0), image->sizeAt(1), image->sizeAt(2), image->sizeAt(3)}):image->reshape(image->ordering(), {1, image->sizeAt(0), image->sizeAt(1), image->sizeAt(2)});
auto target = inRank == 4?output->reshape(output->ordering(), {output->sizeAt(0), output->sizeAt(1), output->sizeAt(2), output->sizeAt(3)}):output->reshape(output->ordering(), {1, output->sizeAt(0), output->sizeAt(1), output->sizeAt(2)});

if (block.width() > 1) {
auto newImageSize = INPUT_VARIABLE(1);
REQUIRE_TRUE(newImageSize->lengthOf() == 2, 0, "resize_bilinear: Resize params is a pair of values, not %i.", newImageSize->lengthOf());
Expand All @@ -51,14 +60,19 @@ namespace nd4j {
center = 0 != INT_ARG(2);
}

return helpers::resizeBilinearFunctor(block.launchContext(), image, width, height, center, output);
auto res = helpers::resizeBilinearFunctor(block.launchContext(), &source, width, height, center, &target);
return res;
}

DECLARE_SHAPE_FN(resize_bilinear) {
auto shapeList = SHAPELIST();
auto in = inputShape->at(0);

Nd4jLong* outputShape;
auto inRank = shape::rank(in);
REQUIRE_TRUE(inRank == 4 || inRank == 3, 0, "resize_bilinear: input image should be 4D "
"tensor, but input has rank %i",
inRank);

int width;
int height;
Expand All @@ -75,12 +89,19 @@ namespace nd4j {
height = INT_ARG(1);
}

ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(4), Nd4jLong);
outputShape[0] = 4;
outputShape[1] = in[1];
outputShape[2] = width;
outputShape[3] = height;
outputShape[4] = in[4];
ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(inRank), Nd4jLong);
outputShape[0] = inRank;
if (inRank == 4) {
outputShape[1] = in[1];
outputShape[2] = width;
outputShape[3] = height;
outputShape[4] = in[4];
}
else { // input shape is 3D, so result also should be 3D
outputShape[1] = width;
outputShape[2] = height;
outputShape[3] = in[3];
}
ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));

shapeList->push_back(CONSTANT(outputShape));
Expand Down
34 changes: 25 additions & 9 deletions libnd4j/include/ops/declarable/generic/parity_ops/resize_neighbor.cpp
View file
Expand Up @@ -51,16 +51,25 @@ namespace nd4j {
if (block.numI() == 3)
center = 0 != INT_ARG(2);
}
auto inRank = image->rankOf();
REQUIRE_TRUE(inRank == 4 || inRank == 3, 0, "resize_nearest_neighbor: Input should be 4D tensor, but rank %i occured");
REQUIRE_TRUE(inRank == output->rankOf(), 0, "resize_nearest_neighbor: Input and output ranks should be equals, but %i and %i occured.", inRank, output->rankOf());
auto source = inRank == 4?image->reshape(image->ordering(), {image->sizeAt(0), image->sizeAt(1), image->sizeAt(2), image->sizeAt(3)}):image->reshape(image->ordering(), {1, image->sizeAt(0), image->sizeAt(1), image->sizeAt(2)});
auto target = inRank == 4?output->reshape(output->ordering(), {output->sizeAt(0), output->sizeAt(1), output->sizeAt(2), output->sizeAt(3)}):output->reshape(output->ordering(), {1, output->sizeAt(0), output->sizeAt(1), output->sizeAt(2)});

return helpers::resizeNeighborFunctor(block.launchContext(), image, width, height, center, output);
return helpers::resizeNeighborFunctor(block.launchContext(), &source, width, height, center, &target);
}

DECLARE_SHAPE_FN(resize_nearest_neighbor) {
auto shapeList = SHAPELIST();
auto in = inputShape->at(0);

auto inRank = shape::rank(in);
Nd4jLong* outputShape;

REQUIRE_TRUE(inRank == 4 || inRank == 3, 0, "resize_bilinear: input image should be 4D "
"tensor, but input has rank %i",
inRank);

int width;
int height;
if (block.width() > 1) {
Expand All @@ -75,13 +84,20 @@ namespace nd4j {
width = INT_ARG(0);
height = INT_ARG(1);
}

ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(4), Nd4jLong);
outputShape[0] = 4;
outputShape[1] = in[1];
outputShape[2] = width;
outputShape[3] = height;
outputShape[4] = in[4];

ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(inRank), Nd4jLong);
outputShape[0] = inRank;
if (inRank == 4) {
outputShape[1] = in[1];
outputShape[2] = width;
outputShape[3] = height;
outputShape[4] = in[4];
}
else { // input shape is 3D, so result also should be 3D
outputShape[1] = width;
outputShape[2] = height;
outputShape[3] = in[3];
}
ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));

shapeList->push_back(CONSTANT(outputShape));
Expand Down
2 changes: 1 addition & 1 deletion libnd4j/include/ops/declarable/helpers/cpu/fake_quantization.cpp
View file
Expand Up @@ -47,7 +47,7 @@ namespace helpers {
if (zeroPointFromMin > quantMaxF) {
return static_cast<uint16_t>(quantMax);
}
return nd4j::math::nd4j_round<T,uint16_t>(zeroPointFromMin);
return (uint16_t)nd4j::math::nd4j_round<T,int>(zeroPointFromMin);
}();
// compute nudged min and max with computed nudged zero point
*nudgedMin = (quantMinF - nudged_zero_point) * (*scale);
Expand Down
151 changes: 125 additions & 26 deletions libnd4j/include/ops/declarable/helpers/cpu/image_draw_bounding_boxes.cpp
View file
Expand Up @@ -13,16 +13,52 @@
*
* SPDX-License-Identifier: Apache-2.0
******************************************************************************/
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

//
// @author sgazeos@gmail.com
//
#include <op_boilerplate.h>
#include <NDArray.h>
#include <execution/Threads.h>

namespace nd4j {
namespace ops {
namespace helpers {
typedef std::vector<std::vector<float>> ColorTable_t;
static ColorTable_t DefaultColorTable(int depth) {
std::vector<std::vector<float>> colorTable;
colorTable.emplace_back(std::vector<float>({1, 1, 0, 1})); // 0: yellow
colorTable.emplace_back(std::vector<float>({0, 0, 1, 1})); // 1: blue
colorTable.emplace_back(std::vector<float>({1, 0, 0, 1})); // 2: red
colorTable.emplace_back(std::vector<float>({0, 1, 0, 1})); // 3: lime
colorTable.emplace_back(std::vector<float>({0.5, 0, 0.5, 1})); // 4: purple
colorTable.emplace_back(std::vector<float>({0.5, 0.5, 0, 1})); // 5: olive
colorTable.emplace_back(std::vector<float>({0.5, 0, 0, 1})); // 6: maroon
colorTable.emplace_back(std::vector<float>({0, 0, 0.5, 1})); // 7: navy blue
colorTable.emplace_back(std::vector<float>({0, 1, 1, 1})); // 8: aqua
colorTable.emplace_back(std::vector<float>({1, 0, 1, 1})); // 9: fuchsia

if (depth == 1) {
for (Nd4jLong i = 0; i < colorTable.size(); i++) {
colorTable[i][0] = 1;
}
}
return colorTable;
}

void drawBoundingBoxesFunctor(nd4j::LaunchContext * context, NDArray* images, NDArray* boxes, NDArray* colors, NDArray* output) {
// images - batch of 3D images with BW (last dim = 1), RGB (last dim = 3) or RGBA (last dim = 4) channel set
Expand All @@ -33,44 +69,107 @@ namespace helpers {
// colors - colors for each box given
// set up color for each box as frame
auto batchSize = images->sizeAt(0);
auto boxSize = boxes->sizeAt(0);
auto height = images->sizeAt(1);
auto width = images->sizeAt(2);
auto channels = images->sizeAt(3);
//auto imageList = images->allTensorsAlongDimension({1, 2, 3}); // split images by batch
// auto boxList = boxes->allTensorsAlongDimension({1, 2}); // split boxes by batch
auto colorSet = colors->allTensorsAlongDimension({1});
//auto colorSet = colors->allTensorsAlongDimension({0});
output->assign(images); // fill up all output with input images, then fill up boxes
ColorTable_t colorTable;
if (colors) {
for (auto i = 0; i < colors->sizeAt(0); i++) {
std::vector<float> colorValue(4);
for (auto j = 0; j < 4; j++) {
colorValue[j] = j < colors->sizeAt(1) ? colors->e<float>(i, j) : 1.f;
}
colorTable.emplace_back(colorValue);
}
}
if (colorTable.empty())
colorTable = DefaultColorTable(channels);
// PRAGMA_OMP_PARALLEL_FOR
auto func = PRAGMA_THREADS_FOR {
for (auto batch = 0; batch < batchSize; ++batch) { // loop by batch
// auto image = imageList->at(batch);
const Nd4jLong numBoxes = boxes->sizeAt(1);
for (auto boxIndex = 0; boxIndex < numBoxes; ++boxIndex) {
// box with shape
//auto internalBox = (*boxes)(batch, {0})(boxIndex, {0});//internalBoxes->at(c);
//auto color = colorSet->at(c);
//internalBox.printIndexedBuffer("Current Box");
auto colorIndex = boxIndex % colorTable.size();
auto rowStart = Nd4jLong((height - 1) * boxes->t<float>(batch, boxIndex, 0));
auto rowStartBound = nd4j::math::nd4j_max(Nd4jLong(0), rowStart);
auto rowEnd = Nd4jLong((height - 1) * boxes->t<float>(batch, boxIndex, 2));
auto rowEndBound = nd4j::math::nd4j_min(Nd4jLong(height - 1), rowEnd);
auto colStart = Nd4jLong((width - 1) * boxes->t<float>(batch, boxIndex, 1));
auto colStartBound = nd4j::math::nd4j_max(Nd4jLong(0), colStart);
auto colEnd = Nd4jLong((width - 1) * boxes->t<float>(batch, boxIndex, 3));
auto colEndBound = nd4j::math::nd4j_min(Nd4jLong(width - 1), colEnd);

PRAGMA_OMP_PARALLEL_FOR
for (auto b = 0; b < batchSize; ++b) { // loop by batch
// auto image = imageList->at(b);
if (rowStart > rowEnd || colStart > colEnd) {
nd4j_debug(
"helpers::drawBoundingBoxesFunctor: Bounding box (%lld, %lld, %lld, %lld) is inverted "
"and will not be drawn\n", rowStart, colStart, rowEnd, colEnd);
continue;
}
if (rowStart >= height || rowEnd < 0 || colStart >= width ||
colEnd < 0) {
nd4j_debug(
"helpers::drawBoundingBoxesFunctor: Bounding box (%lld, %lld, %lld, %lld) is completely "
"outside the image and not be drawn\n ", rowStart, colStart, rowEnd, colEnd);
continue;
}

for (auto c = 0; c < colorSet->size(); ++c) {
// box with shape
auto internalBox = (*boxes)(b, {0})(c, {0});//internalBoxes->at(c);
auto color = colorSet->at(c);
auto rowStart = nd4j::math::nd4j_max(Nd4jLong (0), Nd4jLong ((height - 1) * internalBox.e<float>(0)));
auto rowEnd = nd4j::math::nd4j_min(Nd4jLong (height - 1), Nd4jLong ((height - 1) * internalBox.e<float>(2)));
auto colStart = nd4j::math::nd4j_max(Nd4jLong (0), Nd4jLong ((width - 1) * internalBox.e<float>(1)));
auto colEnd = nd4j::math::nd4j_min(Nd4jLong(width - 1), Nd4jLong ((width - 1) * internalBox.e<float>(3)));
for (auto y = rowStart; y <= rowEnd; y++) {
for (auto e = 0; e < color->lengthOf(); ++e) {
output->p(b, y, colStart, e, color->e(e));
output->p(b, y, colEnd, e, color->e(e));
// Draw upper line
if (rowStart >= 0) {
for (auto j = colStartBound; j <= colEndBound; ++j)
for (auto c = 0; c < channels; c++) {
output->p(batch, rowStart, j, c, colorTable[colorIndex][c]);
}
}
}
for (auto x = colStart + 1; x < colEnd; x++) {
for (auto e = 0; e < color->lengthOf(); ++e) {
output->p(b, rowStart, x, e, color->e(e));
output->p(b, rowEnd, x, e, color->e(e));
// Draw bottom line.
if (rowEnd < height) {
for (auto j = colStartBound; j <= colEndBound; ++j)
for (auto c = 0; c < channels; c++) {
output->p(batch, rowEnd, j, c, colorTable[colorIndex][c]);
}
}

// Draw left line.
if (colStart >= 0) {
for (auto i = rowStartBound; i <= rowEndBound; ++i)
for (auto c = 0; c < channels; c++) {
output->p(batch, i, colStart, c, colorTable[colorIndex][c]);
}
}
// Draw right line.
if (colEnd < width) {
for (auto i = rowStartBound; i <= rowEndBound; ++i)
for (auto c = 0; c < channels; c++) {
output->p(batch, i, colEnd, c, colorTable[colorIndex][c]);
}
}
// for (auto y = rowStart; y <= rowEnd; y++) {
// for (auto e = 0; e < channels; ++e) {
// output->p(batch, y, colStart, e, colorTable[colorIndex][e]);
// output->p(batch, y, colEnd, e, colorTable[colorIndex][e]);
// }
// }
// for (auto x = colStart + 1; x < colEnd; x++) {
// for (auto e = 0; e < channels; ++e) {
// output->p(batch, rowStart, x, e, colorTable[colorIndex][e]);
// output->p(batch, rowEnd, x, e, colorTable[colorIndex][e]);
// }
// }
}
}
// delete internalBoxes;
}
delete colorSet;
// delete imageList;
// delete boxList;
}
};
samediff::Threads::parallel_tad(func, 0, batchSize);

}

}
Expand Down

0 comments on commit 13e5c0a

Please sign in to comment.