/* * Copyright (C) 2018 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "CpuOperationUtils.h" #include "OperationResolver.h" #include "OperationsUtils.h" #include <cfloat> #include <cmath> #include <numeric> #include "Tracing.h" namespace android { namespace nn { namespace bbox_ops { namespace { struct BoxEncodingCorner { float x1, y1, x2, y2; }; struct BoxEncodingCenter { float w, h, x, y; }; BoxEncodingCorner toBoxEncodingCorner(const BoxEncodingCenter& ctr) { return {.x1 = ctr.x - ctr.w / 2, .y1 = ctr.y - ctr.h / 2, .x2 = ctr.x + ctr.w / 2, .y2 = ctr.y + ctr.h / 2}; } BoxEncodingCenter toBoxEncodingCenter(const BoxEncodingCorner& cnr) { return {.w = cnr.x2 - cnr.x1, .h = cnr.y2 - cnr.y1, .x = (cnr.x1 + cnr.x2) / 2, .y = (cnr.y1 + cnr.y2) / 2}; } inline bool bboxTransformFloat32(const float* roiData, const Shape& roiShape, const float* bboxDeltasData, const Shape& bboxDeltasShape, const int32_t* batchesData, const Shape& batchesShape, const float* imageInfoData, const Shape& imageInfoDataShape, float* outputData, const Shape& outputShape) { const uint32_t roiLength = 4; const uint32_t imageLength = 2; uint32_t numClasses = getSizeOfDimension(bboxDeltasShape, 1) / roiLength; uint32_t numBatches = getSizeOfDimension(imageInfoDataShape, 0); const float* roiDataEnd = roiData + getNumberOfElements(roiShape); const float* deltas = bboxDeltasData; float* outPtr = outputData; uint32_t roiIndex = 0; for (const float* roiBase = roiData; roiBase < roiDataEnd; roiBase += roiLength, roiIndex++) { uint32_t batchIndex = batchesData[roiIndex]; // Check for malformed data // 1. Invalid batch id // 2. Invalid region: x2 < x1 || y2 < y1 NN_RET_CHECK_GE(batchIndex, 0); NN_RET_CHECK_LT(batchIndex, numBatches); NN_RET_CHECK_LE(roiBase[0], roiBase[2]); NN_RET_CHECK_LE(roiBase[1], roiBase[3]); const float* imageInfoBase = imageInfoData + batchIndex * imageLength; float imageHeight = imageInfoBase[0]; float imageWidth = imageInfoBase[1]; auto roiBefore = toBoxEncodingCenter( {.x1 = roiBase[0], .y1 = roiBase[1], .x2 = roiBase[2], .y2 = roiBase[3]}); for (uint32_t i = 0; i < numClasses; i++) { auto roiAfter = toBoxEncodingCorner({.w = std::exp(deltas[2]) * roiBefore.w, .h = std::exp(deltas[3]) * roiBefore.h, .x = roiBefore.x + deltas[0] * roiBefore.w, .y = roiBefore.y + deltas[1] * roiBefore.h}); BoxEncodingCorner cliped = {.x1 = std::min(std::max(roiAfter.x1, 0.0f), imageWidth), .y1 = std::min(std::max(roiAfter.y1, 0.0f), imageHeight), .x2 = std::min(std::max(roiAfter.x2, 0.0f), imageWidth), .y2 = std::min(std::max(roiAfter.y2, 0.0f), imageHeight)}; outPtr[0] = cliped.x1; outPtr[1] = cliped.y1; outPtr[2] = cliped.x2; outPtr[3] = cliped.y2; deltas += roiLength; outPtr += roiLength; } } return true; } inline bool bboxTransformFloat16(const _Float16* roiData, const Shape& roiShape, const _Float16* bboxDeltasData, const Shape& bboxDeltasShape, const int32_t* batchesData, const Shape& batchesShape, const _Float16* imageInfoData, const Shape& imageInfoDataShape, _Float16* outputData, const Shape& outputShape) { std::vector<float> roi_float32(getNumberOfElements(roiShape)); convertFloat16ToFloat32(roiData, &roi_float32); std::vector<float> delta_float32(getNumberOfElements(bboxDeltasShape)); convertFloat16ToFloat32(bboxDeltasData, &delta_float32); std::vector<float> imageInfo_float32(getNumberOfElements(imageInfoDataShape)); convertFloat16ToFloat32(imageInfoData, &imageInfo_float32); std::vector<float> output_float32(getNumberOfElements(outputShape)); NN_RET_CHECK(bboxTransformFloat32(roi_float32.data(), roiShape, delta_float32.data(), bboxDeltasShape, batchesData, batchesShape, imageInfo_float32.data(), imageInfoDataShape, output_float32.data(), outputShape)); convertFloat32ToFloat16(output_float32, outputData); return true; } inline bool bboxTransformQuant(const uint16_t* roiData, const Shape& roiShape, const uint8_t* bboxDeltasData, const Shape& bboxDeltasShape, const int32_t* batchesData, const Shape& batchesShape, const uint16_t* imageInfoData, const Shape& imageInfoDataShape, uint16_t* outputData, const Shape& outputShape) { std::vector<float> roi_float32(getNumberOfElements(roiShape)); convertQuantToFloat32(roiData, roiShape.scale, roiShape.offset, &roi_float32); std::vector<float> delta_float32(getNumberOfElements(bboxDeltasShape)); convertQuantToFloat32(bboxDeltasData, bboxDeltasShape.scale, bboxDeltasShape.offset, &delta_float32); std::vector<float> imageInfo_float32(getNumberOfElements(imageInfoDataShape)); convertQuantToFloat32(imageInfoData, imageInfoDataShape.scale, imageInfoDataShape.offset, &imageInfo_float32); std::vector<float> output_float32(getNumberOfElements(outputShape)); NN_RET_CHECK(bboxTransformFloat32(roi_float32.data(), roiShape, delta_float32.data(), bboxDeltasShape, batchesData, batchesShape, imageInfo_float32.data(), imageInfoDataShape, output_float32.data(), outputShape)); convertFloat32ToQuant(output_float32, outputShape.scale, outputShape.offset, outputData); return true; } // Taking two indices of bounding boxes, return the intersection-of-union. float getIoUAxisAligned(const float* roi1, const float* roi2) { const float area1 = (roi1[2] - roi1[0]) * (roi1[3] - roi1[1]); const float area2 = (roi2[2] - roi2[0]) * (roi2[3] - roi2[1]); const float x1 = std::max(roi1[0], roi2[0]); const float x2 = std::min(roi1[2], roi2[2]); const float y1 = std::max(roi1[1], roi2[1]); const float y2 = std::min(roi1[3], roi2[3]); const float w = std::max(x2 - x1, 0.0f); const float h = std::max(y2 - y1, 0.0f); const float areaIntersect = w * h; const float areaUnion = area1 + area2 - areaIntersect; return areaIntersect / areaUnion; } } // namespace namespace axis_aligned_bbox_transform { constexpr char kOperationName[] = "AXIS_ALIGNED_BBOX_TRANSFORM"; constexpr uint32_t kNumInputs = 4; constexpr uint32_t kRoiTensor = 0; constexpr uint32_t kDeltaTensor = 1; constexpr uint32_t kBatchesTensor = 2; constexpr uint32_t kImageInfoTensor = 3; constexpr uint32_t kNumOutputs = 1; constexpr uint32_t kOutputTensor = 0; bool validate(const IOperationValidationContext* context) { NN_RET_CHECK_EQ(context->getNumInputs(), kNumInputs); NN_RET_CHECK_EQ(context->getNumOutputs(), kNumOutputs); std::vector<OperandType> inExpectedTypes; auto inputType = context->getInputType(kRoiTensor); if (inputType == OperandType::TENSOR_FLOAT32 || inputType == OperandType::TENSOR_FLOAT16) { inExpectedTypes = {inputType, inputType, OperandType::TENSOR_INT32, inputType}; } else if (inputType == OperandType::TENSOR_QUANT16_ASYMM) { inExpectedTypes = {OperandType::TENSOR_QUANT16_ASYMM, OperandType::TENSOR_QUANT8_ASYMM, OperandType::TENSOR_INT32, OperandType::TENSOR_QUANT16_ASYMM}; } else { LOG(ERROR) << "Unsupported input tensor type for operation " << kOperationName; return false; } NN_RET_CHECK(validateInputTypes(context, inExpectedTypes)); NN_RET_CHECK(validateOutputTypes(context, {inputType})); return validateHalVersion(context, HalVersion::V1_2); } bool prepare(IOperationExecutionContext* context) { Shape roiShape = context->getInputShape(kRoiTensor); Shape bboxDeltasShape = context->getInputShape(kDeltaTensor); Shape batchesShape = context->getInputShape(kBatchesTensor); Shape imageInfoShape = context->getInputShape(kImageInfoTensor); Shape outputShape = context->getOutputShape(kOutputTensor); NN_RET_CHECK_EQ(getNumberOfDimensions(roiShape), 2); NN_RET_CHECK_EQ(getNumberOfDimensions(bboxDeltasShape), 2); NN_RET_CHECK_EQ(getNumberOfDimensions(batchesShape), 1); NN_RET_CHECK_EQ(getNumberOfDimensions(imageInfoShape), 2); // Only numRois can be zero. const uint32_t kRoiDim = 4; uint32_t numRois = getSizeOfDimension(roiShape, 0); uint32_t numClasses = getSizeOfDimension(bboxDeltasShape, 1) / kRoiDim; uint32_t numBatches = getSizeOfDimension(imageInfoShape, 0); NN_RET_CHECK_GT(numClasses, 0); NN_RET_CHECK_GT(numBatches, 0); NN_RET_CHECK_EQ(getSizeOfDimension(roiShape, 1), kRoiDim); NN_RET_CHECK_EQ(getSizeOfDimension(bboxDeltasShape, 0), numRois); NN_RET_CHECK_EQ(getSizeOfDimension(bboxDeltasShape, 1), kRoiDim * numClasses); NN_RET_CHECK_EQ(getSizeOfDimension(batchesShape, 0), numRois); NN_RET_CHECK_EQ(getSizeOfDimension(imageInfoShape, 1), 2); if (roiShape.type == OperandType::TENSOR_QUANT16_ASYMM) { NN_RET_CHECK_EQ(roiShape.scale, 0.125f); NN_RET_CHECK_EQ(roiShape.offset, 0); NN_RET_CHECK_EQ(imageInfoShape.scale, 0.125f); NN_RET_CHECK_EQ(imageInfoShape.offset, 0); } outputShape.type = roiShape.type; outputShape.dimensions = {numRois, numClasses * kRoiDim}; outputShape.scale = 0.125f; outputShape.offset = 0; NN_RET_CHECK(context->setOutputShape(kOutputTensor, outputShape)); return true; } bool execute(IOperationExecutionContext* context) { NNTRACE_TRANS("axisAlignedBBoxTransform"); // Bypass execution in the case of zero-sized input. if (getNumberOfElements(context->getOutputShape(kOutputTensor)) == 0) return true; switch (context->getInputType(kRoiTensor)) { case OperandType::TENSOR_FLOAT16: { return bboxTransformFloat16(context->getInputBuffer<_Float16>(kRoiTensor), context->getInputShape(kRoiTensor), context->getInputBuffer<_Float16>(kDeltaTensor), context->getInputShape(kDeltaTensor), context->getInputBuffer<int32_t>(kBatchesTensor), context->getInputShape(kBatchesTensor), context->getInputBuffer<_Float16>(kImageInfoTensor), context->getInputShape(kImageInfoTensor), context->getOutputBuffer<_Float16>(kOutputTensor), context->getOutputShape(kOutputTensor)); } case OperandType::TENSOR_FLOAT32: { return bboxTransformFloat32(context->getInputBuffer<float>(kRoiTensor), context->getInputShape(kRoiTensor), context->getInputBuffer<float>(kDeltaTensor), context->getInputShape(kDeltaTensor), context->getInputBuffer<int32_t>(kBatchesTensor), context->getInputShape(kBatchesTensor), context->getInputBuffer<float>(kImageInfoTensor), context->getInputShape(kImageInfoTensor), context->getOutputBuffer<float>(kOutputTensor), context->getOutputShape(kOutputTensor)); } case OperandType::TENSOR_QUANT16_ASYMM: { return bboxTransformQuant(context->getInputBuffer<uint16_t>(kRoiTensor), context->getInputShape(kRoiTensor), context->getInputBuffer<uint8_t>(kDeltaTensor), context->getInputShape(kDeltaTensor), context->getInputBuffer<int32_t>(kBatchesTensor), context->getInputShape(kBatchesTensor), context->getInputBuffer<uint16_t>(kImageInfoTensor), context->getInputShape(kImageInfoTensor), context->getOutputBuffer<uint16_t>(kOutputTensor), context->getOutputShape(kOutputTensor)); } default: NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << kOperationName; } } } // namespace axis_aligned_bbox_transform namespace box_with_nms_limit { constexpr char kOperationName[] = "BOX_WITH_NMS_LIMIT"; constexpr uint32_t kNumInputs = 9; constexpr uint32_t kScoreTensor = 0; constexpr uint32_t kRoiTensor = 1; constexpr uint32_t kBatchesTensor = 2; constexpr uint32_t kScoreThresholdScalar = 3; constexpr uint32_t kMaxNumDetectionScalar = 4; constexpr uint32_t kNmsKernelScalar = 5; constexpr uint32_t kIoUThresholdScalar = 6; constexpr uint32_t kSigmaScalar = 7; constexpr uint32_t kNmsScoreThresholdScalar = 8; constexpr uint32_t kNumOutputs = 4; constexpr uint32_t kOutputScoreTensor = 0; constexpr uint32_t kOutputRoiTensor = 1; constexpr uint32_t kOutputClassTensor = 2; constexpr uint32_t kOutputBatchesTensor = 3; namespace { // TODO(xusongw): Reduce code duplication with hard/soft nms path. // Inplace hard NMS within range [select, select + selectLength). uint32_t* hardNmsSingleClass(const float* scoresData, float iouThreshold, int32_t maxNumDetections, std::function<const float*(uint32_t)> getRoiBase, uint32_t* select, uint32_t selectLength) { uint32_t *selectStart = select, *selectEnd = select + selectLength, numDetections = 0; if (maxNumDetections < 0) { maxNumDetections = selectLength; } while (selectStart < selectEnd && numDetections < maxNumDetections) { // find max score and swap to the front auto& maxScore = *std::max_element(selectStart, selectEnd, [&scoresData](const uint32_t& lhs, const uint32_t& rhs) { return scoresData[lhs] < scoresData[rhs]; }); std::swap(maxScore, *selectStart); // Calculate IoU of the rest, swap to the end (disgard) if needed. for (uint32_t* i = selectStart + 1; i < selectEnd; i++) { float iou = getIoUAxisAligned(getRoiBase(*i), getRoiBase(*selectStart)); if (iou >= iouThreshold) { std::swap(*i--, *(--selectEnd)); } } selectStart++; numDetections++; } return selectStart; } void hardNmsMultiClass(const float* scoresData, uint32_t numClasses, uint32_t numRois, float scoreThreshold, float iouThreshold, int32_t maxNumDetections, int32_t maxNumDetectionsPerClass, std::function<const float*(uint32_t)> getRoiBase, std::vector<uint32_t>* select) { // Exclude class 0 (background) for (uint32_t c = 1; c < numClasses; c++) { uint32_t size = select->size(); for (uint32_t b = 0; b < numRois; b++) { const uint32_t index = b * numClasses + c; const float score = scoresData[index]; if (score > scoreThreshold) { select->push_back(index); } } uint32_t* selectStart = select->data() + size; uint32_t selectLength = select->size() - size; uint32_t* selectEnd = hardNmsSingleClass(scoresData, iouThreshold, maxNumDetectionsPerClass, getRoiBase, selectStart, selectLength); select->resize(selectEnd - select->data()); } // Take top maxNumDetections. std::sort(select->begin(), select->end(), [&scoresData](const uint32_t& lhs, const uint32_t& rhs) { return scoresData[lhs] > scoresData[rhs]; }); if (maxNumDetections < 0 || select->size() <= maxNumDetections) { return; } select->resize(maxNumDetections); } // Inplace soft NMS within range [select, select + selectLength). using SoftNmsKernel = std::function<float(float)>; uint32_t* softNmsSingleClass(float* scoresData, float scoreThreshold, int32_t maxNumDetections, std::function<const float*(uint32_t)> getRoiBase, SoftNmsKernel kernel, uint32_t* select, uint32_t selectLength) { uint32_t *selectStart = select, *selectEnd = select + selectLength, numDetections = 0; if (maxNumDetections < 0) { maxNumDetections = selectLength; } while (selectStart < selectEnd && numDetections < maxNumDetections) { // find max score and swap to the front auto& maxScore = *std::max_element(selectStart, selectEnd, [&scoresData](const uint32_t& lhs, const uint32_t& rhs) { return scoresData[lhs] < scoresData[rhs]; }); std::swap(maxScore, *selectStart); // Calculate IoU of the rest, swap to the end (disgard) if needed. for (uint32_t* i = selectStart + 1; i < selectEnd; i++) { float iou = getIoUAxisAligned(getRoiBase(*i), getRoiBase(*selectStart)); scoresData[*i] *= kernel(iou); if (scoresData[*i] < scoreThreshold) { std::swap(*i--, *(--selectEnd)); } } selectStart++; numDetections++; } return selectStart; } void softNmsMultiClass(float* scoresData, uint32_t numClasses, uint32_t numRois, float scoreThreshold, float nmsScoreThreshold, int32_t maxNumDetections, int32_t maxNumDetectionsPerClass, std::function<const float*(uint32_t)> getRoiBase, SoftNmsKernel kernel, std::vector<uint32_t>* select) { // Exclude class 0 (background) for (uint32_t c = 1; c < numClasses; c++) { uint32_t size = select->size(); for (uint32_t b = 0; b < numRois; b++) { const uint32_t index = b * numClasses + c; const float score = scoresData[index]; if (score > scoreThreshold) { select->push_back(index); } } uint32_t* selectStart = select->data() + size; uint32_t selectLength = select->size() - size; uint32_t* selectEnd = softNmsSingleClass(scoresData, nmsScoreThreshold, maxNumDetectionsPerClass, getRoiBase, kernel, selectStart, selectLength); select->resize(selectEnd - select->data()); } // Take top maxNumDetections. std::sort(select->begin(), select->end(), [&scoresData](const uint32_t& lhs, const uint32_t& rhs) { return scoresData[lhs] > scoresData[rhs]; }); if (maxNumDetections < 0 || select->size() <= maxNumDetections) { return; } select->resize(maxNumDetections); } bool boxWithNmsLimitFloat32Compute(float* scoresData, const Shape& scoresShape, const float* roiData, const Shape& roiShape, const int32_t* batchesData, const Shape& batchesShape, float scoreThreshold, int32_t maxNumDetections, int32_t softNmsKernel, float iouThreshold, float sigma, float nmsScoreThreshold, std::vector<uint32_t>* batchSplitIn, std::vector<uint32_t>* batchSplitOut, std::vector<uint32_t>* selected) { SoftNmsKernel kernel = nullptr; if (softNmsKernel == 0) { kernel = [&iouThreshold](float iou) { return iou < iouThreshold ? 1.0f : 0.0f; }; } else if (softNmsKernel == 1) { kernel = [&iouThreshold](float iou) { return iou < iouThreshold ? 1.0f : 1.0f - iou; }; } else if (softNmsKernel == 2) { kernel = [&sigma](float iou) { return std::exp(-1.0f * iou * iou / sigma); }; } else { NN_RET_CHECK_FAIL() << "Unsupported soft NMS kernel " << softNmsKernel; } const uint32_t kRoiDim = 4; uint32_t numRois = getSizeOfDimension(scoresShape, 0); uint32_t numClasses = getSizeOfDimension(scoresShape, 1); // We assume boxes of the same batch are grouped together. std::vector<uint32_t> batch; for (uint32_t i = 0, ind = -1; i < numRois; i++) { if (batchesData[i] == ind) { (batchSplitIn->back())++; } else { ind = batchesData[i]; batchSplitIn->push_back(1); } } float* scoresBase = scoresData; const float* roiBase = roiData; selected->clear(); for (uint32_t b = 0; b < batchSplitIn->size(); b++) { for (uint32_t i = 0; i < batchSplitIn->at(b); i++) { const float* roi = roiBase + i * kRoiDim; // Check for malformed data: invalid region: x2 < x1 || y2 < y1 NN_RET_CHECK_LE(roi[0], roi[2]); NN_RET_CHECK_LE(roi[1], roi[3]); } std::vector<uint32_t> result; softNmsMultiClass(scoresBase, numClasses, batchSplitIn->at(b), scoreThreshold, nmsScoreThreshold, maxNumDetections, maxNumDetections, [&roiBase](uint32_t ind) { return roiBase + ind * kRoiDim; }, kernel, &result); // Sort again by class. std::sort(result.begin(), result.end(), [&scoresBase, numClasses](const uint32_t& lhs, const uint32_t& rhs) { uint32_t lhsClass = lhs % numClasses, rhsClass = rhs % numClasses; return lhsClass == rhsClass ? scoresBase[lhs] > scoresBase[rhs] : lhsClass < rhsClass; }); selected->insert(selected->end(), result.begin(), result.end()); batchSplitOut->push_back(result.size()); scoresBase += batchSplitIn->at(b) * numClasses; roiBase += batchSplitIn->at(b) * numClasses * kRoiDim; } return true; } template <typename T> T castTo(float val, const Shape&) { return val; } template <> uint8_t castTo(float val, const Shape& shape) { int32_t intVal = std::round(val / shape.scale + shape.offset); intVal = std::min<int32_t>(std::max<int32_t>(intVal, std::numeric_limits<uint8_t>::min()), std::numeric_limits<uint8_t>::max()); return static_cast<uint8_t>(intVal); } template <typename T_Score, typename T_Roi> bool boxWithNmsLimitWriteOutput(const std::vector<uint32_t>& selected, const std::vector<uint32_t>& batchSplitIn, const std::vector<uint32_t>& batchSplitOut, const std::vector<float>& scores, IOperationExecutionContext* context) { const uint32_t kRoiDim = 4; Shape scoresShape = context->getInputShape(kScoreTensor); uint32_t numClasses = getSizeOfDimension(scoresShape, 1); // Set output dimensions. uint32_t numOutRois = selected.size(); if (numOutRois == 0) return true; Shape scoresOutShape = context->getOutputShape(kOutputScoreTensor); scoresOutShape.dimensions = {numOutRois}; NN_RET_CHECK(context->setOutputShape(kOutputScoreTensor, scoresOutShape)); Shape roiOutShape = context->getOutputShape(kOutputRoiTensor); roiOutShape.dimensions = {numOutRois, 4}; NN_RET_CHECK(context->setOutputShape(kOutputRoiTensor, roiOutShape)); Shape classesOutShape = context->getOutputShape(kOutputClassTensor); classesOutShape.dimensions = {numOutRois}; NN_RET_CHECK(context->setOutputShape(kOutputClassTensor, classesOutShape)); Shape batchesOutShape = context->getOutputShape(kOutputBatchesTensor); batchesOutShape.dimensions = {numOutRois}; NN_RET_CHECK(context->setOutputShape(kOutputBatchesTensor, batchesOutShape)); // Write outputs. const float* scoresBase = scores.data(); const T_Roi* roiBase = context->getInputBuffer<T_Roi>(kRoiTensor); const int32_t* batchesInPtr = context->getInputBuffer<int32_t>(kBatchesTensor); T_Score* scoresOutPtr = context->getOutputBuffer<T_Score>(kOutputScoreTensor); T_Roi* roiOutPtr = context->getOutputBuffer<T_Roi>(kOutputRoiTensor); int32_t* classesOutPtr = context->getOutputBuffer<int32_t>(kOutputClassTensor); int32_t* batchesOutPtr = context->getOutputBuffer<int32_t>(kOutputBatchesTensor); uint32_t i = 0; for (uint32_t b = 0; b < batchSplitOut.size(); b++) { for (uint32_t j = 0; j < batchSplitOut[b]; j++) { uint32_t index = selected[i++]; *scoresOutPtr++ = castTo<T_Score>(scoresBase[index], scoresOutShape); memcpy(roiOutPtr, roiBase + index * kRoiDim, kRoiDim * sizeof(T_Roi)); roiOutPtr += kRoiDim; *classesOutPtr++ = index % numClasses; *batchesOutPtr++ = *batchesInPtr; } scoresBase += batchSplitIn[b] * numClasses; roiBase += batchSplitIn[b] * numClasses * kRoiDim; batchesInPtr += batchSplitIn[b]; } return true; } bool boxWithNmsLimitFloat32(const float* scoresData, const Shape& scoresShape, const float* roiData, const Shape& roiShape, const int32_t* batchesData, const Shape& batchesShape, float scoreThreshold, int32_t maxNumDetections, int32_t softNmsKernel, float iouThreshold, float sigma, float nmsScoreThreshold, float* scoresOutData, Shape scoresOutShape, float* roiOutData, Shape roiOutShape, int32_t* classesOutData, Shape classesOutShape, int32_t* batchesOutData, const Shape& batchSplitOutShape, IOperationExecutionContext* context) { NNTRACE_TRANS("boxWithNmsLimit"); std::vector<float> scores_float32(getNumberOfElements(scoresShape)); for (uint32_t i = 0; i < scores_float32.size(); i++) { scores_float32[i] = scoresData[i]; } std::vector<uint32_t> selected, batchSplitIn, batchSplitOut; NN_RET_CHECK(boxWithNmsLimitFloat32Compute( scores_float32.data(), scoresShape, roiData, roiShape, batchesData, batchesShape, scoreThreshold, maxNumDetections, softNmsKernel, iouThreshold, sigma, nmsScoreThreshold, &batchSplitIn, &batchSplitOut, &selected)); return boxWithNmsLimitWriteOutput<float, float>(selected, batchSplitIn, batchSplitOut, scores_float32, context); } bool boxWithNmsLimitFloat16(const _Float16* scoresData, const Shape& scoresShape, const _Float16* roiData, const Shape& roiShape, const int32_t* batchesData, const Shape& batchesShape, _Float16 scoreThreshold, int32_t maxNumDetections, int32_t softNmsKernel, _Float16 iouThreshold, _Float16 sigma, _Float16 nmsScoreThreshold, _Float16* scoresOutData, const Shape& scoresOutShape, _Float16* roiOutData, const Shape& roiOutShape, int32_t* classesOutData, const Shape& classesOutShape, int32_t* batchesOutData, const Shape& batchSplitOutShape, IOperationExecutionContext* context) { std::vector<float> scores_float32(getNumberOfElements(scoresShape)); convertFloat16ToFloat32(scoresData, &scores_float32); std::vector<float> roi_float32(getNumberOfElements(roiShape)); convertFloat16ToFloat32(roiData, &roi_float32); std::vector<uint32_t> selected, batchSplitIn, batchSplitOut; NN_RET_CHECK(boxWithNmsLimitFloat32Compute( scores_float32.data(), scoresShape, roi_float32.data(), roiShape, batchesData, batchesShape, scoreThreshold, maxNumDetections, softNmsKernel, iouThreshold, sigma, nmsScoreThreshold, &batchSplitIn, &batchSplitOut, &selected)); return boxWithNmsLimitWriteOutput<_Float16, _Float16>(selected, batchSplitIn, batchSplitOut, scores_float32, context); } bool boxWithNmsLimitQuant(const uint8_t* scoresData, const Shape& scoresShape, const uint16_t* roiData, const Shape& roiShape, const int32_t* batchesData, const Shape& batchesShape, float scoreThreshold, int32_t maxNumDetections, int32_t softNmsKernel, float iouThreshold, float sigma, float nmsScoreThreshold, uint8_t* scoresOutData, const Shape& scoresOutShape, uint16_t* roiOutData, const Shape& roiOutShape, int32_t* classesOutData, const Shape& classesOutShape, int32_t* batchesOutData, const Shape& batchSplitOutShape, IOperationExecutionContext* context) { std::vector<float> scores_float32(getNumberOfElements(scoresShape)); convertQuantToFloat32(scoresData, scoresShape.scale, scoresShape.offset, &scores_float32); std::vector<float> roi_float32(getNumberOfElements(roiShape)); convertQuantToFloat32(roiData, roiShape.scale, roiShape.offset, &roi_float32); std::vector<uint32_t> selected, batchSplitIn, batchSplitOut; NN_RET_CHECK(boxWithNmsLimitFloat32Compute( scores_float32.data(), scoresShape, roi_float32.data(), roiShape, batchesData, batchesShape, scoreThreshold, maxNumDetections, softNmsKernel, iouThreshold, sigma, nmsScoreThreshold, &batchSplitIn, &batchSplitOut, &selected)); return boxWithNmsLimitWriteOutput<uint8_t, uint16_t>(selected, batchSplitIn, batchSplitOut, scores_float32, context); } } // namespace bool validate(const IOperationValidationContext* context) { NN_RET_CHECK_EQ(context->getNumInputs(), kNumInputs); NN_RET_CHECK_EQ(context->getNumOutputs(), kNumOutputs); std::vector<OperandType> inExpectedTypes; std::vector<OperandType> outExpectedTypes; auto inputType = context->getInputType(kScoreTensor); if (inputType == OperandType::TENSOR_FLOAT16) { inExpectedTypes = { OperandType::TENSOR_FLOAT16, OperandType::TENSOR_FLOAT16, OperandType::TENSOR_INT32, OperandType::FLOAT16, OperandType::INT32, OperandType::INT32, OperandType::FLOAT16, OperandType::FLOAT16, OperandType::FLOAT16}; outExpectedTypes = {OperandType::TENSOR_FLOAT16, OperandType::TENSOR_FLOAT16, OperandType::TENSOR_INT32, OperandType::TENSOR_INT32}; } else if (inputType == OperandType::TENSOR_FLOAT32) { inExpectedTypes = { OperandType::TENSOR_FLOAT32, OperandType::TENSOR_FLOAT32, OperandType::TENSOR_INT32, OperandType::FLOAT32, OperandType::INT32, OperandType::INT32, OperandType::FLOAT32, OperandType::FLOAT32, OperandType::FLOAT32}; outExpectedTypes = {OperandType::TENSOR_FLOAT32, OperandType::TENSOR_FLOAT32, OperandType::TENSOR_INT32, OperandType::TENSOR_INT32}; } else if (inputType == OperandType::TENSOR_QUANT8_ASYMM) { inExpectedTypes = {OperandType::TENSOR_QUANT8_ASYMM, OperandType::TENSOR_QUANT16_ASYMM, OperandType::TENSOR_INT32, OperandType::FLOAT32, OperandType::INT32, OperandType::INT32, OperandType::FLOAT32, OperandType::FLOAT32, OperandType::FLOAT32}; outExpectedTypes = {OperandType::TENSOR_QUANT8_ASYMM, OperandType::TENSOR_QUANT16_ASYMM, OperandType::TENSOR_INT32, OperandType::TENSOR_INT32}; } else { NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << kOperationName; } NN_RET_CHECK(validateInputTypes(context, inExpectedTypes)); NN_RET_CHECK(validateOutputTypes(context, outExpectedTypes)); return validateHalVersion(context, HalVersion::V1_2); } bool prepare(IOperationExecutionContext* context) { Shape scoreShape = context->getInputShape(kScoreTensor); Shape roiShape = context->getInputShape(kRoiTensor); Shape batchesShape = context->getInputShape(kBatchesTensor); Shape outputScoreShape = context->getOutputShape(kOutputScoreTensor); Shape outputRoiShape = context->getOutputShape(kOutputRoiTensor); Shape outputClassShape = context->getOutputShape(kOutputClassTensor); Shape outputBatchSplitShape = context->getOutputShape(kOutputBatchesTensor); NN_RET_CHECK(getNumberOfDimensions(scoreShape) == 2); NN_RET_CHECK(getNumberOfDimensions(roiShape) == 2); NN_RET_CHECK(getNumberOfDimensions(batchesShape) == 1); // Only numRois can be zero. const uint32_t kRoiDim = 4; uint32_t numRois = getSizeOfDimension(scoreShape, 0); uint32_t numClasses = getSizeOfDimension(scoreShape, 1); NN_RET_CHECK(getSizeOfDimension(roiShape, 0) == numRois); NN_RET_CHECK(getSizeOfDimension(roiShape, 1) == kRoiDim * numClasses); NN_RET_CHECK(getSizeOfDimension(batchesShape, 0) == numRois); NN_RET_CHECK_GT(numClasses, 1); if (scoreShape.type == OperandType::TENSOR_QUANT8_ASYMM) { NN_RET_CHECK_EQ(roiShape.scale, 0.125f); NN_RET_CHECK_EQ(roiShape.offset, 0); } outputScoreShape.type = scoreShape.type; outputScoreShape.dimensions = {0}; outputScoreShape.scale = scoreShape.scale; outputScoreShape.offset = scoreShape.offset; NN_RET_CHECK(context->setOutputShape(kOutputScoreTensor, outputScoreShape)); outputRoiShape.type = roiShape.type; outputRoiShape.dimensions = {0, 4}; outputRoiShape.scale = 0.125f; outputRoiShape.offset = 0; NN_RET_CHECK(context->setOutputShape(kOutputRoiTensor, outputRoiShape)); outputClassShape.type = OperandType::TENSOR_INT32; outputClassShape.dimensions = {0}; NN_RET_CHECK(context->setOutputShape(kOutputClassTensor, outputClassShape)); outputBatchSplitShape.type = batchesShape.type; outputBatchSplitShape.dimensions = {0}; NN_RET_CHECK(context->setOutputShape(kOutputBatchesTensor, outputBatchSplitShape)); return true; } bool execute(IOperationExecutionContext* context) { NNTRACE_TRANS("boxWithNMSLimit"); // Bypass execution in the case of zero numRois. if (getSizeOfDimension(context->getInputShape(kScoreTensor), 0) == 0) return true; switch (context->getInputType(kScoreTensor)) { case OperandType::TENSOR_FLOAT16: { return boxWithNmsLimitFloat16( context->getInputBuffer<_Float16>(kScoreTensor), context->getInputShape(kScoreTensor), context->getInputBuffer<_Float16>(kRoiTensor), context->getInputShape(kRoiTensor), context->getInputBuffer<int32_t>(kBatchesTensor), context->getInputShape(kBatchesTensor), context->getInputValue<_Float16>(kScoreThresholdScalar), context->getInputValue<int32_t>(kMaxNumDetectionScalar), context->getInputValue<int32_t>(kNmsKernelScalar), context->getInputValue<_Float16>(kIoUThresholdScalar), context->getInputValue<_Float16>(kSigmaScalar), context->getInputValue<_Float16>(kNmsScoreThresholdScalar), context->getOutputBuffer<_Float16>(kOutputScoreTensor), context->getOutputShape(kOutputScoreTensor), context->getOutputBuffer<_Float16>(kOutputRoiTensor), context->getOutputShape(kOutputRoiTensor), context->getOutputBuffer<int32_t>(kOutputClassTensor), context->getOutputShape(kOutputClassTensor), context->getOutputBuffer<int32_t>(kOutputBatchesTensor), context->getOutputShape(kOutputBatchesTensor), context); } case OperandType::TENSOR_FLOAT32: { return boxWithNmsLimitFloat32(context->getInputBuffer<float>(kScoreTensor), context->getInputShape(kScoreTensor), context->getInputBuffer<float>(kRoiTensor), context->getInputShape(kRoiTensor), context->getInputBuffer<int32_t>(kBatchesTensor), context->getInputShape(kBatchesTensor), context->getInputValue<float>(kScoreThresholdScalar), context->getInputValue<int32_t>(kMaxNumDetectionScalar), context->getInputValue<int32_t>(kNmsKernelScalar), context->getInputValue<float>(kIoUThresholdScalar), context->getInputValue<float>(kSigmaScalar), context->getInputValue<float>(kNmsScoreThresholdScalar), context->getOutputBuffer<float>(kOutputScoreTensor), context->getOutputShape(kOutputScoreTensor), context->getOutputBuffer<float>(kOutputRoiTensor), context->getOutputShape(kOutputRoiTensor), context->getOutputBuffer<int32_t>(kOutputClassTensor), context->getOutputShape(kOutputClassTensor), context->getOutputBuffer<int32_t>(kOutputBatchesTensor), context->getOutputShape(kOutputBatchesTensor), context); } case OperandType::TENSOR_QUANT8_ASYMM: { return boxWithNmsLimitQuant(context->getInputBuffer<uint8_t>(kScoreTensor), context->getInputShape(kScoreTensor), context->getInputBuffer<uint16_t>(kRoiTensor), context->getInputShape(kRoiTensor), context->getInputBuffer<int32_t>(kBatchesTensor), context->getInputShape(kBatchesTensor), context->getInputValue<float>(kScoreThresholdScalar), context->getInputValue<int32_t>(kMaxNumDetectionScalar), context->getInputValue<int32_t>(kNmsKernelScalar), context->getInputValue<float>(kIoUThresholdScalar), context->getInputValue<float>(kSigmaScalar), context->getInputValue<float>(kNmsScoreThresholdScalar), context->getOutputBuffer<uint8_t>(kOutputScoreTensor), context->getOutputShape(kOutputScoreTensor), context->getOutputBuffer<uint16_t>(kOutputRoiTensor), context->getOutputShape(kOutputRoiTensor), context->getOutputBuffer<int32_t>(kOutputClassTensor), context->getOutputShape(kOutputClassTensor), context->getOutputBuffer<int32_t>(kOutputBatchesTensor), context->getOutputShape(kOutputBatchesTensor), context); } default: NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << kOperationName; } } } // namespace box_with_nms_limit namespace generate_proposals { constexpr char kOperationName[] = "GENERATE_PROPOSALS"; constexpr uint32_t kNumInputs = 11; constexpr uint32_t kScoreTensor = 0; constexpr uint32_t kDeltaTensor = 1; constexpr uint32_t kAnchorTensor = 2; constexpr uint32_t kImageInfoTensor = 3; constexpr uint32_t kHeightStrideSalar = 4; constexpr uint32_t kWidthStrideScalar = 5; constexpr uint32_t kPreNmsMaxScalar = 6; constexpr uint32_t kPostNmsMaxScalar = 7; constexpr uint32_t kIoUThresholdScalar = 8; constexpr uint32_t kMinSizeScalar = 9; constexpr uint32_t kLayoutScalar = 10; constexpr uint32_t kNumOutputs = 3; constexpr uint32_t kOutputScoreTensor = 0; constexpr uint32_t kOutputRoiTensor = 1; constexpr uint32_t kOutputBatchesTensor = 2; namespace { void filterBoxes(const float* roiBase, const float* imageInfoBase, float minSize, std::vector<uint32_t>* select) { const uint32_t kRoiDim = 4; uint32_t i = 0; for (uint32_t j = 0; j < select->size(); j++) { const float* roiInfo = roiBase + (*select)[j] * kRoiDim; float roiWidth, roiHeight, xRoiCenter, yRoiCenter; roiWidth = roiInfo[2] - roiInfo[0]; roiHeight = roiInfo[3] - roiInfo[1]; xRoiCenter = roiInfo[0] + roiWidth / 2.0f; yRoiCenter = roiInfo[1] + roiHeight / 2.0f; if (roiWidth > minSize && roiHeight > minSize && xRoiCenter < imageInfoBase[1] && yRoiCenter < imageInfoBase[0]) { (*select)[i++] = (*select)[j]; } } select->resize(i); } bool generateProposalsNhwcFloat32Compute(const float* scoresData, const Shape& scoresShape, const float* bboxDeltasData, const Shape& bboxDeltasShape, const float* anchorsData, const Shape& anchorsShape, const float* imageInfoData, const Shape& imageInfoShape, float heightStride, float widthStride, int32_t preNmsTopN, int32_t postNmsTopN, float iouThreshold, float minSize, std::vector<float>* scoresOutData, std::vector<float>* roiOutData, std::vector<int32_t>* batchesOutData) { const uint32_t kRoiDim = 4; uint32_t numBatches = getSizeOfDimension(scoresShape, 0); uint32_t height = getSizeOfDimension(scoresShape, 1); uint32_t width = getSizeOfDimension(scoresShape, 2); uint32_t numAnchors = getSizeOfDimension(scoresShape, 3); uint32_t imageInfoLength = getSizeOfDimension(imageInfoShape, 1); uint32_t batchSize = height * width * numAnchors; uint32_t roiBufferSize = batchSize * kRoiDim; std::vector<float> roiBuffer(roiBufferSize); std::vector<float> roiTransformedBuffer(roiBufferSize); scoresOutData->clear(); roiOutData->clear(); batchesOutData->clear(); // Compute the roi region for each anchor. float* roiBase = roiBuffer.data(); for (uint32_t h = 0; h < height; h++) { float hShift = h * heightStride; for (uint32_t w = 0; w < width; w++) { const float* anchorsBase = anchorsData; float wShift = w * widthStride; for (uint32_t a = 0; a < numAnchors; a++, roiBase += kRoiDim, anchorsBase += kRoiDim) { roiBase[0] = anchorsBase[0] + wShift; roiBase[1] = anchorsBase[1] + hShift; roiBase[2] = anchorsBase[2] + wShift; roiBase[3] = anchorsBase[3] + hShift; } } } const float* scoresBase = scoresData; const float* bboxDeltasBase = bboxDeltasData; const float* imageInfoBase = imageInfoData; // Need to fake some data to satisfy bboxTransform. Shape tempRoiShape = anchorsShape; tempRoiShape.dimensions = {batchSize, kRoiDim}; Shape tempBBoxDeltasShape = bboxDeltasShape; tempBBoxDeltasShape.dimensions = {batchSize, kRoiDim}; std::vector<int32_t> tempBatchSplitData(batchSize, 0); Shape tempbatchSplitShape = {.dimensions = {batchSize}}; Shape tempImageInfoShape = imageInfoShape; tempImageInfoShape.dimensions = {1, imageInfoLength}; for (uint32_t b = 0; b < numBatches; b++) { // Apply bboxDeltas to anchor locations. float tempImageInfo[] = {imageInfoBase[0], imageInfoBase[1]}; if (!bboxTransformFloat32(roiBuffer.data(), tempRoiShape, bboxDeltasBase, tempBBoxDeltasShape, tempBatchSplitData.data(), tempbatchSplitShape, tempImageInfo, tempImageInfoShape, roiTransformedBuffer.data(), tempRoiShape)) { LOG(ERROR) << "BBoxTransform step failed in GENERATE_PROPOSALS op."; return false; } // Find the top preNmsTopN scores. std::vector<uint32_t> select(batchSize); std::iota(select.begin(), select.end(), 0); if (preNmsTopN > 0 && preNmsTopN < select.size()) { std::sort(select.begin(), select.end(), [&scoresBase](const uint32_t lhs, const uint32_t rhs) { return scoresBase[lhs] > scoresBase[rhs]; }); select.resize(preNmsTopN); } // Filter boxes, disgard regions with height or width < minSize. filterBoxes(roiTransformedBuffer.data(), imageInfoBase, minSize, &select); // Apply hard NMS. uint32_t* selectEnd = box_with_nms_limit::hardNmsSingleClass( scoresBase, iouThreshold, postNmsTopN, [&roiTransformedBuffer](uint32_t ind) { return roiTransformedBuffer.data() + ind * kRoiDim; }, select.data(), select.size()); uint32_t selectSize = selectEnd - select.data(); select.resize(selectSize); // Write output. for (auto i : select) { roiOutData->insert(roiOutData->end(), roiTransformedBuffer.begin() + i * kRoiDim, roiTransformedBuffer.begin() + (i + 1) * kRoiDim); scoresOutData->push_back(scoresBase[i]); batchesOutData->push_back(b); } scoresBase += batchSize; bboxDeltasBase += roiBufferSize; imageInfoBase += imageInfoLength; } return true; } bool generateProposalsFloat32Compute(const float* scoresData, const Shape& scoresShape, const float* bboxDeltasData, const Shape& bboxDeltasShape, const float* anchorsData, const Shape& anchorsShape, const float* imageInfoData, const Shape& imageInfoShape, float heightStride, float widthStride, int32_t preNmsTopN, int32_t postNmsTopN, float iouThreshold, float minSize, bool useNchw, std::vector<float>* scoresOutData, std::vector<float>* roiOutData, std::vector<int32_t>* batchesOutData) { InputWithLayout<float> score_nhwc(useNchw), delta_nhwc(useNchw); NN_RET_CHECK(score_nhwc.initialize(scoresData, scoresShape)); NN_RET_CHECK(delta_nhwc.initialize(bboxDeltasData, bboxDeltasShape)); return generateProposalsNhwcFloat32Compute( score_nhwc.getNhwcBuffer(), score_nhwc.getNhwcShape(), delta_nhwc.getNhwcBuffer(), delta_nhwc.getNhwcShape(), anchorsData, anchorsShape, imageInfoData, imageInfoShape, heightStride, widthStride, preNmsTopN, postNmsTopN, iouThreshold, minSize, scoresOutData, roiOutData, batchesOutData); } bool generateProposalsFloat32(const float* scoresData, const Shape& scoresShape, const float* bboxDeltasData, const Shape& bboxDeltasShape, const float* anchorsData, const Shape& anchorsShape, const float* imageInfoData, const Shape& imageInfoShape, float heightStride, float widthStride, int32_t preNmsTopN, int32_t postNmsTopN, float iouThreshold, float minSize, bool useNchw, IOperationExecutionContext* context) { std::vector<float> scoresOut_float32, roiOut_float32; std::vector<int32_t> batchesOut; NN_RET_CHECK(generateProposalsFloat32Compute( scoresData, scoresShape, bboxDeltasData, bboxDeltasShape, anchorsData, anchorsShape, imageInfoData, imageInfoShape, heightStride, widthStride, preNmsTopN, postNmsTopN, iouThreshold, minSize, useNchw, &scoresOut_float32, &roiOut_float32, &batchesOut)); // Set output dimensions. uint32_t numOutRois = scoresOut_float32.size(); if (numOutRois == 0) return true; Shape scoresOutShape = context->getOutputShape(kOutputScoreTensor); scoresOutShape.dimensions = {numOutRois}; NN_RET_CHECK(context->setOutputShape(kOutputScoreTensor, scoresOutShape)); Shape roiOutShape = context->getOutputShape(kOutputRoiTensor); roiOutShape.dimensions = {numOutRois, 4}; NN_RET_CHECK(context->setOutputShape(kOutputRoiTensor, roiOutShape)); Shape batchesOutShape = context->getOutputShape(kOutputBatchesTensor); batchesOutShape.dimensions = {numOutRois}; NN_RET_CHECK(context->setOutputShape(kOutputBatchesTensor, batchesOutShape)); // Write outputs. float* scoresOutData = context->getOutputBuffer<float>(kOutputScoreTensor); for (uint32_t i = 0; i < scoresOut_float32.size(); i++) { scoresOutData[i] = scoresOut_float32[i]; } float* roiOutData = context->getOutputBuffer<float>(kOutputRoiTensor); for (uint32_t i = 0; i < roiOut_float32.size(); i++) { roiOutData[i] = roiOut_float32[i]; } int32_t* batchesOutData = context->getOutputBuffer<int32_t>(kOutputBatchesTensor); for (uint32_t i = 0; i < batchesOut.size(); i++) { batchesOutData[i] = batchesOut[i]; } return true; } bool generateProposalsFloat16(const _Float16* scoresData, const Shape& scoresShape, const _Float16* bboxDeltasData, const Shape& bboxDeltasShape, const _Float16* anchorsData, const Shape& anchorsShape, const _Float16* imageInfoData, const Shape& imageInfoShape, float heightStride, float widthStride, int32_t preNmsTopN, int32_t postNmsTopN, float iouThreshold, float minSize, bool useNchw, IOperationExecutionContext* context) { std::vector<float> score_float32(getNumberOfElements(scoresShape)); convertFloat16ToFloat32(scoresData, &score_float32); std::vector<float> delta_float32(getNumberOfElements(bboxDeltasShape)); convertFloat16ToFloat32(bboxDeltasData, &delta_float32); std::vector<float> anchors_float32(getNumberOfElements(anchorsShape)); convertFloat16ToFloat32(anchorsData, &anchors_float32); std::vector<float> imageInfo_float32(getNumberOfElements(imageInfoShape)); convertFloat16ToFloat32(imageInfoData, &imageInfo_float32); std::vector<float> scoresOut_float32, roiOut_float32; std::vector<int32_t> batchesOut; NN_RET_CHECK(generateProposalsFloat32Compute( score_float32.data(), scoresShape, delta_float32.data(), bboxDeltasShape, anchors_float32.data(), anchorsShape, imageInfo_float32.data(), imageInfoShape, heightStride, widthStride, preNmsTopN, postNmsTopN, iouThreshold, minSize, useNchw, &scoresOut_float32, &roiOut_float32, &batchesOut)); // Set output dimensions. uint32_t numOutRois = scoresOut_float32.size(); if (numOutRois == 0) return true; Shape scoresOutShape = context->getOutputShape(kOutputScoreTensor); scoresOutShape.dimensions = {numOutRois}; NN_RET_CHECK(context->setOutputShape(kOutputScoreTensor, scoresOutShape)); Shape roiOutShape = context->getOutputShape(kOutputRoiTensor); roiOutShape.dimensions = {numOutRois, 4}; NN_RET_CHECK(context->setOutputShape(kOutputRoiTensor, roiOutShape)); Shape batchesOutShape = context->getOutputShape(kOutputBatchesTensor); batchesOutShape.dimensions = {numOutRois}; NN_RET_CHECK(context->setOutputShape(kOutputBatchesTensor, batchesOutShape)); // Write outputs. _Float16* scoresOutData = context->getOutputBuffer<_Float16>(kOutputScoreTensor); convertFloat32ToFloat16(scoresOut_float32, scoresOutData); _Float16* roiOutData = context->getOutputBuffer<_Float16>(kOutputRoiTensor); convertFloat32ToFloat16(roiOut_float32, roiOutData); int32_t* batchesOutData = context->getOutputBuffer<int32_t>(kOutputBatchesTensor); for (uint32_t i = 0; i < batchesOut.size(); i++) { batchesOutData[i] = batchesOut[i]; } return true; } bool generateProposalsQuant(const uint8_t* scoresData, const Shape& scoresShape, const uint8_t* bboxDeltasData, const Shape& bboxDeltasShape, const int16_t* anchorsData, const Shape& anchorsShape, const uint16_t* imageInfoData, const Shape& imageInfoShape, float heightStride, float widthStride, int32_t preNmsTopN, int32_t postNmsTopN, float iouThreshold, float minSize, bool useNchw, IOperationExecutionContext* context) { std::vector<float> score_float32(getNumberOfElements(scoresShape)); convertQuantToFloat32(scoresData, scoresShape.scale, scoresShape.offset, &score_float32); std::vector<float> delta_float32(getNumberOfElements(bboxDeltasShape)); convertQuantToFloat32(bboxDeltasData, bboxDeltasShape.scale, bboxDeltasShape.offset, &delta_float32); std::vector<float> anchors_float32(getNumberOfElements(anchorsShape)); convertQuantToFloat32(anchorsData, anchorsShape.scale, anchorsShape.offset, &anchors_float32); std::vector<float> imageInfo_float32(getNumberOfElements(imageInfoShape)); convertQuantToFloat32(imageInfoData, imageInfoShape.scale, imageInfoShape.offset, &imageInfo_float32); std::vector<float> scoresOut_float32, roiOut_float32; std::vector<int32_t> batchesOut; NN_RET_CHECK(generateProposalsFloat32Compute( score_float32.data(), scoresShape, delta_float32.data(), bboxDeltasShape, anchors_float32.data(), anchorsShape, imageInfo_float32.data(), imageInfoShape, heightStride, widthStride, preNmsTopN, postNmsTopN, iouThreshold, minSize, useNchw, &scoresOut_float32, &roiOut_float32, &batchesOut)); // Set output dimensions. uint32_t numOutRois = scoresOut_float32.size(); if (numOutRois == 0) return true; Shape scoresOutShape = context->getOutputShape(kOutputScoreTensor); scoresOutShape.dimensions = {numOutRois}; NN_RET_CHECK(context->setOutputShape(kOutputScoreTensor, scoresOutShape)); Shape roiOutShape = context->getOutputShape(kOutputRoiTensor); roiOutShape.dimensions = {numOutRois, 4}; NN_RET_CHECK(context->setOutputShape(kOutputRoiTensor, roiOutShape)); Shape batchesOutShape = context->getOutputShape(kOutputBatchesTensor); batchesOutShape.dimensions = {numOutRois}; NN_RET_CHECK(context->setOutputShape(kOutputBatchesTensor, batchesOutShape)); // Write outputs. uint8_t* scoresOutData = context->getOutputBuffer<uint8_t>(kOutputScoreTensor); convertFloat32ToQuant(scoresOut_float32, scoresOutShape.scale, scoresOutShape.offset, scoresOutData); uint16_t* roiOutData = context->getOutputBuffer<uint16_t>(kOutputRoiTensor); convertFloat32ToQuant(roiOut_float32, roiOutShape.scale, roiOutShape.offset, roiOutData); int32_t* batchesOutData = context->getOutputBuffer<int32_t>(kOutputBatchesTensor); for (uint32_t i = 0; i < batchesOut.size(); i++) { batchesOutData[i] = batchesOut[i]; } return true; } } // namespace bool validate(const IOperationValidationContext* context) { NN_RET_CHECK_EQ(context->getNumInputs(), kNumInputs); NN_RET_CHECK_EQ(context->getNumOutputs(), kNumOutputs); std::vector<OperandType> inExpectedTypes; std::vector<OperandType> outExpectedTypes; auto inputType = context->getInputType(kScoreTensor); if (inputType == OperandType::TENSOR_FLOAT16) { inExpectedTypes = {OperandType::TENSOR_FLOAT16, OperandType::TENSOR_FLOAT16, OperandType::TENSOR_FLOAT16, OperandType::TENSOR_FLOAT16, OperandType::FLOAT16, OperandType::FLOAT16, OperandType::INT32, OperandType::INT32, OperandType::FLOAT16, OperandType::FLOAT16, OperandType::BOOL}; outExpectedTypes = {OperandType::TENSOR_FLOAT16, OperandType::TENSOR_FLOAT16, OperandType::TENSOR_INT32}; } else if (inputType == OperandType::TENSOR_FLOAT32) { inExpectedTypes = {OperandType::TENSOR_FLOAT32, OperandType::TENSOR_FLOAT32, OperandType::TENSOR_FLOAT32, OperandType::TENSOR_FLOAT32, OperandType::FLOAT32, OperandType::FLOAT32, OperandType::INT32, OperandType::INT32, OperandType::FLOAT32, OperandType::FLOAT32, OperandType::BOOL}; outExpectedTypes = {OperandType::TENSOR_FLOAT32, OperandType::TENSOR_FLOAT32, OperandType::TENSOR_INT32}; } else if (inputType == OperandType::TENSOR_QUANT8_ASYMM) { inExpectedTypes = {OperandType::TENSOR_QUANT8_ASYMM, OperandType::TENSOR_QUANT8_ASYMM, OperandType::TENSOR_QUANT16_SYMM, OperandType::TENSOR_QUANT16_ASYMM, OperandType::FLOAT32, OperandType::FLOAT32, OperandType::INT32, OperandType::INT32, OperandType::FLOAT32, OperandType::FLOAT32, OperandType::BOOL}; outExpectedTypes = {OperandType::TENSOR_QUANT8_ASYMM, OperandType::TENSOR_QUANT16_ASYMM, OperandType::TENSOR_INT32}; } else { NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << kOperationName; } NN_RET_CHECK(validateInputTypes(context, inExpectedTypes)); NN_RET_CHECK(validateOutputTypes(context, outExpectedTypes)); return validateHalVersion(context, HalVersion::V1_2); } bool prepare(IOperationExecutionContext* context) { bool useNchw = context->getInputValue<bool>(kLayoutScalar); Shape scoreShape = context->getInputShape(kScoreTensor); Shape bboxDeltasShape = context->getInputShape(kDeltaTensor); Shape anchorsShape = context->getInputShape(kAnchorTensor); Shape imageInfoDataShape = context->getInputShape(kImageInfoTensor); Shape outputScoreShape = context->getOutputShape(kOutputScoreTensor); Shape outputRoiShape = context->getOutputShape(kOutputRoiTensor); Shape outputBatchSplitShape = context->getOutputShape(kOutputBatchesTensor); NN_RET_CHECK_EQ(getNumberOfDimensions(scoreShape), 4); NN_RET_CHECK_EQ(getNumberOfDimensions(bboxDeltasShape), 4); NN_RET_CHECK_EQ(getNumberOfDimensions(anchorsShape), 2); NN_RET_CHECK_EQ(getNumberOfDimensions(imageInfoDataShape), 2); const uint32_t kRoiDim = 4; uint32_t numBatches = getSizeOfDimension(scoreShape, 0); uint32_t height = getSizeOfDimension(scoreShape, useNchw ? 2 : 1); uint32_t width = getSizeOfDimension(scoreShape, useNchw ? 3 : 2); uint32_t numAnchors = getSizeOfDimension(scoreShape, useNchw ? 1 : 3); NN_RET_CHECK_EQ(getSizeOfDimension(bboxDeltasShape, 0), numBatches); NN_RET_CHECK_EQ(getSizeOfDimension(bboxDeltasShape, useNchw ? 2 : 1), height); NN_RET_CHECK_EQ(getSizeOfDimension(bboxDeltasShape, useNchw ? 3 : 2), width); NN_RET_CHECK_EQ(getSizeOfDimension(bboxDeltasShape, useNchw ? 1 : 3), numAnchors * kRoiDim); NN_RET_CHECK_EQ(getSizeOfDimension(imageInfoDataShape, 0), numBatches); NN_RET_CHECK_EQ(getSizeOfDimension(imageInfoDataShape, 1), 2); NN_RET_CHECK_EQ(getSizeOfDimension(anchorsShape, 0), numAnchors); NN_RET_CHECK_EQ(getSizeOfDimension(anchorsShape, 1), kRoiDim); if (scoreShape.type == OperandType::TENSOR_QUANT8_ASYMM) { NN_RET_CHECK_EQ(anchorsShape.scale, 0.125f); NN_RET_CHECK_EQ(imageInfoDataShape.scale, 0.125f); NN_RET_CHECK_EQ(imageInfoDataShape.offset, 0); } outputScoreShape.type = scoreShape.type; outputScoreShape.dimensions = {0}; outputScoreShape.scale = scoreShape.scale; outputScoreShape.offset = scoreShape.offset; NN_RET_CHECK(context->setOutputShape(kOutputScoreTensor, outputScoreShape)); outputRoiShape.dimensions = {0, 4}; if (scoreShape.type == OperandType::TENSOR_QUANT8_ASYMM) { outputRoiShape.scale = 0.125f; outputRoiShape.offset = 0; } NN_RET_CHECK(context->setOutputShape(kOutputRoiTensor, outputRoiShape)); outputBatchSplitShape.dimensions = {0}; NN_RET_CHECK(context->setOutputShape(kOutputBatchesTensor, outputBatchSplitShape)); return true; } bool execute(IOperationExecutionContext* context) { NNTRACE_TRANS("generateProposals"); switch (context->getInputType(kScoreTensor)) { case OperandType::TENSOR_FLOAT16: { return generateProposalsFloat16(context->getInputBuffer<_Float16>(kScoreTensor), context->getInputShape(kScoreTensor), context->getInputBuffer<_Float16>(kDeltaTensor), context->getInputShape(kDeltaTensor), context->getInputBuffer<_Float16>(kAnchorTensor), context->getInputShape(kAnchorTensor), context->getInputBuffer<_Float16>(kImageInfoTensor), context->getInputShape(kImageInfoTensor), context->getInputValue<_Float16>(kHeightStrideSalar), context->getInputValue<_Float16>(kWidthStrideScalar), context->getInputValue<int32_t>(kPreNmsMaxScalar), context->getInputValue<int32_t>(kPostNmsMaxScalar), context->getInputValue<_Float16>(kIoUThresholdScalar), context->getInputValue<_Float16>(kMinSizeScalar), context->getInputValue<bool>(kLayoutScalar), context); } case OperandType::TENSOR_FLOAT32: { return generateProposalsFloat32(context->getInputBuffer<float>(kScoreTensor), context->getInputShape(kScoreTensor), context->getInputBuffer<float>(kDeltaTensor), context->getInputShape(kDeltaTensor), context->getInputBuffer<float>(kAnchorTensor), context->getInputShape(kAnchorTensor), context->getInputBuffer<float>(kImageInfoTensor), context->getInputShape(kImageInfoTensor), context->getInputValue<float>(kHeightStrideSalar), context->getInputValue<float>(kWidthStrideScalar), context->getInputValue<int32_t>(kPreNmsMaxScalar), context->getInputValue<int32_t>(kPostNmsMaxScalar), context->getInputValue<float>(kIoUThresholdScalar), context->getInputValue<float>(kMinSizeScalar), context->getInputValue<bool>(kLayoutScalar), context); } case OperandType::TENSOR_QUANT8_ASYMM: { return generateProposalsQuant(context->getInputBuffer<uint8_t>(kScoreTensor), context->getInputShape(kScoreTensor), context->getInputBuffer<uint8_t>(kDeltaTensor), context->getInputShape(kDeltaTensor), context->getInputBuffer<int16_t>(kAnchorTensor), context->getInputShape(kAnchorTensor), context->getInputBuffer<uint16_t>(kImageInfoTensor), context->getInputShape(kImageInfoTensor), context->getInputValue<float>(kHeightStrideSalar), context->getInputValue<float>(kWidthStrideScalar), context->getInputValue<int32_t>(kPreNmsMaxScalar), context->getInputValue<int32_t>(kPostNmsMaxScalar), context->getInputValue<float>(kIoUThresholdScalar), context->getInputValue<float>(kMinSizeScalar), context->getInputValue<bool>(kLayoutScalar), context); } default: NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << kOperationName; } } } // namespace generate_proposals namespace detection_postprocess { constexpr char kOperationName[] = "DETECTION_POSTPROCESS"; constexpr uint32_t kNumInputs = 14; constexpr uint32_t kScoreTensor = 0; constexpr uint32_t kDeltaTensor = 1; constexpr uint32_t kAnchorTensor = 2; constexpr uint32_t kScaleYScalar = 3; constexpr uint32_t kScaleXScalar = 4; constexpr uint32_t kScaleHScalar = 5; constexpr uint32_t kScaleWScalar = 6; constexpr uint32_t kUseRegularNmsScalar = 7; constexpr uint32_t kMaxNumDetectionScalar = 8; constexpr uint32_t kMaxClassesPerDetectionScalar = 9; constexpr uint32_t kMaxNumDetectionPerClassScalar = 10; constexpr uint32_t kScoreThresholdScalar = 11; constexpr uint32_t kIoUThresholdScalar = 12; constexpr uint32_t kIsBGInLabelScalar = 13; constexpr uint32_t kNumOutputs = 4; constexpr uint32_t kOutputScoreTensor = 0; constexpr uint32_t kOutputRoiTensor = 1; constexpr uint32_t kOutputClassTensor = 2; constexpr uint32_t kOutputDetectionTensor = 3; namespace { bool detectionPostprocessFloat32( const float* scoreData, const Shape& scoreShape, const float* deltaData, const Shape& deltaShape, const float* anchorData, const Shape& anchorShape, float scaleY, float scaleX, float scaleH, float scaleW, bool useRegularNms, int32_t maxNumDetections, int32_t maxClassesPerDetection, int32_t maxNumDetectionsPerClass, float iouThreshold, float scoreThreshold, bool isBGInLabel, float* scoreOutData, const Shape& scoreOutShape, float* roiOutData, const Shape& roiOutShape, int32_t* classOutData, const Shape& classOutShape, int32_t* detectionOutData, const Shape& detectionOutShape) { const uint32_t kRoiDim = 4; uint32_t numBatches = getSizeOfDimension(scoreShape, 0); uint32_t numAnchors = getSizeOfDimension(scoreShape, 1); uint32_t numClasses = getSizeOfDimension(scoreShape, 2); uint32_t lengthBoxEncoding = getSizeOfDimension(deltaShape, 2); uint32_t numOutDetection = getSizeOfDimension(scoreOutShape, 1); memset(scoreOutData, 0, getNumberOfElements(scoreOutShape) * sizeof(float)); memset(roiOutData, 0, getNumberOfElements(roiOutShape) * sizeof(float)); memset(classOutData, 0, getNumberOfElements(classOutShape) * sizeof(int32_t)); memset(detectionOutData, 0, getNumberOfElements(detectionOutShape) * sizeof(int32_t)); const float* scoreBase = scoreData; const float* deltaBase = deltaData; float* scoreOutBase = scoreOutData; float* roiOutBase = roiOutData; int32_t* classOutBase = classOutData; std::vector<float> roiBuffer(numAnchors * kRoiDim); std::vector<float> scoreBuffer(numAnchors); for (uint32_t b = 0; b < numBatches; b++) { const float* anchorBase = anchorData; for (uint32_t a = 0; a < numAnchors; a++) { float yCtr = anchorBase[0] + anchorBase[2] * deltaBase[0] / scaleY; float xCtr = anchorBase[1] + anchorBase[3] * deltaBase[1] / scaleX; float hHalf = anchorBase[2] * std::exp(deltaBase[2] / scaleH) * 0.5f; float wHalf = anchorBase[3] * std::exp(deltaBase[3] / scaleW) * 0.5f; roiBuffer[a * kRoiDim] = yCtr - hHalf; roiBuffer[a * kRoiDim + 1] = xCtr - wHalf; roiBuffer[a * kRoiDim + 2] = yCtr + hHalf; roiBuffer[a * kRoiDim + 3] = xCtr + wHalf; anchorBase += kRoiDim; deltaBase += lengthBoxEncoding; } if (useRegularNms) { std::vector<uint32_t> select; box_with_nms_limit::hardNmsMultiClass( scoreBase, numClasses, numAnchors, scoreThreshold, iouThreshold, maxNumDetections, maxNumDetectionsPerClass, [&roiBuffer, numClasses](uint32_t ind) { return roiBuffer.data() + (ind / numClasses) * kRoiDim; }, &select); for (uint32_t i = 0; i < select.size(); i++) { uint32_t ind = select[i]; scoreOutBase[i] = scoreBase[ind]; memcpy(roiOutBase + i * kRoiDim, &roiBuffer[(ind / numClasses) * kRoiDim], kRoiDim * sizeof(float)); classOutBase[i] = (ind % numClasses) - (isBGInLabel ? 0 : 1); } *detectionOutData++ = select.size(); } else { uint32_t numOutClasses = std::min<uint32_t>(numClasses - 1, maxClassesPerDetection); std::vector<float> maxScores(numAnchors); for (uint32_t a = 0; a < numAnchors; a++) { maxScores[a] = *std::max_element(scoreBase + a * numClasses + 1, scoreBase + (a + 1) * numClasses); } std::vector<uint32_t> select; for (uint32_t a = 0; a < numAnchors; a++) { if (maxScores[a] > scoreThreshold) { select.push_back(a); } } uint32_t* selectEnd = box_with_nms_limit::hardNmsSingleClass( maxScores.data(), iouThreshold, maxNumDetections, [&roiBuffer](uint32_t ind) { return roiBuffer.data() + ind * kRoiDim; }, select.data(), select.size()); select.resize(selectEnd - select.data()); float* scoreOutPtr = scoreOutBase; float* roiOutPtr = roiOutBase; int32_t* classOutPtr = classOutBase; for (auto i : select) { const float* score = scoreBase + i * numClasses; std::vector<uint32_t> scoreInds(numClasses - 1); std::iota(scoreInds.begin(), scoreInds.end(), 1); std::sort(scoreInds.begin(), scoreInds.end(), [&score](const uint32_t lhs, const uint32_t rhs) { return score[lhs] > score[rhs]; }); for (uint32_t c = 0; c < numOutClasses; c++) { *scoreOutPtr++ = score[scoreInds[c]]; memcpy(roiOutPtr, &roiBuffer[i * kRoiDim], kRoiDim * sizeof(float)); roiOutPtr += kRoiDim; *classOutPtr++ = scoreInds[c] - (isBGInLabel ? 0 : 1); } } *detectionOutData++ = select.size() * numOutClasses; } scoreBase += numAnchors * numClasses; scoreOutBase += numOutDetection; roiOutBase += numOutDetection * kRoiDim; classOutBase += numOutDetection; } return true; } bool detectionPostprocessFloat16( const _Float16* scoreData, const Shape& scoreShape, const _Float16* deltaData, const Shape& deltaShape, const _Float16* anchorData, const Shape& anchorShape, float scaleY, float scaleX, float scaleH, float scaleW, bool useRegularNms, int32_t maxNumDetections, int32_t maxClassesPerDetection, int32_t maxNumDetectionsPerClass, float iouThreshold, float scoreThreshold, bool isBGInLabel, _Float16* scoreOutData, const Shape& scoreOutShape, _Float16* roiOutData, const Shape& roiOutShape, int32_t* classOutData, const Shape& classOutShape, int32_t* detectionOutData, const Shape& detectionOutShape) { std::vector<float> scores_float32(getNumberOfElements(scoreShape)); convertFloat16ToFloat32(scoreData, &scores_float32); std::vector<float> delta_float32(getNumberOfElements(deltaShape)); convertFloat16ToFloat32(deltaData, &delta_float32); std::vector<float> anchor_float32(getNumberOfElements(anchorShape)); convertFloat16ToFloat32(anchorData, &anchor_float32); std::vector<float> outputScore_float32(getNumberOfElements(scoreOutShape)); std::vector<float> outputRoi_float32(getNumberOfElements(roiOutShape)); NN_RET_CHECK(detectionPostprocessFloat32( scores_float32.data(), scoreShape, delta_float32.data(), deltaShape, anchor_float32.data(), anchorShape, scaleY, scaleX, scaleH, scaleW, useRegularNms, maxNumDetections, maxClassesPerDetection, maxNumDetectionsPerClass, iouThreshold, scoreThreshold, isBGInLabel, outputScore_float32.data(), scoreOutShape, outputRoi_float32.data(), roiOutShape, classOutData, classOutShape, detectionOutData, detectionOutShape)); convertFloat32ToFloat16(outputScore_float32, scoreOutData); convertFloat32ToFloat16(outputRoi_float32, roiOutData); return true; } } // namespace bool validate(const IOperationValidationContext* context) { NN_RET_CHECK_EQ(context->getNumInputs(), kNumInputs); NN_RET_CHECK_EQ(context->getNumOutputs(), kNumOutputs); std::vector<OperandType> inExpectedTypes; std::vector<OperandType> outExpectedTypes; auto inputType = context->getInputType(kScoreTensor); if (inputType == OperandType::TENSOR_FLOAT16) { inExpectedTypes = {OperandType::TENSOR_FLOAT16, OperandType::TENSOR_FLOAT16, OperandType::TENSOR_FLOAT16, OperandType::FLOAT16, OperandType::FLOAT16, OperandType::FLOAT16, OperandType::FLOAT16, OperandType::BOOL, OperandType::INT32, OperandType::INT32, OperandType::INT32, OperandType::FLOAT16, OperandType::FLOAT16, OperandType::BOOL}; } else if (inputType == OperandType::TENSOR_FLOAT32) { inExpectedTypes = {OperandType::TENSOR_FLOAT32, OperandType::TENSOR_FLOAT32, OperandType::TENSOR_FLOAT32, OperandType::FLOAT32, OperandType::FLOAT32, OperandType::FLOAT32, OperandType::FLOAT32, OperandType::BOOL, OperandType::INT32, OperandType::INT32, OperandType::INT32, OperandType::FLOAT32, OperandType::FLOAT32, OperandType::BOOL}; } else { NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << kOperationName; } NN_RET_CHECK(validateInputTypes(context, inExpectedTypes)); NN_RET_CHECK(validateOutputTypes( context, {inputType, inputType, OperandType::TENSOR_INT32, OperandType::TENSOR_INT32})); return validateHalVersion(context, HalVersion::V1_2); } bool prepare(IOperationExecutionContext* context) { Shape scoreShape = context->getInputShape(kScoreTensor); Shape deltasShape = context->getInputShape(kDeltaTensor); Shape anchorsShape = context->getInputShape(kAnchorTensor); Shape outputScoreShape = context->getOutputShape(kOutputScoreTensor); Shape outputRoiShape = context->getOutputShape(kOutputRoiTensor); Shape outputClassShape = context->getOutputShape(kOutputClassTensor); Shape outputDetectionShape = context->getOutputShape(kOutputDetectionTensor); NN_RET_CHECK_EQ(getNumberOfDimensions(scoreShape), 3); NN_RET_CHECK_EQ(getNumberOfDimensions(deltasShape), 3); NN_RET_CHECK_EQ(getNumberOfDimensions(anchorsShape), 2); const uint32_t kRoiDim = 4; uint32_t numBatches = getSizeOfDimension(scoreShape, 0); uint32_t numAnchors = getSizeOfDimension(scoreShape, 1); uint32_t numClasses = getSizeOfDimension(scoreShape, 2); uint32_t lengthBoxEncoding = getSizeOfDimension(deltasShape, 2); uint32_t maxNumDetections = context->getInputValue<int32_t>(kMaxNumDetectionScalar); uint32_t maxClassesPerDetection = context->getInputValue<int32_t>(kMaxClassesPerDetectionScalar); uint32_t numOutDetections = maxNumDetections; NN_RET_CHECK_EQ(getSizeOfDimension(deltasShape, 0), numBatches); NN_RET_CHECK_EQ(getSizeOfDimension(deltasShape, 1), numAnchors); NN_RET_CHECK_EQ(getSizeOfDimension(anchorsShape, 0), numAnchors); NN_RET_CHECK_EQ(getSizeOfDimension(anchorsShape, 1), kRoiDim); if (scoreShape.type == OperandType::TENSOR_FLOAT32) { NN_RET_CHECK_GT(context->getInputValue<float>(kScaleYScalar), 0); NN_RET_CHECK_GT(context->getInputValue<float>(kScaleXScalar), 0); NN_RET_CHECK_GT(context->getInputValue<float>(kScaleHScalar), 0); NN_RET_CHECK_GT(context->getInputValue<float>(kScaleWScalar), 0); NN_RET_CHECK_GE(context->getInputValue<float>(kScoreThresholdScalar), 0); NN_RET_CHECK_GE(context->getInputValue<float>(kIoUThresholdScalar), 0); } else if (scoreShape.type == OperandType::TENSOR_FLOAT16) { NN_RET_CHECK(context->getInputValue<_Float16>(kScaleYScalar) > 0); NN_RET_CHECK(context->getInputValue<_Float16>(kScaleXScalar) > 0); NN_RET_CHECK(context->getInputValue<_Float16>(kScaleHScalar) > 0); NN_RET_CHECK(context->getInputValue<_Float16>(kScaleWScalar) > 0); NN_RET_CHECK(context->getInputValue<_Float16>(kScoreThresholdScalar) >= 0); NN_RET_CHECK(context->getInputValue<_Float16>(kIoUThresholdScalar) >= 0); } else { NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << kOperationName; } NN_RET_CHECK_GT(numClasses, 1); NN_RET_CHECK_GE(lengthBoxEncoding, 4); NN_RET_CHECK_GT(maxNumDetections, 0); if (context->getInputValue<bool>(kUseRegularNmsScalar)) { NN_RET_CHECK_GT(context->getInputValue<int32_t>(kMaxNumDetectionPerClassScalar), 0); } else { NN_RET_CHECK_GT(maxClassesPerDetection, 0); numOutDetections *= maxClassesPerDetection; } outputScoreShape.type = scoreShape.type; outputScoreShape.dimensions = {numBatches, numOutDetections}; NN_RET_CHECK(context->setOutputShape(kOutputScoreTensor, outputScoreShape)); outputRoiShape.type = anchorsShape.type; outputRoiShape.dimensions = {numBatches, numOutDetections, 4}; NN_RET_CHECK(context->setOutputShape(kOutputRoiTensor, outputRoiShape)); outputClassShape.type = OperandType::TENSOR_INT32; outputClassShape.dimensions = {numBatches, numOutDetections}; NN_RET_CHECK(context->setOutputShape(kOutputClassTensor, outputClassShape)); outputDetectionShape.type = OperandType::TENSOR_INT32; outputDetectionShape.dimensions = {numBatches}; NN_RET_CHECK(context->setOutputShape(kOutputDetectionTensor, outputDetectionShape)); return true; } bool execute(IOperationExecutionContext* context) { NNTRACE_TRANS("detectionPostProcess"); switch (context->getInputType(kScoreTensor)) { case OperandType::TENSOR_FLOAT16: { return detectionPostprocessFloat16( context->getInputBuffer<_Float16>(kScoreTensor), context->getInputShape(kScoreTensor), context->getInputBuffer<_Float16>(kDeltaTensor), context->getInputShape(kDeltaTensor), context->getInputBuffer<_Float16>(kAnchorTensor), context->getInputShape(kAnchorTensor), context->getInputValue<_Float16>(kScaleYScalar), context->getInputValue<_Float16>(kScaleXScalar), context->getInputValue<_Float16>(kScaleHScalar), context->getInputValue<_Float16>(kScaleWScalar), context->getInputValue<bool>(kUseRegularNmsScalar), context->getInputValue<int32_t>(kMaxNumDetectionScalar), context->getInputValue<int32_t>(kMaxClassesPerDetectionScalar), context->getInputValue<int32_t>(kMaxNumDetectionPerClassScalar), context->getInputValue<_Float16>(kIoUThresholdScalar), context->getInputValue<_Float16>(kScoreThresholdScalar), context->getInputValue<bool>(kIsBGInLabelScalar), context->getOutputBuffer<_Float16>(kOutputScoreTensor), context->getOutputShape(kOutputScoreTensor), context->getOutputBuffer<_Float16>(kOutputRoiTensor), context->getOutputShape(kOutputRoiTensor), context->getOutputBuffer<int32_t>(kOutputClassTensor), context->getOutputShape(kOutputClassTensor), context->getOutputBuffer<int32_t>(kOutputDetectionTensor), context->getOutputShape(kOutputDetectionTensor)); } case OperandType::TENSOR_FLOAT32: { return detectionPostprocessFloat32( context->getInputBuffer<float>(kScoreTensor), context->getInputShape(kScoreTensor), context->getInputBuffer<float>(kDeltaTensor), context->getInputShape(kDeltaTensor), context->getInputBuffer<float>(kAnchorTensor), context->getInputShape(kAnchorTensor), context->getInputValue<float>(kScaleYScalar), context->getInputValue<float>(kScaleXScalar), context->getInputValue<float>(kScaleHScalar), context->getInputValue<float>(kScaleWScalar), context->getInputValue<bool>(kUseRegularNmsScalar), context->getInputValue<int32_t>(kMaxNumDetectionScalar), context->getInputValue<int32_t>(kMaxClassesPerDetectionScalar), context->getInputValue<int32_t>(kMaxNumDetectionPerClassScalar), context->getInputValue<float>(kIoUThresholdScalar), context->getInputValue<float>(kScoreThresholdScalar), context->getInputValue<bool>(kIsBGInLabelScalar), context->getOutputBuffer<float>(kOutputScoreTensor), context->getOutputShape(kOutputScoreTensor), context->getOutputBuffer<float>(kOutputRoiTensor), context->getOutputShape(kOutputRoiTensor), context->getOutputBuffer<int32_t>(kOutputClassTensor), context->getOutputShape(kOutputClassTensor), context->getOutputBuffer<int32_t>(kOutputDetectionTensor), context->getOutputShape(kOutputDetectionTensor)); } default: NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << kOperationName; } } } // namespace detection_postprocess } // namespace bbox_ops NN_REGISTER_OPERATION(AXIS_ALIGNED_BBOX_TRANSFORM, bbox_ops::axis_aligned_bbox_transform::kOperationName, bbox_ops::axis_aligned_bbox_transform::validate, bbox_ops::axis_aligned_bbox_transform::prepare, bbox_ops::axis_aligned_bbox_transform::execute, .allowZeroSizedInput = true); NN_REGISTER_OPERATION(BOX_WITH_NMS_LIMIT, bbox_ops::box_with_nms_limit::kOperationName, bbox_ops::box_with_nms_limit::validate, bbox_ops::box_with_nms_limit::prepare, bbox_ops::box_with_nms_limit::execute, .allowZeroSizedInput = true); NN_REGISTER_OPERATION(GENERATE_PROPOSALS, bbox_ops::generate_proposals::kOperationName, bbox_ops::generate_proposals::validate, bbox_ops::generate_proposals::prepare, bbox_ops::generate_proposals::execute); NN_REGISTER_OPERATION(DETECTION_POSTPROCESSING, bbox_ops::detection_postprocess::kOperationName, bbox_ops::detection_postprocess::validate, bbox_ops::detection_postprocess::prepare, bbox_ops::detection_postprocess::execute); } // namespace nn } // namespace android