Skip to content

Commit

Permalink
add rtdetr
Browse files Browse the repository at this point in the history
  • Loading branch information
@ZHEQIUSHUI
ZHEQIUSHUI committed Nov 7, 2023
1 parent 1c3f7ef commit c1aa806
Show file tree
Hide file tree
Showing 3 changed files with 279 additions and 6 deletions.
1 change: 1 addition & 0 deletions examples/ax650/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ if (AXERA_TARGET_CHIP MATCHES "ax650" OR AXERA_TARGET_CHIP MATCHES "ax620e")
axera_example(ax_dinov2 ax_dinov2_steps.cc)
axera_example(ax_simcc_pose ax_simcc_pose_steps.cc)
axera_example(ax_glpdepth ax_glpdepth_steps.cc)
axera_example(ax_rtdetr ax_rtdetr_steps.cc)

axera_example(ax_imgproc ax_imgproc_steps.cc)
axera_example(ax_model_info ax_model_info.cc)
Expand Down
272 changes: 272 additions & 0 deletions examples/ax650/ax_rtdetr_steps.cc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,272 @@
/*
* AXERA is pleased to support the open source community by making ax-samples available.
*
* Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved.
*
* Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* https://opensource.org/licenses/BSD-3-Clause
*
* Unless required by applicable law or agreed to in writing, software distributed
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/

/*
* Author: ZHEQIUSHUI
*/

#include <cstdio>
#include <cstring>
#include <numeric>

#include <opencv2/opencv.hpp>
#include "base/common.hpp"
#include "base/detection.hpp"
#include "middleware/io.hpp"

#include "utilities/args.hpp"
#include "utilities/cmdline.hpp"
#include "utilities/file.hpp"
#include "utilities/timer.hpp"

#include <ax_sys_api.h>
#include <ax_engine_api.h>

const int DEFAULT_IMG_H = 640;
const int DEFAULT_IMG_W = 640;

const char* CLASS_NAMES[] = {
"person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light",
"fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
"elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
"skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard",
"tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
"sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch",
"potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone",
"microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear",
"hair drier", "toothbrush"};

const int DEFAULT_LOOP_COUNT = 1;

const float PROB_THRESHOLD = 0.1f;

namespace ax
{
void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const cv::Mat& mat, int input_w, int input_h, const std::vector<float>& time_costs)
{
std::vector<detection::Object> proposals, objects;

timer timer_postprocess;
auto& info = io_info->pOutputs[0];
float* output_prob = (float*)io_data->pOutputs[0].pVirAddr;
float* output_bbox = (float*)io_data->pOutputs[1].pVirAddr;

for (size_t i = 0; i < info.pShape[1]; i++)
{
int maxid = -1;
float maxval = -FLT_MAX;
for (size_t j = 0; j < io_info->pOutputs[0].pShape[2]; j++)
{
if (output_prob[j] > maxval)
{
maxval = output_prob[j];
maxid = j;
}
}
maxval = detection::sigmoid(maxval);

if (maxval > PROB_THRESHOLD)
{
detection::Object obj;
obj.label = maxid;
obj.prob = maxval;
obj.rect.x = (output_bbox[0] - 0.5 * output_bbox[2]) * input_w;
obj.rect.y = (output_bbox[1] - 0.5 * output_bbox[3]) * input_h;
obj.rect.width = output_bbox[2] * input_w;
obj.rect.height = output_bbox[3] * input_h;
objects.push_back(obj);
}
output_prob += io_info->pOutputs[0].pShape[2];
output_bbox += io_info->pOutputs[1].pShape[2];
}

detection::get_out_bbox(objects, input_h, input_w, mat.rows, mat.cols);
fprintf(stdout, "post process cost time:%.2f ms \n", timer_postprocess.cost());
fprintf(stdout, "--------------------------------------\n");
auto total_time = std::accumulate(time_costs.begin(), time_costs.end(), 0.f);
auto min_max_time = std::minmax_element(time_costs.begin(), time_costs.end());
fprintf(stdout,
"Repeat %d times, avg time %.2f ms, max_time %.2f ms, min_time %.2f ms\n",
(int)time_costs.size(),
total_time / (float)time_costs.size(),
*min_max_time.second,
*min_max_time.first);
fprintf(stdout, "--------------------------------------\n");
fprintf(stdout, "detection num: %zu\n", objects.size());

detection::draw_objects(mat, objects, CLASS_NAMES, "rtdetr_out");
}

bool run_model(const std::string& model, const std::vector<uint8_t>& data, const int& repeat, cv::Mat& mat, int input_h, int input_w)
{
// 1. init engine
#ifdef AXERA_TARGET_CHIP_AX620E
auto ret = AX_ENGINE_Init();
#else
AX_ENGINE_NPU_ATTR_T npu_attr;
memset(&npu_attr, 0, sizeof(npu_attr));
npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE;
auto ret = AX_ENGINE_Init(&npu_attr);
#endif
if (0 != ret)
{
return ret;
}

// 2. load model
std::vector<char> model_buffer;
if (!utilities::read_file(model, model_buffer))
{
fprintf(stderr, "Read Run-Joint model(%s) file failed.\n", model.c_str());
return false;
}

// 3. create handle
AX_ENGINE_HANDLE handle;
ret = AX_ENGINE_CreateHandle(&handle, model_buffer.data(), model_buffer.size());
SAMPLE_AX_ENGINE_DEAL_HANDLE
fprintf(stdout, "Engine creating handle is done.\n");

// 4. create context
ret = AX_ENGINE_CreateContext(handle);
SAMPLE_AX_ENGINE_DEAL_HANDLE
fprintf(stdout, "Engine creating context is done.\n");

// 5. set io
AX_ENGINE_IO_INFO_T* io_info;
ret = AX_ENGINE_GetIOInfo(handle, &io_info);
SAMPLE_AX_ENGINE_DEAL_HANDLE
fprintf(stdout, "Engine get io info is done. \n");

// 6. alloc io
AX_ENGINE_IO_T io_data;
ret = middleware::prepare_io(io_info, &io_data, std::make_pair(AX_ENGINE_ABST_DEFAULT, AX_ENGINE_ABST_CACHED));
SAMPLE_AX_ENGINE_DEAL_HANDLE
fprintf(stdout, "Engine alloc io is done. \n");

// 7. insert input
ret = middleware::push_input(data, &io_data, io_info);
SAMPLE_AX_ENGINE_DEAL_HANDLE_IO
fprintf(stdout, "Engine push input is done. \n");
fprintf(stdout, "--------------------------------------\n");

// 8. warn up
for (int i = 0; i < 5; ++i)
{
AX_ENGINE_RunSync(handle, &io_data);
}

// 9. run model
std::vector<float> time_costs(repeat, 0);
for (int i = 0; i < repeat; ++i)
{
timer tick;
ret = AX_ENGINE_RunSync(handle, &io_data);
time_costs[i] = tick.cost();
SAMPLE_AX_ENGINE_DEAL_HANDLE_IO
}

// 10. get result
post_process(io_info, &io_data, mat, input_w, input_h, time_costs);
fprintf(stdout, "--------------------------------------\n");

middleware::free_io(&io_data);
return AX_ENGINE_DestroyHandle(handle);
}
} // namespace ax

int main(int argc, char* argv[])
{
cmdline::parser cmd;
cmd.add<std::string>("model", 'm', "joint file(a.k.a. joint model)", true, "");
cmd.add<std::string>("image", 'i', "image file", true, "");
cmd.add<std::string>("size", 'g', "input_h, input_w", false, std::to_string(DEFAULT_IMG_H) + "," + std::to_string(DEFAULT_IMG_W));

cmd.add<int>("repeat", 'r', "repeat count", false, DEFAULT_LOOP_COUNT);
cmd.parse_check(argc, argv);

// 0. get app args, can be removed from user's app
auto model_file = cmd.get<std::string>("model");
auto image_file = cmd.get<std::string>("image");

auto model_file_flag = utilities::file_exist(model_file);
auto image_file_flag = utilities::file_exist(image_file);

if (!model_file_flag | !image_file_flag)
{
auto show_error = [](const std::string& kind, const std::string& value) {
fprintf(stderr, "Input file %s(%s) is not exist, please check it.\n", kind.c_str(), value.c_str());
};

if (!model_file_flag) { show_error("model", model_file); }
if (!image_file_flag) { show_error("image", image_file); }

return -1;
}

auto input_size_string = cmd.get<std::string>("size");

std::array<int, 2> input_size = {DEFAULT_IMG_H, DEFAULT_IMG_W};

auto input_size_flag = utilities::parse_string(input_size_string, input_size);

if (!input_size_flag)
{
auto show_error = [](const std::string& kind, const std::string& value) {
fprintf(stderr, "Input %s(%s) is not allowed, please check it.\n", kind.c_str(), value.c_str());
};

show_error("size", input_size_string);

return -1;
}

auto repeat = cmd.get<int>("repeat");

// 1. print args
fprintf(stdout, "--------------------------------------\n");
fprintf(stdout, "model file : %s\n", model_file.c_str());
fprintf(stdout, "image file : %s\n", image_file.c_str());
fprintf(stdout, "img_h, img_w : %d %d\n", input_size[0], input_size[1]);
fprintf(stdout, "--------------------------------------\n");

// 2. read image & resize & transpose
std::vector<uint8_t> image(input_size[0] * input_size[1] * 3, 0);
cv::Mat mat = cv::imread(image_file);
if (mat.empty())
{
fprintf(stderr, "Read image failed.\n");
return -1;
}
common::get_input_data_letterbox(mat, image, input_size[0], input_size[1], true);

// 3. sys_init
AX_SYS_Init();

// 4. - engine model - can only use AX_ENGINE** inside
{
// AX_ENGINE_NPUReset(); // todo ??
ax::run_model(model_file, image, repeat, mat, input_size[0], input_size[1]);

// 4.3 engine de init
AX_ENGINE_Deinit();
// AX_ENGINE_NPUReset();
}
// 4. - engine model -

AX_SYS_Deinit();
return 0;
}
12 changes: 6 additions & 6 deletions examples/ax650/middleware/io.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,12 +174,12 @@ namespace middleware
{AX_ENGINE_DT_UINT14_PACKED, "UINT14_PACKED"},
{AX_ENGINE_DT_UINT16_PACKED, "UINT16_PACKED"},
};
printf("input size: %d\n", io_info->nInputSize);
printf("\ninput size: %d\n", io_info->nInputSize);
for (uint32_t i = 0; i < io_info->nInputSize; ++i)
{
// print shape info,like [batchsize x channel x height x width]
auto& info = io_info->pInputs[i];
printf("name: %s [%s]\n ", info.pName, data_type[info.eDataType]);
printf(" name: \e[1;32m%8s \e[1;34m[%s]\e[0m\n \e[1;31m", info.pName, data_type[info.eDataType]);
for (size_t s = 0; s < info.nShapeSize; s++)
{
printf("%d", info.pShape[s]);
Expand All @@ -188,15 +188,15 @@ namespace middleware
printf(" x ");
}
}
printf("\n");
printf("\e[0m\n\n");
}

printf("output size: %d\n", io_info->nOutputSize);
printf("\noutput size: %d\n", io_info->nOutputSize);
for (uint32_t i = 0; i < io_info->nOutputSize; ++i)
{
// print shape info,like [batchsize x channel x height x width]
auto& info = io_info->pOutputs[i];
printf("name: %s [%s]\n ", info.pName, data_type[info.eDataType]);
printf(" name: \e[1;32m%8s \e[1;34m[%s]\e[0m\n \e[1;31m", info.pName, data_type[info.eDataType]);
for (size_t s = 0; s < info.nShapeSize; s++)
{
printf("%d", info.pShape[s]);
Expand All @@ -205,7 +205,7 @@ namespace middleware
printf(" x ");
}
}
printf("\n");
printf("\e[0m\n\n");
}
}

Expand Down

0 comments on commit c1aa806

Please sign in to comment.