#include "eptz_control.h" #include "uvc_log.h" static RockchipRga gRkRga; struct eptz_frame_info eptz_info; static int get_rga_format(PixelFormat f) { static std::map rga_format_map = { {PIX_FMT_YUV420P, RK_FORMAT_YCbCr_420_P}, {PIX_FMT_NV12, RK_FORMAT_YCbCr_420_SP}, {PIX_FMT_NV21, RK_FORMAT_YCrCb_420_SP}, {PIX_FMT_YUV422P, RK_FORMAT_YCbCr_422_P}, {PIX_FMT_NV16, RK_FORMAT_YCbCr_422_SP}, {PIX_FMT_NV61, RK_FORMAT_YCrCb_422_SP}, {PIX_FMT_YUYV422, -1}, {PIX_FMT_UYVY422, -1}, {PIX_FMT_RGB565, RK_FORMAT_RGB_565}, {PIX_FMT_BGR565, -1}, {PIX_FMT_RGB888, RK_FORMAT_BGR_888}, {PIX_FMT_BGR888, RK_FORMAT_RGB_888}, {PIX_FMT_ARGB8888, RK_FORMAT_BGRA_8888}, {PIX_FMT_ABGR8888, RK_FORMAT_RGBA_8888}}; auto it = rga_format_map.find(f); if (it != rga_format_map.end()) return it->second; return -1; } static int rga_blit(std::shared_ptr src, std::shared_ptr dst, ImageRect *src_rect, ImageRect *dst_rect, int rotate) { if (!src) return -EINVAL; if (!dst) return -EINVAL; rga_info_t src_info, dst_info; memset(&src_info, 0, sizeof(src_info)); src_info.fd = src->GetFD(); if (src_info.fd < 0) src_info.virAddr = src->GetPtr(); src_info.mmuFlag = 1; src_info.rotation = rotate; if (src_rect) { rga_set_rect(&src_info.rect, src_rect->x, src_rect->y, src_rect->w, src_rect->h, src->GetVirWidth(), src->GetVirHeight(), get_rga_format(src->GetPixelFormat())); } else { LOG_ERROR("%s %d src_rect error \n", __FUNCTION__, __LINE__); } memset(&dst_info, 0, sizeof(dst_info)); dst_info.fd = dst->GetFD(); if (dst_info.fd < 0) dst_info.virAddr = dst->GetPtr(); dst_info.mmuFlag = 1; if (dst_rect) { rga_set_rect(&dst_info.rect, dst_rect->x, dst_rect->y, dst_rect->w, dst_rect->h, dst->GetVirWidth(), dst->GetVirHeight(), get_rga_format(dst->GetPixelFormat())); } else { LOG_ERROR("%s %d dst_rect error\n", __FUNCTION__, __LINE__); } int ret = gRkRga.RkRgaBlit(&src_info, &dst_info, NULL); if (ret) { dst->SetValidSize(0); LOG_ERROR("Fail to RkRgaBlit, ret=%d\n", ret); } else { size_t valid_size = CalPixFmtSize(dst->GetPixelFormat(), dst->GetVirWidth(), dst->GetVirHeight(), 16); dst->SetValidSize(valid_size); if (src->GetUSTimeStamp() > dst->GetUSTimeStamp()) dst->SetUSTimeStamp(src->GetUSTimeStamp()); } return ret; } static char *enable_skip_frame = nullptr; bool do_rockx(easymedia::Flow *f, easymedia::MediaBufferVector &input_vector) { if (enable_skip_frame) { int static count = 0; int static divisor = atoi(enable_skip_frame); if (count++ % divisor != 0) return -1; if (count == 10000) count = 0; } assert(sizeof(float) == 4); RockxFlow *flow = (RockxFlow *)f; auto input = std::static_pointer_cast(input_vector[0]); if (!input) return false; rockx_image_t input_image; input_image.width = input->GetWidth(); input_image.height = input->GetHeight(); input_image.data = (uint8_t *)input->GetPtr(); input_image.pixel_format = ROCKX_PIXEL_FORMAT_YUV420SP_NV12; // ROCKX_PIXEL_FORMAT_RGB888; auto &handles = flow->rockx_handles; rockx_handle_t &face_det_handle = handles[0]; rockx_handle_t &object_track_handle = handles[1]; rockx_object_array_t face_array; rockx_object_array_t face_array_track; memset(&face_array, 0, sizeof(rockx_object_array_t)); rockx_ret_t ret = rockx_face_detect(face_det_handle, &input_image, &face_array, nullptr); if (ret != ROCKX_RET_SUCCESS) { LOG_ERROR("rockx_face_detect error %d\n", ret); return false; } if (face_array.count <= 0) return false; ret = rockx_object_track(object_track_handle, input_image.width, input_image.height, 4, &face_array, &face_array_track); if (ret != ROCKX_RET_SUCCESS) { LOG_ERROR("rockx_face_detect error %d\n", ret); return false; } size_t ret_buf_size = face_array_track.count * sizeof(struct aligned_rockx_face_rect); auto ret_buf = easymedia::MediaBuffer::Alloc(ret_buf_size); if (!ret_buf) return false; auto face_rects = (struct aligned_rockx_face_rect *)ret_buf->GetPtr(); memset(face_rects, 0, ret_buf_size); for (int i = 0; i < face_array_track.count; i++) { auto array = &face_rects[i]; array->left = face_array_track.object[i].box.left; array->top = face_array_track.object[i].box.top; array->right = face_array_track.object[i].box.right; array->bottom = face_array_track.object[i].box.bottom; memcpy(array->score, &face_array_track.object[i].score, 4); } ret_buf->SetSize(ret_buf_size); ret_buf->SetValidSize(face_array_track.count); do { current_focus_state = count_rectXY( ret_buf, arrayXY, lastXY, eptz_info.src_width, eptz_info.src_height, eptz_info.dst_width, eptz_info.dst_height); } while (0); return true; } RockxFlow::RockxFlow() { easymedia::SlotMap sm; sm.thread_model = easymedia::Model::SYNC; sm.mode_when_full = easymedia::InputMode::DROPFRONT; sm.input_slots.push_back(0); sm.input_maxcachenum.push_back(1); sm.fetch_block.push_back(true); sm.output_slots.push_back(0); // if (hand_over_big_pic) sm.hold_input.push_back(easymedia::HoldInputMode::INHERIT_FORM_INPUT); sm.process = do_rockx; if (!InstallSlotMap(sm, "rockx", -1)) { LOG_ERROR("Fail to InstallSlotMap, %s\n", "rockx"); SetError(-EINVAL); return; } std::vector models; void *config = nullptr; size_t config_size = 0; models.push_back(ROCKX_MODULE_FACE_DETECTION_V3); models.push_back(ROCKX_MODULE_OBJECT_TRACK); for (size_t i = 0; i < models.size(); i++) { rockx_handle_t npu_handle = nullptr; rockx_module_t &model = models[i]; rockx_ret_t ret = rockx_create(&npu_handle, model, config, config_size); if (ret != ROCKX_RET_SUCCESS) { LOG_ERROR("init rockx module %d error=%d\n", model, ret); SetError(-EINVAL); return; } rockx_handles.push_back(npu_handle); } enable_skip_frame = getenv("ENABLE_EPTZ_SKIP_FRAME"); } bool do_dynamic_clip(easymedia::Flow *f, easymedia::MediaBufferVector &input_vector) { auto flow = static_cast(f); auto input = input_vector[0]; if (!input) return false; auto src = std::static_pointer_cast(input); // fmt,w,h,vw,vh ImageInfo out_img_info = {PIX_FMT_NV12, flow->dst_width, flow->dst_height, flow->dst_width, flow->dst_height}; const auto &info = out_img_info; std::shared_ptr out_buffer; if (info.vir_width > 0 && info.vir_height > 0) { size_t size = CalPixFmtSize(out_img_info.pix_fmt, out_img_info.vir_width, out_img_info.vir_height, 16); // size_t size = CalPixFmtSize(info); auto &&mb = easymedia::MediaBuffer::Alloc2( size, easymedia::MediaBuffer::MemType::MEM_HARD_WARE); out_buffer = std::make_shared(mb, info); } auto dst = std::static_pointer_cast(out_buffer); ImageRect *src_rect = (ImageRect *)malloc(sizeof(ImageRect)); ImageRect *dst_rect = (ImageRect *)malloc(sizeof(ImageRect)); if (!src_rect || !dst_rect) return false; output_result(src_rect, dst_rect); rga_blit(src, dst, src_rect, dst_rect, 0); out: if (src_rect) free(src_rect); src_rect = nullptr; if (dst_rect) free(dst_rect); dst_rect = nullptr; return flow->SetOutput(dst, 0); } DynamicClipFlow::DynamicClipFlow(uint32_t dst_w, uint32_t dst_h) : dst_width(dst_w), dst_height(dst_h) { easymedia::SlotMap sm; sm.thread_model = easymedia::Model::ASYNCCOMMON; sm.mode_when_full = easymedia::InputMode::DROPFRONT; sm.input_slots.push_back(0); sm.input_maxcachenum.push_back(1); sm.output_slots.push_back(0); sm.process = do_dynamic_clip; if (!InstallSlotMap(sm, "dynamic_clip", -1)) { LOG_ERROR("Fail to InstallSlotMap, %s\n", "dynamic_clip"); SetError(-EINVAL); return; } } std::shared_ptr eptz_source = nullptr; std::shared_ptr rknn = nullptr; std::shared_ptr dclip = nullptr; int eptz_config(int stream_width, int stream_height, int eptz_width, int eptz_height) { eptz_info.src_width = stream_width; eptz_info.src_height = stream_height; eptz_info.dst_width = eptz_width; eptz_info.dst_height = eptz_height; eptz_info.threshold_x = (eptz_info.src_width - eptz_info.dst_width) / 4; eptz_info.threshold_y = (eptz_info.src_height - eptz_info.dst_height) / 4; if (eptz_info.dst_width == 1920) { eptz_info.iterate_x = 6; eptz_info.iterate_y = 3; } else if (eptz_info.dst_width == 1280) { eptz_info.iterate_x = 4; eptz_info.iterate_y = 2; } else { eptz_info.iterate_x = 2; eptz_info.iterate_y = 2; } LOG_INFO("%s: eptz_info src_wh [%d %d] dst_wh[%d %d], threshold_xy[%d %d] " "iterate_xy[%d %d]\n", __func__, eptz_info.src_width, eptz_info.src_height, eptz_info.dst_width, eptz_info.dst_height, eptz_info.threshold_x, eptz_info.threshold_y, eptz_info.iterate_x, eptz_info.iterate_y); tempXY = new float[4]{(float)(eptz_info.src_width - eptz_info.dst_width) / 2, (float)(eptz_info.src_height - eptz_info.dst_height) / 2, (float)(eptz_info.dst_width), (float)(eptz_info.dst_height)}; arrayXY = new float[4]{(float)(eptz_info.src_width - eptz_info.dst_width) / 2, (float)(eptz_info.src_height - eptz_info.dst_height) / 2, (float)(eptz_info.dst_width), (float)(eptz_info.dst_height)}; lastXY = new float[4]{(float)(eptz_info.src_width - eptz_info.dst_width) / 2, (float)(eptz_info.src_height - eptz_info.dst_height) / 2, (float)(eptz_info.dst_width), (float)(eptz_info.dst_height)}; do { std::string flow_name("source_stream"); std::string flow_param(""); PARAM_STRING_APPEND(flow_param, KEY_NAME, "v4l2_capture_stream"); PARAM_STRING_APPEND(flow_param, KEK_THREAD_SYNC_MODEL, KEY_SYNC); PARAM_STRING_APPEND(flow_param, KEK_INPUT_MODEL, KEY_DROPFRONT); PARAM_STRING_APPEND_TO(flow_param, KEY_INPUT_CACHE_NUM, 5); std::string stream_param = ""; PARAM_STRING_APPEND_TO(stream_param, KEY_USE_LIBV4L2, 1); PARAM_STRING_APPEND(stream_param, KEY_DEVICE, "rkispp_scale2"); // PARAM_STRING_APPEND(param, KEY_SUB_DEVICE, sub_input_path); PARAM_STRING_APPEND(stream_param, KEY_V4L2_CAP_TYPE, KEY_V4L2_C_TYPE(VIDEO_CAPTURE)); PARAM_STRING_APPEND(stream_param, KEY_V4L2_MEM_TYPE, KEY_V4L2_M_TYPE(MEMORY_DMABUF)); PARAM_STRING_APPEND_TO(stream_param, KEY_FRAMES, 3); // if not set, default is 2 PARAM_STRING_APPEND(stream_param, KEY_OUTPUTDATATYPE, "image:nv12"); PARAM_STRING_APPEND_TO(stream_param, KEY_BUFFER_WIDTH, 640); PARAM_STRING_APPEND_TO(stream_param, KEY_BUFFER_HEIGHT, 480); PARAM_STRING_APPEND_TO(stream_param, KEY_BUFFER_VIR_WIDTH, 640); PARAM_STRING_APPEND_TO(stream_param, KEY_BUFFER_VIR_HEIGHT, 480); flow_param = easymedia::JoinFlowParam(flow_param, 1, stream_param); LOG_INFO("\n#VideoCapture flow param:\n%s\n", flow_param.c_str()); eptz_source = easymedia::REFLECTOR(Flow)::Create( flow_name.c_str(), flow_param.c_str()); if (!eptz_source) { LOG_ERROR("Create flow %s failed\n", flow_name.c_str()); return -1; } } while (0); rknn = std::make_shared(); if (!rknn || rknn->GetError()) { LOG_ERROR("Fail to create rockx flow\n"); return -1; } // dynamic_clip dclip = std::make_shared(eptz_info.dst_width, eptz_info.dst_height); if (!dclip || dclip->GetError()) { LOG_ERROR("Fail to create dynamic_clip flow\n"); return -1; } return 0; } std::shared_ptr create_flow(const std::string &flow_name, const std::string &flow_param, const std::string &elem_param) { auto &¶m = easymedia::JoinFlowParam(flow_param, 1, elem_param); auto ret = easymedia::REFLECTOR(Flow)::Create( flow_name.c_str(), param.c_str()); if (!ret) LOG_ERROR("Create flow %s failed\n", flow_name.c_str()); return ret; }