#include "video_render.h" #include "SpBase.h" #include "Event.h" #include "videoutil.h" #include "libvideoqueue.h" #include "rvc_media_common.h" #include "fileutil.h" #include "video_common/ffmpeg_api_cpp_adapter.h" #include "cv.h" #include "highgui.h" #include "y2k_time.h" static int translate_image_resolution(video_frame** dstframe, const int iwidth, const int iheight, const video_frame* psrcframe) { int i = -1; if (iwidth == psrcframe->width && iheight == psrcframe->height) { return i; } video_frame* pframe = video_frame_new(iwidth, iheight, VIDEO_FORMAT_RGB24); video_frame_fill_black(pframe); SwsContext* sws = sws_getCachedContext(NULL, psrcframe->width, psrcframe->height, PIX_FMT_BGR24, iwidth, iheight, PIX_FMT_BGR24, SWS_LANCZOS, NULL, NULL, NULL); sws_scale(sws, psrcframe->data, psrcframe->linesize, 0, psrcframe->height, pframe->data, pframe->linesize); sws_freeContext(sws); *dstframe = pframe; i = 0; return i; } static int rvc_video_shm_enqueue(Clibvideoqueue* shm_queue, video_frame* frame, int flags, int iframeid) { videoq_frame tmp_frm; tmp_frm.data = frame->data[0]; tmp_frm.framesize = frame->width * frame->height * 3; tmp_frm.format = VIDEOQ_FORMAT_RGB24; tmp_frm.width = frame->width; tmp_frm.height = frame->height; tmp_frm.iframeid = iframeid; unsigned int nowtime = y2k_time_now(); if (!shm_queue->InsertVideo(&tmp_frm, flags, nowtime)){ DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("caution: rvc render shm_queue video insert shm video failed!"); return Error_Unexpect; } else{ //DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render shm_queue video insert shm video ok, and nowtime is %d, and frame id is %d.", nowtime, iframeid); return Error_Succeed; } } static bool get_local_video_frame(video_frame** frame, int itype, Clibvideoqueue* local_video_queue, int iwidth, int iheight, IplImage* personimage, IplImage* personmask) { bool result = false; video_frame* tmp_frame_preview = NULL; tmp_frame_preview = video_frame_new(iwidth, iheight, VIDEO_FORMAT_RGB24); if (NULL == tmp_frame_preview) { return result; } videoq_frame frm; frm.data = tmp_frame_preview->data[0]; static bool blog = true; if (blog) { int ivideo_width = 0; int ivideo_height = 0; int isize = local_video_queue->GetFrameSize(ivideo_width, ivideo_height); DbgWithLink(LOG_LEVEL_DEBUG, LOG_TYPE_SYSTEM)("isize = %d, ivideo_width = %d, ivideo_height = %d.", isize, ivideo_width, ivideo_height); blog = false; } result = local_video_queue->GetVideo(&frm, VIDEOQUEUE_FLAG_HORIZONTAL_FLIP); if (result){ if (1 == itype) { if (NULL != personimage && NULL != personmask) { IplImage* img = cvCreateImageHeader(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 3); img->imageData = (char*)frm.data; if (frm.width != personimage->width) { IplImage* tmp = cvCreateImage(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 3); IplImage* tmpmask = cvCreateImage(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 1); cvResize(personimage, tmp); cvResize(personmask, tmpmask); cvAdd(img, tmp, img, tmpmask); cvReleaseImage(&tmp); cvReleaseImage(&tmpmask); } else { cvAdd(img, personimage, img, personmask); } cvReleaseImageHeader(&img); } } *frame = tmp_frame_preview; } else { video_frame_delete(tmp_frame_preview); tmp_frame_preview = NULL; } return result; } int rvc_remote_video_render(rvc_video_render_t* prender, void* videoframe) { if (NULL != prender && NULL != videoframe) { if (NULL != prender->premote_render) { video_frame* echoframe = NULL; if (0 == translate_image_resolution(&echoframe, prender->render_param.iremote_view_cx, prender->render_param.iremote_view_cy, (const video_frame*)videoframe)) { prender->premote_render->RenderVideoFrame(echoframe, RVC_FLIP_VERTICAL); video_frame_delete(echoframe); echoframe = NULL; } else { prender->premote_render->RenderVideoFrame((video_frame*)videoframe, RVC_FLIP_VERTICAL); } } } return 0; } static int rvc_getrender_videoqueue(Clibvideoqueue** render_video_queue, int* render_width, int* render_height, RVC_RendererFlip* renderflip, rvc_video_render_t* param) { int iret = -1; int irender_camera = CAMERA_TYPE_ENV; if (param->render_param.ilocal_view_cx < param->render_param.ilocal_view_cy) { irender_camera = CAMERA_TYPE_OPT; } if (CAMERA_TYPE_ENV == irender_camera) { if (ePreview_Queue == param->render_param.erender_queue) { *render_video_queue = new Clibvideoqueue(REC_COMMON_VIDEO_ENV_SHM_PREVIEW_QUEUE); *render_width = REC_COMMON_VIDEO_PREVIEW_WIDTH; *render_height = REC_COMMON_VIDEO_PREVIEW_HEIGHT; } else { *render_video_queue = new Clibvideoqueue(REC_COMMON_VIDEO_ENV_SHM_SNAPSHOT_QUEUE); *render_width = REC_COMMON_VIDEO_SNAPSHOT_WIDTH; *render_height = REC_COMMON_VIDEO_SNAPSHOT_HEIGHT; DbgWithLink(LOG_LEVEL_DEBUG, LOG_TYPE_SYSTEM)("use sanpshot queue."); } #ifdef RVC_OS_WIN *renderflip = RVC_FLIP_HORIZONTAL; #endif // RVC_OS_WIN } else { if (ePreview_Queue == param->render_param.erender_queue) { *render_video_queue = new Clibvideoqueue(REC_COMMON_VIDEO_OPT_SHM_PREVIEW_QUEUE); *render_width = REC_COMMON_VIDEO_PREVIEW_HEIGHT; *render_height = REC_COMMON_VIDEO_PREVIEW_WIDTH; } else { *render_video_queue = new Clibvideoqueue(REC_COMMON_VIDEO_OPT_SHM_SNAPSHOT_QUEUE); *render_width = REC_COMMON_VIDEO_SNAPSHOT_HEIGHT; *render_height = REC_COMMON_VIDEO_SNAPSHOT_WIDTH; DbgWithLink(LOG_LEVEL_DEBUG, LOG_TYPE_SYSTEM)("use sanpshot queue."); } *renderflip = RVC_FLIP_NONE; } iret = 0; return iret; } static int rvc_getrxkimage(IplImage** personimage, IplImage** personmask) { int iret = -1; char strPersonPath[MAX_PATH] = { 0 }; rvc_snprintf(strPersonPath, MAX_PATH, "./bin/rxk.jpg"); if (ExistsFile(strPersonPath)) { *personimage = cvLoadImage(strPersonPath); *personmask = cvLoadImage(strPersonPath, 0); iret = 0; } return iret; } #ifdef RVC_OS_WIN static unsigned int __stdcall rvc_videorender_func(void* arg) #else static void* rvc_videorender_func(void* arg) #endif { rvc_video_render_t* param = (rvc_video_render_t*)arg; int ilocal_video_fresh_time = param->render_param.ilocal_fresh_time; bool bremote_video = false; Clibvideoqueue* local_video_queue = NULL; RVC_RendererFlip renderflip = RVC_FLIP_VERTICAL; int iwidth = 0; int iheight = 0; rvc_getrender_videoqueue(&local_video_queue, &iwidth, &iheight, &renderflip, param); int ivideo_width = 0; int ivideo_height = 0; int isize = local_video_queue->GetFrameSize(ivideo_width, ivideo_height); if (ivideo_width > 0 && ivideo_height > 0) { iwidth = ivideo_width; iheight = ivideo_height; } IplImage* personimage = NULL; IplImage* personmask = NULL; rvc_getrxkimage(&personimage, &personmask); if (NULL != param->plocal_render) { videorender_param_t tparam = { 0 }; tparam.icx = param->render_param.ilocal_view_x; tparam.icy = param->render_param.ilocal_view_y; tparam.uwidth = param->render_param.ilocal_view_cx; tparam.uheight = param->render_param.ilocal_view_cy; tparam.ivideoformat = VIDEO_FORMAT_RGB24; if (0 == param->plocal_render->VideoRenderSetParam(&tparam)) { param->plocal_render->HideVideoWindow(); } else { return 0; } } if (0 != param->render_param.iremote_view_cx && 0 != param->render_param.iremote_view_cy) { if (NULL != param->premote_render) { videorender_param_t tparam_remote = { 0 }; tparam_remote.icx = param->render_param.iremote_view_x; tparam_remote.icy = param->render_param.iremote_view_y; tparam_remote.uwidth = param->render_param.iremote_view_cx; tparam_remote.uheight = param->render_param.iremote_view_cy; tparam_remote.ivideoformat = VIDEO_FORMAT_RGB24; if (0 != param->premote_render->VideoRenderSetParam(&tparam_remote)) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render set param failed."); } bremote_video = true; } else { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("remote video render is null."); } } if (NULL != param->plocal_render) { bool bshow_local = false; bool bset = true; param->plocal_render->StartVideoRender(); ivideo_width = 0; ivideo_height = 0; isize = local_video_queue->GetFrameSize(ivideo_width, ivideo_height); DbgWithLink(LOG_LEVEL_DEBUG, LOG_TYPE_SYSTEM)("%s:%d isize = %d, ivideo_width = %d, ivideo_height = %d.", __FUNCTION__, __LINE__, isize, ivideo_width, ivideo_height); for (; ; ) { #ifdef RVC_OS_WIN DWORD dwRet = WaitForSingleObject(param->local_render_stop_event, ilocal_video_fresh_time); if (WAIT_TIMEOUT == dwRet) { #else struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); long unsec = ts.tv_nsec + (1000 * 1000 * ilocal_video_fresh_time); ts.tv_sec += (unsec / 1000000000); ts.tv_nsec = (unsec % 1000000000); if (0 != sem_timedwait(¶m->local_render_stop_sem, &ts) && (ETIMEDOUT == errno)){ #endif video_frame* local_video_frame = NULL; int iwindowstate = param->cb.on_window_type(param->cb.user_data); get_local_video_frame(&local_video_frame, iwindowstate, local_video_queue, iwidth, iheight, personimage, personmask); if (NULL != local_video_frame) { video_frame* localframe = NULL; if (0 == translate_image_resolution(&localframe, param->render_param.ilocal_view_cx, param->render_param.ilocal_view_cy, local_video_frame)) { param->plocal_render->RenderVideoFrame(localframe, renderflip); video_frame_delete(localframe); localframe = NULL; } else { param->plocal_render->RenderVideoFrame(local_video_frame, renderflip); } bshow_local = true; video_frame_delete(local_video_frame); local_video_frame = NULL; } } else { param->plocal_render->HideVideoWindow(); break; } if (bset) { if (bshow_local) { param->plocal_render->ShowVideoWindow(); if (bremote_video) { param->premote_render->ShowVideoWindow(); } bset = false; } } } param->plocal_render->StopVideoRender(); } if (NULL != personimage) { cvReleaseImage(&personimage); } if (NULL != personmask) { cvReleaseImage(&personmask); } return 0; } static int rvc_get_remote_videoframe(video_frame** remote_frame) { int iret = -1; char strImgPath[MAX_PATH] = { 0 }; rvc_snprintf(strImgPath, MAX_PATH, "%s", "./bin/agent.jpg"); int irecord_video_frame_width = REC_COMMON_VIDEO_SNAPSHOT_PREVIEW_WIDTH; int irecord_video_frame_heigt = REC_COMMON_VIDEO_SNAPSHOT_PREVIEW_HEIGHT; *remote_frame = video_frame_new(irecord_video_frame_width, irecord_video_frame_heigt, VIDEO_FORMAT_RGB24); video_frame_fill_black(*remote_frame); if (ExistsFileA(strImgPath)) { IplImage* img = cvLoadImage(strImgPath, 1); videoq_frame* vframe = new videoq_frame; if (NULL != img) { vframe->format = VIDEOQ_FORMAT_RGB24; vframe->framesize = img->imageSize; vframe->width = img->width; vframe->height = img->height; vframe->data = new unsigned char[img->imageSize]; memcpy(vframe->data, img->imageData, img->imageSize); cvReleaseImage(&img); } SwsContext* sws = sws_getContext(vframe->width, vframe->height, PIX_FMT_BGR24, irecord_video_frame_width, irecord_video_frame_heigt, PIX_FMT_BGR24, SWS_POINT, NULL, NULL, NULL); uint8_t* src_data[4] = { (unsigned char*)vframe->data + (vframe->height - 1) * vframe->width * 3, NULL, NULL, NULL }; int src_linesize[4] = { -vframe->width * 3, 0, 0, 0 }; unsigned char* dst[4] = { (*remote_frame)->data[0],NULL,NULL,NULL }; int dst_linesize[4] = { (*remote_frame)->linesize[0], 0, 0, 0 }; sws_scale(sws, src_data, src_linesize, 0, vframe->height, dst, dst_linesize); sws_freeContext(sws); if (vframe) { delete vframe->data; delete vframe; vframe = NULL; } iret = 0; } return iret; } #ifdef RVC_OS_WIN static unsigned int __stdcall rvc_remote_videorender_func(void* arg) #else static void* rvc_remote_videorender_func(void* arg) #endif { rvc_video_render_t* param = (rvc_video_render_t*)arg; int iremote_video_fresh_time = param->render_param.iremote_fresh_time; Clibvideoqueue* video_shm_q_remote = new Clibvideoqueue(REC_COMMON_VIDEO_REMOTE_SHM_RTP_QUEUE); video_frame* remote_frame = NULL; rvc_get_remote_videoframe(&remote_frame); video_frame* showframe = NULL; if (-1 == translate_image_resolution(&showframe, param->render_param.iremote_view_cx, param->render_param.iremote_view_cy, remote_frame)) { showframe = video_frame_new(param->render_param.iremote_view_cx, param->render_param.iremote_view_cy, VIDEO_FORMAT_RGB24); video_frame_fill_black(showframe); video_frame_copy(showframe, remote_frame); } int iframeid = 0; int rc = rvc_video_shm_enqueue(video_shm_q_remote, remote_frame, VIDEOQUEUE_FLAG_VERTICAL_FLIP, iframeid++); if (rc != Error_Succeed) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("insert agent picture to remote video queue failed."); } for (; ; ) { #ifdef RVC_OS_WIN DWORD dwRet = WaitForSingleObject(param->remote_render_stop_event, iremote_video_fresh_time); if (WAIT_TIMEOUT == dwRet){ #else struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); long unsec = ts.tv_nsec + (1000 * 1000 * iremote_video_fresh_time); ts.tv_sec += (unsec / 1000000000); ts.tv_nsec = (unsec % 1000000000); if (0 != sem_timedwait(¶m->remote_render_stop_sem, &ts) && (ETIMEDOUT == errno)){ #endif // RVC_OS_WIN rvc_remote_video_render(param, showframe); if (NULL != video_shm_q_remote){ rc = rvc_video_shm_enqueue(video_shm_q_remote, remote_frame, VIDEOQUEUE_FLAG_VERTICAL_FLIP, iframeid++); if (rc != Error_Succeed){ DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("insert agent picture to remote video queue failed."); } } } else { break; } } video_frame_delete(showframe); showframe = NULL; video_frame_delete(remote_frame); remote_frame = NULL; return 0; } #ifdef RVC_OS_WIN static unsigned int __stdcall rvc_optcam_videorender_func(void* arg) #else static void* rvc_optcam_videorender_func(void* arg) #endif { rvc_video_render_t* param = (rvc_video_render_t*)arg; int iremote_video_fresh_time = param->render_param.iremote_fresh_time; Clibvideoqueue* local_optvideo_queue = new Clibvideoqueue(REC_COMMON_VIDEO_OPT_SHM_SNAPSHOT_QUEUE); int iwidth = REC_COMMON_VIDEO_SNAPSHOT_HEIGHT; int iheight = REC_COMMON_VIDEO_SNAPSHOT_WIDTH; RVC_RendererFlip renderflip = RVC_FLIP_NONE; for (; ; ) { #ifdef RVC_OS_WIN DWORD dwRet = WaitForSingleObject(param->remote_render_stop_event, iremote_video_fresh_time); if (WAIT_TIMEOUT == dwRet) { #else struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); long unsec = ts.tv_nsec + (1000 * 1000 * iremote_video_fresh_time); ts.tv_sec += (unsec / 1000000000); ts.tv_nsec = (unsec % 1000000000); if (0 != sem_timedwait(¶m->remote_render_stop_sem, &ts) && (ETIMEDOUT == errno)) { #endif // RVC_OS_WIN video_frame* local_video_frame = NULL; get_local_video_frame(&local_video_frame, 0, local_optvideo_queue, iwidth, iheight, NULL, NULL); if (NULL != local_video_frame) { video_frame* localframe = NULL; if (0 == translate_image_resolution(&localframe, param->render_param.iremote_view_cx, param->render_param.iremote_view_cy, local_video_frame)) { param->premote_render->RenderVideoFrame(localframe, renderflip); video_frame_delete(localframe); localframe = NULL; } else { param->premote_render->RenderVideoFrame(local_video_frame, renderflip); } video_frame_delete(local_video_frame); local_video_frame = NULL; } } else { break; } } return 0; } int rvc_start_video_render(rvc_video_render_t* prender, eVideoRenderMode eMode, rvc_video_box_move_callback_t* cb) { int errcode = -1; if (NULL == prender) { return errcode; } #ifdef RVC_OS_WIN prender->local_render_stop_event = CreateEventA(NULL, false, false, NULL); if (NULL == prender->local_render_stop_event) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create local stop event failed!"); return Error_Resource; } prender->local_render_thread = (HANDLE)_beginthreadex(NULL, 0, rvc_videorender_func, prender, 0, NULL); if (NULL == prender->local_render_thread) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create local video render thread failed."); return Error_Resource; } if (eRender_LocalRemote == eMode || eRender_Local_Both == eMode) { prender->remote_render_stop_event = CreateEventA(NULL, false, false, NULL); if (NULL == prender->remote_render_stop_event) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create remote stop event failed!"); return Error_Resource; } if (eRender_LocalRemote == eMode) { prender->remote_render_thread = (HANDLE)_beginthreadex(NULL, 0, rvc_remote_videorender_func, prender, 0, NULL); } else { prender->remote_render_thread = (HANDLE)_beginthreadex(NULL, 0, rvc_optcam_videorender_func, prender, 0, NULL); } if (NULL == prender->remote_render_thread) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create remote video render thread failed."); return Error_Resource; } } errcode = 0; #else if (0 != sem_init(&prender->local_render_stop_sem, 0, 0)) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create local stop sem failed!"); return Error_Resource; } errcode = pthread_create(&prender->local_render_threadid, NULL, rvc_videorender_func, prender); if (Error_Succeed != errcode) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create local video render thread failed."); return Error_Resource; } if (eRender_LocalRemote == eMode || eRender_Local_Both == eMode) { if (0 != sem_init(&prender->remote_render_stop_sem, 0, 0)) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create remote stop sem failed!"); return Error_Resource; } if (eRender_LocalRemote == eMode) { errcode = pthread_create(&prender->remote_render_threadid, NULL, rvc_remote_videorender_func, prender); } else { errcode = pthread_create(&prender->remote_render_threadid, NULL, rvc_videorender_func, prender); } if (Error_Succeed != errcode) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create remote video render thread failed."); return Error_Resource; } } #endif // RVC_OS_WIN return errcode; } int rvc_stop_video_render(rvc_video_render_t* prender) { if (NULL == prender) { return Error_Param; } #ifdef RVC_OS_WIN if (prender->local_render_stop_event) { SetEvent(prender->local_render_stop_event); } rvc_stop_remote_video_render(prender); if (NULL != prender->local_render_thread) { WaitForSingleObject(prender->local_render_thread, INFINITE); CloseHandle(prender->local_render_thread); prender->local_render_thread = NULL; } if (NULL != prender->local_render_stop_event) { CloseHandle(prender->local_render_stop_event); prender->local_render_stop_event = NULL; } #else sem_post(&prender->local_render_stop_sem); rvc_stop_remote_video_render(prender); if (prender->local_render_threadid > 0) { if (0 == pthread_join(prender->local_render_threadid, NULL)) { prender->local_render_threadid = 0; } else { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render thread pthread join error for %s.", strerror(errno)); } } if (NULL != &prender->local_render_stop_sem) { sem_destroy(&prender->local_render_stop_sem); } #endif // RVC_OS_WIN return Error_Succeed; } int rvc_stop_remote_video_render(rvc_video_render_t* prender) { #ifdef RVC_OS_WIN if (NULL != prender->remote_render_thread) { SetEvent(prender->remote_render_stop_event); WaitForSingleObject(prender->remote_render_thread, INFINITE); CloseHandle(prender->remote_render_thread); prender->remote_render_thread = NULL; } if (NULL != prender->remote_render_stop_event) { CloseHandle(prender->remote_render_stop_event); prender->remote_render_stop_event = NULL; } #else if (prender->remote_render_threadid > 0) { sem_post(&prender->remote_render_stop_sem); if (0 == pthread_join(prender->remote_render_threadid, NULL)) { prender->remote_render_threadid = 0; } else { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("remote video render thread pthread join error for %s.", strerror(errno)); } } if (NULL != &prender->remote_render_stop_sem) { sem_destroy(&prender->remote_render_stop_sem); } #endif return Error_Succeed; }