Unity下如何实现RTMP或RTSP流播放和录制

技术背景

在探讨Unity平台RTMP或RTSP直播流数据播放和录制之前，我们先简单回顾下RTSP或RTMP直播流数据在Unity平台的播放流程：

1. 通过Native RTSP或RTSP直播播放SDK回调RGB/YUV420/NV12等其中的一种未压缩的图像格式；
2. Unity下创建相应的RGB/YUV420等Shader；
3. Unity从各个平台获取图像数据来填充纹理即可。

Unity3D环境下，我们之前已覆盖以下模块：

• Windows平台RTMP直播推送模块（采集Unity窗体、摄像头或屏幕）;
• Windows平台RTMP|RTSP直播播放模块；
• Linux平台RTMP直播推送模块（采集Unity窗体、Unity声音）;
• Linux平台RTMP|RTSP直播播放模块；
• Android平台RTMP直播推送模块（采集Unity窗体、麦克风或Unity声音）；
• Android平台RTMP|RTSP直播播放模块；
• iOS平台RTMP|RTSP直播播放模块。

技术实现

本文主要介绍流数据录制，实际上，录制相对直播播放来说，更简单一些，毕竟不需要解码绘制。本文以Windows平台下Unity为例，介绍下流数据的录制流程，无图无真相：

开始录像

我们可指定文件录制规则，如是否需要录制纯音频或纯视频、单个录制文件大小限制、文件录制目录等参数设定，并设置录像回调事件：

/*
 * SmartPlayerWinMono.cs
 *
 * WebSite: http://daniusdk.com
 * Created by DaniuSDK on 2018/05/10.
 */
private void StartRecorder(int sel)
{
  Debug.Log("StartRecorder++, sel: " + sel);

  if (videoctrl[sel].is_recording_)
  {
    Debug.Log("StartRecorder, already started.. sel: " + sel);
    return;
  }

  if (!videoctrl[sel].is_playing_)
  {
    if (!OpenPlayerHandle(sel))
    {
      Debug.LogError("call OpenPlayerHandle failed..");
      return;
    }
  }

  bool is_rec_video = true;
  bool is_rec_audio = true;
  NTSmartPlayerSDK.NT_SP_SetRecorderVideo(videoctrl[sel].player_handle_, is_rec_video ? 1 : 0);
  NTSmartPlayerSDK.NT_SP_SetRecorderAudio(videoctrl[sel].player_handle_, is_rec_audio ? 1 : 0);

  String rec_dir = "D:\\Rec";     //录像目录可自行指定
  String rec_name_file_prefix_= "daniu" + sel.ToString();
  UInt32 max_file_size = 200 * 1024; // 单位是KByte, 默认200MB
  bool is_append_date = true;
  bool is_append_time = true;
  bool is_audio_transcode_aac = true;

  UInt32 ret = NTSmartPlayerSDK.NT_SP_SetRecorderDirectory(videoctrl[sel].player_handle_, rec_dir);
  if (NT.NTBaseCodeDefine.NT_ERC_OK != ret)
  {
    Debug.LogError("设置录像目录失败，请确保目录存在且是英文目录");
    return;
  }

  NTSmartPlayerSDK.NT_SP_SetRecorderFileMaxSize(videoctrl[sel].player_handle_, max_file_size);

  NT_SP_RecorderFileNameRuler rec_name_ruler = new NT_SP_RecorderFileNameRuler();

  rec_name_ruler.type_ = 0;
  rec_name_ruler.file_name_prefix_ = rec_name_file_prefix_;
  rec_name_ruler.append_date_ = is_append_date ? 1 : 0;
  rec_name_ruler.append_time_ = is_append_time ? 1 : 0;

  NTSmartPlayerSDK.NT_SP_SetRecorderFileNameRuler(videoctrl[sel].player_handle_, ref rec_name_ruler);

  NTSmartPlayerSDK.NT_SP_SetRecorderAudioTranscodeAAC(videoctrl[sel].player_handle_, is_audio_transcode_aac ? 1 : 0);

  videoctrl[sel].record_call_back_ = new SP_SDKRecorderCallBack(NT_SP_SDKRecorderCallBack);
  NTSmartPlayerSDK.NT_SP_SetRecorderCallBack(videoctrl[sel].player_handle_, IntPtr.Zero, videoctrl[sel].record_call_back_);
  videoctrl[sel].set_record_call_back_ = new VideoControl.SetRecordCallBack(RecordCallBack);

  if (NT.NTBaseCodeDefine.NT_ERC_OK != NTSmartPlayerSDK.NT_SP_StartRecorder(videoctrl[sel].player_handle_))
  {
    Debug.LogError("call NT_SP_StartRecorder failed..");
    return;
  }

  videoctrl[sel].is_recording_ = true;
}

这里需要注意的是，OpenPlayerHandle()针对播放端的设定，同一个实例，播放和录像仅需设置一次，具体实现如下：

private bool OpenPlayerHandle(int sel)
{
  if (videoctrl[sel].player_handle_ != IntPtr.Zero)
    return true;

  window_handle_ = IntPtr.Zero;

  if (videoctrl[sel].player_handle_ == IntPtr.Zero)
  {
    videoctrl[sel].player_handle_ = new IntPtr();
    UInt32 ret_open = NTSmartPlayerSDK.NT_SP_Open(out videoctrl[sel].player_handle_, window_handle_, 0, IntPtr.Zero);
    if (ret_open != 0)
    {
      videoctrl[sel].player_handle_ = IntPtr.Zero;
      Debug.LogError("call NT_SP_Open failed, sel: " + sel);
      return false;
    }
  }

  videoctrl[sel].event_call_back_ = new SP_SDKEventCallBack(NT_SP_SDKEventCallBack);
  NTSmartPlayerSDK.NT_SP_SetEventCallBack(videoctrl[sel].player_handle_, window_handle_, videoctrl[sel].event_call_back_);
  videoctrl[sel].sdk_event_call_back_ = new VideoControl.SetEventCallBack(SDKEventCallBack);

  if (IntPtr.Zero == videoctrl[sel].player_handle_)
    return false;

  /* ++ 播放前参数配置可加在此处 ++ */

  int play_buffer_time_ = 100;
  NTSmartPlayerSDK.NT_SP_SetBuffer(videoctrl[sel].player_handle_, play_buffer_time_);                 //设置buffer time

  int is_using_tcp = 0;        //TCP模式
  NTSmartPlayerSDK.NT_SP_SetRTSPTcpMode(videoctrl[sel].player_handle_, is_using_tcp);

  int timeout = 10;
  NTSmartPlayerSDK.NT_SP_SetRtspTimeout(videoctrl[sel].player_handle_, timeout);

  int is_auto_switch_tcp_udp = 1;
  NTSmartPlayerSDK.NT_SP_SetRtspAutoSwitchTcpUdp(videoctrl[sel].player_handle_, is_auto_switch_tcp_udp);

  Boolean is_mute_ = false;
  NTSmartPlayerSDK.NT_SP_SetMute(videoctrl[sel].player_handle_, is_mute_ ? 1 : 0);                    //是否启动播放的时候静音

  int is_fast_startup = 1;
  NTSmartPlayerSDK.NT_SP_SetFastStartup(videoctrl[sel].player_handle_, is_fast_startup);              //设置快速启动模式

  Boolean is_low_latency_ = false;
  NTSmartPlayerSDK.NT_SP_SetLowLatencyMode(videoctrl[sel].player_handle_, is_low_latency_ ? 1 : 0);    //设置是否启用低延迟模式

  //设置旋转角度(设置0， 90， 180， 270度有效，其他值无效)
  int rotate_degrees = 0;
  NTSmartPlayerSDK.NT_SP_SetRotation(videoctrl[sel].player_handle_, rotate_degrees);

  int volume = 100;
  NTSmartPlayerSDK.NT_SP_SetAudioVolume(videoctrl[sel].player_handle_, volume);  //设置播放音量, 范围是[0, 100], 0是静音，100是最大音量, 默认是100

  // 设置上传下载报速度
  int is_report = 0;
  int report_interval = 2;
  NTSmartPlayerSDK.NT_SP_SetReportDownloadSpeed(videoctrl[sel].player_handle_, is_report, report_interval);

  //设置播放URL
  NTSmartPlayerSDK.NT_SP_SetURL(videoctrl[sel].player_handle_, videoctrl[sel].playback_url_);
  /* -- 播放前参数配置可加在此处 -- */

  return true;
}

停止录像

停止录制的时候，如果当前流数据没有在播放，可直接调用Close接口，释放实例并把player_handle置null。

private void StopRecorder(int sel)
{
  Debug.Log("StopRecorder++, sel: " + sel);

  if (videoctrl[sel].player_handle_ == IntPtr.Zero)
  {
    return;
  }

  NTSmartPlayerSDK.NT_SP_StopRecorder(videoctrl[sel].player_handle_);

  videoctrl[sel].is_recording_ = false;

  if (!videoctrl[sel].is_playing_)
  {
    NTSmartPlayerSDK.NT_SP_Close(videoctrl[sel].player_handle_);
    videoctrl[sel].player_handle_ = IntPtr.Zero;
  }
}

开始播放

接下来，我们回顾下播放流程，播放端我们以回调YUV数据为例：

public void StartPlayer(int sel)
{
  Debug.Log("StartPlayer++, sel: " + sel);

  if (videoctrl[sel].is_playing_)
  {
    Debug.Log("StartPlayer, already started.. sel: " + sel);
    return;
  }

  lock (videoctrl[sel].frame_lock_)
  {
    videoctrl[sel].cur_video_frame_ = null;
  }

  if (!videoctrl[sel].is_recording_)
  {
    if (!OpenPlayerHandle(sel))
    {
      Debug.LogError("call OpenPlayerHandle failed, sel:" + sel);
      return;
    }
  }

  if (is_enable_hardware_decoder_)
  {
    NTSmartPlayerSDK.NT_SP_SetH264HardwareDecoder(videoctrl[sel].player_handle_, is_support_h264_hardware_decoder_ ? 1 : 0, 0);
    NTSmartPlayerSDK.NT_SP_SetH265HardwareDecoder(videoctrl[sel].player_handle_, is_support_h265_hardware_decoder_ ? 1 : 0, 0);
  }
  else
  {
    NTSmartPlayerSDK.NT_SP_SetH264HardwareDecoder(videoctrl[sel].player_handle_, 0, 0);
    NTSmartPlayerSDK.NT_SP_SetH265HardwareDecoder(videoctrl[sel].player_handle_, 0, 0);
  }

  //video frame callback (YUV/RGB)
  videoctrl[sel].sdk_video_frame_call_back_ = new VideoControl.SetVideoFrameCallBack(SDKVideoFrameCallBack);
  videoctrl[sel].video_frame_call_back_ = new SP_SDKVideoFrameCallBack(NT_SP_SetVideoFrameCallBack);
  NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(videoctrl[sel].player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, window_handle_, videoctrl[sel].video_frame_call_back_);

  UInt32 flag = NTSmartPlayerSDK.NT_SP_StartPlay(videoctrl[sel].player_handle_);

  if (flag == DANIULIVE_RETURN_OK)
  {
    videoctrl[sel].is_need_get_frame_ = true;
    Debug.Log("NT_SP_StartPlay succeed, sel:" + sel);
  }
  else
  {
    videoctrl[sel].is_need_get_frame_ = false;
    Debug.LogError("NT_SP_StartPlay failed, sel:" + sel);
  }

  videoctrl[sel].is_playing_ = true;
}

停止播放

停止播放的时候和录像一样，如果当前流数据没有在录制状态，可直接调用Close接口，释放实例并把player_handle置null。

private void StopPlayer(int sel)
{
  Debug.Log("StopPlayer++, sel: " + sel);

  videoctrl[sel].is_need_get_frame_ = false;
  videoctrl[sel].is_need_init_texture_ = false;

  if (videoctrl[sel].player_handle_ == IntPtr.Zero)
  {
    return;
  }

  UInt32 flag = NTSmartPlayerSDK.NT_SP_StopPlay(videoctrl[sel].player_handle_);
  if (flag == DANIULIVE_RETURN_OK)
  {
    Debug.Log("call NT_SP_StopPlay succeed, sel: " + sel);
  }
  else
  {
    Debug.LogError("call NT_SP_StopPlay failed, sel: " + sel);
  }

  if (!videoctrl[sel].is_recording_)
  {
    NTSmartPlayerSDK.NT_SP_Close(videoctrl[sel].player_handle_);
    videoctrl[sel].player_handle_ = IntPtr.Zero;
  }

  videoctrl[sel].is_playing_ = false;
}

总结

Unity下实现RTMP或RTSP流录像，相对来说技术难度不大，甚至不需要操作UI即可完成。需要注意的是，Windows平台录制特别是多路录制受限于磁盘IO写入等，如果录制多路，可能会存在硬件性能瓶颈。此外，录像和播放，共用一个player实例即可，播放和录像，设计需要考虑逻辑完全分离，比如可只播放或只录制，亦或播放的过程中随时录制，或者录像的过程中随时播放。