diff --git a/.DS_Store b/.DS_Store index a5f17d6..4027a5f 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/src/Codec.md b/src/Codec.md new file mode 100644 index 0000000..d182fb1 --- /dev/null +++ b/src/Codec.md @@ -0,0 +1,106 @@ +## Codec + +### pjmedia_codec_factory + +```c +struct pjmedia_codec_factory +{ + /** Entries to put this structure in the codec manager list. */ + PJ_DECL_LIST_MEMBER(struct pjmedia_codec_factory); + + /** The factory's private data. */ + void *factory_data; + + /** Operations to the factory. */ + pjmedia_codec_factory_op *op; + +}; + +``` + +### pjmedia_codec_factory_op + +```c +/** + * This structure describes operations that must be supported by codec + * factories. + */ +typedef struct pjmedia_codec_factory_op +{ + /** + * Check whether the factory can create codec with the specified + * codec info. + * + * @param factory The codec factory. + * @param info The codec info. + * + * @return PJ_SUCCESS if this factory is able to create an + * instance of codec with the specified info. + */ + pj_status_t (*test_alloc)(pjmedia_codec_factory *factory, + const pjmedia_codec_info *info ); + + /** + * Create default attributes for the specified codec ID. This function + * can be called by application to get the capability of the codec. + * + * @param factory The codec factory. + * @param info The codec info. + * @param attr The attribute to be initialized. + * + * @return PJ_SUCCESS if success. + */ + pj_status_t (*default_attr)(pjmedia_codec_factory *factory, + const pjmedia_codec_info *info, + pjmedia_codec_param *attr ); + + /** + * Enumerate supported codecs that can be created using this factory. + * + * @param factory The codec factory. + * @param count On input, specifies the number of elements in + * the array. On output, the value will be set to + * the number of elements that have been initialized + * by this function. + * @param info The codec info array, which contents will be + * initialized upon return. + * + * @return PJ_SUCCESS on success. + */ + pj_status_t (*enum_info)(pjmedia_codec_factory *factory, + unsigned *count, + pjmedia_codec_info codecs[]); + + /** + * Create one instance of the codec with the specified codec info. + * + * @param factory The codec factory. + * @param info The codec info. + * @param p_codec Pointer to receive the codec instance. + * + * @return PJ_SUCCESS on success. + */ + pj_status_t (*alloc_codec)(pjmedia_codec_factory *factory, + const pjmedia_codec_info *info, + pjmedia_codec **p_codec); + + /** + * This function is called by codec manager to return a particular + * instance of codec back to the codec factory. + * + * @param factory The codec factory. + * @param codec The codec instance to be returned. + * + * @return PJ_SUCCESS on success. + */ + pj_status_t (*dealloc_codec)(pjmedia_codec_factory *factory, + pjmedia_codec *codec ); + + /** + * This callback will be called to deinitialize and destroy this factory. + */ + pj_status_t (*destroy)(void); + +} pjmedia_codec_factory_op; +``` + diff --git a/src/Port.md b/src/Port.md new file mode 100644 index 0000000..31f0cfc --- /dev/null +++ b/src/Port.md @@ -0,0 +1,668 @@ +# Port & Audio + +## 结构体 + +### pjmedia_port + +```c + +/** + * Port interface. + */ +typedef struct pjmedia_port +{ + pjmedia_port_info info; /**< Port information. */ + + /** Port data can be used by the port creator to attach arbitrary + * value to be associated with the port. + */ + struct port_data { + void *pdata; /**< Pointer data. */ + long ldata; /**< Long data. */ + } port_data; + + /** + * Group lock. + * + * This is optional, but if this port is registered to the audio/video + * conference bridge, the bridge will create one if the port has none. + */ + pj_grp_lock_t *grp_lock; + + /** + * Get clock source. + * This should only be called by #pjmedia_port_get_clock_src(). + */ + pjmedia_clock_src* (*get_clock_src)(struct pjmedia_port *this_port, + pjmedia_dir dir); + + /** + * Sink interface. + * This should only be called by #pjmedia_port_put_frame(). + */ + pj_status_t (*put_frame)(struct pjmedia_port *this_port, + pjmedia_frame *frame); + + /** + * Source interface. + * This should only be called by #pjmedia_port_get_frame(). + */ + pj_status_t (*get_frame)(struct pjmedia_port *this_port, + pjmedia_frame *frame); + + /** + * Called to destroy this port. + */ + pj_status_t (*on_destroy)(struct pjmedia_port *this_port); + +} pjmedia_port; +``` + +### pjmedia_snd_port + +```c +struct pjmedia_snd_port +{ + int rec_id; + int play_id; + pj_uint32_t aud_caps; + pjmedia_aud_param aud_param; + pjmedia_aud_stream *aud_stream; + pjmedia_dir dir; + pjmedia_port *port; + + pjmedia_clock_src cap_clocksrc, + play_clocksrc; + + unsigned clock_rate; + unsigned channel_count; + unsigned samples_per_frame; + unsigned bits_per_sample; + unsigned options; + unsigned prm_ec_options; + + + /* audio frame preview callbacks */ + void *user_data; + pjmedia_aud_play_cb on_play_frame; + pjmedia_aud_rec_cb on_rec_frame; +}; +``` + +### pjmedia_snd_port_param + +```c +/** + * This structure specifies the parameters to create the sound port. + * Use pjmedia_snd_port_param_default() to initialize this structure with + * default values (mostly zeroes) + */ +typedef struct pjmedia_snd_port_param +{ + /** + * Base structure. + */ + pjmedia_aud_param base; + + /** + * Sound port creation options. + */ + unsigned options; + + /** + * Echo cancellation options/flags. + */ + unsigned ec_options; + + /** + * Arbitrary user data for playback and record preview callbacks below. + */ + void *user_data; + + /** + * Optional callback for audio frame preview right before queued to + * the speaker. + * Notes: + * - application MUST NOT block or perform long operation in the callback + * as the callback may be executed in sound device thread + * - when using software echo cancellation, application MUST NOT modify + * the audio data from within the callback, otherwise the echo canceller + * will not work properly. + * - the return value of the callback will be ignored + */ + pjmedia_aud_play_cb on_play_frame; + + /** + * Optional callback for audio frame preview recorded from the microphone + * before being processed by any media component such as software echo + * canceller. + * Notes: + * - application MUST NOT block or perform long operation in the callback + * as the callback may be executed in sound device thread + * - when using software echo cancellation, application MUST NOT modify + * the audio data from within the callback, otherwise the echo canceller + * will not work properly. + * - the return value of the callback will be ignored + */ + pjmedia_aud_rec_cb on_rec_frame; + +} pjmedia_snd_port_param; +``` + +### pjmedia_aud_param + +```c +/** + * This structure specifies the parameters to open the audio stream. + */ +typedef struct pjmedia_aud_param +{ + /** + * The audio direction. This setting is mandatory. + */ + pjmedia_dir dir; + + /** + * The audio recorder device ID. This setting is mandatory if the audio + * direction includes input/capture direction. + */ + pjmedia_aud_dev_index rec_id; + + /** + * The audio playback device ID. This setting is mandatory if the audio + * direction includes output/playback direction. + */ + pjmedia_aud_dev_index play_id; + + /** + * Clock rate/sampling rate. This setting is mandatory. + */ + unsigned clock_rate; + + /** + * Number of channels. This setting is mandatory. + */ + unsigned channel_count; + + /** + * Number of samples per frame. This setting is mandatory. + */ + unsigned samples_per_frame; + + /** + * Number of bits per sample. This setting is mandatory. + */ + unsigned bits_per_sample; + + /** + * This flags specifies which of the optional settings are valid in this + * structure. The flags is bitmask combination of pjmedia_aud_dev_cap. + */ + unsigned flags; + + /** + * Set the audio format. This setting is optional, and will only be used + * if PJMEDIA_AUD_DEV_CAP_EXT_FORMAT is set in the flags. + */ + pjmedia_format ext_fmt; + + /** + * Input latency, in milliseconds. This setting is optional, and will + * only be used if PJMEDIA_AUD_DEV_CAP_INPUT_LATENCY is set in the flags. + */ + unsigned input_latency_ms; + + /** + * Input latency, in milliseconds. This setting is optional, and will + * only be used if PJMEDIA_AUD_DEV_CAP_OUTPUT_LATENCY is set in the flags. + */ + unsigned output_latency_ms; + + /** + * Input volume setting, in percent. This setting is optional, and will + * only be used if PJMEDIA_AUD_DEV_CAP_INPUT_VOLUME_SETTING is set in + * the flags. + */ + unsigned input_vol; + + /** + * Output volume setting, in percent. This setting is optional, and will + * only be used if PJMEDIA_AUD_DEV_CAP_OUTPUT_VOLUME_SETTING is set in + * the flags. + */ + unsigned output_vol; + + /** + * Set the audio input route/source. This setting is optional, and + * will only be used if PJMEDIA_AUD_DEV_CAP_INPUT_ROUTE/ + * PJMEDIA_AUD_DEV_CAP_INPUT_SOURCE is set in the flags. + */ + pjmedia_aud_dev_route input_route; + + /** + * Set the audio output route. This setting is optional, and will only be + * used if PJMEDIA_AUD_DEV_CAP_OUTPUT_ROUTE is set in the flags. + */ + pjmedia_aud_dev_route output_route; + + /** + * Enable/disable echo canceller, if the device supports it. This setting + * is optional, and will only be used if PJMEDIA_AUD_DEV_CAP_EC is set in + * the flags. + */ + pj_bool_t ec_enabled; + + /** + * Set echo canceller tail length in milliseconds, if the device supports + * it. This setting is optional, and will only be used if + * PJMEDIA_AUD_DEV_CAP_EC_TAIL is set in the flags. + */ + unsigned ec_tail_ms; + + /** + * Enable/disable PLC. This setting is optional, and will only be used + * if PJMEDIA_AUD_DEV_CAP_PLC is set in the flags. + */ + pj_bool_t plc_enabled; + + /** + * Enable/disable CNG. This setting is optional, and will only be used + * if PJMEDIA_AUD_DEV_CAP_CNG is set in the flags. + */ + pj_bool_t cng_enabled; + + /** + * Enable/disable VAD. This setting is optional, and will only be used + * if PJMEDIA_AUD_DEV_CAP_VAD is set in the flags. + */ + pj_bool_t vad_enabled; + +} pjmedia_aud_param; +``` + +### 设备抽象 + +#### pjmedia_aud_stream 在设备的进一步抽象中绑定rec_cb play_cb + +```c +/** + * This structure describes the audio device stream. + */ +struct pjmedia_aud_stream +{ + /** Internal data to be initialized by audio subsystem */ + struct { + /** Driver index */ + unsigned drv_idx; + } sys; + + /** Operations */ + pjmedia_aud_stream_op *op; +}; + +``` + +#### pjmedia_aud_stream_op + +```c +/** + * Sound stream operations. + */ +typedef struct pjmedia_aud_stream_op +{ + /** + * See #pjmedia_aud_stream_get_param() + */ + pj_status_t (*get_param)(pjmedia_aud_stream *strm, + pjmedia_aud_param *param); + + /** + * See #pjmedia_aud_stream_get_cap() + */ + pj_status_t (*get_cap)(pjmedia_aud_stream *strm, + pjmedia_aud_dev_cap cap, + void *value); + + /** + * See #pjmedia_aud_stream_set_cap() + */ + pj_status_t (*set_cap)(pjmedia_aud_stream *strm, + pjmedia_aud_dev_cap cap, + const void *value); + + /** + * See #pjmedia_aud_stream_start() + */ + pj_status_t (*start)(pjmedia_aud_stream *strm); + + /** + * See #pjmedia_aud_stream_stop(). + */ + pj_status_t (*stop)(pjmedia_aud_stream *strm); + + /** + * See #pjmedia_aud_stream_destroy(). + */ + pj_status_t (*destroy)(pjmedia_aud_stream *strm); + +} pjmedia_aud_stream_op; +``` + +#### factory相关 + +```c + +/** + * Sound device factory operations. + */ +typedef struct pjmedia_aud_dev_factory_op +{ + /** + * Initialize the audio device factory. + * + * @param f The audio device factory. + */ + pj_status_t (*init)(pjmedia_aud_dev_factory *f); + + /** + * Close this audio device factory and release all resources back to the + * operating system. + * + * @param f The audio device factory. + */ + pj_status_t (*destroy)(pjmedia_aud_dev_factory *f); + + /** + * Get the number of audio devices installed in the system. + * + * @param f The audio device factory. + */ + unsigned (*get_dev_count)(pjmedia_aud_dev_factory *f); + + /** + * Get the audio device information and capabilities. + * + * @param f The audio device factory. + * @param index Device index. + * @param info The audio device information structure which will be + * initialized by this function once it returns + * successfully. + */ + pj_status_t (*get_dev_info)(pjmedia_aud_dev_factory *f, + unsigned index, + pjmedia_aud_dev_info *info); + + /** + * Initialize the specified audio device parameter with the default + * values for the specified device. + * + * @param f The audio device factory. + * @param index Device index. + * @param param The audio device parameter. + */ + pj_status_t (*default_param)(pjmedia_aud_dev_factory *f, + unsigned index, + pjmedia_aud_param *param); + + /** + * Open the audio device and create audio stream. See + * #pjmedia_aud_stream_create() + */ + pj_status_t (*create_stream)(pjmedia_aud_dev_factory *f, + const pjmedia_aud_param *param, + pjmedia_aud_rec_cb rec_cb, + pjmedia_aud_play_cb play_cb, + void *user_data, + pjmedia_aud_stream **p_aud_strm); + + /** + * Refresh the list of audio devices installed in the system. + * + * @param f The audio device factory. + */ + pj_status_t (*refresh)(pjmedia_aud_dev_factory *f); + +} pjmedia_aud_dev_factory_op; + + +/** + * This structure describes an audio device factory. + */ +struct pjmedia_aud_dev_factory +{ + /** Internal data to be initialized by audio subsystem. */ + struct { + /** Driver index */ + unsigned drv_idx; + } sys; + + /** Operations */ + pjmedia_aud_dev_factory_op *op; +}; +``` + + + +## 相关方法 + +### 初始化方法 + +#### （snd_port_param）pjmedia_snd_port_param_default + +```c +/* Initialize with default values (zero) */ +PJ_DEF(void) pjmedia_snd_port_param_default(pjmedia_snd_port_param *prm) +{ + pj_bzero(prm, sizeof(*prm)); +} +``` + +#### pjmedia_stream_get_port + +``` +PJ_DEF(pj_status_t) pjmedia_stream_get_port( pjmedia_stream *stream, + pjmedia_port **p_port ) +{ + *p_port = &stream->port; + return PJ_SUCCESS; +} +``` + +stream->port，返回p_port + +#### （aud_dev_default_param）pjmedia_aud_dev_default_param + +```c +/* API: Initialize the audio device parameters with default values for the + * specified device. + */ +PJ_DEF(pj_status_t) pjmedia_aud_dev_default_param(pjmedia_aud_dev_index id, + pjmedia_aud_param *param) +{ + pjmedia_aud_dev_factory *f; + unsigned index; + pj_status_t status; + + PJ_ASSERT_RETURN(param && id!=PJMEDIA_AUD_INVALID_DEV, PJ_EINVAL); + PJ_ASSERT_RETURN(aud_subsys.pf, PJMEDIA_EAUD_INIT); + + status = lookup_dev(id, &f, &index); //Internal: lookup valid device id + if (status != PJ_SUCCESS) + return status; + + status = f->op->default_param(f, index, param); + if (status != PJ_SUCCESS) + return status; + + /* Normalize device IDs */ + make_global_index(f->sys.drv_idx, ¶m->rec_id); + make_global_index(f->sys.drv_idx, ¶m->play_id); + + return PJ_SUCCESS; +} +``` + + + +### Create port + +#### pjmedia_snd_port_create + +初始化pjmedia_snd_port_param的各种信息，调用pjmedia_snd_port_create2 + +```c +pj_status_t pjmedia_snd_port_create_player(pj_pool_t *pool, int dev_id, unsigned int clock_rate, unsigned int channel_count, unsigned int samples_per_frame, unsigned int bits_per_sample, unsigned int options, pjmedia_snd_port **p_port) +``` + +Create unidirectional sound device port for playing audio streams with the specified parameters. + +**参数:** +`pool` – Pool to allocate sound port structure. +`index` – Device index, or -1 to let the library choose the first available device. +`clock_rate` – Sound device's clock rate to set. +`channel_count` – Set number of channels, 1 for mono, or 2 for stereo. The channel count determines the format of the frame. +`samples_per_frame` – Number of samples per frame. +`bits_per_sample` – Set the number of bits per sample. The normal value for this parameter is 16 bits per sample. +`options` – Options flag. +`p_port` – Pointer to receive the sound device port instance. + +#### pjmedia_snd_port_create2 + +用pjmedia_snd_port_creat中传入的pjmedia_snd_port_param，初始化pjmedia_snd_port（此时声音port创建），最后调用start_sound_device，Start the sound stream. + +#### start_sound_device + +- Get device caps：获取dev_id，根据id调用函数pjmedia_aud_dev_get_info获取pjmedia_aud_dev_info + +- Process EC settings + +- Open the device + + 设置了两个回调snd_rec_cb、snd_play_cb分别为 rec_cb、play_cb + + ```c + status = pjmedia_aud_stream_create(¶m_copy, + snd_rec_cb, + snd_play_cb, + snd_port, + &snd_port->aud_stream); + ``` + +- Start sound stream.：pjmedia_aud_stream_start(snd_port->aud_stream); + +#### pjmedia_aud_stream_create + +先通过lookup_dev搜索rec_id、play_id设备对应的工厂，然后通过工厂创建设备f->op->create_stream并设置给设备两个回调函数 rec_cb、play_cb给设备抽象pjmedia_aud_stream的ca_cb、pb_cb，lookup_dev中aud_subsys已经存储了所有的音频设备是在创建媒体端点endpoint时初始化举例来说：初始化的时候创建媒体端点pjmedia_endpt，同时初始化了音频子系统aud_subsys，把各种类型的音频设备工厂添加到全局变量[static](https://so.csdn.net/so/search?q=static&spm=1001.2101.3001.7020) pjmedia_aud_subsys aud_subsys;。这样当创建设备时，就可以遍历这些工厂，寻找合适的工厂，通过工厂创建设备实例。比如alsa类型的设备在alsa_dev.c + +#### pjmedia_aud_stream_start + +以alsa为例，在alsa_dev.c中，调用alsa_stream_start函数，在alsa_dev.c alsa_stream_start中，会创建播放和采集两条线程。 + +```c +static pj_status_t alsa_stream_start (pjmedia_aud_stream *s) +{ + struct alsa_stream *stream = (struct alsa_stream*)s; + pj_status_t status = PJ_SUCCESS; + + stream->quit = 0; + if (stream->param.dir & PJMEDIA_DIR_PLAYBACK) { + status = pj_thread_create (stream->pool, + "alsasound_playback", + pb_thread_func, + stream, + 0, //ZERO, + 0, + &stream->pb_thread); + + + if (stream->param.dir & PJMEDIA_DIR_CAPTURE) { + status = pj_thread_create (stream->pool, + "alsasound_playback", + ca_thread_func, + stream, + 0, //ZERO, + 0, + &stream->ca_thread); + + } + + return status; +} +``` + +以播放线程为例 + +```c +static int pb_thread_func (void *arg) +{ + struct alsa_stream* stream = (struct alsa_stream*) arg; + snd_pcm_t* pcm = stream->pb_pcm; + int size = stream->pb_buf_size; + snd_pcm_uframes_t nframes = stream->pb_frames; + void* user_data = stream->user_data; + char* buf = stream->pb_buf; + pj_timestamp tstamp; + int result; + + pj_bzero (buf, size); + tstamp.u64 = 0; + + + snd_pcm_prepare (pcm); + + while (!stream->quit) { + pjmedia_frame frame; + + frame.type = PJMEDIA_FRAME_TYPE_AUDIO; + frame.buf = buf; + frame.size = size; + frame.timestamp.u64 = tstamp.u64; + frame.bit_info = 0; + + result = stream->pb_cb (user_data, &frame); + if (result != PJ_SUCCESS || stream->quit) + break; + + if (frame.type != PJMEDIA_FRAME_TYPE_AUDIO) + pj_bzero (buf, size); + + result = snd_pcm_writei (pcm, buf, nframes); + if (result == -EPIPE) { + PJ_LOG (4,(THIS_FILE, "pb_thread_func: underrun!")); + snd_pcm_prepare (pcm); + } else if (result < 0) { + PJ_LOG (4,(THIS_FILE, "pb_thread_func: error writing data!")); + } + + tstamp.u64 += nframes; + } + + snd_pcm_drain (pcm); + TRACE_((THIS_FILE, "pb_thread_func: Stopped")); + return PJ_SUCCESS; +} +``` + +播放线程先通过回调拿到待播放的音频数据stream->pb_cb ，然后写到声卡snd_pcm_writei。pb_cb就是sound_port.c中的play_cb，来看下play_cb的流程。 + +```c +static pj_status_t play_cb(void *user_data, pjmedia_frame *frame) +{ + pjmedia_snd_port *snd_port = (pjmedia_snd_port*) user_data; + pjmedia_port *port; + const unsigned required_size = (unsigned)frame->size; + pj_status_t status; + + port = snd_port->port; + status = pjmedia_port_get_frame(port, frame); + + /* Invoke preview callback */ + if (snd_port->on_play_frame) + (*snd_port->on_play_frame)(snd_port->user_data, frame); + + return PJ_SUCCESS; +} +``` + +通过pjmedia_port* port获取一帧数据 + diff --git a/src/SDP.md b/src/SDP.md index 0475669..2299916 100644 --- a/src/SDP.md +++ b/src/SDP.md @@ -1,5 +1,7 @@ ## SDP +[SDP协议简介](https://www.jianshu.com/p/94b118b8fd97) + ### pjmedia_sdp_session ```c diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 58be04b..b83a3bd 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -2,13 +2,23 @@ - [sip协议概述](./SIP.md) - [pjsip文档翻译](./pjsip_doc.md) -- [原理机制]() +- [make call流程]() - [状态机](./State.md) - - [make call相关流程图]() + - [流程图]() - [make call流程图](html/make_call.html) - [Create an UAC dialog](html/pjsip_dlg_create_uac.html) - [Outgoing call callback when media transport creation is completed](html/on_make_call_med_tp_complete.html) - [tsx_on_state_null状态处理](html/tsx_on_state_null.html) +- [pjmedia学习]() + - [发送数据过程](./发送数据全流程.md) + - [接收数据过程](./数据全流程.md) + - [初始化过程](./全流程思考.md) + - [相关数据结构]() + - [ioqueue_epoll](./ioqueue_epoll.md) + - [epoll](./epoll学习.md) + - [pjmedia_transport](pjmedia_transport.md) + - [pjmedia_port](./Port.md) + - [pjmedia_stream](./Stream.md) - [常见数据结构与工具]() - [pjsua_call](./pjsua_call.md) - [pjsua_acc](./pjsua_acc.md) diff --git a/src/Stream.md b/src/Stream.md new file mode 100644 index 0000000..96ac8a9 --- /dev/null +++ b/src/Stream.md @@ -0,0 +1,422 @@ +## Stream + +### pjmedia_stream_info + +对应sdp中的 m=字段 （媒体名称和传输地址） + +```makefile +m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 126 +//m=audio说明本会话包含音频，9代表音频使用端口9来传输，但是在webrtc中一现在一般不使用，如果设置为0，代表不 +//传输音频,UDP/TLS/RTP/SAVPF是表示用户来传输音频支持的协议，udp，tls,rtp代表使用udp来传输rtp包，并使用tls加密 +//SAVPF代表使用srtcp的反馈机制来控制通信过程,后台111 103 104 9 0 8 106 105 13 126表示本会话音频支持的编码，后台几行会有详细补充说明 +``` + + + +```c +/** + * This structure describes media stream information. Each media stream + * corresponds to one "m=" line in SDP session descriptor, and it has + * its own RTP/RTCP socket pair. + */ +typedef struct pjmedia_stream_info +{ + pjmedia_type type; /**< Media type (audio, video) */ + pjmedia_tp_proto proto; /**< Transport protocol (RTP/AVP, etc.) */ + pjmedia_dir dir; /**< Media direction. */ + pj_sockaddr local_addr; /**< Local RTP address */ + pj_sockaddr rem_addr; /**< Remote RTP address */ + pj_sockaddr rem_rtcp; /**< Optional remote RTCP address. If + sin_family is zero, the RTP address + will be calculated from RTP. */ + pj_bool_t rtcp_mux; /**< Use RTP and RTCP multiplexing. */ +#if defined(PJMEDIA_HAS_RTCP_XR) && (PJMEDIA_HAS_RTCP_XR != 0) + pj_bool_t rtcp_xr_enabled; + /**< Specify whether RTCP XR is enabled.*/ + pj_uint32_t rtcp_xr_interval; /**< RTCP XR interval. */ + pj_sockaddr rtcp_xr_dest;/**events的事件类型 + + 功能：用于控制某个文件描述符上的事件，可以注册事件，修改事件，删除事件。 + + @epfd：由epoll_create生成的epoll专用的文件描述符； + + @op：要进行的操作，EPOLL_CTL_ADD注册、EPOLL_CTL_MOD修改、EPOLL_CTL_DEL删除； + + @fd：关联的文件描述符； + + @event：指向epoll_event的指针； + + 成功：0；失败：-1 + + +3、epoll_wait函数 + 函数声明:int epoll_wait(int epfd,struct epoll_event * events,int maxevents,int timeout) + + 功能：该函数用于轮询I/O事件的发生； + + @epfd：由epoll_create生成的epoll专用的文件描述符； + + @epoll_event：用于回传代处理事件的数组； + + @maxevents：每次能处理的事件数； + + @timeout：等待I/O事件发生的超时值； + + 成功：返回发生的事件数；失败：-1 + + + +`epoll_ctl` 是用于控制 epoll 实例的系统调用之一，它用于向 epoll 实例注册或删除事件，并指定相应的文件描述符和事件类型。 + +`epoll_ctl` 的原型如下： + +``` +cCopy code +int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event); +``` + +其中参数的含义如下： + +- `epfd`：epoll 实例的文件描述符，通过 `epoll_create` 创建得到。 + +- ``` + op + ``` + + ：操作类型，可以是以下几种值之一： + + - `EPOLL_CTL_ADD`：添加一个新的文件描述符到 epoll 实例中。 + - `EPOLL_CTL_MOD`：修改一个已注册的文件描述符的事件类型。 + - `EPOLL_CTL_DEL`：从 epoll 实例中删除一个文件描述符。 + +- `fd`：要添加、修改或删除的文件描述符。 + +- `event`：一个指向 `struct epoll_event` 结构体的指针，用于描述要注册的事件类型和相关的数据。 + +`struct epoll_event` 结构体定义如下： + +``` +cCopy code + + +解释struct epoll_event { + uint32_t events; // 事件类型，可以是 EPOLLIN、EPOLLOUT、EPOLLERR 等 + epoll_data_t data; // 事件关联的数据，通常是一个联合体 +}; +``` + +`epoll_ctl` 的主要作用是管理 epoll 实例中的文件描述符和事件，可以添加、修改或删除特定的文件描述符以及相应的事件类型。通过这个系统调用，可以实现高效的 I/O 多路复用机制，使得应用程序能够监视多个文件描述符上的事件，并在事件发生时做出相应的处理。 + + + +```c +#include +#include +#include +#include + +#define MAX_EVENTS 10 + +int main() { + int epoll_fd, nfds, i; + struct epoll_event event, events[MAX_EVENTS]; + + // 创建 epoll 实例 + epoll_fd = epoll_create1(0); + if (epoll_fd == -1) { + perror("epoll_create1"); + exit(EXIT_FAILURE); + } + + // 打开一个文件描述符（这里以标准输入描述符为例） + int stdin_fd = STDIN_FILENO; + + // 配置要监听的事件 + event.events = EPOLLIN; // 监听读事件 + event.data.fd = stdin_fd; // 将标准输入描述符与事件关联 + + // 将事件添加到 epoll 实例中 + if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, stdin_fd, &event) == -1) { + perror("epoll_ctl: EPOLL_CTL_ADD"); + close(epoll_fd); + exit(EXIT_FAILURE); + } + + // 等待事件发生 + while (1) { + nfds = epoll_wait(epoll_fd, events, MAX_EVENTS, -1); + if (nfds == -1) { + perror("epoll_wait"); + close(epoll_fd); + exit(EXIT_FAILURE); + } + + // 处理所有发生的事件 + for (i = 0; i < nfds; ++i) { + if (events[i].events & EPOLLIN) { + printf("Event on file descriptor %d: EPOLLIN\n", events[i].data.fd); + + // 从标准输入读取数据并进行处理 + char buffer[1024]; + ssize_t bytes_read = read(events[i].data.fd, buffer, sizeof(buffer)); + if (bytes_read == -1) { + perror("read"); + close(epoll_fd); + exit(EXIT_FAILURE); + } + // 处理读取的数据 + // ... + } + } + } + + // 关闭 epoll 实例 + close(epoll_fd); + + return 0; +} + +``` + diff --git a/src/html/pjmedia_transport_udp_attach.html b/src/html/pjmedia_transport_udp_attach.html new file mode 100644 index 0000000..de2fa19 --- /dev/null +++ b/src/html/pjmedia_transport_udp_attach.html @@ -0,0 +1,11 @@ + + + + +pjmedia_transport_udp_attach + + +
+ + + \ No newline at end of file diff --git a/src/img/20190917203551691.png b/src/img/20190917203551691.png new file mode 100644 index 0000000..7e97ffe Binary files /dev/null and b/src/img/20190917203551691.png differ diff --git a/src/ioqueue_epoll.md b/src/ioqueue_epoll.md new file mode 100644 index 0000000..437faa9 --- /dev/null +++ b/src/ioqueue_epoll.md @@ -0,0 +1,472 @@ +## ioqueue_epoll + +### pj_ioqueue_t + +ioqueue的整体结构pj_ioqueue_t，使用epoll的在ioqueue_epoll.c 下列出ioqueue相关的`DECLARE_COMMON_IOQUEUE宏`、`pj_ioqueue_cfg`、`pj_ioqueue_t` + +```c +#define DECLARE_COMMON_IOQUEUE \ + pj_lock_t *lock; \ + pj_bool_t auto_delete_lock; \ + pj_ioqueue_cfg cfg; + +/** + * Additional settings that can be given during ioqueue creation. Application + * MUST initialize this structure with #pj_ioqueue_cfg_default(). + */ +typedef struct pj_ioqueue_cfg +{ + /** + * Specify flags to control e.g. how events are handled when epoll backend + * is used on Linux. The values are combination of pj_ioqueue_epoll_flag. + * The default value is PJ_IOQUEUE_DEFAULT_EPOLL_FLAGS, which by default + * is set to PJ_IOQUEUE_EPOLL_AUTO. This setting will be ignored for other + * ioqueue backends. + */ + unsigned epoll_flags; + + /** + * Default concurrency for the handles registered to this ioqueue. Setting + * this to non-zero enables a handle to process more than one operations + * at the same time using different threads. Default is + * PJ_IOQUEUE_DEFAULT_ALLOW_CONCURRENCY. This setting is equivalent to + * calling pj_ioqueue_set_default_concurrency() after creating the ioqueue. + */ + pj_bool_t default_concurrency; + +} pj_ioqueue_cfg; + +/* + * This describes the I/O queue. + */ +struct pj_ioqueue_t +{ + DECLARE_COMMON_IOQUEUE + + unsigned max, count; + //pj_ioqueue_key_t hlist; + pj_ioqueue_key_t active_list; + int epfd; + //struct epoll_event *events; + //struct queue *queue; + +#if PJ_IOQUEUE_HAS_SAFE_UNREG + pj_mutex_t *ref_cnt_mutex; + pj_ioqueue_key_t closing_list; + pj_ioqueue_key_t free_list; +#endif +}; +``` + +有三个 pj_ioqueue_key_t类型的队列active_list、closing_list、free_list， + +### pj_ioqueue_key_t + +```c +#define DECLARE_COMMON_KEY \ + PJ_DECL_LIST_MEMBER(struct pj_ioqueue_key_t); \ + pj_ioqueue_t *ioqueue; \ + pj_grp_lock_t *grp_lock; \ + pj_lock_t *lock; \ + pj_bool_t inside_callback; \ + pj_bool_t destroy_requested; \ + pj_bool_t allow_concurrent; \ + pj_sock_t fd; \ + int fd_type; \ + void *user_data; \ + pj_ioqueue_callback cb; \ + int connecting; \ + struct read_operation read_list; \ + struct write_operation write_list; \ + struct accept_operation accept_list; \ + UNREG_FIELDS + +/* + * This describes each key. + */ +struct pj_ioqueue_key_t +{ + DECLARE_COMMON_KEY + struct epoll_event ev; +}; + +``` + +pj_ioqueue_key_t中出现了epoll_event 是linux中结构 + +### epoll_event + +```c +typedef [union](https://so.csdn.net/so/search?q=union&spm=1001.2101.3001.7020) epoll_data { + void *ptr; + int fd; + __uint32_t u32; + __uint64_t u64; +} epoll_data_t;//保存触发事件的某个文件描述符相关的数据 + +struct epoll_event { + __uint32_t events; /* [epoll](https://so.csdn.net/so/search?q=epoll&spm=1001.2101.3001.7020) event */ + epoll_data_t data; /* User data variable */ +}; +``` + +### pj_ioqueue_callback + +接下来介绍I/O结束的回调函数 + +```c +/** + * This structure describes the callbacks to be called when I/O operation + * completes. + */ +typedef struct pj_ioqueue_callback +{ + /** + * This callback is called when #pj_ioqueue_recv or #pj_ioqueue_recvfrom + * completes. + * + * @param key The key. + * @param op_key Operation key. + * @param bytes_read >= 0 to indicate the amount of data read, + * otherwise negative value containing the error + * code. To obtain the pj_status_t error code, use + * (pj_status_t code = -bytes_read). + */ + void (*on_read_complete)(pj_ioqueue_key_t *key, + pj_ioqueue_op_key_t *op_key, + pj_ssize_t bytes_read); + + /** + * This callback is called when #pj_ioqueue_send or #pj_ioqueue_sendto + * completes. + * + * @param key The key. + * @param op_key Operation key. + * @param bytes_sent >= 0 to indicate the amount of data written, + * otherwise negative value containing the error + * code. To obtain the pj_status_t error code, use + * (pj_status_t code = -bytes_sent). + */ + void (*on_write_complete)(pj_ioqueue_key_t *key, + pj_ioqueue_op_key_t *op_key, + pj_ssize_t bytes_sent); + + /** + * This callback is called when #pj_ioqueue_accept completes. + * + * @param key The key. + * @param op_key Operation key. + * @param sock Newly connected socket. + * @param status Zero if the operation completes successfully. + */ + void (*on_accept_complete)(pj_ioqueue_key_t *key, + pj_ioqueue_op_key_t *op_key, + pj_sock_t sock, + pj_status_t status); + + /** + * This callback is called when #pj_ioqueue_connect completes. + * + * @param key The key. + * @param status PJ_SUCCESS if the operation completes successfully. + */ + void (*on_connect_complete)(pj_ioqueue_key_t *key, + pj_status_t status); +} pj_ioqueue_callback; +``` + +### pj_ioqueue_op_key_t + +```c +typedef struct pj_ioqueue_op_key_t +{ + void *internal__[32]; /**< Internal I/O Queue data. */ + void *activesock_data; /**< Active socket data. */ + void *user_data; /**< Application data. */ +} pj_ioqueue_op_key_t; +``` + +## 初始化相关 + +### pj_ioqueue_create2 + +pj_ioqueue_create（空实现调用pj_ioqueue_create2） + +```c +status = pj_ioqueue_create( endpt->pool, PJSIP_MAX_TRANSPORTS, &endpt->ioqueue); +if (status != PJ_SUCCESS) { + goto on_error; +} +``` + +初始化ioqueue的空间、ioqueue->lock、ioqueue->auto_delete_lock、ioqueue->cfg（pj_ioqueue_cfg）、ioqueue->max、ioqueue->count、ioqueue->cfg.epoll_flags（epoll type）、ioqueue->ref_cnt_mutex、ioqueue->free_list（对max_fd个key初始化key->ref_count、key->lock然后加入freelist）、ioqueue->closing_list、ioqueue->epfd（epoll fd调用epoll create） + +### ioqueue_init_key + +```c +static pj_status_t ioqueue_init_key( pj_pool_t *pool, + pj_ioqueue_t *ioqueue, + pj_ioqueue_key_t *key, + pj_sock_t sock, + pj_grp_lock_t *grp_lock, + void *user_data, + const pj_ioqueue_callback *cb) +``` + +初始化key的key->ioqueue、key->fd、key->user_data（这里放到是transport_udp）、key->read_list、key->write_list、key->accept_list、key->connecting = 0、key->cb（callback）、key->closing 、key->allow_concurrent（pj_ioqueue_set_concurrency）、key->fd_type、key->grp_lock + +### pj_ioqueue_register_sock2 + +pjmedia_transport_udp_create3->pjmedia_transport_udp_attach->pj_ioqueue_register_sock2 + +os_ioctl 设置socket to nonblocking. + +从ioqueue的freelist中取得key，ioqueue_init_key，然后初始化key->ev（epoll_event类型 data.ptr 是key本身）, os_epoll_ctl注册 fd-events监听其socket事件。 + +```c +PJ_DEF(pj_status_t) pj_ioqueue_register_sock2(pj_pool_t *pool, + pj_ioqueue_t *ioqueue, + pj_sock_t sock, + pj_grp_lock_t *grp_lock, + void *user_data, + const pj_ioqueue_callback *cb, + pj_ioqueue_key_t **p_key) + + status = pj_ioqueue_register_sock2(pool, ioqueue, tp->rtp_sock, grp_lock, + tp, &rtp_cb, &tp->rtp_key); +``` + +注意在`pjmedia_transport_udp_attach` 中调用pj_ioqueue_register_sock2传入的是tp->rtp_key，pj_ioqueue_register_sock2返回后，会将绑定socket后的key保存在rtp_key中 + +## ioqueue 读取写入流程 + +ioqueue 读取全流程 + +pj_ioqueue_poll，发现事件-》ioqueue_dispatch_read_event-》on_rx_request-》on_rx_request + +### pj_ioqueue_poll + +事件触发后执行主要使用os_epoll_wait，执行分发回调 + +一般会另开一个工作线程，不停循环，执行epoll_wait以监听读取/写入rtp包的事件请求 + +```c +while(1){ + pj_ioqueue_poll() +} +``` + +下面详细介绍pj_ioqueue_poll + +先调用os_epoll_wait，事件放到events中，遍历所有events。根据read,write事件放入到queue中，queue结构如下： + +```c +struct queue +{ + pj_ioqueue_key_t *key; + enum ioqueue_event_type event_type; +}; + +enum ioqueue_event_type +{ + NO_EVENT, + READABLE_EVENT = 1, + WRITEABLE_EVENT = 2, + EXCEPTION_EVENT = 4, +}; +``` + +最后在遍历queue，根据事件执行相应的处理函数 + +READABLE_EVENT：ioqueue_dispatch_read_event + +WRITEABLE_EVENT：ioqueue_dispatch_write_event + +EXCEPTION_EVENT：ioqueue_dispatch_exception_e + +### ioqueue_dispatch_read_event 读取操作 + +首先要看一下read_operation + +```c +struct read_operation +{ + PJ_DECL_LIST_MEMBER(struct read_operation); + pj_ioqueue_operation_e op; + + void *buf; + pj_size_t size; + unsigned flags; + pj_sockaddr_t *rmt_addr; + int *rmt_addrlen; +}; +``` + +```c + +/** + * Types of pending I/O Queue operation. This enumeration is only used + * internally within the ioqueue. + */ +typedef enum pj_ioqueue_operation_e +{ + PJ_IOQUEUE_OP_NONE = 0, /**< No operation. */ + PJ_IOQUEUE_OP_READ = 1, /**< read() operation. */ + PJ_IOQUEUE_OP_RECV = 2, /**< recv() operation. */ + PJ_IOQUEUE_OP_RECV_FROM = 4, /**< recvfrom() operation. */ + PJ_IOQUEUE_OP_WRITE = 8, /**< write() operation. */ + PJ_IOQUEUE_OP_SEND = 16, /**< send() operation. */ + PJ_IOQUEUE_OP_SEND_TO = 32, /**< sendto() operation. */ +#if defined(PJ_HAS_TCP) && PJ_HAS_TCP != 0 + PJ_IOQUEUE_OP_ACCEPT = 64, /**< accept() operation. */ + PJ_IOQUEUE_OP_CONNECT = 128 /**< connect() operation. */ +#endif /* PJ_HAS_TCP */ +} pj_ioqueue_operation_e; +``` + + + +众多read_operation会挂在key中`struct read_operation read_list;` 上通过`key_has_pending_read` 判断是否有pending的read操作 + +具体流程： + +先看key read_list上有没有pending_read,有的话，从read_list取出，根据read_list的read_op确定读入大小，pj_sock_recvfrom接受数据的函数，将数据读入到read_op中，最后调用on_read_complete，回调函数已在pj_ioqueue_register_sock2时设置过，传入read_op + +```c +(*h->cb.on_read_complete)(h, + (pj_ioqueue_op_key_t*)read_op, + bytes_read); +``` + + + +### ioqueue_dispatch_write_event写操作 + +与ioqueue_dispatch_read_event相似，先看write_operation + +```c +struct write_operation +{ + PJ_DECL_LIST_MEMBER(struct write_operation); + pj_ioqueue_operation_e op; + + char *buf; + pj_size_t size; + pj_ssize_t written; + unsigned flags; + pj_sockaddr_in rmt_addr; + int rmt_addrlen; +}; +``` + +众多write_operation会挂在key中`struct write_operation write_list;` 上通过`key_has_pending_write` 判断是否有pending的write操作 + +具体流程： + +先看key write_list上有没有pending_write,有的话，从write_list取出，根据write_list的 write_op确定写大小，要写入的数据，将数据写入调用pj_sock_send函数Transmit data to the socket.，最后调用on_write_complete，回调函数已在pj_ioqueue_register_sock2时设置过，传入write_op + +```c +if (h->cb.on_write_complete && !IS_CLOSING(h)) { + (*h->cb.on_write_complete)(h, + (pj_ioqueue_op_key_t*)write_op, + write_op->written); + } +``` + +### on_rx_rtp 读取操作 + +on_rx_rtp是被下面调用 + +```c +(*h->cb.on_read_complete)(h, + (pj_ioqueue_op_key_t*)read_op, + bytes_read); +``` + +这里有一个很有意思的问题，就是read_op 在on_rx_rtp中竟然没有使用，下面来分析一下原因 + +``` +udp = (struct transport_udp*) pj_ioqueue_get_user_data(key); + +``` + +我们的read_op是从readlist中取出的，readlist对于读操作添加是依靠pj_ioqueue_recvfrom函数，在data is not immediately available时将read_op加入readlist + +```c +read_op->op = PJ_IOQUEUE_OP_RECV_FROM; +read_op->buf = buffer; +read_op->size = *length; +read_op->flags = flags; +read_op->rmt_addr = addr; +read_op->rmt_addrlen = addrlen; + +pj_ioqueue_lock_key(key); +/* Check again. Handle may have been closed after the previous check + * in multithreaded app. If we add bad handle to the set it will + * corrupt the ioqueue set. See #913 + */ +if (IS_CLOSING(key)) { + pj_ioqueue_unlock_key(key); + return PJ_ECANCELLED; +} +pj_list_insert_before(&key->read_list, read_op); +ioqueue_add_to_set(key->ioqueue, key, READABLE_EVENT); +``` + +这里read_op->buf = buffer;的buffer，来自udp->rtp_pkt,相当于直接写入了rtp_pkt,所以不用read_op了。 + +on_rx_rtp是一个while循环，条件如下status来自pj_ioqueue_recvfrom的结果 + +```c +status != PJ_EPENDING && status != PJ_ECANCELLED && + udp->started +``` + +#### call_rtp_cb + +在while循环里，先执行call_rtp_cb，设置pjmedia_tp_cb_param param;，调用(*cb2)(¶m);cb2由transport_attach2-》tp_attach设置为stream.c ::on_rx_rtp。注意param.pkt = udp->rtp_pkt;，这里rtp_pkt其实就是ioqueue_dispatch_read_event中read_op->buf中读到的数据rtp包 + +```c +/* Call RTP cb. */ +static void call_rtp_cb(struct transport_udp *udp, pj_ssize_t bytes_read, + pj_bool_t *rem_switch) +{ + void (*cb)(void*,void*,pj_ssize_t); + void (*cb2)(pjmedia_tp_cb_param*); + void *user_data; + + cb = udp->rtp_cb; + cb2 = udp->rtp_cb2; + user_data = udp->user_data; + + if (cb2) { + pjmedia_tp_cb_param param; + + param.user_data = user_data; + param.pkt = udp->rtp_pkt; + param.size = bytes_read; + param.src_addr = &udp->rtp_src_addr; + param.rem_switch = PJ_FALSE; + (*cb2)(¶m); + if (rem_switch) + *rem_switch = param.rem_switch; + } else if (cb) { + (*cb)(user_data, udp->rtp_pkt, bytes_read); + } +} +``` + + param.user_data = user_data; 注意这个user_data，是pjmedia_stream *stream + +#### pj_ioqueue_recvfrom + +接下来是调用pj_ioqueue_recvfrom，至于为什么明明ioqueue_dispatch_read_event已经读取了数据，此时还在读取数据，是因为可能有新的rtp包到达，pj_ioqueue_recvfrom查看有没有到达的包，如果有就调用pj_sock_recvfrom继续读读到udp->rtp_pkt，如果没有加到readlist中，返回PJ_EPENDING，结束on_rx_rtp中的while循环。 + +### on_rx_rtp::cb2 + +Stream.c中的回调 tp_attach中设置该回调 + +该函数处理接收到的rtp包, 解析成payload和head + +Put "good" packet to jitter buffer，需要先把payload解析成frame，再把frame放入jitter buffer + diff --git a/src/pj_ioqueue.md b/src/pj_ioqueue.md new file mode 100644 index 0000000..8c26c0d --- /dev/null +++ b/src/pj_ioqueue.md @@ -0,0 +1,65 @@ +## pj_ioqueue + +### pj_ioqueue_callback + +```c +/** + * This structure describes the callbacks to be called when I/O operation + * completes. + */ +typedef struct pj_ioqueue_callback +{ + /** + * This callback is called when #pj_ioqueue_recv or #pj_ioqueue_recvfrom + * completes. + * + * @param key The key. + * @param op_key Operation key. + * @param bytes_read >= 0 to indicate the amount of data read, + * otherwise negative value containing the error + * code. To obtain the pj_status_t error code, use + * (pj_status_t code = -bytes_read). + */ + void (*on_read_complete)(pj_ioqueue_key_t *key, + pj_ioqueue_op_key_t *op_key, + pj_ssize_t bytes_read); + + /** + * This callback is called when #pj_ioqueue_send or #pj_ioqueue_sendto + * completes. + * + * @param key The key. + * @param op_key Operation key. + * @param bytes_sent >= 0 to indicate the amount of data written, + * otherwise negative value containing the error + * code. To obtain the pj_status_t error code, use + * (pj_status_t code = -bytes_sent). + */ + void (*on_write_complete)(pj_ioqueue_key_t *key, + pj_ioqueue_op_key_t *op_key, + pj_ssize_t bytes_sent); + + /** + * This callback is called when #pj_ioqueue_accept completes. + * + * @param key The key. + * @param op_key Operation key. + * @param sock Newly connected socket. + * @param status Zero if the operation completes successfully. + */ + void (*on_accept_complete)(pj_ioqueue_key_t *key, + pj_ioqueue_op_key_t *op_key, + pj_sock_t sock, + pj_status_t status); + + /** + * This callback is called when #pj_ioqueue_connect completes. + * + * @param key The key. + * @param status PJ_SUCCESS if the operation completes successfully. + */ + void (*on_connect_complete)(pj_ioqueue_key_t *key, + pj_status_t status); +} pj_ioqueue_callback; +``` + diff --git a/src/pjmedia.md b/src/pjmedia.md new file mode 100644 index 0000000..8497139 --- /dev/null +++ b/src/pjmedia.md @@ -0,0 +1,803 @@ +## pjmedia总述 + +转载：[pjmedia_HYQ458941968的博客-CSDN博客](https://blog.csdn.net/hyq458941968/category_11380864.html) + +从对象关系来看： + +**一次session，多个endpoint参与，彼此发送stream，每个stream两个channel分别收发。音频设备port对象采集播放声音，transport最终发送接收信号。** + +1、 pjmedia_endpt，代表一个媒体端点，端点可以理解为一个节点，可以是服务器或者客户端，一个设备一般只会有唯一一个端点，而且在初始化的时候创建。 + +2、pjmedia_session，代表一次会话，一个会话可以只有两个，也可以有多个参与者会议。 + +3、pjmedia_stream，代表一个流，在于一方进行通讯时，一般可以有音频流、视频流。 + +4、pjmedia_channel，只有一个方向的媒体流，也就是说，对应音频流，要发送通道和接收通道两个channel。 + +以上是范围从大到小的媒体对象，下面介绍其它对象。 + +5、pjmedia_transport，传输对象，封装了所有从网络接收数据和发送数据到网络的操作，可以是UDP、SRTP、ICE等传输方式。 + +6、pjmedia_snd_port，音频设备对象，最终的音频裸流都要在设备上播放，从麦克风采集，此对象封装所有设备操作。 + +从数据流来看： + +![img](img/20190917203551691.png) + +左边为最底层，音频设备的播放和采集，通过回调接口，调用stream.c生产或消化数据，最后通过transport传输接口进行发送接收。当然，其中还夹杂着前后处理、编解码、抖动缓冲区等，后面的章节会对各个对象及相关的音频处理进行描述。 + +最后来分析一下实例代码simpleua.c中对pjmedia的使用方法。 + +### **初始化** + +在main函数中，前半部分主要是对sip的初始化，对pjmedia的初始化大概从358行开始 + +#### 1、创建媒体端点endpoint + +```cpp +static pjmedia_endpt *g_med_endpt; /* Media endpoint. */ + +#if PJ_HAS_THREADS + status = pjmedia_endpt_create(&cp.factory, NULL, 1, &g_med_endpt); +#else + status = pjmedia_endpt_create(&cp.factory, + pjsip_endpt_get_ioqueue(g_endpt), + 0, &g_med_endpt); +#endif +``` + +这里我们默认看有多线程的流程。 + +#### 2、创建udp网络接口transport + +```cpp +static pjmedia_transport *g_med_transport[MAX_MEDIA_CNT]; + + /* + * Create media transport used to send/receive RTP/RTCP socket. + * One media transport is needed for each call. Application may + * opt to re-use the same media transport for subsequent calls. + */ + for (i = 0; i < PJ_ARRAY_SIZE(g_med_transport); ++i) { + status = pjmedia_transport_udp_create3(g_med_endpt, AF, NULL, NULL, + RTP_PORT + i*2, 0, + &g_med_transport[i]); +``` + +可以看出，虽然初始化还没有建立媒体通讯，但是预先创建了若干传输对象。 + +#### 3、loop处理sip事件 + +```cpp +/* Loop until one call is completed */ +for (;!g_complete;) { + pj_time_val timeout = {0, 10}; + pjsip_endpt_handle_events(g_endpt, &timeout); +} +``` + +初始化的最后是一个循环等待处理sip对象的操作。 + +### **建立媒体通讯** + +当sip,sdp协商成功后，则要开始媒体通讯，发生在回调函数call_on_media_update + +#### 1、创建并启动流对象stream + +```cpp +static pjmedia_stream *g_med_stream; /* Call's audio stream. */ + + /* Create stream info based on the media audio SDP. */ + status = pjmedia_stream_info_from_sdp(&stream_info, inv->dlg->pool, + g_med_endpt, + local_sdp, remote_sdp, 0); + + /* Create new audio media stream, passing the stream info, and also the + * media socket that we created earlier. + */ + status = pjmedia_stream_create(g_med_endpt, inv->dlg->pool, &stream_info, + g_med_transport[0], NULL, &g_med_stream); + + /* Start the audio stream */ + status = pjmedia_stream_start(g_med_stream); +``` + +这个实例并没有创建session，而是直接创建流对象。先从sip协商的sdp中获取媒体信息pjmedia_stream_info_from_sdp，然后根据这些信息创建流对象pjmedia_stream_create，并紧接着启动流pjmedia_stream_start。这里只有音频流，视频流暂不纳入分析范围。 + +#### 2、启动网络传输transport + +```cpp + /* Start the UDP media transport */ + pjmedia_transport_media_start(g_med_transport[0], 0, 0, 0, 0); +``` + +前面讲到，初始化的时候预创建了若干传输对象，但是并没有启动，等到协商成功后，才启动网络传输。 + +#### 3、创建音频设备对象Sound_device + +```cpp +static pjmedia_snd_port *g_snd_port; /* Sound device. */ + + /* Get the media port interface of the audio stream. + * Media port interface is basicly a struct containing get_frame() and + * put_frame() function. With this media port interface, we can attach + * the port interface to conference bridge, or directly to a sound + * player/recorder device. + */ + pjmedia_stream_get_port(g_med_stream, &media_port); + + /* Create sound port */ + pjmedia_snd_port_create(inv->pool, + PJMEDIA_AUD_DEFAULT_CAPTURE_DEV, + PJMEDIA_AUD_DEFAULT_PLAYBACK_DEV, + PJMEDIA_PIA_SRATE(&media_port->info),/* clock rate */ + PJMEDIA_PIA_CCNT(&media_port->info),/* channel count */ + PJMEDIA_PIA_SPF(&media_port->info), /* samples per frame*/ + PJMEDIA_PIA_BITS(&media_port->info),/* bits per sample */ + 0, + &g_snd_port); + + status = pjmedia_snd_port_connect(g_snd_port, media_port); +``` + +这里先从流对象取出媒体接口pjmedia_stream_get_port，其中的媒体接口包含了数据回调，这些后面分析数据流再重点讲，然后创建pjmedia_snd_port_create并启动pjmedia_snd_port_connect音频设备对象。 + +### **结束销毁** + +```cpp + /* Destroy audio ports. Destroy the audio port first + * before the stream since the audio port has threads + * that get/put frames to the stream. + */ + if (g_snd_port) + pjmedia_snd_port_destroy(g_snd_port); + + + /* Destroy streams */ + if (g_med_stream) + pjmedia_stream_destroy(g_med_stream); + + /* Destroy media transports */ + for (i = 0; i < MAX_MEDIA_CNT; ++i) { + if (g_med_transport[i]) + pjmedia_transport_close(g_med_transport[i]); + } + + /* Deinit pjmedia endpoint */ + if (g_med_endpt) + pjmedia_endpt_destroy(g_med_endpt); + + /* Deinit pjsip endpoint */ + if (g_endpt) + pjsip_endpt_destroy(g_endpt); + + /* Release pool */ + if (pool) + pj_pool_release(pool); +``` + +当主线程退出loop时，则销毁所有创建的对象。 + +## pjmedia_endpt + +simpleua.c在进行媒体相关初始化时，首先创建媒体端点，看看媒体端点的数据结构和创建流程。 + +```cpp +#if PJ_HAS_THREADS + status = pjmedia_endpt_create(&cp.factory, NULL, 1, &g_med_endpt); +#else + status = pjmedia_endpt_create(&cp.factory, + pjsip_endpt_get_ioqueue(g_endpt), + 0, &g_med_endpt); +#endif +``` + +### **pjmedia_endpt结构体** + +```cpp +/** Concrete declaration of media endpoint. */ +struct pjmedia_endpt +{ + /** Pool. */ + pj_pool_t *pool; + + /** Pool factory. */ + pj_pool_factory *pf; + + /** Codec manager. */ + pjmedia_codec_mgr codec_mgr; + + /** IOqueue instance. */ + pj_ioqueue_t *ioqueue; + + /** Do we own the ioqueue? */ + pj_bool_t own_ioqueue; + + /** Number of threads. */ + unsigned thread_cnt; + + /** IOqueue polling thread, if any. */ + pj_thread_t *thread[MAX_THREADS]; + + /** To signal polling thread to quit. */ + pj_bool_t quit_flag; + + /** Is telephone-event enable */ + pj_bool_t has_telephone_event; + + /** List of exit callback. */ + exit_cb exit_cb_list; +}; +``` + +ioqueue：用于绑定socket，然后异步接收数据； + +thread[]：线程数组，存储工作线程。 + +codec_mgr：codec的管理者 + +exit_cb_list：退出时的回调链表 + +quit_flag：置位则工作线程都退出 + +### **创建端点** + +```cpp +/** + * Initialize and get the instance of media endpoint. + */ +PJ_DEF(pj_status_t) pjmedia_endpt_create2(pj_pool_factory *pf, + pj_ioqueue_t *ioqueue, + unsigned worker_cnt, + pjmedia_endpt **p_endpt) +{ + pj_pool_t *pool; + pjmedia_endpt *endpt; + unsigned i; + pj_status_t status; + + + pool = pj_pool_create(pf, "med-ept", 512, 512, NULL); + if (!pool) + return PJ_ENOMEM; + + endpt = PJ_POOL_ZALLOC_T(pool, struct pjmedia_endpt); + endpt->pool = pool; + endpt->pf = pf; + endpt->ioqueue = ioqueue; + endpt->thread_cnt = worker_cnt; + endpt->has_telephone_event = PJ_TRUE; + + + /* Init codec manager. */ + status = pjmedia_codec_mgr_init(&endpt->codec_mgr, endpt->pf); + if (status != PJ_SUCCESS) + goto on_error; + + /* Initialize exit callback list. */ + pj_list_init(&endpt->exit_cb_list); + + /* Create ioqueue if none is specified. */ + if (endpt->ioqueue == NULL) { + + endpt->own_ioqueue = PJ_TRUE; + + status = pj_ioqueue_create( endpt->pool, PJ_IOQUEUE_MAX_HANDLES, + &endpt->ioqueue); + if (status != PJ_SUCCESS) + goto on_error; + + } + + /* Create worker threads if asked. */ + for (i=0; ipool, "media", &worker_proc, + endpt, 0, 0, &endpt->thread[i]); + if (status != PJ_SUCCESS) + goto on_error; + } + + + *p_endpt = endpt; + return PJ_SUCCESS; + +on_error: + + /* Destroy threads */ + for (i=0; ithread_cnt; ++i) { + if (endpt->thread[i]) { + pj_thread_destroy(endpt->thread[i]); + } + } + + /* Destroy internal ioqueue */ + if (endpt->ioqueue && endpt->own_ioqueue) + pj_ioqueue_destroy(endpt->ioqueue); + + pjmedia_codec_mgr_destroy(&endpt->codec_mgr); + //pjmedia_aud_subsys_shutdown(); + pj_pool_release(pool); + return status; +} +``` + +1、创建内存池med-ept + +2、申请媒体端点结构体内存，并把外部的ioqueue地址存到结构体 endpt->ioqueue = ioqueue; + +3、初始化编码器manager pjmedia_codec_mgr_init(&endpt->codec_mgr, endpt->pf) + +4、如果初入的ioqueue为空，则创建一个新的ioqueue + +5、创建工作线程组，线程函数是worker_proc + +**工作线程** + +```cpp +/** + * Worker thread proc. + */ +static int PJ_THREAD_FUNC worker_proc(void *arg) +{ + pjmedia_endpt *endpt = (pjmedia_endpt*) arg; + + while (!endpt->quit_flag) { + pj_time_val timeout = { 0, 500 }; + pj_ioqueue_poll(endpt->ioqueue, &timeout); + } + + return 0; +} +``` + +工作线程就是每500ms调用一次pj_ioqueue_pool。刚创建媒体端点时，ioqueue还没有注册socket，所以不会监控到数据。 + +## pjmedia_transport + +媒体传输封装了网络收发细节，pjmedia_transport可以是udp、srtp、ice等，这里以udp为例。 + +### **结构体pjmedia_transport** + +```cpp +/** + * This structure declares media transport. A media transport is called + * by the stream to transmit a packet, and will notify stream when + * incoming packet is arrived. + */ +struct pjmedia_transport +{ + /** Transport name (for logging purpose). */ + char name[PJ_MAX_OBJ_NAME]; + + /** Transport type. */ + pjmedia_transport_type type; + + /** Transport's "virtual" function table. */ + pjmedia_transport_op *op; + + /** Application/user data */ + void *user_data; +}; +``` + +type：传输类型，上面讲过，这里以udp为例。 + +op：操作集，每种传输类型实现了同一组接口。 + +user_data：应用层用户数据 + +### pjmedia_transport_op + +操作集是核心，这里列举重要的一些函数。[pjmedia_transport_op](https://docs.pjsip.org/en/latest/api/generated/pjmedia/group/group__PJMEDIA__TRANSPORT.html#_CPPv420pjmedia_transport_op) + +```cpp +/** + * This structure describes the operations for the stream transport. + */ +struct pjmedia_transport_op +{ + + /** + * This function is called by the stream when the transport is about + * to be used by the stream for the first time, and it tells the transport + * about remote RTP address to send the packet and some callbacks to be + * called for incoming packets. This function exists for backwards + * compatibility. Transports should implement attach2 instead. + * + * Application should call #pjmedia_transport_attach() instead of + * calling this function directly. + */ + pj_status_t (*attach)(pjmedia_transport *tp, + void *user_data, + const pj_sockaddr_t *rem_addr, + const pj_sockaddr_t *rem_rtcp, + unsigned addr_len, + void (*rtp_cb)(void *user_data, + void *pkt, + pj_ssize_t size), + void (*rtcp_cb)(void *user_data, + void *pkt, + pj_ssize_t size)); + + /** + * This function is called by the stream to send RTP packet using the + * transport. + * + * Application should call #pjmedia_transport_send_rtp() instead of + * calling this function directly. + */ + pj_status_t (*send_rtp)(pjmedia_transport *tp, + const void *pkt, + pj_size_t size); + + /** + * Prepare the transport for a new media session. + * + * Application should call #pjmedia_transport_media_create() instead of + * calling this function directly. + */ + pj_status_t (*media_create)(pjmedia_transport *tp, + pj_pool_t *sdp_pool, + unsigned options, + const pjmedia_sdp_session *remote_sdp, + unsigned media_index); + + /** + * This function is called by application to start the transport + * based on local and remote SDP. + * + * Application should call #pjmedia_transport_media_start() instead of + * calling this function directly. + */ + pj_status_t (*media_start) (pjmedia_transport *tp, + pj_pool_t *tmp_pool, + const pjmedia_sdp_session *sdp_local, + const pjmedia_sdp_session *sdp_remote, + unsigned media_index); + + /** + * This function is called by application to stop the transport. + * + * Application should call #pjmedia_transport_media_stop() instead of + * calling this function directly. + */ + pj_status_t (*media_stop) (pjmedia_transport *tp); + + + /** + * This function can be called to destroy this transport. + * + * Application should call #pjmedia_transport_close() instead of + * calling this function directly. + */ + pj_status_t (*destroy)(pjmedia_transport *tp); + + /** + * This function is called by the stream when the transport is about + * to be used by the stream for the first time, and it tells the transport + * about remote RTP address to send the packet and some callbacks to be + * called for incoming packets. + * + * Application should call #pjmedia_transport_attach2() instead of + * calling this function directly. + */ + pj_status_t (*attach2)(pjmedia_transport *tp, + pjmedia_transport_attach_param *att_param); +}; +``` + +这里主要看attach2，这个函数传入rtp和rtcp的回调函数指针，当从网络收到数据时，会通过该回调通知。 + +### **创建udp媒体传输** + +在simpleua.c初始化时，创建完媒体端点pjmedia_endpt后，还会预先创建好udp媒体传输 + +```cpp + /* + * Create media transport used to send/receive RTP/RTCP socket. + * One media transport is needed for each call. Application may + * opt to re-use the same media transport for subsequent calls. + */ + for (i = 0; i < PJ_ARRAY_SIZE(g_med_transport); ++i) { + status = pjmedia_transport_udp_create3(g_med_endpt, AF, NULL, NULL, + RTP_PORT + i*2, 0, + &g_med_transport[i]); +``` + +pjmedia_transport_udp_create最终调用pjmedia_transport_udp_create3，这个函数先创建rtp和rtcp两个socket，然后调用pjmedia_transport_udp_attach。 + +#### 调用流： + +pjmedia_transport_udp_create3、pjmedia_transport_udp_attach、pj_ioqueue_register_sock2 + +### pjmedia_transport_udp_create3 + +```cpp +/** + * Create UDP stream transport. + */ +PJ_DEF(pj_status_t) pjmedia_transport_udp_create3(pjmedia_endpt *endpt, + int af, + const char *name, + const pj_str_t *addr, + int port, + unsigned options, + pjmedia_transport **p_tp) +{ + pjmedia_sock_info si; + pj_status_t status; + + + /* Sanity check */ + PJ_ASSERT_RETURN(endpt && port && p_tp, PJ_EINVAL); + + + pj_bzero(&si, sizeof(pjmedia_sock_info)); + si.rtp_sock = si.rtcp_sock = PJ_INVALID_SOCKET; + + /* Create RTP socket */ + status = pj_sock_socket(af, pj_SOCK_DGRAM(), 0, &si.rtp_sock); + if (status != PJ_SUCCESS) + goto on_error; + + /* Bind RTP socket */ + status = pj_sockaddr_init(af, &si.rtp_addr_name, addr, (pj_uint16_t)port); + if (status != PJ_SUCCESS) + goto on_error; + + status = pj_sock_bind(si.rtp_sock, &si.rtp_addr_name, + pj_sockaddr_get_len(&si.rtp_addr_name)); + if (status != PJ_SUCCESS) + goto on_error; + + + /* Create RTCP socket */ + status = pj_sock_socket(af, pj_SOCK_DGRAM(), 0, &si.rtcp_sock); + if (status != PJ_SUCCESS) + goto on_error; + + /* Bind RTCP socket */ + status = pj_sockaddr_init(af, &si.rtcp_addr_name, addr, + (pj_uint16_t)(port+1)); + if (status != PJ_SUCCESS) + goto on_error; + + status = pj_sock_bind(si.rtcp_sock, &si.rtcp_addr_name, + pj_sockaddr_get_len(&si.rtcp_addr_name)); + if (status != PJ_SUCCESS) + goto on_error; + + + /* Create UDP transport by attaching socket info */ + return pjmedia_transport_udp_attach( endpt, name, &si, options, p_tp); + + +on_error: + if (si.rtp_sock != PJ_INVALID_SOCKET) + pj_sock_close(si.rtp_sock); + if (si.rtcp_sock != PJ_INVALID_SOCKET) + pj_sock_close(si.rtcp_sock); + return status; +} +``` + +Create & Bind RTP & RTCP socket、Create UDP transport by attaching socket info + +### transport_udp + +```c +struct transport_udp +{ + pjmedia_transport base; /**< Base transport. */ + + pj_pool_t *pool; /**< Memory pool */ + unsigned options; /**< Transport options. */ + unsigned media_options; /**< Transport media options. */ + void *user_data; /**< Only valid when attached */ + //pj_bool_t attached; /**< Has attachment? */ + pj_bool_t started; /**< Has started? */ + pj_sockaddr rem_rtp_addr; /**< Remote RTP address */ + pj_sockaddr rem_rtcp_addr; /**< Remote RTCP address */ + int addr_len; /**< Length of addresses. */ + void (*rtp_cb)( void*, /**< To report incoming RTP. */ + void*, + pj_ssize_t); + void (*rtp_cb2)(pjmedia_tp_cb_param*); /**< To report incoming RTP. */ + void (*rtcp_cb)( void*, /**< To report incoming RTCP. */ + void*, + pj_ssize_t); + + unsigned tx_drop_pct; /**< Percent of tx pkts to drop. */ + unsigned rx_drop_pct; /**< Percent of rx pkts to drop. */ + pj_ioqueue_t *ioqueue; /**< Ioqueue instance. */ + + pj_sock_t rtp_sock; /**< RTP socket */ + pj_sockaddr rtp_addr_name; /**< Published RTP address. */ + pj_ioqueue_key_t *rtp_key; /**< RTP socket key in ioqueue */ + pj_ioqueue_op_key_t rtp_read_op; /**< Pending read operation */ + unsigned rtp_write_op_id;/**< Next write_op to use */ + pending_write rtp_pending_write[MAX_PENDING]; /**< Pending write */ + pj_sockaddr rtp_src_addr; /**< Actual packet src addr. */ + int rtp_addrlen; /**< Address length. */ + char rtp_pkt[RTP_LEN];/**< Incoming RTP packet buffer */ + + pj_bool_t enable_rtcp_mux;/**< Enable RTP & RTCP multiplexing?*/ + pj_bool_t use_rtcp_mux; /**< Use RTP & RTCP multiplexing? */ + pj_sock_t rtcp_sock; /**< RTCP socket */ + pj_sockaddr rtcp_addr_name; /**< Published RTCP address. */ + pj_sockaddr rtcp_src_addr; /**< Actual source RTCP address. */ + unsigned rtcp_src_cnt; /**< How many pkt from this addr. */ + int rtcp_addr_len; /**< Length of RTCP src address. */ + pj_ioqueue_key_t *rtcp_key; /**< RTCP socket key in ioqueue */ + pj_ioqueue_op_key_t rtcp_read_op; /**< Pending read operation */ + pj_ioqueue_op_key_t rtcp_write_op; /**< Pending write operation */ + char rtcp_pkt[RTCP_LEN];/**< Incoming RTCP packet buffer */ +}; + +``` + +### pjmedia_transport_udp_attach + +[pjmedia_transport_udp_attach](html/pjmedia_transport_udp_attach.html) + +```c +/** + * Create UDP stream transport from existing socket info. + */ +PJ_DEF(pj_status_t) pjmedia_transport_udp_attach( pjmedia_endpt *endpt, + const char *name, + const pjmedia_sock_info *si, + unsigned options, + pjmedia_transport **p_tp) +{ + struct transport_udp *tp; + pj_pool_t *pool; + pj_ioqueue_t *ioqueue; + pj_ioqueue_callback rtp_cb, rtcp_cb; + pj_grp_lock_t *grp_lock; + pj_status_t status; + + + /* Sanity check */ + PJ_ASSERT_RETURN(endpt && si && p_tp, PJ_EINVAL); + + /* Get ioqueue instance */ + ioqueue = pjmedia_endpt_get_ioqueue(endpt); + + if (name==NULL) + name = "udp%p"; + + /* Create transport structure */ + pool = pjmedia_endpt_create_pool(endpt, name, 512, 512); + if (!pool) + return PJ_ENOMEM; + + tp = PJ_POOL_ZALLOC_T(pool, struct transport_udp); + tp->pool = pool; + tp->options = options; + pj_memcpy(tp->base.name, pool->obj_name, PJ_MAX_OBJ_NAME); + tp->base.op = &transport_udp_op; + tp->base.type = PJMEDIA_TRANSPORT_TYPE_UDP; + + /* Copy socket infos */ + tp->rtp_sock = si->rtp_sock; + tp->rtp_addr_name = si->rtp_addr_name; + tp->rtcp_sock = si->rtcp_sock; + tp->rtcp_addr_name = si->rtcp_addr_name; + + /* If address is 0.0.0.0, use host's IP address */ + if (!pj_sockaddr_has_addr(&tp->rtp_addr_name)) { + pj_sockaddr hostip; + + status = pj_gethostip(tp->rtp_addr_name.addr.sa_family, &hostip); + if (status != PJ_SUCCESS) + goto on_error; + + pj_memcpy(pj_sockaddr_get_addr(&tp->rtp_addr_name), + pj_sockaddr_get_addr(&hostip), + pj_sockaddr_get_addr_len(&hostip)); + } + + /* Same with RTCP */ + if (!pj_sockaddr_has_addr(&tp->rtcp_addr_name)) { + pj_memcpy(pj_sockaddr_get_addr(&tp->rtcp_addr_name), + pj_sockaddr_get_addr(&tp->rtp_addr_name), + pj_sockaddr_get_addr_len(&tp->rtp_addr_name)); + } + + /* Create group lock */ + status = pj_grp_lock_create(pool, NULL, &grp_lock); + if (status != PJ_SUCCESS) + goto on_error; + + pj_grp_lock_add_ref(grp_lock); + tp->base.grp_lock = grp_lock; + + /* Setup RTP socket with the ioqueue */ + pj_bzero(&rtp_cb, sizeof(rtp_cb)); + rtp_cb.on_read_complete = &on_rx_rtp; + rtp_cb.on_write_complete = &on_rtp_data_sent; + + status = pj_ioqueue_register_sock2(pool, ioqueue, tp->rtp_sock, grp_lock, + tp, &rtp_cb, &tp->rtp_key); + if (status != PJ_SUCCESS) + goto on_error; + + /* Disallow concurrency so that detach() and destroy() are + * synchronized with the callback. + * + * Note that we still need this even after group lock is added to + * maintain the above behavior. + */ + status = pj_ioqueue_set_concurrency(tp->rtp_key, PJ_FALSE); + if (status != PJ_SUCCESS) + goto on_error; + + /* Setup RTCP socket with ioqueue */ + pj_bzero(&rtcp_cb, sizeof(rtcp_cb)); + rtcp_cb.on_read_complete = &on_rx_rtcp; + + status = pj_ioqueue_register_sock2(pool, ioqueue, tp->rtcp_sock, grp_lock, + tp, &rtcp_cb, &tp->rtcp_key); + if (status != PJ_SUCCESS) + goto on_error; + + status = pj_ioqueue_set_concurrency(tp->rtcp_key, PJ_FALSE); + if (status != PJ_SUCCESS) + goto on_error; + + tp->ioqueue = ioqueue; + + /* Done */ + *p_tp = &tp->base; + return PJ_SUCCESS; + + +on_error: + transport_destroy(&tp->base); + return status; +} + + +``` + +创建transport_udp PJ_POOL_ZALLOC_T，Copy socket infos 到transport_udp、 Setup RTP/RTCP socket with the ioqueue（设置两个回调函数on_rx_rtp、on_rtp_data_sent、on_rx_rtcp）。 + +udp_attach先申请UDP媒体传输结构体transport_udp *tp的内存，注意，此结构体包含了媒体传输pjmedia_transport和一些回调，但是这些回调还没有设置。其中的操作集指向transport_udp_op。接着把socket注册到媒体端点中的io队列，io队列的读完成回调是on_rx_rtp。从这里可以知道，从网络读到数据时，会调用transport_udp.c中的on_rx_rtp，而在这个回调里，会再调用transport_udp中的回调rtp_cb和rtp_cb2，而这两个回调，创建的时候还没有设置，要等到调用操作集的attach才会设置。 + +注意，这里有两个attach的地方，一个是创建的时候，调用pjmedia_transport_udp_attach，这个attach会把socket注册到ioqueue，同时ioqueue的读完成回调为transport_udp.c中的on_rx_rtp。 + +### pj_ioqueue_register_sock2 + +注册一个套接字到I/O队列框架。当一个套接字注册到IO队列时，它可以被修改为使用非阻塞IO。如果被修改了，就不能保证在套接字取消注册后会恢复这种修改。 + +- **pool** – To allocate the resource for the specified handle, which must be valid until the handle/key is unregistered from I/O Queue. +- **ioque** – The I/O Queue. +- **sock** – The socket. +- **user_data** – User data to be associated with the key, which can be retrieved later. +- **cb** – Callback to be called when I/O opertion completes. +- **key** – Pointer to receive the key to be associated with this socket. Subsequent I/O queue operation will need this key. + +第二个attach是操作集的attach，看看上面提到的操作集 + +```cpp +static pjmedia_transport_op transport_udp_op = +{ + &transport_get_info, + &transport_attach, + &transport_detach, + &transport_send_rtp, + &transport_send_rtcp, + &transport_send_rtcp2, + &transport_media_create, + &transport_encode_sdp, + &transport_media_start, + &transport_media_stop, + &transport_simulate_lost, + &transport_destroy, + &transport_attach2 +}; +``` + +第二个attach是transport_attach/transport_attach2，这个attach会再传入rtp_cb和rtcp_cb，而这两个回调，会被on_rx_rtp调用，所以这里有两个回调。总结数据流方向，从网络收到数据，最后会进入attach传入的rtp_cb。但是这个rtp_cb什么时候设置，设置的是谁，这个是在 stream 中实现，下一篇再说。 + diff --git a/src/pjmedia_format.md b/src/pjmedia_format.md new file mode 100644 index 0000000..14efac6 --- /dev/null +++ b/src/pjmedia_format.md @@ -0,0 +1,77 @@ +## pjmedia_format + +This structure contains all the information needed to completely describe a media. + +This structure is put in \a *detail* field of #pjmedia_format to describe detail information about an audio media. + +```c +/** + * This structure contains all the information needed to completely describe + * a media. + */ +typedef struct pjmedia_format +{ + /** + * The format id that specifies the audio sample or video pixel format. + * Some well known formats ids are declared in pjmedia_format_id + * enumeration. + * + * @see pjmedia_format_id + */ + pj_uint32_t id; + + /** + * The top-most type of the media, as an information. + */ + pjmedia_type type; + + /** + * The type of detail structure in the \a detail pointer. + */ + pjmedia_format_detail_type detail_type; + + /** + * Detail section to describe the media. + */ + union + { + /** + * Detail section for audio format. + */ + pjmedia_audio_format_detail aud; + + /** + * Detail section for video format. + */ + pjmedia_video_format_detail vid; + + /** + * Reserved area for user-defined format detail. + */ + char user[PJMEDIA_FORMAT_DETAIL_USER_SIZE]; + } det; + +} pjmedia_format; +``` + + + +## pjmedia_audio_format_detail + +```c +/** + * This structure is put in \a detail field of #pjmedia_format to describe + * detail information about an audio media. + */ +typedef struct pjmedia_audio_format_detail +{ + unsigned clock_rate; /**< Audio clock rate in samples or Hz. */ + unsigned channel_count; /**< Number of channels. */ + unsigned frame_time_usec;/**< Frame interval, in microseconds. */ + unsigned bits_per_sample;/**< Number of bits per sample. */ + pj_uint32_t avg_bps; /**< Average bitrate */ + pj_uint32_t max_bps; /**< Maximum bitrate */ +} pjmedia_audio_format_detail; + +``` + diff --git a/src/pjmedia_port.md b/src/pjmedia_port.md new file mode 100644 index 0000000..0972db4 --- /dev/null +++ b/src/pjmedia_port.md @@ -0,0 +1,82 @@ +## pjmedia_port + +媒体端口（用pjmedia_port“类”表示）提供了一个通用和可扩展的框架，用于实现媒体元素。媒体元素本身可以是媒体源、接收端或处理元素。媒体端口接口基本上具有以下属性： + +- 媒体端口信息（pjmedia_port_info）用于描述媒体端口的属性（采样率、通道数等）， +- 可选的指向从端口获取帧的函数指针（get_frame()接口），将被pjmedia_port_get_frame()公共API调用， +- 以及可选的指向将帧存储到端口的函数指针（put_frame()接口），将被pjmedia_port_put_frame()公共API调用。 + + + +get_frame()和put_frame()接口当然只需要在媒体端口发出和/或接收媒体帧时才需要实现。 + +媒体端口是被动的“对象”。默认情况下，没有工作线程来运行媒体流。应用程序（或其他PJMEDIA组件，如时钟/定时中所述）必须积极地从媒体端口调用pjmedia_port_get_frame()或pjmedia_port_put_frame()来检索/存储媒体帧。 + +一些媒体端口（如会议桥和重采样端口）可能与其他端口相互连接（或封装），以执行端口的组合任务，而另一些代表媒体的最终源/汇终止。互连意味着上游媒体端口将调用其下游媒体端口的get_frame()和put_frame()。为了实现这一点，媒体端口需要具有相同的格式，其中格式被定义为音频媒体的采样格式、时钟速率、通道数、每样本位数和每帧样本数的组合。 + +```c + +/** + * Port interface. + */ +typedef struct pjmedia_port +{ + pjmedia_port_info info; /**< Port information. */ + + /** Port data can be used by the port creator to attach arbitrary + * value to be associated with the port. + */ + struct port_data { + void *pdata; /**< Pointer data. */ + long ldata; /**< Long data. */ + } port_data; + + /** + * Group lock. + * + * This is optional, but if this port is registered to the audio/video + * conference bridge, the bridge will create one if the port has none. + */ + pj_grp_lock_t *grp_lock; + + /** + * Get clock source. + * This should only be called by #pjmedia_port_get_clock_src(). + */ + pjmedia_clock_src* (*get_clock_src)(struct pjmedia_port *this_port, + pjmedia_dir dir); + + /** + * Sink interface. + * This should only be called by #pjmedia_port_put_frame(). + */ + pj_status_t (*put_frame)(struct pjmedia_port *this_port, + pjmedia_frame *frame); + + /** + * Source interface. + * This should only be called by #pjmedia_port_get_frame(). + */ + pj_status_t (*get_frame)(struct pjmedia_port *this_port, + pjmedia_frame *frame); + + /** + * Called to destroy this port. + */ + pj_status_t (*on_destroy)(struct pjmedia_port *this_port); + +} pjmedia_port; +``` + +## pjmedia_port_info + +```c +typedef struct pjmedia_port_info +{ + pj_str_t name; /**< Port name. */ + pj_uint32_t signature; /**< Port signature. */ + pjmedia_dir dir; /**< Port direction. */ + pjmedia_format fmt; /**< Format. */ +} pjmedia_port_info; +``` + diff --git a/src/pjmedia_transport.md b/src/pjmedia_transport.md new file mode 100644 index 0000000..10bab4a --- /dev/null +++ b/src/pjmedia_transport.md @@ -0,0 +1,587 @@ +## Transport + +媒体传输封装了网络收发细节，pjmedia_transport可以是udp、srtp、ice等，这里以udp为例。 + +### 相关结构体 + +#### **结构体pjmedia_transport** + +```cpp +/** + * This structure declares media transport. A media transport is called + * by the stream to transmit a packet, and will notify stream when + * incoming packet is arrived. + */ +struct pjmedia_transport +{ + /** Transport name (for logging purpose). */ + char name[PJ_MAX_OBJ_NAME]; + + /** Transport type. */ + pjmedia_transport_type type; + + /** Transport's "virtual" function table. */ + pjmedia_transport_op *op; + + /** Application/user data */ + void *user_data; +}; +``` + +type：传输类型，上面讲过，这里以udp为例。 + +op：操作集，每种传输类型实现了同一组接口。 + +user_data：应用层用户数据 + +#### pjmedia_transport_op + +操作集是核心，这里列举重要的一些函数。[pjmedia_transport_op](https://docs.pjsip.org/en/latest/api/generated/pjmedia/group/group__PJMEDIA__TRANSPORT.html#_CPPv420pjmedia_transport_op) + +```cpp +/** + * This structure describes the operations for the stream transport. + */ +struct pjmedia_transport_op +{ + + /** + * This function is called by the stream when the transport is about + * to be used by the stream for the first time, and it tells the transport + * about remote RTP address to send the packet and some callbacks to be + * called for incoming packets. This function exists for backwards + * compatibility. Transports should implement attach2 instead. + * + * Application should call #pjmedia_transport_attach() instead of + * calling this function directly. + */ + pj_status_t (*attach)(pjmedia_transport *tp, + void *user_data, + const pj_sockaddr_t *rem_addr, + const pj_sockaddr_t *rem_rtcp, + unsigned addr_len, + void (*rtp_cb)(void *user_data, + void *pkt, + pj_ssize_t size), + void (*rtcp_cb)(void *user_data, + void *pkt, + pj_ssize_t size)); + + /** + * This function is called by the stream to send RTP packet using the + * transport. + * + * Application should call #pjmedia_transport_send_rtp() instead of + * calling this function directly. + */ + pj_status_t (*send_rtp)(pjmedia_transport *tp, + const void *pkt, + pj_size_t size); + + /** + * Prepare the transport for a new media session. + * + * Application should call #pjmedia_transport_media_create() instead of + * calling this function directly. + */ + pj_status_t (*media_create)(pjmedia_transport *tp, + pj_pool_t *sdp_pool, + unsigned options, + const pjmedia_sdp_session *remote_sdp, + unsigned media_index); + + /** + * This function is called by application to start the transport + * based on local and remote SDP. + * + * Application should call #pjmedia_transport_media_start() instead of + * calling this function directly. + */ + pj_status_t (*media_start) (pjmedia_transport *tp, + pj_pool_t *tmp_pool, + const pjmedia_sdp_session *sdp_local, + const pjmedia_sdp_session *sdp_remote, + unsigned media_index); + + /** + * This function is called by application to stop the transport. + * + * Application should call #pjmedia_transport_media_stop() instead of + * calling this function directly. + */ + pj_status_t (*media_stop) (pjmedia_transport *tp); + + + /** + * This function can be called to destroy this transport. + * + * Application should call #pjmedia_transport_close() instead of + * calling this function directly. + */ + pj_status_t (*destroy)(pjmedia_transport *tp); + + /** + * This function is called by the stream when the transport is about + * to be used by the stream for the first time, and it tells the transport + * about remote RTP address to send the packet and some callbacks to be + * called for incoming packets. + * + * Application should call #pjmedia_transport_attach2() instead of + * calling this function directly. + */ + pj_status_t (*attach2)(pjmedia_transport *tp, + pjmedia_transport_attach_param *att_param); +}; +``` + +op的默认初始化方法 + +```c +static pjmedia_transport_op transport_udp_op = +{ + &transport_get_info, + &transport_attach, + &transport_detach, + &transport_send_rtp, + &transport_send_rtcp, + &transport_send_rtcp2, + &transport_media_create, + &transport_encode_sdp, + &transport_media_start, + &transport_media_stop, + &transport_simulate_lost, + &transport_destroy, + &transport_attach2 +}; +``` + +这里主要看attach2，这个函数传入rtp和rtcp的回调函数指针，当从网络收到数据时，会通过该回调通知。 + +#### pjmedia_transport_attach_param + +```c +/** + * This structure describes the data passed when calling + * #pjmedia_transport_attach2(). + */ +struct pjmedia_transport_attach_param +{ + /** + * The media stream. + */ + void *stream; + + /** + * Indicate the stream type, either it's audio (PJMEDIA_TYPE_AUDIO) + * or video (PJMEDIA_TYPE_VIDEO). + */ + pjmedia_type media_type; + + /** + * Remote RTP address to send RTP packet to. + */ + pj_sockaddr rem_addr; + + /** + * Optional remote RTCP address. If the argument is NULL + * or if the address is zero, the RTCP address will be + * calculated from the RTP address (which is RTP port plus one). + */ + pj_sockaddr rem_rtcp; + + /** + * Length of the remote address. + */ + unsigned addr_len; + + /** + * Arbitrary user data to be set when the callbacks are called. + */ + void *user_data; + + /** + * Callback to be called when RTP packet is received on the transport. + */ + void (*rtp_cb)(void *user_data, void *pkt, pj_ssize_t); + + /** + * Callback to be called when RTCP packet is received on the transport. + */ + void (*rtcp_cb)(void *user_data, void *pkt, pj_ssize_t); + + /** + * Callback to be called when RTP packet is received on the transport. + */ + void (*rtp_cb2)(pjmedia_tp_cb_param *param); + +}; + +``` + +#### transport_udp + +```c +struct transport_udp +{ + pjmedia_transport base; /**< Base transport. */ + + pj_pool_t *pool; /**< Memory pool */ + unsigned options; /**< Transport options. */ + unsigned media_options; /**< Transport media options. */ + void *user_data; /**< Only valid when attached */ + //pj_bool_t attached; /**< Has attachment? */ + pj_bool_t started; /**< Has started? */ + pj_sockaddr rem_rtp_addr; /**< Remote RTP address */ + pj_sockaddr rem_rtcp_addr; /**< Remote RTCP address */ + int addr_len; /**< Length of addresses. */ + void (*rtp_cb)( void*, /**< To report incoming RTP. */ + void*, + pj_ssize_t); + void (*rtp_cb2)(pjmedia_tp_cb_param*); /**< To report incoming RTP. */ + void (*rtcp_cb)( void*, /**< To report incoming RTCP. */ + void*, + pj_ssize_t); + + unsigned tx_drop_pct; /**< Percent of tx pkts to drop. */ + unsigned rx_drop_pct; /**< Percent of rx pkts to drop. */ + pj_ioqueue_t *ioqueue; /**< Ioqueue instance. */ + + pj_sock_t rtp_sock; /**< RTP socket */ + pj_sockaddr rtp_addr_name; /**< Published RTP address. */ + pj_ioqueue_key_t *rtp_key; /**< RTP socket key in ioqueue */ + pj_ioqueue_op_key_t rtp_read_op; /**< Pending read operation */ + unsigned rtp_write_op_id;/**< Next write_op to use */ + pending_write rtp_pending_write[MAX_PENDING]; /**< Pending write */ + pj_sockaddr rtp_src_addr; /**< Actual packet src addr. */ + int rtp_addrlen; /**< Address length. */ + char rtp_pkt[RTP_LEN];/**< Incoming RTP packet buffer */ + + pj_bool_t enable_rtcp_mux;/**< Enable RTP & RTCP multiplexing?*/ + pj_bool_t use_rtcp_mux; /**< Use RTP & RTCP multiplexing? */ + pj_sock_t rtcp_sock; /**< RTCP socket */ + pj_sockaddr rtcp_addr_name; /**< Published RTCP address. */ + pj_sockaddr rtcp_src_addr; /**< Actual source RTCP address. */ + unsigned rtcp_src_cnt; /**< How many pkt from this addr. */ + int rtcp_addr_len; /**< Length of RTCP src address. */ + pj_ioqueue_key_t *rtcp_key; /**< RTCP socket key in ioqueue */ + pj_ioqueue_op_key_t rtcp_read_op; /**< Pending read operation */ + pj_ioqueue_op_key_t rtcp_write_op; /**< Pending write operation */ + char rtcp_pkt[RTCP_LEN];/**< Incoming RTCP packet buffer */ +}; + +``` + +#### pj_ioqueue_callback + +```c +/** + * This structure describes the callbacks to be called when I/O operation + * completes. + */ +typedef struct pj_ioqueue_callback +{ + /** + * This callback is called when #pj_ioqueue_recv or #pj_ioqueue_recvfrom + * completes. + * + * @param key The key. + * @param op_key Operation key. + * @param bytes_read >= 0 to indicate the amount of data read, + * otherwise negative value containing the error + * code. To obtain the pj_status_t error code, use + * (pj_status_t code = -bytes_read). + */ + void (*on_read_complete)(pj_ioqueue_key_t *key, + pj_ioqueue_op_key_t *op_key, + pj_ssize_t bytes_read); + + /** + * This callback is called when #pj_ioqueue_send or #pj_ioqueue_sendto + * completes. + * + * @param key The key. + * @param op_key Operation key. + * @param bytes_sent >= 0 to indicate the amount of data written, + * otherwise negative value containing the error + * code. To obtain the pj_status_t error code, use + * (pj_status_t code = -bytes_sent). + */ + void (*on_write_complete)(pj_ioqueue_key_t *key, + pj_ioqueue_op_key_t *op_key, + pj_ssize_t bytes_sent); + + /** + * This callback is called when #pj_ioqueue_accept completes. + * + * @param key The key. + * @param op_key Operation key. + * @param sock Newly connected socket. + * @param status Zero if the operation completes successfully. + */ + void (*on_accept_complete)(pj_ioqueue_key_t *key, + pj_ioqueue_op_key_t *op_key, + pj_sock_t sock, + pj_status_t status); + + /** + * This callback is called when #pj_ioqueue_connect completes. + * + * @param key The key. + * @param status PJ_SUCCESS if the operation completes successfully. + */ + void (*on_connect_complete)(pj_ioqueue_key_t *key, + pj_status_t status); +} pj_ioqueue_callback; +``` + + + +### **创建udp media transport** + +在simpleua.c初始化时，创建完媒体端点pjmedia_endpt后，还会预先创建好udp媒体传输在main.c在sdp和invite session之前 + +```cpp + /* + * Create media transport used to send/receive RTP/RTCP socket. + * One media transport is needed for each call. Application may + * opt to re-use the same media transport for subsequent calls. + */ + for (i = 0; i < PJ_ARRAY_SIZE(g_med_transport); ++i) { + status = pjmedia_transport_udp_create3(g_med_endpt, AF, NULL, NULL, + RTP_PORT + i*2, 0, + &g_med_transport[i]); +``` + +pjmedia_transport_udp_create最终调用pjmedia_transport_udp_create3，这个函数先创建rtp和rtcp两个socket，然后调用pjmedia_transport_udp_attach。 + +#### transport_udp_create调用流 + +pjmedia_transport_udp_create3 (Create & Bind RTP & RTCP socket)->pjmedia_transport_udp_attach(创建transport_udp,初始化socket infos，pj_ioqueue_register_sock2：注册socket到 I/O queue同时设置回调函数，以便异步接受消息) + +##### pjmedia_transport_udp_create3 + +```cpp +/** + * Create UDP stream transport. + */ +PJ_DEF(pj_status_t) pjmedia_transport_udp_create3(pjmedia_endpt *endpt, + int af, + const char *name, + const pj_str_t *addr, + int port, + unsigned options, + pjmedia_transport **p_tp) +{ + pjmedia_sock_info si; + pj_status_t status; + + + /* Sanity check */ + PJ_ASSERT_RETURN(endpt && port && p_tp, PJ_EINVAL); + + + pj_bzero(&si, sizeof(pjmedia_sock_info)); + si.rtp_sock = si.rtcp_sock = PJ_INVALID_SOCKET; + + /* Create RTP socket */ + status = pj_sock_socket(af, pj_SOCK_DGRAM(), 0, &si.rtp_sock); + if (status != PJ_SUCCESS) + goto on_error; + + /* Bind RTP socket */ + status = pj_sockaddr_init(af, &si.rtp_addr_name, addr, (pj_uint16_t)port); + if (status != PJ_SUCCESS) + goto on_error; + + status = pj_sock_bind(si.rtp_sock, &si.rtp_addr_name, + pj_sockaddr_get_len(&si.rtp_addr_name)); + if (status != PJ_SUCCESS) + goto on_error; + + + /* Create RTCP socket */ + status = pj_sock_socket(af, pj_SOCK_DGRAM(), 0, &si.rtcp_sock); + if (status != PJ_SUCCESS) + goto on_error; + + /* Bind RTCP socket */ + status = pj_sockaddr_init(af, &si.rtcp_addr_name, addr, + (pj_uint16_t)(port+1)); + if (status != PJ_SUCCESS) + goto on_error; + + status = pj_sock_bind(si.rtcp_sock, &si.rtcp_addr_name, + pj_sockaddr_get_len(&si.rtcp_addr_name)); + if (status != PJ_SUCCESS) + goto on_error; + + + /* Create UDP transport by attaching socket info */ + return pjmedia_transport_udp_attach( endpt, name, &si, options, p_tp); + + +on_error: + if (si.rtp_sock != PJ_INVALID_SOCKET) + pj_sock_close(si.rtp_sock); + if (si.rtcp_sock != PJ_INVALID_SOCKET) + pj_sock_close(si.rtcp_sock); + return status; +} +``` + +Create & Bind RTP & RTCP socket、Create UDP transport by attaching socket info + +##### pjmedia_transport_udp_attach + +[pjmedia_transport_udp_attach](html/pjmedia_transport_udp_attach.html) + +```c +/** + * Create UDP stream transport from existing socket info. + */ +PJ_DEF(pj_status_t) pjmedia_transport_udp_attach( pjmedia_endpt *endpt, + const char *name, + const pjmedia_sock_info *si, + unsigned options, + pjmedia_transport **p_tp) +{ + struct transport_udp *tp; + pj_pool_t *pool; + pj_ioqueue_t *ioqueue; + pj_ioqueue_callback rtp_cb, rtcp_cb; + pj_grp_lock_t *grp_lock; + pj_status_t status; + + + /* Sanity check */ + PJ_ASSERT_RETURN(endpt && si && p_tp, PJ_EINVAL); + + /* Get ioqueue instance */ + ioqueue = pjmedia_endpt_get_ioqueue(endpt); + + if (name==NULL) + name = "udp%p"; + + /* Create transport structure */ + pool = pjmedia_endpt_create_pool(endpt, name, 512, 512); + if (!pool) + return PJ_ENOMEM; + + tp = PJ_POOL_ZALLOC_T(pool, struct transport_udp); + tp->pool = pool; + tp->options = options; + pj_memcpy(tp->base.name, pool->obj_name, PJ_MAX_OBJ_NAME); + tp->base.op = &transport_udp_op; + tp->base.type = PJMEDIA_TRANSPORT_TYPE_UDP; + + /* Copy socket infos */ + tp->rtp_sock = si->rtp_sock; + tp->rtp_addr_name = si->rtp_addr_name; + tp->rtcp_sock = si->rtcp_sock; + tp->rtcp_addr_name = si->rtcp_addr_name; + + /* If address is 0.0.0.0, use host's IP address */ + if (!pj_sockaddr_has_addr(&tp->rtp_addr_name)) { + pj_sockaddr hostip; + + status = pj_gethostip(tp->rtp_addr_name.addr.sa_family, &hostip); + if (status != PJ_SUCCESS) + goto on_error; + + pj_memcpy(pj_sockaddr_get_addr(&tp->rtp_addr_name), + pj_sockaddr_get_addr(&hostip), + pj_sockaddr_get_addr_len(&hostip)); + } + + /* Same with RTCP */ + if (!pj_sockaddr_has_addr(&tp->rtcp_addr_name)) { + pj_memcpy(pj_sockaddr_get_addr(&tp->rtcp_addr_name), + pj_sockaddr_get_addr(&tp->rtp_addr_name), + pj_sockaddr_get_addr_len(&tp->rtp_addr_name)); + } + + /* Create group lock */ + status = pj_grp_lock_create(pool, NULL, &grp_lock); + if (status != PJ_SUCCESS) + goto on_error; + + pj_grp_lock_add_ref(grp_lock); + tp->base.grp_lock = grp_lock; + + /* Setup RTP socket with the ioqueue */ + pj_bzero(&rtp_cb, sizeof(rtp_cb)); + rtp_cb.on_read_complete = &on_rx_rtp; + rtp_cb.on_write_complete = &on_rtp_data_sent; + + status = pj_ioqueue_register_sock2(pool, ioqueue, tp->rtp_sock, grp_lock, + tp, &rtp_cb, &tp->rtp_key); + if (status != PJ_SUCCESS) + goto on_error; + + /* Disallow concurrency so that detach() and destroy() are + * synchronized with the callback. + * + * Note that we still need this even after group lock is added to + * maintain the above behavior. + */ + status = pj_ioqueue_set_concurrency(tp->rtp_key, PJ_FALSE); + if (status != PJ_SUCCESS) + goto on_error; + + /* Setup RTCP socket with ioqueue */ + pj_bzero(&rtcp_cb, sizeof(rtcp_cb)); + rtcp_cb.on_read_complete = &on_rx_rtcp; + + status = pj_ioqueue_register_sock2(pool, ioqueue, tp->rtcp_sock, grp_lock, + tp, &rtcp_cb, &tp->rtcp_key); + if (status != PJ_SUCCESS) + goto on_error; + + status = pj_ioqueue_set_concurrency(tp->rtcp_key, PJ_FALSE); + if (status != PJ_SUCCESS) + goto on_error; + + tp->ioqueue = ioqueue; + + /* Done */ + *p_tp = &tp->base; + return PJ_SUCCESS; + + +on_error: + transport_destroy(&tp->base); + return status; +} + + +``` + +创建transport_udp ：PJ_POOL_ZALLOC_T，Copy socket infos 到transport_udp、 Setup RTP/RTCP socket with the ioqueue（设置回调函数on_rx_rtp、on_rtp_data_sent、on_rx_rtcp）。 + +udp_attach先申请UDP媒体传输结构体transport_udp *tp的内存，注意，此结构体包含了媒体传输pjmedia_transport和一些回调，但是这些回调还没有设置。其中的操作集指向transport_udp_op。接着把socket注册到媒体端点中的io队列，io队列的读完成回调是on_rx_rtp。从这里可以知道，从网络读到数据时，会调用transport_udp.c中的on_rx_rtp，而在这个回调里，会再调用transport_udp中的回调rtp_cb和rtp_cb2，而这两个回调，创建的时候还没有设置，要等到调用操作集的attach才会设置。 + +注意，这里有两个attach的地方，一个是创建的时候，调用pjmedia_transport_udp_attach，这个attach会把socket注册到ioqueue，同时ioqueue的读完成回调为transport_udp.c中的on_rx_rtp。 + +##### pj_ioqueue_register_sock2 + +注册一个套接字到I/O队列框架。当一个套接字注册到IO队列时，它可以被修改为使用非阻塞IO。如果被修改了，就不能保证在套接字取消注册后会恢复这种修改。 + +- **pool** – To allocate the resource for the specified handle, which must be valid until the handle/key is unregistered from I/O Queue. +- **ioque** – The I/O Queue. +- **sock** – The socket. +- **user_data** – User data to be associated with the key, which can be retrieved later. +- **cb** – Callback to be called when I/O opertion completes. +- **key** – Pointer to receive the key to be associated with this socket. Subsequent I/O queue operation will need this key. + +第二个attach是transport_attach/transport_attach2，这个attach会再传入rtp_cb和rtcp_cb，而这两个回调，会被on_rx_rtp调用，所以这里有两个回调。总结数据流方向，从网络收到数据，最后会进入attach传入的rtp_cb。但是这个rtp_cb什么时候设置，设置的是谁，这个是在 stream 中实现，下一篇再说。 + +#### pjmedia_transport_attach2调用流 + +pjmedia_transport_attach2-》transport_attach2-》tp_attach + +tp_attach最重要就是设置了pjmedia_transport 的cb2 为on_rx_request，还有Copy remote RTP address + + + + + diff --git a/src/pjsip_doc.md b/src/pjsip_doc.md index d224049..4cbff23 100644 --- a/src/pjsip_doc.md +++ b/src/pjsip_doc.md @@ -2798,7 +2798,7 @@ on_tsx_state_changed：每当dialog中的任何transaction更改了其状态时 on_rx_offer：当invite session收到来自对等方的新提议时，将调用此回调。应用程序通过调用pjsip_inv_set_sdp_answer（）设置本地应答。此功能不会发送外发消息。它只是保留SDP协商过程的答案，并将包含在后续发送的响应或请求中。此回调是可选的。如果未指定，默认行为是与会话的初始功能协商远程提供。 -on_media_update：此回调在SDP提供/应答会话完成后调用。status参数指定pjmedia_sdp_neg_negotiate（）返回的offer/answer的状态。这个回调是可选的（从框架的角度来看），但是所有有用的应用程序通常都需要实现这个回调。 +on_media_update：此回调在SDP协商完后调用。status参数指定pjmedia_sdp_neg_negotiate（）返回的offer/answer的状态。这个回调是可选的（从框架的角度来看），但是所有有用的应用程序通常都需要实现这个回调。 ### 12.2.4 Session Creation and Termination diff --git "a/src/\345\205\250\346\265\201\347\250\213\346\200\235\350\200\203.md" "b/src/\345\205\250\346\265\201\347\250\213\346\200\235\350\200\203.md" new file mode 100644 index 0000000..f5ce30c --- /dev/null +++ "b/src/\345\205\250\346\265\201\347\250\213\346\200\235\350\200\203.md" @@ -0,0 +1,46 @@ +## 初始化全流程思考 + +一个transport 一个 stream一个port两个channel， + +![img](img/20190917203551691.png) + +一、再开启invite session之前 + +1. 创建media endpoint +2. 创建transport放在g_med_transport数组，注册绑定socket + +二、loop处理sip事件 + +三、SDP协商完后回调 + +1. 创建stream，先获取stream_info（从sdp协商中pjmedia_stream_info_from_sdp） + + - 1、申请媒体流空间 + + 2、初始化流的若干参数 + + 3、codec管理者及codec相关的操作 + + 4、设置第一组回调put_frame和get_frame，这组回调是音频设备port要用的 + + 5、创建jitterbuffer，这个后面会单独讲 + + 6、创建编码通道和解码通道 + + 7、调用上一节中提到的媒体传输attach，传入第2组回调on_rx_rtp和on_rx_rtcp + +2. pjmedia_stream_start，channel开，Start the audio stream + + 编码通道和解码通道pause =0； + +3. pjmedia_transport_media_start，UDP media transport开 + + Start the audio stream，tp->start = 1 + +4. port创建，创建设备，开启设备 Open audio stream object + + + + + +Callback when SDP negotiation has completed. \ No newline at end of file diff --git "a/src/\345\217\221\351\200\201\346\225\260\346\215\256\345\205\250\346\265\201\347\250\213.md" "b/src/\345\217\221\351\200\201\346\225\260\346\215\256\345\205\250\346\265\201\347\250\213.md" new file mode 100644 index 0000000..51e7196 --- /dev/null +++ "b/src/\345\217\221\351\200\201\346\225\260\346\215\256\345\205\250\346\265\201\347\250\213.md" @@ -0,0 +1,717 @@ +# 发送数据全流程 + +## 1、发送的线程（在声音设备端直接获取声卡数据） + +先看发送的线程 ca_thread_func，这里我们以alsa_dev为例 + +```c +static int ca_thread_func (void *arg) +{ + struct alsa_stream* stream = (struct alsa_stream*) arg; + snd_pcm_t* pcm = stream->ca_pcm; + int size = stream->ca_buf_size; + snd_pcm_uframes_t nframes = stream->ca_frames; + void* user_data = stream->user_data; + char* buf = stream->ca_buf; + pj_timestamp tstamp; + int result; + struct sched_param param; + pthread_t* thid; + + thid = (pthread_t*) pj_thread_get_os_handle (pj_thread_this()); + param.sched_priority = sched_get_priority_max (SCHED_RR); + PJ_LOG (5,(THIS_FILE, "ca_thread_func(%u): Set thread priority " + "for audio capture thread.", + (unsigned)syscall(SYS_gettid))); + result = pthread_setschedparam (*thid, SCHED_RR, ¶m); + if (result) { + if (result == EPERM) + PJ_LOG (5,(THIS_FILE, "Unable to increase thread priority, " + "root access needed.")); + else + PJ_LOG (5,(THIS_FILE, "Unable to increase thread priority, " + "error: %d", + result)); + } + + pj_bzero (buf, size); + tstamp.u64 = 0; + + TRACE_((THIS_FILE, "ca_thread_func(%u): Started", + (unsigned)syscall(SYS_gettid))); + + snd_pcm_prepare (pcm); + + while (!stream->quit) { + pjmedia_frame frame; + + pj_bzero (buf, size); + result = snd_pcm_readi (pcm, buf, nframes); + if (result == -EPIPE) { + PJ_LOG (4,(THIS_FILE, "ca_thread_func: overrun!")); + snd_pcm_prepare (pcm); + continue; + } else if (result < 0) { + PJ_LOG (4,(THIS_FILE, "ca_thread_func: error reading data!")); + } + if (stream->quit) + break; + + frame.type = PJMEDIA_FRAME_TYPE_AUDIO; + frame.buf = (void*) buf; + frame.size = size; + frame.timestamp.u64 = tstamp.u64; + frame.bit_info = 0; + + result = stream->ca_cb (user_data, &frame); + if (result != PJ_SUCCESS || stream->quit) + break; + + tstamp.u64 += nframes; + } + snd_pcm_drop(pcm); + TRACE_((THIS_FILE, "ca_thread_func: Stopped")); + + return PJ_SUCCESS; +} +``` + +该线程的主体部分在不停的while循环，我们来看一次循环的内容： + +调用 `snd_pcm_readi (pcm, buf, nframes);` 读网卡的数据到buf中。然后将读出的数据和一系列属性包装成一个frame。然后调用`stream->ca_cb (user_data, &frame);` + +在初始化（pjmedia_aud_stream_create->f->op->create_stream）中我们知道stream->ca_cb 我们设置为 `rec_cb` 接下来我们来看`rec_cb` 在sound_port.c中. + +另外我们需要知道`ca_cb` 中的`user_data` 到底是什么 这可以看我们之前初始化流程的分析：pjmedia_snd_port_create-》pjmedia_snd_port_create2-》start_sound_device-》pjmedia_aud_stream_create-》f->op->create_stream-》alsa_factory_create_stream + + 到pjmedia_aud_stream_create这步我们看出user_data是我们创建的`snd_port` + +## 2、rec_cb （音频流端） + +```c +/* + * The callback called by sound recorder when it has finished capturing a + * frame. + */ +static pj_status_t rec_cb(void *user_data, pjmedia_frame *frame) +{ + pjmedia_snd_port *snd_port = (pjmedia_snd_port*) user_data; + pjmedia_port *port; + + pjmedia_clock_src_update(&snd_port->cap_clocksrc, &frame->timestamp); + + /* Invoke preview callback */ + if (snd_port->on_rec_frame) + (*snd_port->on_rec_frame)(snd_port->user_data, frame); + + port = snd_port->port; + if (port == NULL) + return PJ_SUCCESS; + + /* Cancel echo */ + if (snd_port->ec_state && !snd_port->ec_suspended) { + pjmedia_echo_capture(snd_port->ec_state, (pj_int16_t*) frame->buf, 0); + } + + pjmedia_port_put_frame(port, frame); + + + return PJ_SUCCESS; +} +``` + +首先看`snd_port->on_rec_frame`这个是可选项，这里没有使用 + +其次就是最重要的调用pjmedia_port_put_frame，看一下这里的port参数，`port = snd_port->port;` 是pjmedia_snd_port类型snd_port中的pjmedia_port类型属性，snd_port->port是在`pjmedia_snd_port_connect`中初始化的先看一下pjmedia_snd_port_connect + +```c +/* + * Connect a port. + */ +PJ_DEF(pj_status_t) pjmedia_snd_port_connect( pjmedia_snd_port *snd_port, + pjmedia_port *port) +{ + pjmedia_audio_format_detail *afd; + + PJ_ASSERT_RETURN(snd_port && port, PJ_EINVAL); + + afd = pjmedia_format_get_audio_format_detail(&port->info.fmt, PJ_TRUE); + + /* Check that port has the same configuration as the sound device + * port. + */ + if (afd->clock_rate != snd_port->clock_rate) + return PJMEDIA_ENCCLOCKRATE; + + if (PJMEDIA_AFD_SPF(afd) != snd_port->samples_per_frame) + return PJMEDIA_ENCSAMPLESPFRAME; + + if (afd->channel_count != snd_port->channel_count) + return PJMEDIA_ENCCHANNEL; + + if (afd->bits_per_sample != snd_port->bits_per_sample) + return PJMEDIA_ENCBITS; + + /* Port is okay. */ + snd_port->port = port; + return PJ_SUCCESS; +} +``` + +注意这里的参数`pjmedia_port *port`来自pjmedia_stream stream的port，这个port还有一点特殊即port->port_data.pdata; 其实是指向stream，即stream中有port，port也有办法指向stream，这在put_frame中会遇到。 + +<::>再回来看pjmedia_port_put_frame该函数其实是port->put_frame callback的封装，会直接调用port->put_frame，这个回调函数的初始化在`pjmedia_stream_create` 完成将port->put_frame 初始化为 stream->port.put_frame = &put_frame;，所以接下来我们看put_frame + +```c + +/** + * Put a frame to the port (and subsequent downstream ports). + */ +PJ_DEF(pj_status_t) pjmedia_port_put_frame( pjmedia_port *port, + pjmedia_frame *frame ) +{ + PJ_ASSERT_RETURN(port && frame, PJ_EINVAL); + + if (port->put_frame) + return port->put_frame(port, frame); + else + return PJ_EINVALIDOP; +} +``` + +## 3、put_frame （port端） + +```c +/** + * put_frame() + * + * This callback is called by upstream component when it has PCM frame + * to transmit. This function encodes the PCM frame, pack it into + * RTP packet, and transmit to peer. + */ +static pj_status_t put_frame( pjmedia_port *port, + pjmedia_frame *frame ) +{ + pjmedia_stream *stream = (pjmedia_stream*) port->port_data.pdata; + pjmedia_frame tmp_zero_frame; + unsigned samples_per_frame; + + samples_per_frame = stream->enc_samples_per_pkt; + + /* https://github.com/pjsip/pjproject/issues/56: + * when input is PJMEDIA_FRAME_TYPE_NONE, feed zero PCM frame + * instead so that encoder can decide whether or not to transmit + * silence frame. + */ + if (frame->type == PJMEDIA_FRAME_TYPE_NONE) { + pj_memcpy(&tmp_zero_frame, frame, sizeof(pjmedia_frame)); + frame = &tmp_zero_frame; + + tmp_zero_frame.buf = NULL; + tmp_zero_frame.size = samples_per_frame * 2; + tmp_zero_frame.type = PJMEDIA_FRAME_TYPE_AUDIO; + } + + /* If VAD is temporarily disabled during creation, enable it + * after transmitting for VAD_SUSPEND_SEC seconds. + */ + if (stream->vad_enabled != stream->codec_param.setting.vad && + (stream->tx_duration - stream->ts_vad_disabled) > + PJMEDIA_PIA_SRATE(&stream->port.info) * + PJMEDIA_STREAM_VAD_SUSPEND_MSEC / 1000) + { + stream->codec_param.setting.vad = stream->vad_enabled; + pjmedia_codec_modify(stream->codec, &stream->codec_param); + PJ_LOG(4,(stream->port.info.name.ptr,"VAD re-enabled")); + } + + + /* If encoder has different ptime than decoder, then the frame must + * be passed through the encoding buffer via rebuffer() function. + */ + if (stream->enc_buf != NULL) { + pjmedia_frame tmp_rebuffer_frame; + pj_status_t status = PJ_SUCCESS; + + /* Copy original frame to temporary frame since we need + * to modify it. + */ + pj_memcpy(&tmp_rebuffer_frame, frame, sizeof(pjmedia_frame)); + + /* Loop while we have full frame in enc_buffer */ + for (;;) { + pj_status_t st; + + /* Run rebuffer() */ + rebuffer(stream, &tmp_rebuffer_frame); + + /* Process this frame */ + st = put_frame_imp(port, &tmp_rebuffer_frame); + if (st != PJ_SUCCESS) + status = st; + + /* If we still have full frame in the buffer, re-run + * rebuffer() with NULL frame. + */ + if (stream->enc_buf_count >= stream->enc_samples_per_pkt) { + + tmp_rebuffer_frame.type = PJMEDIA_FRAME_TYPE_NONE; + + } else { + + /* Otherwise break */ + break; + } + } + + return status; + + } else { + return put_frame_imp(port, frame); + } +} +``` + +重点看一下`put_frame_imp` 对frame的处理，函数比较长，我们来分段看一下 + +1. 函数开始时，从 `port` 参数中获取了指向 `pjmedia_stream` 结构的指针 `stream`，并且从 `stream` 中获取了编码通道 `enc`。 + + ```c + pjmedia_stream *stream = (pjmedia_stream*) port->port_data.pdata; + pjmedia_channel *channel = stream->enc; + ``` + +2. 如果流启用了保活机制，函数会检查距离上次发送数据包的时间间隔，如果超过了指定的保活间隔，会发送一个保活数据包。 + +3. 然后，函数会根据帧的类型计算帧中的样本数 `ts_len`，并根据是否存在特定的编码器问题来确定 RTP 时间戳的长度 `rtp_ts_len`。 + + ```c + /* Number of samples in the frame */ + if (frame->type == PJMEDIA_FRAME_TYPE_AUDIO) + ts_len = ((unsigned)frame->size >> 1) / + stream->codec_param.info.channel_cnt; + else if (frame->type == PJMEDIA_FRAME_TYPE_EXTENDED) + ts_len = PJMEDIA_PIA_SPF(&stream->port.info) / + PJMEDIA_PIA_CCNT(&stream->port.info); + else + ts_len = 0; + + #if defined(PJMEDIA_HANDLE_G722_MPEG_BUG) && (PJMEDIA_HANDLE_G722_MPEG_BUG!=0) + /* Handle special case for audio codec with RTP timestamp inconsistence + * e.g: G722, MPEG audio. + */ + if (stream->has_g722_mpeg_bug) + rtp_ts_len = stream->rtp_tx_ts_len_per_pkt; + else + rtp_ts_len = ts_len; + #else + rtp_ts_len = ts_len; + #endif + ``` + +4. 如果编码通道被暂停，函数会更新 RTP 会话的时间戳，并在需要时发送 RTCP SR/RR 报告。 + + ```c + /* Don't do anything if stream is paused, except updating RTP timestamp */ + if (channel->paused) { + stream->enc_buf_pos = stream->enc_buf_count = 0; + + /* Update RTP session's timestamp. */ + status = pjmedia_rtp_encode_rtp( &channel->rtp, 0, 0, 0, rtp_ts_len, + NULL, NULL); + + /* Update RTCP stats with last RTP timestamp. */ + stream->rtcp.stat.rtp_tx_last_ts = pj_ntohl(channel->rtp.out_hdr.ts); + + /* Check if now is the time to transmit RTCP SR/RR report. + * We only do this when the decoder is paused, + * because otherwise check_tx_rtcp() will be handled by on_rx_rtp(). + */ + if (stream->dec->paused) { + check_tx_rtcp(stream, pj_ntohl(channel->rtp.out_hdr.ts)); + } + + return PJ_SUCCESS; + } + ``` + +5. 接着，函数会增加传输时长，初始化输出帧缓冲区，并检查是否有 DTMF 数字在队列中，如果有则发送数字，否则对音频帧进行编码。 + + ```c + if (stream->tx_dtmf_count) { + int first=0, last=0; + + create_dtmf_payload(stream, &frame_out, 0, &first, &last); + + /* Encapsulate into RTP packet. Note that: + * - RTP marker should be set on the beginning of a new event + * - RTP timestamp is constant for the same packet. + */ + status = pjmedia_rtp_encode_rtp( &channel->rtp, + stream->tx_event_pt, first, + (int)frame_out.size, + (first ? rtp_ts_len : 0), + (const void**)&rtphdr, + &rtphdrlen); + + if (last) { + /* This is the last packet for the event. + * Increment the RTP timestamp of the RTP session, for next + * RTP packets. + */ + inc_timestamp = stream->dtmf_duration + + ((DTMF_EBIT_RETRANSMIT_CNT-1) * + stream->rtp_tx_ts_len_per_pkt) + - rtp_ts_len; + } + + } + ``` + + + +6. 如果音频帧的缓冲区为空，则发送一段静音，保持 NAT 绑定。 + + ```c + else if (frame->type == PJMEDIA_FRAME_TYPE_AUDIO && + frame->buf == NULL && + stream->port.info.fmt.id == PJMEDIA_FORMAT_L16 && + (stream->dir & PJMEDIA_DIR_ENCODING)) + { + pjmedia_frame silence_frame; + + pj_bzero(&silence_frame, sizeof(silence_frame)); + silence_frame.buf = stream->zero_frame; + silence_frame.size = stream->enc_samples_per_pkt * 2; + silence_frame.type = PJMEDIA_FRAME_TYPE_AUDIO; + silence_frame.timestamp.u32.lo = pj_ntohl(stream->enc->rtp.out_hdr.ts); + + /* Encode! */ + status = pjmedia_codec_encode( stream->codec, &silence_frame, + channel->out_pkt_size - + sizeof(pjmedia_rtp_hdr), + &frame_out); + if (status != PJ_SUCCESS) { + LOGERR_((stream->port.info.name.ptr, status, + "Codec encode() error")); + return status; + } + + /* Encapsulate. */ + status = pjmedia_rtp_encode_rtp( &channel->rtp, + channel->pt, 0, + (int)frame_out.size, rtp_ts_len, + (const void**)&rtphdr, + &rtphdrlen); + + + } + ``` + + + +7. 如果音频帧不为空，则对音频帧进行编码，并将 RTP 头封装到输出包中。 + + ```c + else if ((frame->type == PJMEDIA_FRAME_TYPE_AUDIO && + frame->buf != NULL) || + (frame->type == PJMEDIA_FRAME_TYPE_EXTENDED)) + { + /* Encode! */ + status = pjmedia_codec_encode( stream->codec, frame, + channel->out_pkt_size - + sizeof(pjmedia_rtp_hdr), + &frame_out); + if (status != PJ_SUCCESS) { + LOGERR_((stream->port.info.name.ptr, status, + "Codec encode() error")); + return status; + } + + /* Encapsulate. */ + status = pjmedia_rtp_encode_rtp( &channel->rtp, + channel->pt, 0, + (int)frame_out.size, rtp_ts_len, + (const void**)&rtphdr, + &rtphdrlen); + + } + ``` + + + +8. 最后，函数会根据当前是否正在流式传输来设置 RTP 标记位，并将 RTP 包发送到传输层。 + + ```c + + /* Copy RTP header to the beginning of packet */ + pj_memcpy(channel->out_pkt, rtphdr, sizeof(pjmedia_rtp_hdr)); + + /* Special case for DTMF: timestamp remains constant for + * the same event, and is only updated after a complete event + * has been transmitted. + */ + if (inc_timestamp) { + pjmedia_rtp_encode_rtp( &channel->rtp, stream->tx_event_pt, 0, + 0, inc_timestamp, NULL, NULL); + } + + /* Set RTP marker bit if currently not streaming */ + if (stream->is_streaming == PJ_FALSE) { + pjmedia_rtp_hdr *rtp = (pjmedia_rtp_hdr*) channel->out_pkt; + + rtp->m = 1; + PJ_LOG(5,(stream->port.info.name.ptr,"Start talksprut..")); + } + + stream->is_streaming = PJ_TRUE; + + /* Send the RTP packet to the transport. */ + status = pjmedia_transport_send_rtp(stream->transport, channel->out_pkt, + frame_out.size + + sizeof(pjmedia_rtp_hdr)); + ``` + +9. 在发送 RTP 包之后，函数会更新一些统计信息，并且如果启用了保活机制，则记录最后发送数据包的时间。 + +我们来仔细梳理一下rtp包发送的流程 + +（1）先设置frame_out.buf 对应的偏置 `((char*)channel->out_pkt) + sizeof(pjmedia_rtp_hdr);` + +（2）调用pjmedia_codec_encode，对frame编码结果输出到frame_out中，此时frame_out.buf获取到rtp payload，相应地(char*)channel->out_pkt) + sizeof(pjmedia_rtp_hdr处获取到rtp payload + +（3）添加rtp头部，调用pjmedia_rtp_encode_rtp，hannel->rtp->out_hdr，并将头部拷贝至(char*)channel->out_pkt)处，至此channel->out_pkt存放地为编码后的rtp包 + +（4）Send the RTP packet to the transport. 调用pjmedia_transport_send_rtp + +我们就主要看pjmedia_transport_send_rtp了，其实就是op->send_rtp 的封装，由初始化可知send_rtp为transport_send_rtcp + +```c +PJ_INLINE(pj_status_t) pjmedia_transport_send_rtp(pjmedia_transport *tp, + const void *pkt, + pj_size_t size) +{ + return (*tp->op->send_rtp)(tp, pkt, size); +} + +``` + +## 4、transport_send_rtp + +直接调用transport_send_rtcp2 + +```c +/* Called by application to send RTP packet */ +static pj_status_t transport_send_rtp( pjmedia_transport *tp, + const void *pkt, + pj_size_t size) +{ + struct transport_udp *udp = (struct transport_udp*)tp; + pj_ssize_t sent; + unsigned id; + struct pending_write *pw; + pj_status_t status; + + /* Must be attached */ + //PJ_ASSERT_RETURN(udp->attached, PJ_EINVALIDOP); + + /* Check that the size is supported */ + PJ_ASSERT_RETURN(size <= PJMEDIA_MAX_MTU, PJ_ETOOBIG); + + if (!udp->started) { + return PJ_SUCCESS; + } + + /* Simulate packet lost on TX direction */ + if (udp->tx_drop_pct) { + if ((pj_rand() % 100) <= (int)udp->tx_drop_pct) { + PJ_LOG(5,(udp->base.name, + "TX RTP packet dropped because of pkt lost " + "simulation")); + return PJ_SUCCESS; + } + } + + + id = udp->rtp_write_op_id; + pw = &udp->rtp_pending_write[id]; + if (pw->is_pending) { + /* There is still currently pending operation for this buffer. */ + PJ_LOG(4,(udp->base.name, "Too many pending write operations")); + return PJ_EBUSY; + } + pw->is_pending = PJ_TRUE; + + /* We need to copy packet to our buffer because when the + * operation is pending, caller might write something else + * to the original buffer. + */ + pj_memcpy(pw->buffer, pkt, size); + + sent = size; + status = pj_ioqueue_sendto( udp->rtp_key, + &udp->rtp_pending_write[id].op_key, + pw->buffer, &sent, 0, + &udp->rem_rtp_addr, + udp->addr_len); + + if (status != PJ_EPENDING) { + /* Send operation has completed immediately. Clear the flag. */ + pw->is_pending = PJ_FALSE; + } + + udp->rtp_write_op_id = (udp->rtp_write_op_id + 1) % + PJ_ARRAY_SIZE(udp->rtp_pending_write); + + if (status==PJ_SUCCESS || status==PJ_EPENDING) + return PJ_SUCCESS; + + return status; +} +``` + +udp->rtp_write_op_id;是当前write操作可用的id，udp->rtp_pending_write是Pending write对象用于指示udp->rtp_pending_write[udp->rtp_write_op_id]是否有pending的write 操作如果有pending 返回。没有的话将该id处置为pending `pw->is_pending = PJ_TRUE; ` 调用pj_ioqueue_sendto 最后需要将id+1 + +## 5、pj_ioqueue_sendto + +参数key来着 udp->rtp_key 在 pj_ioqueue_register_sock2中初始化，绑定到rtp socket + +参数op_key来自 transport_udp::udp->rtp_pending_write[id].op_key，在transport_media_start中设置为空 + +如果key对应的writelist不为空，直接发送调用pj_sock_sendto + +```c + if (pj_list_empty(&key->write_list)) { + /* + * See if data can be sent immediately. + */ + sent = *length; + status = pj_sock_sendto(key->fd, data, &sent, flags, addr, addrlen); + + } +``` + +否则初始化write_operation:: write_op 主要需要将要发送的rtp数据保存在write_op中，然后挂在key对应的writelist队列上 + +```c + write_op->op = PJ_IOQUEUE_OP_SEND_TO; + write_op->buf = (char*)data; + write_op->size = *length; + write_op->written = 0; + write_op->flags = flags; + pj_memcpy(&write_op->rmt_addr, addr, addrlen); + write_op->rmt_addrlen = addrlen; + + pj_ioqueue_lock_key(key); + /* Check again. Handle may have been closed after the previous check + * in multithreaded app. If we add bad handle to the set it will + * corrupt the ioqueue set. See #913 + */ + if (IS_CLOSING(key)) { + pj_ioqueue_unlock_key(key); + return PJ_ECANCELLED; + } + pj_list_insert_before(&key->write_list, write_op); +``` + +调用ioqueue_add_to_set进而调用ioqueue_add_to_set2 + +```c +static void ioqueue_add_to_set2(pj_ioqueue_t *ioqueue, + pj_ioqueue_key_t *key, + unsigned event_types ) +{ + pj_uint32_t events = key->ev.events; + + if (event_types & READABLE_EVENT) + events |= EPOLLIN; + if (event_types & WRITEABLE_EVENT) + events |= EPOLLOUT; + if (event_types & EXCEPTION_EVENT) + events |= EPOLLERR; + + if (events != key->ev.events) + update_epoll_event_set(ioqueue, key, events); +} +``` + + 根据 event_types 设置 events 调用 update_epoll_event_set 这里event_types是WRITEABLE_EVENT + +```c +static void update_epoll_event_set(pj_ioqueue_t *ioqueue, + pj_ioqueue_key_t *key, + pj_uint32_t events) +{ + int rc; + /* From epoll_ctl(2): + * EPOLLEXCLUSIVE may be used only in an EPOLL_CTL_ADD operation; + * attempts to employ it with EPOLL_CTL_MOD yield an error. + */ + if (key->ev.events & EPOLLEXCLUSIVE) { + rc = os_epoll_ctl(ioqueue->epfd, EPOLL_CTL_DEL, key->fd, &key->ev); + key->ev.events = events; + rc = os_epoll_ctl(ioqueue->epfd, EPOLL_CTL_ADD, key->fd, &key->ev); + } else { + key->ev.events = events; + rc = os_epoll_ctl(ioqueue->epfd, EPOLL_CTL_MOD, key->fd, &key->ev); + } + + if (rc != 0) { + pj_status_t status = pj_get_os_error(); + PJ_PERROR(1,(THIS_FILE, status, + "epol_ctl(MOD) error (events=0x%x)", events)); + } +} +``` + +这里修改监测rtp socket事件EPOLL_CTL_MOD修改为EPOLLOUT 触发 + +## 6、ioqueue_epoll参与 + +这里不一定是上述追踪的rtp包 + +当pj_ioqueue_poll工作线程 调用os_epoll_wait 发现监测的EPOLLOUT写触发，调用ioqueue_dispatch_write_event写操作，在ioqueue_dispatch_write_event中先看key write_list上有没有pending_write,有的话，从write_list取出，根据write_list的 write_op确定写大小，要写入的数据，将数据写入调用pj_sock_send函数Transmit data to the socket.，最后调用on_write_complete，回调函数已在pj_ioqueue_register_sock2时设置过，传入write_op为on_rtp_data_sent + +```c +if (h->cb.on_write_complete && !IS_CLOSING(h)) { + (*h->cb.on_write_complete)(h, + (pj_ioqueue_op_key_t*)write_op, + write_op->written); + } +``` + +```c +static void on_rtp_data_sent(pj_ioqueue_key_t *key, + pj_ioqueue_op_key_t *op_key, + pj_ssize_t bytes_sent) +{ + struct transport_udp *udp; + unsigned i; + + PJ_UNUSED_ARG(bytes_sent); + + udp = (struct transport_udp*) pj_ioqueue_get_user_data(key); + + for (i = 0; i < PJ_ARRAY_SIZE(udp->rtp_pending_write); ++i) { + if (&udp->rtp_pending_write[i].op_key == op_key) { + udp->rtp_pending_write[i].is_pending = PJ_FALSE; + break; + } + } +} +``` + +遍历transport_udp udp中的rtp_pending_write 找到与目标 op_key一致的位置，将is_pending置为false完成 + + + +## 附：相互指向关系 + +stream.user_data-》snd_port， + +snd_port.port-》port + +port->port_data.pdata;-》stream + +stream->transport-》pjmedia_transport + diff --git "a/src/\346\216\245\346\224\266\346\225\260\346\215\256\345\205\250\346\265\201\347\250\213.md" "b/src/\346\216\245\346\224\266\346\225\260\346\215\256\345\205\250\346\265\201\347\250\213.md" new file mode 100644 index 0000000..16e822b --- /dev/null +++ "b/src/\346\216\245\346\224\266\346\225\260\346\215\256\345\205\250\346\265\201\347\250\213.md" @@ -0,0 +1,653 @@ +# 接收数据全流程 + +## 1、接受的线程（在声音设备端直接向声卡写入数据） + +先看发送的线程 pb_thread_func，这里我们以alsa_dev为例 + +```c +static int pb_thread_func (void *arg) +{ + struct alsa_stream* stream = (struct alsa_stream*) arg; + snd_pcm_t* pcm = stream->pb_pcm; + int size = stream->pb_buf_size; + snd_pcm_uframes_t nframes = stream->pb_frames; + void* user_data = stream->user_data; + char* buf = stream->pb_buf; + pj_timestamp tstamp; + int result; + + pj_bzero (buf, size); + tstamp.u64 = 0; + + TRACE_((THIS_FILE, "pb_thread_func(%u): Started", + (unsigned)syscall(SYS_gettid))); + + snd_pcm_prepare (pcm); + + while (!stream->quit) { + pjmedia_frame frame; + + frame.type = PJMEDIA_FRAME_TYPE_AUDIO; + frame.buf = buf; + frame.size = size; + frame.timestamp.u64 = tstamp.u64; + frame.bit_info = 0; + + result = stream->pb_cb (user_data, &frame); + if (result != PJ_SUCCESS || stream->quit) + break; + + if (frame.type != PJMEDIA_FRAME_TYPE_AUDIO) + pj_bzero (buf, size); + + result = snd_pcm_writei (pcm, buf, nframes); + if (result == -EPIPE) { + PJ_LOG (4,(THIS_FILE, "pb_thread_func: underrun!")); + snd_pcm_prepare (pcm); + } else if (result < 0) { + PJ_LOG (4,(THIS_FILE, "pb_thread_func: error writing data!")); + } + + tstamp.u64 += nframes; + } + + snd_pcm_drop(pcm); + TRACE_((THIS_FILE, "pb_thread_func: Stopped")); + return PJ_SUCCESS; +} +``` + +该线程的主体部分在不停的while循环，我们来看一次循环的内容： + +调用 `stream->pb_cb (user_data, &frame);` 其中user_data 依然是我们创建的`snd_port`，frame存放最终接收的一个frame即Frame to store samples.，需要先对frame的类型等信息进行设置。 + +调用` snd_pcm_writei (pcm, buf, nframes);` 将数据写入声卡 + +## 2、play_cb （音频流端） + +pjmedia/src/pjmedia/sound_port.c + +```c +/* + * The callback called by sound player when it needs more samples to be + * played. + */ +static pj_status_t play_cb(void *user_data, pjmedia_frame *frame) +{ + pjmedia_snd_port *snd_port = (pjmedia_snd_port*) user_data; + pjmedia_port *port; + const unsigned required_size = (unsigned)frame->size; + pj_status_t status; + + pjmedia_clock_src_update(&snd_port->play_clocksrc, &frame->timestamp); + + port = snd_port->port; + if (port == NULL) + goto no_frame; + + status = pjmedia_port_get_frame(port, frame); + if (status != PJ_SUCCESS) + goto no_frame; + + if (frame->type != PJMEDIA_FRAME_TYPE_AUDIO) + goto no_frame; + + /* Must supply the required samples */ + pj_assert(frame->size == required_size); + + if (snd_port->ec_state) { + if (snd_port->ec_suspended) { + snd_port->ec_suspended = PJ_FALSE; + //pjmedia_echo_state_reset(snd_port->ec_state); + PJ_LOG(4,(THIS_FILE, "EC activated")); + } + snd_port->ec_suspend_count = 0; + pjmedia_echo_playback(snd_port->ec_state, (pj_int16_t*)frame->buf); + } + + /* Invoke preview callback */ + if (snd_port->on_play_frame) + (*snd_port->on_play_frame)(snd_port->user_data, frame); + + return PJ_SUCCESS; + +no_frame: + frame->type = PJMEDIA_FRAME_TYPE_AUDIO; + frame->size = required_size; + pj_bzero(frame->buf, frame->size); + + if (snd_port->ec_state && !snd_port->ec_suspended) { + ++snd_port->ec_suspend_count; + if (snd_port->ec_suspend_count > snd_port->ec_suspend_limit) { + snd_port->ec_suspended = PJ_TRUE; + PJ_LOG(4,(THIS_FILE, "EC suspended because of inactivity")); + } + if (snd_port->ec_state) { + /* To maintain correct delay in EC */ + pjmedia_echo_playback(snd_port->ec_state, (pj_int16_t*)frame->buf); + } + } + + /* Invoke preview callback */ + if (snd_port->on_play_frame) + (*snd_port->on_play_frame)(snd_port->user_data, frame); + + return PJ_SUCCESS; +} + +``` + +首先看`snd_port->on_play_frame`这个是可选项，这里没有使用 + +其次就是最重要的调用pjmedia_port_get_frame，看一下这里的port参数，`port = snd_port->port;` 是pjmedia_snd_port类型snd_port中的pjmedia_port类型属性，snd_port->port是在`pjmedia_snd_port_connect`中初始化的先看一下pjmedia_snd_port_connect + +```c +/* + * Connect a port. + */ +PJ_DEF(pj_status_t) pjmedia_snd_port_connect( pjmedia_snd_port *snd_port, + pjmedia_port *port) +{ + pjmedia_audio_format_detail *afd; + + PJ_ASSERT_RETURN(snd_port && port, PJ_EINVAL); + + afd = pjmedia_format_get_audio_format_detail(&port->info.fmt, PJ_TRUE); + + /* Check that port has the same configuration as the sound device + * port. + */ + if (afd->clock_rate != snd_port->clock_rate) + return PJMEDIA_ENCCLOCKRATE; + + if (PJMEDIA_AFD_SPF(afd) != snd_port->samples_per_frame) + return PJMEDIA_ENCSAMPLESPFRAME; + + if (afd->channel_count != snd_port->channel_count) + return PJMEDIA_ENCCHANNEL; + + if (afd->bits_per_sample != snd_port->bits_per_sample) + return PJMEDIA_ENCBITS; + + /* Port is okay. */ + snd_port->port = port; + return PJ_SUCCESS; +} +``` + +注意这里的参数`pjmedia_port *port`来自pjmedia_stream stream的port，这个port还有一点特殊即port->port_data.pdata; 其实是指向stream，即stream中有port，port也有办法指向stream，这在put_frame中会遇到。 + +<::>再回来看pjmedia_port_put_frame该函数其实是port->put_frame callback的封装，会直接调用port->put_frame，这个回调函数的初始化在`pjmedia_stream_create` 完成将port->put_frame 初始化为 stream->get_frame = &get_frame;，所以接下来我们看put_frame + +```c +/** + * Get a frame from the port (and subsequent downstream ports). + */ +PJ_DEF(pj_status_t) pjmedia_port_get_frame( pjmedia_port *port, + pjmedia_frame *frame ) +{ + PJ_ASSERT_RETURN(port && frame, PJ_EINVAL); + + if (port->get_frame) + return port->get_frame(port, frame); + else { + frame->type = PJMEDIA_FRAME_TYPE_NONE; + return PJ_EINVALIDOP; + } +} +``` + +## 3、get_frame （port端） + +pjmedia/src/pjmedia/stream.c + +1. 首先，函数从 `port` 中获取与当前流相关联的 `stream` 和 `channel`。 + + ```c + pjmedia_stream *stream = (pjmedia_stream*) port->port_data.pdata; + pjmedia_channel *channel = stream->dec; + ``` + +2. 然后，函数检查通道是否处于暂停状态，如果是，则设置帧类型为 `PJMEDIA_FRAME_TYPE_NONE`，表示没有帧可用，并返回 `PJ_SUCCESS`。 + + ```c + /* Return no frame is channel is paused */ + if (channel->paused) { + frame->type = PJMEDIA_FRAME_TYPE_NONE; + return PJ_SUCCESS; + } + ``` + + + +3. 接着，函数检查是否处于软启动计数状态。如果是，它首先检查软启动计数是否为 `PJMEDIA_STREAM_SOFT_START`，如果是，则重置抖动缓冲区。然后递减软启动计数，并返回 `PJ_SUCCESS`。 + + ```c + if (stream->soft_start_cnt) { + if (stream->soft_start_cnt == PJMEDIA_STREAM_SOFT_START) { + PJ_LOG(4,(stream->port.info.name.ptr, + "Resetting jitter buffer in stream playback start")); + pj_mutex_lock( stream->jb_mutex ); + pjmedia_jbuf_reset(stream->jb); + pj_mutex_unlock( stream->jb_mutex ); + } + --stream->soft_start_cnt; + frame->type = PJMEDIA_FRAME_TYPE_NONE; + return PJ_SUCCESS; + } + ``` + + + +4. 接下来，函数从抖动缓冲区中获取帧并解码，直到获得足够的帧以满足编解码器的 ptime 要求。 + + - 函数首先锁定抖动缓冲区的互斥锁。 + + ```c + pj_mutex_lock( stream->jb_mutex ); + ``` + + + + - 接着，函数计算所需的样本数，并根据解码器的 ptime 要求计算每帧的样本数。 + + ```c + samples_required = PJMEDIA_PIA_SPF(&stream->port.info); + samples_per_frame = stream->dec_ptime * + stream->codec_param.info.clock_rate * + stream->codec_param.info.channel_cnt / + stream->dec_ptime_denum / + 1000; + p_out_samp = (pj_int16_t*) frame->buf; + ``` + + + + - 然后，函数循环获取帧，直到获得足够的样本数。在每次循环中，它会尝试从抖动缓冲区获取帧pjmedia_jbuf_get_frame2放在channel->out_pkt，并根据获取的帧类型进行不同的处理。 + + ```c + for (samples_count=0; samples_count < samples_required;) { + char frame_type; + pj_size_t frame_size = channel->out_pkt_size; + pj_uint32_t bit_info; + + if (stream->dec_buf && stream->dec_buf_pos < stream->dec_buf_count) { + unsigned nsamples_req = samples_required - samples_count; + unsigned nsamples_avail = stream->dec_buf_count - + stream->dec_buf_pos; + unsigned nsamples_copy = PJ_MIN(nsamples_req, nsamples_avail); + + pjmedia_copy_samples(p_out_samp + samples_count, + stream->dec_buf + stream->dec_buf_pos, + nsamples_copy); + samples_count += nsamples_copy; + stream->dec_buf_pos += nsamples_copy; + continue; + } + + /* Get frame from jitter buffer. */ + pjmedia_jbuf_get_frame2(stream->jb, channel->out_pkt, &frame_size, + &frame_type, &bit_info); + ``` + + - 如果帧类型为 `PJMEDIA_JB_MISSING_FRAME`，表示丢失帧，则尝试激活 PLC 进行丢帧处理。如果 PLC 激活成功，则填充丢失的样本，并增加 PLC 计数。 + + - 如果帧类型为 `PJMEDIA_JB_ZERO_EMPTY_FRAME`，表示抖动缓冲区为空。函数会尝试激活 PLC 进行丢帧处理，然后填充零样本，以平滑淡出。 + + - 如果帧类型为 `PJMEDIA_JB_ZERO_PREFETCH_FRAME`，表示抖动缓冲区正在预取数据。函数会尝试激活 PLC 进行丢帧处理，然后填充零样本。 + + - 如果帧类型为 `PJMEDIA_JB_NORMAL_FRAME`，表示获得了正常的帧。函数会解码帧pjmedia_codec_decode 得到frame_out并将其放入播放缓冲区即传入的参数frame中。 + + ```c + /* Got "NORMAL" frame from jitter buffer */ + pjmedia_frame frame_in, frame_out; + pj_bool_t use_dec_buf = PJ_FALSE; + + stream->plc_cnt = 0; + + /* Decode */ + frame_in.buf = channel->out_pkt; + frame_in.size = frame_size; + frame_in.bit_info = bit_info; + frame_in.type = PJMEDIA_FRAME_TYPE_AUDIO; /* ignored */ + + frame_out.buf = p_out_samp + samples_count; + frame_out.size = frame->size - samples_count*BYTES_PER_SAMPLE; + if (stream->dec_buf && + bit_info * sizeof(pj_int16_t) > frame_out.size) + { + stream->dec_buf_pos = 0; + stream->dec_buf_count = bit_info; + + use_dec_buf = PJ_TRUE; + frame_out.buf = stream->dec_buf; + frame_out.size = stream->dec_buf_size; + } + + status = pjmedia_codec_decode( stream->codec, &frame_in, + (unsigned)frame_out.size, + &frame_out); + if (status != 0) { + LOGERR_((port->info.name.ptr, status, + "codec decode() error")); + + if (use_dec_buf) { + pjmedia_zero_samples(stream->dec_buf, + stream->dec_buf_count); + } else { + pjmedia_zero_samples(p_out_samp + samples_count, + samples_per_frame); + } + } else if (use_dec_buf) { + stream->dec_buf_count = (unsigned)frame_out.size / + sizeof(pj_int16_t); + } + + if (stream->jb_last_frm != frame_type) { + /* Report changing frame type event */ + PJ_LOG(5,(stream->port.info.name.ptr, + "Jitter buffer starts returning normal frames " + "(after %d empty/lost)", + stream->jb_last_frm_cnt)); + + stream->jb_last_frm = frame_type; + stream->jb_last_frm_cnt = 1; + } else { + stream->jb_last_frm_cnt++; + } + if (!use_dec_buf) + samples_count += samples_per_frame; + + ``` + + + +5. 最后，函数解锁抖动缓冲区的互斥锁，并根据获取的样本数设置帧类型和大小，并返回 `PJ_SUCCESS`。 + +### （1）pjmedia_jbuf_get_frame3 + +```c +/* + * Get frame from jitter buffer. + */ +PJ_DEF(void) pjmedia_jbuf_get_frame3(pjmedia_jbuf *jb, + void *frame, + pj_size_t *size, + char *p_frame_type, + pj_uint32_t *bit_info, + pj_uint32_t *ts, + int *seq) +{ + if (jb->jb_prefetching) { + + /* Can't return frame because jitter buffer is filling up + * minimum prefetch. + */ + + //pj_bzero(frame, jb->jb_frame_size); + *p_frame_type = PJMEDIA_JB_ZERO_PREFETCH_FRAME; + if (size) + *size = 0; + + TRACE__((jb->jb_name.ptr, "GET prefetch_cnt=%d/%d", + jb_framelist_eff_size(&jb->jb_framelist), jb->jb_prefetch)); + + jb->jb_empty++; + + } else { + + pjmedia_jb_frame_type ftype = PJMEDIA_JB_NORMAL_FRAME; + pj_bool_t res; + + /* Try to retrieve a frame from frame list */ + res = jb_framelist_get(&jb->jb_framelist, frame, size, &ftype, + bit_info, ts, seq); + if (res) { + /* We've successfully retrieved a frame from the frame list, but + * the frame could be a blank frame! + */ + if (ftype == PJMEDIA_JB_NORMAL_FRAME) { + *p_frame_type = PJMEDIA_JB_NORMAL_FRAME; + } else { + *p_frame_type = PJMEDIA_JB_MISSING_FRAME; + jb->jb_lost++; + } + + /* Store delay history at the first GET */ + if (jb->jb_last_op == JB_OP_PUT) { + unsigned cur_size; + + /* We've just retrieved one frame, so add one to cur_size */ + cur_size = jb_framelist_eff_size(&jb->jb_framelist) + 1; + pj_math_stat_update(&jb->jb_delay, + cur_size * jb->jb_frame_ptime / + jb->jb_frame_ptime_denum); + } + } else { + /* Jitter buffer is empty */ + if (jb->jb_prefetch) + jb->jb_prefetching = PJ_TRUE; + + //pj_bzero(frame, jb->jb_frame_size); + *p_frame_type = PJMEDIA_JB_ZERO_EMPTY_FRAME; + if (size) + *size = 0; + + jb->jb_empty++; + } + } + + jb->jb_level++; + jbuf_update(jb, JB_OP_GET); +} + +``` + +1. 首先，函数检查抖动缓冲区是否正在预取数据。如果是，则表示抖动缓冲区正在填充，此时无法返回帧，因此将帧类型设置为 `PJMEDIA_JB_ZERO_PREFETCH_FRAME`，并将帧大小设置为 0。 +2. 如果抖动缓冲区不在预取状态，则尝试从帧列表中获取帧。如果成功获取到帧，则将帧类型设置为 `PJMEDIA_JB_NORMAL_FRAME`，表示正常帧。如果获取到的是空白帧，则将帧类型设置为 `PJMEDIA_JB_MISSING_FRAME`，并增加抖动缓冲区丢失帧的计数。 +3. 如果无法从帧列表中获取帧，表示抖动缓冲区为空。如果抖动缓冲区允许预取，则将预取状态设置为 `PJ_TRUE`。然后，将帧类型设置为 `PJMEDIA_JB_ZERO_EMPTY_FRAME`，并将帧大小设置为 0。 +4. 最后，函数更新抖动缓冲区的级别，并调用 `jbuf_update` 函数更新抖动缓冲区的操作。 + +## 4、ioqueue_epoll参与 + +这里不一定是上述追踪的rtp包 + +当pj_ioqueue_poll工作线程 调用os_epoll_wait 发现监测的读触发，调用ioqueue_dispatch_read_event写操作，在ioqueue_dispatch_read_event中先看key read_list上有没有pending_read,有的话，从read_list取出，根据read_list的read_op确定读入大小，pj_sock_recvfrom接受数据的函数，将数据读入到read_op中，最后调用on_read_complete，回调函数已在pj_ioqueue_register_sock2时设置过，传入read_op为 on_rx_rtp + +```c +(*h->cb.on_read_complete)(h, + (pj_ioqueue_op_key_t*)read_op, + bytes_read); +``` + + + +### （1）on_rx_rtp 读取操作 + +on_rx_rtp是被下面调用 + +```c +(*h->cb.on_read_complete)(h, + (pj_ioqueue_op_key_t*)read_op, + bytes_read); +``` + +这里有一个很有意思的问题，就是read_op 在on_rx_rtp中竟然没有使用，下面来分析一下原因 + +``` +udp = (struct transport_udp*) pj_ioqueue_get_user_data(key); + +``` + +我们的read_op是从readlist中取出的，readlist对于读操作添加是依靠pj_ioqueue_recvfrom函数，在data is not immediately available时将read_op加入readlist + +```c +read_op->op = PJ_IOQUEUE_OP_RECV_FROM; +read_op->buf = buffer; +read_op->size = *length; +read_op->flags = flags; +read_op->rmt_addr = addr; +read_op->rmt_addrlen = addrlen; + +pj_ioqueue_lock_key(key); +/* Check again. Handle may have been closed after the previous check + * in multithreaded app. If we add bad handle to the set it will + * corrupt the ioqueue set. See #913 + */ +if (IS_CLOSING(key)) { + pj_ioqueue_unlock_key(key); + return PJ_ECANCELLED; +} +pj_list_insert_before(&key->read_list, read_op); +ioqueue_add_to_set(key->ioqueue, key, READABLE_EVENT); +``` + +这里read_op->buf = buffer;的buffer，来自udp->rtp_pkt,相当于直接写入了rtp_pkt,所以不用read_op了。 + +on_rx_rtp是一个while循环，条件如下status来自pj_ioqueue_recvfrom的结果 + +```c +status != PJ_EPENDING && status != PJ_ECANCELLED && + udp->started +``` + +#### a. call_rtp_cb + +在while循环里，先执行call_rtp_cb，设置pjmedia_tp_cb_param param;，调用(*cb2)(¶m);cb2由transport_attach2-》tp_attach设置为stream.c ::on_rx_rtp。注意param.pkt = udp->rtp_pkt;，这里rtp_pkt其实就是ioqueue_dispatch_read_event中read_op->buf中读到的数据rtp包 + +```c +/* Call RTP cb. */ +static void call_rtp_cb(struct transport_udp *udp, pj_ssize_t bytes_read, + pj_bool_t *rem_switch) +{ + void (*cb)(void*,void*,pj_ssize_t); + void (*cb2)(pjmedia_tp_cb_param*); + void *user_data; + + cb = udp->rtp_cb; + cb2 = udp->rtp_cb2; + user_data = udp->user_data; + + if (cb2) { + pjmedia_tp_cb_param param; + + param.user_data = user_data; + param.pkt = udp->rtp_pkt; + param.size = bytes_read; + param.src_addr = &udp->rtp_src_addr; + param.rem_switch = PJ_FALSE; + (*cb2)(¶m); + if (rem_switch) + *rem_switch = param.rem_switch; + } else if (cb) { + (*cb)(user_data, udp->rtp_pkt, bytes_read); + } +} +``` + + param.user_data = user_data; 注意这个user_data，是pjmedia_stream *stream + +#### b. pj_ioqueue_recvfrom + +```c +/* + * pj_ioqueue_recvfrom() + * + * Start asynchronous recvfrom() from the socket. + */ +PJ_DEF(pj_status_t) pj_ioqueue_recvfrom( pj_ioqueue_key_t *key, + pj_ioqueue_op_key_t *op_key, + void *buffer, + pj_ssize_t *length, + unsigned flags, + pj_sockaddr_t *addr, + int *addrlen) +{ + struct read_operation *read_op; + + PJ_ASSERT_RETURN(key && op_key && buffer && length, PJ_EINVAL); + PJ_CHECK_STACK(); + + /* Check if key is closing. */ + if (IS_CLOSING(key)) + return PJ_ECANCELLED; + + read_op = (struct read_operation*)op_key; + PJ_ASSERT_RETURN(read_op->op == PJ_IOQUEUE_OP_NONE, PJ_EPENDING); + read_op->op = PJ_IOQUEUE_OP_NONE; + + /* Try to see if there's data immediately available. + */ + if ((flags & PJ_IOQUEUE_ALWAYS_ASYNC) == 0) { + pj_status_t status; + pj_ssize_t size; + + size = *length; + status = pj_sock_recvfrom(key->fd, buffer, &size, flags, + addr, addrlen); + if (status == PJ_SUCCESS) { + /* Yes! Data is available! */ + *length = size; + return PJ_SUCCESS; + } else { + /* If error is not EWOULDBLOCK (or EAGAIN on Linux), report + * the error to caller. + */ + if (status != PJ_STATUS_FROM_OS(PJ_BLOCKING_ERROR_VAL)) + return status; + } + } + + flags &= ~(PJ_IOQUEUE_ALWAYS_ASYNC); + + /* + * No data is immediately available. + * Must schedule asynchronous operation to the ioqueue. + */ + read_op->op = PJ_IOQUEUE_OP_RECV_FROM; + read_op->buf = buffer; + read_op->size = *length; + read_op->flags = flags; + read_op->rmt_addr = addr; + read_op->rmt_addrlen = addrlen; + + pj_ioqueue_lock_key(key); + /* Check again. Handle may have been closed after the previous check + * in multithreaded app. If we add bad handle to the set it will + * corrupt the ioqueue set. See #913 + */ + if (IS_CLOSING(key)) { + pj_ioqueue_unlock_key(key); + return PJ_ECANCELLED; + } + pj_list_insert_before(&key->read_list, read_op); + ioqueue_add_to_set(key->ioqueue, key, READABLE_EVENT); + pj_ioqueue_unlock_key(key); + + return PJ_EPENDING; +} + +``` + +接下来是调用pj_ioqueue_recvfrom，至于为什么明明ioqueue_dispatch_read_event已经读取了数据，此时还在读取数据，是因为可能有新的rtp包到达，pj_ioqueue_recvfrom查看有没有到达的包，如果有就调用pj_sock_recvfrom继续读读到udp->rtp_pkt，如果没有加到readlist中，返回PJ_EPENDING，结束on_rx_rtp中的while循环。 + +### （2）on_rx_rtp::cb2 + +Stream.c中的回调 tp_attach中设置该回调 + +该函数处理接收到的rtp包, 解析成payload和head + +Put "good" packet to jitter buffer，需要先把payload解析成frame，再把frame放入jitter buffer + + + +## 附：相互指向关系 + +stream.user_data-》snd_port， + +snd_port.port-》port + +port->port_data.pdata-》stream + +stream->transport-》pjmedia_transport + diff --git "a/\346\265\201\347\250\213\345\233\276/.$call_on_media_update.drawio.bkp" "b/\346\265\201\347\250\213\345\233\276/.$call_on_media_update.drawio.bkp" new file mode 100644 index 0000000..6c2506f --- /dev/null +++ "b/\346\265\201\347\250\213\345\233\276/.$call_on_media_update.drawio.bkp" @@ -0,0 +1,100 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git "a/\346\265\201\347\250\213\345\233\276/call_on_media_update.drawio" "b/\346\265\201\347\250\213\345\233\276/call_on_media_update.drawio" new file mode 100644 index 0000000..6657f19 --- /dev/null +++ "b/\346\265\201\347\250\213\345\233\276/call_on_media_update.drawio" @@ -0,0 +1,103 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git "a/\346\265\201\347\250\213\345\233\276/pjmedia_transport_udp_attach.drawio" "b/\346\265\201\347\250\213\345\233\276/pjmedia_transport_udp_attach.drawio" new file mode 100644 index 0000000..4d455e6 --- /dev/null +++ "b/\346\265\201\347\250\213\345\233\276/pjmedia_transport_udp_attach.drawio" @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git "a/\346\265\201\347\250\213\345\233\276/unknown.drawio" "b/\346\265\201\347\250\213\345\233\276/unknown.drawio" new file mode 100644 index 0000000..31305d2 --- /dev/null +++ "b/\346\265\201\347\250\213\345\233\276/unknown.drawio" @@ -0,0 +1,231 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +