GStreamer playbin 多音轨切换是如何实现的？

Posted 2023-03-22

GStreamer 的playbin代码中明显看到是支持多路音视频的，只有音视频的第一个流才会创建preroll queue，而其它的都不做连接。当用户想切换到音轨2的时候，具体的实现流程是什么样的呢？
比如：
是unlink现在的queue1，再link queue2吗？
用户切换音轨又通过什么接口调用呢？

回答详细额外加分

参考技术A 那大海飞扬的优雅的浪花
和笑在白帆颊上哈哈的酒窝
我迷失了方向，从软绵绵的
睡垫上升起来，
离开我的卧室。本回答被提问者采纳

C++ gstreamer函数使用总结

1、GSteamer的基本API的使用

gst_init（）初始化GStreamer 。

gst_parse_launch（）从文本描述快速构建管道 。

playbin创建自动播放管道。

gst_element_set_state（）通知GStreamer开始播放 。

gst_element_get_bus（）和gst_bus_timed_pop_filtered（）来释放资源     

#include <iostream>
#include <gst/gst.h>
#include <glib.h>

int main(int argc, char *argv[]) 
    GstElement *pipeline;
    GstBus *bus;
    GstMessage *msg;

    /* Initialize GStreamer */
    gst_init(&argc, &argv);
    //初始化gstream

    /* Build the pipeline */
    pipeline =gst_parse_launch("playbin uri=file:///D:/gstream/1.mp4",NULL);
    //gst_parse_launch使用系统预设的管道来处理流媒体。gst_parse_launch创建的是一个由playbin单元素组成的管道

    /* Start playing */
    gst_element_set_state(pipeline, GST_STATE_PLAYING);
    //将我们的元素设置为playing状态才能开始播放

    /* Wait until error or EOS */
    bus = gst_element_get_bus(pipeline);
    msg =gst_bus_timed_pop_filtered(bus, GST_CLOCK_TIME_NONE,GST_MESSAGE_ERROR );  //| GST_MESSAGE_EOS
    //遇到错误或者播放完毕以后gst_bus_timed_pop_filtered（）会返回一条消息
    /* Free resources */
    if (msg != NULL) 
        gst_message_unref(msg);
        //需要使用gst_message_unref（）将msg释放,此函数专门清除gst_bus_timed_pop_filtered
        gst_object_unref(bus);
        gst_element_set_state(pipeline, GST_STATE_NULL);
        gst_object_unref(pipeline);
    
    printf("finish");
    return 0;

2、创建元件并且链接起来

#include <gst/gst.h>

int main(int argc, char *argv[]) 
    GstElement *pipeline, *source, *sink;
    GstBus *bus;
    GstMessage *msg;
    GstStateChangeReturn ret;

    /* Initialize GStreamer */
    gst_init(&argc, &argv);

    /* Create the elements */
    source = gst_element_factory_make("videotestsrc", "source");
    sink = gst_element_factory_make("autovideosink", "sink");
    //创建元件，参数：元件的类型，元件名称
    //videotestsrc是一个源元素（它产生数据），它创建一个测试视频模式。
    //autovideosink是一个接收器元素（它消耗数据），它在窗口上显示它接收到的图像。

    /* Create the empty pipeline */
    pipeline = gst_pipeline_new("test-pipeline");
    //创建管道，管道是一种特殊类型的bin，（估计就是箱柜）

    if (!pipeline || !source || !sink) 
        g_printerr("Not all elements could be created.
");
        return -1;
    

    /* Build the pipeline */
    gst_bin_add_many(GST_BIN(pipeline), source, sink, NULL);
    //向管道中添加元件，以null结尾，添加单个和可以，函数是：gst_bin_add()
    if (gst_element_link(source, sink) != TRUE) 
        g_printerr("Elements could not be linked.
");
        gst_object_unref(pipeline);
        return -1;
    

    /* Modify the source's properties */
    g_object_set(source, "pattern", 0, NULL);
    //修改元件的属性

    /* Start playing */
    ret = gst_element_set_state(pipeline, GST_STATE_PLAYING);
    if (ret == GST_STATE_CHANGE_FAILURE) 
        g_printerr("Unable to set the pipeline to the playing state.
");
        gst_object_unref(pipeline);
        return -1;
    
    //设置管道开始工作
    //调用gst_element_set_state()，并且检查其返回值是否有错误。

    /* Wait until error or EOS */
    bus = gst_element_get_bus(pipeline);
    //获取pipeline的总线
    msg = gst_bus_timed_pop_filtered(bus, GST_CLOCK_TIME_NONE, GST_MESSAGE_ERROR );
    //gst_bus_timed_pop_filtered()等待执行结束并返回GstMessage

    /* Parse message：
    GstMessage是一种非常通用的结构，
    通过使用 GST_MESSAGE_TYPE()宏可以获得其中的消息
    */
    if (msg != NULL) 
        GError *err;
        gchar *debug_info;

        switch (GST_MESSAGE_TYPE(msg)) 
        case GST_MESSAGE_ERROR:
            gst_message_parse_error(msg, &err, &debug_info);
            g_printerr("Error received from element %s: %s
", GST_OBJECT_NAME(msg->src), err->message);
            g_printerr("Debugging information: %s
", debug_info ? debug_info : "none");
            g_clear_error(&err);
            g_free(debug_info);
            break;
        case GST_MESSAGE_EOS:
            g_print("End-Of-Stream reached.
");
            break;
        default:
            /* We should not reach here because we only asked for ERRORs and EOS */
            g_printerr("Unexpected message received.
");
            break;
        
        gst_message_unref(msg);
    

    /* Free resources */
    gst_object_unref(bus);
    gst_element_set_state(pipeline, GST_STATE_NULL);
    gst_object_unref(pipeline);
    return 0;

3、添加衬垫，添加回调，手动链接衬垫

#include <gst/gst.h>

/* Structure to contain all our information, so we can pass it to callbacks */
typedef struct _CustomData 
    GstElement *pipeline;
    GstElement *source;
    GstElement *convert;
    GstElement *resample;
    GstElement *sink;
 CustomData;
//先建立一个结构，里面放了一个pipeline指针和四个元件指针

/* Handler for the pad-added signal */
static void pad_added_handler(GstElement *src, GstPad *pad, CustomData *data);
//声明一个函数，叫添加衬垫的函数pad_added_handler。
int main(int argc, char *argv[]) 
    CustomData data;
    GstBus *bus;
    GstMessage *msg;
    GstStateChangeReturn ret;
    gboolean terminate = FALSE;

    /* Initialize GStreamer */
    gst_init(&argc, &argv);
    //同样需要先初始化

    /* Create the elements */
    data.source = gst_element_factory_make("uridecodebin", "source");
    data.convert = gst_element_factory_make("audioconvert", "convert");
    data.resample = gst_element_factory_make("audioresample", "resample");
    data.sink = gst_element_factory_make("autoaudiosink", "sink");

    /* Create the empty pipeline */
    data.pipeline = gst_pipeline_new("test-pipeline");
    //先把data里的信息创建出来，创建了一个pipeline和四个元件

    if (!data.pipeline || !data.source || !data.convert || !data.resample || !data.sink) 
        g_printerr("Not all elements could be created.
");
        return -1;
    

    /* Build the pipeline. Note that we are NOT linking the source at this
    * point. We will do it later. */
    gst_bin_add_many(GST_BIN(data.pipeline), data.source, data.convert, data.resample, data.sink, NULL);
    if (!gst_element_link_many(data.convert, data.resample, data.sink, NULL)) 
        g_printerr("Elements could not be linked.
");
        gst_object_unref(data.pipeline);
        return -1;
    

    /* Set the URI to play */
    g_object_set(data.source, "uri", "https://www.freedesktop.org/software/gstreamer-sdk/data/media/sintel_trailer-480p.webm", NULL);
    //大约是将source元件的衬垫链接到某个网址上

    /* Connect to the pad-added signal */
    g_signal_connect(data.source, "pad-added", G_CALLBACK(pad_added_handler), &data);
    //GSignals是GStreamer中的关键点。它们使您可以在发生事情时（通过回调）得到通知，所以我们为source元件添加了一个回调
    //这个回调好像没有传递参数啊喂，好吧，官方是真么说的：src是GstElement触发信号的。在此示例中，它只能是uridecodebin。newpad是刚刚添加到src元素中的，我理解为哦我们为source添加回调这件事就是增加了一个衬垫，data是当作指针来传递信号的

    /* Start playing */
    ret = gst_element_set_state(data.pipeline, GST_STATE_PLAYING);
    if (ret == GST_STATE_CHANGE_FAILURE) 
        g_printerr("Unable to set the pipeline to the playing state.
");
        gst_object_unref(data.pipeline);
        return -1;
    

    /* Listen to the bus */
    bus = gst_element_get_bus(data.pipeline);
    do 
        msg = gst_bus_timed_pop_filtered(bus, GST_CLOCK_TIME_NONE, GST_MESSAGE_ANY);
        //等待执行结束并且返回
        //顺带说一句，以前的老语法是GST_MESSAGE_STATE_CHANGED | GST_MESSAGE_ERROR | GST_MESSAGE_EOS这样的，所以下文中的case用的是这几个错误信息，但是现在这个语法不被支持了。嗯嗯
        /* Parse message */
        if (msg != NULL) 
            GError *err;
            gchar *debug_info;

            switch (GST_MESSAGE_TYPE(msg)) 
            case GST_MESSAGE_ERROR:
                gst_message_parse_error(msg, &err, &debug_info);
                g_printerr("Error received from element %s: %s
", GST_OBJECT_NAME(msg->src), err->message);
                g_printerr("Debugging information: %s
", debug_info ? debug_info : "none");
                g_clear_error(&err);
                g_free(debug_info);
                terminate = TRUE;
                break;
            case GST_MESSAGE_EOS:
                g_print("End-Of-Stream reached.
");
                terminate = TRUE;
                break;
            case GST_MESSAGE_STATE_CHANGED:
                /* We are only interested in state-changed messages from the pipeline */
                if (GST_MESSAGE_SRC(msg) == GST_OBJECT(data.pipeline)) 
                    GstState old_state, new_state, pending_state;
                    gst_message_parse_state_changed(msg, &old_state, &new_state, &pending_state);
                    g_print("Pipeline state changed from %s to %s:
",
                        gst_element_state_get_name(old_state), gst_element_state_get_name(new_state));
                
                break;
            default:
                /* We should not reach here */
                g_printerr("Unexpected message received.
");
                break;
            
            gst_message_unref(msg);
        
     while (!terminate);
    //只要不中止，就一直监视执行结束的状态

    /* Free resources */
    gst_object_unref(bus);
    gst_element_set_state(data.pipeline, GST_STATE_NULL);
    gst_object_unref(data.pipeline);
    return 0;


/* This function will be called by the pad-added signal */
static void pad_added_handler(GstElement *src, GstPad *new_pad, CustomData *data) 
    GstPad *sink_pad = gst_element_get_static_pad(data->convert, "sink");
    //pipeline的链接顺序是：source-convert-resample-sink，我们为source添加了回调，然后此处在回调内部获取了convert的对应的衬垫
    GstPadLinkReturn ret;
    GstCaps *new_pad_caps = NULL;
    GstStructure *new_pad_struct = NULL;
    const gchar *new_pad_type = NULL;

    g_print("Received new pad '%s' from '%s':
", GST_PAD_NAME(new_pad), GST_ELEMENT_NAME(src));

    /* If our converter is already linked, we have nothing to do here */
    if (gst_pad_is_linked(sink_pad)) 
        g_print("We are already linked. Ignoring.
");
        goto exit;
    
    //此处应该是检查新为source添加的衬垫是不是已经链接到了convert衬垫

    /* Check the new pad's type */
    new_pad_caps = gst_pad_get_current_caps(new_pad);
    new_pad_struct = gst_caps_get_structure(new_pad_caps, 0);
    new_pad_type = gst_structure_get_name(new_pad_struct);
    if (!g_str_has_prefix(new_pad_type, "audio/x-raw")) 
        g_print("It has type '%s' which is not raw audio. Ignoring.
", new_pad_type);
        goto exit;
    
    //检查这个衬垫当前输出的数据类型，经过一番解析，如果发现里面没有"audio/x-raw"，那说明这不是解码音频的

    /* Attempt the link */
    ret = gst_pad_link(new_pad, sink_pad);
    if (GST_PAD_LINK_FAILED(ret)) 
        g_print("Type is '%s' but link failed.
", new_pad_type);
    
    else 
        g_print("Link succeeded (type '%s').
", new_pad_type);
    
    //如果两个衬垫没链接，那就人为地链接起来

exit:
    //这个语法就厉害了，首先定义了一个exit标号，如果前文中goto exit;那转到的就将会是此处
    /* Unreference the new pad's caps, if we got them */
    if (new_pad_caps != NULL)
        gst_caps_unref(new_pad_caps);

    /* Unreference the sink pad */
    gst_object_unref(sink_pad);

4、打印gstreamer的版本信息

#include <iostream>
#include <gst/gst.h>
#include <glib.h>

//#include <gst/gst.h>
int main(int argc,char *argv[])

    const gchar *nano_str;
    guint major, minor, micro, nano;
    gst_init(&argc, &argv);
    gst_version(&major, &minor, &micro, &nano);
    if (nano == 1)
        nano_str = "(CVS)";
    else if (nano == 2)
        nano_str = "(Prerelease)";
    else
        nano_str = "";
    printf("This program is linked against GStreamer %d.%d.%d %s
",major, minor, micro, nano_str);
    return 0;

5、gstreamer封装的argparse

#include <iostream>
#include <gst/gst.h>
#include <glib.h>

#include <gst/gst.h>
int main(int argc,char *argv[])

    gboolean silent = FALSE;
    gchar *savefile = NULL;
    GOptionContext *ctx;
    GError *err = NULL;
    GOptionEntry entries[] = 
         "silent", 's', 0, G_OPTION_ARG_NONE, &silent,"do not output status information", NULL ,
         "output", 'o', 0, G_OPTION_ARG_STRING, &savefile,"save xml representation of pipeline to FILE and exit", "FILE" ,
         NULL 
    ;
    ctx = g_option_context_new("- Your application");
    g_option_context_add_main_entries(ctx, entries, NULL);
    g_option_context_add_group(ctx, gst_init_get_option_group());
    if (!g_option_context_parse(ctx, &argc, &argv, &err)) 
        g_print("Failed to initialize: %s
", err->message);
        g_error_free(err);
        return 1;
    
    printf("Run me with --help to see the Application options appended.
");
    return 0;

6、创建gst元件对象

#include <iostream>

#include <gst/gst.h>
#include <glib.h>


int main(int argc, char *argv[])

    GstElement *element;
    gchar *name;
    /* init GStreamer */
    gst_init(&argc, &argv);
    /* create element */
    element = gst_element_factory_make("fakesrc", "source");
    //创建一个jst元件
    if (!element) 
        g_print("Failed to create element of type 'fakesrc'
");
        return -1;
    
    else 
        g_print("gstelement ok!
");
    
    element = gst_element_factory_make("fakesrc", "source");
    /* get name */
    g_object_get(G_OBJECT(element), "name", &name, NULL);
    //g_object_get获取gobject对象的名字属性
    g_print("The name of the element is '%s'.
", name);
    g_free(name);
    gst_object_unref(GST_OBJECT(element));
    //释放jst元件，必须手动释放
    return 0;

元件的四种状态：

GST_STATE_NULL: 默认状态

        该状态将会回收所有被该元件占用的资源。

GST_STATE_READY: 准备状态

        元件会得到所有所需的全局资源，这些全局资源将被通过该元 件的数据流所使用。例如打开设备、分配缓存等。但在这种状态下，数据流仍未开始被处 理，所 以数据流的位置信息应该自动置 0。如果数据流先前被打开过，它应该被关闭，并且其位置信 息、特性信息应该被重新置为初始状态。

GST_STATE_PAUSED: 暂停状态

        在这种状态下，元件已经对流开始了处理，但此刻暂停了处理。因此该 状态下元件可以修改流的位置信息，读取或者处理流数据，以及一旦状态变为 PLAYING，流可 以重放数据流。这种情况下，时钟是禁止运行的。总之， PAUSED 状态除了不能运行时钟外， 其它与 PLAYING 状态一模一样。处于 PAUSED 状态的元件会很快变换到 PLAYING 状态。举 例来说，视频或音频输出元件会等待数据的到来，并将它们压入队列。一旦状态改变，元件就会 处理接收到的数据。同样，视频接收元件能够播放数据的第 一帧。(因为这并不会影响时钟)。自 动加载器（Autopluggers）可以对已经加载进管道的插件进行这种状态转换。其它更多的像 codecs 或者 filters 这种元件不需要在这个状态上做任何事情。

GST_STATE_PLAYING: 

        PLAYING 状态除了当前运行时钟外，其它与 PAUSED 状态一模一 样。你可以通过函数 gst_element_set_state()来改变一个元件的状态。你如果显式地改变一个元件 的状态，GStreamer 可能会 使它在内部经过一些中间状态。例如你将一个元件从 NULL 状态设 置为 PLAYING 状态，GStreamer 在其内部会使得元件经历过 READY 以及 PAUSED 状态。 当处于 GST_STATE_PLAYING 状态，管道会自动处理数据。它们不需要任何形式的迭代。

7、查看插件

#include <iostream>
#include <gst/gst.h>
#include <glib.h>

int main(int argc,char *argv[])

    GstElementFactory *factory;
    //声明插件，插件是GstElementFactory

    /* init GStreamer */
    gst_init(&argc, &argv);
    /* get factory */
    factory = gst_element_factory_find("audiotestsrc");
    //寻找系统里是否有这个插件
    if (!factory) 
        g_print("You don't have the 'audiotestsrc' element installed!
");
        return -1;
    
    /* display information */
    g_print("The '%s' element is a member of the category %s.
"
        "Description: %s
",
        gst_plugin_feature_get_name(GST_PLUGIN_FEATURE(factory)),
        gst_element_factory_get_klass(factory),
        gst_element_factory_get_description(factory));
    //打印出插件的信息
    return 0;

这个功能就像是命令行里的如下命令：

gst-inspect-1.0   audiotestsrc

#gst-inspect-1.0 加插件名

8、链接元件

#include <iostream>
#include <gst/gst.h>
#include <glib.h>

#include <gst/gst.h>
int
main(int argc,
    char *argv[])

    GstElement *pipeline;
    GstElement *source, *filter, *sink;
    //声明一个源元件，过滤元件和接收元件
    /* init */
    gst_init(&argc, &argv);
    /* create pipeline */
    pipeline = gst_pipeline_new("my-pipeline");
    /* create elements */
    source = gst_element_factory_make("fakesrc", "source");
    filter = gst_element_factory_make("identity", "filter");
    sink = gst_element_factory_make("fakesink", "sink");
    //选择3个插件创建3个不同的元件
    /* must add elements to pipeline before linking them */
    gst_bin_add_many(GST_BIN(pipeline), source, filter, sink, NULL);
    /* link */
    if (!gst_element_link_many(source, filter, sink, NULL)) 
        g_warning("Failed to link elements!");
    
    return 0;

9、箱柜（箱柜本身是一个元件，但是它内部还可以是一串链接起来的元件）

#include <iostream>
#include <gst/gst.h>
#include <glib.h>

#include <gst/gst.h>
int
main(int argc,
    char *argv[])

    GstElement *bin, *pipeline, *source, *sink;
    /* init */
    gst_init(&argc, &argv);
    /* create */
    pipeline = gst_pipeline_new("my_pipeline");
    bin = gst_pipeline_new("my_bin");
    //创建箱柜：gst_pipeline_new和gst_bin_new

    source = gst_element_factory_make("fakesrc", "source");
    sink = gst_element_factory_make("fakesink", "sink");
    /* set up pipeline */
    gst_bin_add_many(GST_BIN(bin), source, sink, NULL);
    gst_bin_add(GST_BIN(pipeline), bin);
    //添加元件到箱柜
    gst_bin_remove(GST_BIN(bin),sink);
    //从箱柜中移除元件，移除的元件自动被销毁，

    gst_element_link(source, sink);
    //链接元件，因为sink元件被我移除了，所以可能实际上运行不起来

    gst_object_unref(GST_OBJECT(source));
    gst_object_unref(GST_OBJECT(sink));
    return 0;

10、bus总线

获取bus总线：gst_pipeline_get_bus

在总线上添加一个回调函数（官方语言叫watch）：gst_bus_add_watch

#include <gst/gst.h>
static GMainLoop *loop;
static gboolean
my_bus_callback(GstBus *bus,GstMessage *message,gpointer data)

    g_print("Got %s message
", GST_MESSAGE_TYPE_NAME(message));
    switch (GST_MESSAGE_TYPE(message)) 
    case GST_MESSAGE_ERROR: 
        GError *err;
        gchar *debug;
        gst_message_parse_error(message, &err, &debug);
        g_print("Error: %s
", err->message);
        g_error_free(err);
        g_free(debug);
        g_main_loop_quit(loop);
        break;
    
    case GST_MESSAGE_EOS:
        /* end-of-stream */
        g_main_loop_quit(loop);
        break;
    default:
        /* unhandled message */
        g_print("something happend!
");
        break;
    
    /* we want to be notified again the next time there is a message
    * on the bus, so returning TRUE (FALSE means we want to stop watching
    * for messages on the bus and our callback should not be called again)
    */
    return TRUE;

gint main(gint argc,gchar *argv[])

    GstElement *pipeline;
    GstBus *bus;
    /* init */
    gst_init(&argc, &argv);
    /* create pipeline, add handler */
    pipeline = gst_pipeline_new("my_pipeline");
    /* adds a watch for new message on our pipeline's message bus to
    * the default GLib main context, which is the main context that our
    * GLib main loop is attached to below
    */
    bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline));
    //首先获取总线
    gst_bus_add_watch(bus, my_bus_callback, NULL);
    //然后添加一个消息处理器：设置消息处理器到管道的总线上gst_bus_add_watch ()
    gst_object_unref(bus);
    
    /* create a mainloop that runs/iterates the default GLib main context
    * (context NULL), in other words: makes the context check if anything
    * it watches for has happened. When a message has been posted on the
    * bus, the default main context will automatically call our
    * my_bus_callback() function to notify us of that message.
    * The main loop will be run until someone calls g_main_loop_quit()
    */
    loop = g_main_loop_new(NULL, FALSE);
    g_main_loop_run(loop);
    /* clean up */
    gst_element_set_state(pipeline, GST_STATE_NULL);
    /*gst_element_unref(pipeline);
    gst_main_loop_unref(loop);*/
    return 0;