@xverizex

Почему и как исправить эту ошибку с выводом видео в виджет gtk с помощью ffmpeg?

Уже несколько дней работаю над этим. Сначала делал на gstreamer, но в этой библиотеке видео начинает обрабатываться только когда поток присваиваешь виджету, например gtk_drawing_area. а я хочу передавать видео по сети и так как не смог найти решения в gstreamer, решил попробовать ffmpeg. В общем написал такой код. и сразу покажу что получилось.
613107d139ebc395593957.png
/* dating_chat-window.c
 *
 * Copyright 2021 cf
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "dating-chat-window.h"
#include <libavdevice/avdevice.h>
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
#include <libavutil/avutil.h>
#include <libavutil/imgutils.h>
#include <libavutil/pixfmt.h>

struct _DatingChatWindow
{
  GtkApplicationWindow  parent_instance;

  AVFormatContext *ctx;
  AVCodecContext *codec_ctx;
  struct SwsContext *sws;
  int video_stream;
  AVFrame *frame;
  AVFrame *frame_rgb;
  uint8_t *buffer;
  int num;
  /* Template widgets */
  GtkWidget *area;

};

G_DEFINE_TYPE (DatingChatWindow, dating_chat_window, GTK_TYPE_APPLICATION_WINDOW)

static void
dating_chat_window_class_init (DatingChatWindowClass *klass)
{
  GtkWidgetClass *widget_class = GTK_WIDGET_CLASS (klass);
  (void) widget_class;
}

gboolean
drawing_render_cb (GtkWidget *widget, cairo_t *cr, gpointer data) {
	DatingChatWindow *self = (DatingChatWindow *) data;

	AVPacket packet;
	int frame_finished;

	av_read_frame (self->ctx, &packet);

	if (packet.stream_index == self->video_stream) {
		static int a = 1;
		avcodec_send_packet (self->codec_ctx, &packet);
		avcodec_receive_frame (self->codec_ctx, self->frame);
		int width, height;
		gtk_widget_get_size_request (self->area, &width, &height);

#if 0
		av_image_fill_arrays (
					self->frame_rgb->data,
					self->frame_rgb->linesize,
					self->buffer,
					AV_PIX_FMT_RGB24,
					self->codec_ctx->width,
					self->codec_ctx->height,
					1
				);
#endif
#if 1
		uint8_t *rgb[1] = { self->buffer };
		int rgb_stride[4] = { self->codec_ctx->width * 3, 0, 0, 0 };
		sws_scale (
				self->sws,
				(const uint8_t * const *) self->frame->data,
				self->frame->linesize,
				0,
				height,
				rgb,
				rgb_stride
			  );

#endif
#if 0
		int rt = av_image_copy_to_buffer (self->buffer,
				self->num,
				(const uint8_t * const *) self->frame_rgb->data,
				self->frame_rgb->linesize,
		  		AV_PIX_FMT_YUYV422,
				//AV_PIX_FMT_RGB24,
				width,
				height,
				1
				);

#endif
		int stride = cairo_format_stride_for_width (CAIRO_FORMAT_RGB24, self->codec_ctx->width);

		cairo_surface_t *surface = cairo_image_surface_create_for_data (
				self->buffer,
				CAIRO_FORMAT_RGB24,
				self->codec_ctx->width,
				self->codec_ctx->height,
				stride);

		cairo_set_source_surface (cr, surface, 0, 0);
		cairo_paint (cr);
		cairo_surface_finish (surface);

	}

	av_packet_unref (&packet);

	return FALSE;
}

gboolean
timeout_frame (gpointer data) {
	DatingChatWindow *self = (DatingChatWindow *) data;

	gtk_widget_queue_draw (self->area);

	return G_SOURCE_CONTINUE;
}

static void
dating_chat_window_init (DatingChatWindow *self)
{


	self->area = gtk_drawing_area_new ( );

	

  avdevice_register_all ();
  AVInputFormat *input_format = av_find_input_format ("v4l2");
  AVFormatContext *ctx = NULL;
  AVCodec *codec_in = NULL;
  AVCodecContext *codec_ctx = NULL;

  AVDictionary *options = NULL;
  av_dict_set (&options, "video_size", "640x480", 0);
  av_dict_set (&options, "framerate", "30", 0);
  //av_dict_set (&options, "pixel_format", "mjpeg", 0);
  av_dict_set (&options, "pixel_format", "yuyv422", 0);


  int ret = avformat_open_input (&ctx, "/dev/video0", input_format, &options);

  avformat_find_stream_info (ctx, NULL);

  int video_stream_id = -1;

  for (int i = 0; i < ctx->nb_streams; i++) {
	  if (ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
		  printf ("found: %d\n", i);
		  video_stream_id = i;
		  break;
	  }
  }
  self->video_stream = video_stream_id;

  printf ("codec id: %d\n", ctx->streams[video_stream_id]->codecpar->codec_id);
  codec_in = avcodec_find_decoder (ctx->streams[video_stream_id]->codecpar->codec_id);
  if (codec_in == NULL) {
	  fprintf (stderr, "decoder not found\n");
	  exit (-1);
  }

  codec_ctx = avcodec_alloc_context3 (codec_in);
  self->codec_ctx = codec_ctx;
  printf ("codec ctx: %p\n", codec_ctx);
  codec_ctx->width = 640;
  codec_ctx->height = 480;
  codec_ctx->sample_rate = 1000 / 33;
  codec_ctx->pix_fmt = AV_PIX_FMT_YUYV422;

  ret = avcodec_open2 ((AVCodecContext *) codec_ctx, codec_in, &options);
  char *buf = calloc (512, 1);
  av_strerror (ret, buf, 512);
  printf ("avcodec open2: %d %s\n", ret, buf);
  free (buf);

  AVFrame *frame = NULL;
  AVFrame *frame_rgb = NULL;

  frame = av_frame_alloc ( );
  frame_rgb = av_frame_alloc ( );

  self->frame = frame;
  self->frame_rgb = frame_rgb;

  uint8_t *buffer = NULL;

  int numbytes;

  printf ("codec_ctx->width: %d\n", codec_ctx->width);
  printf ("codec_ctx->height: %d\n", codec_ctx->height);

  numbytes = av_image_get_buffer_size (AV_PIX_FMT_RGB24, codec_ctx->width, codec_ctx->height, 1);
  self->num = numbytes;
  self->buffer = (uint8_t *) av_malloc (numbytes);



  self->ctx = ctx;
#if 1
  self->sws = sws_getContext (codec_ctx->width, 
		  codec_ctx->height,
		  codec_ctx->pix_fmt,
		  codec_ctx->width,
		  codec_ctx->height,
		  AV_PIX_FMT_RGB24,
		  SWS_BILINEAR,
		  NULL,
		  NULL,
		  NULL
		  );
#endif

  g_signal_connect (self->area, "draw", G_CALLBACK (drawing_render_cb), self);
  gtk_widget_set_size_request (self->area, 640, 480);
  gtk_widget_set_visible (self->area, TRUE);

  gtk_container_add (GTK_CONTAINER (self), self->area);

  //g_idle_add (timeout_frame, self);
  g_timeout_add ( 1000 / codec_ctx->sample_rate, timeout_frame, self);
}


Я вроде правильно делаю всё, но не работает как надо. процесс конвертирования выполняется с помощью sws_ctx. но он как то странно это делает.
  • Вопрос задан
  • 116 просмотров
Решения вопроса 1
@xverizex Автор вопроса
всё, я смог сделать. теперь надо поработать над утечками памяти, если они есть.
вот код.
/* dating_chat-window.c
 *
 * Copyright 2021 cf
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "dating-chat-window.h"
#include <libavdevice/avdevice.h>
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
#include <libavutil/avutil.h>
#include <libavutil/imgutils.h>
#include <libavutil/pixfmt.h>

struct _DatingChatWindow
{
  GtkApplicationWindow  parent_instance;

  AVFormatContext *ctx;
  AVCodecContext *codec_ctx;
  struct SwsContext *sws;
  int video_stream;
  AVFrame *frame;
  AVFrame *frame_rgb;
  uint8_t *buffer;
  uint8_t *b;
  int num;
  /* Template widgets */
  GtkWidget *area;

};

G_DEFINE_TYPE (DatingChatWindow, dating_chat_window, GTK_TYPE_APPLICATION_WINDOW)

static void
dating_chat_window_class_init (DatingChatWindowClass *klass)
{
  GtkWidgetClass *widget_class = GTK_WIDGET_CLASS (klass);
  (void) widget_class;
}

gboolean
drawing_render_cb (GtkWidget *widget, cairo_t *cr, gpointer data) {
	DatingChatWindow *self = (DatingChatWindow *) data;

	AVPacket packet;
	int frame_finished;

	av_read_frame (self->ctx, &packet);

	if (packet.stream_index == self->video_stream) {
		static int a = 1;
		avcodec_send_packet (self->codec_ctx, &packet);
		avcodec_receive_frame (self->codec_ctx, self->frame);
		int width, height;
		gtk_widget_get_size_request (self->area, &width, &height);

		uint8_t *rgb[4] = { self->buffer, 0, 0, 0 };
		int rgb_stride[4] = { self->codec_ctx->width * 3, 0, 0, 0 };
		memset ( self->buffer, 255, self->num);

		self->sws = sws_getCachedContext (
				self->sws,
				width,
				height,
				AV_PIX_FMT_YUYV422,
				width,
				height,
				AV_PIX_FMT_RGB24,
				SWS_BICUBIC,
				0,
				0,
				0
				);

		sws_scale (
				self->sws,
				(const uint8_t * const *) self->frame->data,
				self->frame->linesize,
				0,
				height,
				rgb,
				rgb_stride
			  );

		int ind = 0;
		for (int i = 0; i < self->num;) {
			self->b[ind + 0] = self->buffer[i + 2];
			self->b[ind + 1] = self->buffer[i + 1];
			self->b[ind + 2] = self->buffer[i + 0];
			self->b[ind + 3] = 255;
			ind += 4;
			i += 3;
		}

		int stride = cairo_format_stride_for_width (CAIRO_FORMAT_RGB24, width);

		cairo_surface_t *surface = cairo_image_surface_create_for_data (
				self->b,
				CAIRO_FORMAT_RGB24,
				self->codec_ctx->width,
				self->codec_ctx->height,
				stride);

		cairo_set_source_surface (cr, surface, 0, 0);
		cairo_paint (cr);
		cairo_surface_finish (surface);

	}

	av_packet_unref (&packet);

	return FALSE;
}

gboolean
timeout_frame (gpointer data) {
	DatingChatWindow *self = (DatingChatWindow *) data;

	gtk_widget_queue_draw (self->area);

	return G_SOURCE_CONTINUE;
}

static void
dating_chat_window_init (DatingChatWindow *self)
{


	self->area = gtk_drawing_area_new ( );

	

  avdevice_register_all ();
  AVInputFormat *input_format = av_find_input_format ("v4l2");
  AVFormatContext *ctx = NULL;
  AVCodec *codec_in = NULL;
  AVCodecContext *codec_ctx = NULL;

  AVDictionary *options = NULL;
  av_dict_set (&options, "video_size", "640x480", 0);
  av_dict_set (&options, "framerate", "30", 0);
  //av_dict_set (&options, "pixel_format", "mjpeg", 0);
  av_dict_set (&options, "pixel_format", "yuyv422", 0);


  int ret = avformat_open_input (&ctx, "/dev/video0", input_format, &options);

  avformat_find_stream_info (ctx, NULL);

  int video_stream_id = -1;

  for (int i = 0; i < ctx->nb_streams; i++) {
	  if (ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
		  printf ("found: %d\n", i);
		  video_stream_id = i;
		  break;
	  }
  }
  self->video_stream = video_stream_id;

  printf ("codec id: %d\n", ctx->streams[video_stream_id]->codecpar->codec_id);
  codec_in = avcodec_find_decoder (ctx->streams[video_stream_id]->codecpar->codec_id);
  if (codec_in == NULL) {
	  fprintf (stderr, "decoder not found\n");
	  exit (-1);
  }

  codec_ctx = avcodec_alloc_context3 (codec_in);
  self->codec_ctx = codec_ctx;
  printf ("codec ctx: %p\n", codec_ctx);
  codec_ctx->width = 640;
  codec_ctx->height = 480;
  codec_ctx->sample_rate = 1000 / 33;
  codec_ctx->pix_fmt = AV_PIX_FMT_YUYV422;

  ret = avcodec_open2 ((AVCodecContext *) codec_ctx, codec_in, &options);
  char *buf = calloc (512, 1);
  av_strerror (ret, buf, 512);
  printf ("avcodec open2: %d %s\n", ret, buf);
  free (buf);

  AVFrame *frame = NULL;
  AVFrame *frame_rgb = NULL;

  frame = av_frame_alloc ( );
  frame_rgb = av_frame_alloc ( );

  self->frame = frame;
  self->frame_rgb = frame_rgb;

  uint8_t *buffer = NULL;

  int numbytes;

  printf ("codec_ctx->width: %d\n", codec_ctx->width);
  printf ("codec_ctx->height: %d\n", codec_ctx->height);

  numbytes = av_image_get_buffer_size (AV_PIX_FMT_RGB24, codec_ctx->width, codec_ctx->height, 1);
  g_print ("numbytes: %d\n", numbytes);
  self->num = numbytes;
  self->buffer = (uint8_t *) av_malloc (numbytes);
  self->b = (uint8_t *) av_malloc (numbytes);



  self->ctx = ctx;
#if 1
  self->sws = sws_getContext (codec_ctx->width, 
		  codec_ctx->height,
		  AV_PIX_FMT_YUYV422,
		  codec_ctx->width,
		  codec_ctx->height,
		  AV_PIX_FMT_RGB24,
		  SWS_BICUBIC,
		  NULL,
		  NULL,
		  NULL
		  );
#endif

  g_signal_connect (self->area, "draw", G_CALLBACK (drawing_render_cb), self);
  gtk_widget_set_size_request (self->area, 640, 480);
  gtk_widget_set_visible (self->area, TRUE);

  gtk_container_add (GTK_CONTAINER (self), self->area);

  //g_idle_add (timeout_frame, self);
  g_timeout_add ( codec_ctx->sample_rate, timeout_frame, self);
}
Ответ написан
Комментировать
Пригласить эксперта
Ваш ответ на вопрос

Войдите, чтобы написать ответ

Войти через центр авторизации
Похожие вопросы