fix(ffmpeg): fix ffmpeg decoder assert (#8128)

Signed-off-by: pengyiqiang <pengyiqiang@xiaomi.com>
Co-authored-by: pengyiqiang <pengyiqiang@xiaomi.com>

docs/src/details/libs/ffmpeg.rst

@@ -19,7 +19,11 @@ For a detailed introduction, see:  https://www.ffmpeg.org
 Installing FFmpeg
 *****************
 
-Download the FFmpeg library from `its official download page
+.. code-block:: shell
+
+    sudo apt install libavformat-dev libavcodec-dev libswscale-dev libavutil-dev
+
+Or download the FFmpeg library from `its official download page
 <https://www.ffmpeg.org/download.html>`__, then install it:
 
 .. code-block:: shell

examples/libs/ffmpeg/lv_example_ffmpeg_1.c

@@ -1,6 +1,6 @@
 #include "../../lv_examples.h"
 #if LV_BUILD_EXAMPLES
-#if LV_USE_FFMPEG
+#if LV_USE_FFMPEG && LV_FFMPEG_PLAYER_USE_LV_FS
 
 /**
  * Open an image from a file
@@ -11,7 +11,7 @@ void lv_example_ffmpeg_1(void)
      *to open the image, unlike `lv_ffmpeg_player_set_src` which depends on
      *the setting of `LV_FFMPEG_PLAYER_USE_LV_FS`.*/
     lv_obj_t * img = lv_image_create(lv_screen_active());
-    lv_image_set_src(img, "./lvgl/examples/libs/ffmpeg/ffmpeg.png");
+    lv_image_set_src(img, "A:lvgl/examples/libs/ffmpeg/ffmpeg.png");
     lv_obj_center(img);
 }
 

examples/libs/ffmpeg/lv_example_ffmpeg_2.c

@@ -2,6 +2,12 @@
 #if LV_BUILD_EXAMPLES
 #if LV_USE_FFMPEG
 
+#if LV_FFMPEG_PLAYER_USE_LV_FS
+    #define PATH_PREFIX "A:"
+#else
+    #define PATH_PREFIX "./"
+#endif
+
 /**
  * Open a video from a file
  */
@@ -12,7 +18,7 @@ void lv_example_ffmpeg_2(void)
     /*It will use the LVGL filesystem abstraction (not the OS filesystem)
      *if `LV_FFMPEG_PLAYER_USE_LV_FS` is set.*/
     lv_obj_t * player = lv_ffmpeg_player_create(lv_screen_active());
-    lv_ffmpeg_player_set_src(player, "./lvgl/examples/libs/ffmpeg/birds.mp4");
+    lv_ffmpeg_player_set_src(player, PATH_PREFIX "lvgl/examples/libs/ffmpeg/birds.mp4");
     lv_ffmpeg_player_set_auto_restart(player, true);
     lv_ffmpeg_player_set_cmd(player, LV_FFMPEG_PLAYER_CMD_START);
     lv_obj_center(player);

scripts/install-prerequisites.sh

@@ -13,5 +13,7 @@ sudo apt install gcc gcc-multilib g++-multilib ninja-build \
     libpng-dev:i386 libjpeg-dev:i386 libfreetype6-dev:i386 \
     ruby-full gcovr cmake  python3 libinput-dev libxkbcommon-dev \
     libdrm-dev pkg-config wayland-protocols libwayland-dev libwayland-bin \
-    libwayland-dev:i386 libxkbcommon-dev:i386 libudev-dev
+    libwayland-dev:i386 libxkbcommon-dev:i386 libudev-dev \
+    libavformat-dev libavcodec-dev libswscale-dev libavutil-dev \
+    libavformat-dev:i386 libavcodec-dev:i386 libswscale-dev:i386 libavutil-dev:i386
 pip3 install pypng lz4 kconfiglib

src/libs/ffmpeg/lv_ffmpeg.c

@@ -9,6 +9,7 @@
 #include "lv_ffmpeg_private.h"
 #if LV_USE_FFMPEG != 0
 #include "../../draw/lv_image_decoder_private.h"
+#include "../../draw/lv_draw_buf_private.h"
 #include "../../core/lv_obj_class_private.h"
 
 #include <libavcodec/avcodec.h>
@@ -60,6 +61,7 @@ struct ffmpeg_context_s {
     enum AVPixelFormat video_dst_pix_fmt;
     bool has_alpha;
     lv_draw_buf_t draw_buf;
+    lv_draw_buf_handlers_t draw_buf_handlers;
 };
 
 #pragma pack(1)
@@ -127,11 +129,22 @@ void lv_ffmpeg_init(void)
 
     dec->name = DECODER_NAME;
 
-#if LV_FFMPEG_AV_DUMP_FORMAT == 0
+#if LV_FFMPEG_DUMP_FORMAT == 0
     av_log_set_level(AV_LOG_QUIET);
 #endif
 }
 
+void lv_ffmpeg_deinit(void)
+{
+    lv_image_decoder_t * dec = NULL;
+    while((dec = lv_image_decoder_get_next(dec)) != NULL) {
+        if(dec->info_cb == decoder_info) {
+            lv_image_decoder_delete(dec);
+            break;
+        }
+    }
+}
+
 int lv_ffmpeg_get_frame_num(const char * path)
 {
     int ret = -1;
@@ -169,26 +182,25 @@ lv_result_t lv_ffmpeg_player_set_src(lv_obj_t * obj, const char * path)
     player->ffmpeg_ctx = ffmpeg_open_file(path, LV_FFMPEG_PLAYER_USE_LV_FS);
 
     if(!player->ffmpeg_ctx) {
-        LV_LOG_ERROR("ffmpeg file open failed: %s", path);
         goto failed;
     }
 
     if(ffmpeg_image_allocate(player->ffmpeg_ctx) < 0) {
         LV_LOG_ERROR("ffmpeg image allocate failed");
         ffmpeg_close(player->ffmpeg_ctx);
+        player->ffmpeg_ctx = NULL;
         goto failed;
     }
 
     bool has_alpha = player->ffmpeg_ctx->has_alpha;
     int width = player->ffmpeg_ctx->video_dec_ctx->width;
     int height = player->ffmpeg_ctx->video_dec_ctx->height;
-    uint32_t data_size = 0;
 
-    data_size = width * height * 4;
     uint8_t * data = ffmpeg_get_image_data(player->ffmpeg_ctx);
     lv_color_format_t cf = has_alpha ? LV_COLOR_FORMAT_ARGB8888 : LV_COLOR_FORMAT_NATIVE;
     uint32_t stride = width * lv_color_format_get_size(cf);
-    lv_memzero(data, stride * height);
+    uint32_t data_size = stride * height;
+    lv_memzero(data, data_size);
 
     player->imgdsc.header.w = width;
     player->imgdsc.header.h = height;
@@ -322,11 +334,24 @@ static lv_result_t decoder_open(lv_image_decoder_t * decoder, lv_image_decoder_d
 
         dsc->user_data = ffmpeg_ctx;
         lv_draw_buf_t * decoded = &ffmpeg_ctx->draw_buf;
-        decoded->header = dsc->header;
-        decoded->header.flags |= LV_IMAGE_FLAGS_MODIFIABLE;
-        decoded->data = img_data;
-        decoded->data_size = (uint32_t)dsc->header.stride * dsc->header.h;
-        decoded->unaligned_data = NULL;
+        lv_draw_buf_init(
+            decoded,
+            dsc->header.w,
+            dsc->header.h,
+            dsc->header.cf,
+            dsc->header.stride,
+            img_data,
+            dsc->header.stride * dsc->header.h);
+        lv_draw_buf_set_flag(decoded, LV_IMAGE_FLAGS_MODIFIABLE);
+
+        /* Empty handlers to avoid decoder asserts */
+        lv_draw_buf_handlers_init(&ffmpeg_ctx->draw_buf_handlers, NULL, NULL, NULL, NULL, NULL, NULL);
+        decoded->handlers = &ffmpeg_ctx->draw_buf_handlers;
+
+        if(dsc->args.premultiply && ffmpeg_ctx->has_alpha) {
+            lv_draw_buf_premultiply(decoded);
+        }
+
         dsc->decoded = decoded;
 
         /* The image is fully decoded. Return with its pointer */
@@ -409,8 +434,6 @@ static int ffmpeg_output_video_frame(struct ffmpeg_context_s * ffmpeg_ctx)
         goto failed;
     }
 
-    LV_LOG_TRACE("video_frame coded_n:%d", frame->coded_picture_number);
-
     /* copy decoded frame to destination buffer:
      * this is required since rawvideo expects non aligned data
      */
@@ -732,15 +755,20 @@ static struct ffmpeg_context_s * ffmpeg_open_file(const char * path, bool is_lv_
         return NULL;
     }
 
-    struct ffmpeg_context_s * ffmpeg_ctx = calloc(1, sizeof(struct ffmpeg_context_s));
-
+    struct ffmpeg_context_s * ffmpeg_ctx = lv_malloc_zeroed(sizeof(struct ffmpeg_context_s));
+    LV_ASSERT_MALLOC(ffmpeg_ctx);
     if(ffmpeg_ctx == NULL) {
         LV_LOG_ERROR("ffmpeg_ctx malloc failed");
         goto failed;
     }
 
     if(is_lv_fs_path) {
-        lv_fs_open(&(ffmpeg_ctx->lv_file), path, LV_FS_MODE_RD);    /* image_decoder_get_info says the file truly exists. */
+        const lv_fs_res_t fs_res = lv_fs_open(&(ffmpeg_ctx->lv_file), path, LV_FS_MODE_RD);
+        if(fs_res != LV_FS_RES_OK) {
+            LV_LOG_WARN("Could not open file: %s, res: %d", path, fs_res);
+            lv_free(ffmpeg_ctx);
+            return NULL;
+        }
 
         ffmpeg_ctx->io_ctx = ffmpeg_open_io_context(&(ffmpeg_ctx->lv_file));     /* Save the buffer pointer to free it later */
 
@@ -784,7 +812,7 @@ static struct ffmpeg_context_s * ffmpeg_open_file(const char * path, bool is_lv_
         ffmpeg_ctx->video_dst_pix_fmt = (ffmpeg_ctx->has_alpha ? AV_PIX_FMT_BGRA : AV_PIX_FMT_TRUE_COLOR);
     }
 
-#if LV_FFMPEG_AV_DUMP_FORMAT != 0
+#if LV_FFMPEG_DUMP_FORMAT
     /* dump input information to stderr */
     av_dump_format(ffmpeg_ctx->fmt_ctx, 0, path, 0);
 #endif
@@ -891,7 +919,7 @@ static void ffmpeg_close(struct ffmpeg_context_s * ffmpeg_ctx)
         av_free(ffmpeg_ctx->io_ctx);
         lv_fs_close(&(ffmpeg_ctx->lv_file));
     }
-    free(ffmpeg_ctx);
+    lv_free(ffmpeg_ctx);
 
     LV_LOG_INFO("ffmpeg_ctx closed");
 }

src/libs/ffmpeg/lv_ffmpeg.h

@@ -44,6 +44,11 @@ typedef enum {
  */
 void lv_ffmpeg_init(void);
 
+/**
+ * De-initialize FFMPEG image decoder
+ */
+void lv_ffmpeg_deinit(void);
+
 /**
  * Get the number of frames contained in the file
  * @param path image or video file name

src/lv_init.c

@@ -357,6 +357,11 @@ void lv_init(void)
     lv_fs_uefi_init();
 #endif
 
+    /*Use the earlier initialized position of FFmpeg decoder as a fallback decoder*/
+#if LV_USE_FFMPEG
+    lv_ffmpeg_init();
+#endif
+
 #if LV_USE_LODEPNG
     lv_lodepng_init();
 #endif
@@ -381,12 +386,6 @@ void lv_init(void)
     lv_svg_decoder_init();
 #endif
 
-    /*Make FFMPEG last because the last converter will be checked first and
-     *it's superior to any other */
-#if LV_USE_FFMPEG
-    lv_ffmpeg_init();
-#endif
-
 #if LV_USE_XML
     lv_xml_init();
 #endif

tests/CMakeLists.txt

@@ -358,6 +358,21 @@ endif()
 find_package(PNG REQUIRED)
 include_directories(${PNG_INCLUDE_DIR})
 
+if(NOT WIN32)
+    # FFmpeg is optional for the image and video test cases
+    find_package(PkgConfig)
+    pkg_check_modules(FFMPEG libavformat libavcodec libswscale libavutil)
+    if(FFMPEG_FOUND)
+        include_directories(${FFMPEG_INCLUDE_DIRS})
+    else()
+        message("FFmpeg not found, defaulting to 0")
+        add_definitions(-DLV_USE_FFMPEG=0)
+    endif()
+else()
+    message("Disable FFmpeg for Windows")
+    add_definitions(-DLV_USE_FFMPEG=0)
+endif()
+
 # libfreetype is required for the font test case
 find_package(Freetype REQUIRED)
 include_directories(${FREETYPE_INCLUDE_DIRS})
@@ -479,6 +494,7 @@ foreach( test_case_fname ${TEST_CASE_FILES} )
             ${LIBDRM_LIBRARIES}
             ${LIBINPUT_LIBRARIES}
             ${JPEG_LIBRARIES}
+            ${FFMPEG_LIBRARIES}
             m
             pthread
             ${TEST_LIBS})

tests/src/lv_test_conf_full.h

@@ -79,7 +79,12 @@
 #define LV_USE_BMP          1
 #define LV_USE_TJPGD        1
 #ifndef _WIN32
-    #define LV_USE_LIBJPEG_TURBO   1
+    #define LV_USE_LIBJPEG_TURBO       1
+#endif
+#ifndef LV_USE_FFMPEG
+    #define LV_USE_FFMPEG              1
+    #define LV_FFMPEG_DUMP_FORMAT      1
+    #define LV_FFMPEG_PLAYER_USE_LV_FS 1
 #endif
 #define LV_USE_GIF          1
 #define LV_USE_QRCODE       1

tests/src/test_cases/libs/test_ffmpeg.c

@@ -0,0 +1,146 @@
+#if LV_BUILD_TEST
+#include "../lvgl.h"
+#include "../../lvgl_private.h"
+
+#include "unity/unity.h"
+
+#if LV_USE_FFMPEG
+
+void setUp(void)
+{
+    /* Function run before every test */
+}
+
+void tearDown(void)
+{
+    lv_obj_clean(lv_screen_active());
+}
+
+static void create_image_item(lv_obj_t * parent, const void * src, const char * text)
+{
+    lv_obj_t * cont = lv_obj_create(parent);
+    lv_obj_remove_style_all(cont);
+    lv_obj_set_size(cont, 300, 200);
+    lv_obj_set_flex_flow(cont, LV_FLEX_FLOW_COLUMN);
+    lv_obj_set_flex_align(cont, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER);
+
+    lv_obj_t * img = lv_image_create(cont);
+    lv_image_set_src(img, src);
+
+    lv_obj_t * label = lv_label_create(cont);
+    lv_label_set_text(label, text);
+}
+
+static void create_images(void)
+{
+    lv_obj_t * screen = lv_screen_active();
+    lv_obj_clean(screen);
+    lv_obj_set_flex_flow(screen, LV_FLEX_FLOW_ROW_WRAP);
+    lv_obj_set_flex_align(screen, LV_FLEX_ALIGN_SPACE_AROUND, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER);
+
+    create_image_item(screen, "A:src/test_assets/test_img_lvgl_logo.png", "PNG File (32 bit)");
+    create_image_item(screen, "A:src/test_assets/test_img_lvgl_logo_png_no_ext", "PNG File (32 bit) No Extension");
+    create_image_item(screen, "A:src/test_assets/test_img_lvgl_logo_8bit_palette.png", "PNG File (8 bit palette)");
+}
+
+void test_ffmpeg_image_decoder_1(void)
+{
+    /* Temporarily remove other decoder */
+#if LV_USE_LODEPNG
+    lv_lodepng_deinit();
+#endif
+
+#if LV_USE_LIBPNG
+    lv_libpng_deinit();
+#endif
+
+    create_images();
+
+    /* Should decode images consistently with other PNG decoders  */
+    TEST_ASSERT_EQUAL_SCREENSHOT("libs/ffmpeg_1.png");
+
+    size_t mem_before = lv_test_get_free_mem();
+    for(uint32_t i = 0; i < 20; i++) {
+        create_images();
+
+        lv_obj_invalidate(lv_screen_active());
+        lv_refr_now(NULL);
+    }
+
+    TEST_ASSERT_EQUAL_SCREENSHOT("libs/ffmpeg_1.png");
+
+    TEST_ASSERT_MEM_LEAK_LESS_THAN(mem_before, 32);
+
+    /* Re-add other decoder */
+#if LV_USE_LODEPNG
+    lv_lodepng_init();
+#endif
+
+#if LV_USE_LIBPNG
+    lv_libpng_init();
+#endif
+
+    lv_obj_clean(lv_screen_active());
+}
+
+void test_ffmpeg_player_1(void)
+{
+    lv_obj_t * player = lv_ffmpeg_player_create(lv_screen_active());
+    lv_ffmpeg_player_set_auto_restart(player, true);
+    lv_obj_center(player);
+
+    lv_ffmpeg_player_set_src(player, "A:ERROR_FILE.mp4");
+    lv_ffmpeg_player_set_cmd(player, LV_FFMPEG_PLAYER_CMD_START);
+    TEST_ASSERT_EQUAL_SCREENSHOT("libs/ffmpeg_player_error_file.png");
+
+    /* Video: test_video_birds.mp4 Update frame rate 25FPS */
+    lv_ffmpeg_player_set_src(player, "A:src/test_assets/test_video_birds.mp4");
+
+    /* Not started, it should be in the black screen  */
+    TEST_ASSERT_EQUAL_SCREENSHOT("libs/ffmpeg_player_frame_0.png");
+
+    lv_ffmpeg_player_set_cmd(player, LV_FFMPEG_PLAYER_CMD_START);
+
+    lv_test_wait(400);
+
+    TEST_ASSERT_EQUAL_SCREENSHOT("libs/ffmpeg_player_frame_1.png");
+
+    lv_test_wait(400);
+
+    TEST_ASSERT_EQUAL_SCREENSHOT("libs/ffmpeg_player_frame_2.png");
+
+    lv_ffmpeg_player_set_cmd(player, LV_FFMPEG_PLAYER_CMD_PAUSE);
+
+    lv_test_wait(400);
+
+    /* Paused, it should be in the same frame  */
+    TEST_ASSERT_EQUAL_SCREENSHOT("libs/ffmpeg_player_frame_2.png");
+
+    lv_ffmpeg_player_set_cmd(player, LV_FFMPEG_PLAYER_CMD_RESUME);
+
+    TEST_ASSERT_EQUAL_SCREENSHOT("libs/ffmpeg_player_frame_3.png");
+
+    lv_obj_delete(player);
+}
+
+#else
+
+void setUp(void)
+{
+}
+
+void tearDown(void)
+{
+}
+
+void test_ffmpeg_image_decoder_1(void)
+{
+}
+
+void test_ffmpeg_player_1(void)
+{
+}
+
+#endif /* LV_USE_FFMPEG */
+
+#endif /* LV_BUILD_TEST */