HandBrake · dashsantosh-mcw · Jul 16, 2025 · galad87 · Aug 7, 2025 · dashsantosh-mcw
diff --git a/contrib/ffmpeg/A23-fix-d3d11-static-pool-size-error.patch b/contrib/ffmpeg/A23-fix-d3d11-static-pool-size-error.patch
@@ -0,0 +1,74 @@
+diff --git a/libavutil/hwcontext_d3d11va.c b/libavutil/hwcontext_d3d11va.c
+index 9831f530c1..dac659ea9a 100644
+--- a/libavutil/hwcontext_d3d11va.c
++++ b/libavutil/hwcontext_d3d11va.c
+@@ -85,6 +85,9 @@ typedef struct D3D11VAFramesContext {
+     int nb_surfaces;
+     int nb_surfaces_used;
+
++    int max_retries;
++    int retries;
++
+     DXGI_FORMAT format;
+
+     ID3D11Texture2D *staging_texture;
+@@ -260,8 +263,10 @@ static AVBufferRef *d3d11va_pool_alloc(void *opaque, size_t size)
+     ID3D11Texture2D_GetDesc(hwctx->texture, &texDesc);
+
+     if (s->nb_surfaces_used >= texDesc.ArraySize) {
+-        av_log(ctx, AV_LOG_ERROR, "Static surface pool size exceeded.\n");
+-        return NULL;
++        if (s->retries >= s->max_retries) {
++            av_log(ctx, AV_LOG_ERROR, "Static surface pool size exceeded.\n");
++        }
++        return NULL;
+     }
+
+     ID3D11Texture2D_AddRef(hwctx->texture);
+@@ -343,20 +348,34 @@ static int d3d11va_frames_init(AVHWFramesContext *ctx)
+ static int d3d11va_get_buffer(AVHWFramesContext *ctx, AVFrame *frame)
+ {
+     AVD3D11FrameDescriptor *desc;
++    D3D11VAFramesContext       *s = ctx->hwctx;
++    s->retries = 0;
++    s->max_retries = 50;
+
+-    frame->buf[0] = av_buffer_pool_get(ctx->pool);
+-    if (!frame->buf[0])
+-        return AVERROR(ENOMEM);
+-
+-    desc = (AVD3D11FrameDescriptor *)frame->buf[0]->data;
+-
+-    frame->data[0] = (uint8_t *)desc->texture;
+-    frame->data[1] = (uint8_t *)desc->index;
+-    frame->format  = AV_PIX_FMT_D3D11;
+-    frame->width   = ctx->width;
+-    frame->height  = ctx->height;
++    /**
++     * Loop until a buffer becomes available from the pool.
++     * In a full hardware pipeline, all buffers may be temporarily in use by
++     * other modules (encoder/filter/decoder). Rather than immediately failing
++     * with ENOMEM, we wait for a buffer to be released back to the pool, which
++     * maintains pipeline flow and prevents unnecessary allocation failures
++     * during normal operation.
++     */
++    while (s->retries < s->max_retries) {
++        frame->buf[0] = av_buffer_pool_get(ctx->pool);
++        if (frame->buf[0]) {
++            desc = (AVD3D11FrameDescriptor *)frame->buf[0]->data;
++            frame->data[0] = (uint8_t *)desc->texture;
++            frame->data[1] = (uint8_t *)desc->index;
++            frame->format  = AV_PIX_FMT_D3D11;
++            frame->width   = ctx->width;
++            frame->height  = ctx->height;
++            return 0;
++        }
+
+-    return 0;
++        av_usleep(1000);
++    }
++ 
++    return AVERROR(ENOMEM);
+ }
+
+ static int d3d11va_transfer_get_formats(AVHWFramesContext *ctx,
diff --git a/contrib/ffmpeg/module.defs b/contrib/ffmpeg/module.defs
@@ -65,6 +65,7 @@ FFMPEG.CONFIGURE.extra = \
     --enable-filter=crop \
     --enable-filter=scale \
     --enable-filter=zscale \
+    --enable-filter=scale_d3d11 \
     --enable-filter=pad \
     --enable-filter=transpose \
     --enable-filter=hflip \

diff --git a/libhb/cropscale.c b/libhb/cropscale.c
@@ -132,6 +132,12 @@ static int crop_scale_init(hb_filter_object_t * filter, hb_filter_init_t * init)
             hb_dict_set_string(avsettings, "format", av_get_pix_fmt_name(init->pix_fmt));
             hb_dict_set(avfilter, "scale_cuda", avsettings);
         }
+        else if (init->hw_pix_fmt == AV_PIX_FMT_D3D11)
+        {
+            hb_dict_set_int(avsettings, "width", width);
+            hb_dict_set_int(avsettings, "height", height);
+            hb_dict_set(avfilter, "scale_d3d11", avsettings);
+        }
         else if (hb_av_can_use_zscale(init->pix_fmt,
                                       init->geometry.width, init->geometry.height,
                                       width, height))

diff --git a/libhb/decavcodec.c b/libhb/decavcodec.c
@@ -1469,6 +1469,13 @@ int reinit_video_filters(hb_work_private_t * pv)
             hb_dict_set(settings, "format", hb_value_string(av_get_pix_fmt_name(pv->job->input_pix_fmt)));
             hb_avfilter_append_dict(filters, "scale_cuda", settings);
         }
+        else if (pv->frame->hw_frames_ctx && pv->job->hw_pix_fmt == AV_PIX_FMT_D3D11)
+        {
+            hb_dict_set(settings, "width", hb_value_int(orig_width));
+            hb_dict_set(settings, "height", hb_value_int(orig_height));
+            hb_dict_set(settings, "format", hb_value_string(av_get_pix_fmt_name(pv->job->input_pix_fmt)));
+            hb_avfilter_append_dict(filters, "scale_d3d11", settings);
+        }
         else if (hb_av_can_use_zscale(pv->frame->format,
                                       pv->frame->width, pv->frame->height,
                                       orig_width, orig_height))
@@ -1898,6 +1905,13 @@ static int decavcodecvInit( hb_work_object_t * w, hb_job_t * job )
         }
 #endif
 
+#if HB_PROJECT_FEATURE_MF
+        if (w->hw_accel && w->hw_accel->type == AV_HWDEVICE_TYPE_D3D11VA)
+        {
+           pv->context->extra_hw_frames = 30;
+        }
+#endif
+
         if ( hb_avcodec_open( pv->context, pv->codec, &av_opts, pv->threads ) )
         {
             av_dict_free( &av_opts );

diff --git a/libhb/format.c b/libhb/format.c
@@ -65,6 +65,13 @@ static int format_init(hb_filter_object_t *filter, hb_filter_init_t *init)
         hb_dict_set(avfilter, "vpp_qsv", avsettings);
     }
     else
+#elif HB_PROJECT_FEATURE_MF
+    if (init->hw_pix_fmt == AV_PIX_FMT_D3D11)
+    {
+        hb_dict_set_string(avsettings, "format", format);
+        hb_dict_set(avfilter, "scale_d3d11", avsettings);
+    }
+    else
 #endif
     {
         if (init->hw_pix_fmt == AV_PIX_FMT_CUDA)

diff --git a/libhb/handbrake/common.h b/libhb/handbrake/common.h
@@ -237,8 +237,10 @@ struct hb_encoder_s
 
 #ifdef __LIBHB__
 
-#define HB_HWACCEL_CAP_SCAN   0x1
-#define HB_HWACCEL_CAP_ROTATE 0x2
+#define HB_HWACCEL_CAP_SCAN                 0x01
+#define HB_HWACCEL_CAP_ROTATE               0x02
+#define HB_HWACCEL_CAP_FORMAT_REQUIRED      0x04
+
 
 struct hb_hwaccel_s
 {

diff --git a/libhb/handbrake/mf_common.h b/libhb/handbrake/mf_common.h
@@ -34,5 +34,6 @@ int hb_mf_h264_available();
 int hb_mf_h265_available();
 int hb_mf_av1_available();
 int hb_check_mf_available();
+int hb_mf_are_filters_supported(hb_list_t *filters);
 
 #endif // HANDBRAKE_MF_COMMON_H
diff --git a/libhb/hbavfilter.c b/libhb/hbavfilter.c
@@ -345,6 +345,22 @@ void hb_avfilter_combine( hb_list_t * list)
                 }
             }
             else
+#endif
+#if HB_PROJECT_FEATURE_MF
+            // Concat d3d11 settings as one scale_d3d11 filter to optimize pipeline
+            hb_dict_t * avfilter_settings_dict = hb_value_array_get(avfilter->settings, 0);
+            hb_dict_t * cur_settings_dict = hb_value_array_get(settings, 0);
+            if (cur_settings_dict && avfilter_settings_dict && hb_dict_get(avfilter_settings_dict, "scale_d3d11"))
+            {
+                hb_dict_t *avfilter_settings_dict_d3d11 = hb_dict_get(avfilter_settings_dict, "scale_d3d11");
+                hb_dict_t *cur_settings_dict_d3d11 = hb_dict_get(cur_settings_dict, "scale_d3d11");
+                if (avfilter_settings_dict_d3d11 && cur_settings_dict_d3d11)
+                {
+                    hb_dict_merge(avfilter_settings_dict_d3d11, cur_settings_dict_d3d11);
+
+                }
+            }
+            else
 #endif
             {
                 hb_value_array_concat(avfilter->settings, settings);

diff --git a/libhb/mf_common.c b/libhb/mf_common.c
@@ -223,8 +223,64 @@ int hb_directx_available()
     return 1;
 }
 
+int hb_mf_are_filters_supported(hb_list_t *filters)
+{
+    if (filters == NULL || hb_list_count(filters) == 0)
+    {
+        hb_log("D3D11: No filters to process");
+        return 1;
+    }
+
+    for (int i = 0; i < hb_list_count(filters); i++)
+    {
+        hb_filter_object_t *filter = hb_list_item(filters, i);
+        if (filter == NULL)
+        {
+            hb_log("D3D11: Null filter object encountered");
+            return 0;
+        }
+
+        switch (filter->id)
+        {
+            case HB_FILTER_CROP_SCALE:
+                hb_log("D3D11: Scaling filter supported");
+                break;
+            case HB_FILTER_FORMAT:
+                hb_log("D3D11: Format supported");
+                break;
+
+            case HB_FILTER_AVFILTER:
+                hb_log("D3D11: AVFilter filter supported");
+                break;
+
+            case HB_FILTER_VFR:
+                {
+                    int mode = hb_dict_get_int(filter->settings, "mode");
+                    hb_log("Checking VFR mode: %d", mode);
+                    if (mode == 2)
+                    {
+                        hb_log("D3D11: Unsupported VFR mode %d detected", mode);
+                        return 0;
+                    }
+                    hb_log("D3D11: VFR mode %d supported", mode);
+                    continue;
+                }
+
+            default:
+                hb_log("D3D11: Unsupported filter %s (id: %d)", filter->name, filter->id);
+                return 0;
+        }
+    }
+
+    hb_log("D3D11: All filters are supported");
+    return 1;
+}
+
 static const int mf_encoders[] =
 {
+    HB_VCODEC_FFMPEG_MF_H264,
+    HB_VCODEC_FFMPEG_MF_H265,
+    HB_VCODEC_FFMPEG_MF_AV1,
     HB_VCODEC_INVALID
 };
 
@@ -235,7 +291,8 @@ hb_hwaccel_t hb_hwaccel_mf =
     .encoders   = mf_encoders,
     .type       = AV_HWDEVICE_TYPE_D3D11VA,
     .hw_pix_fmt = AV_PIX_FMT_D3D11,
-    .caps       = HB_HWACCEL_CAP_SCAN
+    .can_filter = hb_mf_are_filters_supported,
+    .caps       = HB_HWACCEL_CAP_SCAN | HB_HWACCEL_CAP_FORMAT_REQUIRED
 };
 
 #else // HB_PROJECT_FEATURE_MF

diff --git a/libhb/work.c b/libhb/work.c
@@ -1455,7 +1455,10 @@ static void sanitize_filter_list_post(hb_job_t *job)
     }
 #endif
 
-    if (hb_video_encoder_pix_fmt_is_supported(job->vcodec, job->input_pix_fmt, job->encoder_profile) == 0)
+    hb_hwaccel_t *hwaccel = hb_get_hwaccel(job->hw_decode);
+
+    if (hb_video_encoder_pix_fmt_is_supported(job->vcodec, job->input_pix_fmt, job->encoder_profile) == 0 ||
+        (job->hw_pix_fmt != AV_PIX_FMT_NONE && hwaccel && (hwaccel->caps & HB_HWACCEL_CAP_FORMAT_REQUIRED)))
     {
         // Some encoders require a specific input pixel format
         // that could be different from the current pipeline format.