From 088809721d6dcbae6d63e23b28873ae96243c45e Mon Sep 17 00:00:00 2001
From: Eugene Zemtsov <eugene@chromium.org>
Date: Wed, 01 Apr 2026 12:48:27 -0700
Subject: [PATCH] media: Fix OpenH264 encoder silent failure on oversized frames

OpenH264 silently fails during preprocessing when a frame's area
exceeds its internal macroblock limit (MAX_MBS_PER_FRAME). This
can result in the encoder reading and encoding uninitialized memory.

This change updates NeedsManualResizing() to force manual downscaling
for frames that exceed this limit before passing them to OpenH264.

Bug: 497952533
Change-Id: I5e042ac380ee79be5a7a6e515ccc88948cf19c26
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/7718887
Commit-Queue: Eugene Zemtsov <eugene@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1608727}
---

diff --git a/media/video/openh264_video_encoder.cc b/media/video/openh264_video_encoder.cc
index c29a9fce..f83ab4a 100644
--- a/media/video/openh264_video_encoder.cc
+++ b/media/video/openh264_video_encoder.cc
@@ -148,6 +148,18 @@
   }
 }
 
+// OpenH264 silently fails during preprocessing when a frame's area
+// exceeds its internal macroblock limit (MAX_MBS_PER_FRAME in
+// third_party/openh264). We must manually resize such frames.
+// MAX_MBS_PER_FRAME is 36864.
+constexpr int kOpenH264MaxMBs = 36864;
+
+bool IsFrameSizeTooLarge(const gfx::Size& frame_size) {
+  int mb_width = (frame_size.width() + 15) / 16;
+  int mb_height = (frame_size.height() + 15) / 16;
+  return mb_width * mb_height > kOpenH264MaxMBs;
+}
+
 // OpenH264 can resize frames automatically as long as
 // - the input and output aspect ratios are the same and
 // - the input is larger than the output in both dimensions.
@@ -156,6 +168,10 @@
     return true;
   }
 
+  if (IsFrameSizeTooLarge(src)) {
+    return true;
+  }
+
   if (dst.width() > src.width() || dst.height() > src.height()) {
     return true;
   }
@@ -249,6 +265,14 @@
                       "Unsupported frame size which is less than 16"));
     return;
   }
+
+  if (IsFrameSizeTooLarge(options.frame_size)) {
+    std::move(done_cb).Run(EncoderStatus(
+        EncoderStatus::Codes::kEncoderUnsupportedConfig,
+        "Configured frame size exceeds OpenH264 max macroblocks"));
+    return;
+  }
+
   SetUpOpenH264Params(
       profile_, options,
       VideoColorSpace::FromGfxColorSpace(last_frame_color_space_), &params);
@@ -514,6 +538,20 @@
     return;
   }
 
+  if (options.frame_size.width() < 16 || options.frame_size.height() < 16) {
+    std::move(done_cb).Run(
+        EncoderStatus(EncoderStatus::Codes::kEncoderUnsupportedConfig,
+                      "Unsupported frame size which is less than 16"));
+    return;
+  }
+
+  if (IsFrameSizeTooLarge(options.frame_size)) {
+    std::move(done_cb).Run(EncoderStatus(
+        EncoderStatus::Codes::kEncoderUnsupportedConfig,
+        "Configured frame size exceeds OpenH264 max macroblocks"));
+    return;
+  }
+
   SEncParamExt params = {};
   if (int err = codec_->GetDefaultParams(&params)) {
     std::move(done_cb).Run(
diff --git a/media/video/software_video_encoder_test.cc b/media/video/software_video_encoder_test.cc
index 3e879b8..fc86111f 100644
--- a/media/video/software_video_encoder_test.cc
+++ b/media/video/software_video_encoder_test.cc
@@ -1372,6 +1372,79 @@
   EXPECT_EQ(outputs_count, 3);
 }
 
+TEST_P(H264VideoEncoderTest, OversizedFrameSilentFailure) {
+  VideoEncoder::Options options = CreateDefaultOptions();
+  options.frame_size = gfx::Size(1024, 1024);
+  options.bitrate = Bitrate::ConstantBitrate(1000000u);
+  options.framerate = 25;
+  if (codec_ == VideoCodec::kH264) {
+    options.avc.produce_annexb = true;
+  }
+
+  int total_decoded_frames = 0;
+
+  scoped_refptr<VideoFrame> reference_frame;
+
+  VideoEncoder::OutputCB encoder_output_cb = base::BindLambdaForTesting(
+      [&, this](VideoEncoderOutput output,
+                std::optional<VideoEncoder::CodecDescription> desc) {
+        auto buffer = DecoderBuffer::FromArray(std::move(output.data));
+        buffer->set_timestamp(output.timestamp);
+        buffer->set_is_key_frame(output.key_frame);
+        decoder_->Decode(std::move(buffer), DecoderStatusCB());
+      });
+
+  VideoDecoder::OutputCB decoder_output_cb =
+      base::BindLambdaForTesting([&](scoped_refptr<VideoFrame> decoded_frame) {
+        ASSERT_TRUE(reference_frame);
+
+        EXPECT_EQ(decoded_frame->timestamp(), reference_frame->timestamp());
+        EXPECT_EQ(decoded_frame->visible_rect().size(),
+                  reference_frame->visible_rect().size());
+
+        // This validates that the encoder actually encoded our input frame,
+        // and didn't just read uninitialized memory.
+        if (decoded_frame->format() == reference_frame->format()) {
+          // Allow up to 1% of pixels to be different due to compression
+          // artifacts.
+          const int kAllowedDifferentPixels =
+              reference_frame->visible_rect().size().GetArea() / 100;
+          EXPECT_LE(CountDifferentPixels(*decoded_frame, *reference_frame, 10),
+                    kAllowedDifferentPixels);
+        }
+        ++total_decoded_frames;
+      });
+
+  PrepareDecoder(options.frame_size, std::move(decoder_output_cb));
+
+  encoder_->Initialize(profile_, options, /*info_cb=*/base::DoNothing(),
+                       std::move(encoder_output_cb),
+                       ValidateStatusThenQuitCB());
+  RunUntilQuit();
+
+  // Create frames with identical colors but vastly different sizes.
+  constexpr uint32_t kXorMask = 0x123456;
+  auto oversized_frame = CreateFrame(gfx::Size(4096, 4096), pixel_format_,
+                                     base::TimeDelta(), kXorMask);
+  ASSERT_TRUE(oversized_frame) << "Failed to allocate oversized frame";
+  oversized_frame->set_color_space(gfx::ColorSpace::CreateREC709());
+
+  reference_frame = CreateFrame(options.frame_size, pixel_format_,
+                                base::TimeDelta(), kXorMask);
+  ASSERT_TRUE(reference_frame) << "Failed to allocate reference frame";
+
+  encoder_->Encode(std::move(oversized_frame),
+                   VideoEncoder::EncodeOptions(false),
+                   ValidateStatusThenQuitCB());
+  RunUntilQuit();
+
+  encoder_->Flush(ValidateStatusThenQuitCB());
+  RunUntilQuit();
+  DecodeAndWaitForStatus(DecoderBuffer::CreateEOSBuffer());
+
+  EXPECT_EQ(total_decoded_frames, 1);
+}
+
 TEST_P(H264VideoEncoderTest, AnnexB) {
   int outputs_count = 0;
   VideoEncoder::Options options = CreateDefaultOptions();

diff --git a/media/video/software_video_encoder_test.cc b/media/video/software_video_encoder_test.cc
index 3e879b8..fc86111f 100644
--- a/media/video/software_video_encoder_test.cc
+++ b/media/video/software_video_encoder_test.cc
@@ -1372,6 +1372,79 @@
   EXPECT_EQ(outputs_count, 3);
 }
 
+TEST_P(H264VideoEncoderTest, OversizedFrameSilentFailure) {
+  VideoEncoder::Options options = CreateDefaultOptions();
+  options.frame_size = gfx::Size(1024, 1024);
+  options.bitrate = Bitrate::ConstantBitrate(1000000u);
+  options.framerate = 25;
+  if (codec_ == VideoCodec::kH264) {
+    options.avc.produce_annexb = true;
+  }
+
+  int total_decoded_frames = 0;
+
+  scoped_refptr<VideoFrame> reference_frame;
+
+  VideoEncoder::OutputCB encoder_output_cb = base::BindLambdaForTesting(
+      [&, this](VideoEncoderOutput output,
+                std::optional<VideoEncoder::CodecDescription> desc) {
+        auto buffer = DecoderBuffer::FromArray(std::move(output.data));
+        buffer->set_timestamp(output.timestamp);
+        buffer->set_is_key_frame(output.key_frame);
+        decoder_->Decode(std::move(buffer), DecoderStatusCB());
+      });
+
+  VideoDecoder::OutputCB decoder_output_cb =
+      base::BindLambdaForTesting([&](scoped_refptr<VideoFrame> decoded_frame) {
+        ASSERT_TRUE(reference_frame);
+
+        EXPECT_EQ(decoded_frame->timestamp(), reference_frame->timestamp());
+        EXPECT_EQ(decoded_frame->visible_rect().size(),
+                  reference_frame->visible_rect().size());
+
+        // This validates that the encoder actually encoded our input frame,
+        // and didn't just read uninitialized memory.
+        if (decoded_frame->format() == reference_frame->format()) {
+          // Allow up to 1% of pixels to be different due to compression
+          // artifacts.
+          const int kAllowedDifferentPixels =
+              reference_frame->visible_rect().size().GetArea() / 100;
+          EXPECT_LE(CountDifferentPixels(*decoded_frame, *reference_frame, 10),
+                    kAllowedDifferentPixels);
+        }
+        ++total_decoded_frames;
+      });
+
+  PrepareDecoder(options.frame_size, std::move(decoder_output_cb));
+
+  encoder_->Initialize(profile_, options, /*info_cb=*/base::DoNothing(),
+                       std::move(encoder_output_cb),
+                       ValidateStatusThenQuitCB());
+  RunUntilQuit();
+
+  // Create frames with identical colors but vastly different sizes.
+  constexpr uint32_t kXorMask = 0x123456;
+  auto oversized_frame = CreateFrame(gfx::Size(4096, 4096), pixel_format_,
+                                     base::TimeDelta(), kXorMask);
+  ASSERT_TRUE(oversized_frame) << "Failed to allocate oversized frame";
+  oversized_frame->set_color_space(gfx::ColorSpace::CreateREC709());
+
+  reference_frame = CreateFrame(options.frame_size, pixel_format_,
+                                base::TimeDelta(), kXorMask);
+  ASSERT_TRUE(reference_frame) << "Failed to allocate reference frame";
+
+  encoder_->Encode(std::move(oversized_frame),
+                   VideoEncoder::EncodeOptions(false),
+                   ValidateStatusThenQuitCB());
+  RunUntilQuit();
+
+  encoder_->Flush(ValidateStatusThenQuitCB());
+  RunUntilQuit();
+  DecodeAndWaitForStatus(DecoderBuffer::CreateEOSBuffer());
+
+  EXPECT_EQ(total_decoded_frames, 1);
+}
+
 TEST_P(H264VideoEncoderTest, AnnexB) {
   int outputs_count = 0;
   VideoEncoder::Options options = CreateDefaultOptions();