From 96c76fee15bb6ea844fa8b3684ee23b49e4011f0 Mon Sep 17 00:00:00 2001
From: Erik Språng <sprang@chromium.org>
Date: Tue, 14 Jul 2026 02:55:58 -0700
Subject: [PATCH] WebCodecs: Validate H.264 resolution against configured level.

This CL adds validation that the configured resolution conforms to the
the AVC specification for that selected level used (see e.g.
https://en.wikipedia.org/wiki/Advanced_Video_Coding#Levels).

In particular, we validate sections A.3.1 (e), (f) and (g) of the
ITU-T H.264 specification which limits both the max total number of
macroblocks as well as the maximum width and height.

Bug: 504690157
Change-Id: I3ec3d75b1f21b44f299431b540a3d68c4864127a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/8037024
Reviewed-by: Eugene Zemtsov <eugene@chromium.org>
Auto-Submit: Erik Språng <sprang@chromium.org>
Commit-Queue: Erik Språng <sprang@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1661715}
---
diff --git a/media/video/openh264_video_encoder.cc b/media/video/openh264_video_encoder.cc
index 3d158ba1..599a66d 100644
--- a/media/video/openh264_video_encoder.cc
+++ b/media/video/openh264_video_encoder.cc
@@ -5,6 +5,7 @@
 #include "media/video/openh264_video_encoder.h"
 
 #include <algorithm>
+#include <cmath>
 #include <limits>
 #include <numeric>
 
@@ -18,6 +19,8 @@
 #include "media/base/video_aspect_ratio.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_util.h"
+#include "media/parsers/h264_level_limits.h"
+#include "media/parsers/h264_parser.h"
 #include "media/video/video_encoder_info.h"
 
 namespace media {
@@ -155,8 +158,8 @@
 constexpr int kOpenH264MaxMBs = 36864;
 
 bool IsFrameSizeTooLarge(const gfx::Size& frame_size) {
-  int mb_width = (frame_size.width() + 15) / 16;
-  int mb_height = (frame_size.height() + 15) / 16;
+  uint64_t mb_width = (static_cast<uint64_t>(frame_size.width()) + 15) / 16;
+  uint64_t mb_height = (static_cast<uint64_t>(frame_size.height()) + 15) / 16;
   return mb_width * mb_height > kOpenH264MaxMBs;
 }
 
@@ -180,6 +183,38 @@
          VideoAspectRatio::PAR(dst.width(), dst.height());
 }
 
+// Validates that the frame size is within the limits defined by H.264
+// Level 6.1.
+EncoderStatus ValidateH264Resolution(const gfx::Size& frame_size) {
+  uint8_t level = H264SPS::kLevelIDC6p1;
+  uint32_t max_fs = H264LevelToMaxFS(level);
+  if (max_fs == 0) {
+    return EncoderStatus(EncoderStatus::Codes::kEncoderUnsupportedConfig,
+                         "Invalid H.264 level");
+  }
+
+  uint64_t mb_width = (static_cast<uint64_t>(frame_size.width()) + 15) / 16;
+  uint64_t mb_height = (static_cast<uint64_t>(frame_size.height()) + 15) / 16;
+  uint64_t mb_count = mb_width * mb_height;
+
+  if (mb_count > max_fs) {
+    return EncoderStatus(
+        EncoderStatus::Codes::kEncoderUnsupportedConfig,
+        "Configured frame size exceeds H.264 level max macroblocks");
+  }
+
+  // Aspect ratio constraint from H.264 standard Annex A:
+  // PicWidthInMbs <= Sqrt(MaxFS * 8)
+  // FrameHeightInMbs <= Sqrt(MaxFS * 8)
+  double max_mb_dim = std::sqrt(static_cast<double>(max_fs) * 8.0);
+  if (mb_width > max_mb_dim || mb_height > max_mb_dim) {
+    return EncoderStatus(EncoderStatus::Codes::kEncoderUnsupportedConfig,
+                         "Configured aspect ratio exceeds H.264 level limits");
+  }
+
+  return EncoderStatus::Codes::kOk;
+}
+
 }  // namespace
 
 OpenH264VideoEncoder::ISVCEncoderDeleter::ISVCEncoderDeleter() = default;
@@ -273,6 +308,12 @@
     return;
   }
 
+  if (auto status = ValidateH264Resolution(options.frame_size);
+      !status.is_ok()) {
+    std::move(done_cb).Run(status);
+    return;
+  }
+
   SetUpOpenH264Params(
       profile_, options,
       VideoColorSpace::FromGfxColorSpace(last_frame_color_space_), &params);
@@ -552,6 +593,12 @@
     return;
   }
 
+  if (auto status = ValidateH264Resolution(options.frame_size);
+      !status.is_ok()) {
+    std::move(done_cb).Run(status);
+    return;
+  }
+
   SEncParamExt params = {};
   if (int err = codec_->GetDefaultParams(&params)) {
     std::move(done_cb).Run(
diff --git a/media/video/software_video_encoder_test.cc b/media/video/software_video_encoder_test.cc
index d25df6ba0..e092c6a 100644
--- a/media/video/software_video_encoder_test.cc
+++ b/media/video/software_video_encoder_test.cc
@@ -1739,4 +1739,92 @@
   EXPECT_EQ(GetDefaultVideoEncodeBitrate({1280, 720}, 1000u), 20'000'000u);
 }
 
+#if BUILDFLAG(ENABLE_OPENH264)
+class OpenH264VideoEncoderResolutionTest : public ::testing::Test {
+ public:
+  OpenH264VideoEncoderResolutionTest() = default;
+
+  void SetUp() override { encoder_ = std::make_unique<OpenH264VideoEncoder>(); }
+
+  void TearDown() override { encoder_.reset(); }
+
+ protected:
+  base::test::TaskEnvironment task_environment_;
+  std::unique_ptr<OpenH264VideoEncoder> encoder_;
+};
+
+TEST_F(OpenH264VideoEncoderResolutionTest, HighestValidResolution) {
+  // 4096x2304 is exactly 36864 macroblocks, which is the OpenH264 limit.
+  VideoEncoder::Options options;
+  options.frame_size = gfx::Size(4096, 2304);
+
+  base::RunLoop run_loop;
+  encoder_->Initialize(H264PROFILE_BASELINE, options,
+                       /*info_cb=*/base::DoNothing(),
+                       /*output_cb=*/base::DoNothing(),
+                       base::BindLambdaForTesting([&](EncoderStatus status) {
+                         EXPECT_TRUE(status.is_ok());
+                         run_loop.Quit();
+                       }));
+  run_loop.Run();
+}
+
+TEST_F(OpenH264VideoEncoderResolutionTest, ResolutionExceedingMaxMBs) {
+  // 4097x2304 is 37008 macroblocks, exceeding the OpenH264 limit of 36864.
+  VideoEncoder::Options options;
+  options.frame_size = gfx::Size(4097, 2304);
+
+  base::RunLoop run_loop;
+  encoder_->Initialize(
+      H264PROFILE_BASELINE, options, /*info_cb=*/base::DoNothing(),
+      /*output_cb=*/base::DoNothing(),
+      base::BindLambdaForTesting([&](EncoderStatus status) {
+        EXPECT_EQ(status.code(),
+                  EncoderStatus::Codes::kEncoderUnsupportedConfig);
+        run_loop.Quit();
+      }));
+  run_loop.Run();
+}
+
+TEST_F(OpenH264VideoEncoderResolutionTest, WidthExceeding6p1AspectRatioLimit) {
+  // A resolution that is extremely wide (e.g. 17280x256) has 1080x16
+  // macroblocks. This is only 17280 macroblocks (well within OpenH264's 36864
+  // limit), but its width (1080 MBs) exceeds the Level 6.1 limit of Sqrt(139264
+  // * 8) = 1055.
+  VideoEncoder::Options options;
+  options.frame_size = gfx::Size(17280, 256);
+
+  base::RunLoop run_loop;
+  encoder_->Initialize(
+      H264PROFILE_BASELINE, options, /*info_cb=*/base::DoNothing(),
+      /*output_cb=*/base::DoNothing(),
+      base::BindLambdaForTesting([&](EncoderStatus status) {
+        EXPECT_EQ(status.code(),
+                  EncoderStatus::Codes::kEncoderUnsupportedConfig);
+        run_loop.Quit();
+      }));
+  run_loop.Run();
+}
+
+TEST_F(OpenH264VideoEncoderResolutionTest, HeightExceeding6p1AspectRatioLimit) {
+  // A resolution that is extremely tall (e.g. 256x17280) has 16x1080
+  // macroblocks. This is only 17280 macroblocks (well within OpenH264's 36864
+  // limit), but its height (1080 MBs) exceeds the Level 6.1 limit of
+  // Sqrt(139264 * 8) = 1055.
+  VideoEncoder::Options options;
+  options.frame_size = gfx::Size(256, 17280);
+
+  base::RunLoop run_loop;
+  encoder_->Initialize(
+      H264PROFILE_BASELINE, options, /*info_cb=*/base::DoNothing(),
+      /*output_cb=*/base::DoNothing(),
+      base::BindLambdaForTesting([&](EncoderStatus status) {
+        EXPECT_EQ(status.code(),
+                  EncoderStatus::Codes::kEncoderUnsupportedConfig);
+        run_loop.Quit();
+      }));
+  run_loop.Run();
+}
+#endif
+
 }  // namespace media
diff --git a/media/video/BUILD.gn b/media/video/BUILD.gn
index 2b2771c7..b45f3f5b 100644
--- a/media/video/BUILD.gn
+++ b/media/video/BUILD.gn
@@ -80,6 +80,7 @@
       "openh264_video_encoder.h",
     ]
     public_deps += [ "//third_party/openh264:encoder" ]
+    deps += [ "//media/parsers" ]
   }
 
   if (is_apple) {

diff --git a/media/video/software_video_encoder_test.cc b/media/video/software_video_encoder_test.cc
index d25df6ba0..e092c6a 100644
--- a/media/video/software_video_encoder_test.cc
+++ b/media/video/software_video_encoder_test.cc
@@ -1739,4 +1739,92 @@
   EXPECT_EQ(GetDefaultVideoEncodeBitrate({1280, 720}, 1000u), 20'000'000u);
 }
 
+#if BUILDFLAG(ENABLE_OPENH264)
+class OpenH264VideoEncoderResolutionTest : public ::testing::Test {
+ public:
+  OpenH264VideoEncoderResolutionTest() = default;
+
+  void SetUp() override { encoder_ = std::make_unique<OpenH264VideoEncoder>(); }
+
+  void TearDown() override { encoder_.reset(); }
+
+ protected:
+  base::test::TaskEnvironment task_environment_;
+  std::unique_ptr<OpenH264VideoEncoder> encoder_;
+};
+
+TEST_F(OpenH264VideoEncoderResolutionTest, HighestValidResolution) {
+  // 4096x2304 is exactly 36864 macroblocks, which is the OpenH264 limit.
+  VideoEncoder::Options options;
+  options.frame_size = gfx::Size(4096, 2304);
+
+  base::RunLoop run_loop;
+  encoder_->Initialize(H264PROFILE_BASELINE, options,
+                       /*info_cb=*/base::DoNothing(),
+                       /*output_cb=*/base::DoNothing(),
+                       base::BindLambdaForTesting([&](EncoderStatus status) {
+                         EXPECT_TRUE(status.is_ok());
+                         run_loop.Quit();
+                       }));
+  run_loop.Run();
+}
+
+TEST_F(OpenH264VideoEncoderResolutionTest, ResolutionExceedingMaxMBs) {
+  // 4097x2304 is 37008 macroblocks, exceeding the OpenH264 limit of 36864.
+  VideoEncoder::Options options;
+  options.frame_size = gfx::Size(4097, 2304);
+
+  base::RunLoop run_loop;
+  encoder_->Initialize(
+      H264PROFILE_BASELINE, options, /*info_cb=*/base::DoNothing(),
+      /*output_cb=*/base::DoNothing(),
+      base::BindLambdaForTesting([&](EncoderStatus status) {
+        EXPECT_EQ(status.code(),
+                  EncoderStatus::Codes::kEncoderUnsupportedConfig);
+        run_loop.Quit();
+      }));
+  run_loop.Run();
+}
+
+TEST_F(OpenH264VideoEncoderResolutionTest, WidthExceeding6p1AspectRatioLimit) {
+  // A resolution that is extremely wide (e.g. 17280x256) has 1080x16
+  // macroblocks. This is only 17280 macroblocks (well within OpenH264's 36864
+  // limit), but its width (1080 MBs) exceeds the Level 6.1 limit of Sqrt(139264
+  // * 8) = 1055.
+  VideoEncoder::Options options;
+  options.frame_size = gfx::Size(17280, 256);
+
+  base::RunLoop run_loop;
+  encoder_->Initialize(
+      H264PROFILE_BASELINE, options, /*info_cb=*/base::DoNothing(),
+      /*output_cb=*/base::DoNothing(),
+      base::BindLambdaForTesting([&](EncoderStatus status) {
+        EXPECT_EQ(status.code(),
+                  EncoderStatus::Codes::kEncoderUnsupportedConfig);
+        run_loop.Quit();
+      }));
+  run_loop.Run();
+}
+
+TEST_F(OpenH264VideoEncoderResolutionTest, HeightExceeding6p1AspectRatioLimit) {
+  // A resolution that is extremely tall (e.g. 256x17280) has 16x1080
+  // macroblocks. This is only 17280 macroblocks (well within OpenH264's 36864
+  // limit), but its height (1080 MBs) exceeds the Level 6.1 limit of
+  // Sqrt(139264 * 8) = 1055.
+  VideoEncoder::Options options;
+  options.frame_size = gfx::Size(256, 17280);
+
+  base::RunLoop run_loop;
+  encoder_->Initialize(
+      H264PROFILE_BASELINE, options, /*info_cb=*/base::DoNothing(),
+      /*output_cb=*/base::DoNothing(),
+      base::BindLambdaForTesting([&](EncoderStatus status) {
+        EXPECT_EQ(status.code(),
+                  EncoderStatus::Codes::kEncoderUnsupportedConfig);
+        run_loop.Quit();
+      }));
+  run_loop.Run();
+}
+#endif
+
 }  // namespace media