From f4c4aaf00cd981baf98fd81d37823d1032582d12 Mon Sep 17 00:00:00 2001
From: Shrek Shao <shrekshao@google.com>
Date: Tue, 17 Mar 2026 14:04:32 -0700
Subject: [PATCH] Fix D3D11 integer overflows in streaming vertex buffer path

The D3D11 backend's streaming vertex buffer path had several integer
arithmetic errors that could lead to under-reservation or out-of-bounds
memory access when using large base instance values.

Key changes:
- mathutil.h: Added UnsignedCeilDivide64 to handle 64-bit values safely.
- Renderer11/Renderer9: Updated getVertexSpaceRequired to use size_t and
CheckedNumeric for instance calculations, preventing 32-bit truncation.
- VertexBuffer: Updated storeDynamicAttribute to use CheckedNumeric when
calculating adjustedCount for instanced attributes with a base instance.
- VertexDataManager: Changed firstVertexIndex from GLint to size_t to
prevent negative index wrapping and added overflow checks for vertex
buffer size validation.
- Added a regression test (D3D11OverflowTest.cpp) to verify that large
baseInstance values do not cause crashes in the D3D11 backend.

This is a Gemini generated CL with manual modifications.

Bug: chromium:489791424
Change-Id: I86d13594ee6da6238d8c6583c5268b4bd2ee2658
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/7675790
Auto-Submit: Shrek Shao <shrekshao@google.com>
Commit-Queue: Shahbaz Youssefi <syoussefi@chromium.org>
Reviewed-by: Geoff Lang <geofflang@chromium.org>
Reviewed-by: Shahbaz Youssefi <syoussefi@chromium.org>
---

diff --git a/src/common/mathutil.h b/src/common/mathutil.h
index 0a67c2c..4f5b5c5 100644
--- a/src/common/mathutil.h
+++ b/src/common/mathutil.h
@@ -1538,6 +1538,12 @@
     return (divided + ((value % divisor == 0) ? 0 : 1));
 }
 
+inline constexpr uint64_t UnsignedCeilDivide64(uint64_t value, uint64_t divisor)
+{
+    uint64_t divided = value / divisor;
+    return (divided + ((value % divisor == 0) ? 0 : 1));
+}
+
 #if defined(__has_builtin)
 #    define ANGLE_HAS_BUILTIN(x) __has_builtin(x)
 #else
diff --git a/src/common/mathutil_unittest.cpp b/src/common/mathutil_unittest.cpp
index a66d881..b2e0a5c 100644
--- a/src/common/mathutil_unittest.cpp
+++ b/src/common/mathutil_unittest.cpp
@@ -1112,4 +1112,28 @@
     EXPECT_NEAR((normalizedToFloat<26>(-16777216)), -0.5f, 0.00000003);
 }
 
+// Test UnsignedCeilDivide
+TEST(MathUtilTest, UnsignedCeilDivide)
+{
+    EXPECT_EQ(0u, rx::UnsignedCeilDivide(0u, 5u));
+    EXPECT_EQ(1u, rx::UnsignedCeilDivide(1u, 5u));
+    EXPECT_EQ(1u, rx::UnsignedCeilDivide(4u, 5u));
+    EXPECT_EQ(1u, rx::UnsignedCeilDivide(5u, 5u));
+    EXPECT_EQ(2u, rx::UnsignedCeilDivide(6u, 5u));
+}
+
+// Test UnsignedCeilDivide64
+TEST(MathUtilTest, UnsignedCeilDivide64)
+{
+    EXPECT_EQ(0ull, rx::UnsignedCeilDivide64(0ull, 5ull));
+    EXPECT_EQ(1ull, rx::UnsignedCeilDivide64(1ull, 5ull));
+    EXPECT_EQ(1ull, rx::UnsignedCeilDivide64(4ull, 5ull));
+    EXPECT_EQ(1ull, rx::UnsignedCeilDivide64(5ull, 5ull));
+    EXPECT_EQ(2ull, rx::UnsignedCeilDivide64(6ull, 5ull));
+
+    uint64_t largeValue = 0xFFFFFFFF00000000ull;
+    EXPECT_EQ(0xFFFFFFFFull, rx::UnsignedCeilDivide64(largeValue, 0x100000000ull));
+    EXPECT_EQ(0x100000000ull, rx::UnsignedCeilDivide64(largeValue + 1, 0x100000000ull));
+}
+
 }  // anonymous namespace
diff --git a/src/libANGLE/renderer/d3d/RendererD3D.h b/src/libANGLE/renderer/d3d/RendererD3D.h
index 11e9e59..23183f0 100644
--- a/src/libANGLE/renderer/d3d/RendererD3D.h
+++ b/src/libANGLE/renderer/d3d/RendererD3D.h
@@ -160,7 +160,7 @@
                                                  const gl::VertexBinding &binding,
                                                  size_t count,
                                                  GLsizei instances,
-                                                 GLuint baseInstance,
+                                                 uint64_t baseInstance,
                                                  unsigned int *bytesRequiredOut) const = 0;
 };
 
diff --git a/src/libANGLE/renderer/d3d/VertexBuffer.cpp b/src/libANGLE/renderer/d3d/VertexBuffer.cpp
index acb27ad..051fc7f 100644
--- a/src/libANGLE/renderer/d3d/VertexBuffer.cpp
+++ b/src/libANGLE/renderer/d3d/VertexBuffer.cpp
@@ -96,7 +96,7 @@
                                                       const gl::VertexBinding &binding,
                                                       size_t count,
                                                       GLsizei instances,
-                                                      GLuint baseInstance,
+                                                      uint64_t baseInstance,
                                                       unsigned int *spaceInBytesOut) const
 {
     unsigned int spaceRequired = 0;
@@ -169,10 +169,10 @@
     const gl::VertexAttribute &attrib,
     const gl::VertexBinding &binding,
     gl::VertexAttribType currentValueType,
-    GLint start,
+    size_t start,
     size_t count,
     GLsizei instances,
-    GLuint baseInstance,
+    uint64_t baseInstance,
     unsigned int *outStreamOffset,
     const uint8_t *sourceData)
 {
@@ -187,17 +187,22 @@
 
     mReservedSpace = 0;
 
-    size_t adjustedCount = count;
-    GLuint divisor       = binding.getDivisor();
+    angle::CheckedNumeric<size_t> checkedCount = count;
+    GLuint divisor                             = binding.getDivisor();
 
     if (instances != 0 && divisor != 0)
     {
         // The attribute is an instanced attribute and it's an draw instance call
         // Extra number of elements are copied at the beginning to make sure
         // the driver is referencing the correct data with non-zero baseInstance
-        adjustedCount += UnsignedCeilDivide(baseInstance, divisor);
+        checkedCount += UnsignedCeilDivide64(static_cast<uint64_t>(baseInstance),
+                                             static_cast<uint64_t>(divisor));
     }
 
+    ANGLE_CHECK(GetImplAs<ContextD3D>(context), checkedCount.IsValid(),
+                "New vertex buffer size would result in an overflow.", GL_OUT_OF_MEMORY);
+
+    size_t adjustedCount = checkedCount.ValueOrDie();
     ANGLE_TRY(mVertexBuffer->storeVertexAttributes(context, attrib, binding, currentValueType,
                                                    start, adjustedCount, instances, mWritePosition,
                                                    sourceData));
@@ -217,7 +222,7 @@
                                                                  const gl::VertexBinding &binding,
                                                                  size_t count,
                                                                  GLsizei instances,
-                                                                 GLuint baseInstance)
+                                                                 uint64_t baseInstance)
 {
     unsigned int requiredSpace = 0;
     ANGLE_TRY(mFactory->getVertexSpaceRequired(context, attrib, binding, count, instances,
@@ -286,7 +291,7 @@
 angle::Result StaticVertexBufferInterface::storeStaticAttribute(const gl::Context *context,
                                                                 const gl::VertexAttribute &attrib,
                                                                 const gl::VertexBinding &binding,
-                                                                GLint start,
+                                                                size_t start,
                                                                 GLsizei count,
                                                                 GLsizei instances,
                                                                 const uint8_t *sourceData)
diff --git a/src/libANGLE/renderer/d3d/VertexBuffer.h b/src/libANGLE/renderer/d3d/VertexBuffer.h
index 45a16c6..4d0038f 100644
--- a/src/libANGLE/renderer/d3d/VertexBuffer.h
+++ b/src/libANGLE/renderer/d3d/VertexBuffer.h
@@ -50,7 +50,7 @@
                                                 const gl::VertexAttribute &attrib,
                                                 const gl::VertexBinding &binding,
                                                 gl::VertexAttribType currentValueType,
-                                                GLint start,
+                                                size_t start,
                                                 size_t count,
                                                 GLsizei instances,
                                                 unsigned int offset,
@@ -102,7 +102,7 @@
                                    const gl::VertexBinding &binding,
                                    size_t count,
                                    GLsizei instances,
-                                   GLuint baseInstance,
+                                   uint64_t baseInstance,
                                    unsigned int *spaceInBytesOut) const;
     BufferFactoryD3D *const mFactory;
     VertexBuffer *mVertexBuffer;
@@ -122,10 +122,10 @@
                                         const gl::VertexAttribute &attrib,
                                         const gl::VertexBinding &binding,
                                         gl::VertexAttribType currentValueType,
-                                        GLint start,
+                                        size_t start,
                                         size_t count,
                                         GLsizei instances,
-                                        GLuint baseInstance,
+                                        uint64_t baseInstance,
                                         unsigned int *outStreamOffset,
                                         const uint8_t *sourceData);
 

diff --git a/src/common/mathutil_unittest.cpp b/src/common/mathutil_unittest.cpp
index a66d881..b2e0a5c 100644
--- a/src/common/mathutil_unittest.cpp
+++ b/src/common/mathutil_unittest.cpp
@@ -1112,4 +1112,28 @@
     EXPECT_NEAR((normalizedToFloat<26>(-16777216)), -0.5f, 0.00000003);
 }
 
+// Test UnsignedCeilDivide
+TEST(MathUtilTest, UnsignedCeilDivide)
+{
+    EXPECT_EQ(0u, rx::UnsignedCeilDivide(0u, 5u));
+    EXPECT_EQ(1u, rx::UnsignedCeilDivide(1u, 5u));
+    EXPECT_EQ(1u, rx::UnsignedCeilDivide(4u, 5u));
+    EXPECT_EQ(1u, rx::UnsignedCeilDivide(5u, 5u));
+    EXPECT_EQ(2u, rx::UnsignedCeilDivide(6u, 5u));
+}
+
+// Test UnsignedCeilDivide64
+TEST(MathUtilTest, UnsignedCeilDivide64)
+{
+    EXPECT_EQ(0ull, rx::UnsignedCeilDivide64(0ull, 5ull));
+    EXPECT_EQ(1ull, rx::UnsignedCeilDivide64(1ull, 5ull));
+    EXPECT_EQ(1ull, rx::UnsignedCeilDivide64(4ull, 5ull));
+    EXPECT_EQ(1ull, rx::UnsignedCeilDivide64(5ull, 5ull));
+    EXPECT_EQ(2ull, rx::UnsignedCeilDivide64(6ull, 5ull));
+
+    uint64_t largeValue = 0xFFFFFFFF00000000ull;
+    EXPECT_EQ(0xFFFFFFFFull, rx::UnsignedCeilDivide64(largeValue, 0x100000000ull));
+    EXPECT_EQ(0x100000000ull, rx::UnsignedCeilDivide64(largeValue + 1, 0x100000000ull));
+}
+
 }  // anonymous namespace
diff --git a/src/tests/angle_end2end_tests.gni b/src/tests/angle_end2end_tests.gni
index 314ac14..bce0a90 100644
--- a/src/tests/angle_end2end_tests.gni
+++ b/src/tests/angle_end2end_tests.gni
@@ -39,6 +39,7 @@
   "gl_tests/AtomicCounterBufferTest.cpp",
   "gl_tests/AttributeLayoutTest.cpp",
   "gl_tests/BPTCCompressedTextureTest.cpp",
+  "gl_tests/BaseInstanceOverflowTest.cpp",
   "gl_tests/BindGeneratesResourceTest.cpp",
   "gl_tests/BindUniformLocationTest.cpp",
   "gl_tests/BlendFuncExtendedTest.cpp",
diff --git a/src/tests/angle_end2end_tests_expectations.txt b/src/tests/angle_end2end_tests_expectations.txt
index 11cb020..02db357 100644
--- a/src/tests/angle_end2end_tests_expectations.txt
+++ b/src/tests/angle_end2end_tests_expectations.txt
@@ -407,6 +407,7 @@
 448658630 MAC OPENGL : GLSLTestLoops.ForContinueInConstSwitch/* = SKIP
 448658630 MAC OPENGL : GLSLTestLoops.ForContinueInSwitchComplex/* = SKIP
 475587478 MAC OPENGL : VertexAttributeTestES3.MaxAttribsWithBuiltInAttribs/* = SKIP
+494270619 MAC OPENGL : BaseInstanceOverflowTest.BaseInstanceOverflow/* = SKIP
 
 // BlitFramebufferTest.ScissoredMultisampleStencil failures
 42262159 MAC INTEL OPENGL : BlitFramebufferTest.ScissoredMultisampleStencil/* = SKIP
diff --git a/src/tests/gl_tests/BaseInstanceOverflowTest.cpp b/src/tests/gl_tests/BaseInstanceOverflowTest.cpp
new file mode 100644
index 0000000..51a4e64
--- /dev/null
+++ b/src/tests/gl_tests/BaseInstanceOverflowTest.cpp
@@ -0,0 +1,87 @@
+//
+// Copyright 2026 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+// BaseInstanceOverflowTest: Reproduces integer overflow in vertex buffer streaming path.
+
+#include "test_utils/ANGLETest.h"
+#include "test_utils/gl_raii.h"
+
+using namespace angle;
+
+class BaseInstanceOverflowTest : public ANGLETest<>
+{
+  protected:
+    BaseInstanceOverflowTest()
+    {
+        setWindowWidth(1);
+        setWindowHeight(1);
+        setConfigRedBits(8);
+        setConfigGreenBits(8);
+        setConfigBlueBits(8);
+        setConfigAlphaBits(8);
+    }
+};
+
+// Regression Test (crbug.com/489791424)
+// Reproduces integer overflow in vertex buffer streaming path.
+// The bug occurs when baseInstance is large, causing a wrap-around in reservation
+// while the copy path uses 64-bit math.
+TEST_P(BaseInstanceOverflowTest, BaseInstanceOverflow)
+{
+    ANGLE_SKIP_TEST_IF(!EnsureGLExtensionEnabled("GL_ANGLE_base_vertex_base_instance"));
+
+    // We need a dynamic vertex attribute to trigger the streaming path.
+    // In D3D11, using an unaligned offset forces the streaming path.
+    GLBuffer buffer;
+    glBindBuffer(GL_ARRAY_BUFFER, buffer);
+    std::vector<GLfloat> data(1000, 1.0f);
+    glBufferData(GL_ARRAY_BUFFER, data.size() * sizeof(GLfloat), data.data(), GL_DYNAMIC_DRAW);
+
+    GLProgram program;
+    program.makeRaster(essl1_shaders::vs::Simple(), essl1_shaders::fs::Red());
+    glUseProgram(program);
+
+    GLint posLoc = glGetAttribLocation(program, essl1_shaders::PositionAttrib());
+    glEnableVertexAttribArray(posLoc);
+
+    // Unaligned offset (1 byte) forces streaming path in D3D11.
+    glVertexAttribPointer(posLoc, 4, GL_FLOAT, GL_FALSE, 16, reinterpret_cast<void *>(1));
+    glVertexAttribDivisor(posLoc, 1);
+
+    // Trigger overflow by using a large baseInstance.
+    // elementCount calculation in Renderer11::getVertexSpaceRequired:
+    // (instances + baseInstance) / divisor
+    // If instances = 300, baseInstance = 0xFFFFFF00, divisor = 1:
+    // 300 + 0xFFFFFF00 = 0x10000002C.
+    // 32-bit truncation results in 0x2C = 44 elements reserved.
+    // However, StreamingVertexBufferInterface::storeDynamicAttribute uses 64-bit size_t:
+    // adjustedCount = 300 + 0xFFFFFF00 = 4,294,967,340 elements copied.
+    GLuint baseInstance   = 0xFFFFFF00;
+    GLsizei instanceCount = 300;
+
+    // This call is expected to crash the GPU process without the fix.
+    // With the fix, the large baseInstance is caught by validation in VertexDataManager (D3D11)
+    // or ContextGL (OpenGL) because it exceeds the source buffer's bounds, returning
+    // GL_INVALID_OPERATION (D3D11) or avoiding the draw (OpenGL fallback).
+    glDrawArraysInstancedBaseInstanceANGLE(GL_TRIANGLES, 0, 3, instanceCount, baseInstance);
+
+    if (isD3D11Renderer())
+    {
+        EXPECT_GL_ERROR(GL_INVALID_OPERATION);
+    }
+    else
+    {
+        EXPECT_GL_NO_ERROR();
+    }
+}
+
+GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(BaseInstanceOverflowTest);
+ANGLE_INSTANTIATE_TEST(BaseInstanceOverflowTest,
+                       ES3_D3D11().enable(Feature::AlwaysEnableEmulatedMultidrawExtensions),
+                       ES3_OPENGL().enable(Feature::AlwaysEnableEmulatedMultidrawExtensions),
+                       ES3_OPENGLES().enable(Feature::AlwaysEnableEmulatedMultidrawExtensions),
+                       ES3_VULKAN().enable(Feature::AlwaysEnableEmulatedMultidrawExtensions),
+                       ES3_METAL().enable(Feature::AlwaysEnableEmulatedMultidrawExtensions));
diff --git a/src/tests/perf_tests/IndexDataManagerTest.cpp b/src/tests/perf_tests/IndexDataManagerTest.cpp
index 5d9856a..8084612 100644
--- a/src/tests/perf_tests/IndexDataManagerTest.cpp
+++ b/src/tests/perf_tests/IndexDataManagerTest.cpp
@@ -66,7 +66,7 @@
                                      const gl::VertexBinding &,
                                      size_t,
                                      GLsizei,
-                                     GLuint,
+                                     uint64_t,
                                      unsigned int *));
 
     // Dependency injection