Protect against integer overflows in the VertexBuffer class by validating the reserved space.

Issue 444

Signed-off-by: Jamie Madil
Signed-off-by: Shannon Woods
Author: Geoff Lang

3 files changed, 33 insertions, 8 deletions

diff --git a/src/libGLESv2/renderer/VertexBuffer.cpp b/src/libGLESv2/renderer/VertexBuffer.cpp
index 16e14865..a8589465 100644
--- a/src/libGLESv2/renderer/VertexBuffer.cpp
+++ b/src/libGLESv2/renderer/VertexBuffer.cpp
@@ -125,14 +125,30 @@ int VertexBufferInterface::storeRawData(const void* data, unsigned int size)
     return oldWritePos;
 }
 
-void VertexBufferInterface::reserveVertexSpace(const gl::VertexAttribute &attribute, GLsizei count, GLsizei instances)
+bool VertexBufferInterface::reserveVertexSpace(const gl::VertexAttribute &attribute, GLsizei count, GLsizei instances)
 {
-    mReservedSpace += mVertexBuffer->getSpaceRequired(attribute, count, instances);
+    unsigned int requiredSpace = mVertexBuffer->getSpaceRequired(attribute, count, instances);
+
+    // Protect against integer overflow
+    if (mReservedSpace + requiredSpace < mReservedSpace)
+    {
+         return false;
+    }
+
+    mReservedSpace += requiredSpace;
+    return true;
 }
 
-void VertexBufferInterface::reserveRawDataSpace(unsigned int size)
+bool VertexBufferInterface::reserveRawDataSpace(unsigned int size)
 {
+    // Protect against integer overflow
+    if (mReservedSpace + size < mReservedSpace)
+    {
+         return false;
+    }
+
     mReservedSpace += size;
+    return true;
 }
 
 VertexBuffer* VertexBufferInterface::getVertexBuffer() const
diff --git a/src/libGLESv2/renderer/VertexBuffer.h b/src/libGLESv2/renderer/VertexBuffer.h
index 6ecffd0c..c474b05c 100644
--- a/src/libGLESv2/renderer/VertexBuffer.h
+++ b/src/libGLESv2/renderer/VertexBuffer.h
@@ -60,8 +60,8 @@ class VertexBufferInterface
     VertexBufferInterface(rx::Renderer *renderer, bool dynamic);
     virtual ~VertexBufferInterface();
 
-    void reserveVertexSpace(const gl::VertexAttribute &attribute, GLsizei count, GLsizei instances);
-    void reserveRawDataSpace(unsigned int size);
+    bool reserveVertexSpace(const gl::VertexAttribute &attribute, GLsizei count, GLsizei instances);
+    bool reserveRawDataSpace(unsigned int size);
 
     unsigned int getBufferSize() const;
 
diff --git a/src/libGLESv2/renderer/VertexDataManager.cpp b/src/libGLESv2/renderer/VertexDataManager.cpp
index ec858572..fd869b10 100644
--- a/src/libGLESv2/renderer/VertexDataManager.cpp
+++ b/src/libGLESv2/renderer/VertexDataManager.cpp
@@ -116,12 +116,18 @@ GLenum VertexDataManager::prepareVertexData(const gl::VertexAttribute attribs[],
                     if (staticBuffer->getBufferSize() == 0)
                     {
                         int totalCount = elementsInBuffer(attribs[i], buffer->size());
-                        staticBuffer->reserveVertexSpace(attribs[i], totalCount, 0);
+                        if (!staticBuffer->reserveVertexSpace(attribs[i], totalCount, 0))
+                        {
+                            return GL_OUT_OF_MEMORY;
+                        }
                     }
                 }
                 else
                 {
-                    mStreamingBuffer->reserveVertexSpace(attribs[i], count, instances);
+                    if (!mStreamingBuffer->reserveVertexSpace(attribs[i], count, instances))
+                    {
+                        return GL_OUT_OF_MEMORY;
+                    }
                 }
             }
         }
@@ -217,7 +223,10 @@ GLenum VertexDataManager::prepareVertexData(const gl::VertexAttribute attribs[],
                     mCurrentValue[i][3] != attribs[i].mCurrentValue[3])
                 {
                     unsigned int requiredSpace = sizeof(float) * 4;
-                    buffer->reserveRawDataSpace(requiredSpace);
+                    if (!buffer->reserveRawDataSpace(requiredSpace))
+                    {
+                        return GL_OUT_OF_MEMORY;
+                    }
                     int streamOffset = buffer->storeRawData(attribs[i].mCurrentValue, requiredSpace);
                     if (streamOffset == -1)
                     {