Bug 1352681 - Overflow checking on SAB reference count. r=sfink

--HG--
extra : rebase_source : 86549600fd1fc755c490546052e1bbd40326331d

js/src/js.msg

@@ -434,6 +434,7 @@ MSG_DEF(JSMSG_SC_UNSUPPORTED_TYPE,     0, JSEXN_TYPEERR, "unsupported type for s
MSG_DEF(JSMSG_SC_NOT_CLONABLE,         1, JSEXN_TYPEERR, "{0} cannot be cloned in this context")
MSG_DEF(JSMSG_SC_SAB_TRANSFERABLE,     0, JSEXN_TYPEERR, "SharedArrayBuffer must not be in the transfer list")
MSG_DEF(JSMSG_SC_SAB_DISABLED,         0, JSEXN_TYPEERR, "SharedArrayBuffer not cloned - shared memory disabled in receiver")
MSG_DEF(JSMSG_SC_SAB_REFCNT_OFLO,      0, JSEXN_TYPEERR, "SharedArrayBuffer has too many references")

// Debugger
MSG_DEF(JSMSG_ASSIGN_FUNCTION_OR_NULL, 1, JSEXN_TYPEERR, "value assigned to {0} must be a function or null")

js/src/shell/js.cpp

@@ -5515,25 +5515,31 @@ GetSharedArrayBuffer(JSContext* cx, unsigned argc, Value* vp)
{
    CallArgs args = CallArgsFromVp(argc, vp);
    JSObject* newObj = nullptr;
    bool rval = true;

    sharedArrayBufferMailboxLock->lock();
    SharedArrayRawBuffer* buf = sharedArrayBufferMailbox;
    if (buf) {
        buf->addReference();
        // Shared memory is enabled globally in the shell: there can't be a worker
        // that does not enable it if the main thread has it.
        MOZ_ASSERT(cx->compartment()->creationOptions().getSharedMemoryAndAtomicsEnabled());
        newObj = SharedArrayBufferObject::New(cx, buf);
        if (!newObj) {
            buf->dropReference();
            rval = false;

    {
        sharedArrayBufferMailboxLock->lock();
        auto unlockMailbox = MakeScopeExit([]() { sharedArrayBufferMailboxLock->unlock(); });

        SharedArrayRawBuffer* buf = sharedArrayBufferMailbox;
        if (buf) {
            if (!buf->addReference()) {
                JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, JSMSG_SC_SAB_REFCNT_OFLO);
                return false;
            }

            // Shared memory is enabled globally in the shell: there can't be a worker
            // that does not enable it if the main thread has it.
            MOZ_ASSERT(cx->compartment()->creationOptions().getSharedMemoryAndAtomicsEnabled());
            newObj = SharedArrayBufferObject::New(cx, buf);
            if (!newObj) {
                buf->dropReference();
                return false;
            }
        }
    }
    sharedArrayBufferMailboxLock->unlock();

    args.rval().setObjectOrNull(newObj);
    return rval;
    return true;
}

static bool
@@ -5547,18 +5553,24 @@ SetSharedArrayBuffer(JSContext* cx, unsigned argc, Value* vp)
    }
    else if (args.get(0).isObject() && args[0].toObject().is<SharedArrayBufferObject>()) {
        newBuffer = args[0].toObject().as<SharedArrayBufferObject>().rawBufferObject();
        newBuffer->addReference();
        if (!newBuffer->addReference()) {
            JS_ReportErrorASCII(cx, "Reference count overflow on SharedArrayBuffer");
            return false;
        }
    } else {
        JS_ReportErrorASCII(cx, "Only a SharedArrayBuffer can be installed in the global mailbox");
        return false;
    }

    sharedArrayBufferMailboxLock->lock();
    SharedArrayRawBuffer* oldBuffer = sharedArrayBufferMailbox;
    if (oldBuffer)
        oldBuffer->dropReference();
    sharedArrayBufferMailbox = newBuffer;
    sharedArrayBufferMailboxLock->unlock();
    {
        sharedArrayBufferMailboxLock->lock();
        auto unlockMailbox = MakeScopeExit([]() { sharedArrayBufferMailboxLock->unlock(); });

        SharedArrayRawBuffer* oldBuffer = sharedArrayBufferMailbox;
        if (oldBuffer)
            oldBuffer->dropReference();
        sharedArrayBufferMailbox = newBuffer;
    }

    args.rval().setUndefined();
    return true;

js/src/vm/SharedArrayObject.cpp

@@ -165,16 +165,30 @@ SharedArrayRawBuffer::New(JSContext* cx, uint32_t length)
    return rawbuf;
}

void
bool
SharedArrayRawBuffer::addReference()
{
    MOZ_ASSERT(this->refcount_ > 0);
    ++this->refcount_; // Atomic.
    MOZ_RELEASE_ASSERT(this->refcount_ > 0);

    // Be careful never to overflow the refcount field.
    for (;;) {
        uint32_t old_refcount = this->refcount_;
        uint32_t new_refcount = old_refcount+1;
        if (new_refcount == 0)
            return false;
        if (this->refcount_.compareExchange(old_refcount, new_refcount))
            return true;
    }
}

void
SharedArrayRawBuffer::dropReference()
{
    // Normally if the refcount is zero then the memory will have been unmapped
    // and this test may just crash, but if the memory has been retained for any
    // reason we will catch the underflow here.
    MOZ_RELEASE_ASSERT(this->refcount_ > 0);

    // Drop the reference to the buffer.
    uint32_t refcount = --this->refcount_; // Atomic.
    if (refcount)

js/src/vm/SharedArrayObject.h

@@ -92,7 +92,7 @@ class SharedArrayRawBuffer

    uint32_t refcount() const { return refcount_; }

    void addReference();
    MOZ_MUST_USE bool addReference();
    void dropReference();
};

js/src/vm/StructuredClone.cpp

@@ -1163,7 +1163,10 @@ JSStructuredCloneWriter::writeSharedArrayBuffer(HandleObject obj)

    // Avoids a race condition where the parent thread frees the buffer
    // before the child has accepted the transferable.
    rawbuf->addReference();
    if (!rawbuf->addReference()) {
        JS_ReportErrorNumberASCII(context(), GetErrorMessage, nullptr, JSMSG_SC_SAB_REFCNT_OFLO);
        return false;
    }

    intptr_t p = reinterpret_cast<intptr_t>(rawbuf);
    return out.writePair(SCTAG_SHARED_ARRAY_BUFFER_OBJECT, static_cast<uint32_t>(sizeof(p))) &&