Fixes CVE-2021-43527.
Copied from <https://hg.mozilla.org/projects/nss/rev/dea71cbef9e03636f37c6cb120f8deccce6e17dd>,
but with the file names adjusted to allow easy use within GNU Guix.

# HG changeset patch
# User Dennis Jackson <djackson@mozilla.com>
# Date 1637577642 0
# Node ID dea71cbef9e03636f37c6cb120f8deccce6e17dd
# Parent  da3d22d708c9cc0a32cff339658aeb627575e371
Bug 1737470 - Ensure DER encoded signatures are within size limits. r=jschanck,mt,bbeurdouche,rrelyea

Differential Revision: https://phabricator.services.mozilla.com/D129514

--- a/nss/lib/cryptohi/secvfy.c
+++ b/nss/lib/cryptohi/secvfy.c
@@ -159,58 +159,89 @@ verifyPKCS1DigestInfo(const VFYContext *
     SECItem pkcs1DigestInfo;
     pkcs1DigestInfo.data = cx->pkcs1RSADigestInfo;
     pkcs1DigestInfo.len = cx->pkcs1RSADigestInfoLen;
     return _SGN_VerifyPKCS1DigestInfo(
         cx->hashAlg, digest, &pkcs1DigestInfo,
         PR_FALSE /*XXX: unsafeAllowMissingParameters*/);
 }
 
+static unsigned int
+checkedSignatureLen(const SECKEYPublicKey *pubk)
+{
+    unsigned int sigLen = SECKEY_SignatureLen(pubk);
+    if (sigLen == 0) {
+        /* Error set by SECKEY_SignatureLen */
+        return sigLen;
+    }
+    unsigned int maxSigLen;
+    switch (pubk->keyType) {
+        case rsaKey:
+        case rsaPssKey:
+            maxSigLen = (RSA_MAX_MODULUS_BITS + 7) / 8;
+            break;
+        case dsaKey:
+            maxSigLen = DSA_MAX_SIGNATURE_LEN;
+            break;
+        case ecKey:
+            maxSigLen = 2 * MAX_ECKEY_LEN;
+            break;
+        default:
+            PORT_SetError(SEC_ERROR_UNSUPPORTED_KEYALG);
+            return 0;
+    }
+    if (sigLen > maxSigLen) {
+        PORT_SetError(SEC_ERROR_INVALID_KEY);
+        return 0;
+    }
+    return sigLen;
+}
+
 /*
  * decode the ECDSA or DSA signature from it's DER wrapping.
  * The unwrapped/raw signature is placed in the buffer pointed
  * to by dsig and has enough room for len bytes.
  */
 static SECStatus
 decodeECorDSASignature(SECOidTag algid, const SECItem *sig, unsigned char *dsig,
                        unsigned int len)
 {
     SECItem *dsasig = NULL; /* also used for ECDSA */
-    SECStatus rv = SECSuccess;
 
-    if ((algid != SEC_OID_ANSIX9_DSA_SIGNATURE) &&
-        (algid != SEC_OID_ANSIX962_EC_PUBLIC_KEY)) {
-        if (sig->len != len) {
-            PORT_SetError(SEC_ERROR_BAD_DER);
-            return SECFailure;
+    /* Safety: Ensure algId is as expected and that signature size is within maxmimums */
+    if (algid == SEC_OID_ANSIX9_DSA_SIGNATURE) {
+        if (len > DSA_MAX_SIGNATURE_LEN) {
+            goto loser;
         }
-
-        PORT_Memcpy(dsig, sig->data, sig->len);
-        return SECSuccess;
+    } else if (algid == SEC_OID_ANSIX962_EC_PUBLIC_KEY) {
+        if (len > MAX_ECKEY_LEN * 2) {
+            goto loser;
+        }
+    } else {
+        goto loser;
     }
 
-    if (algid == SEC_OID_ANSIX962_EC_PUBLIC_KEY) {
-        if (len > MAX_ECKEY_LEN * 2) {
-            PORT_SetError(SEC_ERROR_BAD_DER);
-            return SECFailure;
-        }
+    /* Decode and pad to length */
+    dsasig = DSAU_DecodeDerSigToLen((SECItem *)sig, len);
+    if (dsasig == NULL) {
+        goto loser;
     }
-    dsasig = DSAU_DecodeDerSigToLen((SECItem *)sig, len);
-
-    if ((dsasig == NULL) || (dsasig->len != len)) {
-        rv = SECFailure;
-    } else {
-        PORT_Memcpy(dsig, dsasig->data, dsasig->len);
+    if (dsasig->len != len) {
+        SECITEM_FreeItem(dsasig, PR_TRUE);
+        goto loser;
     }
 
-    if (dsasig != NULL)
-        SECITEM_FreeItem(dsasig, PR_TRUE);
-    if (rv == SECFailure)
-        PORT_SetError(SEC_ERROR_BAD_DER);
-    return rv;
+    PORT_Memcpy(dsig, dsasig->data, len);
+    SECITEM_FreeItem(dsasig, PR_TRUE);
+
+    return SECSuccess;
+
+loser:
+    PORT_SetError(SEC_ERROR_BAD_DER);
+    return SECFailure;
 }
 
 const SEC_ASN1Template hashParameterTemplate[] =
     {
       { SEC_ASN1_SEQUENCE, 0, NULL, sizeof(SECItem) },
       { SEC_ASN1_OBJECT_ID, 0 },
       { SEC_ASN1_SKIP_REST },
       { 0 }
@@ -276,17 +307,17 @@ sec_GetEncAlgFromSigAlg(SECOidTag sigAlg
  *
  * Returns: SECSuccess if the algorithm was acceptable, SECFailure if the
  *	algorithm was not found or was not a signing algorithm.
  */
 SECStatus
 sec_DecodeSigAlg(const SECKEYPublicKey *key, SECOidTag sigAlg,
                  const SECItem *param, SECOidTag *encalgp, SECOidTag *hashalg)
 {
-    int len;
+    unsigned int len;
     PLArenaPool *arena;
     SECStatus rv;
     SECItem oid;
     SECOidTag encalg;
 
     PR_ASSERT(hashalg != NULL);
     PR_ASSERT(encalgp != NULL);
 
@@ -461,58 +492,62 @@ vfy_CreateContext(const SECKEYPublicKey 
     cx->wincx = wincx;
     cx->hasSignature = (sig != NULL);
     cx->encAlg = encAlg;
     cx->hashAlg = hashAlg;
     cx->key = SECKEY_CopyPublicKey(key);
     cx->pkcs1RSADigestInfo = NULL;
     rv = SECSuccess;
     if (sig) {
-        switch (type) {
-            case rsaKey:
-                rv = recoverPKCS1DigestInfo(hashAlg, &cx->hashAlg,
-                                            &cx->pkcs1RSADigestInfo,
-                                            &cx->pkcs1RSADigestInfoLen,
-                                            cx->key,
-                                            sig, wincx);
-                break;
-            case rsaPssKey:
-                sigLen = SECKEY_SignatureLen(key);
-                if (sigLen == 0) {
-                    /* error set by SECKEY_SignatureLen */
-                    rv = SECFailure;
+        rv = SECFailure;
+        if (type == rsaKey) {
+            rv = recoverPKCS1DigestInfo(hashAlg, &cx->hashAlg,
+                                        &cx->pkcs1RSADigestInfo,
+                                        &cx->pkcs1RSADigestInfoLen,
+                                        cx->key,
+                                        sig, wincx);
+        } else {
+            sigLen = checkedSignatureLen(key);
+            /* Check signature length is within limits */
+            if (sigLen == 0) {
+                /* error set by checkedSignatureLen */
+                rv = SECFailure;
+                goto loser;
+            }
+            if (sigLen > sizeof(cx->u)) {
+                PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
+                rv = SECFailure;
+                goto loser;
+            }
+            switch (type) {
+                case rsaPssKey:
+                    if (sig->len != sigLen) {
+                        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
+                        rv = SECFailure;
+                        goto loser;
+                    }
+                    PORT_Memcpy(cx->u.buffer, sig->data, sigLen);
+                    rv = SECSuccess;
                     break;
-                }
-                if (sig->len != sigLen) {
-                    PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
+                case ecKey:
+                case dsaKey:
+                    /* decodeECorDSASignature will check sigLen == sig->len after padding */
+                    rv = decodeECorDSASignature(encAlg, sig, cx->u.buffer, sigLen);
+                    break;
+                default:
+                    /* Unreachable */
                     rv = SECFailure;
-                    break;
-                }
-                PORT_Memcpy(cx->u.buffer, sig->data, sigLen);
-                break;
-            case dsaKey:
-            case ecKey:
-                sigLen = SECKEY_SignatureLen(key);
-                if (sigLen == 0) {
-                    /* error set by SECKEY_SignatureLen */
-                    rv = SECFailure;
-                    break;
-                }
-                rv = decodeECorDSASignature(encAlg, sig, cx->u.buffer, sigLen);
-                break;
-            default:
-                rv = SECFailure;
-                PORT_SetError(SEC_ERROR_UNSUPPORTED_KEYALG);
-                break;
+                    goto loser;
+            }
+        }
+        if (rv != SECSuccess) {
+            goto loser;
         }
     }
 
-    if (rv)
-        goto loser;
-
     /* check hash alg again, RSA may have changed it.*/
     if (HASH_GetHashTypeByOidTag(cx->hashAlg) == HASH_AlgNULL) {
         /* error set by HASH_GetHashTypeByOidTag */
         goto loser;
     }
     /* check the policy on the hash algorithm. Do this after
      * the rsa decode because some uses of this function get hash implicitly
      * from the RSA signature itself. */
@@ -645,21 +680,26 @@ VFY_EndWithSignature(VFYContext *cx, SEC
     if (cx->hashcx == NULL) {
         PORT_SetError(SEC_ERROR_INVALID_ARGS);
         return SECFailure;
     }
     (*cx->hashobj->end)(cx->hashcx, final, &part, sizeof(final));
     switch (cx->key->keyType) {
         case ecKey:
         case dsaKey:
-            dsasig.data = cx->u.buffer;
-            dsasig.len = SECKEY_SignatureLen(cx->key);
+            dsasig.len = checkedSignatureLen(cx->key);
             if (dsasig.len == 0) {
                 return SECFailure;
             }
+            if (dsasig.len > sizeof(cx->u)) {
+                PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
+                return SECFailure;
+            }
+            dsasig.data = cx->u.buffer;
+
             if (sig) {
                 rv = decodeECorDSASignature(cx->encAlg, sig, dsasig.data,
                                             dsasig.len);
                 if (rv != SECSuccess) {
                     PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
                     return SECFailure;
                 }
             }
@@ -681,18 +721,23 @@ VFY_EndWithSignature(VFYContext *cx, SEC
                                                        cx->params,
                                                        &mech);
                 PORT_DestroyCheapArena(&tmpArena);
                 if (rv != SECSuccess) {
                     return SECFailure;
                 }
 
                 rsasig.data = cx->u.buffer;
-                rsasig.len = SECKEY_SignatureLen(cx->key);
+                rsasig.len = checkedSignatureLen(cx->key);
                 if (rsasig.len == 0) {
+                    /* Error set by checkedSignatureLen */
+                    return SECFailure;
+                }
+                if (rsasig.len > sizeof(cx->u)) {
+                    PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
                     return SECFailure;
                 }
                 if (sig) {
                     if (sig->len != rsasig.len) {
                         PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
                         return SECFailure;
                     }
                     PORT_Memcpy(rsasig.data, sig->data, rsasig.len);
@@ -744,37 +789,42 @@ VFY_End(VFYContext *cx)
 static SECStatus
 vfy_VerifyDigest(const SECItem *digest, const SECKEYPublicKey *key,
                  const SECItem *sig, SECOidTag encAlg, SECOidTag hashAlg,
                  void *wincx)
 {
     SECStatus rv;
     VFYContext *cx;
     SECItem dsasig; /* also used for ECDSA */
-
     rv = SECFailure;
 
     cx = vfy_CreateContext(key, sig, encAlg, hashAlg, NULL, wincx);
     if (cx != NULL) {
         switch (key->keyType) {
             case rsaKey:
                 rv = verifyPKCS1DigestInfo(cx, digest);
+                /* Error (if any) set by verifyPKCS1DigestInfo */
                 break;
-            case dsaKey:
             case ecKey:
+            case dsaKey:
                 dsasig.data = cx->u.buffer;
-                dsasig.len = SECKEY_SignatureLen(cx->key);
+                dsasig.len = checkedSignatureLen(cx->key);
                 if (dsasig.len == 0) {
+                    /* Error set by checkedSignatureLen */
+                    rv = SECFailure;
                     break;
                 }
-                if (PK11_Verify(cx->key, &dsasig, (SECItem *)digest, cx->wincx) !=
-                    SECSuccess) {
+                if (dsasig.len > sizeof(cx->u)) {
                     PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
-                } else {
-                    rv = SECSuccess;
+                    rv = SECFailure;
+                    break;
+                }
+                rv = PK11_Verify(cx->key, &dsasig, (SECItem *)digest, cx->wincx);
+                if (rv != SECSuccess) {
+                    PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
                 }
                 break;
             default:
                 break;
         }
         VFY_DestroyContext(cx, PR_TRUE);
     }
     return rv;