SignedXml checksignature returns false

I've looked at other posts on here regarding this issue and none of them seem to address my situation.

I've been trying to verify a SAML assertion for the last week and I have 2 clients that have sent me SAML but I cannot verify it.

The main process is we get a base64 encoded assertion and I decode it. Load it into an XmlDocment with PreserveWhitespace = true.

The verify method is

  public static bool Verify(X509Certificate2 cert, XmlElement xmlElement, SignedXml signedXml)
  {
       bool flag;
       try
       {
           KeyInfo keyInfo = new KeyInfo();
           var clause = new KeyInfoX509Data(cert);
           keyInfo.AddClause(clause);

            XmlElement signatureElement = GetSignatureElement(xmlElement);
            if (signatureElement == null)
            {
                string message = "The XML does not contain a signature.";
                throw new SAMLSignatureException(message);
            }
            signedXml.LoadXml(signatureElement);
            if (keyInfo != null)
            {
                signedXml.KeyInfo = keyInfo;
            }
            SetSigningKeyFromKeyInfo(signedXml);
            flag = signedXml.CheckSignature(cert.PublicKey.Key);
        }
        catch (Exception exception)
        {
            throw new SAMLSignatureException("Failed to verify the XML signature.", exception);
        }
        return flag;
    }

 private static void SetSigningKeyFromKeyInfo(SignedXml signedXml)
    {
        IEnumerator enumerator = signedXml.KeyInfo.GetEnumerator();
        while (enumerator.MoveNext())
        {
            if (enumerator.Current is KeyInfoX509Data)
            {
                var current = (KeyInfoX509Data) enumerator.Current;
                if (current.Certificates.Count != 0)
                {
                    var certificate = (X509Certificate) current.Certificates[0];
                    var certificate2 = new X509Certificate2(certificate);