diff --git a/test/config-next/ca.json b/test/config-next/ca.json index 37f69f6bbd7..3c1accbf7f5 100644 --- a/test/config-next/ca.json +++ b/test/config-next/ca.json @@ -77,7 +77,7 @@ "active": true, "issuerURL": "http://ca.example.org:4502/int-ecdsa-a", "ocspURL": "http://ca.example.org:4002/", - "crlURLBase": "http://ca.example.org:4501/ecdsa-a/", + "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/43104258997432926/", "location": { "configFile": "test/certs/webpki/int-ecdsa-a.pkcs11.json", "certFile": "test/certs/webpki/int-ecdsa-a.cert.pem", @@ -88,7 +88,7 @@ "active": true, "issuerURL": "http://ca.example.org:4502/int-ecdsa-b", "ocspURL": "http://ca.example.org:4002/", - "crlURLBase": "http://ca.example.org:4501/ecdsa-b/", + "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/17302365692836921/", "location": { "configFile": "test/certs/webpki/int-ecdsa-b.pkcs11.json", "certFile": "test/certs/webpki/int-ecdsa-b.cert.pem", @@ -99,7 +99,7 @@ "active": false, "issuerURL": "http://ca.example.org:4502/int-ecdsa-c", "ocspURL": "http://ca.example.org:4002/", - "crlURLBase": "http://ca.example.org:4501/ecdsa-c/", + "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/56560759852043581/", "location": { "configFile": "test/certs/webpki/int-ecdsa-c.pkcs11.json", "certFile": "test/certs/webpki/int-ecdsa-c.cert.pem", @@ -110,7 +110,7 @@ "active": true, "issuerURL": "http://ca.example.org:4502/int-rsa-a", "ocspURL": "http://ca.example.org:4002/", - "crlURLBase": "http://ca.example.org:4501/rsa-a/", + "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/29947985078257530/", "location": { "configFile": "test/certs/webpki/int-rsa-a.pkcs11.json", "certFile": "test/certs/webpki/int-rsa-a.cert.pem", @@ -121,7 +121,7 @@ "active": true, "issuerURL": "http://ca.example.org:4502/int-rsa-b", "ocspURL": "http://ca.example.org:4002/", - "crlURLBase": "http://ca.example.org:4501/rsa-b/", + "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/6762885421992935/", "location": { "configFile": "test/certs/webpki/int-rsa-b.pkcs11.json", "certFile": "test/certs/webpki/int-rsa-b.cert.pem", @@ -132,7 +132,7 @@ "active": false, "issuerURL": "http://ca.example.org:4502/int-rsa-c", "ocspURL": "http://ca.example.org:4002/", - "crlURLBase": "http://ca.example.org:4501/rsa-c/", + "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/56183656833365902/", "location": { "configFile": "test/certs/webpki/int-rsa-c.pkcs11.json", "certFile": "test/certs/webpki/int-rsa-c.cert.pem", diff --git a/test/config-next/crl-updater.json b/test/config-next/crl-updater.json index 86f7e601d3d..ae2a90ac333 100644 --- a/test/config-next/crl-updater.json +++ b/test/config-next/crl-updater.json @@ -53,7 +53,7 @@ "features": {} }, "syslog": { - "stdoutlevel": 6, + "stdoutlevel": 4, "sysloglevel": -1 }, "openTelemetry": { diff --git a/test/config-next/ocsp-responder.json b/test/config-next/ocsp-responder.json index bae65304459..4e506323328 100644 --- a/test/config-next/ocsp-responder.json +++ b/test/config-next/ocsp-responder.json @@ -53,9 +53,9 @@ ], "liveSigningPeriod": "60h", "timeout": "4.9s", - "maxInflightSignings": 2, - "maxSigningWaiters": 1, "shutdownStopTimeout": "10s", + "maxInflightSignings": 20, + "maxSigningWaiters": 100, "requiredSerialPrefixes": [ "7f" ], diff --git a/test/config/ca.json b/test/config/ca.json index 809d626ac34..32c5214524b 100644 --- a/test/config/ca.json +++ b/test/config/ca.json @@ -62,7 +62,7 @@ "active": true, "issuerURL": "http://ca.example.org:4502/int-ecdsa-a", "ocspURL": "http://ca.example.org:4002/", - "crlURLBase": "http://ca.example.org:4501/ecdsa-a/", + "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/43104258997432926/", "location": { "configFile": "test/certs/webpki/int-ecdsa-a.pkcs11.json", "certFile": "test/certs/webpki/int-ecdsa-a.cert.pem", @@ -73,7 +73,7 @@ "active": true, "issuerURL": "http://ca.example.org:4502/int-ecdsa-b", "ocspURL": "http://ca.example.org:4002/", - "crlURLBase": "http://ca.example.org:4501/ecdsa-b/", + "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/17302365692836921/", "location": { "configFile": "test/certs/webpki/int-ecdsa-b.pkcs11.json", "certFile": "test/certs/webpki/int-ecdsa-b.cert.pem", @@ -84,7 +84,7 @@ "active": false, "issuerURL": "http://ca.example.org:4502/int-ecdsa-c", "ocspURL": "http://ca.example.org:4002/", - "crlURLBase": "http://ca.example.org:4501/ecdsa-c/", + "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/56560759852043581/", "location": { "configFile": "test/certs/webpki/int-ecdsa-c.pkcs11.json", "certFile": "test/certs/webpki/int-ecdsa-c.cert.pem", @@ -95,7 +95,7 @@ "active": true, "issuerURL": "http://ca.example.org:4502/int-rsa-a", "ocspURL": "http://ca.example.org:4002/", - "crlURLBase": "http://ca.example.org:4501/rsa-a/", + "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/29947985078257530/", "location": { "configFile": "test/certs/webpki/int-rsa-a.pkcs11.json", "certFile": "test/certs/webpki/int-rsa-a.cert.pem", @@ -106,7 +106,7 @@ "active": true, "issuerURL": "http://ca.example.org:4502/int-rsa-b", "ocspURL": "http://ca.example.org:4002/", - "crlURLBase": "http://ca.example.org:4501/rsa-b/", + "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/6762885421992935/", "location": { "configFile": "test/certs/webpki/int-rsa-b.pkcs11.json", "certFile": "test/certs/webpki/int-rsa-b.cert.pem", @@ -117,7 +117,7 @@ "active": false, "issuerURL": "http://ca.example.org:4502/int-rsa-c", "ocspURL": "http://ca.example.org:4002/", - "crlURLBase": "http://ca.example.org:4501/rsa-c/", + "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/56183656833365902/", "location": { "configFile": "test/certs/webpki/int-rsa-c.pkcs11.json", "certFile": "test/certs/webpki/int-rsa-c.cert.pem", diff --git a/test/config/crl-updater.json b/test/config/crl-updater.json index 21f3603bb5d..d89ee38461c 100644 --- a/test/config/crl-updater.json +++ b/test/config/crl-updater.json @@ -53,7 +53,7 @@ "features": {} }, "syslog": { - "stdoutlevel": 6, - "sysloglevel": 6 + "stdoutlevel": 4, + "sysloglevel": 4 } } diff --git a/test/config/ocsp-responder.json b/test/config/ocsp-responder.json index c67aa41f736..2e4ba1461a5 100644 --- a/test/config/ocsp-responder.json +++ b/test/config/ocsp-responder.json @@ -59,6 +59,7 @@ "liveSigningPeriod": "60h", "timeout": "4.9s", "shutdownStopTimeout": "10s", + "maxInflightSignings": 20, "debugAddr": ":8005", "requiredSerialPrefixes": [ "7f" diff --git a/test/integration/crl_test.go b/test/integration/crl_test.go index fc7cc28a01a..c567047d962 100644 --- a/test/integration/crl_test.go +++ b/test/integration/crl_test.go @@ -11,6 +11,7 @@ import ( "path" "path/filepath" "strings" + "sync" "testing" "time" @@ -21,10 +22,16 @@ import ( "github.com/letsencrypt/boulder/test/vars" ) +// crlUpdaterMu controls access to `runUpdater`, because two crl-updaters running +// at once will result in errors trying to lease shards that are already leased. +var crlUpdaterMu sync.Mutex + // runUpdater executes the crl-updater binary with the -runOnce flag, and // returns when it completes. func runUpdater(t *testing.T, configFile string) { t.Helper() + crlUpdaterMu.Lock() + defer crlUpdaterMu.Unlock() binPath, err := filepath.Abs("bin/boulder") test.AssertNotError(t, err, "computing boulder binary path") diff --git a/test/integration/revocation_test.go b/test/integration/revocation_test.go index c6ae66d73e2..093b9ccd8c7 100644 --- a/test/integration/revocation_test.go +++ b/test/integration/revocation_test.go @@ -8,16 +8,23 @@ import ( "crypto/elliptic" "crypto/rand" "crypto/x509" + "encoding/hex" + "encoding/json" + "encoding/pem" "fmt" "io" "net/http" + "os" + "path" "strings" + "sync" "testing" "time" "github.com/eggsampler/acme/v3" "golang.org/x/crypto/ocsp" + "github.com/letsencrypt/boulder/crl/idp" "github.com/letsencrypt/boulder/test" ocsp_helper "github.com/letsencrypt/boulder/test/ocsp/helper" ) @@ -33,6 +40,119 @@ func isPrecert(cert *x509.Certificate) bool { return false } +// getALLCRLs fetches and parses each certificate for each configured CA. +// Returns a map from issuer SKID (hex) to a list of that issuer's CRLs. +func getAllCRLs(t *testing.T) map[string][]*x509.RevocationList { + t.Helper() + b, err := os.ReadFile(path.Join(os.Getenv("BOULDER_CONFIG_DIR"), "ca.json")) + if err != nil { + t.Fatalf("reading CA config: %s", err) + } + + var conf struct { + CA struct { + Issuance struct { + Issuers []struct { + CRLURLBase string + Location struct { + CertFile string + } + } + } + } + } + + err = json.Unmarshal(b, &conf) + if err != nil { + t.Fatalf("unmarshaling CA config: %s", err) + } + + ret := make(map[string][]*x509.RevocationList) + + for _, issuer := range conf.CA.Issuance.Issuers { + issuerPEMBytes, err := os.ReadFile(issuer.Location.CertFile) + if err != nil { + t.Fatalf("reading CRL issuer: %s", err) + } + + block, _ := pem.Decode(issuerPEMBytes) + issuerCert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("parsing CRL issuer: %s", err) + } + + issuerSKID := hex.EncodeToString(issuerCert.SubjectKeyId) + + // 10 is the number of shards configured in test/config*/crl-updater.json + for i := range 10 { + crlURL := fmt.Sprintf("%s%d.crl", issuer.CRLURLBase, i+1) + resp, err := http.Get(crlURL) + if err != nil { + t.Fatalf("getting CRL from %s: %s", crlURL, err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("fetching %s: status code %d", crlURL, resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("reading CRL from %s: %s", crlURL, err) + } + resp.Body.Close() + + list, err := x509.ParseRevocationList(body) + if err != nil { + t.Fatalf("parsing CRL from %s: %s (bytes: %x)", crlURL, err, body) + } + + err = list.CheckSignatureFrom(issuerCert) + if err != nil { + t.Errorf("checking CRL signature on %s from %s: %s", + crlURL, issuerCert.Subject, err) + } + + idpURIs, err := idp.GetIDPURIs(list.Extensions) + if err != nil { + t.Fatalf("getting IDP URIs: %s", err) + } + if len(idpURIs) != 1 { + t.Errorf("CRL at %s: expected 1 IDP URI, got %s", crlURL, idpURIs) + } + if idpURIs[0] != crlURL { + t.Errorf("fetched CRL from %s, got IDP of %s (should be same)", crlURL, idpURIs[0]) + } + + ret[issuerSKID] = append(ret[issuerSKID], list) + } + } + return ret +} + +func checkRevoked(t *testing.T, revocations map[string][]*x509.RevocationList, cert *x509.Certificate, reason int) { + t.Helper() + akid := hex.EncodeToString(cert.AuthorityKeyId) + if len(revocations[akid]) == 0 { + t.Errorf("no CRLs found for authorityKeyID %s", akid) + } + var matches []x509.RevocationListEntry + var count int + for _, list := range revocations[akid] { + for _, entry := range list.RevokedCertificateEntries { + count++ + if entry.SerialNumber.Cmp(cert.SerialNumber) == 0 { + matches = append(matches, entry) + } + } + } + if len(matches) == 0 { + t.Errorf("searching for %x in CRLs: no entry on combined CRLs of length %d", cert.SerialNumber, count) + } + for _, match := range matches { + if match.ReasonCode != reason { + t.Errorf("revoked certificate %x: got reason %d, want %d", cert.SerialNumber, match.ReasonCode, reason) + } + } +} + // TestRevocation tests that a certificate can be revoked using all of the // RFC 8555 revocation authentication mechanisms. It does so for both certs and // precerts (with no corresponding final cert), and for both the Unspecified and @@ -59,135 +179,161 @@ func TestRevocation(t *testing.T) { kind certKind } - var testCases []testCase - for _, kind := range []certKind{precert, finalcert} { - for _, reason := range []int{ocsp.Unspecified, ocsp.KeyCompromise} { - for _, method := range []authMethod{byAccount, byAuth, byKey} { - testCases = append(testCases, testCase{ - method: method, - reason: reason, - kind: kind, - // We do not expect any of these revocation requests to error. - // The ones done byAccount will succeed as requested, but will not - // result in the key being blocked for future issuance. - // The ones done byAuth will succeed, but will be overwritten to have - // reason code 5 (cessationOfOperation). - // The ones done byKey will succeed, but will be overwritten to have - // reason code 1 (keyCompromise), and will block the key. - }) - } - } - } + issueAndRevoke := func(tc testCase) *x509.Certificate { + issueClient, err := makeClient() + test.AssertNotError(t, err, "creating acme client") - for _, tc := range testCases { - name := fmt.Sprintf("%s_%d_%s", tc.kind, tc.reason, tc.method) - t.Run(name, func(t *testing.T) { - issueClient, err := makeClient() - test.AssertNotError(t, err, "creating acme client") + certKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + test.AssertNotError(t, err, "creating random cert key") - certKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - test.AssertNotError(t, err, "creating random cert key") + domain := random_domain() - domain := random_domain() + // Try to issue a certificate for the name. + var cert *x509.Certificate + switch tc.kind { + case finalcert: + res, err := authAndIssue(issueClient, certKey, []string{domain}, true) + test.AssertNotError(t, err, "authAndIssue failed") + cert = res.certs[0] + + case precert: + // Make sure the ct-test-srv will reject generating SCTs for the domain, + // so we only get a precert and no final cert. + err := ctAddRejectHost(domain) + test.AssertNotError(t, err, "adding ct-test-srv reject host") + + _, err = authAndIssue(issueClient, certKey, []string{domain}, true) + test.AssertError(t, err, "expected error from authAndIssue, was nil") + if !strings.Contains(err.Error(), "urn:ietf:params:acme:error:serverInternal") || + !strings.Contains(err.Error(), "SCT embedding") { + t.Fatal(err) + } - // Try to issue a certificate for the name. - var cert *x509.Certificate - switch tc.kind { - case finalcert: - res, err := authAndIssue(issueClient, certKey, []string{domain}, true) - test.AssertNotError(t, err, "authAndIssue failed") - cert = res.certs[0] - - case precert: - // Make sure the ct-test-srv will reject generating SCTs for the domain, - // so we only get a precert and no final cert. - err := ctAddRejectHost(domain) - test.AssertNotError(t, err, "adding ct-test-srv reject host") - - _, err = authAndIssue(issueClient, certKey, []string{domain}, true) - test.AssertError(t, err, "expected error from authAndIssue, was nil") - if !strings.Contains(err.Error(), "urn:ietf:params:acme:error:serverInternal") || - !strings.Contains(err.Error(), "SCT embedding") { - t.Fatal(err) - } + // Instead recover the precertificate from CT. + cert, err = ctFindRejection([]string{domain}) + if err != nil || cert == nil { + t.Fatalf("couldn't find rejected precert for %q", domain) + } + // And make sure the cert we found is in fact a precert. + if !isPrecert(cert) { + t.Fatal("precert was missing poison extension") + } - // Instead recover the precertificate from CT. - cert, err = ctFindRejection([]string{domain}) - if err != nil || cert == nil { - t.Fatalf("couldn't find rejected precert for %q", domain) - } - // And make sure the cert we found is in fact a precert. - if !isPrecert(cert) { - t.Fatal("precert was missing poison extension") - } + default: + t.Fatalf("unrecognized cert kind %q", tc.kind) + } - default: - t.Fatalf("unrecognized cert kind %q", tc.kind) - } + // Initially, the cert should have a Good OCSP response. + ocspConfig := ocsp_helper.DefaultConfig.WithExpectStatus(ocsp.Good) + _, err = ocsp_helper.ReqDER(cert.Raw, ocspConfig) + test.AssertNotError(t, err, "requesting OCSP for precert") - // Initially, the cert should have a Good OCSP response. - ocspConfig := ocsp_helper.DefaultConfig.WithExpectStatus(ocsp.Good) - _, err = ocsp_helper.ReqDER(cert.Raw, ocspConfig) - test.AssertNotError(t, err, "requesting OCSP for precert") + // Set up the account and key that we'll use to revoke the cert. + var revokeClient *client + var revokeKey crypto.Signer + switch tc.method { + case byAccount: + // When revoking by account, use the same client and key as were used + // for the original issuance. + revokeClient = issueClient + revokeKey = revokeClient.PrivateKey + + case byAuth: + // When revoking by auth, create a brand new client, authorize it for + // the same domain, and use that account and key for revocation. Ignore + // errors from authAndIssue because all we need is the auth, not the + // issuance. + revokeClient, err = makeClient() + test.AssertNotError(t, err, "creating second acme client") + _, _ = authAndIssue(revokeClient, certKey, []string{domain}, true) + revokeKey = revokeClient.PrivateKey - // Set up the account and key that we'll use to revoke the cert. - var revokeClient *client - var revokeKey crypto.Signer - switch tc.method { - case byAccount: - // When revoking by account, use the same client and key as were used - // for the original issuance. - revokeClient = issueClient - revokeKey = revokeClient.PrivateKey + case byKey: + // When revoking by key, create a brand new client and use it with + // the cert's key for revocation. + revokeClient, err = makeClient() + test.AssertNotError(t, err, "creating second acme client") + revokeKey = certKey + + default: + t.Fatalf("unrecognized revocation method %q", tc.method) + } - case byAuth: - // When revoking by auth, create a brand new client, authorize it for - // the same domain, and use that account and key for revocation. Ignore - // errors from authAndIssue because all we need is the auth, not the - // issuance. - revokeClient, err = makeClient() - test.AssertNotError(t, err, "creating second acme client") - _, _ = authAndIssue(revokeClient, certKey, []string{domain}, true) - revokeKey = revokeClient.PrivateKey + // Revoke the cert using the specified key and client. + err = revokeClient.RevokeCertificate( + revokeClient.Account, + cert, + revokeKey, + tc.reason, + ) + test.AssertNotError(t, err, "revocation should have succeeded") - case byKey: - // When revoking by key, create a brand new client and use it with - // the cert's key for revocation. - revokeClient, err = makeClient() - test.AssertNotError(t, err, "creating second acme client") - revokeKey = certKey + return cert + } - default: - t.Fatalf("unrecognized revocation method %q", tc.method) + // revocationCheck represents a deferred that a specific certificate is revoked. + // + // We defer these checks for performance reasons: we want to run crl-updater once, + // after all certificates have been revoked. + type revocationCheck func(t *testing.T, allCRLs map[string][]*x509.RevocationList) + var revocationChecks []revocationCheck + var rcMu sync.Mutex + var wg sync.WaitGroup + + for _, kind := range []certKind{precert, finalcert} { + for _, reason := range []int{ocsp.Unspecified, ocsp.KeyCompromise} { + for _, method := range []authMethod{byAccount, byAuth, byKey} { + wg.Add(1) + go func() { + defer wg.Done() + cert := issueAndRevoke(testCase{ + method: method, + reason: reason, + kind: kind, + // We do not expect any of these revocation requests to error. + // The ones done byAccount will succeed as requested, but will not + // result in the key being blocked for future issuance. + // The ones done byAuth will succeed, but will be overwritten to have + // reason code 5 (cessationOfOperation). + // The ones done byKey will succeed, but will be overwritten to have + // reason code 1 (keyCompromise), and will block the key. + }) + + // If the request was made by demonstrating control over the + // names, the reason should be overwritten to CessationOfOperation (5), + // and if the request was made by key, then the reason should be set to + // KeyCompromise (1). + expectedReason := reason + switch method { + case byAuth: + expectedReason = ocsp.CessationOfOperation + case byKey: + expectedReason = ocsp.KeyCompromise + default: + } + + check := func(t *testing.T, allCRLs map[string][]*x509.RevocationList) { + _, err := ocsp_helper.ReqDER(cert.Raw, ocsp_helper.DefaultConfig.WithExpectStatus(ocsp.Revoked).WithExpectReason(expectedReason)) + test.AssertNotError(t, err, "requesting OCSP for revoked cert") + + checkRevoked(t, allCRLs, cert, expectedReason) + } + + rcMu.Lock() + revocationChecks = append(revocationChecks, check) + rcMu.Unlock() + }() } + } + } - // Revoke the cert using the specified key and client. - err = revokeClient.RevokeCertificate( - revokeClient.Account, - cert, - revokeKey, - tc.reason, - ) + wg.Wait() - test.AssertNotError(t, err, "revocation should have succeeded") + runUpdater(t, path.Join(os.Getenv("BOULDER_CONFIG_DIR"), "crl-updater.json")) + allCRLs := getAllCRLs(t) - // Check the OCSP response for the certificate again. It should now be - // revoked. If the request was made by demonstrating control over the - // names, the reason should be overwritten to CessationOfOperation (5), - // and if the request was made by key, then the reason should be set to - // KeyCompromise (1). - ocspConfig = ocsp_helper.DefaultConfig.WithExpectStatus(ocsp.Revoked) - switch tc.method { - case byAuth: - ocspConfig = ocspConfig.WithExpectReason(ocsp.CessationOfOperation) - case byKey: - ocspConfig = ocspConfig.WithExpectReason(ocsp.KeyCompromise) - default: - ocspConfig = ocspConfig.WithExpectReason(tc.reason) - } - _, err = ocsp_helper.ReqDER(cert.Raw, ocspConfig) - test.AssertNotError(t, err, "requesting OCSP for revoked cert") - }) + for _, check := range revocationChecks { + check(t, allCRLs) } } diff --git a/test/startservers.py b/test/startservers.py index 93d0c25bcee..2d94c53e5df 100644 --- a/test/startservers.py +++ b/test/startservers.py @@ -84,7 +84,7 @@ ('akamai-test-srv',)), Service('s3-test-srv', 4501, None, None, - ('./bin/s3-test-srv', '--listen', 'localhost:4501'), + ('./bin/s3-test-srv', '--listen', ':4501'), None), Service('crl-storer', 9667, None, None,