diff --git a/test/extended/imageapis/limitrange_admission.go b/test/extended/imageapis/limitrange_admission.go index 56dd45bec5f0..ab90ac4d8214 100644 --- a/test/extended/imageapis/limitrange_admission.go +++ b/test/extended/imageapis/limitrange_admission.go @@ -16,16 +16,16 @@ import ( quotautil "github.com/openshift/origin/pkg/quota/util" imagesutil "github.com/openshift/origin/test/extended/images" exutil "github.com/openshift/origin/test/extended/util" - testutil "github.com/openshift/origin/test/util" ) const limitRangeName = "limits" -var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/registry/serial][local] Image limit range", func() { +var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/registry/serial] Image limit range", func() { defer g.GinkgoRecover() + var oc = exutil.NewCLI("limitrange-admission", exutil.KubeConfigPath()) - g.JustBeforeEach(func() { + g.BeforeEach(func() { g.By("waiting for default service account") err := exutil.WaitForServiceAccount(oc.KubeClient().Core().ServiceAccounts(oc.Namespace()), "default") o.Expect(err).NotTo(o.HaveOccurred()) @@ -34,49 +34,30 @@ var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/regis o.Expect(err).NotTo(o.HaveOccurred()) }) - // needs to be run at the of of each It; cannot be run in AfterEach which is run after the project - // is destroyed - tearDown := func(oc *exutil.CLI) { - g.By(fmt.Sprintf("Deleting limit range %s", limitRangeName)) - oc.AdminKubeClient().Core().LimitRanges(oc.Namespace()).Delete(limitRangeName, nil) - - deleteTestImagesAndStreams(oc) - } - - g.It(fmt.Sprintf("[Skipped] should deny a push of built image exceeding %s limit", imageapi.LimitTypeImage), func() { - g.Skip("FIXME: fill image metadata for schema1 in the registry") - - defer tearDown(oc) - - dClient, err := testutil.NewDockerClient() - o.Expect(err).NotTo(o.HaveOccurred()) - - _, err = createLimitRangeOfType(oc, imageapi.LimitTypeImage, kapi.ResourceList{ + g.It(fmt.Sprintf("should deny a push of built image exceeding %s limit", imageapi.LimitTypeImage), func() { + _, err := createLimitRangeOfType(oc, imageapi.LimitTypeImage, kapi.ResourceList{ kapi.ResourceStorage: resource.MustParse("10Ki"), }) o.Expect(err).NotTo(o.HaveOccurred()) g.By(fmt.Sprintf("trying to push an image exceeding size limit with just 1 layer")) - err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "middle", 16000, 1, false) + err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "middle", 16000, 1, false) o.Expect(err).NotTo(o.HaveOccurred()) g.By(fmt.Sprintf("trying to push an image exceeding size limit in total")) - err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "middle", 16000, 5, false) + err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "middle", 16000, 5, false) o.Expect(err).NotTo(o.HaveOccurred()) g.By(fmt.Sprintf("trying to push an image with one big layer below size limit")) - err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "small", 8000, 1, true) + err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "small", 8000, 1, true) o.Expect(err).NotTo(o.HaveOccurred()) g.By(fmt.Sprintf("trying to push an image below size limit")) - err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "small", 8000, 2, true) + err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "small", 8000, 2, true) o.Expect(err).NotTo(o.HaveOccurred()) }) g.It(fmt.Sprintf("should deny a push of built image exceeding limit on %s resource", imageapi.ResourceImageStreamImages), func() { - - defer tearDown(oc) - limits := kapi.ResourceList{ imageapi.ResourceImageStreamTags: resource.MustParse("0"), imageapi.ResourceImageStreamImages: resource.MustParse("0"), @@ -84,37 +65,34 @@ var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/regis _, err := createLimitRangeOfType(oc, imageapi.LimitTypeImageStream, limits) o.Expect(err).NotTo(o.HaveOccurred()) - dClient, err := testutil.NewDockerClient() - o.Expect(err).NotTo(o.HaveOccurred()) - g.By(fmt.Sprintf("trying to push image exceeding limits %v", limits)) - err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "refused", imageSize, 1, false) + err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "refused", imageSize, 1, false) o.Expect(err).NotTo(o.HaveOccurred()) limits, err = bumpLimit(oc, imageapi.ResourceImageStreamImages, "1") o.Expect(err).NotTo(o.HaveOccurred()) g.By(fmt.Sprintf("trying to push image below limits %v", limits)) - err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "first", imageSize, 2, true) + err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "first", imageSize, 2, true) o.Expect(err).NotTo(o.HaveOccurred()) g.By(fmt.Sprintf("trying to push image exceeding limits %v", limits)) - err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "second", imageSize, 2, false) + err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "second", imageSize, 2, false) o.Expect(err).NotTo(o.HaveOccurred()) g.By(fmt.Sprintf("trying to push image below limits %v to another image stream", limits)) - err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "another", "second", imageSize, 1, true) + err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "another", "second", imageSize, 1, true) o.Expect(err).NotTo(o.HaveOccurred()) limits, err = bumpLimit(oc, imageapi.ResourceImageStreamImages, "2") o.Expect(err).NotTo(o.HaveOccurred()) g.By(fmt.Sprintf("trying to push image below limits %v", limits)) - err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "another", "third", imageSize, 1, true) + err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "another", "third", imageSize, 1, true) o.Expect(err).NotTo(o.HaveOccurred()) g.By(fmt.Sprintf("trying to push image exceeding limits %v", limits)) - err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "another", "fourth", imageSize, 1, false) + err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "another", "fourth", imageSize, 1, false) o.Expect(err).NotTo(o.HaveOccurred()) g.By(`removing tag "second" from "another" image stream`) @@ -122,14 +100,11 @@ var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/regis o.Expect(err).NotTo(o.HaveOccurred()) g.By(fmt.Sprintf("trying to push image below limits %v", limits)) - err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "another", "replenish", imageSize, 1, true) + err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "another", "replenish", imageSize, 1, true) o.Expect(err).NotTo(o.HaveOccurred()) }) g.It(fmt.Sprintf("should deny a docker image reference exceeding limit on %s resource", imageapi.ResourceImageStreamTags), func() { - - defer tearDown(oc) - tag2Image, err := buildAndPushTestImagesTo(oc, "src", "tag", 2) o.Expect(err).NotTo(o.HaveOccurred()) @@ -187,15 +162,12 @@ var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/regis }) g.It(fmt.Sprintf("should deny an import of a repository exceeding limit on %s resource", imageapi.ResourceImageStreamTags), func() { - maxBulkImport, err := getMaxImagesBulkImportedPerRepository() if err != nil { g.Skip(err.Error()) return } - defer tearDown(oc) - s1tag2Image, err := buildAndPushTestImagesTo(oc, "src1st", "tag", maxBulkImport+1) s2tag2Image, err := buildAndPushTestImagesTo(oc, "src2nd", "t", 2) o.Expect(err).NotTo(o.HaveOccurred()) @@ -234,15 +206,11 @@ var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/regis // buildAndPushTestImagesTo builds a given number of test images. The images are pushed to a new image stream // of given name under where X is a number of image starting from 1. func buildAndPushTestImagesTo(oc *exutil.CLI, isName string, tagPrefix string, numberOfImages int) (tag2Image map[string]imageapi.Image, err error) { - dClient, err := testutil.NewDockerClient() - if err != nil { - return - } tag2Image = make(map[string]imageapi.Image) for i := 1; i <= numberOfImages; i++ { tag := fmt.Sprintf("%s%d", tagPrefix, i) - dgst, _, err := imagesutil.BuildAndPushImageOfSizeWithDocker(oc, dClient, isName, tag, imageSize, 2, g.GinkgoWriter, true, true) + err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), isName, tag, imageSize, 2, true) if err != nil { return nil, err } @@ -250,9 +218,6 @@ func buildAndPushTestImagesTo(oc *exutil.CLI, isName string, tagPrefix string, n if err != nil { return nil, err } - if dgst != ist.Image.Name { - return nil, fmt.Errorf("digest of built image does not match stored: %s != %s", dgst, ist.Image.Name) - } tag2Image[tag] = ist.Image } @@ -316,7 +281,7 @@ func bumpLimit(oc *exutil.CLI, resourceName kapi.ResourceName, limit string) (ka func getMaxImagesBulkImportedPerRepository() (int, error) { max := os.Getenv("MAX_IMAGES_BULK_IMPORTED_PER_REPOSITORY") if len(max) == 0 { - return 0, fmt.Errorf("MAX_IMAGES_BULK_IMAGES_IMPORTED_PER_REPOSITORY is not set") + return 0, fmt.Errorf("MAX_IMAGES_BULK_IMPORTED_PER_REPOSITORY is not set") } return strconv.Atoi(max) } diff --git a/test/extended/images/helper.go b/test/extended/images/helper.go index 7133520a86c8..341e0252b924 100644 --- a/test/extended/images/helper.go +++ b/test/extended/images/helper.go @@ -2,12 +2,14 @@ package images import ( "bytes" + "context" cryptorand "crypto/rand" "crypto/tls" "fmt" "io" "io/ioutil" "net/http" + "net/url" "os" "path" "regexp" @@ -15,9 +17,10 @@ import ( "strings" "time" + distribution "github.com/docker/distribution" + "github.com/docker/distribution/manifest/schema2" dockerclient "github.com/fsouza/go-dockerclient" g "github.com/onsi/ginkgo" - godigest "github.com/opencontainers/go-digest" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -25,17 +28,19 @@ import ( knet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" + e2e "k8s.io/kubernetes/test/e2e/framework" imageapi "github.com/openshift/origin/pkg/image/apis/image" imagetypedclientset "github.com/openshift/origin/pkg/image/generated/internalclientset/typed/image/internalversion" + "github.com/openshift/origin/pkg/image/registryclient" exutil "github.com/openshift/origin/test/extended/util" testutil "github.com/openshift/origin/test/util" ) const ( // There are coefficients used to multiply layer data size to get a rough size of uploaded blob. - layerSizeMultiplierForDocker18 = 2.0 layerSizeMultiplierForLatestDocker = 0.8 defaultLayerSize = 1024 digestSHA256GzippedEmptyTar = godigest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4") @@ -214,8 +219,10 @@ func BuildAndPushImageOfSizeWithBuilder( } buildLog, logsErr := br.Logs() - if match := reSuccessfulBuild.FindStringSubmatch(buildLog); len(match) > 1 { - defer dClient.RemoveImageExtended(match[1], dockerclient.RemoveImageOptions{Force: true}) + if dClient != nil { + if match := reSuccessfulBuild.FindStringSubmatch(buildLog); len(match) > 1 { + defer dClient.RemoveImageExtended(match[1], dockerclient.RemoveImageOptions{Force: true}) + } } if !shouldSucceed { @@ -431,26 +438,35 @@ func pushImageWithDocker( return imageDigest, nil } +func newRandomBlob(size uint64) ([]byte, error) { + var letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") + + data := make([]byte, size) + if _, err := cryptorand.Read(data); err != nil { + return nil, err + } + + for i := range data { + data[i] = letters[uint(data[i])%uint(len(letters))] + } + + return data, nil +} + // createRandomBlob creates a random data with bytes from `letters` in order to let docker take advantage of // compression. Resulting layer size will be different due to file metadata overhead and compression. func createRandomBlob(dest string, size uint64) error { - var letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") - f, err := os.Create(dest) if err != nil { return err } defer f.Close() - data := make([]byte, size) - if _, err = cryptorand.Read(data); err != nil { + data, err := newRandomBlob(size) + if err != nil { return err } - for i := range data { - data[i] = letters[uint(data[i])%uint(len(letters))] - } - f.Write(data) return nil } @@ -815,3 +831,96 @@ func (c *CleanUpContainer) Run() { } } } + +func uploadBlob(ctx context.Context, repo distribution.Repository, blob []byte) (distribution.Descriptor, error) { + bs := repo.Blobs(ctx) + fmt.Fprintf(g.GinkgoWriter, "uploading blob (%d bytes)...\n", len(blob)) + return bs.Put(ctx, "", blob) +} + +func uploadManifest(ctx context.Context, repo distribution.Repository, tag string, config distribution.Descriptor, layers []distribution.Descriptor) (godigest.Digest, error) { + ms, err := repo.Manifests(ctx) + if err != nil { + return "", err + } + manifest, err := schema2.FromStruct(schema2.Manifest{ + Versioned: schema2.SchemaVersion, + Config: config, + Layers: layers, + }) + if err != nil { + return "", err + } + fmt.Fprintf(g.GinkgoWriter, "uploading manifest...\n") + return ms.Put(ctx, manifest, distribution.WithTag(tag)) +} + +// BuildAndPushMockImage tries to build a fake image of wanted size and number +// of layers. Built image is stored as an image stream tag :. If +// shouldSucceed is false, a push is expected to fail with a denied error. +// Returned is an image digest, and an error if any. +func BuildAndPushMockImage( + oc *exutil.CLI, + name, tag string, + size uint64, + numberOfLayers int, +) (godigest.Digest, error) { + e2e.Logf("Uploading mock image to %s:%s (~%d bytes, %d layers)...", name, tag, size, numberOfLayers) + + layerSize := (size + uint64(numberOfLayers) - 1) / uint64(numberOfLayers) // round up + + rt, err := rest.TransportFor(&rest.Config{}) + if err != nil { + return "", err + } + insecureRT, err := rest.TransportFor(&rest.Config{TLSClientConfig: rest.TLSClientConfig{Insecure: true}}) + if err != nil { + return "", err + } + + registryHost, err := GetDockerRegistryURL(oc) + if err != nil { + return "", err + } + registryURL, err := url.Parse(fmt.Sprintf("http://%s/", registryHost)) + if err != nil { + return "", err + } + + out, err := oc.Run("whoami").Args("-t").Output() + if err != nil { + return "", err + } + token := strings.TrimSpace(out) + + creds := registryclient.NewBasicCredentials() + creds.Add(&url.URL{Host: registryHost}, "unused", token) + registryContext := registryclient.NewContext(rt, insecureRT).WithCredentials(creds) + + ctx := context.Background() + insecure := true + repo, err := registryContext.Repository(ctx, registryURL, name, insecure) + if err != nil { + return "", err + } + + configDesc, err := uploadBlob(ctx, repo, []byte("{}")) + if err != nil { + return "", fmt.Errorf("unable to upload dummy config: %v", err) + } + + var layers []distribution.Descriptor + for i := 0; i < numberOfLayers; i++ { + blob, err := newRandomBlob(layerSize) + if err != nil { + return "", err + } + desc, err := uploadBlob(ctx, repo, blob) + if err != nil { + return "", fmt.Errorf("unable to upload random blob: %v", err) + } + layers = append(layers, desc) + } + + return uploadManifest(ctx, repo, tag, configDesc, layers) +}