Skip to content

Commit

Permalink
Make registry extended tests dockerless
Browse files Browse the repository at this point in the history
  • Loading branch information
Oleg Bulatov committed Nov 5, 2018
1 parent 45c74af commit abc5c15
Show file tree
Hide file tree
Showing 2 changed files with 139 additions and 65 deletions.
71 changes: 18 additions & 53 deletions test/extended/imageapis/limitrange_admission.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,16 @@ import (
quotautil "github.com/openshift/origin/pkg/quota/util"
imagesutil "github.com/openshift/origin/test/extended/images"
exutil "github.com/openshift/origin/test/extended/util"
testutil "github.com/openshift/origin/test/util"
)

const limitRangeName = "limits"

var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/registry/serial][local] Image limit range", func() {
var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/registry/serial] Image limit range", func() {
defer g.GinkgoRecover()

var oc = exutil.NewCLI("limitrange-admission", exutil.KubeConfigPath())

g.JustBeforeEach(func() {
g.BeforeEach(func() {
g.By("waiting for default service account")
err := exutil.WaitForServiceAccount(oc.KubeClient().Core().ServiceAccounts(oc.Namespace()), "default")
o.Expect(err).NotTo(o.HaveOccurred())
Expand All @@ -34,102 +34,77 @@ var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/regis
o.Expect(err).NotTo(o.HaveOccurred())
})

// needs to be run at the of of each It; cannot be run in AfterEach which is run after the project
// is destroyed
tearDown := func(oc *exutil.CLI) {
g.By(fmt.Sprintf("Deleting limit range %s", limitRangeName))
oc.AdminKubeClient().Core().LimitRanges(oc.Namespace()).Delete(limitRangeName, nil)

deleteTestImagesAndStreams(oc)
}

g.It(fmt.Sprintf("[Skipped] should deny a push of built image exceeding %s limit", imageapi.LimitTypeImage), func() {
g.Skip("FIXME: fill image metadata for schema1 in the registry")

defer tearDown(oc)

dClient, err := testutil.NewDockerClient()
o.Expect(err).NotTo(o.HaveOccurred())

_, err = createLimitRangeOfType(oc, imageapi.LimitTypeImage, kapi.ResourceList{
g.It(fmt.Sprintf("should deny a push of built image exceeding %s limit", imageapi.LimitTypeImage), func() {
_, err := createLimitRangeOfType(oc, imageapi.LimitTypeImage, kapi.ResourceList{
kapi.ResourceStorage: resource.MustParse("10Ki"),
})
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("trying to push an image exceeding size limit with just 1 layer"))
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "middle", 16000, 1, false)
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "middle", 16000, 1, false)
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("trying to push an image exceeding size limit in total"))
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "middle", 16000, 5, false)
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "middle", 16000, 5, false)
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("trying to push an image with one big layer below size limit"))
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "small", 8000, 1, true)
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "small", 8000, 1, true)
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("trying to push an image below size limit"))
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "small", 8000, 2, true)
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "small", 8000, 2, true)
o.Expect(err).NotTo(o.HaveOccurred())
})

g.It(fmt.Sprintf("should deny a push of built image exceeding limit on %s resource", imageapi.ResourceImageStreamImages), func() {

defer tearDown(oc)

limits := kapi.ResourceList{
imageapi.ResourceImageStreamTags: resource.MustParse("0"),
imageapi.ResourceImageStreamImages: resource.MustParse("0"),
}
_, err := createLimitRangeOfType(oc, imageapi.LimitTypeImageStream, limits)
o.Expect(err).NotTo(o.HaveOccurred())

dClient, err := testutil.NewDockerClient()
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("trying to push image exceeding limits %v", limits))
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "refused", imageSize, 1, false)
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "refused", imageSize, 1, false)
o.Expect(err).NotTo(o.HaveOccurred())

limits, err = bumpLimit(oc, imageapi.ResourceImageStreamImages, "1")
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("trying to push image below limits %v", limits))
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "first", imageSize, 2, true)
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "first", imageSize, 2, true)
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("trying to push image exceeding limits %v", limits))
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "second", imageSize, 2, false)
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "second", imageSize, 2, false)
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("trying to push image below limits %v to another image stream", limits))
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "another", "second", imageSize, 1, true)
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "another", "second", imageSize, 1, true)
o.Expect(err).NotTo(o.HaveOccurred())

limits, err = bumpLimit(oc, imageapi.ResourceImageStreamImages, "2")
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("trying to push image below limits %v", limits))
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "another", "third", imageSize, 1, true)
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "another", "third", imageSize, 1, true)
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("trying to push image exceeding limits %v", limits))
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "another", "fourth", imageSize, 1, false)
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "another", "fourth", imageSize, 1, false)
o.Expect(err).NotTo(o.HaveOccurred())

g.By(`removing tag "second" from "another" image stream`)
err = oc.ImageClient().Image().ImageStreamTags(oc.Namespace()).Delete("another:second", nil)
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("trying to push image below limits %v", limits))
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "another", "replenish", imageSize, 1, true)
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "another", "replenish", imageSize, 1, true)
o.Expect(err).NotTo(o.HaveOccurred())
})

g.It(fmt.Sprintf("should deny a docker image reference exceeding limit on %s resource", imageapi.ResourceImageStreamTags), func() {

defer tearDown(oc)

tag2Image, err := buildAndPushTestImagesTo(oc, "src", "tag", 2)
o.Expect(err).NotTo(o.HaveOccurred())

Expand Down Expand Up @@ -187,15 +162,12 @@ var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/regis
})

g.It(fmt.Sprintf("should deny an import of a repository exceeding limit on %s resource", imageapi.ResourceImageStreamTags), func() {

maxBulkImport, err := getMaxImagesBulkImportedPerRepository()
if err != nil {
g.Skip(err.Error())
return
}

defer tearDown(oc)

s1tag2Image, err := buildAndPushTestImagesTo(oc, "src1st", "tag", maxBulkImport+1)
s2tag2Image, err := buildAndPushTestImagesTo(oc, "src2nd", "t", 2)
o.Expect(err).NotTo(o.HaveOccurred())
Expand Down Expand Up @@ -234,25 +206,18 @@ var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/regis
// buildAndPushTestImagesTo builds a given number of test images. The images are pushed to a new image stream
// of given name under <tagPrefix><X> where X is a number of image starting from 1.
func buildAndPushTestImagesTo(oc *exutil.CLI, isName string, tagPrefix string, numberOfImages int) (tag2Image map[string]imageapi.Image, err error) {
dClient, err := testutil.NewDockerClient()
if err != nil {
return
}
tag2Image = make(map[string]imageapi.Image)

for i := 1; i <= numberOfImages; i++ {
tag := fmt.Sprintf("%s%d", tagPrefix, i)
dgst, _, err := imagesutil.BuildAndPushImageOfSizeWithDocker(oc, dClient, isName, tag, imageSize, 2, g.GinkgoWriter, true, true)
err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), isName, tag, imageSize, 2, true)
if err != nil {
return nil, err
}
ist, err := oc.ImageClient().Image().ImageStreamTags(oc.Namespace()).Get(isName+":"+tag, metav1.GetOptions{})
if err != nil {
return nil, err
}
if dgst != ist.Image.Name {
return nil, fmt.Errorf("digest of built image does not match stored: %s != %s", dgst, ist.Image.Name)
}
tag2Image[tag] = ist.Image
}

Expand Down Expand Up @@ -316,7 +281,7 @@ func bumpLimit(oc *exutil.CLI, resourceName kapi.ResourceName, limit string) (ka
func getMaxImagesBulkImportedPerRepository() (int, error) {
max := os.Getenv("MAX_IMAGES_BULK_IMPORTED_PER_REPOSITORY")
if len(max) == 0 {
return 0, fmt.Errorf("MAX_IMAGES_BULK_IMAGES_IMPORTED_PER_REPOSITORY is not set")
return 0, fmt.Errorf("MAX_IMAGES_BULK_IMPORTED_PER_REPOSITORY is not set")
}
return strconv.Atoi(max)
}
133 changes: 121 additions & 12 deletions test/extended/images/helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,40 +2,45 @@ package images

import (
"bytes"
"context"
cryptorand "crypto/rand"
"crypto/tls"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
"regexp"
"strconv"
"strings"
"time"

distribution "github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema2"
dockerclient "github.com/fsouza/go-dockerclient"
g "github.com/onsi/ginkgo"

godigest "github.com/opencontainers/go-digest"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kerrors "k8s.io/apimachinery/pkg/util/errors"
knet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/rest"
"k8s.io/client-go/util/retry"
e2e "k8s.io/kubernetes/test/e2e/framework"

imageapi "github.com/openshift/origin/pkg/image/apis/image"
imagetypedclientset "github.com/openshift/origin/pkg/image/generated/internalclientset/typed/image/internalversion"
"github.com/openshift/origin/pkg/image/registryclient"
exutil "github.com/openshift/origin/test/extended/util"
testutil "github.com/openshift/origin/test/util"
)

const (
// There are coefficients used to multiply layer data size to get a rough size of uploaded blob.
layerSizeMultiplierForDocker18 = 2.0
layerSizeMultiplierForLatestDocker = 0.8
defaultLayerSize = 1024
digestSHA256GzippedEmptyTar = godigest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")
Expand Down Expand Up @@ -214,8 +219,10 @@ func BuildAndPushImageOfSizeWithBuilder(
}
buildLog, logsErr := br.Logs()

if match := reSuccessfulBuild.FindStringSubmatch(buildLog); len(match) > 1 {
defer dClient.RemoveImageExtended(match[1], dockerclient.RemoveImageOptions{Force: true})
if dClient != nil {
if match := reSuccessfulBuild.FindStringSubmatch(buildLog); len(match) > 1 {
defer dClient.RemoveImageExtended(match[1], dockerclient.RemoveImageOptions{Force: true})
}
}

if !shouldSucceed {
Expand Down Expand Up @@ -431,26 +438,35 @@ func pushImageWithDocker(
return imageDigest, nil
}

func newRandomBlob(size uint64) ([]byte, error) {
var letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")

data := make([]byte, size)
if _, err := cryptorand.Read(data); err != nil {
return nil, err
}

for i := range data {
data[i] = letters[uint(data[i])%uint(len(letters))]
}

return data, nil
}

// createRandomBlob creates a random data with bytes from `letters` in order to let docker take advantage of
// compression. Resulting layer size will be different due to file metadata overhead and compression.
func createRandomBlob(dest string, size uint64) error {
var letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")

f, err := os.Create(dest)
if err != nil {
return err
}
defer f.Close()

data := make([]byte, size)
if _, err = cryptorand.Read(data); err != nil {
data, err := newRandomBlob(size)
if err != nil {
return err
}

for i := range data {
data[i] = letters[uint(data[i])%uint(len(letters))]
}

f.Write(data)
return nil
}
Expand Down Expand Up @@ -815,3 +831,96 @@ func (c *CleanUpContainer) Run() {
}
}
}

func uploadBlob(ctx context.Context, repo distribution.Repository, blob []byte) (distribution.Descriptor, error) {
bs := repo.Blobs(ctx)
fmt.Fprintf(g.GinkgoWriter, "uploading blob (%d bytes)...\n", len(blob))
return bs.Put(ctx, "", blob)
}

func uploadManifest(ctx context.Context, repo distribution.Repository, tag string, config distribution.Descriptor, layers []distribution.Descriptor) (godigest.Digest, error) {
ms, err := repo.Manifests(ctx)
if err != nil {
return "", err
}
manifest, err := schema2.FromStruct(schema2.Manifest{
Versioned: schema2.SchemaVersion,
Config: config,
Layers: layers,
})
if err != nil {
return "", err
}
fmt.Fprintf(g.GinkgoWriter, "uploading manifest...\n")
return ms.Put(ctx, manifest, distribution.WithTag(tag))
}

// BuildAndPushMockImage tries to build a fake image of wanted size and number
// of layers. Built image is stored as an image stream tag <name>:<tag>. If
// shouldSucceed is false, a push is expected to fail with a denied error.
// Returned is an image digest, and an error if any.
func BuildAndPushMockImage(
oc *exutil.CLI,
name, tag string,
size uint64,
numberOfLayers int,
) (godigest.Digest, error) {
e2e.Logf("Uploading mock image to %s:%s (~%d bytes, %d layers)...", name, tag, size, numberOfLayers)

layerSize := (size + uint64(numberOfLayers) - 1) / uint64(numberOfLayers) // round up

rt, err := rest.TransportFor(&rest.Config{})
if err != nil {
return "", err
}
insecureRT, err := rest.TransportFor(&rest.Config{TLSClientConfig: rest.TLSClientConfig{Insecure: true}})
if err != nil {
return "", err
}

registryHost, err := GetDockerRegistryURL(oc)
if err != nil {
return "", err
}
registryURL, err := url.Parse(fmt.Sprintf("http://%s/", registryHost))
if err != nil {
return "", err
}

out, err := oc.Run("whoami").Args("-t").Output()
if err != nil {
return "", err
}
token := strings.TrimSpace(out)

creds := registryclient.NewBasicCredentials()
creds.Add(&url.URL{Host: registryHost}, "unused", token)
registryContext := registryclient.NewContext(rt, insecureRT).WithCredentials(creds)

ctx := context.Background()
insecure := true
repo, err := registryContext.Repository(ctx, registryURL, name, insecure)
if err != nil {
return "", err
}

configDesc, err := uploadBlob(ctx, repo, []byte("{}"))
if err != nil {
return "", fmt.Errorf("unable to upload dummy config: %v", err)
}

var layers []distribution.Descriptor
for i := 0; i < numberOfLayers; i++ {
blob, err := newRandomBlob(layerSize)
if err != nil {
return "", err
}
desc, err := uploadBlob(ctx, repo, blob)
if err != nil {
return "", fmt.Errorf("unable to upload random blob: %v", err)
}
layers = append(layers, desc)
}

return uploadManifest(ctx, repo, tag, configDesc, layers)
}

0 comments on commit abc5c15

Please sign in to comment.