Skip to content

Commit

Permalink
summarize by hash id (#9)
Browse files Browse the repository at this point in the history
* summarize by hash id

* update readme
  • Loading branch information
tomoyamachi authored Dec 26, 2019
1 parent 5e3b6fb commit eba21c0
Show file tree
Hide file tree
Showing 8 changed files with 1,071 additions and 53 deletions.
64 changes: 40 additions & 24 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,36 +32,52 @@ $ trivy $IMAGENAME:$(dockertags -limit 1 -format json $IMAGENAME | jq -r .[0].ta
## Examples

```bash
$ dockertags goodwithtech/dockle
+---------+-------+----------------------+-------------+
| TAG | SIZE | CREATED AT | UPLOADED AT |
+---------+-------+----------------------+-------------+
| latest | 21.1M | 2019-12-16T14:05:18Z | NULL |
| v0.2.4 | 21.1M | 2019-12-05T05:18:04Z | NULL |
| v0.2.3 | 21.1M | 2019-11-17T15:03:10Z | NULL |
| v0.2.2 | 21.1M | 2019-11-17T14:45:53Z | NULL |
......
| v0.0.18 | 20.4M | 2019-06-10T18:31:45Z | NULL |
+---------+-------+----------------------+-------------+
$ dockertags alpine
+----------+------+----------------------+-------------+
| TAG | SIZE | CREATED AT | UPLOADED AT |
+----------+------+----------------------+-------------+
| 3 | 2.7M | 2019-12-24T20:40:57Z | NULL |
| 3.11 | | | |
| latest | | | |
| 3.11.2 | | | |
+----------+------+----------------------+-------------+
| edge | 2.7M | 2019-12-20T00:41:30Z | NULL |
| 20191219 | | | |
+----------+------+----------------------+-------------+
| 3.11.0 | 2.7M | 2019-12-20T00:41:21Z | NULL |
+----------+------+----------------------+-------------+
| 20191114 | 2.7M | 2019-11-14T22:41:11Z | NULL |
+----------+------+----------------------+-------------+
| 3.10 | 2.7M | 2019-10-21T18:41:18Z | NULL |
| 3.10.3 | | | |
+----------+------+----------------------+-------------+
| 20190925 | 2.7M | 2019-09-25T22:40:50Z | NULL |
+----------+------+----------------------+-------------+
| 3.10.2 | 2.7M | 2019-08-20T21:40:57Z | NULL |
+----------+------+----------------------+-------------+
| 3.8 | 2.1M | 2019-08-20T06:41:01Z | NULL |
| 3.8.4 | | | |
+----------+------+----------------------+-------------+
| 20190809 | 2.7M | 2019-08-09T21:41:13Z | NULL |
+----------+------+----------------------+-------------+
| 3.10.1 | 2.7M | 2019-07-11T22:41:17Z | NULL |
+----------+------+----------------------+-------------+



# You can set limit, filter and format
$ dockertags -limit 2 -contain v -contain 2 -format json goodwithtech/dockle
$ dockertags -limit 1 -contain latest -format json alpine
[
{
"tags": [
"v0.2.4"
],
"byte": 22154435,
"created_at": "2019-12-05T05:18:04.174078Z",
"uploaded_at": null
},
{
"tags": [
"v0.2.3"
"latest",
"3.11.2",
"3.11",
"3"
],
"byte": 22154435,
"created_at": "2019-11-17T15:03:10.914092Z",
"uploaded_at": null
"byte": 2801778,
"created_at": "2019-12-24T20:40:57.918177Z",
"uploaded_at": "0001-01-01T00:00:00Z"
}
]
```
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ require (
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
github.com/google/go-cmp v0.3.1
github.com/moul/http2curl v1.0.0 // indirect
github.com/olekukonko/tablewriter v0.0.2-0.20190618033246-cc27d85e17ce
github.com/opencontainers/go-digest v1.0.0-rc1
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
Expand Down
7 changes: 6 additions & 1 deletion internal/report/table.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package report
import (
"io"
"os"
"sort"
"strings"
"time"

Expand All @@ -21,14 +22,18 @@ func (w TableWriter) Write(tags types.ImageTags) (err error) {
table.SetHeader([]string{"Tag", "Size", "Created At", "Uploaded At"})

for _, tag := range tags {
targets := utils.StrByLen(tag.Tags)
sort.Sort(targets)

table.Append([]string{
strings.Join(tag.Tags, ","),
strings.Join(targets, tablewriter.NEWLINE),
getBytesize(tag.Byte),
ttos(tag.CreatedAt),
ttos(tag.UploadedAt),
})
}
table.SetAlignment(tablewriter.ALIGN_LEFT)
table.SetRowLine(true)
table.Render()

return nil
Expand Down
15 changes: 15 additions & 0 deletions internal/utils/strlen.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package utils

type StrByLen []string

func (a StrByLen) Len() int {
return len(a)
}

func (a StrByLen) Less(i, j int) bool {
return len(a[i]) < len(a[j])
}

func (a StrByLen) Swap(i, j int) {
a[i], a[j] = a[j], a[i]
}
103 changes: 75 additions & 28 deletions pkg/provider/dockerhub/dockerhub.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package dockerhub
import (
"context"
"fmt"
"sort"

"golang.org/x/sync/errgroup"

Expand Down Expand Up @@ -32,6 +33,19 @@ type ImageSummary struct {
Name string `json:"name"`
FullSize int `json:"full_size"`
LastUpdated string `json:"last_updated"`
Images Images `json:"images"`
}

type Images []Image
type Image struct {
Digest string `json:"digest"`
Architecture string `json:"architecture"`
}

func (t Images) Len() int { return len(t) }
func (t Images) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
func (t Images) Less(i, j int) bool {
return (t[i].Digest) > (t[j].Digest)
}

func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (types.ImageTags, error) {
Expand All @@ -46,14 +60,18 @@ func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt *
if err != nil {
return nil, err
}
imageTags := p.convertResultToTag(tagResp.Results)
if reqOpt.MaxCount > 0 && len(imageTags) > reqOpt.MaxCount {
return imageTags, nil
}
// imageTags := p.convertResultToTag(tagResp.Results)
// if reqOpt.MaxCount > 0 && len(imageTags) > reqOpt.MaxCount {
// return imageTags, nil
// }

// create all in one []ImageSummary
totalTagSummary := tagResp.Results

lastPage := calcMaxRequestPage(tagResp.Count, reqOpt.MaxCount, filterOpt)
// create ch (page - 1), already fetched first page,
tagsPerPage := make(chan types.ImageTags, lastPage-1)
//tagsPerPage := make(chan types.ImageTags, lastPage-1)
tagsPerPage := make(chan []ImageSummary, lastPage-1)
eg := errgroup.Group{}
for page := 2; page <= lastPage; page++ {
page := page
Expand All @@ -62,7 +80,7 @@ func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt *
if err != nil {
return err
}
tagsPerPage <- p.convertResultToTag(tagResp.Results)
tagsPerPage <- tagResp.Results
return nil
})
}
Expand All @@ -73,32 +91,59 @@ func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt *
for page := 2; page <= lastPage; page++ {
select {
case tags := <-tagsPerPage:
imageTags = append(imageTags, tags...)
totalTagSummary = append(totalTagSummary, tags...)
}
}
return imageTags, nil
return p.convertResultToTag(totalTagSummary), nil
}

func (p *DockerHub) convertResultToTag(summaries []ImageSummary) types.ImageTags {
tags := []types.ImageTag{}
for _, detail := range summaries {
if detail.Name == "" {
// TODO : refactor it

// create map : key is image hash
pools := map[string]types.ImageTag{}
for _, imageSummary := range summaries {
if imageSummary.Name == "" {
log.Logger.Debugf("no tag data :%v", imageSummary)
continue
}
createdAt, _ := time.Parse(time.RFC3339Nano, detail.LastUpdated)
tagNames := []string{detail.Name}
if !utils.MatchConditionTags(p.filterOpt, tagNames) {
if len(imageSummary.Images) == 0 {
log.Logger.Debugf("no image layer data :%v", imageSummary)
continue
}
tags = append(tags, types.ImageTag{
Tags: tagNames,
Byte: detail.FullSize,
CreatedAt: createdAt,
})
sort.Sort(imageSummary.Images)
firstHash := imageSummary.Images[0].Digest
target, ok := pools[firstHash]
// create first one if not exist
if !ok {
pools[firstHash] = createImageTag(imageSummary)
continue
}
// update exist ImageTag
target.Tags = append(target.Tags, imageSummary.Name)
pools[firstHash] = target
}

tags := []types.ImageTag{}
for _, imageTag := range pools {
if !utils.MatchConditionTags(p.filterOpt, imageTag.Tags) {
continue
}
tags = append(tags, imageTag)
}
return tags
}

func createImageTag(is ImageSummary) types.ImageTag {
createdAt, _ := time.Parse(time.RFC3339Nano, is.LastUpdated)
tagNames := []string{is.Name}
return types.ImageTag{
Tags: tagNames,
Byte: is.FullSize,
CreatedAt: createdAt,
}
}

// getTagResponse returns the tags for a specific repository.
// curl 'https://registry.hub.docker.com/v2/repositories/library/debian/tags/'
func getTagResponse(ctx context.Context, auth dockertypes.AuthConfig, timeout time.Duration, repository string, page int) (tagsResponse, error) {
Expand All @@ -113,13 +158,15 @@ func getTagResponse(ctx context.Context, auth dockertypes.AuthConfig, timeout ti
}

func calcMaxRequestPage(totalCnt, needCnt int, option *types.FilterOption) int {
maxPage := totalCnt/types.ImagePerPage + 1
if needCnt == 0 || len(option.Contain) != 0 {
return maxPage
}
needPage := needCnt/types.ImagePerPage + 1
if needPage >= maxPage {
return maxPage
}
return needPage
// TODO : currently always fetch all pages for show alias
return totalCnt/types.ImagePerPage + 1
// maxPage := totalCnt/types.ImagePerPage + 1
// if needCnt == 0 || len(option.Contain) != 0 {
// return maxPage
// }
// needPage := needCnt/types.ImagePerPage + 1
// if needPage >= maxPage {
// return maxPage
// }
// return needPage
}
85 changes: 85 additions & 0 deletions pkg/provider/dockerhub/dockerhub_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
package dockerhub

import (
"encoding/json"
"io/ioutil"
"sort"
"testing"

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"

"github.com/goodwithtech/dockertags/internal/types"
)

func TestScanImage(t *testing.T) {
testcases := map[string]struct {
filePath string
filterOpt types.FilterOption
expected types.ImageTags
}{
"debian page1": {
filePath: "./testdata/page1.json",
filterOpt: types.FilterOption{},
expected: types.ImageTags{
types.ImageTag{
Tags: []string{"unstable-slim", "unstable-20191118-slim"},
},
types.ImageTag{
Tags: []string{"unstable", "unstable-20191118"},
},
types.ImageTag{
Tags: []string{"testing-slim", "testing-20191118-slim"},
},
types.ImageTag{
Tags: []string{"testing-backports"},
},
types.ImageTag{
Tags: []string{"testing", "testing-20191118"},
},
types.ImageTag{
Tags: []string{"stretch-slim"},
},
},
},
"debian filter slim": {
filePath: "./testdata/page1.json",
filterOpt: types.FilterOption{Contain: []string{"slim"}},
expected: types.ImageTags{
types.ImageTag{
Tags: []string{"unstable-slim", "unstable-20191118-slim"},
},
types.ImageTag{
Tags: []string{"testing-slim", "testing-20191118-slim"},
},
types.ImageTag{
Tags: []string{"stretch-slim"},
},
},
},
}

for tc, v := range testcases {
dockerHub := DockerHub{filterOpt: &v.filterOpt}
var data tagsResponse
file, err := ioutil.ReadFile(v.filePath)
if err != nil {
t.Errorf("readfile error: %w", err)
continue
}
json.Unmarshal(file, &data)
actual := dockerHub.convertResultToTag(data.Results)
opts := []cmp.Option{
cmp.Transformer("Sort", func(in []string) []string {
out := append([]string{}, in...) // Copy input to avoid mutating it
sort.Strings(out)
return out
}),
cmpopts.IgnoreFields(types.ImageTag{}, "Byte", "CreatedAt"),
}
sort.Sort(actual)
if diff := cmp.Diff(v.expected, actual, opts...); diff != "" {
t.Errorf("%s: diff %v", tc, diff)
}
}
}
Loading

0 comments on commit eba21c0

Please sign in to comment.