From d85dab3ce2c0b057cd5146d28b2123c16163da99 Mon Sep 17 00:00:00 2001 From: Pete Wall Date: Fri, 20 Aug 2021 16:49:02 -0500 Subject: [PATCH] Properly paginate the product list Signed-off-by: Pete Wall --- cmd/curl.go | 15 +++++-- cmd/products.go | 76 +++++++++++++++++++++------------- cmd/products_test.go | 91 ++++++++++++++++++++++++++++++++++++++++- lib/marketplace.go | 27 +++++++----- lib/marketplace_test.go | 36 ++++++++++++++-- 5 files changed, 197 insertions(+), 48 deletions(-) diff --git a/cmd/curl.go b/cmd/curl.go index ddf2706..408fbfd 100644 --- a/cmd/curl.go +++ b/cmd/curl.go @@ -14,22 +14,29 @@ import ( ) func init() { - rootCmd.AddCommand(CurlCmd) + rootCmd.AddCommand(curlCmd) + curlCmd.SetOut(curlCmd.OutOrStdout()) } -var CurlCmd = &cobra.Command{ +var curlCmd = &cobra.Command{ Use: "curl", Hidden: true, PreRunE: GetRefreshToken, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { cmd.SilenceUsage = true - req, err := MarketplaceConfig.MakeGetRequest(args[0], url.Values{}) + + requestUrl, err := url.Parse(args[0]) + if err != nil { + return err + } + + req, err := MarketplaceConfig.MakeGetRequest(requestUrl.Path, requestUrl.Query()) if err != nil { return err } - cmd.Printf("Sending %s request to %s...\n", req.Method, req.URL.String()) + cmd.PrintErrf("Sending %s request to %s...\n", req.Method, req.URL.String()) resp, err := Client.Do(req) if err != nil { return err diff --git a/cmd/products.go b/cmd/products.go index bcafd34..fc959d6 100644 --- a/cmd/products.go +++ b/cmd/products.go @@ -17,6 +17,8 @@ import ( "github.com/vmware-labs/marketplace-cli/v2/models" ) +var allOrgs bool = false + func init() { rootCmd.AddCommand(ProductCmd) ProductCmd.AddCommand(ListProductsCmd) @@ -24,6 +26,7 @@ func init() { ProductCmd.PersistentFlags().StringVarP(&OutputFormat, "output-format", "f", FormatTable, "Output format") ListProductsCmd.Flags().StringVar(&SearchTerm, "search-text", "", "Filter by text") + ListProductsCmd.Flags().BoolVarP(&allOrgs, "all-orgs", "a", false, "Show products from all organizations") GetProductCmd.Flags().StringVarP(&ProductSlug, "product", "p", "", "Product slug") _ = GetProductCmd.MarkFlagRequired("product") @@ -46,6 +49,10 @@ type ListProductResponsePayload struct { Message string `json:"string"` StatusCode int `json:"statuscode"` Products []*models.Product `json:"dataList"` + Params struct { + ProductCount int `json:"itemsnumber"` + Pagination *Pagination `json:"pagination"` + } `json:"params"` } var ListProductsCmd = &cobra.Command{ @@ -53,44 +60,55 @@ var ListProductsCmd = &cobra.Command{ Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { values := url.Values{ - "pagination": Pagination(0, 20), - "ownOrg": []string{"true"}, + "managed": []string{strconv.FormatBool(!allOrgs)}, } if SearchTerm != "" { values.Set("search", SearchTerm) } - req, err := MarketplaceConfig.MakeGetRequest("/api/v1/products", values) - if err != nil { - cmd.SilenceUsage = true - return fmt.Errorf("preparing the request for the list of products failed: %w", err) - } - - resp, err := Client.Do(req) - if err != nil { - cmd.SilenceUsage = true - return fmt.Errorf("sending the request for the list of products failed: %w", err) - } - - if resp.StatusCode != http.StatusOK { - cmd.SilenceUsage = true - return fmt.Errorf("getting the list of products failed: (%d) %s", resp.StatusCode, resp.Status) - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - cmd.SilenceUsage = true - return fmt.Errorf("failed to read the list of products: %w", err) + var products []*models.Product + totalProducts := 1 + pagination := &Pagination{ + Page: 1, + PageSize: 20, } - response := &ListProductResponse{} - err = json.Unmarshal(body, response) - if err != nil { - cmd.SilenceUsage = true - return fmt.Errorf("failed to parse the list of products: %w", err) + for ; len(products) < totalProducts; pagination.Page += 1 { + req, err := MarketplaceConfig.MakeGetRequest("/api/v1/products", values) + if err != nil { + cmd.SilenceUsage = true + return fmt.Errorf("preparing the request for the list of products failed: %w", err) + } + + req.URL = pagination.Apply(req.URL) + resp, err := Client.Do(req) + if err != nil { + cmd.SilenceUsage = true + return fmt.Errorf("sending the request for the list of products failed: %w", err) + } + + if resp.StatusCode != http.StatusOK { + cmd.SilenceUsage = true + return fmt.Errorf("getting the list of products failed: (%d) %s", resp.StatusCode, resp.Status) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + cmd.SilenceUsage = true + return fmt.Errorf("failed to read the list of products: %w", err) + } + + response := &ListProductResponse{} + err = json.Unmarshal(body, response) + if err != nil { + cmd.SilenceUsage = true + return fmt.Errorf("failed to parse the list of products: %w", err) + } + totalProducts = response.Response.Params.ProductCount + products = append(products, response.Response.Products...) } - err = RenderProductList(OutputFormat, response.Response.Products, cmd.OutOrStdout()) + err := RenderProductList(OutputFormat, products, cmd.OutOrStdout()) if err != nil { cmd.SilenceUsage = true return fmt.Errorf("failed to render the list of products: %w", err) diff --git a/cmd/products_test.go b/cmd/products_test.go index ef0dad0..31f9a61 100644 --- a/cmd/products_test.go +++ b/cmd/products_test.go @@ -88,7 +88,7 @@ var _ = Describe("Products", func() { request := httpClient.DoArgsForCall(0) Expect(request.Method).To(Equal("GET")) Expect(request.URL.Path).To(Equal("/api/v1/products")) - Expect(request.URL.Query().Get("pagination")).To(Equal("{\"page\":0,\"pagesize\":20}")) + Expect(request.URL.Query().Get("pagination")).To(Equal("{\"page\":1,\"pageSize\":20}")) }) By("outputting the response", func() { @@ -119,6 +119,95 @@ var _ = Describe("Products", func() { }) }) + Context("Multiple pages of results", func() { + BeforeEach(func() { + var products []*models.Product + for i := 0; i < 30; i += 1 { + product := CreateFakeProduct( + "", + fmt.Sprintf("My Super Product %d", i), + fmt.Sprintf("my-super-product-%d", i), + "PENDING") + AddVerions(product, "1.0.0") + products = append(products, product) + } + + response1 := &cmd.ListProductResponse{ + Response: &cmd.ListProductResponsePayload{ + Products: products[:20], + StatusCode: http.StatusOK, + Params: struct { + ProductCount int `json:"itemsnumber"` + Pagination *lib.Pagination `json:"pagination"` + }{ + ProductCount: len(products), + Pagination: &lib.Pagination{ + Enabled: true, + Page: 1, + PageSize: 20, + }, + }, + Message: "testing", + }, + } + response2 := &cmd.ListProductResponse{ + Response: &cmd.ListProductResponsePayload{ + Products: products[20:], + StatusCode: http.StatusOK, + Params: struct { + ProductCount int `json:"itemsnumber"` + Pagination *lib.Pagination `json:"pagination"` + }{ + ProductCount: len(products), + Pagination: &lib.Pagination{ + Enabled: true, + Page: 1, + PageSize: 20, + }, + }, + Message: "testing", + }, + } + responseBytes, err := json.Marshal(response1) + Expect(err).ToNot(HaveOccurred()) + + httpClient.DoReturnsOnCall(0, &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader(responseBytes)), + }, nil) + + responseBytes, err = json.Marshal(response2) + Expect(err).ToNot(HaveOccurred()) + + httpClient.DoReturnsOnCall(1, &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader(responseBytes)), + }, nil) + }) + + It("returns all results", func() { + err := cmd.ListProductsCmd.RunE(cmd.ListProductsCmd, []string{}) + Expect(err).ToNot(HaveOccurred()) + + By("sending the correct requests", func() { + Expect(httpClient.DoCallCount()).To(Equal(2)) + request := httpClient.DoArgsForCall(0) + Expect(request.Method).To(Equal("GET")) + Expect(request.URL.Path).To(Equal("/api/v1/products")) + Expect(request.URL.Query().Get("pagination")).To(Equal("{\"page\":1,\"pageSize\":20}")) + + request = httpClient.DoArgsForCall(1) + Expect(request.Method).To(Equal("GET")) + Expect(request.URL.Path).To(Equal("/api/v1/products")) + Expect(request.URL.Query().Get("pagination")).To(Equal("{\"page\":2,\"pageSize\":20}")) + }) + + By("outputting the response", func() { + Expect(stdout).To(Say("TOTAL COUNT: 30")) + }) + }) + }) + Context("Error fetching products", func() { BeforeEach(func() { httpClient.DoReturns(nil, fmt.Errorf("request failed")) diff --git a/lib/marketplace.go b/lib/marketplace.go index c467190..f8c4cbe 100644 --- a/lib/marketplace.go +++ b/lib/marketplace.go @@ -4,7 +4,6 @@ package lib import ( - "encoding/json" "fmt" "io" "net/http" @@ -35,21 +34,29 @@ var ( } ) -type PaginationObject struct { +type Pagination struct { + Enabled bool `json:"enabled"` Page int32 `json:"page"` PageSize int32 `json:"pagesize"` } -func (p PaginationObject) ToUrlValue() []string { - data, _ := json.Marshal(p) - return []string{string(data)} +func (p Pagination) QueryString() string { + return fmt.Sprintf("pagination={%%22page%%22:%d,%%22pageSize%%22:%d}", p.Page, p.PageSize) } -func Pagination(page, pageSize int32) []string { - return PaginationObject{ - Page: page, - PageSize: pageSize, - }.ToUrlValue() +func (p Pagination) Apply(input *url.URL) *url.URL { + values := input.Query() + delete(values, "pagination") + + output := *input + output.RawQuery = values.Encode() + if len(values) == 0 { + output.RawQuery = p.QueryString() + } else { + output.RawQuery += "&" + p.QueryString() + } + + return &output } func (m *MarketplaceConfiguration) MakeRequest(method, path string, params url.Values, header map[string]string, content io.Reader) (*http.Request, error) { diff --git a/lib/marketplace_test.go b/lib/marketplace_test.go index 3c81396..b787e13 100644 --- a/lib/marketplace_test.go +++ b/lib/marketplace_test.go @@ -19,10 +19,38 @@ var TestConfig = &lib.MarketplaceConfiguration{ } var _ = Describe("Pagination", func() { - It("returns a valid pagination URL value", func() { - pagination := lib.Pagination(1, 25) - Expect(pagination).To(HaveLen(1)) - Expect(pagination[0]).To(Equal(`{"page":1,"pagesize":25}`)) + Describe("Apply", func() { + It("modifies a URL to add pagination with a very specific encoding format", func() { + pagination := lib.Pagination{ + Page: 1, + PageSize: 25, + } + + By("appending to an existing query", func() { + baseUrl := &url.URL{ + Scheme: "https", + Host: "marketplace.vmware.com", + Path: "/api/products", + } + baseUrl.RawQuery = url.Values{ + "price": []string{"free"}, + }.Encode() + paginatedUrl := pagination.Apply(baseUrl) + Expect(paginatedUrl.RawQuery).To(Equal("price=free&pagination={%22page%22:1,%22pageSize%22:25}")) + Expect(paginatedUrl.String()).To(Equal("https://marketplace.vmware.com/api/products?price=free&pagination={%22page%22:1,%22pageSize%22:25}")) + }) + + By("working with urls without an existing query", func() { + baseUrl := &url.URL{ + Scheme: "https", + Host: "marketplace.vmware.com", + Path: "/api/products", + } + paginatedUrl := pagination.Apply(baseUrl) + Expect(paginatedUrl.RawQuery).To(Equal("pagination={%22page%22:1,%22pageSize%22:25}")) + Expect(paginatedUrl.String()).To(Equal("https://marketplace.vmware.com/api/products?pagination={%22page%22:1,%22pageSize%22:25}")) + }) + }) }) })