Feature/master merge 20201119 (#18)

Update: Bring master new commits to Daemn branch 

This work was done by Dae.mn Team.
Co-authored-by: Akram Dweikat [email protected]
Co-authored-by: Akram Dweikat [email protected]
Co-authored-by: Damien Jade Duff [email protected]
Co-authored-by: Joe Major [email protected]

aws/cloudformation-templates/base/tables.yaml

@@ -44,14 +44,21 @@ Resources:
     Type: AWS::DynamoDB::Table
     Properties: 
       AttributeDefinitions: 
-        - 
-          AttributeName: "id"
+        - AttributeName: "id"
+          AttributeType: "S"
+        - AttributeName: "name"
           AttributeType: "S"
       KeySchema: 
-        - 
-          AttributeName: "id"
+        - AttributeName: "id"
           KeyType: "HASH"
       BillingMode: "PAY_PER_REQUEST"
+      GlobalSecondaryIndexes:
+        - IndexName: name-index
+          KeySchema:
+            - AttributeName: "name"
+              KeyType: "HASH"
+          Projection:
+            ProjectionType: ALL
 
   ExperimentStrategyTable:
     Type: AWS::DynamoDB::Table

src/.env

@@ -6,10 +6,16 @@ APPDATA=/tmp
 AWS_REGION=us-west-2
 AWS_DEFAULT_REGION=us-west-2
 
+# For services that depend on DDB, you can run it locally or connect to 
+# the real DDB running in your AWS account. To connect to it locally, 
+# uncomment the following line. Otherwise, comment the following line 
+# and setup your AWS credentials in environment variables.
+DDB_ENDPOINT_OVERRIDE=http://ddb:3001
+
 # Product service variables:
-# DynamoDB table names
-DDB_TABLE_PRODUCTS=
-DDB_TABLE_CATEGORIES=
+# DynamoDB table names (if connecting to DDB in your AWS account, change accordingly)
+DDB_TABLE_PRODUCTS=products
+DDB_TABLE_CATEGORIES=categories
 # Root URL to use when building fully qualified URLs to product detail view
 WEB_ROOT_URL=http://localhost:8080
 # Image root URL to use when building fully qualified URLs to product images

src/README.md

@@ -14,6 +14,8 @@ Docker Compose will load the [.env](./.env) file to resolve environment variable
 
 Some services, such as the [products](./products) and [recommendations](./recommendations) services, need to access AWS services running in your AWS account from your local machine. Given the differences between these container setups, different approaches are needed to pass in the AWS credentials needed to make these connections. For example, for the recommendations service we can map your local `~./.aws` configuration directory into the container's `/root` directory so the AWS SDK in the container can pick up the credentials it needs. Alternatively, since the products service is packaged from a [scratch image](https://hub.docker.com/_/scratch), credentials must be passed using the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` environment variables. In this case, rather than setting these variables in `.env` and risk exposing these values, consider setting these three variables in your shell environment. The following command can be used to obtain a session token which can be used to set your environment variables in your shell.
 
+> DynamoDB is one dependency that can be run locally rather than connecting to the real DynamoDB. This makes it easier to run services like the [products](./products) service completely local to you computer.
+
 ```console
 foo@bar:~$ aws sts get-session-token
 ```

src/aws-lambda/retaildemostore-lambda-load-products/src/main.go

@@ -23,17 +23,11 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-type MyEvent struct {
-	Table    string `json:"table"`
-	Bucket   string `json:"bucket"`
-	File     string `json:"file"`
-	Datatype string `json:"datatype"`
-}
-
 // Product Struct
 // using omitempty as a DynamoDB optimization to create indexes
 type Product struct {
 	ID             string  `json:"id" yaml:"id"`
+	URL            string  `json:"url" yaml:"url"`
 	SK             string  `json:"sk" yaml:"sk"`
 	Name           string  `json:"name" yaml:"name"`
 	Category       string  `json:"category" yaml:"category"`
@@ -43,6 +37,7 @@ type Product struct {
 	Image          string  `json:"image" yaml:"image"`
 	Featured       string  `json:"featured,omitempty" yaml:"featured,omitempty"`
 	GenderAffinity string  `json:"gender_affinity,omitempty" yaml:"gender_affinity,omitempty"`
+	CurrentStock   int     `json:"current_stock" yaml:"current_stock"`
 }
 
 // Products Array
@@ -81,6 +76,7 @@ var (
 	returnstring string
 )
 
+// DynamoDBPutItem - upserts item in DDB table
 func DynamoDBPutItem(item map[string]*dynamodb.AttributeValue, ddbtable string) {
 	input := &dynamodb.PutItemInput{
 		Item:      item,
@@ -102,7 +98,7 @@ func loadData(s3bucket, s3file, ddbtable, datatype string) (string, error) {
 
 	localfile := "/tmp/load.yaml"
 
-	log.Println("Attempting to load "+datatype+"file: ", s3bucket, s3file, localfile)
+	log.Println("Attempting to load "+datatype+" file: ", s3bucket, s3file, localfile)
 
 	file, err := os.Create(localfile)
 	if err != nil {
@@ -158,9 +154,10 @@ func loadData(s3bucket, s3file, ddbtable, datatype string) (string, error) {
 
 	log.Println("Loaded in ", time.Since(start))
 
-	return "data loaded from" + s3bucket + s3file + ddbtable + datatype + string(time.Since(start)), nil
+	return "data loaded from" + s3bucket + s3file + ddbtable + datatype + time.Since(start).String(), nil
 }
 
+// HandleRequest - handles Lambda request
 func HandleRequest(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) {
 	Bucket, _ := event.ResourceProperties["Bucket"].(string)
 	File, _ := event.ResourceProperties["File"].(string)

src/docker-compose.yml

@@ -11,6 +11,15 @@ services:
       - dev-net
     ports:
       - "8003:80"
+
+  ddb:
+    image: amazon/dynamodb-local
+    container_name: ddb
+    command: -jar DynamoDBLocal.jar -port 3001
+    networks:
+      - dev-net
+    ports:
+      - 3001:3001
 
   orders:
     container_name: orders
@@ -23,14 +32,18 @@ services:
 
   products:
     container_name: products
+    depends_on:
+      - ddb
     environment:
       - AWS_REGION
       - AWS_ACCESS_KEY_ID
       - AWS_SECRET_ACCESS_KEY
       - AWS_SESSION_TOKEN
       - DDB_TABLE_PRODUCTS
       - DDB_TABLE_CATEGORIES
+      - DDB_ENDPOINT_OVERRIDE
       - IMAGE_ROOT_URL
+      - WEB_ROOT_URL
     build:
       context: ./products
     networks:

src/products/Dockerfile

@@ -4,6 +4,8 @@ RUN apk add --no-cache git bash
 RUN go get -u github.com/gorilla/mux
 RUN go get -u gopkg.in/yaml.v2
 RUN go get -u github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute
+RUN go get -u github.com/jinzhu/copier
+RUN go get -u github.com/google/uuid
 COPY src/products-service/*.* /src/
 COPY src/products-service/data/*.* /src/data/
 RUN CGO_ENABLED=0 go build -o /bin/products-service

src/products/README.md

@@ -1,12 +1,12 @@
 # Retail Demo Store Products Service
 
-The Products web service provides a RESTful API for retrieving product information. The [Web UI](../web-ui) makes calls to this service when a user is viewing products and categories.
+The Products web service provides a RESTful API for retrieving product information. The [Web UI](../web-ui) makes calls to this service when a user is viewing products and categories and the Personalize workshop connects to this service to retrieve product information for building the items dataset.
 
-When deployed to AWS, CodePipeline is used to build and deploy the Products service as a Docker container to Amazon ECS behind an Application Load Balancer. The Products service can also be run locally in a Docker container. This makes it easier to iterate on and test changes locally before commiting.
+When deployed to AWS, CodePipeline is used to build and deploy the Products service as a Docker container in Amazon ECS behind an Application Load Balancer. The Products service can also be run locally in a Docker container. This makes it easier to iterate on and test changes locally before commiting.
 
 ## Local Development
 
-The Products service can be built and run locally (in Docker) using Docker Compose. See the [local development instructions](../) for details. **From the `../src` directory**, run the following command to build and deploy the service locally.
+The Products service can be built and run locally (in Docker) using Docker Compose. See the [local development instructions](../) for details. Since the Products service has a dependency on DynamoDB as its datastore, you can either connect to DynamoDB in your AWS account or run DynamoDB [locally](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.html) (default). The [docker-compose.yml](../docker-compose.yml) and default [.env](../.env) is already setup to run DynamoDB locally in Docker. If you want to connect to the real DynamoDB instead, you will need to configure your AWS credentials and comment the `DDB_ENDPOINT_OVERRIDE` environment variable since it is checked first. **From the `../src` directory**, run the following command to build and deploy the service locally.
 
 ```console
 foo@bar:~$ docker-compose up --build products

src/products/src/products-service/aws.go

@@ -4,25 +4,41 @@
 package main
 
 import (
+	"log"
 	"os"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/dynamodb"
-	"github.com/aws/aws-sdk-go/service/ssm"
 )
 
-//var sess *session.Session
 var sess, err = session.NewSession(&aws.Config{})
 
-// read DynamoDB tables env variable
-var ddb_table_products = os.Getenv("DDB_TABLE_PRODUCTS")
-var ddb_table_categories = os.Getenv("DDB_TABLE_CATEGORIES")
+// DynamoDB table names passed via environment
+var ddbTableProducts = os.Getenv("DDB_TABLE_PRODUCTS")
+var ddbTableCategories = os.Getenv("DDB_TABLE_CATEGORIES")
 
-var dynamoclient = dynamodb.New(sess)
-var ssm_client = ssm.New(sess)
+// Allow DDB endpoint to be overridden to support amazon/dynamodb-local
+var ddbEndpointOverride = os.Getenv("DDB_ENDPOINT_OVERRIDE")
+var runningLocal bool
 
-// Connect Stuff
+var dynamoClient *dynamodb.DynamoDB
+
+// Initialize clients
 func init() {
-
+	if len(ddbEndpointOverride) > 0 {
+		runningLocal = true
+		log.Println("Creating DDB client with endpoint override: ", ddbEndpointOverride)
+		creds := credentials.NewStaticCredentials("does", "not", "matter")
+		awsConfig := &aws.Config{
+			Credentials: creds,
+			Region:      aws.String("us-east-1"),
+			Endpoint:    aws.String(ddbEndpointOverride),
+		}
+		dynamoClient = dynamodb.New(sess, awsConfig)
+	} else {
+		runningLocal = false
+		dynamoClient = dynamodb.New(sess)
+	}
 }

src/products/src/products-service/category.go

@@ -4,12 +4,16 @@
 package main
 
 // Category Struct
+// IMPORTANT: if you change the shape of this struct, be sure to update the retaildemostore-lambda-load-products Lambda too!
 type Category struct {
 	ID    string `json:"id" yaml:"id"`
 	URL   string `json:"url" yaml:"url"`
 	Name  string `json:"name" yaml:"name"`
 	Image string `json:"image" yaml:"image"`
 }
 
+// Initialized - indicates if instance has been initialized or not
+func (c *Category) Initialized() bool { return c != nil && len(c.ID) > 0 }
+
 // Categories Array
 type Categories []Category