Merge pull request #215 from dxe/alex/devcontainer-docker-compose

Use containerized database in devcontainer and all unit tests

.devcontainer/compose.extend.yaml

@@ -0,0 +1,26 @@
+# Docker Compose configuration for development environment, for users of the
+# devcontainer. This file defines the devcontainer container itself as well as
+# adding configuration to other docker compose configuration.
+
+services:
+  # https://code.visualstudio.com/docs/devcontainers/create-dev-container
+  devcontainer:
+    image: "mcr.microsoft.com/devcontainers/base:1.0.9-bookworm"
+    volumes:
+      # Mounts the project folder to '/workspace', referenced by
+      # 'workspaceFolder' in devcontainer.json.
+      - ..:/workspace:cached
+    networks:
+      - dev-net
+    # Override default command to keep devcontainer alive as default command
+    # could exit immediately. Command taken from VS Code devcontainer docs.
+    command: /bin/sh -c "while sleep 1000; do :; done"
+    environment:
+      - DB_PROTOCOL=tcp(mysql:3306)
+  mysql:  # extending server/compose.yaml
+    networks: # make accessible to devcontainer via network
+      - dev-net
+
+networks:
+  dev-net:
+    driver: bridge

.devcontainer/devcontainer.json

@@ -2,9 +2,14 @@
 // README at: https://github.com/devcontainers/templates/tree/main/src/debian
 {
 	"name": "DxE ADB",
-	"build": {
-		"dockerfile": "Dockerfile"
-	},
+
+	// https://code.visualstudio.com/docs/devcontainers/create-dev-container#_use-docker-compose
+	"dockerComposeFile": ["../server/compose.yaml", "./compose.extend.yaml"],
+	"service": "devcontainer", // defined in dockerComposeFile
+	"runServices": ["devcontainer", "mysql"], // defined in dockerComposeFile
+	"workspaceFolder": "/workspace",
+	"shutdownAction": "stopCompose",
+	"remoteUser": "vscode",
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
 	"features": {
@@ -43,7 +48,4 @@
 
 	// Configure tool-specific properties.
 	// "customizations": {},
-
-	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
-	// "remoteUser": "root"
 }

.github/workflows/main.yml

@@ -36,6 +36,8 @@ jobs:
           mysql -h127.0.0.1 -P3306 -uroot -proot -e "GRANT ALL PRIVILEGES ON *.* to adb_user@localhost;"
           mysql -h127.0.0.1 -P3306 -uroot -proot -e "CREATE DATABASE adb_db CHARACTER SET utf8 COLLATE utf8_general_ci;"
           mysql -h127.0.0.1 -P3306 -uroot -proot -e "CREATE DATABASE adb_test_db CHARACTER SET utf8 COLLATE utf8_general_ci;"
+      - name: Set environment variables for tests
+        run: echo "DB_PROTOCOL=tcp(127.0.0.1:3306)" >> $GITHUB_ENV
       - name: Run tests
         run: |
           ./hooks/pre-commit

README.md

@@ -35,7 +35,7 @@ If you are using the devcontainer, just run this command in the container:
 make dev_db
 ```
 
-This command will log you into the database if you run the comman in the same
+This command will log you into the database if you run the command in the same
 container as the database:
 
 ```bash
@@ -55,9 +55,18 @@ make run_all
 
 Access the web app at http://localhost:8080.
 
-### Format and test
+### Code formatting
 
-Please run `make fmt` and `make test` before sending a pull request.
+Please run `make fmt` before sending a pull request.
+
+### Test
+
+Please run `make test` before sending a pull request.
+
+Note the tests take several minutes to run, even though some of them
+finish quickly. To check if any tests are hanging, you can add "-v -p 1" after
+"go test" args in the Makefile to show individual test names and to run them
+in serial.
 
 ### JS
 

server/compose.yaml

@@ -1,4 +1,4 @@
-version: "3.1"
+name: dxe-adb
 
 services:
   mysql:

server/scripts/create_db_wrapper/create_db.go

@@ -54,7 +54,7 @@ func createEventsDevDB() string {
 }
 
 func createDevDB(name string) {
-	db := model.NewDB(config.DBUser + ":" + config.DBPassword + "@/" + name + "?multiStatements=true")
+	db := model.NewDB(config.DataSourceBase + "/" + name + "?multiStatements=true")
 	defer db.Close()
 	model.WipeDatabase(db)
 	insertStatement := fmt.Sprintf(`

server/src/config/config.go

@@ -9,10 +9,11 @@ import (
 )
 
 var (
-	DBUser     = mustGetenv("DB_USER", "adb_user", true)
-	DBPassword = mustGetenv("DB_PASSWORD", "adbpassword", true)
-	DBName     = mustGetenv("DB_NAME", "adb_db", true)
-	DBProtocol = mustGetenv("DB_PROTOCOL", "", true)
+	DBUser         = mustGetenv("DB_USER", "adb_user", true)
+	DBPassword     = mustGetenv("DB_PASSWORD", "adbpassword", true)
+	DBName         = mustGetenv("DB_NAME", "adb_db", true)
+	DBProtocol     = mustGetenv("DB_PROTOCOL", "", true)
+	DataSourceBase = DBUser + ":" + DBPassword + "@" + DBProtocol
 
 	Port        = mustGetenv("PORT", "8080", true)
 	MembersPort = mustGetenv("MEMBERS_PORT", "8081", true)
@@ -133,15 +134,15 @@ func mustGetenvAsBool(key string) bool {
 }
 
 func DBDataSource() string {
-	connectionString := DBUser + ":" + DBPassword + "@" + DBProtocol + "/" + DBName + "?parseTime=true&charset=utf8mb4"
+	connectionString := DataSourceBase + "/" + DBName + "?parseTime=true&charset=utf8mb4"
 	if IsProd {
 		return connectionString + "&tls=true"
 	}
 	return connectionString
 }
 
 func DBTestDataSource() string {
-	return DBUser + ":" + DBPassword + "@/adb_test_db?parseTime=true"
+	return DataSourceBase + "/adb_test_db?parseTime=true"
 }
 
 var staticResourcesHash = strconv.FormatInt(rand.NewSource(time.Now().UnixNano()).Int63(), 10)

server/src/form_processor/main_test.go

@@ -0,0 +1,61 @@
+package form_processor
+
+import (
+	"database/sql"
+	"log"
+	"os"
+	"testing"
+
+	"github.com/dxe/adb/config"
+	"github.com/dxe/adb/model"
+	"github.com/jmoiron/sqlx"
+)
+
+// Do not use same DB as used by `models` package because go tests in different
+// packages run in parallel.
+const dbName = "adb_test_forms_db"
+
+// createTempDb creates a database for testing. Please call `dropTempDb` when
+// tests finish.
+func createTempDb(name string) {
+	db, err := sql.Open("mysql", config.DataSourceBase+"/")
+	if err != nil {
+		log.Fatalf("Error connecting to database: %v", err)
+	}
+	defer db.Close()
+
+	// Create the database
+	createDBQuery := "CREATE DATABASE IF NOT EXISTS " + name
+	_, err = db.Exec(createDBQuery)
+	if err != nil {
+		log.Fatalf("Error creating database: %v", err)
+	}
+}
+func dropTempDb(name string) {
+	db, err := sql.Open("mysql", config.DataSourceBase+"/")
+	if err != nil {
+		log.Fatalf("Error connecting to database: %v", err)
+	}
+	defer db.Close()
+
+	// Create the database
+	createDBQuery := "DROP DATABASE " + name
+	_, err = db.Exec(createDBQuery)
+	if err != nil {
+		log.Fatalf("Error creating database: %v", err)
+	}
+}
+func useTestDb() *sqlx.DB {
+	db := model.NewDB(config.DataSourceBase + "/" + dbName + "?parseTime=true")
+	model.WipeDatabase(db)
+	return db
+}
+
+func TestMain(m *testing.M) {
+	// Use a dedicated database for this package's tests. Tests in this package
+	// must run in serial, but may run in parallel with tests in other packages.
+	createTempDb(dbName)
+	defer dropTempDb(dbName)
+
+	os.Exit(m.Run())
+}

server/src/form_processor/process_forms_test.go

@@ -1,57 +1,23 @@
 package form_processor
 
 import (
-	"github.com/jmoiron/sqlx"
-	"github.com/lestrrat-go/test-mysqld"
 	"testing"
+
+	"github.com/jmoiron/sqlx"
 )
 
 /* Common utils */
 type activist struct {
 	id int
 }
 
-func createTables(t *testing.T) (*mysqltest.TestMysqld, *sqlx.DB) {
-	/* Set up MySQL */
-	mysqld, err := mysqltest.NewMysqld(nil)
-	if err != nil {
-		t.Fatalf("failed to start mysqld: %s", err)
-	}
-	db, err := sqlx.Open("mysql", mysqld.Datasource("test", "", "", 0))
-	if err != nil {
-		t.Fatalf("failed to open MySQL connection: %s", err)
-	}
-
-	/* Crate tables */
-	_, err = db.Exec(createTableFormApplicationQuery)
-	if err != nil {
-		t.Fatalf("createTableFormApplicationQuery failed: %s", err)
-	}
-	_, err = db.Exec(createTableActivistsQuery)
-	if err != nil {
-		t.Fatalf("createTableActivistsQuery failed: %s", err)
-	}
-	_, err = db.Exec(createTableWorkingGroupMembersQuery)
-	if err != nil {
-		t.Fatalf("createTableWorkingGroupMembersQuery failed: %s", err)
-	}
-	_, err = db.Exec(createTableCircleMembersQuery)
-	if err != nil {
-		t.Fatalf("createTableCircleMembersQuery failed: %s", err)
-	}
-	_, err = db.Exec(createTableFormInterestQuery)
-	if err != nil {
-		t.Fatalf("createTableFormInterestQuery failed: %s", err)
-	}
-	return mysqld, db
-}
-
 func verifyFormWasMarkedAsProcessed(t *testing.T, db *sqlx.DB, query string) {
 	rawActivists, err := db.Query(getActivistsQuery)
-	defer rawActivists.Close()
 	if err != nil {
 		t.Fatalf("getActivistsQuery failed: %s", err)
 	}
+	defer rawActivists.Close()
+
 	var activists []activist
 	for rawActivists.Next() {
 		var activist activist
@@ -73,8 +39,8 @@ func verifyFormWasMarkedAsProcessed(t *testing.T, db *sqlx.DB, query string) {
 /* Form application tests */
 func TestProcessFormApplicationForNoMatchingActivist(t *testing.T) {
 	/* Set up */
-	mysqld, db := createTables(t)
-	defer mysqld.Stop()
+	db := useTestDb()
+	defer db.Close()
 	_, err := db.Query(insertIntoFormApplicationQuery)
 	if err != nil {
 		t.Fatalf("insertIntoFormApplicationQuery failed: %s", err)
@@ -89,8 +55,9 @@ func TestProcessFormApplicationForNoMatchingActivist(t *testing.T) {
 
 func TestProcessFormApplicationForActivistMatchingOnName(t *testing.T) {
 	/* Set up */
-	mysqld, db := createTables(t)
-	defer mysqld.Stop()
+	db := useTestDb()
+	defer db.Close()
+
 	_, err := db.Exec(insertActivistQuery, "name1")
 	if err != nil {
 		t.Fatalf("insertActivistQuery failed: %s", err)
@@ -109,8 +76,9 @@ func TestProcessFormApplicationForActivistMatchingOnName(t *testing.T) {
 
 func TestProcessFormApplicationForActivistMatchingOnEmail(t *testing.T) {
 	/* Set up */
-	mysqld, db := createTables(t)
-	defer mysqld.Stop()
+	db := useTestDb()
+	defer db.Close()
+
 	_, err := db.Exec(insertActivistQuery, "non-matching_name")
 	if err != nil {
 		t.Fatalf("insertActivistQuery failed: %s", err)
@@ -129,8 +97,8 @@ func TestProcessFormApplicationForActivistMatchingOnEmail(t *testing.T) {
 
 func TestProcessFormApplicationForMultipleMatchingActivistsOnEmail(t *testing.T) {
 	/* Set up */
-	mysqld, db := createTables(t)
-	defer mysqld.Stop()
+	db := useTestDb()
+	defer db.Close()
 	_, err := db.Exec(insertActivistQuery, "non-matching_name1")
 	if err != nil {
 		t.Fatalf("insertActivistQuery failed: %s", err)
@@ -153,8 +121,8 @@ func TestProcessFormApplicationForMultipleMatchingActivistsOnEmail(t *testing.T)
 /* Form interest tests */
 func TestProcessFormInterestForNoMatchingActivist(t *testing.T) {
 	/* Set up */
-	mysqld, db := createTables(t)
-	defer mysqld.Stop()
+	db := useTestDb()
+	defer db.Close()
 	_, err := db.Exec(insertIntoFormInterestQuery)
 	if err != nil {
 		t.Fatalf("insertIntoFormInterestQuery failed: %s", err)
@@ -169,8 +137,8 @@ func TestProcessFormInterestForNoMatchingActivist(t *testing.T) {
 
 func TestProcessFormInterestForActivistMatchingOnName(t *testing.T) {
 	/* Set up */
-	mysqld, db := createTables(t)
-	defer mysqld.Stop()
+	db := useTestDb()
+	defer db.Close()
 	_, err := db.Exec(insertActivistQuery, "name1")
 	if err != nil {
 		t.Fatalf("insertActivistQuery failed: %s", err)
@@ -189,8 +157,8 @@ func TestProcessFormInterestForActivistMatchingOnName(t *testing.T) {
 
 func TestProcessFormInterestForActivistMatchingOnEmail(t *testing.T) {
 	/* Set up */
-	mysqld, db := createTables(t)
-	defer mysqld.Stop()
+	db := useTestDb()
+	defer db.Close()
 	_, err := db.Exec(insertActivistQuery, "non-matching_name")
 	if err != nil {
 		t.Fatalf("insertActivistQuery failed: %s", err)
@@ -209,8 +177,8 @@ func TestProcessFormInterestForActivistMatchingOnEmail(t *testing.T) {
 
 func TestProcessFormInterestForMultipleMatchingActivistsOnEmail(t *testing.T) {
 	/* Set up */
-	mysqld, db := createTables(t)
-	defer mysqld.Stop()
+	db := useTestDb()
+	defer db.Close()
 	_, err := db.Exec(insertActivistQuery, "non-matching_name1")
 	if err != nil {
 		t.Fatalf("insertActivistQuery failed: %s", err)