Organize codebase files into folders and install MySql in devcontaine…

…r automatically (#207)

* Move files into new directories server, server/src, and create_db_wrapper

TESTED with `make run_all` and the new `make docker_run` command to simulate the production environment.

* Automtically install MySql in devcontainer for use by existing Go unit tests.

TESTED by building the devcontainer, following setup instructions from README and finally running `make test`.

* Fix GitHub workflow after organizing files

* Format webpack.config.js after moving it to frontend/ directory

.devcontainer/Dockerfile

@@ -0,0 +1,7 @@
+FROM mcr.microsoft.com/devcontainers/base:bullseye
+
+# Install MySql for Go backend unit tests. The tests expect MySql to be
+# installed directly in the development environment rather than in another
+# docker continer such as the dev database used for manual testing.
+COPY install-mysql.sh .
+RUN chmod +x ./install-mysql.sh && ./install-mysql.sh

.devcontainer/devcontainer.json

@@ -1,9 +1,10 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 // README at: https://github.com/devcontainers/templates/tree/main/src/debian
 {
-	"name": "Debian",
-	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-	"image": "mcr.microsoft.com/devcontainers/base:bullseye",
+	"name": "DxE ADB",
+	"build": {
+		"dockerfile": "Dockerfile"
+	},
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
 	"features": {
@@ -28,7 +29,8 @@
 			"extensions": [
 				"ms-vscode.makefile-tools",
 				"golang.go",
-				"Vue.volar"
+				"Vue.volar",
+				"GitHub.vscode-github-actions"
 			]
 		}
 	},

.devcontainer/install-mysql.sh

@@ -0,0 +1,31 @@
+# Original: https://gist.github.com/soubrunorocha/ec30b7704d737a1797b0281e97967834
+
+# Fail if any line fails and return an error so that building this Dockerfile
+# will not swallow the script's errors.
+set -e
+
+#set the root password
+DEFAULTPASS=""
+
+#set some config to avoid prompting
+sudo debconf-set-selections <<EOF
+mysql-community-server mysql-community-server/root-pass password $DEFAULTPASS
+mysql-community-server mysql-community-server/re-root-pass password $DEFAULTPASS
+EOF
+
+#get the mysql repository via wget
+wget https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb
+
+#set debian frontend to not prompt
+export DEBIAN_FRONTEND=noninteractive
+
+#config the package
+sudo -E dpkg -i mysql-apt-config_0.8.29-1_all.deb
+
+#update apt to get mysql repository
+sudo apt update
+
+#install mysql according to previous config
+sudo -E apt install mysql-server mysql-client --assume-yes --force-yes
+
+rm mysql-apt-config_0.8.29-1_all.deb

.github/workflows/main.yml

@@ -25,7 +25,9 @@ jobs:
           go version
       - name: Install deps
         run: |
+          cd ./frontend
           npm ci --legacy-peer-deps
+          cd ../server/src
           go mod download
       - name: Start database
         run: |
@@ -37,6 +39,7 @@ jobs:
       - name: Run tests
         run: |
           ./hooks/pre-commit
+          cd ./server/src
           go test github.com/dxe/adb/...
       - name: Deploy image to ECR
         if: ${{ github.event_name == 'push' }}  # don't deploy branches

.gitignore

@@ -5,7 +5,6 @@
 adb
 adb-config
 adb.db
-dist
-docker-compose.yml
-node_modules
+frontend/dist
+frontend/node_modules
 npm-debug.log

Dockerfile

@@ -2,44 +2,29 @@
 
 FROM golang:latest AS build-api
 WORKDIR /src
-COPY go.mod go.sum ./
+COPY server/src ./
 RUN GOFLAGS=-mod=readonly GOPROXY=https://proxy.golang.org go mod download
-COPY main.go ./
-COPY config config/
-COPY google_groups_sync google_groups_sync/
-COPY survey_mailer survey_mailer/
-COPY international_mailer international_mailer/
-COPY mailer mailer/
-COPY event_sync event_sync/
-COPY form_processor form_processor/
-COPY members members/
-COPY model model/
-COPY discord discord/
-COPY mailing_list_signup mailing_list_signup/
 RUN CGO_ENABLED=0 go build -o adb
 
-
 ## Build web UI frontend.   
 
 FROM node:16 AS build-ui
+
 WORKDIR /src
-COPY package.json package-lock.json ./
+COPY frontend ./
 RUN npm ci --legacy-peer-deps
-COPY tsconfig.json webpack.config.js ./
-COPY frontend frontend/
 RUN npm run build
 
-
 ## Assemble composite server container.
 
 FROM alpine:latest
 RUN apk add --no-cache ca-certificates tzdata
 RUN addgroup -S adb && adduser -S adb -G adb
 
 WORKDIR /app
-COPY run.sh ./
-COPY static static/
-COPY templates templates/
+COPY server/run.sh ./
+COPY server/templates templates/
+COPY frontend/static static/
 COPY --from=build-api /src/adb ./
 COPY --from=build-ui /src/dist dist/
 

Makefile

@@ -2,56 +2,71 @@
 
 # Runs the application.
 run_all:
-	npm run dev-build
+	cd frontend && npm run dev-build
 	$(MAKE) run
 
 # Just start the go program without recompiling the JS.
 run:
-	go install # Install first so that we keep cached build objects around.
+	cd server/src && go install # Install first so that we keep cached build objects around.
+
+	cd server/src; \
+	export TEMPLATES_DIRECTORY=../templates; \
+	export STATIC_DIRECTORY=../../frontend/static; \
+	export DIST_DIRECTORY=../../frontend/dist; \
 	go run main.go
 
 # Builds the frontend JS.
 js:
-	npm run dev-build
+	cd frontend && npm run dev-build
 
 # Automatically rebuild the JS when you edit a JS file. This is more
 # convenient then manually running `make run_all` every time you
 # update the JS. You'll need to do this in a separate terminal.
 watch:
-	npm run watch
+	cd frontend && npm run watch
 
 # Wipe and re-create the dev databases. See the readme for more
 # details.
 dev_db:
-	./scripts/create_db_wrapper.sh
+	cd server/scripts/create_db_wrapper && ./create_db_wrapper.sh
 
 # Install all deps for this project.
 deps:
-	npm install --legacy-peer-deps
-	go get -t github.com/dxe/adb/...
+	cd frontend && npm install --legacy-peer-deps
+	cd server/src && go get -t github.com/dxe/adb/...
 
 # Run all tests
 test:
-	go test ./...
+	cd server/src && go test ./...
 
 # Clean all built outputs
 clean:
-	rm -f adb
-	rm -rf dist
+	rm -f server/adb
+	rm -rf frontend/dist
 
 # Set git hooks
 set_git_hooks:
 	if [ ! -h .git/hooks/pre-commit ] ; then ln -s ../../hooks/pre-commit .git/hooks/pre-commit ; fi
 	if [ ! -h .git/hooks/pre-push ] ; then ln -s ../../hooks/pre-push .git/hooks/pre-push ; fi
 
+
+# Test docker image
+docker_run:
+	docker build . -t dxe/adb
+	docker container run --rm -p 8080:8080 -it --name adbtest dxe/adb
+
+# Open shell inside docker container while it's running
+docker_shell:
+	docker exec -it adbtest /bin/ash
+
 # Build the project for production.
 prod_build: clean set_git_hooks
-	./scripts/pull_adb_config.sh
-	npm run build
-	env GOOS=linux GOARCH=amd64 go build
+	cd server && ./scripts/pull_adb_config.sh
+	cd frontend && npm run build
+	cd server/src && env GOOS=linux GOARCH=amd64 go build
 
 # Reformat source files.
 # Keep in sync with hooks/pre-commit.
 fmt:
-	gofmt -w `find . -name '*.go'`
-	npx prettier --write frontend/*.{ts,vue}
+	cd server && gofmt -w `find . -name '*.go'`
+	cd frontend && npx prettier --write *.{ts,vue,js}

README.md

@@ -25,7 +25,7 @@ make deps
 There is now a Docker container to run MySQL locally, so just run the following:
 
 ```bash
-docker compose up -d
+( cd server/ && docker compose up -d )
 make dev_db
 ```
 
@@ -47,7 +47,7 @@ Please run `make fmt` and `make test` before sending a pull request.
 
 This project uses webpack to compile our frontend files. Frontend
 files that need to be compiled are in `frontend/`, and the compiled
-outputs are in `dist/`.
+outputs are in `frontend/dist/`.
 
  * package.json: file with all frontend dependencies
 

tsconfig.json → frontend/tsconfig.json

@@ -12,9 +12,9 @@
     "types": ["jquery"],
   },
   "include": [
-    "./frontend/*",
+    "./*",
   ],
   "exclude": [
-    "./frontend/external/**/*",
+    "./external/**/*",
   ],
 }

webpack.config.js → frontend/webpack.config.js

@@ -1,13 +1,13 @@
-var path = require('path')
-var webpack = require('webpack')
-var VueLoaderPlugin = require('vue-loader/lib/plugin')
+var path = require('path');
+var webpack = require('webpack');
+var VueLoaderPlugin = require('vue-loader/lib/plugin');
 
 module.exports = {
   // List of bundles to create. If you want to add a new page, you'll
   // need to also add it here.
   entry: {
-    adb: './frontend/adb',
-    flash_message: './frontend/flash_message',
+    adb: './adb',
+    flash_message: './flash_message',
   },
 
   output: {
@@ -17,9 +17,7 @@ module.exports = {
     libraryTarget: 'var',
   },
 
-  plugins: [
-    new VueLoaderPlugin(),
-  ],
+  plugins: [new VueLoaderPlugin()],
 
   module: {
     rules: [
@@ -33,7 +31,7 @@ module.exports = {
         exclude: /node_modules/,
         options: {
           appendTsSuffixTo: [/\.vue$/],
-          reportFiles: ['!frontend/external/**/*.ts'],
+          reportFiles: ['!external/**/*.ts'],
         },
       },
       {
@@ -46,24 +44,21 @@ module.exports = {
       },
       {
         test: /\.css$/,
-        use: [
-          { loader: "style-loader" },
-          { loader: "css-loader" }
-        ]
-      }
-    ]
+        use: [{ loader: 'style-loader' }, { loader: 'css-loader' }],
+      },
+    ],
   },
   resolve: {
     extensions: ['.js', '.ts'],
     alias: {
-      'vue$': 'vue/dist/vue.esm.js'
+      vue$: 'vue/dist/vue.esm.js',
     },
   },
   devServer: {
     historyApiFallback: true,
-    noInfo: true
+    noInfo: true,
   },
   performance: {
-    hints: false
+    hints: false,
   },
-}
+};

scripts/create_db_wrapper.sh → server/scripts/create_db_wrapper/create_db_wrapper.sh

@@ -7,4 +7,4 @@ if [ -z ${DXE_DEV_EMAIL+x} ]; then
     echo "Please enter your development email or set DXE_DEV_EMAIL to never see this message again: "
     read DXE_DEV_EMAIL
 fi
-go run ./scripts/create_db.go --dev-email="${DXE_DEV_EMAIL}"
+go run ./create_db.go --dev-email="${DXE_DEV_EMAIL}"

server/scripts/create_db_wrapper/go.mod

@@ -0,0 +1,7 @@
+module github.com/dxe/adb/scripts/create_db_wrapper
+
+go 1.13
+
+require github.com/dxe/adb v0.0.0
+
+replace github.com/dxe/adb => ../../src