Demo and versioning fixes

.dockerignore

@@ -22,9 +22,7 @@ tools
 build.sh
 LICENSE
 Makefile
-README.md
 
-mlte/frontend/nuxt-app/package-lock.json
 mlte/frontend/nuxt-app/.nuxt/
 mlte/frontend/nuxt-app/.output/
 mlte/frontend/nuxt-app/dist/

.github/workflows/ci.yaml

@@ -26,15 +26,15 @@ jobs:
       - name: Format with black
         run: make check-format
       - name: Lint with flake8
-        run: make check-lint
+        run: make lint
       - name: Typecheck with mypy
-        run: make check-typecheck
+        run: make typecheck
       - name: Check docs
         run: make docs 
       - name: Execute unit tests
         run: make test
       - name: Vet schemas
-        run: make vet
+        run: make check-schema
 
   docker:
     runs-on: ubuntu-latest

Makefile

@@ -1,5 +1,26 @@
 # Automation of various common tasks
 
+# -----------------------------------------------------------------------------
+# Schema Generation / Vetting
+# -----------------------------------------------------------------------------
+
+.PHONY: schema
+schema:
+	poetry run python tools/schema.py generate mlte --verbose
+
+.PHONY: check-schema
+check-schema:
+	poetry run python tools/schema.py vet mlte --verbose
+
+# -----------------------------------------------------------------------------
+# Doc building/checking.
+# -----------------------------------------------------------------------------
+
+# Doc generation.
+.PHONY: docs
+docs:
+	cd docs && poetry run mkdocs build --strict
+
 # -----------------------------------------------------------------------------
 # QA
 # -----------------------------------------------------------------------------
@@ -48,37 +69,13 @@ lint:
 	poetry run flake8 test/
 	poetry run flake8 tools/
 
-.PHONY: check-lint
-check-lint: lint
-
 # Typecheck all source code
 .PHONY: typecheck
 typecheck:
 	poetry run mypy mlte/
 	poetry run mypy test/
 	poetry run mypy tools/
 
-.PHONY: check-typecheck
-check-typecheck: typecheck
-
-# Doc generation.
-.PHONY: docs
-docs:
-	cd docs && poetry run mkdocs build --strict
-
-# Clean cache files
-.PHONY: clean
-clean: 
-	rm -r -f .mypy_cache .pytest_cache
-
-# All quality assurance
-.PHONY: qa
-qa: isort format lint typecheck
-
-# Check all QA tasks
-.PHONY: check
-check: check-isort check-format check-lint check-typecheck
-
 # -----------------------------------------------------------------------------
 # Unit Tests
 # -----------------------------------------------------------------------------
@@ -88,32 +85,28 @@ check: check-isort check-format check-lint check-typecheck
 test:
 	poetry run pytest --cov=mlte test 
 
-# Open coverage results in a browser
-.PHONY: cov-results
-cov-results:
-	coverage html && open htmlcov/index.html
-
 # Demo Jupyter Notebook tests
 .PHONY: demo-test
 demo-test:
 	bash demo/simple/test.sh
 	bash demo/scenarios/test.sh
 
 # -----------------------------------------------------------------------------
-# Schema Generation / Vetting
+# Shorthand actions and checks needed to update and review for pushing.
 # -----------------------------------------------------------------------------
 
-.PHONY: gen
-gen:
-	poetry run python tools/schema.py generate mlte --verbose
+# All quality assurance, as well as schema generation
+.PHONY: qa
+qa: schema isort format lint typecheck docs
 
-.PHONY: vet
-vet:
-	poetry run python tools/schema.py vet mlte --verbose
+# Check all QA tasks
+.PHONY: check-qa
+check-qa: check-schema check-isort check-format lint typecheck docs
 
+# Clean cache files
+.PHONY: clean
+clean: 
+	rm -r -f .mypy_cache .pytest_cache
 
-# -----------------------------------------------------------------------------
-# All actions and checks needed to update and review for pushing.
-# -----------------------------------------------------------------------------
 .PHONY: ci
-ci: clean gen qa docs test
+ci: clean check-qa test

demo/scenarios/demo_script.txt

@@ -1,14 +1,15 @@
-Pre requirements:
-
- - Have environment running; to do this, execute run_environment.sh from demo/scenarios.
- - Have already executed all notebooks that will be shown, so the results can be seen in the notebooks themselves.
+Pre-Demo Steps:
+ - Execute the Requirements and Demos steps (and optionally the Python Version Support step if needed) in the Setup section in https://mlte.readthedocs.io/en/latest/development/#quickstart. 
+ - Execute all notebooks in demo/scenarios so that the results can be seen in the notebooks themselves. This can be done in VSCode, or in another program that runs Jupyter Notebooks. Be sure to run the notebooks in the virtual environment created at the beginning, so that all dependencies are installed.
+ - Ensure that the Docker daemon is running.
+ - Execute run_environment.sh from demo/scenarios to get the frontend and backend working.
 
 Demo:
  - Start with slides explaining what MLTE is.
  - Continue slides including a diagram of the whole process. Use this diagram later with more slides to indicate where in the process we are as we move forward through the demo.
- - Log into frontend (browser, localhost:8000), show Negotiation Card. Explain important parts of negotiation card and how it is refined.
- - Open up Jupyter Notebook with Spec (the one in demo/scenarios/1_requirements.ipynb. Note that we may need a slightly modified version if we add a data quality QAS. Skip the Negotiation Card definition, but do show the context initialization, and then the Spec definition. Walk though it without getting into too many details.
+ - Log into frontend (browser, localhost:8000, user=admin, pwd = admin1234), show Negotiation Card. Explain important parts of negotiation card and how it is refined.
+ - Open the Jupyter Notebook with Spec (the one in demo/scenarios/1_requirements.ipynb. Skip the Negotiation Card definition, but do show the context initialization, and then the Spec definition. Walk though it without getting into too many details.
  - Go back to frontend. Show Test Catalog, and show how people would use it find an example of how to measure certain QAS.
- - Open up one Jupyter Notebooks with QAS test measurements, the demo/scenarios/2a_evidence_fairness one, and walk through it.
- - Open up the demo/scenarios/3_report.ipynb Notebook, and show the process for validating and generating the report.
+ - Open one of the Jupyter Notebooks with QAS test measurements, the demo/scenarios/2a_evidence_fairness one, and walk through it.
+ - Open the demo/scenarios/3_report.ipynb Notebook, and show the process for validating and generating the report.
  - Go back to frontend, and show the report results.

demo/scenarios/get_model.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-wget 'https://docs.google.com/uc?export=download&id=15kAII1kOPGIAI46OP01ecNkq4tdf5yXw' -O ./model/model_f_a.h5
+curl -o ./model/model_f_a.h5 -L "https://docs.google.com/uc?export=download&id=15kAII1kOPGIAI46OP01ecNkq4tdf5yXw"

docker/Dockerfile.backend

@@ -4,21 +4,12 @@
 
 FROM mlte-base
 
-# Install deps.
-WORKDIR /mnt/app
-COPY pyproject.toml /mnt/app
-COPY poetry.lock /mnt/app
-RUN poetry install --no-root
+# Create the default storage location.
+RUN mkdir /mnt/store
 
-# Copy and install mlte
-COPY mlte /mnt/app/mlte
+# Install MLTE.
+WORKDIR /mnt/app/
 RUN poetry install
 
-# Create the default storage location
-RUN mkdir /mnt/store
-
-# Run the mlte backend server
+# Set the MLTE backend server as the entry point.
 ENTRYPOINT ["poetry", "run", "mlte", "backend"]
-
-# By default, run a local filesystem store
-#CMD ["--store-uri", "fs:///mnt/store"]

docker/Dockerfile.frontend

@@ -2,39 +2,40 @@
 #
 # Image definition for MLTE frontend container.
 
-FROM mlte-base
+# First build the static website in a separate image.
+FROM mlte-python AS build
 
-# Set up node.
+# Set up Node.js
 RUN apt-get update && apt-get install -y npm
 
-# Set up dependencies.
+# Install Javascript dependencies.
 WORKDIR /mnt/app/mlte/frontend/nuxt-app
 COPY mlte/frontend/nuxt-app/package.json /mnt/app/mlte/frontend/nuxt-app/package.json
-#RUN npm i --package-lock-only
-RUN npm install && npm cache clean --force
+COPY mlte/frontend/nuxt-app/package-lock.json /mnt/app/mlte/frontend/nuxt-app/package-lock.json
+RUN npm ci && npm cache clean --force
 
-# Setup USWDS design dependencies.
+# Build USWDS design dependencies.
 COPY mlte/frontend/nuxt-app/assets /mnt/app/mlte/frontend/nuxt-app/assets
 COPY mlte/frontend/nuxt-app/gulpfile.js /mnt/app/mlte/frontend/nuxt-app/
 RUN npx gulp init
 
-# Install Python deps.
-WORKDIR /mnt/app
-COPY pyproject.toml /mnt/app
-COPY poetry.lock /mnt/app
-RUN poetry install --no-root
-
-# Create static server.
+# Build the static website, including .env file with specific configs for frontend.
 COPY mlte/frontend/nuxt-app /mnt/app/mlte/frontend/nuxt-app
 COPY docker/deployment/.env.frontend /mnt/app/mlte/frontend/nuxt-app/.env
 WORKDIR /mnt/app/mlte/frontend/nuxt-app
 RUN npm run build
 
-# Copy and install rest of MLTE.
-COPY mlte /mnt/app/mlte
+###############################################################################
+
+# Create the final frontend image, with the static website.
+FROM mlte-base
+
+# Copy the static web site back into the MLTE folder structure.
+COPY --from=build /mnt/app/mlte/frontend/nuxt-app/.output /mnt/app/mlte/frontend/nuxt-app/.output
+
+# Install MLTE.
 WORKDIR /mnt/app/
 RUN poetry install
 
-# Run the mlte frontend server
-WORKDIR /mnt/app
+# Set the MLTE frontend server as the entry point.
 ENTRYPOINT ["poetry", "run", "mlte", "ui"]

docker/Dockerfile.mlte

@@ -1,26 +1,24 @@
 # Dockerfile.mlte
 #
-# Image definition for MLTE base container.
+# Image definition for MLTE common code, to be used by backend and frontend.
 
-FROM python:3.10.14-slim-bookworm
+FROM mlte-python
 
 # Base deps for compiling wheels.
 RUN apt-get update && apt-get install -y build-essential
 
-# Set up certificates for any proxies that can get in the middle of curl/wget commands during the build
-# NOTE: put any CA certificates needed for a proxy in the ./certs folder in the root of this repo, in PEM format
-# but with a .crt extensions, so they can be loaded into the container and used for SSL connections properly.
-RUN apt-get install -y ca-certificates
-RUN mkdir /certs
-COPY ./certs/ /certs/
-RUN if [ -n "$(ls -A /certs/*.crt)" ]; then \
-      cp -rf /certs/*.crt /usr/local/share/ca-certificates/; \
-      update-ca-certificates; \
-    fi
+# Ensure we are not buffering output for clarity.
+ENV PYTHONUNBUFFERED=1
 
 # Setup poetry to handle dependencies.
 RUN pip install poetry
 
-ENV PYTHONUNBUFFERED=1
+# Install Python deps.
+WORKDIR /mnt/app
+COPY pyproject.toml /mnt/app
+COPY poetry.lock /mnt/app
+COPY README.md /mnt/app
+RUN poetry install --no-root
 
-# NOTE: MLTE code is not copied and left for the specific sub dockerfiles to do, to allow caching their specific dependencies.
+# Copy the rest of the MLTE code.
+COPY mlte /mnt/app/mlte

docker/Dockerfile.python

@@ -0,0 +1,16 @@
+# Dockerfile.python
+#
+# Image definition for base Python image with proxy support.
+
+FROM python:3.10.14-slim-bookworm
+
+# Set up certificates for any proxies that can get in the middle of curl/wget commands during the build
+# NOTE: put any CA certificates needed for a proxy in the ./certs folder in the root of this repo, in PEM format
+# but with a .crt extensions, so they can be loaded into the container and used for SSL connections properly.
+RUN apt-get install -y ca-certificates
+RUN mkdir /certs
+COPY ./certs/ /certs/
+RUN if [ -n "$(ls -A /certs/*.crt)" ]; then \
+      cp -rf /certs/*.crt /usr/local/share/ca-certificates/; \
+      update-ca-certificates; \
+    fi

docker/build.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 # Build base first.
 cd ..
+docker build -t mlte-python . -f docker/Dockerfile.python
 docker build -t mlte-base . -f docker/Dockerfile.mlte
 cd docker