Merge branch 'master' into feature/validators

mlte-team · Jan 6, 2025 · 592bf6a · 592bf6a
2 parents 46b8f1d + e93af42
commit 592bf6a
Show file tree

Hide file tree

Showing 21 changed files with 99 additions and 482 deletions.
diff --git a/demo/scenarios/1_requirements.ipynb b/demo/scenarios/1_requirements.ipynb
diff --git a/demo/scenarios/2_evidence.ipynb b/demo/scenarios/2_evidence.ipynb
@@ -38,7 +38,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "env",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },
@@ -54,12 +54,7 @@
    "pygments_lexer": "ipython3",
    "version": "3.9.16"
   },
-  "orig_nbformat": 4,
-  "vscode": {
-   "interpreter": {
-    "hash": "82adda432962015d5f71beb9387a99f24d390514e497c776c87ff3434daf7312"
-   }
-  }
+  "orig_nbformat": 4
  },
  "nbformat": 4,
  "nbformat_minor": 2

diff --git a/demo/scenarios/2a_evidence_fairness.ipynb b/demo/scenarios/2a_evidence_fairness.ipynb
@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -40,7 +40,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -183,7 +183,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": ".venv-3.9",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },

diff --git a/demo/scenarios/2b_evidence_robustness.ipynb b/demo/scenarios/2b_evidence_robustness.ipynb
@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -41,7 +41,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -347,7 +347,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": ".venv-3.9",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },

diff --git a/demo/scenarios/2c_evidence_performance.ipynb b/demo/scenarios/2c_evidence_performance.ipynb
@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -93,7 +93,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": ".venv-3.9",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },

diff --git a/demo/scenarios/2d_evidence_interpretability.ipynb b/demo/scenarios/2d_evidence_interpretability.ipynb
@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -41,7 +41,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -73,7 +73,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -84,7 +84,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -175,7 +175,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -191,7 +191,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": ".venv-3.9",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },

diff --git a/demo/scenarios/2e_evidence_accuracy.ipynb b/demo/scenarios/2e_evidence_accuracy.ipynb
@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -41,7 +41,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -82,7 +82,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -132,7 +132,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": ".venv-3.9",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },

diff --git a/demo/scenarios/2f_evidence_interoperability.ipynb b/demo/scenarios/2f_evidence_interoperability.ipynb
@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -41,7 +41,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -96,7 +96,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },
@@ -110,7 +110,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.15"
+   "version": "3.9.16"
   }
  },
  "nbformat": 4,

diff --git a/demo/scenarios/2g_evidence_resilience.ipynb b/demo/scenarios/2g_evidence_resilience.ipynb
@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -41,7 +41,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -92,7 +92,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": ".venv-3.9",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },

diff --git a/demo/scenarios/2h_evidence_monitorability.ipynb b/demo/scenarios/2h_evidence_monitorability.ipynb
@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -40,7 +40,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -110,7 +110,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },
@@ -124,7 +124,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.15"
+   "version": "3.9.16"
   }
  },
  "nbformat": 4,

diff --git a/demo/scenarios/2i_evidence_performance_time.ipynb b/demo/scenarios/2i_evidence_performance_time.ipynb
@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -41,7 +41,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -92,7 +92,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": ".venv-3.9",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },

diff --git a/demo/scenarios/3_report.ipynb b/demo/scenarios/3_report.ipynb
@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -32,7 +32,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -62,7 +62,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -112,7 +112,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -144,7 +144,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": ".venv-3.9",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },

diff --git a/demo/scenarios/copy_nc.sh b/demo/scenarios/copy_nc.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+MODEL="OxfordFlower"
+VERSION="0.0.1"
+
+# Copy sample negotiation card to our working store space. Create folders if needed.
+mkdir -p store/models/$MODEL/$VERSION
+cp ../sample_store/models/$MODEL/$VERSION/*.json ./store/models/$MODEL/$VERSION
diff --git a/demo/scenarios/demo_script.txt b/demo/scenarios/demo_script.txt
@@ -0,0 +1,14 @@
+Pre requirements:
+
+ - Have environment running; to do this, execute run_environment.sh from demo/scenarios.
+ - Have already executed all notebooks that will be shown, so the results can be seen in the notebooks themselves.
+
+Demo:
+ - Start with slides explaining what MLTE is.
+ - Continue slides including a diagram of the whole process. Use this diagram later with more slides to indicate where in the process we are as we move forward through the demo.
+ - Log into frontend (browser, localhost:8000), show Negotiation Card. Explain important parts of negotiation card and how it is refined.
+ - Open up Jupyter Notebook with Spec (the one in demo/scenarios/1_requirements.ipynb. Note that we may need a slightly modified version if we add a data quality QAS. Skip the Negotiation Card definition, but do show the context initialization, and then the Spec definition. Walk though it without getting into too many details.
+ - Go back to frontend. Show Test Catalog, and show how people would use it find an example of how to measure certain QAS.
+ - Open up one Jupyter Notebooks with QAS test measurements, the demo/scenarios/2a_evidence_fairness one, and walk through it.
+ - Open up the demo/scenarios/3_report.ipynb Notebook, and show the process for validating and generating the report.
+ - Go back to frontend, and show the report results.
diff --git a/demo/scenarios/run_environment.sh b/demo/scenarios/run_environment.sh
@@ -3,9 +3,7 @@
 MODEL="OxfordFlower"
 VERSION="0.0.1"
 
-# Copy sample negotiation card to our working store space. Create folders if needed.
-mkdir -p store/models/$MODEL/$VERSION
-cp ../sample_store/models/$MODEL/$VERSION/*.json ./store/models/$MODEL/$VERSION
+source copy_nc.sh
 
 # Set env vars to not use a relational DB, but a file store, and point to the one here.
 export STORE_TYPE=fs