Add bubble_env to benchmark.

CHANGELOG.md

@@ -12,6 +12,8 @@ Copy and pasting the git commit messages is __NOT__ enough.
 ### Added
 - Added a zoo agent, named Control-and-Supervised-Learning, from NeurIPS 2022 submission. This zoo agent runs in benchmark `driving_smarts==0.0`.
 - Added a zoo agent, named Discrete Soft Actor Critic, from NeurIPS 2022 submission. This zoo agent runs in benchmark `driving_smarts==0.0`.
+- Added `"driving_smarts_full==0.0"` benchmark to the set of available benchmarks.
+- Added `"driving-smarts-competition-bubble-env-v0"` to the set of available environments. This requires installing `bubble_env` in order to run.
 ### Changed
 ### Deprecated
 ### Fixed
@@ -31,6 +33,7 @@ Copy and pasting the git commit messages is __NOT__ enough.
 - Individualised the agent instances in the `benchmark_runner_v0.py`.
 - Made driving_smarts_competition_v0 env configurable through supply of `AgentInterface`.
 - Observation of driving_smarts_competition_v0 env was fixed to be of type `ObservationOptions.unformatted`.
+- The V0 benchmark runner now uses `load_config` from the benchmarks module.
 ### Deprecated
 ### Fixed
 - Fixed an exit error that occurs when envision attempts to close down.

smarts/benchmark/__init__.py

@@ -25,6 +25,8 @@
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
+from smarts.core.utils.resources import load_yaml_config as _load_config
+
 BENCHMARK_LISTING_FILE = Path(__file__).parent.absolute() / "benchmark_listing.yaml"
 
 
@@ -118,3 +120,10 @@ def list_benchmarks(benchmark_listing):
     from smarts.core.utils.resources import load_yaml_config_with_substitution
 
     return load_yaml_config_with_substitution(Path(benchmark_listing))
+
+
+def load_config(path: Path) -> Optional[Dict[str, Any]]:
+    """Load in a benchmark config."""
+    if isinstance(path, (str,)):
+        path = Path(path)
+    return _load_config(path)

smarts/benchmark/benchmark_listing.yaml

@@ -10,4 +10,13 @@ benchmarks:
         requirements: ["ray<=2.2.0,>2.0"]
         params: 
           benchmark_config: ${{smarts.benchmark.driving_smarts.v0}}/config.yaml
+  driving_smarts_full:
+    name: "Driving SMARTS Full"
+    versions:
+      -
+        version: 0.0
+        entrypoint: "smarts.benchmark.entrypoints.benchmark_runner_v0.benchmark_from_configs"
+        requirements: ["ray<=2.2.0,>2.0"]
+        params: 
+          benchmark_config: ${{smarts.benchmark.driving_smarts_full.v0}}/config.yaml
 

smarts/benchmark/driving_smarts/__init__.py

@@ -19,14 +19,3 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
-from pathlib import Path
-from typing import Any, Dict, Optional
-
-from smarts.core.utils.resources import load_yaml_config as _load_config
-
-
-def load_config(path: Path) -> Optional[Dict[str, Any]]:
-    """Load in a benchmark config."""
-    if isinstance(path, (str,)):
-        path = Path(path)
-    return _load_config(path)

smarts/benchmark/driving_smarts_full/__init__.py

@@ -0,0 +1,21 @@
+# MIT License
+#
+# Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.

smarts/benchmark/driving_smarts_full/v0/__init__.py

@@ -0,0 +1,24 @@
+# MIT License
+#
+# Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+from pathlib import Path
+
+DEFAULT_CONFIG = str((Path(__file__).parent / "config.yaml").absolute())

smarts/benchmark/driving_smarts_full/v0/config.yaml

@@ -0,0 +1,53 @@
+# smarts/benchmark/driving_smarts_v0/config.yaml
+---
+benchmark:
+  name: "Driving SMARTS V0"
+  message: | # A useful message given at the start of the benchmark.
+    This is a cleaned up version of the Driving SMARTS benchmark.
+
+      Using `TargetPose` agent action has an applied 28m/s cap for agent motion.
+      Using `RelativeTargetPose` agent action, the constraint is inbuilt into the action space.
+
+      If bubble_env is not installed, please install the bubble_env repository:
+
+      ```bash
+      # set $REPOS to wherever you wish to store the repository.
+      git lfs clone https://bitbucket.org/malban/bubble_env.git $REPOS/bubble_env
+      # read $REPOS/README.md and follow those instructions
+      pip install $REPOS/bubble_env
+      ```
+
+      For history see: 
+        - https://codalab.lisn.upsaclay.fr/competitions/6618
+        - https://smarts-project.github.io/archive/2022_nips_driving_smarts/
+  eval_episodes: 50
+  shared_env_kwargs:
+    seed: 42
+    headless: true
+  envs:
+    bubble: # reserved for bubble env
+      loc: "smarts.env:driving-smarts-competition-bubble-env-v0"
+      scenarios:
+        - 6
+        - 4
+        - 42
+    standard:
+      loc: "smarts.env:driving-smarts-competition-v0"
+      scenarios:
+        - 1_to_2lane_left_turn_c
+        - 1_to_2lane_left_turn_t
+        - 3lane_merge_multi_agent
+        - 3lane_merge_single_agent
+        - 3lane_cruise_multi_agent
+        - 3lane_cruise_single_agent
+        - 3lane_cut_in
+        - 3lane_overtake
+      kwargs:
+        seed: 42
+    #   kwargs:
+    # naturalistic: # reserved for driving_smarts_competition_naturalistic-v0
+    #   loc: "smarts.env:driving_smarts_competition_naturalistic-v0"
+    #   kwargs:
+    #     scenario_dirs:
+    #       - "./scenarios/naturalistic/waymo"
+    #       - "./scenarios/naturalistic/ngsim"

smarts/benchmark/entrypoints/benchmark_runner_v0.py

@@ -27,7 +27,7 @@
 import psutil
 import ray
 
-from smarts.benchmark.driving_smarts import load_config
+from smarts.benchmark import load_config
 from smarts.core.utils.logging import suppress_output
 from smarts.env.gymnasium.wrappers.metrics import Metrics, Score
 from smarts.zoo import registry as agent_registry

smarts/env/gymnasium/__init__.py

@@ -32,6 +32,12 @@
         entry_point="smarts.env.gymnasium.driving_smarts_competition_env:driving_smarts_competition_v0_env",
         disable_env_checker=True,
     )
+
+    register(
+        id="driving-smarts-competition-bubble-env-v0",
+        entry_point="smarts.env.gymnasium.bubble_env:checked_bubble_env_v0",
+        disable_env_checker=True,
+    )
 except ModuleNotFoundError:
     import warnings
 

smarts/env/gymnasium/bubble_env.py

@@ -0,0 +1,46 @@
+from textwrap import dedent
+from typing import Optional
+
+import gymnasium as gym
+
+
+def checked_bubble_env_v0(
+    traffic_mode="traffic_A",
+    action_space="Direct",
+    img_meters: int = 64,
+    img_pixels: int = 256,
+    headless: bool = True,
+    seed: int = 42,
+    **kwargs,
+):
+
+    try:
+        import bubble_env_contrib
+    except ImportError as err:
+        raise ImportError(
+            """
+            Bubble env is not installed.
+            
+            If bubble_env is not installed, please install the bubble_env repository:
+
+            ```bash
+            # set $REPOS to wherever you wish to store the repository.
+            git lfs clone https://bitbucket.org/malban/bubble_env.git $REPOS/bubble_env
+            # read $REPOS/README.md and follow those instructions
+            cd -
+            pip install $REPOS/bubble_env
+            ```
+            """,
+            "Install bubble_env",
+        ) from err
+
+    env = gym.make(
+        "bubble_env_contrib:bubble_env-v1",
+        action_space=action_space,
+        img_meters=img_meters,
+        img_pixels=img_pixels,
+        headless=headless,
+        seed=seed,
+        traffic_mode=traffic_mode,
+    )
+    return env