fix(carla_run): sync and color mapping

drivellava/carla/client.py

@@ -24,7 +24,7 @@ def __init__(
         host="127.0.0.1",
         port=2000,
         sync=True,
-        autopilot=True,
+        autopilot=False,
         width=256,
         height=128,
         rolename="hero",
@@ -45,7 +45,7 @@ def __init__(
             settings = self.sim_world.get_settings()
             # if not settings.synchronous_mode:
             settings.synchronous_mode = True
-            settings.fixed_delta_seconds = 3.0
+            settings.fixed_delta_seconds = 1.0
             self.sim_world.apply_settings(settings)
 
             traffic_manager = self.client.get_trafficmanager()
@@ -127,8 +127,8 @@ def set_car_controls(
         """
         Set the car controls
         """
-        return
-        controls.speed_index = 1
+        # return
+        # controls.speed_index = 1
         if controls.speed_index == 0:
             self.world.player.enable_constant_velocity(
                 carla.Vector3D(0, 0, 0)  # 0 Km/h
@@ -144,13 +144,13 @@ def set_car_controls(
                 2.0
                 * (
                     float(controls.trajectory_index)
-                    / trajectory_encoder.num_trajectory_templates
+                    / (trajectory_encoder.num_trajectory_templates - 1)
                 )
                 - 1.0
-            )
+            ) * 0.7
 
-            if abs(steering_angle) < 0.2:
-                steering_angle = 0.0
+            # if abs(steering_angle) < 0.2:
+            #     steering_angle = 0.0
 
             print("steering_angle", steering_angle)
 

drivellava/gpt/gpt_vision.py

@@ -48,7 +48,7 @@ def llm_client_factory(llm_provider: str):
 
 
 class GPTVision:
-    def __init__(self, max_history=10):
+    def __init__(self, max_history=1):
         self.client = llm_client_factory(settings.system.SYSTEM_LLM_PROVIDER)
 
         self.previous_messages = GPTState()
@@ -77,18 +77,23 @@ def step(
         trajectory_templates, colors = (
             self.trajectory_encoder.get_colors_left_to_right()
         )
-        color_map = {i: colors[i] for i in range(len(colors))}
+        offset = (self.num_trajectory_templates - 1) // 2
+        color_map = {i - offset: colors[i] for i in range(len(colors))}
+        center = (self.num_trajectory_templates - 1) // 2 - offset
+        first = 0 - offset
+        last = self.num_trajectory_templates - 1 - offset
         traj_str = dedent(
             f"""
             The trajectories are numbered from:
-            [0,{self.num_trajectory_templates-1}]
+            [{first},{last}]
 
-            They are labelled from left to right.
-            Select trajectory 0 to get the left most.
-            Select trajectory {(self.num_trajectory_templates-1)//2} to get a
-             more centered trajectory
-            Select trajectory {self.num_trajectory_templates-1} to get the
-             right most.
+            They are labelled from left to right with a BGR color mapping.
+            Select trajectory {first} (color: {color_map[first]}) to get the \
+            left most.
+            Select trajectory {center} (color: {color_map[center]}) to get \
+            a centered trajectory
+            Select trajectory {last} (color: {color_map[last]}) to get the \
+            right most.
 
             Color Mapping (B,G,R):
             {str(color_map)}
@@ -150,7 +155,7 @@ def step(
         response = self.client.chat.completions.create(
             model=VISION_MODEL,
             messages=prompt,
-            max_tokens=800,
+            max_tokens=300,
         )
 
         desc = response.choices[0].message.content
@@ -169,7 +174,8 @@ def step(
 
         # If number of messages is greater than max_history,
         # remove the oldest message. Do not remove the system message
-        if len(self.previous_messages) > self.max_history:
+        # The -2 is to account for the [system message, setup message]
+        if len(self.previous_messages) - 2 > self.max_history:
             self.previous_messages.pop(1)
 
         print(desc)
@@ -201,6 +207,8 @@ def step(
             max_tokens=300,
         )
 
+        gpt_controls.trajectory_index += offset
+
         print("gpt:", gpt_controls)
 
         return gpt_controls

drivellava/gpt/prompts.py

@@ -15,7 +15,7 @@
 You can select one or more of the following actions as your next immediate \
 action:
  - trajectory_index: select one of the trajectories from those drawn
- - speed_index: select a speed index from one of those provided
+ - speed_index: Select 0 to stop the vehicle and 1 to move the vehicle
 
 Trajectories: {traj_str}
 

drivellava/scripts/carla_run.py

@@ -69,8 +69,8 @@ def start_carla():
 
 def main():  # pragma: no cover
     """
-    Use the Image from the drone and the SLAM map to feed as input to GPT
-    Return the drone controls
+    Use the Image from the sim and the map to feed as input to GPT
+    Return the vehicle controls to the sim and step
     """
 
     mission = settings.system.SYSTEM_MISSION
@@ -134,12 +134,16 @@ def main():  # pragma: no cover
                 gpt_input_q.put(data)
 
             # Get GPT Controls
-            if not gpt_output_q.empty():
+            if not gpt_output_q.empty() or (
+                settings.system.GPT_WAIT
+                and time.time() * 1000 - last_update > 10.0 * 1000
+            ):
                 gpt_controls_dict = gpt_output_q.get()
                 gpt_controls = DroneControls(**gpt_controls_dict)
 
                 client.set_car_controls(gpt_controls, gpt.trajectory_encoder)
                 gpt.previous_messages.timestamp = last_update
+                last_update = gpt_controls.timestamp
 
                 template_trajectory = trajectory_templates[
                     gpt_controls.trajectory_index

drivellava/settings.py

@@ -50,6 +50,7 @@ class SystemSettings:
     )
     TRAJECTORY_SIZE: int = str_to_int(os.getenv("TRAJECTORY_SIZE", "20"), 20)
     GPT_ENABLED: bool = str_to_bool(os.getenv("GPT_ENABLED", "True"))
+    GPT_WAIT: bool = str_to_bool(os.getenv("GPT_WAIT", "True"))
 
 
 class Settings: