[Question] Pytorch Profile Error

[Dingjifeng]

Pytorch Profile Error:

Thanks for your work. But when I want to profile with pytorch I encounterd an error:

Traceback (most recent call last):
  File "/ssd/public/dingjifeng/IsaacLab/test/lab/tiled_camera.py", line 363, in <module>
    main()
  File "/ssd/public/dingjifeng/IsaacLab/test/lab/tiled_camera.py", line 358, in main
    run_simulator(sim, scene)
  File "/ssd/public/dingjifeng/IsaacLab/test/lab/tiled_camera.py", line 183, in run_simulator
    with torch.profiler.profile(
  File "/home/public/data/dingjifeng/isaac-sim-4.2.0/exts/omni.isaac.ml_archive/pip_prebundle/torch/profiler/profiler.py", line 706, in __exit__
    self.stop()
  File "/home/public/data/dingjifeng/isaac-sim-4.2.0/exts/omni.isaac.ml_archive/pip_prebundle/torch/profiler/profiler.py", line 722, in stop
    self._transit_action(self.current_action, None)
  File "/home/public/data/dingjifeng/isaac-sim-4.2.0/exts/omni.isaac.ml_archive/pip_prebundle/torch/profiler/profiler.py", line 751, in _transit_action
    action()
  File "/home/public/data/dingjifeng/isaac-sim-4.2.0/exts/omni.isaac.ml_archive/pip_prebundle/torch/profiler/profiler.py", line 206, in stop_trace
    self.profiler.__exit__(None, None, None)
  File "/home/public/data/dingjifeng/isaac-sim-4.2.0/exts/omni.isaac.ml_archive/pip_prebundle/torch/autograd/profiler.py", line 352, in __exit__
    self.kineto_results = _disable_profiler()
RuntimeError: !stack.empty() INTERNAL ASSERT FAILED at "../torch/csrc/autograd/profiler_python.cpp":969, please report a bug to PyTorch. Python replay stack is empty.

And the python code as follows:

import argparse
import nvtx
import torch

from omni.isaac.lab.app import AppLauncher

# add argparse arguments
parser = argparse.ArgumentParser(description="This script demonstrates the different camera sensor implementations.")
parser.add_argument("--num_envs", type=int, default=4, help="Number of environments to spawn.")
parser.add_argument("--disable_fabric", action="store_true", help="Disable Fabric API and use USD instead.")
# append AppLauncher cli args
AppLauncher.add_app_launcher_args(parser)
# parse the arguments
args_cli = parser.parse_args()
num_envs = args_cli.num_envs
print(f"[INFO]: num_envs = {num_envs}")
# launch omniverse app
app_launcher = AppLauncher({"headless":True,"num_envs":num_envs,"enable_cameras":True})
simulation_app = app_launcher.app

"""Rest everything follows."""

import matplotlib.pyplot as plt
import numpy as np
import os
import torch

import omni.isaac.lab.sim as sim_utils
from omni.isaac.lab.assets import ArticulationCfg, AssetBaseCfg
from omni.isaac.lab.scene import InteractiveScene, InteractiveSceneCfg
from omni.isaac.lab.sensors import CameraCfg, RayCasterCameraCfg, TiledCameraCfg
from omni.isaac.lab.sensors.ray_caster import patterns
from omni.isaac.lab.terrains import TerrainImporterCfg
from omni.isaac.lab.utils import configclass

##
# Pre-defined configs
##
from omni.isaac.lab.terrains.config.rough import ROUGH_TERRAINS_CFG  # isort:skip
from omni.isaac.lab_assets.anymal import ANYMAL_C_CFG  # isort: skip

@configclass
class SensorsSceneCfg(InteractiveSceneCfg):
    """Design the scene with sensors on the robot."""

    # ground plane
    ground = TerrainImporterCfg(
        prim_path="/World/ground",
        max_init_terrain_level=None,
        terrain_type="generator",
        terrain_generator=ROUGH_TERRAINS_CFG.replace(color_scheme="random"),
        visual_material=None,
        debug_vis=False,
    )

    # # lights
    # dome_light = AssetBaseCfg(
    #     prim_path="/World/Light", spawn=sim_utils.DomeLightCfg(intensity=2000.0, color=(0.75, 0.75, 0.75))
    # )

    # # robot
    robot: ArticulationCfg = ANYMAL_C_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot")
    
    camera01 = CameraCfg(
        prim_path="{ENV_REGEX_NS}/Robot/base/front_cam1",
        update_period=0.1,
        height=480,
        width=640,
        data_types=["rgb", "distance_to_image_plane"],
        spawn=sim_utils.PinholeCameraCfg(
            focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(0.1, 1.0e5)
        ),
        offset=CameraCfg.OffsetCfg(pos=(0.510, 0.0, 0.015), rot=(0.5, -0.5, 0.5, -0.5), convention="ros"),
    )
    
    # # sensors
    camera = CameraCfg(
        prim_path="{ENV_REGEX_NS}/Robot/base/front_cam",
        update_period=0.1,
        height=480,
        width=640,
        data_types=["rgb", "distance_to_image_plane"],
        spawn=sim_utils.PinholeCameraCfg(
            focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(0.1, 1.0e5)
        ),
        offset=CameraCfg.OffsetCfg(pos=(0.510, 0.0, 0.015), rot=(0.5, -0.5, 0.5, -0.5), convention="ros"),
    )
    
    tiled_camera = TiledCameraCfg(
        prim_path="{ENV_REGEX_NS}/Robot/base/front_cam",
        update_period=0.1,
        height=480,
        width=640,
        data_types=["rgb", "distance_to_image_plane"],
        # spawn=sim_utils.PinholeCameraCfg(
        #     focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(0.1, 1.0e5)
        # ),
        spawn=None,  # the camera is already spawned in the scene
        offset=TiledCameraCfg.OffsetCfg(pos=(0.510, 0.0, 0.015), rot=(0.5, -0.5, 0.5, -0.5), convention="ros"),
    )
    
    # raycast_camera = RayCasterCameraCfg(
    #     prim_path="{ENV_REGEX_NS}/Robot/base",
    #     mesh_prim_paths=["/World/ground"],
    #     update_period=0.1,
    #     offset=RayCasterCameraCfg.OffsetCfg(pos=(0.510, 0.0, 0.015), rot=(0.5, -0.5, 0.5, -0.5), convention="ros"),
    #     data_types=["distance_to_image_plane", "normals"],
    #     pattern_cfg=patterns.PinholeCameraPatternCfg(
    #         focal_length=24.0,
    #         horizontal_aperture=20.955,
    #         height=480,
    #         width=640,
    #     ),
    # )
def run_simulator(sim: sim_utils.SimulationContext, scene: InteractiveScene):
    """Run the simulator."""
    # Define simulation stepping
    sim_dt = sim.get_physics_dt()
    sim_time = 0.0
    count = 0

    # Create output directory to save images
    output_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "output")
    os.makedirs(output_dir, exist_ok=True)
    
    
    times = []
    with torch.profiler.profile(
        activities=[torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA],
        record_shapes=True,
        with_stack=True,
        # schedule=torch.profiler.schedule(wait=0,warmup=0,active=10,repeat=1),
        on_trace_ready=save_as_chrome_trace,
    ) as prof:
        # Simulate physics
        while simulation_app.is_running():

            # Reset
            # if count % 500 == 0:
            #     # reset counter
            #     count = 0
            #     # reset the scene entities
            #     # root state
            #     # we offset the root state by the origin since the states are written in simulation world frame
            #     # if this is not done, then the robots will be spawned at the (0, 0, 0) of the simulation world
            #     root_state = scene["robot"].data.default_root_state.clone()
            #     root_state[:, :3] += scene.env_origins
            #     scene["robot"].write_root_link_pose_to_sim(root_state[:, :7])
            #     scene["robot"].write_root_com_velocity_to_sim(root_state[:, 7:])
            #     # set joint positions with some noise
            #     joint_pos, joint_vel = (
            #         scene["robot"].data.default_joint_pos.clone(),
            #         scene["robot"].data.default_joint_vel.clone(),
            #     )
            #     joint_pos += torch.rand_like(joint_pos) * 0.1
            #     scene["robot"].write_joint_state_to_sim(joint_pos, joint_vel)
            #     # clear internal buffers
            #     scene.reset()
            #     print("[INFO]: Resetting robot state...")
            
            # Apply default actions to the robot
            # -- generate actions/commands
            # targets = scene["robot"].data.default_joint_pos
            # -- apply action to the robot
            # scene["robot"].set_joint_position_target(targets)
            # -- write data to sim
            # scene.write_data_to_sim()
            # perform step
            
            camera0 = scene["camera01"]
            
            simulation_app.update()
            import time
            torch.cuda.synchronize()
            start = time.perf_counter()
            sim.step()
            # update sim-time
            sim_time += sim_dt
            count += 1
            # update buffers
            scene.update(sim_dt)
            
            torch.cuda.synchronize()
            end = time.perf_counter()
            times.append((end - start) * 1000)
            print(f"one step: {(end - start) * 1000}ms")
            
            # print information from the sensors
            print("-------------------------------")
            print(scene["terrain"])
            # print("Received shape of rgb   image: ", scene["camera"].data.output["rgb"].shape)
            # print("Received shape of depth image: ", scene["camera"].data.output["distance_to_image_plane"].shape)
            # print("-------------------------------")
            # print(scene["tiled_camera"])
            # print("Received shape of rgb   image: ", scene["tiled_camera"].data.output["rgb"].shape)
            # print("Received shape of depth image: ", scene["tiled_camera"].data.output["distance_to_image_plane"].shape)
            # print("-------------------------------")
            # print(scene["raycast_camera"])
            # print("Received shape of depth: ", scene["raycast_camera"].data.output["distance_to_image_plane"].shape)
            # print("Received shape of normals: ", scene["raycast_camera"].data.output["normals"].shape)

            # save every 10th image (for visualization purposes only)
            # note: saving images will slow down the simulation
            # if count % 10 == 0:
            # #     # compare generated RGB images across different cameras
            tiled_rgb_images = scene["tiled_camera"].data.output["rgb"]
            rgb_images = scene["camera"].data.output["rgb"]
            
            
            #save rgb image
            
            assert(len(tiled_rgb_images) == num_envs)
            #     print(len(rgb_images))
            #     print(rgb_images[0].shape)
                
                
            from PIL import Image
            import numpy as np

            #     # 假设 rgb_images 是 numpy 数组，形状为 (H, W, C)
            #     # 确保数据类型为 uint8（像素值范围应在 0-255）
            tiled_rgb_images = np.clip(tiled_rgb_images[0].cpu().numpy(), 0, 255).astype(np.uint8)
            rgb_images = np.clip(rgb_images[0].cpu().numpy(),0,255).astype(np.uint8)
            
            #     # 将 numpy 数组转换为 Image 对象
            tiled_image = Image.fromarray(tiled_rgb_images)
            rgb_images = Image.fromarray(rgb_images)

            #     # 保存图像到文件
            tiled_image.save("tiled_output_image.png")  
            rgb_images.save("output_image.png")
                
                # rgb_images = [scene["camera"].data.output["rgb"][0, ..., :3], scene["tiled_camera"].data.output["rgb"][0]]
                # save_images_grid(
                #     rgb_images,
                #     subtitles=["TiledCamera"],
                #     # subtitles=["Camera", "TiledCamera"],
                #     title="RGB Image: Cam0",
                #     filename=os.path.join(output_dir, "rgb", f"{count:04d}.jpg"),
                # )

            #     # compare generated Depth images across different cameras
            #     depth_images = [
            #         scene["camera"].data.output["distance_to_image_plane"][0],
            #         scene["tiled_camera"].data.output["distance_to_image_plane"][0, ..., 0],
            #         scene["raycast_camera"].data.output["distance_to_image_plane"][0],
            #     ]
            #     save_images_grid(
            #         depth_images,
            #         cmap="turbo",
            #         subtitles=["Camera", "TiledCamera", "RaycasterCamera"],
            #         title="Depth Image: Cam0",
            #         filename=os.path.join(output_dir, "distance_to_camera", f"{count:04d}.jpg"),
            #     )

                # save all tiled RGB images
                # tiled_images = scene["tiled_camera"].data.output["rgb"]
                # save_images_grid(
                #     tiled_images,
                #     subtitles=[f"Cam{i}" for i in range(tiled_images.shape[0])],
                #     title="Tiled RGB Image",
                #     filename=os.path.join(output_dir, "tiled_rgb", f"{count:04d}.jpg"),
                # )

                # # save all camera RGB images
                # cam_images = scene["camera"].data.output["rgb"][..., :3]
                # save_images_grid(
                #     cam_images,
                #     subtitles=[f"Cam{i}" for i in range(cam_images.shape[0])],
                #     title="Camera RGB Image",
                #     filename=os.path.join(output_dir, "cam_rgb", f"{count:04d}.jpg"),
                # )
            torch.cuda.synchronize()
            prof.step()
            if count == 100:
                break
    
    total = 0
    for t in times:
        total += t
    print(f"AVG Step time: {total / len(times)}")
        
        


def main():
    """Main function."""
    # Initialize the simulation context
    sim_cfg = sim_utils.SimulationCfg(dt=0.005, device=args_cli.device, use_fabric=not args_cli.disable_fabric)
    print(sim_cfg)
    sim = sim_utils.SimulationContext(sim_cfg)
    # Set main camera
    sim.set_camera_view(eye=[0, 3.5, 3.5], target=[1.0, 1.0, 1.0])
    # design scene
    scene_cfg = SensorsSceneCfg(num_envs=args_cli.num_envs, env_spacing=2.0)
    print(scene_cfg)
    scene = InteractiveScene(scene_cfg)
    # Play the simulator
    sim.reset()
    # Now we are ready!
    print("[INFO]: Setup complete...")
    # Run the simulator
    run_simulator(sim, scene)


if __name__ == "__main__":
    # run the main function
    main()
    # close sim app
    simulation_app.close()

Plz Help me if you encountered the same error. Thanks

[RandomOakForest]

Thanks for submitting this. It doesn't seem spawn=None would be allowed in the tiled camera configuration. Could you try following the example here as a first step?

[ykk624]

I encountered a same error "self.kineto_results = _disable_profiler()" when I run other model with other project.
Set "with_stack=False" is OK.
我跑其他模型时，也遇到这个错误。将 profiler采集参数里的 with_stack 置为False，即可正常运行。
但是采集到的流水文件，只有算子aten::，没有python侧的代码调用。

如果您解决了这个问题，也可以分享下。