Skip to content

Pybullet camera

Simulation / PyBullet / tutorials

camera 
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
import time

import cv2
import numpy as np
import pybullet as p
import pybullet_data

p.connect(p.GUI)
p.setAdditionalSearchPath(pybullet_data.getDataPath())

p.configureDebugVisualizer(p.COV_ENABLE_RGB_BUFFER_PREVIEW, 0)
p.configureDebugVisualizer(p.COV_ENABLE_DEPTH_BUFFER_PREVIEW, 0)
p.configureDebugVisualizer(p.COV_ENABLE_SEGMENTATION_MARK_PREVIEW, 0)

p.setGravity(0, 0, -9.81)
p.loadURDF("plane.urdf")
p.loadURDF("cube_small.urdf", [0, 0, 1])

width = 640
height = 480

view_matrix = p.computeViewMatrix(
    cameraEyePosition=[2, -3, 2],
    cameraTargetPosition=[0, 0, 0.5],
    cameraUpVector=[0, 0, 1],
)

projection_matrix = p.computeProjectionMatrixFOV(
    fov=60,
    aspect=width / height,
    nearVal=0.1,
    farVal=100,
)

while True:
    p.stepSimulation()

    _, _, rgba, depth, seg = p.getCameraImage(
        width=width,
        height=height,
        viewMatrix=view_matrix,
        projectionMatrix=projection_matrix,
        renderer=p.ER_BULLET_HARDWARE_OPENGL,
    )

    rgba = np.array(rgba, dtype=np.uint8).reshape(height, width, 4)

    # PyBullet gives RGBA, OpenCV expects BGR
    bgr = cv2.cvtColor(rgba, cv2.COLOR_RGBA2BGR)

    cv2.imshow("PyBullet Camera", bgr)

    if cv2.waitKey(1) & 0xFF in (ord("q"), 27):
        break

    time.sleep(1 / 240)

cv2.destroyAllWindows()
p.disconnect()

computeProjectionMatrixFOV

1
projection_matrix = p.computeProjectionMatrixFOV(fov, aspect, near, far)

Creates a projection matrix for a virtual camera using a perspective (field-of-view) projection.
Parameters:

  • fov: Field of view in degrees (vertical angle).
  • aspect: Aspect ratio (width/height).
  • near: Near clipping plane distance.
  • far: Far clipping plane distance.

Result: Returns a 4x4 matrix that transforms 3D points into 2D camera space, simulating a real camera lens.

getCameraImage

1
_, _, rgb, _, _ = p.getCameraImage(width, height, view_matrix, projection_matrix, renderer=p.ER_BULLET_HARDWARE_OPENGL)

Renders an image from the simulation as seen by the virtual camera. Parameters:

  • width, height: Output image size.
  • view_matrix: Defines the camera’s position and orientation (where it looks from and to).
  • projection_matrix: Defines how the 3D scene is projected onto the 2D image (from computeProjectionMatrixFOV).
  • renderer: Rendering backend (here, OpenGL).

Returns: A tuple containing image data, including the RGB image (rgb), depth, and segmentation masks.

alt text

opencv view 
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
import cv2
import numpy as np
import pybullet as p
import pybullet_data
import time

# Initialize PyBullet
physicsClient = p.connect(p.GUI)
p.setAdditionalSearchPath(pybullet_data.getDataPath())

# Create ground plane
planeId = p.loadURDF("plane.urdf")

# Add texture to the ground
textureId = p.loadTexture("tarmac.png")
p.changeVisualShape(planeId, -1, textureUniqueId=textureId)

# Set gravity and run simulation
p.setGravity(0, 0, -9.81)

camera_pos = [0, 0, 10]  # Start at x=0, y=0, z=10m
velocity = 5.0  # 5 m/s in x-direction
width, height = 640, 480
fov, aspect, near, far = 60, width/height, 0.1, 100.0
projection_matrix = p.computeProjectionMatrixFOV(fov, aspect, near, far)
target_pos = [camera_pos[0], camera_pos[1], 0]  # Look downward

while True:
    p.stepSimulation()
    view_matrix = p.computeViewMatrix(camera_pos, target_pos, [0, 1, 0])  # Up vector: y-axis
    _, _, rgb, _, _ = p.getCameraImage(width, height, view_matrix, projection_matrix,
                                    renderer=p.ER_BULLET_HARDWARE_OPENGL)

    rgb = np.reshape(rgb, (height, width, 4))[:, :, :3]  # Remove alpha channel
    rgb = cv2.cvtColor(rgb, cv2.COLOR_RGBA2BGR)

    # Write to video
    cv2.imshow('Camera Feed', rgb)
    key = cv2.waitKey(1)

    if key == ord('q'):
        break

    time.sleep(1./30.)
cv2.destroyAllWindows()
p.disconnect()