Your first AI loop: see, pick, sort

python -m venv .venv && source .venv/bin/activate
pip install "cyberwave[ml,camera]"

export CYBERWAVE_API_KEY="your_api_key_here"
# Only if your token spans several workspaces:
export CYBERWAVE_WORKSPACE_ID="your_workspace_uuid"

import time

import cv2
from cyberwave import Cyberwave

# Example poses in degrees, good enough to watch in the Playground.
# On a real SO-101, record your own in the twin's Saved poses panel.
HOME   = {"shoulder_pan": 0, "shoulder_lift": 0, "elbow_flex": 0, "wrist_flex": 0, "gripper": 0}
REACH  = {"shoulder_lift": 45, "elbow_flex": -35, "wrist_flex": 50}
LIFT   = {"shoulder_lift": 10, "elbow_flex": -10, "wrist_flex": 20}
OPEN   = {"gripper": 35}
CLOSED = {"gripper": 0}
BINS   = {"cup": -90, "bottle": 90}   # base rotation (shoulder_pan) of each bin
AIM_RANGE = 45                        # frame edges map to ±45° of base rotation

cw = Cyberwave()                          # reads CYBERWAVE_API_KEY
arm = cw.twin("the-robot-studio/so101")   # creates or reuses "Quickstart Environment"
cw.affect("playground")                   # free kinematic simulation; "live" for a real arm
arm.policy.ensure_attached()              # the script becomes the arm's controller
time.sleep(2)                             # give the controller a moment to come up


def move(pose, pause=0.8):
    arm.set_joints(pose, degrees=True)
    time.sleep(pause)                     # let the arm arrive before the next move


def pick_and_sort(label, x_center, frame_width):
    # Left edge of the frame -> -AIM_RANGE, right edge -> +AIM_RANGE.
    aim = (x_center / frame_width - 0.5) * 2 * AIM_RANGE
    print(f"{label} at x={x_center:.0f}px -> aim {aim:+.0f} deg, bin {BINS[label]:+d} deg")
    move({"shoulder_pan": aim, **OPEN})   # turn toward the object, open the gripper
    move(REACH)                           # go down to it
    move(CLOSED)                          # grasp
    move(LIFT)                            # lift clear of the table
    move({"shoulder_pan": BINS[label]})   # carry it to its bin
    move(OPEN)                            # drop
    move(HOME)                            # back to home, ready for the next one


move(HOME)
model = cw.models.load("yolo26n.pt")      # runs locally; weights download on first use
camera = cv2.VideoCapture(0)              # default webcam

waiting_for_clear = False                 # don't pick the same object twice
try:
    while True:
        ok, frame = camera.read()         # BGR numpy array
        if not ok:
            raise RuntimeError("No frame from webcam 0. Is another app using it?")

        result = model.predict(frame, confidence=0.5, classes=list(BINS))

        if len(result) == 0:
            waiting_for_clear = False     # workspace is empty: ready for the next object
        elif not waiting_for_clear:
            target = max(result, key=lambda d: d.confidence)
            x_center = (target.bbox.x1 + target.bbox.x2) / 2
            pick_and_sort(target.label, x_center, frame.shape[1])
            waiting_for_clear = True

        time.sleep(0.3)
except KeyboardInterrupt:
    pass
finally:
    camera.release()
    cw.disconnect()

python pick_and_sort.py