AgentBay:UI class

Golang

func (ui *UI) GetClickableUIElements(timeoutMs int) (*UIElementsResult, error)

Parameter:

• timeoutMs (int): Timeout in milliseconds. Defaults to 2000 ms if <= 0.

Return values:

• *UIElementsResult: Contains the clickable UI elements and request ID.

• error: Error if the operation fails.

UIElementsResult struct:

type UIElementsResult struct {
    RequestID string      // A unique request identifier for debugging.
    Elements  []*UIElement // An array of UI elements.
}

type UIElement struct {
    Bounds     string       // The bounds of the element.
    ClassName  string       // The CSS class name.
    Text       string       // The text content.
    Type       string       // The element type.
    ResourceId string       // The resource ID.
    Index      int          // The element index.
    IsParent   bool         // Indicates whether the element is a parent element.
    Children   []*UIElement // The child elements.
}

Python

def get_all_ui_elements(timeout_ms: int = 2000) -> List[Dict[str, Any]]

Parameter:

• timeout_ms (int, optional): Timeout in milliseconds. Default: 2000 ms.

Return value:

• List[Dict[str, Any]]: All UI elements with parsing details.

Exception:

• AgentBayError: Raised if the operation fails.

TypeScript

getClickableUIElements(timeoutMs?: number): Promise<string>

Parameter:

• timeoutMs (number, optional): Timeout in milliseconds. Default: 2000 ms.

Return value:

• Promise<string>: String representation of clickable UI elements.

Exception:

• APIError: Thrown if the operation fails.

Golang

func (ui *UI) GetAllUIElements(timeoutMs int) (*UIElementsResult, error)

Parameter:

• timeoutMs (int): Timeout in milliseconds. Defaults to 2000 ms if <= 0.

Return values:

• *UIElementsResult: Contains all UI elements and the request ID.

• error: Error if the operation fails.

Python

def get_all_ui_elements(timeout_ms: int = 2000) -> List[Dict[str, Any]]

Parameter:

• timeout_ms (int, optional): Timeout in milliseconds. Default: 2000 ms.

Return value:

• List[Dict[str, Any]]: All UI elements with parsing details.

Exception:

• AgentBayError: Raised if the operation fails.

TypeScript

getAllUIElements(timeoutMs?: number): Promise<string>

Parameter:

• timeoutMs (number, optional): Timeout in milliseconds. Default: 2000 ms.

Return value:

• Promise<string>: String representation of all UI elements.

Exception:

• APIError: Thrown if the operation fails.

Golang

func (ui *UI) SendKey(key int) (*KeyActionResult, error)

Parameter:

• key (int): Key code to send. Use KeyCode constants.

Return values:

• *KeyActionResult: Contains the operation status and request ID.

• error: Error if the operation fails.

KeyActionResult struct:

type KeyActionResult struct {
    RequestID string // A unique request identifier for debugging.
    Success   bool   // Indicates whether the key was sent successfully.
}

Python

def send_key(key: int) -> bool

Parameter:

• key (int): Key code to send. Use KeyCode constants.

Return value:

• bool: True if the key was sent successfully.

Exception:

• AgentBayError: Raised if the operation fails.

Example:

# Send the Home keypress
success = agent_bay.ui.send_key(KeyCode.HOME)
print("Keypress send result:", success)

TypeScript

sendKey(key: number): Promise<string>

Parameter:

• key (number): Key code to send. Use KeyCode constants.

Return value:

• Promise<string>: Response text on success.

Exception:

• APIError: Thrown on failure.

Example:

// Send the Home keypress
const result = await agentBay.ui.sendKey(KeyCode.HOME);
console.log("Keypress send result:", result);

Golang

func (ui *UI) InputText(text string) (*TextInputResult, error)

Parameter:

• text (string): Text to enter.

Return values:

• *TextInputResult: Contains the entered text and request ID.

• error: Error if the operation fails.

TextInputResult struct:

type TextInputResult struct {
    RequestID string // A unique request identifier for debugging.
    Text      string // The entered text.
}

Python

def input_text(text: str) -> None

Parameter:

• text (str): Text to enter.

Exception:

• AgentBayError: Raised if the operation fails.

Example:

# Enter the text "Hello"
agent_bay.ui.input_text("Hello")

TypeScript

inputText(text: string): Promise<string>

Parameter:

• text (string): Text to enter.

Return value:

• Promise<string>: Response text on success.

Exception:

• APIError: Thrown on failure.

Example:

// Enter the text "Hello"
const result = await agentBay.ui.inputText("Hello");
console.log("Text input result:", result);

Golang

func (ui *UI) Swipe(startX, startY, endX, endY, durationMs int) (*SwipeResult, error)

Parameters:

• startX (int): Start X coordinate.

• startY (int): Start Y coordinate.

• endX (int): End X coordinate.

• endY (int): End Y coordinate.

• durationMs (int): Swipe duration in milliseconds.

Return values:

• *SwipeResult: Contains the operation status and request ID.

• error: Error if the operation fails.

SwipeResult struct:

type SwipeResult struct {
    RequestID string // A unique request identifier for debugging.
    Success   bool   // Indicates whether the swipe was successful.
}

Python

def swipe(start_x: int, start_y: int, end_x: int, end_y: int, duration_ms: int = 300) -> None

Parameters:

• start_x (int): Start X coordinate.

• start_y (int): Start Y coordinate.

• end_x (int): End X coordinate.

• end_y (int): End Y coordinate.

• duration_ms (int, optional): Swipe duration in milliseconds. Default: 300 ms.

Exception:

• AgentBayError: Raised if the operation fails.

Example:

# Swipe from (100, 200) to (300, 400)
agent_bay.ui.swipe(100, 200, 300, 400)

TypeScript

swipe(
  startX: number,
  startY: number,
  endX: number,
  endY: number,
  durationMs?: number
): Promise<string>

Parameters:

• startX (number): Start X coordinate.

• startY (number): Start Y coordinate.

• endX (number): End X coordinate.

• endY (number): End Y coordinate.

• durationMs (number, optional): Swipe duration in milliseconds. Default: 300 ms.

Return value:

• Promise<string>: Response text on success.

Exception:

• APIError: Thrown on failure.

Example:

// Swipe from (100, 200) to (300, 400)
const result = await agentBay.ui.swipe(100, 200, 300, 400);
console.log("Swipe operation result:", result);

Golang

func (ui *UI) Click(x, y int, button string) (*UIResult, error)

Parameters:

• x (int): X coordinate.

• y (int): Y coordinate.

• button (string): Mouse button. Defaults to left if empty.

Return values:

• *UIResult: Contains the operation status, component ID, and request ID.

• error: Error if the operation fails.

UIResult struct:

type UIResult struct {
    RequestID  string // A unique request identifier for debugging.
    ComponentID string // The component ID, if applicable.
    Success    bool   // Indicates whether the operation was successful.
}

Python

def click(x: int, y: int, button: str = "left") -> None

Parameters:

• x (int): X coordinate.

• y (int): Y coordinate.

• button (str, optional): Mouse button. Default: left.

Exception:

• AgentBayError: Raised if the operation fails.

Example:

# Left-click the coordinates (200, 300)
agent_bay.ui.click(200, 300)

TypeScript

click(x: number, y: number, button?: string): Promise<string>

Parameters:

• x (number): X coordinate.

• y (number): Y coordinate.

• button (string, optional): Mouse button. Default: left.

Return value:

• Promise<string>: Response text on success.

Exception:

• APIError: Thrown on failure.

Example:

// Left-click the coordinates (500, 800)
const result = await agentBay.ui.click(500, 800);
console.log("Click operation result:", result);

Golang

func (ui *UI) Screenshot() (*UIResult, error)

Return values:

• *UIResult: Contains the operation status and request ID.

• error: Error if the operation fails.

Python

def screenshot() -> str

Return value:

• str: Screenshot data.

Exception:

• AgentBayError: Raised if the operation fails.

Example:

# Take a screenshot
image_data = agent_bay.ui.screenshot()
print("Screenshot data:", image_data[:100])  # Print the first 100 characters

TypeScript

screenshot(): Promise<string>

Return value:

• Promise<string>: Screenshot data on success.

Exception:

• APIError: Thrown on failure.

Example:

// Take a screenshot
const imageData = await agentBay.ui.screenshot();
console.log("Screenshot data:", imageData);