Automation using Java Access Bridge

Question

I can capture text from UI controls (button/Editbox/Checkbox etc) in Java Applications, using Java Access Bridge events. How can I:

1. Set text in a Editbox
2. Click on a button

using Java Access Bridge API calls?

Answer 1

I would subclass the AccessibleContext of your GUI component and provide it with an accessibleAction object. Make AccessibleContext.getAccessibleAction() return your object.

If it's not null, it gives the screen reader a list of supported actions, that can be invoked - by the screen reader - by calling doAction on it.

Answer 2

Here is how I have done it for my project. Create a base class API that calls all PInvokes into JAB WindowsAccessBridge DLL. Make sure you target proper DLL Name if you are on 64 bit os. Use getAccessibleContextFromHWND function to get VmID and Context from Windows Handle. Locate the textbox or button within the Java Window by enumerating children. Once you locate the Control you are looking for in you case TextBox or Button perform the action.

1) Set text

public string Text
{
    get 
    {
        return GetText();
    }
    set
    {
        if (!API.setTextContents(this.VmId, this.Context, value))
            throw new AccessibilityException("Error setting text");
    }
}

private string GetText()
{
    System.Text.StringBuilder sbText = new System.Text.StringBuilder();

    int caretIndex = 0;

    while (true)
    {
        API.AccessibleTextItemsInfo ti = new API.AccessibleTextItemsInfo();
        if (!API.getAccessibleTextItems(this.VmId, this.Context, ref ti, caretIndex))
            throw new AccessibilityException("Error getting accessible text item information");

        if (!string.IsNullOrEmpty(ti.sentence))
            sbText.Append(ti.sentence);
        else               
            break;

        caretIndex = sbText.Length;

    }

    return sbText.ToString().TrimEnd();
}

2) Click on a button

public void Press()
{
    DoAction("click");
}

protected void DoAction(params string[] actions)
{
    API.AccessibleActionsToDo todo = new API.AccessibleActionsToDo()
    {
        actionInfo = new API.AccessibleActionInfo[API.MAX_ACTIONS_TO_DO],
        actionsCount = actions.Length,
    };

    for (int i = 0, n = Math.Min(actions.Length, API.MAX_ACTIONS_TO_DO); i < n; i++)
        todo.actionInfo[i].name = actions[i];

    Int32 failure = 0;
    if (!API.doAccessibleActions(this.VmId, this.Context, ref todo, ref failure))
        throw new AccessibilityException("Error performing action");
}

Core...

public API.AccessBridgeVersionInfo VersionInfo
{
    get { return GetVersionInfo(this.VmId); }
}

public API.AccessibleContextInfo Info
{
    get { return GetContextInfo(this.VmId, this.Context); }
}

public Int64 Context
{
    get;
    protected set;
}

public Int32 VmId
{
    get;
    protected set;
}