app: expose semantic information to Android platforms

Previous changes added semantic API and semantic information to Gio widgets. This change maps the information to Android accessibility classes so that TalkBack can traverse and interact with Gio programs. Signed-off-by: Elias Naur <mail@eliasnaur.com>
2026-07-01 07:35:40 +00:00 · 2021-12-01 13:14:55 +01:00
parent 6b1ca4ca7e
commit e6e69812af
4 changed files with 589 additions and 8 deletions
@@ -16,6 +16,7 @@ import android.graphics.Canvas;
 import android.graphics.Color;
 import android.graphics.Rect;
 import android.os.Build;
+import android.os.Bundle;
 import android.text.Editable;
 import android.util.AttributeSet;
 import android.util.TypedValue;
@@ -37,6 +38,10 @@ import android.view.inputmethod.InputConnection;
 import android.view.inputmethod.InputMethodManager;
 import android.view.inputmethod.EditorInfo;
 import android.text.InputType;
+import android.view.accessibility.AccessibilityNodeProvider;
+import android.view.accessibility.AccessibilityNodeInfo;
+import android.view.accessibility.AccessibilityEvent;
+import android.view.accessibility.AccessibilityManager;

 import java.io.UnsupportedEncodingException;

@@ -49,6 +54,7 @@ public final class GioView extends SurfaceView {
 	private final float scrollXScale;
 	private final float scrollYScale;
 	private int keyboardHint;
+	private AccessibilityManager accessManager;

 	private long nhandle;

@@ -88,6 +94,7 @@ public final class GioView extends SurfaceView {
 			scrollYScale = px;
 		}

+		accessManager = (AccessibilityManager)context.getSystemService(Context.ACCESSIBILITY_SERVICE);
 		nhandle = onCreateView(this);
 		imm = (InputMethodManager)context.getSystemService(Context.INPUT_METHOD_SERVICE);
 		setFocusable(true);
@@ -230,6 +237,53 @@ public final class GioView extends SurfaceView {
 		this.setBarColor(Bar.NAVIGATION, color, luminance);
 	}

+	@Override protected boolean dispatchHoverEvent(MotionEvent event) {
+		if (!accessManager.isTouchExplorationEnabled()) {
+			return super.dispatchHoverEvent(event);
+		}
+		switch (event.getAction()) {
+		case MotionEvent.ACTION_HOVER_ENTER:
+			// Fall through.
+		case MotionEvent.ACTION_HOVER_MOVE:
+			onTouchExploration(nhandle, event.getX(), event.getY());
+			break;
+		case MotionEvent.ACTION_HOVER_EXIT:
+			onExitTouchExploration(nhandle);
+			break;
+		}
+		return true;
+	}
+
+	void sendA11yEvent(int eventType, int viewId) {
+		if (!accessManager.isEnabled()) {
+			return;
+		}
+		AccessibilityEvent event = obtainA11yEvent(eventType, viewId);
+		getParent().requestSendAccessibilityEvent(this, event);
+	}
+
+	AccessibilityEvent obtainA11yEvent(int eventType, int viewId) {
+		AccessibilityEvent event = AccessibilityEvent.obtain(eventType);
+		event.setPackageName(getContext().getPackageName());
+		event.setSource(this, viewId);
+		return event;
+	}
+
+	boolean isA11yActive() {
+		return accessManager.isEnabled();
+	}
+
+	void sendA11yChange(int viewId) {
+		if (!accessManager.isEnabled()) {
+			return;
+		}
+		AccessibilityEvent event = obtainA11yEvent(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED, viewId);
+		if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) {
+			event.setContentChangeTypes(AccessibilityEvent.CONTENT_CHANGE_TYPE_SUBTREE);
+		}
+		getParent().requestSendAccessibilityEvent(this, event);
+	}
+
 	private void dispatchMotionEvent(MotionEvent event) {
 		if (nhandle == 0) {
 			return;
@@ -365,6 +419,11 @@ public final class GioView extends SurfaceView {
 	static private native void onFrameCallback(long handle);
 	static private native boolean onBack(long handle);
 	static private native void onFocusChange(long handle, boolean focus);
+	static private native AccessibilityNodeInfo initializeAccessibilityNodeInfo(long handle, int viewId, int screenX, int screenY, AccessibilityNodeInfo info);
+	static private native void onTouchExploration(long handle, float x, float y);
+	static private native void onExitTouchExploration(long handle);
+	static private native void onA11yFocus(long handle, int viewId);
+	static private native void onClearA11yFocus(long handle, int viewId);

 	private static class InputConnection extends BaseInputConnection {
 		private final Editable editable;
@@ -380,4 +439,42 @@ public final class GioView extends SurfaceView {
 			return editable;
 		}
 	}
+
+	@Override public AccessibilityNodeProvider getAccessibilityNodeProvider() {
+		return new AccessibilityNodeProvider() {
+			private final int[] screenOff = new int[2];
+
+			@Override public AccessibilityNodeInfo createAccessibilityNodeInfo(int viewId) {
+				AccessibilityNodeInfo info = null;
+				if (viewId == View.NO_ID) {
+					info = AccessibilityNodeInfo.obtain(GioView.this);
+					GioView.this.onInitializeAccessibilityNodeInfo(info);
+				} else {
+					info = AccessibilityNodeInfo.obtain(GioView.this, viewId);
+					info.setPackageName(getContext().getPackageName());
+					info.setVisibleToUser(true);
+				}
+				GioView.this.getLocationOnScreen(screenOff);
+				info = GioView.this.initializeAccessibilityNodeInfo(nhandle, viewId, screenOff[0], screenOff[1], info);
+				return info;
+			}
+
+			@Override public boolean performAction(int viewId, int action, Bundle arguments) {
+				if (viewId == View.NO_ID) {
+					return GioView.this.performAccessibilityAction(action, arguments);
+				}
+				switch (action) {
+				case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS:
+					GioView.this.onA11yFocus(nhandle, viewId);
+					GioView.this.sendA11yEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED, viewId);
+					return true;
+				case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS:
+					GioView.this.onClearA11yFocus(nhandle, viewId);
+					GioView.this.sendA11yEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED, viewId);
+					return true;
+				}
+				return false;
+			}
+		};
+	}
 }
@@ -147,6 +147,7 @@ type driver interface {

 	// Close the window.
 	Close()
+
 	// Wakeup wakes up the event loop and sends a WakeupEvent.
 	Wakeup()

@@ -64,6 +64,10 @@ static void jni_CallVoidMethodA(JNIEnv *env, jobject obj, jmethodID methodID, co
 	(*env)->CallVoidMethodA(env, obj, methodID, args);
 }

+static jboolean jni_CallBooleanMethodA(JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args) {
+	return (*env)->CallBooleanMethodA(env, obj, methodID, args);
+}
+
 static jbyte *jni_GetByteArrayElements(JNIEnv *env, jbyteArray arr) {
 	return (*env)->GetByteArrayElements(env, arr, NULL);
 }
@@ -103,6 +107,14 @@ static jobject jni_CallObjectMethodA(JNIEnv *env, jobject obj, jmethodID method,
 static jobject jni_CallStaticObjectMethodA(JNIEnv *env, jclass cls, jmethodID method, jvalue *args) {
 	return (*env)->CallStaticObjectMethodA(env, cls, method, args);
 }
+
+static jclass jni_FindClass(JNIEnv *env, char *name) {
+	return (*env)->FindClass(env, name);
+}
+
+static jobject jni_NewObjectA(JNIEnv *env, jclass cls, jmethodID cons, jvalue *args) {
+	return (*env)->NewObjectA(env, cls, cons, args);
+}
 */
 import "C"

@@ -127,6 +139,8 @@ import (
 	"gioui.org/io/clipboard"
 	"gioui.org/io/key"
 	"gioui.org/io/pointer"
+	"gioui.org/io/router"
+	"gioui.org/io/semantic"
 	"gioui.org/io/system"
 	"gioui.org/unit"
 )
@@ -146,6 +160,12 @@ type window struct {

 	win    *C.ANativeWindow
 	config Config
+
+	semantic struct {
+		hoverID router.SemanticID
+		rootID  router.SemanticID
+		focusID router.SemanticID
+	}
 }

 // gioView hold cached JNI methods for GioView.
@@ -164,6 +184,9 @@ var gioView struct {
 	setStatusColor     C.jmethodID
 	setFullscreen      C.jmethodID
 	unregister         C.jmethodID
+	sendA11yEvent      C.jmethodID
+	sendA11yChange     C.jmethodID
+	isA11yActive       C.jmethodID
 }

 // ViewEvent is sent whenever the Window's underlying Android view
@@ -195,6 +218,57 @@ var android struct {
 	mwriteClipboard   C.jmethodID
 	mreadClipboard    C.jmethodID
 	mwakeupMainThread C.jmethodID
+
+	// android.view.accessibility.AccessibilityNodeInfo class.
+	accessibilityNodeInfo struct {
+		cls C.jclass
+		// addChild(View, int)
+		addChild C.jmethodID
+		// setBoundsInScreen(Rect)
+		setBoundsInScreen C.jmethodID
+		// setText(CharSequence)
+		setText C.jmethodID
+		// setContentDescription(CharSequence)
+		setContentDescription C.jmethodID
+		// setParent(View, int)
+		setParent C.jmethodID
+		// addAction(int)
+		addAction C.jmethodID
+		// setClassName(CharSequence)
+		setClassName C.jmethodID
+		// setCheckable(boolean)
+		setCheckable C.jmethodID
+		// setSelected(boolean)
+		setSelected C.jmethodID
+		// setChecked(boolean)
+		setChecked C.jmethodID
+		// setEnabled(boolean)
+		setEnabled C.jmethodID
+		// setAccessibilityFocused(boolean)
+		setAccessibilityFocused C.jmethodID
+	}
+
+	// android.graphics.Rect class.
+	rect struct {
+		cls C.jclass
+		// (int, int, int, int) constructor.
+		cons C.jmethodID
+	}
+
+	strings struct {
+		// "android.view.View"
+		androidViewView C.jstring
+		// "android.widget.Button"
+		androidWidgetButton C.jstring
+		// "android.widget.CheckBox"
+		androidWidgetCheckBox C.jstring
+		// "android.widget.EditText"
+		androidWidgetEditText C.jstring
+		// "android.widget.RadioButton"
+		androidWidgetRadioButton C.jstring
+		// "android.widget.Switch"
+		androidWidgetSwitch C.jstring
+	}
 }

 // view maps from GioView JNI refenreces to windows.
@@ -216,6 +290,22 @@ var (
 	newAndroidGLESContext   func(w *window) (context, error)
 )

+// AccessibilityNodeProvider.HOST_VIEW_ID.
+const HOST_VIEW_ID = -1
+
+const (
+	// AccessibilityEvent constants.
+	TYPE_VIEW_HOVER_ENTER = 128
+	TYPE_VIEW_HOVER_EXIT  = 256
+)
+
+const (
+	// AccessibilityNodeInfo constants.
+	ACTION_ACCESSIBILITY_FOCUS       = 64
+	ACTION_CLEAR_ACCESSIBILITY_FOCUS = 128
+	ACTION_CLICK                     = 16
+)
+
 func (w *window) NewContext() (context, error) {
 	funcs := []func(w *window) (context, error){newAndroidVulkanContext, newAndroidGLESContext}
 	var firstErr error
@@ -306,9 +396,39 @@ func initJVM(env *C.JNIEnv, gio C.jclass, ctx C.jobject) {
 	}
 	android.appCtx = C.jni_NewGlobalRef(env, ctx)
 	android.gioCls = C.jclass(C.jni_NewGlobalRef(env, C.jobject(gio)))
+
+	cls := findClass(env, "android/view/accessibility/AccessibilityNodeInfo")
+	android.accessibilityNodeInfo.cls = C.jclass(C.jni_NewGlobalRef(env, C.jobject(cls)))
+	android.accessibilityNodeInfo.addChild = getMethodID(env, cls, "addChild", "(Landroid/view/View;I)V")
+	android.accessibilityNodeInfo.setBoundsInScreen = getMethodID(env, cls, "setBoundsInScreen", "(Landroid/graphics/Rect;)V")
+	android.accessibilityNodeInfo.setText = getMethodID(env, cls, "setText", "(Ljava/lang/CharSequence;)V")
+	android.accessibilityNodeInfo.setContentDescription = getMethodID(env, cls, "setContentDescription", "(Ljava/lang/CharSequence;)V")
+	android.accessibilityNodeInfo.setParent = getMethodID(env, cls, "setParent", "(Landroid/view/View;I)V")
+	android.accessibilityNodeInfo.addAction = getMethodID(env, cls, "addAction", "(I)V")
+	android.accessibilityNodeInfo.setClassName = getMethodID(env, cls, "setClassName", "(Ljava/lang/CharSequence;)V")
+	android.accessibilityNodeInfo.setCheckable = getMethodID(env, cls, "setCheckable", "(Z)V")
+	android.accessibilityNodeInfo.setSelected = getMethodID(env, cls, "setSelected", "(Z)V")
+	android.accessibilityNodeInfo.setChecked = getMethodID(env, cls, "setChecked", "(Z)V")
+	android.accessibilityNodeInfo.setEnabled = getMethodID(env, cls, "setEnabled", "(Z)V")
+	android.accessibilityNodeInfo.setAccessibilityFocused = getMethodID(env, cls, "setAccessibilityFocused", "(Z)V")
+
+	cls = findClass(env, "android/graphics/Rect")
+	android.rect.cls = C.jclass(C.jni_NewGlobalRef(env, C.jobject(cls)))
+	android.rect.cons = getMethodID(env, cls, "<init>", "(IIII)V")
 	android.mwriteClipboard = getStaticMethodID(env, gio, "writeClipboard", "(Landroid/content/Context;Ljava/lang/String;)V")
 	android.mreadClipboard = getStaticMethodID(env, gio, "readClipboard", "(Landroid/content/Context;)Ljava/lang/String;")
 	android.mwakeupMainThread = getStaticMethodID(env, gio, "wakeupMainThread", "()V")
+
+	intern := func(s string) C.jstring {
+		ref := C.jni_NewGlobalRef(env, C.jobject(javaString(env, s)))
+		return C.jstring(ref)
+	}
+	android.strings.androidViewView = intern("android.view.View")
+	android.strings.androidWidgetButton = intern("android.widget.Button")
+	android.strings.androidWidgetCheckBox = intern("android.widget.CheckBox")
+	android.strings.androidWidgetEditText = intern("android.widget.EditText")
+	android.strings.androidWidgetRadioButton = intern("android.widget.RadioButton")
+	android.strings.androidWidgetSwitch = intern("android.widget.Switch")
 }

 // JavaVM returns the global JNI JavaVM.
@@ -347,6 +467,9 @@ func Java_org_gioui_GioView_onCreateView(env *C.JNIEnv, class C.jclass, view C.j
 		m.setStatusColor = getMethodID(env, class, "setStatusColor", "(II)V")
 		m.setFullscreen = getMethodID(env, class, "setFullscreen", "(Z)V")
 		m.unregister = getMethodID(env, class, "unregister", "()V")
+		m.sendA11yEvent = getMethodID(env, class, "sendA11yEvent", "(II)V")
+		m.sendA11yChange = getMethodID(env, class, "sendA11yChange", "(I)V")
+		m.isA11yActive = getMethodID(env, class, "isA11yActive", "()Z")
 	})
 	view = C.jni_NewGlobalRef(env, view)
 	wopts := <-mainWindow.out
@@ -389,7 +512,7 @@ func Java_org_gioui_GioView_onStartView(env *C.JNIEnv, class C.jclass, handle C.
 	w := views[handle]
 	w.started = true
 	if w.win != nil {
-		w.setVisible()
+		w.setVisible(env)
 	}
 }

@@ -405,7 +528,7 @@ func Java_org_gioui_GioView_onSurfaceChanged(env *C.JNIEnv, class C.jclass, hand
 	w := views[handle]
 	w.win = C.ANativeWindow_fromSurface(env, surf)
 	if w.started {
-		w.setVisible()
+		w.setVisible(env)
 	}
 }

@@ -420,7 +543,7 @@ func Java_org_gioui_GioView_onConfigurationChanged(env *C.JNIEnv, class C.jclass
 	w := views[view]
 	w.loadConfig(env, class)
 	if w.stage >= system.StageRunning {
-		w.draw(true)
+		w.draw(env, true)
 	}
 }

@@ -434,7 +557,7 @@ func Java_org_gioui_GioView_onFrameCallback(env *C.JNIEnv, class C.jclass, view
 		return
 	}
 	if w.animating {
-		w.draw(false)
+		w.draw(env, false)
 		// Schedule the next draw immediately after this one. Since onFrameCallback runs
 		// on the UI thread, View.invalidate can be used here instead of postInvalidate.
 		callVoidMethod(env, w.view, gioView.invalidate)
@@ -468,10 +591,171 @@ func Java_org_gioui_GioView_onWindowInsets(env *C.JNIEnv, class C.jclass, view C
 		Left:   unit.Px(float32(left)),
 	}
 	if w.stage >= system.StageRunning {
-		w.draw(true)
+		w.draw(env, true)
 	}
 }

+//export Java_org_gioui_GioView_initializeAccessibilityNodeInfo
+func Java_org_gioui_GioView_initializeAccessibilityNodeInfo(env *C.JNIEnv, class C.jclass, view C.jlong, virtID, screenX, screenY C.jint, info C.jobject) C.jobject {
+	w := views[view]
+	semID := w.semIDFor(virtID)
+	sem, found := w.callbacks.LookupSemantic(semID)
+	if found {
+		off := f32.Pt(float32(screenX), float32(screenY))
+		if err := w.initAccessibilityNodeInfo(env, sem, off, info); err != nil {
+			panic(err)
+		}
+	}
+	return info
+}
+
+//export Java_org_gioui_GioView_onTouchExploration
+func Java_org_gioui_GioView_onTouchExploration(env *C.JNIEnv, class C.jclass, view C.jlong, x, y C.jfloat) {
+	w := views[view]
+	semID, _ := w.callbacks.SemanticAt(f32.Pt(float32(x), float32(y)))
+	if w.semantic.hoverID == semID {
+		return
+	}
+	// Android expects ENTER before EXIT.
+	if semID != 0 {
+		callVoidMethod(env, w.view, gioView.sendA11yEvent, TYPE_VIEW_HOVER_ENTER, jvalue(w.virtualIDFor(semID)))
+	}
+	if prevID := w.semantic.hoverID; prevID != 0 {
+		callVoidMethod(env, w.view, gioView.sendA11yEvent, TYPE_VIEW_HOVER_EXIT, jvalue(w.virtualIDFor(prevID)))
+	}
+	w.semantic.hoverID = semID
+}
+
+//export Java_org_gioui_GioView_onExitTouchExploration
+func Java_org_gioui_GioView_onExitTouchExploration(env *C.JNIEnv, class C.jclass, view C.jlong) {
+	w := views[view]
+	if w.semantic.hoverID != 0 {
+		callVoidMethod(env, w.view, gioView.sendA11yEvent, TYPE_VIEW_HOVER_EXIT, jvalue(w.virtualIDFor(w.semantic.hoverID)))
+		w.semantic.hoverID = 0
+	}
+}
+
+//export Java_org_gioui_GioView_onA11yFocus
+func Java_org_gioui_GioView_onA11yFocus(env *C.JNIEnv, class C.jclass, view C.jlong, virtID C.jint) {
+	w := views[view]
+	if semID := w.semIDFor(virtID); semID != w.semantic.focusID {
+		w.semantic.focusID = semID
+		// Android needs invalidate to refresh the TalkBack focus indicator.
+		callVoidMethod(env, w.view, gioView.invalidate)
+	}
+}
+
+//export Java_org_gioui_GioView_onClearA11yFocus
+func Java_org_gioui_GioView_onClearA11yFocus(env *C.JNIEnv, class C.jclass, view C.jlong, virtID C.jint) {
+	w := views[view]
+	if w.semantic.focusID == w.semIDFor(virtID) {
+		w.semantic.focusID = 0
+	}
+}
+
+func (w *window) initAccessibilityNodeInfo(env *C.JNIEnv, sem router.SemanticNode, off f32.Point, info C.jobject) error {
+	for _, ch := range sem.Children {
+		err := callVoidMethod(env, info, android.accessibilityNodeInfo.addChild, jvalue(w.view), jvalue(w.virtualIDFor(ch.ID)))
+		if err != nil {
+			return err
+		}
+	}
+	if sem.ParentID != 0 {
+		if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setParent, jvalue(w.view), jvalue(w.virtualIDFor(sem.ParentID))); err != nil {
+			return err
+		}
+		b := sem.Desc.Bounds.Add(off)
+		rect, err := newObject(env, android.rect.cls, android.rect.cons,
+			jvalue(b.Min.X),
+			jvalue(b.Min.Y),
+			jvalue(b.Max.X),
+			jvalue(b.Max.Y),
+		)
+		if err != nil {
+			return err
+		}
+		if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setBoundsInScreen, jvalue(rect)); err != nil {
+			return err
+		}
+	}
+	d := sem.Desc
+	if l := d.Label; l != "" {
+		jlbl := javaString(env, l)
+		if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setText, jvalue(jlbl)); err != nil {
+			return err
+		}
+	}
+	if d.Description != "" {
+		jd := javaString(env, d.Description)
+		if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setContentDescription, jvalue(jd)); err != nil {
+			return err
+		}
+	}
+	addAction := func(act C.jint) {
+		if err := callVoidMethod(env, info, android.accessibilityNodeInfo.addAction, jvalue(act)); err != nil {
+			panic(err)
+		}
+	}
+	if d.Gestures&router.ClickGesture != 0 {
+		addAction(ACTION_CLICK)
+	}
+	clsName := android.strings.androidViewView
+	selectMethod := android.accessibilityNodeInfo.setChecked
+	checkable := false
+	switch d.Class {
+	case semantic.Button:
+		clsName = android.strings.androidWidgetButton
+	case semantic.CheckBox:
+		checkable = true
+		clsName = android.strings.androidWidgetCheckBox
+	case semantic.Editor:
+		clsName = android.strings.androidWidgetEditText
+	case semantic.RadioButton:
+		selectMethod = android.accessibilityNodeInfo.setSelected
+		clsName = android.strings.androidWidgetRadioButton
+	case semantic.Switch:
+		checkable = true
+		clsName = android.strings.androidWidgetSwitch
+	}
+	if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setClassName, jvalue(clsName)); err != nil {
+		panic(err)
+	}
+	if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setCheckable, jvalue(javaBool(checkable))); err != nil {
+		panic(err)
+	}
+	if err := callVoidMethod(env, info, selectMethod, jvalue(javaBool(d.Selected))); err != nil {
+		panic(err)
+	}
+	if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setEnabled, jvalue(javaBool(!d.Disabled))); err != nil {
+		panic(err)
+	}
+	isFocus := w.semantic.focusID == sem.ID
+	if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setAccessibilityFocused, jvalue(javaBool(isFocus))); err != nil {
+		panic(err)
+	}
+	if isFocus {
+		addAction(ACTION_CLEAR_ACCESSIBILITY_FOCUS)
+	} else {
+		addAction(ACTION_ACCESSIBILITY_FOCUS)
+	}
+	return nil
+}
+
+func (w *window) virtualIDFor(id router.SemanticID) C.jint {
+	// TODO: Android virtual IDs are 32-bit Java integers, but childID is a int64.
+	if id == w.semantic.rootID {
+		return HOST_VIEW_ID
+	}
+	return C.jint(id)
+}
+
+func (w *window) semIDFor(virtID C.jint) router.SemanticID {
+	if virtID == HOST_VIEW_ID {
+		return w.semantic.rootID
+	}
+	return router.SemanticID(virtID)
+}
+
 func (w *window) detach(env *C.JNIEnv) {
 	callVoidMethod(env, w.view, gioView.unregister)
 	w.callbacks.Event(ViewEvent{})
@@ -481,13 +765,13 @@ func (w *window) detach(env *C.JNIEnv) {
 	w.view = 0
 }

-func (w *window) setVisible() {
+func (w *window) setVisible(env *C.JNIEnv) {
 	width, height := C.ANativeWindow_getWidth(w.win), C.ANativeWindow_getHeight(w.win)
 	if width == 0 || height == 0 {
 		return
 	}
 	w.setStage(system.StageRunning)
-	w.draw(true)
+	w.draw(env, true)
 }

 func (w *window) setStage(stage system.Stage) {
@@ -533,7 +817,7 @@ func (w *window) SetAnimating(anim bool) {
 	}
 }

-func (w *window) draw(sync bool) {
+func (w *window) draw(env *C.JNIEnv, sync bool) {
 	size := image.Pt(int(C.ANativeWindow_getWidth(w.win)), int(C.ANativeWindow_getHeight(w.win)))
 	if size != w.config.Size {
 		w.config.Size = size
@@ -556,6 +840,31 @@ func (w *window) draw(sync bool) {
 		},
 		Sync: sync,
 	})
+	a11yActive, err := callBooleanMethod(env, w.view, gioView.isA11yActive)
+	if err != nil {
+		panic(err)
+	}
+	if a11yActive {
+		if newR, oldR := w.callbacks.SemanticRoot(), w.semantic.rootID; newR != oldR {
+			// Remap focus and hover.
+			if oldR == w.semantic.hoverID {
+				w.semantic.hoverID = newR
+			}
+			if oldR == w.semantic.focusID {
+				w.semantic.focusID = newR
+			}
+			w.semantic.rootID = newR
+			callVoidMethod(env, w.view, gioView.sendA11yChange, jvalue(w.virtualIDFor(newR)))
+		}
+		diffs := w.callbacks.RequestSemanticDiffs()
+		for {
+			id := <-diffs
+			if id == 0 {
+				break
+			}
+			callVoidMethod(env, w.view, gioView.sendA11yChange, jvalue(w.virtualIDFor(id)))
+		}
+	}
 }

 type keyMapper func(devId, keyCode C.int32_t) rune
@@ -709,6 +1018,14 @@ func (w *window) SetInputHint(mode key.InputHint) {
 	})
 }

+func javaBool(b bool) C.jboolean {
+	if b {
+		return C.JNI_TRUE
+	} else {
+		return C.JNI_FALSE
+	}
+}
+
 func javaString(env *C.JNIEnv, str string) C.jstring {
 	if str == "" {
 		return 0
@@ -739,11 +1056,21 @@ func callVoidMethod(env *C.JNIEnv, obj C.jobject, method C.jmethodID, args ...jv
 	return exception(env)
 }

+func callBooleanMethod(env *C.JNIEnv, obj C.jobject, method C.jmethodID, args ...jvalue) (bool, error) {
+	res := C.jni_CallBooleanMethodA(env, obj, method, varArgs(args))
+	return res == C.JNI_TRUE, exception(env)
+}
+
 func callObjectMethod(env *C.JNIEnv, obj C.jobject, method C.jmethodID, args ...jvalue) (C.jobject, error) {
 	res := C.jni_CallObjectMethodA(env, obj, method, varArgs(args))
 	return res, exception(env)
 }

+func newObject(env *C.JNIEnv, cls C.jclass, method C.jmethodID, args ...jvalue) (C.jobject, error) {
+	res := C.jni_NewObjectA(env, cls, method, varArgs(args))
+	return res, exception(env)
+}
+
 // exception returns an error corresponding to the pending
 // exception, or nil if no exception is pending. The pending
 // exception is cleared.
@@ -790,6 +1117,12 @@ func goString(env *C.JNIEnv, str C.jstring) string {
 	return string(utf8)
 }

+func findClass(env *C.JNIEnv, name string) C.jclass {
+	cn := C.CString(name)
+	defer C.free(unsafe.Pointer(cn))
+	return C.jni_FindClass(env, cn)
+}
+
 func osMain() {
 }

@@ -10,6 +10,7 @@ import (
 	"runtime"
 	"time"

+	"gioui.org/f32"
 	"gioui.org/gpu"
 	"gioui.org/io/event"
 	"gioui.org/io/pointer"
@@ -61,6 +62,46 @@ type Window struct {
 	callbacks callbacks

 	nocontext bool
+
+	// semantic data, lazily evaluated if requested by a backend to speed up
+	// the cases where semantic data is not needed.
+	semantic struct {
+		// requestDiffs is notified when a backend requests the list of changed
+		// semantic ids.
+		requestDiffs chan struct{}
+		// diffs is sent every changed semantic id when semRequestDiffs is requested,
+		// ending with the zero id.
+		diffs chan router.SemanticID
+		// lookups is sent semantic IDs for lookup.
+		lookups chan router.SemanticID
+		// results is sent the responses for semLookups queries.
+		results chan semanticResult
+		// requestRoots is sent request for the root ID.
+		requestRoots chan struct{}
+		// roots is sent root IDs when requested throught queryRoots.
+		roots chan router.SemanticID
+		// positions is sent positional requests.
+		positions chan f32.Point
+		// positionIDs is sent results for positions requests.
+		positionIDs chan semanticID
+
+		// uptodate tracks whether the fields below are up to date.
+		uptodate bool
+		root     router.SemanticID
+		prevTree []router.SemanticNode
+		tree     []router.SemanticNode
+		ids      map[router.SemanticID]router.SemanticNode
+	}
+}
+
+type semanticID struct {
+	found bool
+	id    router.SemanticID
+}
+
+type semanticResult struct {
+	found bool
+	node  router.SemanticNode
 }

 type callbacks struct {
@@ -114,6 +155,16 @@ func NewWindow(options ...Option) *Window {
 		dead:         make(chan struct{}),
 		nocontext:    cnf.CustomRenderer,
 	}
+	w.semantic.ids = make(map[router.SemanticID]router.SemanticNode)
+	w.semantic.lookups = make(chan router.SemanticID)
+	w.semantic.results = make(chan semanticResult)
+	w.semantic.requestDiffs = make(chan struct{})
+	w.semantic.requestRoots = make(chan struct{})
+	w.semantic.roots = make(chan router.SemanticID)
+	w.semantic.positions = make(chan f32.Point)
+	w.semantic.positionIDs = make(chan semanticID)
+	// Add buffer to limit context switching when the diff is large.
+	w.semantic.diffs = make(chan router.SemanticID, 50)
 	w.callbacks.w = w
 	go w.run(options)
 	return w
@@ -220,6 +271,10 @@ func (w *Window) render(frame *op.Ops, viewport image.Point) error {

 func (w *Window) processFrame(frameStart time.Time, frame *op.Ops) {
 	w.queue.q.Frame(frame)
+	for k := range w.semantic.ids {
+		delete(w.semantic.ids, k)
+	}
+	w.semantic.uptodate = false
 	switch w.queue.q.TextInputState() {
 	case router.TextInputOpen:
 		w.driverDefer(func(d driver) { d.ShowTextInput(true) })
@@ -442,6 +497,30 @@ func (c *callbacks) Event(e event.Event) {
 	}
 }

+// SemanticRoot returns the ID of the semantic root.
+func (c *callbacks) SemanticRoot() router.SemanticID {
+	c.w.semantic.requestRoots <- struct{}{}
+	return <-c.w.semantic.roots
+}
+
+// LookupSemantic looks up a semantic node from an ID. The zero ID denotes the root.
+func (c *callbacks) LookupSemantic(semID router.SemanticID) (router.SemanticNode, bool) {
+	c.w.semantic.lookups <- semID
+	res := <-c.w.semantic.results
+	return res.node, res.found
+}
+
+func (c *callbacks) RequestSemanticDiffs() <-chan router.SemanticID {
+	c.w.semantic.requestDiffs <- struct{}{}
+	return c.w.semantic.diffs
+}
+
+func (c *callbacks) SemanticAt(pos f32.Point) (router.SemanticID, bool) {
+	c.w.semantic.positions <- pos
+	res := <-c.w.semantic.positionIDs
+	return res.id, res.found
+}
+
 func (w *Window) runFuncs(d driver) {
 	// Don't run driver functions if there's no driver.
 	if d == nil {
@@ -523,6 +602,61 @@ func (w *Window) waitFrame() (*op.Ops, bool) {
 	}
 }

+func (w *Window) lookupSemantic(id router.SemanticID) (router.SemanticNode, bool) {
+	w.updateSemantics()
+	n, ok := w.semantic.ids[id]
+	return n, ok
+}
+
+// updateSemantics refreshes the semantics tree, the id to node map and the ids of
+// updated nodes.
+func (w *Window) updateSemantics() {
+	if w.semantic.uptodate {
+		return
+	}
+	w.semantic.uptodate = true
+	w.semantic.prevTree, w.semantic.tree = w.semantic.tree, w.semantic.prevTree
+	w.semantic.tree = w.queue.q.AppendSemantics(w.semantic.tree[:0])
+	w.semantic.root = w.semantic.tree[0].ID
+	for _, n := range w.semantic.tree {
+		w.semantic.ids[n.ID] = n
+	}
+}
+
+// sendSemanticDiffs traverses the previous semantic tree and sends changed ids to
+// w.semDiffs.
+func (w *Window) sendSemanticDiffs() {
+	w.updateSemantics()
+	defer func() {
+		// Mark end of list.
+		w.semantic.diffs <- 0
+	}()
+	if tree := w.semantic.prevTree; len(tree) > 0 {
+		w.collectSemanticDiffs(w.semantic.prevTree[0])
+	}
+}
+
+// collectSemanticDiffs traverses the previous semantic tree, noting changed nodes.
+func (w *Window) collectSemanticDiffs(n router.SemanticNode) {
+	newNode, exists := w.semantic.ids[n.ID]
+	// Ignore deleted nodes, as their disappearance will be reported through an
+	// ancestor node.
+	if !exists {
+		return
+	}
+	diff := newNode.Desc != n.Desc || len(n.Children) != len(newNode.Children)
+	for i, ch := range n.Children {
+		if !diff {
+			newCh := newNode.Children[i]
+			diff = ch.ID != newCh.ID
+		}
+		w.collectSemanticDiffs(ch)
+	}
+	if diff {
+		w.semantic.diffs <- n.ID
+	}
+}
+
 func (w *Window) run(options []Option) {
 	// Some OpenGL drivers don't like being made current on many different
 	// OS threads. Force the Go runtime to map the event loop goroutine to
@@ -563,6 +697,22 @@ func (w *Window) run(options []Option) {
 			w.updateAnimation()
 		case <-wakeups:
 			wakeup()
+		case semID := <-w.semantic.lookups:
+			node, found := w.lookupSemantic(semID)
+			w.semantic.results <- semanticResult{
+				found: found,
+				node:  node,
+			}
+		case <-w.semantic.requestDiffs:
+			w.sendSemanticDiffs()
+		case <-w.semantic.requestRoots:
+			w.semantic.roots <- w.semantic.root
+		case pos := <-w.semantic.positions:
+			sid, exists := w.queue.q.SemanticAt(pos)
+			w.semantic.positionIDs <- semanticID{
+				found: exists,
+				id:    sid,
+			}
 		case e := <-w.in:
 			switch e2 := e.(type) {
 			case system.StageEvent: