diff --git a/app/GioView.java b/app/GioView.java index 328b830a..14639b01 100644 --- a/app/GioView.java +++ b/app/GioView.java @@ -16,6 +16,7 @@ import android.graphics.Canvas; import android.graphics.Color; import android.graphics.Rect; import android.os.Build; +import android.os.Bundle; import android.text.Editable; import android.util.AttributeSet; import android.util.TypedValue; @@ -37,6 +38,10 @@ import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputMethodManager; import android.view.inputmethod.EditorInfo; import android.text.InputType; +import android.view.accessibility.AccessibilityNodeProvider; +import android.view.accessibility.AccessibilityNodeInfo; +import android.view.accessibility.AccessibilityEvent; +import android.view.accessibility.AccessibilityManager; import java.io.UnsupportedEncodingException; @@ -49,6 +54,7 @@ public final class GioView extends SurfaceView { private final float scrollXScale; private final float scrollYScale; private int keyboardHint; + private AccessibilityManager accessManager; private long nhandle; @@ -88,6 +94,7 @@ public final class GioView extends SurfaceView { scrollYScale = px; } + accessManager = (AccessibilityManager)context.getSystemService(Context.ACCESSIBILITY_SERVICE); nhandle = onCreateView(this); imm = (InputMethodManager)context.getSystemService(Context.INPUT_METHOD_SERVICE); setFocusable(true); @@ -230,6 +237,53 @@ public final class GioView extends SurfaceView { this.setBarColor(Bar.NAVIGATION, color, luminance); } + @Override protected boolean dispatchHoverEvent(MotionEvent event) { + if (!accessManager.isTouchExplorationEnabled()) { + return super.dispatchHoverEvent(event); + } + switch (event.getAction()) { + case MotionEvent.ACTION_HOVER_ENTER: + // Fall through. + case MotionEvent.ACTION_HOVER_MOVE: + onTouchExploration(nhandle, event.getX(), event.getY()); + break; + case MotionEvent.ACTION_HOVER_EXIT: + onExitTouchExploration(nhandle); + break; + } + return true; + } + + void sendA11yEvent(int eventType, int viewId) { + if (!accessManager.isEnabled()) { + return; + } + AccessibilityEvent event = obtainA11yEvent(eventType, viewId); + getParent().requestSendAccessibilityEvent(this, event); + } + + AccessibilityEvent obtainA11yEvent(int eventType, int viewId) { + AccessibilityEvent event = AccessibilityEvent.obtain(eventType); + event.setPackageName(getContext().getPackageName()); + event.setSource(this, viewId); + return event; + } + + boolean isA11yActive() { + return accessManager.isEnabled(); + } + + void sendA11yChange(int viewId) { + if (!accessManager.isEnabled()) { + return; + } + AccessibilityEvent event = obtainA11yEvent(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED, viewId); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { + event.setContentChangeTypes(AccessibilityEvent.CONTENT_CHANGE_TYPE_SUBTREE); + } + getParent().requestSendAccessibilityEvent(this, event); + } + private void dispatchMotionEvent(MotionEvent event) { if (nhandle == 0) { return; @@ -365,6 +419,11 @@ public final class GioView extends SurfaceView { static private native void onFrameCallback(long handle); static private native boolean onBack(long handle); static private native void onFocusChange(long handle, boolean focus); + static private native AccessibilityNodeInfo initializeAccessibilityNodeInfo(long handle, int viewId, int screenX, int screenY, AccessibilityNodeInfo info); + static private native void onTouchExploration(long handle, float x, float y); + static private native void onExitTouchExploration(long handle); + static private native void onA11yFocus(long handle, int viewId); + static private native void onClearA11yFocus(long handle, int viewId); private static class InputConnection extends BaseInputConnection { private final Editable editable; @@ -380,4 +439,42 @@ public final class GioView extends SurfaceView { return editable; } } + + @Override public AccessibilityNodeProvider getAccessibilityNodeProvider() { + return new AccessibilityNodeProvider() { + private final int[] screenOff = new int[2]; + + @Override public AccessibilityNodeInfo createAccessibilityNodeInfo(int viewId) { + AccessibilityNodeInfo info = null; + if (viewId == View.NO_ID) { + info = AccessibilityNodeInfo.obtain(GioView.this); + GioView.this.onInitializeAccessibilityNodeInfo(info); + } else { + info = AccessibilityNodeInfo.obtain(GioView.this, viewId); + info.setPackageName(getContext().getPackageName()); + info.setVisibleToUser(true); + } + GioView.this.getLocationOnScreen(screenOff); + info = GioView.this.initializeAccessibilityNodeInfo(nhandle, viewId, screenOff[0], screenOff[1], info); + return info; + } + + @Override public boolean performAction(int viewId, int action, Bundle arguments) { + if (viewId == View.NO_ID) { + return GioView.this.performAccessibilityAction(action, arguments); + } + switch (action) { + case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS: + GioView.this.onA11yFocus(nhandle, viewId); + GioView.this.sendA11yEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED, viewId); + return true; + case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS: + GioView.this.onClearA11yFocus(nhandle, viewId); + GioView.this.sendA11yEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED, viewId); + return true; + } + return false; + } + }; + } } diff --git a/app/os.go b/app/os.go index 160267ff..81fb328d 100644 --- a/app/os.go +++ b/app/os.go @@ -147,6 +147,7 @@ type driver interface { // Close the window. Close() + // Wakeup wakes up the event loop and sends a WakeupEvent. Wakeup() diff --git a/app/os_android.go b/app/os_android.go index d3354621..b7042f59 100644 --- a/app/os_android.go +++ b/app/os_android.go @@ -64,6 +64,10 @@ static void jni_CallVoidMethodA(JNIEnv *env, jobject obj, jmethodID methodID, co (*env)->CallVoidMethodA(env, obj, methodID, args); } +static jboolean jni_CallBooleanMethodA(JNIEnv *env, jobject obj, jmethodID methodID, const jvalue *args) { + return (*env)->CallBooleanMethodA(env, obj, methodID, args); +} + static jbyte *jni_GetByteArrayElements(JNIEnv *env, jbyteArray arr) { return (*env)->GetByteArrayElements(env, arr, NULL); } @@ -103,6 +107,14 @@ static jobject jni_CallObjectMethodA(JNIEnv *env, jobject obj, jmethodID method, static jobject jni_CallStaticObjectMethodA(JNIEnv *env, jclass cls, jmethodID method, jvalue *args) { return (*env)->CallStaticObjectMethodA(env, cls, method, args); } + +static jclass jni_FindClass(JNIEnv *env, char *name) { + return (*env)->FindClass(env, name); +} + +static jobject jni_NewObjectA(JNIEnv *env, jclass cls, jmethodID cons, jvalue *args) { + return (*env)->NewObjectA(env, cls, cons, args); +} */ import "C" @@ -127,6 +139,8 @@ import ( "gioui.org/io/clipboard" "gioui.org/io/key" "gioui.org/io/pointer" + "gioui.org/io/router" + "gioui.org/io/semantic" "gioui.org/io/system" "gioui.org/unit" ) @@ -146,6 +160,12 @@ type window struct { win *C.ANativeWindow config Config + + semantic struct { + hoverID router.SemanticID + rootID router.SemanticID + focusID router.SemanticID + } } // gioView hold cached JNI methods for GioView. @@ -164,6 +184,9 @@ var gioView struct { setStatusColor C.jmethodID setFullscreen C.jmethodID unregister C.jmethodID + sendA11yEvent C.jmethodID + sendA11yChange C.jmethodID + isA11yActive C.jmethodID } // ViewEvent is sent whenever the Window's underlying Android view @@ -195,6 +218,57 @@ var android struct { mwriteClipboard C.jmethodID mreadClipboard C.jmethodID mwakeupMainThread C.jmethodID + + // android.view.accessibility.AccessibilityNodeInfo class. + accessibilityNodeInfo struct { + cls C.jclass + // addChild(View, int) + addChild C.jmethodID + // setBoundsInScreen(Rect) + setBoundsInScreen C.jmethodID + // setText(CharSequence) + setText C.jmethodID + // setContentDescription(CharSequence) + setContentDescription C.jmethodID + // setParent(View, int) + setParent C.jmethodID + // addAction(int) + addAction C.jmethodID + // setClassName(CharSequence) + setClassName C.jmethodID + // setCheckable(boolean) + setCheckable C.jmethodID + // setSelected(boolean) + setSelected C.jmethodID + // setChecked(boolean) + setChecked C.jmethodID + // setEnabled(boolean) + setEnabled C.jmethodID + // setAccessibilityFocused(boolean) + setAccessibilityFocused C.jmethodID + } + + // android.graphics.Rect class. + rect struct { + cls C.jclass + // (int, int, int, int) constructor. + cons C.jmethodID + } + + strings struct { + // "android.view.View" + androidViewView C.jstring + // "android.widget.Button" + androidWidgetButton C.jstring + // "android.widget.CheckBox" + androidWidgetCheckBox C.jstring + // "android.widget.EditText" + androidWidgetEditText C.jstring + // "android.widget.RadioButton" + androidWidgetRadioButton C.jstring + // "android.widget.Switch" + androidWidgetSwitch C.jstring + } } // view maps from GioView JNI refenreces to windows. @@ -216,6 +290,22 @@ var ( newAndroidGLESContext func(w *window) (context, error) ) +// AccessibilityNodeProvider.HOST_VIEW_ID. +const HOST_VIEW_ID = -1 + +const ( + // AccessibilityEvent constants. + TYPE_VIEW_HOVER_ENTER = 128 + TYPE_VIEW_HOVER_EXIT = 256 +) + +const ( + // AccessibilityNodeInfo constants. + ACTION_ACCESSIBILITY_FOCUS = 64 + ACTION_CLEAR_ACCESSIBILITY_FOCUS = 128 + ACTION_CLICK = 16 +) + func (w *window) NewContext() (context, error) { funcs := []func(w *window) (context, error){newAndroidVulkanContext, newAndroidGLESContext} var firstErr error @@ -306,9 +396,39 @@ func initJVM(env *C.JNIEnv, gio C.jclass, ctx C.jobject) { } android.appCtx = C.jni_NewGlobalRef(env, ctx) android.gioCls = C.jclass(C.jni_NewGlobalRef(env, C.jobject(gio))) + + cls := findClass(env, "android/view/accessibility/AccessibilityNodeInfo") + android.accessibilityNodeInfo.cls = C.jclass(C.jni_NewGlobalRef(env, C.jobject(cls))) + android.accessibilityNodeInfo.addChild = getMethodID(env, cls, "addChild", "(Landroid/view/View;I)V") + android.accessibilityNodeInfo.setBoundsInScreen = getMethodID(env, cls, "setBoundsInScreen", "(Landroid/graphics/Rect;)V") + android.accessibilityNodeInfo.setText = getMethodID(env, cls, "setText", "(Ljava/lang/CharSequence;)V") + android.accessibilityNodeInfo.setContentDescription = getMethodID(env, cls, "setContentDescription", "(Ljava/lang/CharSequence;)V") + android.accessibilityNodeInfo.setParent = getMethodID(env, cls, "setParent", "(Landroid/view/View;I)V") + android.accessibilityNodeInfo.addAction = getMethodID(env, cls, "addAction", "(I)V") + android.accessibilityNodeInfo.setClassName = getMethodID(env, cls, "setClassName", "(Ljava/lang/CharSequence;)V") + android.accessibilityNodeInfo.setCheckable = getMethodID(env, cls, "setCheckable", "(Z)V") + android.accessibilityNodeInfo.setSelected = getMethodID(env, cls, "setSelected", "(Z)V") + android.accessibilityNodeInfo.setChecked = getMethodID(env, cls, "setChecked", "(Z)V") + android.accessibilityNodeInfo.setEnabled = getMethodID(env, cls, "setEnabled", "(Z)V") + android.accessibilityNodeInfo.setAccessibilityFocused = getMethodID(env, cls, "setAccessibilityFocused", "(Z)V") + + cls = findClass(env, "android/graphics/Rect") + android.rect.cls = C.jclass(C.jni_NewGlobalRef(env, C.jobject(cls))) + android.rect.cons = getMethodID(env, cls, "", "(IIII)V") android.mwriteClipboard = getStaticMethodID(env, gio, "writeClipboard", "(Landroid/content/Context;Ljava/lang/String;)V") android.mreadClipboard = getStaticMethodID(env, gio, "readClipboard", "(Landroid/content/Context;)Ljava/lang/String;") android.mwakeupMainThread = getStaticMethodID(env, gio, "wakeupMainThread", "()V") + + intern := func(s string) C.jstring { + ref := C.jni_NewGlobalRef(env, C.jobject(javaString(env, s))) + return C.jstring(ref) + } + android.strings.androidViewView = intern("android.view.View") + android.strings.androidWidgetButton = intern("android.widget.Button") + android.strings.androidWidgetCheckBox = intern("android.widget.CheckBox") + android.strings.androidWidgetEditText = intern("android.widget.EditText") + android.strings.androidWidgetRadioButton = intern("android.widget.RadioButton") + android.strings.androidWidgetSwitch = intern("android.widget.Switch") } // JavaVM returns the global JNI JavaVM. @@ -347,6 +467,9 @@ func Java_org_gioui_GioView_onCreateView(env *C.JNIEnv, class C.jclass, view C.j m.setStatusColor = getMethodID(env, class, "setStatusColor", "(II)V") m.setFullscreen = getMethodID(env, class, "setFullscreen", "(Z)V") m.unregister = getMethodID(env, class, "unregister", "()V") + m.sendA11yEvent = getMethodID(env, class, "sendA11yEvent", "(II)V") + m.sendA11yChange = getMethodID(env, class, "sendA11yChange", "(I)V") + m.isA11yActive = getMethodID(env, class, "isA11yActive", "()Z") }) view = C.jni_NewGlobalRef(env, view) wopts := <-mainWindow.out @@ -389,7 +512,7 @@ func Java_org_gioui_GioView_onStartView(env *C.JNIEnv, class C.jclass, handle C. w := views[handle] w.started = true if w.win != nil { - w.setVisible() + w.setVisible(env) } } @@ -405,7 +528,7 @@ func Java_org_gioui_GioView_onSurfaceChanged(env *C.JNIEnv, class C.jclass, hand w := views[handle] w.win = C.ANativeWindow_fromSurface(env, surf) if w.started { - w.setVisible() + w.setVisible(env) } } @@ -420,7 +543,7 @@ func Java_org_gioui_GioView_onConfigurationChanged(env *C.JNIEnv, class C.jclass w := views[view] w.loadConfig(env, class) if w.stage >= system.StageRunning { - w.draw(true) + w.draw(env, true) } } @@ -434,7 +557,7 @@ func Java_org_gioui_GioView_onFrameCallback(env *C.JNIEnv, class C.jclass, view return } if w.animating { - w.draw(false) + w.draw(env, false) // Schedule the next draw immediately after this one. Since onFrameCallback runs // on the UI thread, View.invalidate can be used here instead of postInvalidate. callVoidMethod(env, w.view, gioView.invalidate) @@ -468,10 +591,171 @@ func Java_org_gioui_GioView_onWindowInsets(env *C.JNIEnv, class C.jclass, view C Left: unit.Px(float32(left)), } if w.stage >= system.StageRunning { - w.draw(true) + w.draw(env, true) } } +//export Java_org_gioui_GioView_initializeAccessibilityNodeInfo +func Java_org_gioui_GioView_initializeAccessibilityNodeInfo(env *C.JNIEnv, class C.jclass, view C.jlong, virtID, screenX, screenY C.jint, info C.jobject) C.jobject { + w := views[view] + semID := w.semIDFor(virtID) + sem, found := w.callbacks.LookupSemantic(semID) + if found { + off := f32.Pt(float32(screenX), float32(screenY)) + if err := w.initAccessibilityNodeInfo(env, sem, off, info); err != nil { + panic(err) + } + } + return info +} + +//export Java_org_gioui_GioView_onTouchExploration +func Java_org_gioui_GioView_onTouchExploration(env *C.JNIEnv, class C.jclass, view C.jlong, x, y C.jfloat) { + w := views[view] + semID, _ := w.callbacks.SemanticAt(f32.Pt(float32(x), float32(y))) + if w.semantic.hoverID == semID { + return + } + // Android expects ENTER before EXIT. + if semID != 0 { + callVoidMethod(env, w.view, gioView.sendA11yEvent, TYPE_VIEW_HOVER_ENTER, jvalue(w.virtualIDFor(semID))) + } + if prevID := w.semantic.hoverID; prevID != 0 { + callVoidMethod(env, w.view, gioView.sendA11yEvent, TYPE_VIEW_HOVER_EXIT, jvalue(w.virtualIDFor(prevID))) + } + w.semantic.hoverID = semID +} + +//export Java_org_gioui_GioView_onExitTouchExploration +func Java_org_gioui_GioView_onExitTouchExploration(env *C.JNIEnv, class C.jclass, view C.jlong) { + w := views[view] + if w.semantic.hoverID != 0 { + callVoidMethod(env, w.view, gioView.sendA11yEvent, TYPE_VIEW_HOVER_EXIT, jvalue(w.virtualIDFor(w.semantic.hoverID))) + w.semantic.hoverID = 0 + } +} + +//export Java_org_gioui_GioView_onA11yFocus +func Java_org_gioui_GioView_onA11yFocus(env *C.JNIEnv, class C.jclass, view C.jlong, virtID C.jint) { + w := views[view] + if semID := w.semIDFor(virtID); semID != w.semantic.focusID { + w.semantic.focusID = semID + // Android needs invalidate to refresh the TalkBack focus indicator. + callVoidMethod(env, w.view, gioView.invalidate) + } +} + +//export Java_org_gioui_GioView_onClearA11yFocus +func Java_org_gioui_GioView_onClearA11yFocus(env *C.JNIEnv, class C.jclass, view C.jlong, virtID C.jint) { + w := views[view] + if w.semantic.focusID == w.semIDFor(virtID) { + w.semantic.focusID = 0 + } +} + +func (w *window) initAccessibilityNodeInfo(env *C.JNIEnv, sem router.SemanticNode, off f32.Point, info C.jobject) error { + for _, ch := range sem.Children { + err := callVoidMethod(env, info, android.accessibilityNodeInfo.addChild, jvalue(w.view), jvalue(w.virtualIDFor(ch.ID))) + if err != nil { + return err + } + } + if sem.ParentID != 0 { + if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setParent, jvalue(w.view), jvalue(w.virtualIDFor(sem.ParentID))); err != nil { + return err + } + b := sem.Desc.Bounds.Add(off) + rect, err := newObject(env, android.rect.cls, android.rect.cons, + jvalue(b.Min.X), + jvalue(b.Min.Y), + jvalue(b.Max.X), + jvalue(b.Max.Y), + ) + if err != nil { + return err + } + if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setBoundsInScreen, jvalue(rect)); err != nil { + return err + } + } + d := sem.Desc + if l := d.Label; l != "" { + jlbl := javaString(env, l) + if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setText, jvalue(jlbl)); err != nil { + return err + } + } + if d.Description != "" { + jd := javaString(env, d.Description) + if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setContentDescription, jvalue(jd)); err != nil { + return err + } + } + addAction := func(act C.jint) { + if err := callVoidMethod(env, info, android.accessibilityNodeInfo.addAction, jvalue(act)); err != nil { + panic(err) + } + } + if d.Gestures&router.ClickGesture != 0 { + addAction(ACTION_CLICK) + } + clsName := android.strings.androidViewView + selectMethod := android.accessibilityNodeInfo.setChecked + checkable := false + switch d.Class { + case semantic.Button: + clsName = android.strings.androidWidgetButton + case semantic.CheckBox: + checkable = true + clsName = android.strings.androidWidgetCheckBox + case semantic.Editor: + clsName = android.strings.androidWidgetEditText + case semantic.RadioButton: + selectMethod = android.accessibilityNodeInfo.setSelected + clsName = android.strings.androidWidgetRadioButton + case semantic.Switch: + checkable = true + clsName = android.strings.androidWidgetSwitch + } + if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setClassName, jvalue(clsName)); err != nil { + panic(err) + } + if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setCheckable, jvalue(javaBool(checkable))); err != nil { + panic(err) + } + if err := callVoidMethod(env, info, selectMethod, jvalue(javaBool(d.Selected))); err != nil { + panic(err) + } + if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setEnabled, jvalue(javaBool(!d.Disabled))); err != nil { + panic(err) + } + isFocus := w.semantic.focusID == sem.ID + if err := callVoidMethod(env, info, android.accessibilityNodeInfo.setAccessibilityFocused, jvalue(javaBool(isFocus))); err != nil { + panic(err) + } + if isFocus { + addAction(ACTION_CLEAR_ACCESSIBILITY_FOCUS) + } else { + addAction(ACTION_ACCESSIBILITY_FOCUS) + } + return nil +} + +func (w *window) virtualIDFor(id router.SemanticID) C.jint { + // TODO: Android virtual IDs are 32-bit Java integers, but childID is a int64. + if id == w.semantic.rootID { + return HOST_VIEW_ID + } + return C.jint(id) +} + +func (w *window) semIDFor(virtID C.jint) router.SemanticID { + if virtID == HOST_VIEW_ID { + return w.semantic.rootID + } + return router.SemanticID(virtID) +} + func (w *window) detach(env *C.JNIEnv) { callVoidMethod(env, w.view, gioView.unregister) w.callbacks.Event(ViewEvent{}) @@ -481,13 +765,13 @@ func (w *window) detach(env *C.JNIEnv) { w.view = 0 } -func (w *window) setVisible() { +func (w *window) setVisible(env *C.JNIEnv) { width, height := C.ANativeWindow_getWidth(w.win), C.ANativeWindow_getHeight(w.win) if width == 0 || height == 0 { return } w.setStage(system.StageRunning) - w.draw(true) + w.draw(env, true) } func (w *window) setStage(stage system.Stage) { @@ -533,7 +817,7 @@ func (w *window) SetAnimating(anim bool) { } } -func (w *window) draw(sync bool) { +func (w *window) draw(env *C.JNIEnv, sync bool) { size := image.Pt(int(C.ANativeWindow_getWidth(w.win)), int(C.ANativeWindow_getHeight(w.win))) if size != w.config.Size { w.config.Size = size @@ -556,6 +840,31 @@ func (w *window) draw(sync bool) { }, Sync: sync, }) + a11yActive, err := callBooleanMethod(env, w.view, gioView.isA11yActive) + if err != nil { + panic(err) + } + if a11yActive { + if newR, oldR := w.callbacks.SemanticRoot(), w.semantic.rootID; newR != oldR { + // Remap focus and hover. + if oldR == w.semantic.hoverID { + w.semantic.hoverID = newR + } + if oldR == w.semantic.focusID { + w.semantic.focusID = newR + } + w.semantic.rootID = newR + callVoidMethod(env, w.view, gioView.sendA11yChange, jvalue(w.virtualIDFor(newR))) + } + diffs := w.callbacks.RequestSemanticDiffs() + for { + id := <-diffs + if id == 0 { + break + } + callVoidMethod(env, w.view, gioView.sendA11yChange, jvalue(w.virtualIDFor(id))) + } + } } type keyMapper func(devId, keyCode C.int32_t) rune @@ -709,6 +1018,14 @@ func (w *window) SetInputHint(mode key.InputHint) { }) } +func javaBool(b bool) C.jboolean { + if b { + return C.JNI_TRUE + } else { + return C.JNI_FALSE + } +} + func javaString(env *C.JNIEnv, str string) C.jstring { if str == "" { return 0 @@ -739,11 +1056,21 @@ func callVoidMethod(env *C.JNIEnv, obj C.jobject, method C.jmethodID, args ...jv return exception(env) } +func callBooleanMethod(env *C.JNIEnv, obj C.jobject, method C.jmethodID, args ...jvalue) (bool, error) { + res := C.jni_CallBooleanMethodA(env, obj, method, varArgs(args)) + return res == C.JNI_TRUE, exception(env) +} + func callObjectMethod(env *C.JNIEnv, obj C.jobject, method C.jmethodID, args ...jvalue) (C.jobject, error) { res := C.jni_CallObjectMethodA(env, obj, method, varArgs(args)) return res, exception(env) } +func newObject(env *C.JNIEnv, cls C.jclass, method C.jmethodID, args ...jvalue) (C.jobject, error) { + res := C.jni_NewObjectA(env, cls, method, varArgs(args)) + return res, exception(env) +} + // exception returns an error corresponding to the pending // exception, or nil if no exception is pending. The pending // exception is cleared. @@ -790,6 +1117,12 @@ func goString(env *C.JNIEnv, str C.jstring) string { return string(utf8) } +func findClass(env *C.JNIEnv, name string) C.jclass { + cn := C.CString(name) + defer C.free(unsafe.Pointer(cn)) + return C.jni_FindClass(env, cn) +} + func osMain() { } diff --git a/app/window.go b/app/window.go index 2ac0cda0..95148d13 100644 --- a/app/window.go +++ b/app/window.go @@ -10,6 +10,7 @@ import ( "runtime" "time" + "gioui.org/f32" "gioui.org/gpu" "gioui.org/io/event" "gioui.org/io/pointer" @@ -61,6 +62,46 @@ type Window struct { callbacks callbacks nocontext bool + + // semantic data, lazily evaluated if requested by a backend to speed up + // the cases where semantic data is not needed. + semantic struct { + // requestDiffs is notified when a backend requests the list of changed + // semantic ids. + requestDiffs chan struct{} + // diffs is sent every changed semantic id when semRequestDiffs is requested, + // ending with the zero id. + diffs chan router.SemanticID + // lookups is sent semantic IDs for lookup. + lookups chan router.SemanticID + // results is sent the responses for semLookups queries. + results chan semanticResult + // requestRoots is sent request for the root ID. + requestRoots chan struct{} + // roots is sent root IDs when requested throught queryRoots. + roots chan router.SemanticID + // positions is sent positional requests. + positions chan f32.Point + // positionIDs is sent results for positions requests. + positionIDs chan semanticID + + // uptodate tracks whether the fields below are up to date. + uptodate bool + root router.SemanticID + prevTree []router.SemanticNode + tree []router.SemanticNode + ids map[router.SemanticID]router.SemanticNode + } +} + +type semanticID struct { + found bool + id router.SemanticID +} + +type semanticResult struct { + found bool + node router.SemanticNode } type callbacks struct { @@ -114,6 +155,16 @@ func NewWindow(options ...Option) *Window { dead: make(chan struct{}), nocontext: cnf.CustomRenderer, } + w.semantic.ids = make(map[router.SemanticID]router.SemanticNode) + w.semantic.lookups = make(chan router.SemanticID) + w.semantic.results = make(chan semanticResult) + w.semantic.requestDiffs = make(chan struct{}) + w.semantic.requestRoots = make(chan struct{}) + w.semantic.roots = make(chan router.SemanticID) + w.semantic.positions = make(chan f32.Point) + w.semantic.positionIDs = make(chan semanticID) + // Add buffer to limit context switching when the diff is large. + w.semantic.diffs = make(chan router.SemanticID, 50) w.callbacks.w = w go w.run(options) return w @@ -220,6 +271,10 @@ func (w *Window) render(frame *op.Ops, viewport image.Point) error { func (w *Window) processFrame(frameStart time.Time, frame *op.Ops) { w.queue.q.Frame(frame) + for k := range w.semantic.ids { + delete(w.semantic.ids, k) + } + w.semantic.uptodate = false switch w.queue.q.TextInputState() { case router.TextInputOpen: w.driverDefer(func(d driver) { d.ShowTextInput(true) }) @@ -442,6 +497,30 @@ func (c *callbacks) Event(e event.Event) { } } +// SemanticRoot returns the ID of the semantic root. +func (c *callbacks) SemanticRoot() router.SemanticID { + c.w.semantic.requestRoots <- struct{}{} + return <-c.w.semantic.roots +} + +// LookupSemantic looks up a semantic node from an ID. The zero ID denotes the root. +func (c *callbacks) LookupSemantic(semID router.SemanticID) (router.SemanticNode, bool) { + c.w.semantic.lookups <- semID + res := <-c.w.semantic.results + return res.node, res.found +} + +func (c *callbacks) RequestSemanticDiffs() <-chan router.SemanticID { + c.w.semantic.requestDiffs <- struct{}{} + return c.w.semantic.diffs +} + +func (c *callbacks) SemanticAt(pos f32.Point) (router.SemanticID, bool) { + c.w.semantic.positions <- pos + res := <-c.w.semantic.positionIDs + return res.id, res.found +} + func (w *Window) runFuncs(d driver) { // Don't run driver functions if there's no driver. if d == nil { @@ -523,6 +602,61 @@ func (w *Window) waitFrame() (*op.Ops, bool) { } } +func (w *Window) lookupSemantic(id router.SemanticID) (router.SemanticNode, bool) { + w.updateSemantics() + n, ok := w.semantic.ids[id] + return n, ok +} + +// updateSemantics refreshes the semantics tree, the id to node map and the ids of +// updated nodes. +func (w *Window) updateSemantics() { + if w.semantic.uptodate { + return + } + w.semantic.uptodate = true + w.semantic.prevTree, w.semantic.tree = w.semantic.tree, w.semantic.prevTree + w.semantic.tree = w.queue.q.AppendSemantics(w.semantic.tree[:0]) + w.semantic.root = w.semantic.tree[0].ID + for _, n := range w.semantic.tree { + w.semantic.ids[n.ID] = n + } +} + +// sendSemanticDiffs traverses the previous semantic tree and sends changed ids to +// w.semDiffs. +func (w *Window) sendSemanticDiffs() { + w.updateSemantics() + defer func() { + // Mark end of list. + w.semantic.diffs <- 0 + }() + if tree := w.semantic.prevTree; len(tree) > 0 { + w.collectSemanticDiffs(w.semantic.prevTree[0]) + } +} + +// collectSemanticDiffs traverses the previous semantic tree, noting changed nodes. +func (w *Window) collectSemanticDiffs(n router.SemanticNode) { + newNode, exists := w.semantic.ids[n.ID] + // Ignore deleted nodes, as their disappearance will be reported through an + // ancestor node. + if !exists { + return + } + diff := newNode.Desc != n.Desc || len(n.Children) != len(newNode.Children) + for i, ch := range n.Children { + if !diff { + newCh := newNode.Children[i] + diff = ch.ID != newCh.ID + } + w.collectSemanticDiffs(ch) + } + if diff { + w.semantic.diffs <- n.ID + } +} + func (w *Window) run(options []Option) { // Some OpenGL drivers don't like being made current on many different // OS threads. Force the Go runtime to map the event loop goroutine to @@ -563,6 +697,22 @@ func (w *Window) run(options []Option) { w.updateAnimation() case <-wakeups: wakeup() + case semID := <-w.semantic.lookups: + node, found := w.lookupSemantic(semID) + w.semantic.results <- semanticResult{ + found: found, + node: node, + } + case <-w.semantic.requestDiffs: + w.sendSemanticDiffs() + case <-w.semantic.requestRoots: + w.semantic.roots <- w.semantic.root + case pos := <-w.semantic.positions: + sid, exists := w.queue.q.SemanticAt(pos) + w.semantic.positionIDs <- semanticID{ + found: exists, + id: sid, + } case e := <-w.in: switch e2 := e.(type) { case system.StageEvent: