From 58cdb3e1da2ef8986a72e4a097beb6e31fb03073 Mon Sep 17 00:00:00 2001 From: Elias Naur Date: Thu, 13 Jan 2022 13:04:31 +0100 Subject: [PATCH] app,widget: implement Editor IME support, add Android implementation Fixes: https://todo.sr.ht/~eliasnaur/gio/116 References: https://todo.sr.ht/~eliasnaur/gio/246 Signed-off-by: Elias Naur --- app/GioView.java | 326 ++++++++++++++++++++++++++++++++++++++++++-- app/ime_test.go | 109 +++++++++++++++ app/os.go | 12 +- app/os_android.go | 116 +++++++++++++++- app/os_ios.go | 6 +- app/os_js.go | 4 +- app/os_macos.go | 4 +- app/os_wayland.go | 9 +- app/os_windows.go | 4 +- app/os_x11.go | 9 +- app/window.go | 105 ++++++++++++++ internal/ops/ops.go | 10 +- io/key/key.go | 64 ++++++++- io/router/key.go | 36 ++++- io/router/router.go | 31 ++++- widget/editor.go | 90 +++++++++++- 16 files changed, 880 insertions(+), 55 deletions(-) create mode 100644 app/ime_test.go diff --git a/app/GioView.java b/app/GioView.java index d67503bb..b788caf5 100644 --- a/app/GioView.java +++ b/app/GioView.java @@ -17,7 +17,11 @@ import android.graphics.Color; import android.graphics.Rect; import android.os.Build; import android.os.Bundle; -import android.text.Editable; +import android.os.Handler; +import android.os.SystemClock; +import android.text.TextUtils; +import android.text.Selection; +import android.text.SpannableStringBuilder; import android.util.AttributeSet; import android.util.TypedValue; import android.view.KeyCharacterMap; @@ -33,11 +37,15 @@ import android.view.SurfaceHolder; import android.view.Window; import android.view.WindowInsetsController; import android.view.WindowManager; -import android.view.inputmethod.BaseInputConnection; import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputMethodManager; import android.view.inputmethod.EditorInfo; -import android.text.InputType; +import android.view.inputmethod.CorrectionInfo; +import android.view.inputmethod.CompletionInfo; +import android.view.inputmethod.InputContentInfo; +import android.view.inputmethod.ExtractedText; +import android.view.inputmethod.ExtractedTextRequest; +import android.view.inputmethod.SurroundingText; import android.view.accessibility.AccessibilityNodeProvider; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityEvent; @@ -332,9 +340,18 @@ public final class GioView extends SurfaceView { } @Override public InputConnection onCreateInputConnection(EditorInfo editor) { + Snippet snip = getSnippet(); editor.inputType = this.keyboardHint; editor.imeOptions = EditorInfo.IME_FLAG_NO_FULLSCREEN | EditorInfo.IME_FLAG_NO_EXTRACT_UI; - return new InputConnection(this); + editor.initialSelStart = snip.toChars(imeSelectionStart(nhandle)); + editor.initialSelEnd = snip.toChars(imeSelectionEnd(nhandle)); + int selStart = editor.initialSelStart - snip.offset; + editor.initialCapsMode = TextUtils.getCapsMode(snip.snippet, selStart, this.keyboardHint); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + editor.setInitialSurroundingSubText(snip.snippet, snip.toChars(snip.offset)); + } + imeSetComposingRegion(nhandle, -1, -1); + return new GioInputConnection(); } void setInputHint(int hint) { @@ -342,7 +359,7 @@ public final class GioView extends SurfaceView { return; } this.keyboardHint = hint; - imm.restartInput(this); + restartInput(); } void showTextInput() { @@ -412,6 +429,19 @@ public final class GioView extends SurfaceView { return onBack(nhandle); } + void restartInput() { + imm.restartInput(this); + } + + void updateSelection() { + Snippet snip = getSnippet(); + int selStart = snip.toChars(imeSelectionStart(nhandle)); + int selEnd = snip.toChars(imeSelectionEnd(nhandle)); + int compStart = snip.toChars(imeComposingStart(nhandle)); + int compEnd = snip.toChars(imeComposingEnd(nhandle)); + imm.updateSelection(this, selStart, selEnd, compStart, compEnd); + } + static private native long onCreateView(GioView view); static private native void onDestroyView(long handle); static private native void onStartView(long handle); @@ -431,19 +461,287 @@ public final class GioView extends SurfaceView { static private native void onExitTouchExploration(long handle); static private native void onA11yFocus(long handle, int viewId); static private native void onClearA11yFocus(long handle, int viewId); + static private native void imeSetSnippet(long handle, int start, int end); + static private native String imeSnippet(long handle); + static private native int imeSnippetStart(long handle); + static private native int imeSelectionStart(long handle); + static private native int imeSelectionEnd(long handle); + static private native int imeComposingStart(long handle); + static private native int imeComposingEnd(long handle); + static private native int imeReplace(long handle, int start, int end, String text); + static private native int imeSetSelection(long handle, int start, int end); + static private native int imeSetComposingRegion(long handle, int start, int end); - private static class InputConnection extends BaseInputConnection { - private final Editable editable; + private class GioInputConnection implements InputConnection { + private int batchDepth; - InputConnection(View view) { - // Passing false enables "dummy mode", where the BaseInputConnection - // attempts to convert IME operations to key events. - super(view, false); - editable = Editable.Factory.getInstance().newEditable(""); + @Override public boolean beginBatchEdit() { + batchDepth++; + return true; } - @Override public Editable getEditable() { - return editable; + @Override public boolean endBatchEdit() { + batchDepth--; + return batchDepth > 0; + } + + @Override public boolean clearMetaKeyStates(int states) { + return false; + } + + @Override public boolean commitCompletion(CompletionInfo text) { + return false; + } + + @Override public boolean commitCorrection(CorrectionInfo info) { + return false; + } + + @Override public boolean commitText(CharSequence text, int cursor) { + setComposingText(text, cursor); + return finishComposingText(); + } + + @Override public boolean deleteSurroundingText(int beforeChars, int afterChars) { + // translate before and after to runes. + int selStart = imeSelectionStart(nhandle); + int selEnd = imeSelectionEnd(nhandle); + Snippet snip = getSnippet(); + int before = selStart - snip.toRunes(snip.toChars(selStart) - beforeChars); + int after = selEnd - snip.toRunes(snip.toChars(selEnd) - afterChars); + return deleteSurroundingTextInCodePoints(before, after); + } + + @Override public boolean finishComposingText() { + imeSetComposingRegion(nhandle, -1, -1); + return true; + } + + @Override public int getCursorCapsMode(int reqModes) { + Snippet snip = getSnippet(); + int selStart = imeSelectionStart(nhandle); + return TextUtils.getCapsMode(snip.snippet, snip.toChars(selStart), reqModes); + } + + @Override public ExtractedText getExtractedText(ExtractedTextRequest request, int flags) { + return null; + } + + @Override public CharSequence getSelectedText(int flags) { + Snippet snip = getSnippet(); + int selStart = imeSelectionStart(nhandle); + int selEnd = imeSelectionEnd(nhandle); + String sub = snip.substringRunes(selStart, selEnd); + return sub; + } + + @Override public CharSequence getTextAfterCursor(int n, int flags) { + Snippet snip = getSnippet(); + int selStart = imeSelectionStart(nhandle); + int selEnd = imeSelectionEnd(nhandle); + // n are in Java characters, but in worst case we'll just ask for more runes + // than wanted. + imeSetSnippet(nhandle, selStart - n, selEnd + n); + int start = selEnd; + int end = snip.toRunes(snip.toChars(selEnd) + n); + String ret = snip.substringRunes(start, end); + return ret; + } + + @Override public CharSequence getTextBeforeCursor(int n, int flags) { + Snippet snip = getSnippet(); + int selStart = imeSelectionStart(nhandle); + int selEnd = imeSelectionEnd(nhandle); + // n are in Java characters, but in worst case we'll just ask for more runes + // than wanted. + imeSetSnippet(nhandle, selStart - n, selEnd + n); + int start = snip.toRunes(snip.toChars(selStart) - n); + int end = selStart; + String ret = snip.substringRunes(start, end); + return ret; + } + + @Override public boolean performContextMenuAction(int id) { + return false; + } + + @Override public boolean performEditorAction(int editorAction) { + long eventTime = SystemClock.uptimeMillis(); + // Translate to enter key. + onKeyEvent(nhandle, KeyEvent.KEYCODE_ENTER, '\n', true, eventTime); + onKeyEvent(nhandle, KeyEvent.KEYCODE_ENTER, '\n', false, eventTime); + return true; + } + + @Override public boolean performPrivateCommand(String action, Bundle data) { + return false; + } + + @Override public boolean reportFullscreenMode(boolean enabled) { + return false; + } + + @Override public boolean sendKeyEvent(KeyEvent event) { + boolean pressed = event.getAction() == KeyEvent.ACTION_DOWN; + onKeyEvent(nhandle, event.getKeyCode(), event.getUnicodeChar(), pressed, event.getEventTime()); + return true; + } + + @Override public boolean setComposingRegion(int startChars, int endChars) { + Snippet snip = getSnippet(); + int compStart = snip.toRunes(startChars); + int compEnd = snip.toRunes(endChars); + imeSetComposingRegion(nhandle, compStart, compEnd); + return true; + } + + @Override public boolean setComposingText(CharSequence text, int relCursor) { + int start = imeComposingStart(nhandle); + int end = imeComposingEnd(nhandle); + if (start == -1 || end == -1) { + start = imeSelectionStart(nhandle); + end = imeSelectionEnd(nhandle); + } + String str = text.toString(); + imeReplace(nhandle, start, end, str); + int cursor = start; + int runes = str.codePointCount(0, str.length()); + if (relCursor > 0) { + cursor += runes; + relCursor--; + } + imeSetComposingRegion(nhandle, start, start + runes); + + // Move cursor. + Snippet snip = getSnippet(); + cursor = snip.toRunes(snip.toChars(cursor) + relCursor); + imeSetSelection(nhandle, cursor, cursor); + return true; + } + + @Override public boolean setSelection(int startChars, int endChars) { + Snippet snip = getSnippet(); + int start = snip.toRunes(startChars); + int end = snip.toRunes(endChars); + imeSetSelection(nhandle, start, end); + return true; + } + + /*@Override*/ public boolean requestCursorUpdates(int cursorUpdateMode) { + return false; + } + + /*@Override*/ public void closeConnection() { + } + + /*@Override*/ public Handler getHandler() { + return null; + } + + /*@Override*/ public boolean commitContent(InputContentInfo info, int flags, Bundle opts) { + return false; + } + + /*@Override*/ public boolean deleteSurroundingTextInCodePoints(int before, int after) { + if (after > 0) { + int selEnd = imeSelectionEnd(nhandle); + imeReplace(nhandle, selEnd, selEnd + after, ""); + } + if (before > 0) { + int selStart = imeSelectionStart(nhandle); + imeReplace(nhandle, selStart - before, selStart, ""); + } + return true; + } + + /*@Override*/ public SurroundingText getSurroundingText(int beforeChars, int afterChars, int flags) { + Snippet snip = getSnippet(); + int selStart = imeSelectionStart(nhandle); + int selEnd = imeSelectionEnd(nhandle); + // Expanding in Java characters is ok. + imeSetSnippet(nhandle, selStart - beforeChars, selEnd + afterChars); + return new SurroundingText(snip.snippet, snip.toChars(selStart), snip.toChars(selEnd), snip.toChars(snip.offset)); + } + } + + private Snippet getSnippet() { + Snippet snip = new Snippet(); + snip.snippet = imeSnippet(nhandle); + snip.offset = imeSnippetStart(nhandle); + return snip; + } + + // Snippet is like android.view.inputmethod.SurroundingText but available for Android < 31. + private static class Snippet { + String snippet; + // offset of snippet into the entire editor content. It is in runes because we won't require + // Gio editors to keep track of UTF-16 offsets. The disctinction won't matter in practice because IMEs only + // ever see snippets. + int offset; + + // toRunes converts the Java character index into runes (Java code points). + int toRunes(int chars) { + if (chars == -1) { + return -1; + } + if (chars < this.offset) { + // Assume runes before offset are one Java character each. + return chars; + } + int runes = this.offset; + chars -= this.offset; + if (chars > snippet.length()) { + // Assume runes after snippets are one Java character each. + runes += chars - snippet.length(); + chars = snippet.length(); + } + runes += snippet.codePointCount(0, chars); + return runes; + } + + // toChars converts the rune index into Java characters. + int toChars(int runes) { + if (runes == -1) { + return -1; + } + if (runes < this.offset) { + // Assume runes before offset are one Java character each. + return runes; + } + int chars = this.offset; + runes -= this.offset; + int n = snippet.codePointCount(0, snippet.length()); + if (runes > n) { + // Assume runes after snippets are one Java character each. + chars += runes - n; + runes = n; + } + chars += snippet.offsetByCodePoints(0, runes); + return chars; + } + + // substringRunes returns the substring from start to end in runes. The resuls is + // truncated to the snippet. + String substringRunes(int start, int end) { + start -= this.offset; + end -= this.offset; + int runes = snippet.codePointCount(0, snippet.length()); + if (start < 0) { + start = 0; + } + if (end < 0) { + end = 0; + } + if (start > runes) { + start = runes; + } + if (end > runes) { + end = runes; + } + return snippet.substring( + snippet.offsetByCodePoints(0, start), + snippet.offsetByCodePoints(0, end) + ); } } diff --git a/app/ime_test.go b/app/ime_test.go new file mode 100644 index 00000000..92a12d6c --- /dev/null +++ b/app/ime_test.go @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: Unlicense OR MIT + +package app + +import ( + "testing" + + "gioui.org/font/gofont" + "gioui.org/io/key" + "gioui.org/io/router" + "gioui.org/layout" + "gioui.org/op" + "gioui.org/text" + "gioui.org/unit" + "gioui.org/widget" +) + +func FuzzIME(f *testing.F) { + runes := []rune("Hello, 世界! 🤬 علي،الحسنب北查爾斯頓工廠的安全漏洞已") + f.Add([]byte("20\x0010")) + f.Add([]byte("80000")) + f.Add([]byte("2008\"80\r00")) + f.Add([]byte("20007900002\x02000")) + f.Add([]byte("20007800002\x02000")) + f.Add([]byte("200A02000990\x19002\x17\x0200")) + f.Fuzz(func(t *testing.T, cmds []byte) { + cache := text.NewCache(gofont.Collection()) + e := new(widget.Editor) + e.Focus() + + var r router.Router + gtx := layout.Context{Ops: new(op.Ops), Queue: &r} + // Layout once to register focus. + e.Layout(gtx, cache, text.Font{}, unit.Px(10), nil) + r.Frame(gtx.Ops) + + var state editorState + const ( + cmdReplace = iota + cmdSelect + cmdSnip + maxCmd + ) + const cmdLen = 5 + for len(cmds) >= cmdLen { + rng := key.Range{ + Start: int(cmds[1]), + End: int(cmds[2]), + } + switch cmds[0] % cmdLen { + case cmdReplace: + rstart := int(cmds[3]) % len(runes) + rend := int(cmds[4]) % len(runes) + if rstart > rend { + rstart, rend = rend, rstart + } + replacement := string(runes[rstart:rend]) + state.Replace(rng, replacement) + r.Queue(key.EditEvent{Range: rng, Text: replacement}) + r.Queue(key.SnippetEvent(state.Snippet.Range)) + case cmdSelect: + r.Queue(key.SelectionEvent(rng)) + runes := []rune(e.Text()) + if rng.Start < 0 { + rng.Start = 0 + } + if rng.End < 0 { + rng.End = 0 + } + if rng.Start > len(runes) { + rng.Start = len(runes) + } + if rng.End > len(runes) { + rng.End = len(runes) + } + state.Selection = rng + case cmdSnip: + r.Queue(key.SnippetEvent(rng)) + runes := []rune(e.Text()) + if rng.Start > rng.End { + rng.Start, rng.End = rng.End, rng.Start + } + if rng.Start < 0 { + rng.Start = 0 + } + if rng.End < 0 { + rng.End = 0 + } + if rng.Start > len(runes) { + rng.Start = len(runes) + } + if rng.End > len(runes) { + rng.End = len(runes) + } + state.Snippet = key.Snippet{ + Range: rng, + Text: string(runes[rng.Start:rng.End]), + } + } + cmds = cmds[cmdLen:] + e.Layout(gtx, cache, text.Font{}, unit.Px(10), nil) + r.Frame(gtx.Ops) + newState := r.EditorState() + if newState != state.EditorState { + t.Errorf("IME state: %+v\neditor state: %+v", state.EditorState, newState) + } + } + }) +} diff --git a/app/os.go b/app/os.go index e44b0d51..fdf76c90 100644 --- a/app/os.go +++ b/app/os.go @@ -152,36 +152,28 @@ type driver interface { // SetAnimating sets the animation flag. When the window is animating, // FrameEvents are delivered as fast as the display can handle them. SetAnimating(anim bool) - // ShowTextInput updates the virtual keyboard state. ShowTextInput(show bool) - SetInputHint(mode key.InputHint) - NewContext() (context, error) - // ReadClipboard requests the clipboard content. ReadClipboard() // WriteClipboard requests a clipboard write. WriteClipboard(s string) - // Configure the window. Configure([]Option) - // SetCursor updates the current cursor to name. SetCursor(name pointer.CursorName) - // Raise the window at the top. Raise() - // Close the window. Close() - // Wakeup wakes up the event loop and sends a WakeupEvent. Wakeup() - // Perform actions on the window. Perform(system.Action) + // EditorStateChanged notifies the driver that the editor state changed. + EditorStateChanged(old, new editorState) } type windowRendezvous struct { diff --git a/app/os_android.go b/app/os_android.go index f3529422..543b2d27 100644 --- a/app/os_android.go +++ b/app/os_android.go @@ -188,6 +188,8 @@ var gioView struct { sendA11yEvent C.jmethodID sendA11yChange C.jmethodID isA11yActive C.jmethodID + restartInput C.jmethodID + updateSelection C.jmethodID } // ViewEvent is sent whenever the Window's underlying Android view @@ -471,6 +473,8 @@ func Java_org_gioui_GioView_onCreateView(env *C.JNIEnv, class C.jclass, view C.j m.sendA11yEvent = getMethodID(env, class, "sendA11yEvent", "(II)V") m.sendA11yChange = getMethodID(env, class, "sendA11yChange", "(I)V") m.isA11yActive = getMethodID(env, class, "isA11yActive", "()Z") + m.restartInput = getMethodID(env, class, "restartInput", "()V") + m.updateSelection = getMethodID(env, class, "updateSelection", "()V") }) view = C.jni_NewGlobalRef(env, view) wopts := <-mainWindow.out @@ -490,6 +494,7 @@ func Java_org_gioui_GioView_onCreateView(env *C.JNIEnv, class C.jclass, view C.j views[handle] = w w.loadConfig(env, class) w.Configure(wopts.options) + w.SetInputHint(key.HintAny) w.setStage(system.StagePaused) w.callbacks.Event(ViewEvent{View: uintptr(view)}) return handle @@ -930,7 +935,7 @@ func Java_org_gioui_GioView_onKeyEvent(env *C.JNIEnv, class C.jclass, handle C.j w.callbacks.Event(key.Event{Name: n, State: state}) } if pressed == C.JNI_TRUE && r != 0 && r != '\n' { // Checking for "\n" to prevent duplication with key.NameEnter (gio#224). - w.callbacks.Event(key.EditEvent{Text: string(rune(r))}) + w.callbacks.EditorInsert(string(rune(r))) } } @@ -988,6 +993,107 @@ func Java_org_gioui_GioView_onTouchEvent(env *C.JNIEnv, class C.jclass, handle C }) } +//export Java_org_gioui_GioView_imeSelectionStart +func Java_org_gioui_GioView_imeSelectionStart(env *C.JNIEnv, class C.jclass, handle C.jlong) C.jint { + w := views[handle] + sel := w.callbacks.EditorState().Selection + start := sel.Start + if sel.End < sel.Start { + start = sel.End + } + return C.jint(start) +} + +//export Java_org_gioui_GioView_imeSelectionEnd +func Java_org_gioui_GioView_imeSelectionEnd(env *C.JNIEnv, class C.jclass, handle C.jlong) C.jint { + w := views[handle] + sel := w.callbacks.EditorState().Selection + end := sel.End + if sel.End < sel.Start { + end = sel.Start + } + return C.jint(end) +} + +//export Java_org_gioui_GioView_imeComposingStart +func Java_org_gioui_GioView_imeComposingStart(env *C.JNIEnv, class C.jclass, handle C.jlong) C.jint { + w := views[handle] + comp := w.callbacks.EditorState().compose + start := comp.Start + if e := comp.End; e < start { + start = e + } + return C.jint(start) +} + +//export Java_org_gioui_GioView_imeComposingEnd +func Java_org_gioui_GioView_imeComposingEnd(env *C.JNIEnv, class C.jclass, handle C.jlong) C.jint { + w := views[handle] + comp := w.callbacks.EditorState().compose + end := comp.End + if s := comp.Start; s > end { + end = s + } + return C.jint(end) +} + +//export Java_org_gioui_GioView_imeSnippet +func Java_org_gioui_GioView_imeSnippet(env *C.JNIEnv, class C.jclass, handle C.jlong) C.jstring { + w := views[handle] + snip := w.callbacks.EditorState().Snippet.Text + return javaString(env, snip) +} + +//export Java_org_gioui_GioView_imeSnippetStart +func Java_org_gioui_GioView_imeSnippetStart(env *C.JNIEnv, class C.jclass, handle C.jlong) C.jint { + w := views[handle] + return C.jint(w.callbacks.EditorState().Snippet.Start) +} + +//export Java_org_gioui_GioView_imeSetSnippet +func Java_org_gioui_GioView_imeSetSnippet(env *C.JNIEnv, class C.jclass, handle C.jlong, start, end C.jint) { + w := views[handle] + r := key.Range{Start: int(start), End: int(end)} + w.callbacks.SetEditorSnippet(r) +} + +//export Java_org_gioui_GioView_imeSetSelection +func Java_org_gioui_GioView_imeSetSelection(env *C.JNIEnv, class C.jclass, handle C.jlong, start, end C.jint) { + w := views[handle] + r := key.Range{Start: int(start), End: int(end)} + w.callbacks.SetEditorSelection(r) +} + +//export Java_org_gioui_GioView_imeSetComposingRegion +func Java_org_gioui_GioView_imeSetComposingRegion(env *C.JNIEnv, class C.jclass, handle C.jlong, start, end C.jint) { + w := views[handle] + w.callbacks.SetComposingRegion(key.Range{ + Start: int(start), + End: int(end), + }) +} + +//export Java_org_gioui_GioView_imeReplace +func Java_org_gioui_GioView_imeReplace(env *C.JNIEnv, class C.jclass, handle C.jlong, start, end C.jint, jtext C.jstring) { + w := views[handle] + r := key.Range{Start: int(start), End: int(end)} + text := goString(env, jtext) + w.callbacks.EditorReplace(r, text) +} + +func (w *window) EditorStateChanged(old, new editorState) { + runInJVM(javaVM(), func(env *C.JNIEnv) { + if old.Snippet != new.Snippet { + callVoidMethod(env, w.view, gioView.restartInput) + return + } + if old.Selection != new.Selection { + w.callbacks.SetComposingRegion(key.Range{Start: -1, End: -1}) + } + callVoidMethod(env, w.view, gioView.updateSelection) + }) +} + func (w *window) ShowTextInput(show bool) { runInJVM(javaVM(), func(env *C.JNIEnv) { if show { @@ -1002,6 +1108,7 @@ func (w *window) SetInputHint(mode key.InputHint) { // Constants defined at https://developer.android.com/reference/android/text/InputType. const ( TYPE_NULL = 0 + TYPE_CLASS_TEXT = 1 TYPE_CLASS_NUMBER = 2 TYPE_NUMBER_FLAG_DECIMAL = 8192 TYPE_NUMBER_FLAG_SIGNED = 4096 @@ -1015,14 +1122,9 @@ func (w *window) SetInputHint(mode key.InputHint) { case key.HintNumeric: m = TYPE_CLASS_NUMBER | TYPE_NUMBER_FLAG_DECIMAL | TYPE_NUMBER_FLAG_SIGNED default: - // TYPE_NULL, since TYPE_CLASS_TEXT isn't currently supported. - m = TYPE_NULL + m = TYPE_CLASS_TEXT } - // The TYPE_TEXT_FLAG_NO_SUGGESTIONS and TYPE_TEXT_VARIATION_VISIBLE_PASSWORD are used to fix the - // Samsung keyboard compatibility, forcing to disable the suggests/auto-complete. gio#116. - m = m | TYPE_TEXT_FLAG_NO_SUGGESTIONS | TYPE_TEXT_VARIATION_VISIBLE_PASSWORD - callVoidMethod(env, w.view, gioView.setInputHint, m) }) } diff --git a/app/os_ios.go b/app/os_ios.go index f6fc6147..ed79017f 100644 --- a/app/os_ios.go +++ b/app/os_ios.go @@ -228,9 +228,7 @@ func onDeleteBackward(view C.CFTypeRef) { //export onText func onText(view C.CFTypeRef, str *C.char) { w := views[view] - w.w.Event(key.EditEvent{ - Text: C.GoString(str), - }) + w.w.EditorInsert(C.GoString(str)) } //export onTouch @@ -283,6 +281,8 @@ func (w *window) Configure([]Option) { } } +func (w *window) EditorStateChanged(old, new editorState) {} + func (w *window) Perform(system.Action) {} func (w *window) Raise() {} diff --git a/app/os_js.go b/app/os_js.go index 728ca14a..698efbc4 100644 --- a/app/os_js.go +++ b/app/os_js.go @@ -300,7 +300,7 @@ func (w *window) addHistory() { func (w *window) flushInput() { val := w.tarea.Get("value").String() w.tarea.Set("value", "") - w.w.Event(key.EditEvent{Text: string(val)}) + w.w.EditorInsert(string(val)) } func (w *window) blur() { @@ -481,6 +481,8 @@ func (w *window) animCallback() { } } +func (w *window) EditorStateChanged(old, new editorState) {} + func (w *window) SetAnimating(anim bool) { w.animating = anim if anim && !w.animRequested { diff --git a/app/os_macos.go b/app/os_macos.go index ec98c01c..e8b6ff23 100644 --- a/app/os_macos.go +++ b/app/os_macos.go @@ -350,6 +350,8 @@ func (w *window) SetCursor(name pointer.CursorName) { w.cursor = windowSetCursor(w.cursor, name) } +func (w *window) EditorStateChanged(old, new editorState) {} + func (w *window) ShowTextInput(show bool) {} func (w *window) SetInputHint(_ key.InputHint) {} @@ -412,7 +414,7 @@ func gio_onKeys(view C.CFTypeRef, cstr *C.char, ti C.double, mods C.NSUInteger, func gio_onText(view C.CFTypeRef, cstr *C.char) { str := C.GoString(cstr) w := mustView(view) - w.w.Event(key.EditEvent{Text: str}) + w.w.EditorInsert(str) } //export gio_onMouse diff --git a/app/os_wayland.go b/app/os_wayland.go index 420dae9d..988d1987 100644 --- a/app/os_wayland.go +++ b/app/os_wayland.go @@ -1202,7 +1202,12 @@ func gio_onKeyboardKey(data unsafe.Pointer, keyboard *C.struct_wl_keyboard, seri kc := mapXKBKeycode(uint32(keyCode)) ks := mapXKBKeyState(uint32(state)) for _, e := range w.disp.xkb.DispatchKey(kc, ks) { - w.w.Event(e) + if ee, ok := e.(key.EditEvent); ok { + // There's no support for IME yet. + w.w.EditorInsert(ee.Text) + } else { + w.w.Event(e) + } } if state != C.WL_KEYBOARD_KEY_STATE_PRESSED { return @@ -1642,6 +1647,8 @@ func (w *window) ShowTextInput(show bool) {} func (w *window) SetInputHint(_ key.InputHint) {} +func (w *window) EditorStateChanged(old, new editorState) {} + // Close the window. func (w *window) Close() { w.dead = true diff --git a/app/os_windows.go b/app/os_windows.go index f453c7d4..badb9950 100644 --- a/app/os_windows.go +++ b/app/os_windows.go @@ -243,7 +243,7 @@ func windowProc(hwnd syscall.Handle, msg uint32, wParam, lParam uintptr) uintptr fallthrough case windows.WM_CHAR: if r := rune(wParam); unicode.IsPrint(r) { - w.w.Event(key.EditEvent{Text: string(r)}) + w.w.EditorInsert(string(r)) } // The message is processed. return windows.TRUE @@ -451,6 +451,8 @@ loop: return nil } +func (w *window) EditorStateChanged(old, new editorState) {} + func (w *window) SetAnimating(anim bool) { w.animating = anim } diff --git a/app/os_x11.go b/app/os_x11.go index 0e26c795..636b66ce 100644 --- a/app/os_x11.go +++ b/app/os_x11.go @@ -323,6 +323,8 @@ func (w *x11Window) ShowTextInput(show bool) {} func (w *x11Window) SetInputHint(_ key.InputHint) {} +func (w *x11Window) EditorStateChanged(old, new editorState) {} + // Close the window. func (w *x11Window) Close() { var xev C.XEvent @@ -534,7 +536,12 @@ func (h *x11EventHandler) handleEvents() bool { } kevt := (*C.XKeyPressedEvent)(unsafe.Pointer(xev)) for _, e := range h.w.xkb.DispatchKey(uint32(kevt.keycode), ks) { - w.w.Event(e) + if ee, ok := e.(key.EditEvent); ok { + // There's no support for IME yet. + w.w.EditorInsert(ee.Text) + } else { + w.w.Event(e) + } } case C.ButtonPress, C.ButtonRelease: bevt := (*C.XButtonEvent)(unsafe.Pointer(xev)) diff --git a/app/window.go b/app/window.go index 497d54e8..23101a03 100644 --- a/app/window.go +++ b/app/window.go @@ -9,12 +9,14 @@ import ( "image/color" "runtime" "time" + "unicode/utf8" "gioui.org/f32" "gioui.org/font/gofont" "gioui.org/gpu" "gioui.org/internal/ops" "gioui.org/io/event" + "gioui.org/io/key" "gioui.org/io/pointer" "gioui.org/io/profile" "gioui.org/io/router" @@ -88,6 +90,13 @@ type Window struct { tree []router.SemanticNode ids map[router.SemanticID]router.SemanticNode } + + imeState editorState +} + +type editorState struct { + router.EditorState + compose key.Range } type callbacks struct { @@ -258,6 +267,13 @@ func (w *Window) processFrame(d driver, frameStart time.Time) { if q.ReadClipboard() { d.ReadClipboard() } + oldState := w.imeState + newState := oldState + newState.EditorState = q.EditorState() + if newState != oldState { + w.imeState = newState + d.EditorStateChanged(oldState, newState) + } if q.Profiling() && w.gpu != nil { frameDur := time.Since(frameStart) frameDur = frameDur.Truncate(100 * time.Microsecond) @@ -444,6 +460,95 @@ func (c *callbacks) SemanticAt(pos f32.Point) (router.SemanticID, bool) { return c.w.queue.q.SemanticAt(pos) } +func (c *callbacks) EditorState() editorState { + return c.w.imeState +} + +func (c *callbacks) SetComposingRegion(r key.Range) { + c.w.imeState.compose = r +} + +func (c *callbacks) EditorInsert(text string) { + sel := c.w.imeState.Selection + c.EditorReplace(sel, text) + start := sel.Start + if sel.End < start { + start = sel.End + } + sel.Start = start + utf8.RuneCountInString(text) + sel.End = sel.Start + c.SetEditorSelection(sel) +} + +func (c *callbacks) EditorReplace(r key.Range, text string) { + c.w.imeState.Replace(r, text) + c.Event(key.EditEvent{Range: r, Text: text}) + c.Event(key.SnippetEvent(c.w.imeState.Snippet.Range)) +} + +func (c *callbacks) SetEditorSelection(r key.Range) { + c.w.imeState.Selection = r + c.Event(key.SelectionEvent(r)) +} + +func (c *callbacks) SetEditorSnippet(r key.Range) { + if sn := c.EditorState().Snippet.Range; sn == r { + // No need to expand. + return + } + c.Event(key.SnippetEvent(r)) +} + +func (e *editorState) Replace(r key.Range, text string) { + if r.Start > r.End { + r.Start, r.End = r.End, r.Start + } + runes := []rune(text) + newEnd := r.Start + len(runes) + adjust := func(pos int) int { + switch { + case newEnd < pos && pos <= r.End: + return newEnd + case r.End < pos: + diff := newEnd - r.End + return pos + diff + } + return pos + } + e.Selection.Start = adjust(e.Selection.Start) + e.Selection.End = adjust(e.Selection.End) + e.compose.Start = adjust(e.compose.Start) + e.compose.End = adjust(e.compose.End) + s := e.Snippet + if r.End < s.Start || r.Start > s.End { + // Replacement does not overlap snippet. + e.Snippet.Start = adjust(s.Start) + e.Snippet.End = adjust(s.End) + return + } + // Replacement overlaps; replace content and expand snippet + // to include all of the replacement. + var newSnippet []rune + snippet := []rune(s.Text) + // Append first part of existing snippet. + if end := r.Start - s.Start; end > 0 { + newSnippet = append(newSnippet, snippet[:end]...) + } + // Append replacement. + newSnippet = append(newSnippet, runes...) + // Append last part of existing snippet. + if start := r.End; start < s.End { + newSnippet = append(newSnippet, snippet[start-s.Start:]...) + } + // Adjust snippet range to include replacement. + if r.Start < s.Start { + s.Start = r.Start + } + s.End = s.Start + len(newSnippet) + s.Text = string(newSnippet) + e.Snippet = s +} + func (w *Window) waitAck(d driver) { for { select { diff --git a/internal/ops/ops.go b/internal/ops/ops.go index 37f3b193..21a73eb3 100644 --- a/internal/ops/ops.go +++ b/internal/ops/ops.go @@ -73,6 +73,8 @@ const ( TypeSemanticClass TypeSemanticSelected TypeSemanticDisabled + TypeSnippet + TypeSelection ) type StackID struct { @@ -154,6 +156,8 @@ const ( TypeSemanticClassLen = 2 TypeSemanticSelectedLen = 2 TypeSemanticDisabledLen = 2 + TypeSnippetLen = 1 + 4 + 4 + TypeSelectionLen = 1 + 4 + 4 ) func (op *ClipOp) Decode(data []byte) { @@ -412,14 +416,16 @@ func (t OpType) Size() int { TypeSemanticClassLen, TypeSemanticSelectedLen, TypeSemanticDisabledLen, + TypeSnippetLen, + TypeSelectionLen, }[t-firstOpIndex] } func (t OpType) NumRefs() int { switch t { - case TypeKeyInput, TypeKeyFocus, TypePointerInput, TypeProfile, TypeCall, TypeClipboardRead, TypeClipboardWrite, TypeCursor, TypeSemanticLabel, TypeSemanticDesc: + case TypeKeyInput, TypeKeyFocus, TypePointerInput, TypeProfile, TypeCall, TypeClipboardRead, TypeClipboardWrite, TypeCursor, TypeSemanticLabel, TypeSemanticDesc, TypeSelection: return 1 - case TypeImage, TypeSource, TypeTarget: + case TypeImage, TypeSource, TypeTarget, TypeSnippet: return 2 case TypeOffer: return 3 diff --git a/io/key/key.go b/io/key/key.go index ef8f9111..c984265f 100644 --- a/io/key/key.go +++ b/io/key/key.go @@ -10,6 +10,7 @@ events. package key import ( + "encoding/binary" "fmt" "strings" @@ -40,6 +41,39 @@ type FocusOp struct { Tag event.Tag } +// SelectionOp updates the selection for an input handler. +type SelectionOp struct { + Tag event.Tag + Range +} + +// SnippetOp updates the content snippet for an input handler. +type SnippetOp struct { + Tag event.Tag + Snippet +} + +// Range represents a range of text, such as an editor's selection. +// Start and End are in runes. +type Range struct { + Start int + End int +} + +// Snippet represents a snippet of text content used for communicating between +// an editor and an input method. Offset and Length are in runes. +type Snippet struct { + Range + Text string +} + +// SelectionEvent is generated when an input method changes the selection. +type SelectionEvent Range + +// SnippetEvent is generated when the snippet range is updated by an +// input method. +type SnippetEvent Range + // A FocusEvent is generated when a handler gains or loses // focus. type FocusEvent struct { @@ -60,9 +94,11 @@ type Event struct { State State } -// An EditEvent is generated when text is input. +// An EditEvent requests an edit by an input method. type EditEvent struct { - Text string + // Range specifies the range to replace with Text. + Range Range + Text string } // InputHint changes the on-screen-keyboard type. That hints the @@ -179,9 +215,27 @@ func (h FocusOp) Add(o *op.Ops) { data[0] = byte(ops.TypeKeyFocus) } -func (EditEvent) ImplementsEvent() {} -func (Event) ImplementsEvent() {} -func (FocusEvent) ImplementsEvent() {} +func (s SnippetOp) Add(o *op.Ops) { + data := ops.Write2(&o.Internal, ops.TypeSnippetLen, s.Tag, &s.Text) + data[0] = byte(ops.TypeSnippet) + bo := binary.LittleEndian + bo.PutUint32(data[1:], uint32(s.Range.Start)) + bo.PutUint32(data[5:], uint32(s.Range.End)) +} + +func (s SelectionOp) Add(o *op.Ops) { + data := ops.Write1(&o.Internal, ops.TypeSelectionLen, s.Tag) + data[0] = byte(ops.TypeSelection) + bo := binary.LittleEndian + bo.PutUint32(data[1:], uint32(s.Start)) + bo.PutUint32(data[5:], uint32(s.End)) +} + +func (EditEvent) ImplementsEvent() {} +func (Event) ImplementsEvent() {} +func (FocusEvent) ImplementsEvent() {} +func (SnippetEvent) ImplementsEvent() {} +func (SelectionEvent) ImplementsEvent() {} func (e Event) String() string { return fmt.Sprintf("%v %v %v}", e.Name, e.Modifiers, e.State) diff --git a/io/router/key.go b/io/router/key.go index 9fef7dcd..6677361b 100644 --- a/io/router/key.go +++ b/io/router/key.go @@ -7,6 +7,12 @@ import ( "gioui.org/io/key" ) +// EditorState represents the state of an editor needed by input handlers. +type EditorState struct { + Selection key.Range + Snippet key.Snippet +} + type TextInputState uint8 type keyQueue struct { @@ -14,6 +20,7 @@ type keyQueue struct { handlers map[event.Tag]*keyHandler state TextInputState hint key.InputHint + content EditorState } type keyHandler struct { @@ -88,6 +95,7 @@ func (q *keyQueue) Frame(events *handlerEvents, collector keyCollector) { } } if collector.changed && collector.focus != q.focus { + q.content = EditorState{} if q.focus != nil { events.Add(q.focus, key.FocusEvent{Focus: false}) } @@ -119,16 +127,34 @@ func (k *keyCollector) softKeyboard(show bool) { } } -func (k *keyCollector) inputOp(op key.InputOp) { - h, ok := k.q.handlers[op.Tag] - if !ok { - h = &keyHandler{new: true} - k.q.handlers[op.Tag] = h +func (k *keyCollector) handlerFor(tag event.Tag) *keyHandler { + h, ok := k.q.handlers[tag] + if ok { + return h } + h = &keyHandler{new: true} + k.q.handlers[tag] = h + return h +} + +func (k *keyCollector) inputOp(op key.InputOp) { + h := k.handlerFor(op.Tag) h.visible = true h.hint = op.Hint } +func (k *keyCollector) selectionOp(op key.SelectionOp) { + if op.Tag == k.q.focus { + k.q.content.Selection = op.Range + } +} + +func (k *keyCollector) snippetOp(op key.SnippetOp) { + if op.Tag == k.q.focus { + k.q.content.Snippet = op.Snippet + } +} + func (t TextInputState) String() string { switch t { case TextInputKeep: diff --git a/io/router/router.go b/io/router/router.go index a02f0f6f..768402c1 100644 --- a/io/router/router.go +++ b/io/router/router.go @@ -138,7 +138,7 @@ func (q *Router) Queue(events ...event.Event) bool { q.profile = e case pointer.Event: q.pointer.queue.Push(e, &q.handlers) - case key.EditEvent, key.Event, key.FocusEvent: + case key.EditEvent, key.Event, key.FocusEvent, key.SnippetEvent, key.SelectionEvent: q.key.queue.Push(e, &q.handlers) case clipboard.Event: q.cqueue.Push(e, &q.handlers) @@ -188,6 +188,12 @@ func (q *Router) AppendSemantics(nodes []SemanticNode) []SemanticNode { return q.pointer.queue.AppendSemantics(nodes) } +// EditorState returns the editor state for the focused handler, or the +// zero value if there is none. +func (q *Router) EditorState() EditorState { + return q.key.queue.content +} + func (q *Router) collect() { q.transStack = q.transStack[:0] pc := &q.pointer.collector @@ -197,6 +203,7 @@ func (q *Router) collect() { *kc = keyCollector{q: &q.key.queue} q.key.queue.Reset() var t f32.Affine2D + bo := binary.LittleEndian for encOp, ok := q.reader.Decode(); ok; encOp, ok = q.reader.Decode() { switch ops.OpType(encOp.Data[0]) { case ops.TypeInvalidate: @@ -252,7 +259,6 @@ func (q *Router) collect() { case ops.TypePopPass: pc.popPass() case ops.TypePointerInput: - bo := binary.LittleEndian op := pointer.InputOp{ Tag: encOp.Refs[0].(event.Tag), Grab: encOp.Data[1] != 0, @@ -310,6 +316,27 @@ func (q *Router) collect() { Hint: key.InputHint(encOp.Data[1]), } kc.inputOp(op) + case ops.TypeSnippet: + op := key.SnippetOp{ + Tag: encOp.Refs[0].(event.Tag), + Snippet: key.Snippet{ + Range: key.Range{ + Start: int(int32(bo.Uint32(encOp.Data[1:]))), + End: int(int32(bo.Uint32(encOp.Data[5:]))), + }, + Text: *(encOp.Refs[1].(*string)), + }, + } + kc.snippetOp(op) + case ops.TypeSelection: + op := key.SelectionOp{ + Tag: encOp.Refs[0].(event.Tag), + Range: key.Range{ + Start: int(int32(bo.Uint32(encOp.Data[1:]))), + End: int(int32(bo.Uint32(encOp.Data[5:]))), + }, + } + kc.selectionOp(op) // Semantic ops. case ops.TypeSemanticLabel: diff --git a/widget/editor.go b/widget/editor.go index 725e44d3..f106a4d9 100644 --- a/widget/editor.go +++ b/widget/editor.go @@ -69,6 +69,12 @@ type Editor struct { // spaced intervals to speed up caret seeking. index []combinedPos + // ime tracks the state relevant to input methods. + ime struct { + imeState + scratch []byte + } + caret struct { on bool scroll bool @@ -96,6 +102,12 @@ type Editor struct { prevEvents int } +type imeState struct { + selection key.Range + snippet key.Snippet + start, end int +} + type maskReader struct { // rr is the underlying reader. rr io.RuneReader @@ -336,6 +348,8 @@ func (e *Editor) processKey(gtx layout.Context) { switch ke := ke.(type) { case key.FocusEvent: e.focused = ke.Focus + // Reset IME state. + e.ime.imeState = imeState{} case key.Event: if !e.focused || ke.State != key.Press { break @@ -352,15 +366,23 @@ func (e *Editor) processKey(gtx layout.Context) { e.caret.scroll = true e.scroller.Stop() } + case key.SnippetEvent: + e.updateSnippet(gtx, ke.Start, ke.End) case key.EditEvent: e.caret.scroll = true e.scroller.Stop() - e.append(ke.Text) + e.replace(ke.Range.Start, ke.Range.End, ke.Text) + e.caret.xoff = 0 // Complete a paste event, initiated by Shortcut-V in Editor.command(). case clipboard.Event: e.caret.scroll = true e.scroller.Stop() e.append(ke.Text) + case key.SelectionEvent: + e.caret.scroll = true + e.scroller.Stop() + e.caret.start = e.closestPosition(combinedPos{runes: ke.Start}).runes + e.caret.end = e.closestPosition(combinedPos{runes: ke.End}).runes } if e.rr.Changed() { e.events = append(e.events, ChangeEvent{}) @@ -504,6 +526,23 @@ func (e *Editor) Layout(gtx layout.Context, sh text.Shaper, font text.Font, size e.processEvents(gtx) e.makeValid() + if e.focused { + // Notify IME of selection if it changed. + newSel := key.Range{ + Start: e.caret.start, + End: e.caret.end, + } + if newSel != e.ime.selection { + e.ime.selection = newSel + key.SelectionOp{ + Tag: &e.eventKey, + Range: newSel, + }.Add(gtx.Ops) + } + + e.updateSnippet(gtx, e.ime.start, e.ime.end) + } + if viewSize := gtx.Constraints.Constrain(e.dims.Size); viewSize != e.viewSize { e.viewSize = viewSize e.invalidate() @@ -513,6 +552,47 @@ func (e *Editor) Layout(gtx layout.Context, sh text.Shaper, font text.Font, size return e.layout(gtx, content) } +// updateSnippet adds a key.SnippetOp if the snippet content or position +// have changed. off and len are in runes. +func (e *Editor) updateSnippet(gtx layout.Context, start, end int) { + e.makeValid() + if start > end { + start, end = end, start + } + imeStart := e.closestPosition(combinedPos{runes: start}) + imeEnd := e.closestPosition(combinedPos{runes: end}) + e.ime.start = imeStart.runes + e.ime.end = imeEnd.runes + e.rr.Seek(int64(imeStart.ofs), io.SeekStart) + n := imeEnd.ofs - imeStart.ofs + if n > len(e.ime.scratch) { + e.ime.scratch = make([]byte, n) + } + scratch := e.ime.scratch[:n] + read, _ := e.rr.Read(scratch) + if read != len(scratch) { + panic("e.rr.Read truncated data") + } + newSnip := key.Snippet{ + Range: key.Range{ + Start: e.ime.start, + End: e.ime.end, + }, + Text: e.ime.snippet.Text, + } + if string(scratch) != newSnip.Text { + newSnip.Text = string(scratch) + } + if newSnip == e.ime.snippet { + return + } + e.ime.snippet = newSnip + key.SnippetOp{ + Tag: &e.eventKey, + Snippet: newSnip, + }.Add(gtx.Ops) +} + func (e *Editor) layout(gtx layout.Context, content layout.Widget) layout.Dimensions { // Adjust scrolling for new viewport and layout. e.scrollRel(0, 0) @@ -910,7 +990,11 @@ func (e *Editor) Insert(s string) { func (e *Editor) append(s string) { e.replace(e.caret.start, e.caret.end, s) e.caret.xoff = 0 - e.caret.start += utf8.RuneCountInString(s) + start := e.caret.start + if end := e.caret.end; end < start { + start = end + } + e.caret.start = start + utf8.RuneCountInString(s) e.caret.end = e.caret.start } @@ -940,6 +1024,8 @@ func (e *Editor) replace(start, end int, s string) { } e.caret.start = adjust(e.caret.start) e.caret.end = adjust(e.caret.end) + e.ime.start = adjust(e.ime.start) + e.ime.end = adjust(e.ime.end) e.invalidate() }