joejulian/gio
1
0
Fork 1
mirror of https://git.sr.ht/~eliasnaur/gio synced 2026-07-04 08:55:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

app,widget,io: implement IME positioning

Browse Source 
This change implements reporting of the caret position from Editor, as well
as Windows, macOS, Android support. As a result, the IME composition window
on Windows and macOS is now positioned correctly.

References: https://todo.sr.ht/~eliasnaur/gio/246
Signed-off-by: Elias Naur <mail@eliasnaur.com>
This commit is contained in:
Elias Naur
2022-02-13 16:28:30 +01:00
parent cffa748c6a
commit 31f55232bf
13 changed files with 295 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/GioView.java
+34 -6
View File
@@ -14,6 +14,7 @@ import android.app.FragmentTransaction;
import android.content.Context;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Matrix;
import android.graphics.Rect;
import android.os.Build;
import android.os.Bundle;
@@ -37,14 +38,15 @@ import android.view.SurfaceHolder;
import android.view.Window;
import android.view.WindowInsetsController;
import android.view.WindowManager;
import android.view.inputmethod.InputConnection;
import android.view.inputmethod.InputMethodManager;
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.CorrectionInfo;
import android.view.inputmethod.CompletionInfo;
import android.view.inputmethod.InputContentInfo;
import android.view.inputmethod.CursorAnchorInfo;
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.ExtractedText;
import android.view.inputmethod.ExtractedTextRequest;
import android.view.inputmethod.InputConnection;
import android.view.inputmethod.InputMethodManager;
import android.view.inputmethod.InputContentInfo;
import android.view.inputmethod.SurroundingText;
import android.view.accessibility.AccessibilityNodeProvider;
import android.view.accessibility.AccessibilityNodeInfo;
@@ -441,6 +443,31 @@ public final class GioView extends SurfaceView {
imm.updateSelection(this, selStart, selEnd, compStart, compEnd);
}
void updateCaret(float m00, float m01, float m02, float m10, float m11, float m12, float caretX, float caretTop, float caretBase, float caretBottom) {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
return;
}
Matrix m = new Matrix();
m.setValues(new float[]{m00, m01, m02, m10, m11, m12, 0.0f, 0.0f, 1.0f});
m.setConcat(getMatrix(), m);
int selStart = imeSelectionStart(nhandle);
int selEnd = imeSelectionEnd(nhandle);
int compStart = imeComposingStart(nhandle);
int compEnd = imeComposingEnd(nhandle);
Snippet snip = getSnippet();
String composing = "";
if (compStart != -1) {
composing = snip.substringRunes(compStart, compEnd);
}
CursorAnchorInfo inf = new CursorAnchorInfo.Builder()
.setMatrix(m)
.setComposingText(imeToUTF16(nhandle, compStart), composing)
.setSelectionRange(imeToUTF16(nhandle, selStart), imeToUTF16(nhandle, selEnd))
.setInsertionMarkerLocation(caretX, caretTop, caretBase, caretBottom, 0)
.build();
imm.updateCursorAnchorInfo(this, inf);
}
static private native long onCreateView(GioView view);
static private native void onDestroyView(long handle);
static private native void onStartView(long handle);
@@ -628,7 +655,8 @@ public final class GioView extends SurfaceView {
}
/*@Override*/ public boolean requestCursorUpdates(int cursorUpdateMode) {
return false;
// We always provide cursor updates.
return true;
}
/*@Override*/ public void closeConnection() {
@@ -675,7 +703,7 @@ public final class GioView extends SurfaceView {
private static class Snippet {
String snippet;
// offset of snippet into the entire editor content. It is in runes because we won't require
// Gio editors to keep track of UTF-16 offsets. The disctinction won't matter in practice because IMEs only
// Gio editors to keep track of UTF-16 offsets. The distinction won't matter in practice because IMEs only
// ever see snippets.
int offset;
app/ime_test.go
+3 -1
View File
@@ -78,7 +78,7 @@ func FuzzIME(f *testing.F) {
if rng.End > len(runes) {
rng.End = len(runes)
}
state.Selection = rng
state.Selection.Range = rng
case cmdSnip:
r.Queue(key.SnippetEvent(rng))
runes := []rune(e.Text())
@@ -106,6 +106,8 @@ func FuzzIME(f *testing.F) {
e.Layout(gtx, cache, text.Font{}, unit.Px(10), nil)
r.Frame(gtx.Ops)
newState := r.EditorState()
// We don't track caret position.
state.Selection.Caret = newState.Selection.Caret
if newState != state.EditorState {
t.Errorf("IME state: %+v\neditor state: %+v", state.EditorState, newState)
}
app/internal/windows/windows.go
+73 -34
View File
@@ -15,6 +15,19 @@ import (
syscall "golang.org/x/sys/windows"
)
type CompositionForm struct {
dwStyle uint32
ptCurrentPos Point
rcArea Rect
}
type CandidateForm struct {
dwIndex uint32
dwStyle uint32
ptCurrentPos Point
rcArea Rect
}
type Rect struct {
Left, Top, Right, Bottom int32
}
@@ -95,6 +108,9 @@ const (
GCS_RESULTREADSTR = 0x0200
GCS_RESULTSTR = 0x0800
CFS_POINT = 0x0002
CFS_CANDIDATEPOS = 0x0040
HWND_TOPMOST = ^(uint32(1) - 1) // -1
HTCLIENT = 1
@@ -187,40 +203,41 @@ const (
UNICODE_NOCHAR = 65535
WM_CANCELMODE = 0x001F
WM_CHAR = 0x0102
WM_CLOSE = 0x0010
WM_CREATE = 0x0001
WM_DPICHANGED = 0x02E0
WM_DESTROY = 0x0002
WM_ERASEBKGND = 0x0014
WM_GETMINMAXINFO = 0x0024
WM_IME_COMPOSITION = 0x010F
WM_IME_ENDCOMPOSITION = 0x010E
WM_KEYDOWN = 0x0100
WM_KEYUP = 0x0101
WM_KILLFOCUS = 0x0008
WM_LBUTTONDOWN = 0x0201
WM_LBUTTONUP = 0x0202
WM_MBUTTONDOWN = 0x0207
WM_MBUTTONUP = 0x0208
WM_MOUSEMOVE = 0x0200
WM_MOUSEWHEEL = 0x020A
WM_MOUSEHWHEEL = 0x020E
WM_PAINT = 0x000F
WM_QUIT = 0x0012
WM_SETCURSOR = 0x0020
WM_SETFOCUS = 0x0007
WM_SHOWWINDOW = 0x0018
WM_SIZE = 0x0005
WM_SYSKEYDOWN = 0x0104
WM_SYSKEYUP = 0x0105
WM_RBUTTONDOWN = 0x0204
WM_RBUTTONUP = 0x0205
WM_TIMER = 0x0113
WM_UNICHAR = 0x0109
WM_USER = 0x0400
WM_WINDOWPOSCHANGED = 0x0047
WM_CANCELMODE = 0x001F
WM_CHAR = 0x0102
WM_CLOSE = 0x0010
WM_CREATE = 0x0001
WM_DPICHANGED = 0x02E0
WM_DESTROY = 0x0002
WM_ERASEBKGND = 0x0014
WM_GETMINMAXINFO = 0x0024
WM_IME_COMPOSITION = 0x010F
WM_IME_ENDCOMPOSITION = 0x010E
WM_IME_STARTCOMPOSITION = 0x010D
WM_KEYDOWN = 0x0100
WM_KEYUP = 0x0101
WM_KILLFOCUS = 0x0008
WM_LBUTTONDOWN = 0x0201
WM_LBUTTONUP = 0x0202
WM_MBUTTONDOWN = 0x0207
WM_MBUTTONUP = 0x0208
WM_MOUSEMOVE = 0x0200
WM_MOUSEWHEEL = 0x020A
WM_MOUSEHWHEEL = 0x020E
WM_PAINT = 0x000F
WM_QUIT = 0x0012
WM_SETCURSOR = 0x0020
WM_SETFOCUS = 0x0007
WM_SHOWWINDOW = 0x0018
WM_SIZE = 0x0005
WM_SYSKEYDOWN = 0x0104
WM_SYSKEYUP = 0x0105
WM_RBUTTONDOWN = 0x0204
WM_RBUTTONUP = 0x0205
WM_TIMER = 0x0113
WM_UNICHAR = 0x0109
WM_USER = 0x0400
WM_WINDOWPOSCHANGED = 0x0047
WS_CLIPCHILDREN = 0x00010000
WS_CLIPSIBLINGS = 0x04000000
@@ -338,6 +355,8 @@ var (
_ImmGetCompositionString = imm32.NewProc("ImmGetCompositionStringW")
_ImmNotifyIME = imm32.NewProc("ImmNotifyIME")
_ImmReleaseContext = imm32.NewProc("ImmReleaseContext")
_ImmSetCandidateWindow = imm32.NewProc("ImmSetCandidateWindow")
_ImmSetCompositionWindow = imm32.NewProc("ImmSetCompositionWindow")
)
func AdjustWindowRectEx(r *Rect, dwStyle uint32, bMenu int, dwExStyle uint32) {
@@ -536,6 +555,26 @@ func ImmGetCompositionValue(imc syscall.Handle, key int) int {
return int(int32(val))
}
func ImmSetCompositionWindow(imc syscall.Handle, x, y int) {
f := CompositionForm{
dwStyle: CFS_POINT,
ptCurrentPos: Point{
X: int32(x), Y: int32(y),
},
}
_ImmSetCompositionWindow.Call(uintptr(imc), uintptr(unsafe.Pointer(&f)))
}
func ImmSetCandidateWindow(imc syscall.Handle, x, y int) {
f := CandidateForm{
dwStyle: CFS_CANDIDATEPOS,
ptCurrentPos: Point{
X: int32(x), Y: int32(y),
},
}
_ImmSetCandidateWindow.Call(uintptr(imc), uintptr(unsafe.Pointer(&f)))
}
func SetWindowLong(hwnd syscall.Handle, idx uint32, style uintptr) {
if runtime.GOARCH == "386" {
_SetWindowLong32.Call(uintptr(hwnd), uintptr(idx), style)
app/os_android.go
+14 -2
View File
@@ -123,6 +123,7 @@ import (
"fmt"
"image"
"image/color"
"math"
"os"
"path/filepath"
"reflect"
@@ -190,6 +191,7 @@ var gioView struct {
isA11yActive C.jmethodID
restartInput C.jmethodID
updateSelection C.jmethodID
updateCaret C.jmethodID
}
// ViewEvent is sent whenever the Window's underlying Android view
@@ -475,6 +477,7 @@ func Java_org_gioui_GioView_onCreateView(env *C.JNIEnv, class C.jclass, view C.j
m.isA11yActive = getMethodID(env, class, "isA11yActive", "()Z")
m.restartInput = getMethodID(env, class, "restartInput", "()V")
m.updateSelection = getMethodID(env, class, "updateSelection", "()V")
m.updateCaret = getMethodID(env, class, "updateCaret", "(FFFFFFFFFF)V")
})
view = C.jni_NewGlobalRef(env, view)
wopts := <-mainWindow.out
@@ -1101,10 +1104,19 @@ func (w *window) EditorStateChanged(old, new editorState) {
callVoidMethod(env, w.view, gioView.restartInput)
return
}
if old.Selection != new.Selection {
if old.Selection.Range != new.Selection.Range {
w.callbacks.SetComposingRegion(key.Range{Start: -1, End: -1})
callVoidMethod(env, w.view, gioView.updateSelection)
}
if old.Selection.Transform != new.Selection.Transform || old.Selection.Caret != new.Selection.Caret {
sel := new.Selection
m00, m01, m02, m10, m11, m12 := sel.Transform.Elems()
f := func(v float32) jvalue {
return jvalue(math.Float32bits(v))
}
c := sel.Caret
callVoidMethod(env, w.view, gioView.updateCaret, f(m00), f(m01), f(m02), f(m10), f(m11), f(m12), f(c.Pos.X), f(c.Pos.Y-c.Ascent), f(c.Pos.Y), f(c.Pos.Y+c.Descent))
}
callVoidMethod(env, w.view, gioView.updateSelection)
})
}
app/os_macos.go
+48 -4
View File
@@ -172,6 +172,16 @@ static void discardMarkedText(CFTypeRef viewRef) {
}
}
}
static void invalidateCharacterCoordinates(CFTypeRef viewRef) {
@autoreleasepool {
id<NSTextInputClient> view = (__bridge id<NSTextInputClient>)viewRef;
NSTextInputContext *ctx = [NSTextInputContext currentInputContext];
if (view == [ctx client]) {
[ctx invalidateCharacterCoordinates];
}
}
}
*/
import "C"
@@ -363,8 +373,13 @@ func (w *window) SetCursor(name pointer.CursorName) {
}
func (w *window) EditorStateChanged(old, new editorState) {
C.discardMarkedText(w.view)
w.w.SetComposingRegion(key.Range{Start: -1, End: -1})
if old.Selection.Range != new.Selection.Range || old.Snippet != new.Snippet {
C.discardMarkedText(w.view)
w.w.SetComposingRegion(key.Range{Start: -1, End: -1})
}
if old.Selection.Caret != new.Selection.Caret || old.Selection.Transform != new.Selection.Transform {
C.invalidateCharacterCoordinates(w.view)
}
}
func (w *window) ShowTextInput(show bool) {}
@@ -546,7 +561,7 @@ func gio_setMarkedText(view, cstr C.CFTypeRef, selRange C.NSRange, replaceRange
state := w.w.EditorState()
rng := state.compose
if rng.Start == -1 {
rng = state.Selection
rng = state.Selection.Range
}
if replaceRange.location != C.NSNotFound {
// replaceRange is relative to marked (or selected) text.
@@ -606,7 +621,7 @@ func gio_insertText(view, cstr C.CFTypeRef, crng C.NSRange) {
state := w.w.EditorState()
rng := state.compose
if rng.Start == -1 {
rng = state.Selection
rng = state.Selection.Range
}
if crng.location != C.NSNotFound {
rng = key.Range{
@@ -621,6 +636,35 @@ func gio_insertText(view, cstr C.CFTypeRef, crng C.NSRange) {
w.w.SetEditorSelection(key.Range{Start: pos, End: pos})
}
//export gio_characterIndexForPoint
func gio_characterIndexForPoint(view C.CFTypeRef, p C.NSPoint) C.NSUInteger {
return C.NSNotFound
}
//export gio_firstRectForCharacterRange
func gio_firstRectForCharacterRange(view C.CFTypeRef, crng C.NSRange, actual C.NSRangePointer) C.NSRect {
w := mustView(view)
state := w.w.EditorState()
sel := state.Selection
u16start := state.UTF16Index(sel.Start)
actual.location = C.NSUInteger(u16start)
actual.length = 0
// Transform to NSView local coordinates (lower left origin, undo backing scale).
scale := 1. / float32(C.getViewBackingScale(w.view))
height := float32(C.viewHeight(w.view))
local := f32.Affine2D{}.Scale(f32.Pt(0, 0), f32.Pt(scale, -scale)).Offset(f32.Pt(0, height))
t := local.Mul(sel.Transform)
bounds := f32.Rectangle{
Min: t.Transform(sel.Pos.Sub(f32.Pt(0, sel.Ascent))),
Max: t.Transform(sel.Pos.Add(f32.Pt(0, sel.Descent))),
}.Canon()
sz := bounds.Size()
return C.NSMakeRect(
C.CGFloat(bounds.Min.X), C.CGFloat(bounds.Min.Y),
C.CGFloat(sz.X), C.CGFloat(sz.Y),
)
}
func (w *window) draw() {
w.scale = float32(C.getViewBackingScale(w.view))
wf, hf := float32(C.viewWidth(w.view)), float32(C.viewHeight(w.view))
app/os_macos.m
+7 -5
View File
@@ -176,12 +176,14 @@ static void handleMouse(NSView *view, NSEvent *event, int typ, CGFloat dx, CGFlo
}
gio_insertText((__bridge CFTypeRef)self, (__bridge CFTypeRef)str, replaceRange);
}
- (NSUInteger)characterIndexForPoint:(NSPoint)point {
return NSNotFound;
- (NSUInteger)characterIndexForPoint:(NSPoint)p {
return gio_characterIndexForPoint((__bridge CFTypeRef)self, p);
}
- (NSRect)firstRectForCharacterRange:(NSRange)range
actualRange:(NSRangePointer)actualRange {
return NSZeroRect;
- (NSRect)firstRectForCharacterRange:(NSRange)rng
actualRange:(NSRangePointer)actual {
NSRect r = gio_firstRectForCharacterRange((__bridge CFTypeRef)self, rng, actual);
r = [self convertRect:r toView:nil];
return [[self window] convertRectToScreen:r];
}
@end
app/os_windows.go
+15 -2
View File
@@ -349,6 +349,17 @@ func windowProc(hwnd syscall.Handle, msg uint32, wParam, lParam uintptr) uintptr
}
case _WM_WAKEUP:
w.w.Event(wakeupEvent{})
case windows.WM_IME_STARTCOMPOSITION:
imc := windows.ImmGetContext(w.hwnd)
if imc == 0 {
return windows.TRUE
}
defer windows.ImmReleaseContext(w.hwnd, imc)
sel := w.w.EditorState().Selection
caret := sel.Transform.Transform(sel.Caret.Pos.Add(f32.Pt(0, sel.Caret.Descent)))
icaret := image.Pt(int(caret.X+.5), int(caret.Y+.5))
windows.ImmSetCompositionWindow(imc, icaret.X, icaret.Y)
windows.ImmSetCandidateWindow(imc, icaret.X, icaret.Y)
case windows.WM_IME_COMPOSITION:
imc := windows.ImmGetContext(w.hwnd)
if imc == 0 {
@@ -358,7 +369,7 @@ func windowProc(hwnd syscall.Handle, msg uint32, wParam, lParam uintptr) uintptr
state := w.w.EditorState()
rng := state.compose
if rng.Start == -1 {
rng = state.Selection
rng = state.Selection.Range
}
if rng.Start > rng.End {
rng.Start, rng.End = rng.End, rng.Start
@@ -499,7 +510,9 @@ func (w *window) EditorStateChanged(old, new editorState) {
return
}
defer windows.ImmReleaseContext(w.hwnd, imc)
windows.ImmNotifyIME(imc, windows.NI_COMPOSITIONSTR, windows.CPS_CANCEL, 0)
if old.Selection.Range != new.Selection.Range || old.Snippet != new.Snippet {
windows.ImmNotifyIME(imc, windows.NI_COMPOSITIONSTR, windows.CPS_CANCEL, 0)
}
}
func (w *window) SetAnimating(anim bool) {
app/window.go
+2 -2
View File
@@ -474,7 +474,7 @@ func (c *callbacks) SetComposingRegion(r key.Range) {
}
func (c *callbacks) EditorInsert(text string) {
sel := c.w.imeState.Selection
sel := c.w.imeState.Selection.Range
c.EditorReplace(sel, text)
start := sel.Start
if sel.End < start {
@@ -492,7 +492,7 @@ func (c *callbacks) EditorReplace(r key.Range, text string) {
}
func (c *callbacks) SetEditorSelection(r key.Range) {
c.w.imeState.Selection = r
c.w.imeState.Selection.Range = r
c.Event(key.SelectionEvent(r))
}