Add Android accessibility-based Chrome form fill

android/application_snippets.xml

@@ -10,3 +10,15 @@
 				android:name="android.autofill"
 				android:resource="@xml/keepassgo_autofill_service" />
 		</service>
+		<service
+			android:name="org.julianfamily.keepassgo.KeePassGOAccessibilityService"
+			android:exported="true"
+			android:label="KeePassGO Accessibility Fill"
+			android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE">
+			<intent-filter>
+				<action android:name="android.accessibilityservice.AccessibilityService" />
+			</intent-filter>
+			<meta-data
+				android:name="android.accessibilityservice"
+				android:resource="@xml/keepassgo_accessibility_service" />
+		</service>

android/res/xml/keepassgo_accessibility_service.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
+	android:accessibilityEventTypes="typeViewFocused|typeViewTextChanged|typeWindowContentChanged|typeWindowStateChanged"
+	android:accessibilityFeedbackType="feedbackGeneric"
+	android:accessibilityFlags="flagReportViewIds|flagRetrieveInteractiveWindows"
+	android:canRetrieveWindowContent="true"
+	android:notificationTimeout="100"
+	android:settingsActivity="" />

androidsrc/org/julianfamily/keepassgo/AutofillCacheStore.java

@@ -57,17 +57,21 @@ final class AutofillCacheStore {
         if (filesDir != null) {
             candidates.add(new File(filesDir, "keepassgo/autofill-cache.json"));
             candidates.add(new File(filesDir, ".config/keepassgo/autofill-cache.json"));
+            candidates.add(new File(filesDir, "config/keepassgo/autofill-cache.json"));
         }
         File baseDir = context.getDataDir();
         if (baseDir != null) {
             candidates.add(new File(baseDir, "files/keepassgo/autofill-cache.json"));
             candidates.add(new File(baseDir, "files/.config/keepassgo/autofill-cache.json"));
+            candidates.add(new File(baseDir, "files/config/keepassgo/autofill-cache.json"));
         }
         for (File candidate : candidates) {
             if (candidate.isFile()) {
+                Log.i(TAG, "using autofill cache " + candidate.getAbsolutePath());
                 return candidate;
             }
         }
+        Log.i(TAG, "no autofill cache file in " + candidates);
         return null;
     }
 

androidsrc/org/julianfamily/keepassgo/KeePassGOAccessibilityService.java

@@ -0,0 +1,195 @@
+package org.julianfamily.keepassgo;
+
+import android.accessibilityservice.AccessibilityService;
+import android.os.Build;
+import android.os.Bundle;
+import android.util.Log;
+import android.view.accessibility.AccessibilityEvent;
+import android.view.accessibility.AccessibilityNodeInfo;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public final class KeePassGOAccessibilityService extends AccessibilityService {
+    private static final String TAG = "KeePassGOA11y";
+
+    private String lastFilledSignature = "";
+
+    @Override
+    public void onAccessibilityEvent(AccessibilityEvent event) {
+        try {
+            AccessibilityNodeInfo root = getRootInActiveWindow();
+            if (root == null) {
+                return;
+            }
+            CharSequence packageName = root.getPackageName();
+            if (packageName == null || !"com.android.chrome".contentEquals(packageName)) {
+                return;
+            }
+
+            ChromeForm form = inspectChrome(root);
+            if (form == null || form.passwordField == null) {
+                return;
+            }
+            AutofillCacheStore.Entry entry = AutofillCacheStore.findBestMatch(this, form.url);
+            if (entry == null) {
+                Log.i(TAG, "no accessibility-fill match for " + form.url);
+                return;
+            }
+
+            String signature = form.url + "|" + entry.username + "|" + nodeKey(form.passwordField);
+            if (signature.equals(lastFilledSignature)) {
+                return;
+            }
+
+            boolean filled = false;
+            if (form.usernameField != null) {
+                filled |= setNodeText(form.usernameField, entry.username);
+            }
+            filled |= setNodeText(form.passwordField, entry.password);
+            if (filled) {
+                lastFilledSignature = signature;
+                Log.i(TAG, "filled login form for " + form.url + " with " + entry.username);
+            }
+        } catch (Exception err) {
+            Log.e(TAG, "accessibility fill failed", err);
+        }
+    }
+
+    @Override
+    public void onInterrupt() {
+        Log.i(TAG, "accessibility service interrupted");
+    }
+
+    private static ChromeForm inspectChrome(AccessibilityNodeInfo root) {
+        List<AccessibilityNodeInfo> editables = new ArrayList<>();
+        ChromeForm form = new ChromeForm();
+        walk(root, editables, form);
+        if (form.url.isEmpty()) {
+            return null;
+        }
+        if (form.passwordField == null) {
+            for (AccessibilityNodeInfo node : editables) {
+                if (isPasswordNode(node)) {
+                    form.passwordField = node;
+                    break;
+                }
+            }
+        }
+        if (form.usernameField == null && form.passwordField != null) {
+            for (AccessibilityNodeInfo node : editables) {
+                if (node.equals(form.passwordField)) {
+                    continue;
+                }
+                if (isUsernameNode(node)) {
+                    form.usernameField = node;
+                    break;
+                }
+            }
+            if (form.usernameField == null) {
+                for (AccessibilityNodeInfo node : editables) {
+                    if (node.equals(form.passwordField)) {
+                        continue;
+                    }
+                    form.usernameField = node;
+                    break;
+                }
+            }
+        }
+        return form;
+    }
+
+    private static void walk(
+            AccessibilityNodeInfo node,
+            List<AccessibilityNodeInfo> editables,
+            ChromeForm form
+    ) {
+        if (node == null) {
+            return;
+        }
+        CharSequence viewID = node.getViewIdResourceName();
+        if (viewID != null && viewID.toString().endsWith(":id/url_bar")) {
+            CharSequence text = node.getText();
+            if (text != null) {
+                form.url = text.toString().trim();
+            }
+        }
+        if (node.isEditable()) {
+            editables.add(node);
+            if (form.passwordField == null && isPasswordNode(node)) {
+                form.passwordField = node;
+            } else if (form.usernameField == null && isUsernameNode(node)) {
+                form.usernameField = node;
+            }
+        }
+        for (int i = 0; i < node.getChildCount(); i++) {
+            AccessibilityNodeInfo child = node.getChild(i);
+            if (child != null) {
+                walk(child, editables, form);
+            }
+        }
+    }
+
+    private static boolean isPasswordNode(AccessibilityNodeInfo node) {
+        if (node.isPassword()) {
+            return true;
+        }
+        return nodeMatches(node, "password");
+    }
+
+    private static boolean isUsernameNode(AccessibilityNodeInfo node) {
+        if (isPasswordNode(node)) {
+            return false;
+        }
+        return nodeMatches(node, "username")
+                || nodeMatches(node, "email")
+                || nodeMatches(node, "login")
+                || nodeMatches(node, "user");
+    }
+
+    private static boolean nodeMatches(AccessibilityNodeInfo node, String term) {
+        String lowerTerm = term.toLowerCase();
+        return containsLower(node.getHintText(), lowerTerm)
+                || containsLower(node.getText(), lowerTerm)
+                || containsLower(node.getContentDescription(), lowerTerm)
+                || containsLower(node.getViewIdResourceName(), lowerTerm)
+                || containsLower(node.getPaneTitle(), lowerTerm);
+    }
+
+    private static boolean containsLower(CharSequence value, String term) {
+        return value != null && value.toString().toLowerCase().contains(term);
+    }
+
+    private static boolean setNodeText(AccessibilityNodeInfo node, String value) {
+        if (node == null || !node.isEditable() || value == null) {
+            return false;
+        }
+        Bundle args = new Bundle();
+        args.putCharSequence(
+                AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
+                value
+        );
+        return node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args);
+    }
+
+    private static String nodeKey(AccessibilityNodeInfo node) {
+        CharSequence viewID = node.getViewIdResourceName();
+        if (viewID != null) {
+            return viewID.toString();
+        }
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+            return String.valueOf(node.getUniqueId());
+        }
+        CharSequence hint = node.getHintText();
+        if (hint != null) {
+            return hint.toString();
+        }
+        return String.valueOf(node.hashCode());
+    }
+
+    private static final class ChromeForm {
+        String url = "";
+        AccessibilityNodeInfo usernameField;
+        AccessibilityNodeInfo passwordField;
+    }
+}

androidsrc/org/julianfamily/keepassgo/KeePassGOAutofillService.java

@@ -22,6 +22,18 @@ import java.util.List;
 public final class KeePassGOAutofillService extends AutofillService {
     private static final String TAG = "KeePassGOAutofill";
 
+    @Override
+    public void onConnected() {
+        super.onConnected();
+        Log.i(TAG, "service connected");
+    }
+
+    @Override
+    public void onDisconnected() {
+        Log.i(TAG, "service disconnected");
+        super.onDisconnected();
+    }
+
     @Override
     public void onFillRequest(
             FillRequest request,
@@ -29,8 +41,10 @@ public final class KeePassGOAutofillService extends AutofillService {
             FillCallback callback
     ) {
         try {
+            Log.i(TAG, "fill request flags=" + request.getFlags());
             List<FillContext> contexts = request.getFillContexts();
             if (contexts.isEmpty()) {
+                Log.i(TAG, "fill request had no contexts");
                 callback.onSuccess(null);
                 return;
             }
@@ -38,16 +52,20 @@ public final class KeePassGOAutofillService extends AutofillService {
             AssistStructure structure = contexts.get(contexts.size() - 1).getStructure();
             ParsedFields fields = new ParsedFields();
             String webDomain = parseWindow(structure, fields);
+            Log.i(TAG, "parsed domain=" + webDomain + " usernameId=" + fields.usernameId + " passwordId=" + fields.passwordId);
             if (fields.passwordId == null) {
+                Log.i(TAG, "no password field found");
                 callback.onSuccess(null);
                 return;
             }
 
             AutofillCacheStore.Entry entry = AutofillCacheStore.findBestMatch(this, webDomain);
             if (entry == null) {
+                Log.i(TAG, "no autofill cache match");
                 callback.onSuccess(null);
                 return;
             }
+            Log.i(TAG, "matched entry title=" + entry.title + " user=" + entry.username + " host=" + entry.host);
 
             RemoteViews presentation = new RemoteViews(getPackageName(), android.R.layout.simple_list_item_1);
             presentation.setTextViewText(
@@ -64,6 +82,7 @@ public final class KeePassGOAutofillService extends AutofillService {
             FillResponse response = new FillResponse.Builder()
                     .addDataset(dataset.build())
                     .build();
+            Log.i(TAG, "returning dataset");
             callback.onSuccess(response);
         } catch (Exception err) {
             Log.e(TAG, "fill request failed", err);
@@ -73,6 +92,7 @@ public final class KeePassGOAutofillService extends AutofillService {
 
     @Override
     public void onSaveRequest(SaveRequest request, SaveCallback callback) {
+        Log.i(TAG, "save request");
         callback.onSuccess();
     }