io/semantic: add package for adding semantic descriptions to UI components

Software such as screen readers require semantic descriptions of user
interfaces to effectively present and interact with them. Package
semantic, combined with the existing package clip provide the operations
for Gio programs to describe themselves.

This change implements the semantic package and the routing changes for
accessing semantic trees; follow-ups add semantic information to widgets
and implement mapping semantic tree to platform representations.

Signed-off-by: Elias Naur <mail@eliasnaur.com>

internal/ops/ops.go

@@ -24,7 +24,7 @@ type Ops struct {
 	nextStateID int
 
 	macroStack stack
-	stacks     [4]stack
+	stacks     [5]stack
 }
 
 type OpType byte
@@ -63,6 +63,11 @@ const (
 	TypeCursor
 	TypePath
 	TypeStroke
+	TypeSemanticLabel
+	TypeSemanticDesc
+	TypeSemanticClass
+	TypeSemanticSelected
+	TypeSemanticDisabled
 )
 
 type StackID struct {
@@ -98,6 +103,7 @@ const (
 	ClipStack StackKind = iota
 	TransStack
 	PassStack
+	MetaStack
 )
 
 const (
@@ -107,34 +113,39 @@ const (
 )
 
 const (
-	TypeMacroLen           = 1 + 4 + 4
-	TypeCallLen            = 1 + 4 + 4
-	TypeDeferLen           = 1
-	TypePushTransformLen   = 1 + 4*6
-	TypeTransformLen       = 1 + 1 + 4*6
-	TypePopTransformLen    = 1
-	TypeRedrawLen          = 1 + 8
-	TypeImageLen           = 1
-	TypePaintLen           = 1
-	TypeColorLen           = 1 + 4
-	TypeLinearGradientLen  = 1 + 8*2 + 4*2
-	TypePassLen            = 1
-	TypePopPassLen         = 1
-	TypePointerInputLen    = 1 + 1 + 1*2 + 2*4 + 2*4
-	TypeClipboardReadLen   = 1
-	TypeClipboardWriteLen  = 1
-	TypeKeyInputLen        = 1 + 1
-	TypeKeyFocusLen        = 1 + 1
-	TypeKeySoftKeyboardLen = 1 + 1
-	TypeSaveLen            = 1 + 4
-	TypeLoadLen            = 1 + 4
-	TypeAuxLen             = 1
-	TypeClipLen            = 1 + 4*4 + 1 + 1
-	TypePopClipLen         = 1
-	TypeProfileLen         = 1
-	TypeCursorLen          = 1 + 1
-	TypePathLen            = 8 + 1
-	TypeStrokeLen          = 1 + 4
+	TypeMacroLen            = 1 + 4 + 4
+	TypeCallLen             = 1 + 4 + 4
+	TypeDeferLen            = 1
+	TypePushTransformLen    = 1 + 4*6
+	TypeTransformLen        = 1 + 1 + 4*6
+	TypePopTransformLen     = 1
+	TypeRedrawLen           = 1 + 8
+	TypeImageLen            = 1
+	TypePaintLen            = 1
+	TypeColorLen            = 1 + 4
+	TypeLinearGradientLen   = 1 + 8*2 + 4*2
+	TypePassLen             = 1
+	TypePopPassLen          = 1
+	TypePointerInputLen     = 1 + 1 + 1*2 + 2*4 + 2*4
+	TypeClipboardReadLen    = 1
+	TypeClipboardWriteLen   = 1
+	TypeKeyInputLen         = 1 + 1
+	TypeKeyFocusLen         = 1 + 1
+	TypeKeySoftKeyboardLen  = 1 + 1
+	TypeSaveLen             = 1 + 4
+	TypeLoadLen             = 1 + 4
+	TypeAuxLen              = 1
+	TypeClipLen             = 1 + 4*4 + 1 + 1
+	TypePopClipLen          = 1
+	TypeProfileLen          = 1
+	TypeCursorLen           = 1 + 1
+	TypePathLen             = 8 + 1
+	TypeStrokeLen           = 1 + 4
+	TypeSemanticLabelLen    = 1
+	TypeSemanticDescLen     = 1
+	TypeSemanticClassLen    = 2
+	TypeSemanticSelectedLen = 2
+	TypeSemanticDisabledLen = 2
 )
 
 func (op *ClipOp) Decode(data []byte) {
@@ -354,12 +365,17 @@ func (t OpType) Size() int {
 		TypeCursorLen,
 		TypePathLen,
 		TypeStrokeLen,
+		TypeSemanticLabelLen,
+		TypeSemanticDescLen,
+		TypeSemanticClassLen,
+		TypeSemanticSelectedLen,
+		TypeSemanticDisabledLen,
 	}[t-firstOpIndex]
 }
 
 func (t OpType) NumRefs() int {
 	switch t {
-	case TypeKeyInput, TypeKeyFocus, TypePointerInput, TypeProfile, TypeCall, TypeClipboardRead, TypeClipboardWrite, TypeCursor:
+	case TypeKeyInput, TypeKeyFocus, TypePointerInput, TypeProfile, TypeCall, TypeClipboardRead, TypeClipboardWrite, TypeCursor, TypeSemanticLabel, TypeSemanticDesc:
 		return 1
 	case TypeImage:
 		return 2
@@ -426,6 +442,8 @@ func (t OpType) String() string {
 		return "Path"
 	case TypeStroke:
 		return "Stroke"
+	case TypeSemanticLabel:
+		return "SemanticDescription"
 	default:
 		panic("unknown OpType")
 	}

io/router/pointer.go

@@ -9,6 +9,7 @@ import (
 	"gioui.org/internal/ops"
 	"gioui.org/io/event"
 	"gioui.org/io/pointer"
+	"gioui.org/io/semantic"
 )
 
 type pointerQueue struct {
@@ -20,6 +21,15 @@ type pointerQueue struct {
 	pointers []pointerInfo
 
 	scratch []event.Tag
+
+	semantic struct {
+		idsAssigned bool
+		lastID      SemanticID
+		// contentIDs maps semantic content to a list of semantic IDs
+		// previously assigned. It is used to maintain stable IDs across
+		// frames.
+		contentIDs map[semanticContent][]semanticID
+	}
 }
 
 type hitNode struct {
@@ -64,8 +74,19 @@ type areaOp struct {
 
 type areaNode struct {
 	trans f32.Affine2D
-	next  int
 	area  areaOp
+
+	// Tree indices, with -1 being the sentinel.
+	parent     int
+	firstChild int
+	lastChild  int
+	sibling    int
+
+	semantic struct {
+		valid   bool
+		id      SemanticID
+		content semanticContent
+	}
 }
 
 type areaKind uint8
@@ -87,6 +108,21 @@ type pointerCollector struct {
 	nodeStack []int
 }
 
+type semanticContent struct {
+	tag      event.Tag
+	label    string
+	desc     string
+	class    semantic.ClassOp
+	gestures SemanticGestures
+	selected bool
+	disabled bool
+}
+
+type semanticID struct {
+	id   SemanticID
+	used bool
+}
+
 const (
 	areaRect areaKind = iota
 	areaEllipse
@@ -101,24 +137,42 @@ func (c *pointerCollector) setTrans(t f32.Affine2D) {
 }
 
 func (c *pointerCollector) clip(op ops.ClipOp) {
-	area := -1
-	if i := c.state.nodePlusOne - 1; i != -1 {
-		n := c.q.hitTree[i]
-		area = n.area
-	}
 	kind := areaRect
 	if op.Shape == ops.Ellipse {
 		kind = areaEllipse
 	}
-	areaOp := areaOp{kind: kind, rect: frect(op.Bounds)}
-	c.q.areas = append(c.q.areas, areaNode{trans: c.state.t, next: area, area: areaOp})
+	c.pushArea(kind, frect(op.Bounds))
+}
+
+func (c *pointerCollector) pushArea(kind areaKind, bounds f32.Rectangle) {
+	parentID := c.currentArea()
+	areaID := len(c.q.areas)
+	areaOp := areaOp{kind: kind, rect: bounds}
+	if parentID != -1 {
+		parent := &c.q.areas[parentID]
+		if parent.firstChild == -1 {
+			parent.firstChild = areaID
+		}
+		if siblingID := parent.lastChild; siblingID != -1 {
+			c.q.areas[siblingID].sibling = areaID
+		}
+		parent.lastChild = areaID
+	}
+	an := areaNode{
+		trans:      c.state.t,
+		area:       areaOp,
+		parent:     parentID,
+		sibling:    -1,
+		firstChild: -1,
+		lastChild:  -1,
+	}
+
+	c.q.areas = append(c.q.areas, an)
 	c.nodeStack = append(c.nodeStack, c.state.nodePlusOne-1)
-	c.q.hitTree = append(c.q.hitTree, hitNode{
-		next: c.state.nodePlusOne - 1,
-		area: len(c.q.areas) - 1,
+	c.addHitNode(hitNode{
+		area: areaID,
 		pass: true,
 	})
-	c.state.nodePlusOne = len(c.q.hitTree) - 1 + 1
 }
 
 // frect converts a rectangle to a f32.Rectangle.
@@ -149,19 +203,33 @@ func (c *pointerCollector) popPass() {
 	c.state.pass--
 }
 
-func (c *pointerCollector) inputOp(op pointer.InputOp, events *handlerEvents) {
-	area := -1
+func (c *pointerCollector) currentArea() int {
 	if i := c.state.nodePlusOne - 1; i != -1 {
 		n := c.q.hitTree[i]
-		area = n.area
+		return n.area
 	}
-	c.q.hitTree = append(c.q.hitTree, hitNode{
-		next: c.state.nodePlusOne - 1,
-		area: area,
+	return -1
+}
+
+func (c *pointerCollector) addHitNode(n hitNode) {
+	n.next = c.state.nodePlusOne - 1
+	c.q.hitTree = append(c.q.hitTree, n)
+	c.state.nodePlusOne = len(c.q.hitTree) - 1 + 1
+}
+
+func (c *pointerCollector) inputOp(op pointer.InputOp, events *handlerEvents) {
+	areaID := c.currentArea()
+	area := &c.q.areas[areaID]
+	area.semantic.content.tag = op.Tag
+	if op.Types&(pointer.Press|pointer.Release) != 0 {
+		area.semantic.content.gestures |= ClickGesture
+	}
+	area.semantic.valid = area.semantic.content.gestures != 0
+	c.addHitNode(hitNode{
+		area: areaID,
 		tag:  op.Tag,
 		pass: c.state.pass > 0,
 	})
-	c.state.nodePlusOne = len(c.q.hitTree) - 1 + 1
 	h, ok := c.q.handlers[op.Tag]
 	if !ok {
 		h = new(pointerHandler)
@@ -171,12 +239,47 @@ func (c *pointerCollector) inputOp(op pointer.InputOp, events *handlerEvents) {
 		events.AddNoRedraw(op.Tag, pointer.Event{Type: pointer.Cancel})
 	}
 	h.active = true
-	h.area = area
+	h.area = areaID
 	h.wantsGrab = h.wantsGrab || op.Grab
 	h.types = h.types | op.Types
 	h.scrollRange = op.ScrollBounds
 }
 
+func (c *pointerCollector) semanticLabel(lbl string) {
+	areaID := c.currentArea()
+	area := &c.q.areas[areaID]
+	area.semantic.valid = true
+	area.semantic.content.label = lbl
+}
+
+func (c *pointerCollector) semanticDesc(desc string) {
+	areaID := c.currentArea()
+	area := &c.q.areas[areaID]
+	area.semantic.valid = true
+	area.semantic.content.desc = desc
+}
+
+func (c *pointerCollector) semanticClass(class semantic.ClassOp) {
+	areaID := c.currentArea()
+	area := &c.q.areas[areaID]
+	area.semantic.valid = true
+	area.semantic.content.class = class
+}
+
+func (c *pointerCollector) semanticSelected(selected bool) {
+	areaID := c.currentArea()
+	area := &c.q.areas[areaID]
+	area.semantic.valid = true
+	area.semantic.content.selected = selected
+}
+
+func (c *pointerCollector) semanticDisabled(disabled bool) {
+	areaID := c.currentArea()
+	area := &c.q.areas[areaID]
+	area.semantic.valid = true
+	area.semantic.content.disabled = disabled
+}
+
 func (c *pointerCollector) cursor(name pointer.CursorName) {
 	c.q.cursors = append(c.q.cursors, cursorNode{
 		name: name,
@@ -186,9 +289,110 @@ func (c *pointerCollector) cursor(name pointer.CursorName) {
 
 func (c *pointerCollector) reset(q *pointerQueue) {
 	q.reset()
-	c.state = collectState{}
+	c.resetState()
 	c.nodeStack = c.nodeStack[:0]
 	c.q = q
+	// Add implicit root area for semantic descriptions to hang onto.
+	c.pushArea(areaRect, f32.Rect(-1e6, -1e6, 1e6, 1e6))
+	// Make it semantic to ensure a single semantic root.
+	c.q.areas[0].semantic.valid = true
+}
+
+func (q *pointerQueue) assignSemIDs() {
+	if q.semantic.idsAssigned {
+		return
+	}
+	q.semantic.idsAssigned = true
+	for i, a := range q.areas {
+		if a.semantic.valid {
+			q.areas[i].semantic.id = q.semanticIDFor(a.semantic.content)
+		}
+	}
+}
+
+func (q *pointerQueue) AppendSemantics(nodes []SemanticNode) []SemanticNode {
+	q.assignSemIDs()
+	nodes = q.appendSemanticChildren(nodes, 0)
+	nodes = q.appendSemanticArea(nodes, 0, 0)
+	return nodes
+}
+
+func (q *pointerQueue) appendSemanticArea(nodes []SemanticNode, parentID SemanticID, nodeIdx int) []SemanticNode {
+	areaIdx := nodes[nodeIdx].areaIdx
+	a := q.areas[areaIdx]
+	childStart := len(nodes)
+	nodes = q.appendSemanticChildren(nodes, a.firstChild)
+	childEnd := len(nodes)
+	for i := childStart; i < childEnd; i++ {
+		nodes = q.appendSemanticArea(nodes, a.semantic.id, i)
+	}
+	n := &nodes[nodeIdx]
+	n.ParentID = parentID
+	n.Children = nodes[childStart:childEnd]
+	return nodes
+}
+
+func (q *pointerQueue) appendSemanticChildren(nodes []SemanticNode, areaIdx int) []SemanticNode {
+	if areaIdx == -1 {
+		return nodes
+	}
+	a := q.areas[areaIdx]
+	if semID := a.semantic.id; semID != 0 {
+		cnt := a.semantic.content
+		nodes = append(nodes, SemanticNode{
+			ID: semID,
+			Desc: SemanticDesc{
+				Bounds: f32.Rectangle{
+					Min: a.trans.Transform(a.area.rect.Min),
+					Max: a.trans.Transform(a.area.rect.Max),
+				},
+				Label:       cnt.label,
+				Description: cnt.desc,
+				Class:       cnt.class,
+				Gestures:    cnt.gestures,
+				Selected:    cnt.selected,
+				Disabled:    cnt.disabled,
+			},
+			areaIdx: areaIdx,
+		})
+	} else {
+		nodes = q.appendSemanticChildren(nodes, a.firstChild)
+	}
+	return q.appendSemanticChildren(nodes, a.sibling)
+}
+
+func (q *pointerQueue) semanticIDFor(content semanticContent) SemanticID {
+	ids := q.semantic.contentIDs[content]
+	for i, id := range ids {
+		if !id.used {
+			ids[i].used = true
+			return id.id
+		}
+	}
+	// No prior assigned ID; allocate a new one.
+	q.semantic.lastID++
+	id := semanticID{id: q.semantic.lastID, used: true}
+	if q.semantic.contentIDs == nil {
+		q.semantic.contentIDs = make(map[semanticContent][]semanticID)
+	}
+	q.semantic.contentIDs[content] = append(q.semantic.contentIDs[content], id)
+	return id.id
+}
+
+func (q *pointerQueue) SemanticAt(pos f32.Point) (SemanticID, bool) {
+	q.assignSemIDs()
+	for i := len(q.hitTree) - 1; i >= 0; i-- {
+		n := &q.hitTree[i]
+		hit := q.hit(n.area, pos)
+		if !hit {
+			continue
+		}
+		area := q.areas[n.area]
+		if area.semantic.id != 0 {
+			return area.semantic.id, true
+		}
+	}
+	return 0, false
 }
 
 func (q *pointerQueue) opHit(handlers *[]event.Tag, pos f32.Point) {
@@ -230,7 +434,7 @@ func (q *pointerQueue) hit(areaIdx int, p f32.Point) bool {
 		if !a.area.Hit(p) {
 			return false
 		}
-		areaIdx = a.next
+		areaIdx = a.parent
 	}
 	return true
 }
@@ -248,6 +452,21 @@ func (q *pointerQueue) reset() {
 	q.hitTree = q.hitTree[:0]
 	q.areas = q.areas[:0]
 	q.cursors = q.cursors[:0]
+	q.semantic.idsAssigned = false
+	for k, ids := range q.semantic.contentIDs {
+		for i := len(ids) - 1; i >= 0; i-- {
+			if !ids[i].used {
+				ids = append(ids[:i], ids[i+1:]...)
+			} else {
+				ids[i].used = false
+			}
+		}
+		if len(ids) > 0 {
+			q.semantic.contentIDs[k] = ids
+		} else {
+			delete(q.semantic.contentIDs, k)
+		}
+	}
 }
 
 func (q *pointerQueue) Frame(events *handlerEvents) {

io/router/router.go

@@ -13,6 +13,7 @@ package router
 import (
 	"encoding/binary"
 	"image"
+	"strings"
 	"time"
 
 	"gioui.org/f32"
@@ -22,6 +23,7 @@ import (
 	"gioui.org/io/key"
 	"gioui.org/io/pointer"
 	"gioui.org/io/profile"
+	"gioui.org/io/semantic"
 	"gioui.org/op"
 )
 
@@ -53,6 +55,40 @@ type Router struct {
 	profile      profile.Event
 }
 
+// SemanticNode represents a node in the tree describing the components
+// contained in a frame.
+type SemanticNode struct {
+	ID       SemanticID
+	ParentID SemanticID
+	Children []SemanticNode
+	Desc     SemanticDesc
+
+	areaIdx int
+}
+
+// SemanticDesc provides a semantic description of a UI component.
+type SemanticDesc struct {
+	Class       semantic.ClassOp
+	Description string
+	Label       string
+	Selected    bool
+	Disabled    bool
+	Gestures    SemanticGestures
+	Bounds      f32.Rectangle
+}
+
+// SemanticGestures is a bit-set of supported gestures.
+type SemanticGestures int
+
+const (
+	ClickGesture SemanticGestures = 1 << iota
+)
+
+// SemanticID uniquely identifies a SemanticDescription.
+//
+// By convention, the zero value denotes the non-existent ID.
+type SemanticID uint64
+
 type handlerEvents struct {
 	handlers  map[event.Tag][]event.Event
 	hadEvents bool
@@ -137,6 +173,17 @@ func (q *Router) Cursor() pointer.CursorName {
 	return q.pointer.queue.cursor
 }
 
+// SemanticAt returns the first semantic description under pos, if any.
+func (q *Router) SemanticAt(pos f32.Point) (SemanticID, bool) {
+	return q.pointer.queue.SemanticAt(pos)
+}
+
+// AppendSemantics appends the semantic tree to nodes, and returns the result.
+// The root node is the first added.
+func (q *Router) AppendSemantics(nodes []SemanticNode) []SemanticNode {
+	return q.pointer.queue.AppendSemantics(nodes)
+}
+
 func (q *Router) collect() {
 	q.transStack = q.transStack[:0]
 	pc := &q.pointer.collector
@@ -175,17 +222,12 @@ func (q *Router) collect() {
 			pc.resetState()
 			pc.setTrans(t)
 
-		// Pointer ops.
 		case ops.TypeClip:
 			var op ops.ClipOp
 			op.Decode(encOp.Data)
 			pc.clip(op)
 		case ops.TypePopClip:
 			pc.popArea()
-		case ops.TypePass:
-			pc.pass()
-		case ops.TypePopPass:
-			pc.popPass()
 		case ops.TypeTransform:
 			t2, push := ops.DecodeTransform(encOp.Data)
 			if push {
@@ -198,6 +240,12 @@ func (q *Router) collect() {
 			t = q.transStack[n-1]
 			q.transStack = q.transStack[:n-1]
 			pc.setTrans(t)
+
+		// Pointer ops.
+		case ops.TypePass:
+			pc.pass()
+		case ops.TypePopPass:
+			pc.popPass()
 		case ops.TypePointerInput:
 			bo := binary.LittleEndian
 			op := pointer.InputOp{
@@ -238,6 +286,29 @@ func (q *Router) collect() {
 				Hint: key.InputHint(encOp.Data[1]),
 			}
 			kc.inputOp(op)
+
+		// Semantic ops.
+		case ops.TypeSemanticLabel:
+			lbl := encOp.Refs[0].(*string)
+			pc.semanticLabel(*lbl)
+		case ops.TypeSemanticDesc:
+			desc := encOp.Refs[0].(*string)
+			pc.semanticDesc(*desc)
+		case ops.TypeSemanticClass:
+			class := semantic.ClassOp(encOp.Data[1])
+			pc.semanticClass(class)
+		case ops.TypeSemanticSelected:
+			if encOp.Data[1] != 0 {
+				pc.semanticSelected(true)
+			} else {
+				pc.semanticSelected(false)
+			}
+		case ops.TypeSemanticDisabled:
+			if encOp.Data[1] != 0 {
+				pc.semanticDisabled(true)
+			} else {
+				pc.semanticDisabled(false)
+			}
 		}
 	}
 }
@@ -320,3 +391,11 @@ func decodeInvalidateOp(d []byte) op.InvalidateOp {
 	}
 	return o
 }
+
+func (s SemanticGestures) String() string {
+	var gestures []string
+	if s&ClickGesture != 0 {
+		gestures = append(gestures, "Click")
+	}
+	return strings.Join(gestures, ",")
+}

io/router/semantic_test.go

@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: Unlicense OR MIT
+
+package router
+
+import (
+	"fmt"
+	"image"
+	"reflect"
+	"testing"
+
+	"gioui.org/f32"
+	"gioui.org/io/pointer"
+	"gioui.org/io/semantic"
+	"gioui.org/op"
+	"gioui.org/op/clip"
+)
+
+func TestSemanticTree(t *testing.T) {
+	var (
+		ops op.Ops
+		r   Router
+	)
+	t1 := clip.Rect(image.Rect(0, 0, 75, 75)).Push(&ops)
+	semantic.DescriptionOp("child1").Add(&ops)
+	t1.Pop()
+	t2 := clip.Rect(image.Rect(25, 25, 100, 100)).Push(&ops)
+	semantic.DescriptionOp("child2").Add(&ops)
+	t2.Pop()
+	r.Frame(&ops)
+	tests := []struct {
+		x, y float32
+		desc string
+	}{
+		{24, 24, "child1"},
+		{50, 50, "child2"},
+		{100, 100, ""},
+	}
+	tree := r.AppendSemantics(nil)
+	verifyTree(t, 0, tree[0])
+	for _, test := range tests {
+		p := f32.Pt(test.x, test.y)
+		id, found := r.SemanticAt(p)
+		if !found {
+			t.Errorf("no semantic node at %v", p)
+		}
+		n, found := lookupNode(tree, id)
+		if !found {
+			t.Errorf("no id %d in semantic tree", id)
+		}
+		if got := n.Desc.Description; got != test.desc {
+			t.Errorf("got semantic description %s at %v, expected %s", got, p, test.desc)
+		}
+	}
+
+	// Verify stable IDs.
+	r.Frame(&ops)
+	tree2 := r.AppendSemantics(nil)
+	if !reflect.DeepEqual(tree, tree2) {
+		fmt.Println("First tree:")
+		printTree(0, tree[0])
+		fmt.Println("Second tree:")
+		printTree(0, tree2[0])
+		t.Error("same semantic description lead to differing trees")
+	}
+}
+
+func TestSemanticDescription(t *testing.T) {
+	var ops op.Ops
+	pointer.InputOp{Tag: new(int), Types: pointer.Press | pointer.Release}.Add(&ops)
+	semantic.DescriptionOp("description").Add(&ops)
+	semantic.LabelOp("label").Add(&ops)
+	semantic.Button.Add(&ops)
+	semantic.DisabledOp(true).Add(&ops)
+	semantic.SelectedOp(true).Add(&ops)
+	var r Router
+	r.Frame(&ops)
+	tree := r.AppendSemantics(nil)
+	got := tree[0].Desc
+	exp := SemanticDesc{
+		Class:       1,
+		Description: "description",
+		Label:       "label",
+		Selected:    true,
+		Disabled:    true,
+		Gestures:    ClickGesture,
+		Bounds:      f32.Rectangle{Min: f32.Point{X: -1e+06, Y: -1e+06}, Max: f32.Point{X: 1e+06, Y: 1e+06}},
+	}
+	if got != exp {
+		t.Errorf("semantic description mismatch:\nGot:  %+v\nWant: %+v", got, exp)
+	}
+}
+
+func lookupNode(tree []SemanticNode, id SemanticID) (SemanticNode, bool) {
+	for _, n := range tree {
+		if id == n.ID {
+			return n, true
+		}
+	}
+	return SemanticNode{}, false
+}
+
+func verifyTree(t *testing.T, parent SemanticID, n SemanticNode) {
+	t.Helper()
+	if n.ParentID != parent {
+		t.Errorf("node %d: got parent %d, want %d", n.ID, n.ParentID, parent)
+	}
+	for _, c := range n.Children {
+		verifyTree(t, n.ID, c)
+	}
+}
+
+func printTree(indent int, n SemanticNode) {
+	for i := 0; i < indent; i++ {
+		fmt.Print("\t")
+	}
+	fmt.Printf("%d: %+v\n", n.ID, n.Desc)
+	for _, c := range n.Children {
+		printTree(indent+1, c)
+	}
+}

io/semantic/semantic.go

@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: Unlicense OR MIT
+
+// Package semantic provides operations for semantic descriptions of a user
+// interface, to facilitate presentation and interaction in external software
+// such as screen readers.
+//
+// Semantic descriptions are organized in a tree, with clip operations as
+// nodes. Operations in this package are associated with the current semantic
+// node, that is the most recent pushed clip operation.
+package semantic
+
+import (
+	"gioui.org/internal/ops"
+	"gioui.org/op"
+)
+
+// LabelOp provides the content of a textual component.
+type LabelOp string
+
+// DescriptionOp describes a component.
+type DescriptionOp string
+
+// ClassOp provides the component class.
+type ClassOp int
+
+const (
+	Unknown ClassOp = iota
+	Button
+	CheckBox
+	Editor
+	RadioButton
+	Switch
+)
+
+// SelectedOp describes the selected state for components that have
+// boolean state.
+type SelectedOp bool
+
+// DisabledOp describes the disabled state.
+type DisabledOp bool
+
+func (l LabelOp) Add(o *op.Ops) {
+	s := string(l)
+	data := ops.Write1(&o.Internal, ops.TypeSemanticLabelLen, &s)
+	data[0] = byte(ops.TypeSemanticLabel)
+}
+
+func (d DescriptionOp) Add(o *op.Ops) {
+	s := string(d)
+	data := ops.Write1(&o.Internal, ops.TypeSemanticDescLen, &s)
+	data[0] = byte(ops.TypeSemanticDesc)
+}
+
+func (c ClassOp) Add(o *op.Ops) {
+	data := ops.Write(&o.Internal, ops.TypeSemanticClassLen)
+	data[0] = byte(ops.TypeSemanticClass)
+	data[1] = byte(c)
+}
+
+func (s SelectedOp) Add(o *op.Ops) {
+	data := ops.Write(&o.Internal, ops.TypeSemanticSelectedLen)
+	data[0] = byte(ops.TypeSemanticSelected)
+	if s {
+		data[1] = 1
+	}
+}
+
+func (d DisabledOp) Add(o *op.Ops) {
+	data := ops.Write(&o.Internal, ops.TypeSemanticDisabledLen)
+	data[0] = byte(ops.TypeSemanticDisabled)
+	if d {
+		data[1] = 1
+	}
+}
+
+func (c ClassOp) String() string {
+	switch c {
+	case Unknown:
+		return "Unknown"
+	case Button:
+		return "Button"
+	case CheckBox:
+		return "CheckBox"
+	case Editor:
+		return "Editor"
+	case RadioButton:
+		return "RadioButton"
+	case Switch:
+		return "Switch"
+	default:
+		panic("invalid ClassOp")
+	}
+}