gpu: implement automatic mipmaps for images

All GPU APIs except OpenGL ES 2 can generate mipmaps for textures.
This trades 33% more GPU memory use for improved rendering quality
and speed for downscaled images.

Signed-off-by: Elias Naur <mail@eliasnaur.com>

gpu/gpu.go

@@ -436,7 +436,7 @@ func (r *renderer) texHandle(cache *resourceCache, data imageOpData) driver.Text
 	if tex.tex != nil {
 		return tex.tex
 	}
-	handle, err := r.ctx.NewTexture(driver.TextureFormatSRGBA, data.src.Bounds().Dx(), data.src.Bounds().Dy(), driver.FilterLinear, driver.FilterLinear, driver.BufferBindingTexture)
+	handle, err := r.ctx.NewTexture(driver.TextureFormatSRGBA, data.src.Bounds().Dx(), data.src.Bounds().Dy(), driver.FilterLinearMipmapLinear, driver.FilterLinear, driver.BufferBindingTexture)
 	if err != nil {
 		panic(err)
 	}

gpu/internal/d3d11/d3d11_windows.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"image"
 	"math"
+	"math/bits"
 	"unsafe"
 
 	"golang.org/x/sys/windows"
@@ -58,6 +59,7 @@ type Texture struct {
 
 	width   int
 	height  int
+	mipmap  bool
 	foreign bool
 }
 
@@ -219,17 +221,33 @@ func (b *Backend) NewTexture(format driver.TextureFormat, width, height int, min
 	default:
 		return nil, fmt.Errorf("unsupported texture format %d", format)
 	}
+	bindFlags := convBufferBinding(bindings)
+	miscFlags := uint32(0)
+	mipmap := minFilter == driver.FilterLinearMipmapLinear
+	nmipmaps := 1
+	if mipmap {
+		// Flags required by ID3D11DeviceContext::GenerateMips.
+		bindFlags |= d3d11.BIND_SHADER_RESOURCE | d3d11.BIND_RENDER_TARGET
+		miscFlags |= d3d11.RESOURCE_MISC_GENERATE_MIPS
+		dim := width
+		if height > dim {
+			dim = height
+		}
+		log2 := 32 - bits.LeadingZeros32(uint32(dim)) - 1
+		nmipmaps = log2 + 1
+	}
 	tex, err := b.dev.CreateTexture2D(&d3d11.TEXTURE2D_DESC{
 		Width:     uint32(width),
 		Height:    uint32(height),
-		MipLevels: 1,
+		MipLevels: uint32(nmipmaps),
 		ArraySize: 1,
 		Format:    d3dfmt,
 		SampleDesc: d3d11.DXGI_SAMPLE_DESC{
 			Count:   1,
 			Quality: 0,
 		},
-		BindFlags: convBufferBinding(bindings),
+		BindFlags: bindFlags,
+		MiscFlags: miscFlags,
 	})
 	if err != nil {
 		return nil, err
@@ -247,6 +265,8 @@ func (b *Backend) NewTexture(format driver.TextureFormat, width, height int, min
 			filter = d3d11.FILTER_MIN_MAG_MIP_POINT
 		case minFilter == driver.FilterLinear && magFilter == driver.FilterLinear:
 			filter = d3d11.FILTER_MIN_MAG_LINEAR_MIP_POINT
+		case minFilter == driver.FilterLinearMipmapLinear && magFilter == driver.FilterLinear:
+			filter = d3d11.FILTER_MIN_MAG_MIP_LINEAR
 		default:
 			d3d11.IUnknownRelease(unsafe.Pointer(tex), tex.Vtbl.Release)
 			return nil, fmt.Errorf("unsupported texture filter combination %d, %d", minFilter, magFilter)
@@ -325,7 +345,7 @@ func (b *Backend) NewTexture(format driver.TextureFormat, width, height int, min
 			return nil, err
 		}
 	}
-	return &Texture{backend: b, format: d3dfmt, tex: tex, sampler: sampler, resView: resView, uaView: uaView, renderTarget: fbo, bindings: bindings, width: width, height: height}, nil
+	return &Texture{backend: b, format: d3dfmt, tex: tex, sampler: sampler, resView: resView, uaView: uaView, renderTarget: fbo, bindings: bindings, width: width, height: height, mipmap: mipmap}, nil
 }
 
 func (b *Backend) newInputLayout(vertexShader shader.Sources, layout []driver.InputDesc) (*d3d11.InputLayout, error) {
@@ -609,6 +629,9 @@ func (t *Texture) Upload(offset, size image.Point, pixels []byte, stride int) {
 	}
 	res := (*d3d11.Resource)(unsafe.Pointer(t.tex))
 	t.backend.ctx.UpdateSubresource(res, dst, uint32(stride), uint32(len(pixels)), pixels)
+	if t.mipmap {
+		t.backend.ctx.GenerateMips(t.resView)
+	}
 }
 
 func (t *Texture) Release() {

gpu/internal/driver/driver.go

@@ -166,6 +166,7 @@ const (
 const (
 	FilterNearest TextureFilter = iota
 	FilterLinear
+	FilterLinearMipmapLinear
 )
 
 const (

gpu/internal/metal/metal_darwin.go

@@ -239,6 +239,14 @@ static void blitEncCopyBufferToTexture(CFTypeRef blitEncRef, CFTypeRef bufRef, C
 	}
 }
 
+static void blitEncGenerateMipmapsForTexture(CFTypeRef blitEncRef, CFTypeRef texRef) {
+	@autoreleasepool {
+		id<MTLBlitCommandEncoder> enc = (__bridge id<MTLBlitCommandEncoder>)blitEncRef;
+		id<MTLTexture> tex = (__bridge id<MTLTexture>)texRef;
+		[enc generateMipmapsForTexture: tex];
+	}
+}
+
 static void blitEncCopyTextureToBuffer(CFTypeRef blitEncRef, CFTypeRef texRef, CFTypeRef bufRef, NSUInteger offset, NSUInteger stride, NSUInteger length, MTLSize dims, MTLOrigin orig) {
 	@autoreleasepool {
 		id<MTLBlitCommandEncoder> enc = (__bridge id<MTLBlitCommandEncoder>)blitEncRef;
@@ -269,25 +277,26 @@ static void blitEncCopyBufferToBuffer(CFTypeRef blitEncRef, CFTypeRef srcRef, CF
 	}
 }
 
-static CFTypeRef newTexture(CFTypeRef devRef, NSUInteger width, NSUInteger height, MTLPixelFormat format, MTLTextureUsage usage) {
+static CFTypeRef newTexture(CFTypeRef devRef, NSUInteger width, NSUInteger height, MTLPixelFormat format, MTLTextureUsage usage, int mipmapped) {
 	@autoreleasepool {
 		id<MTLDevice> dev = (__bridge id<MTLDevice>)devRef;
 		MTLTextureDescriptor *mtlDesc = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat: format
 																						   width: width
 																						  height: height
-																				   	   mipmapped: NO];
+																				   	   mipmapped: mipmapped ? YES : NO];
 		mtlDesc.usage = usage;
 		mtlDesc.storageMode =  MTLStorageModePrivate;
 		return CFBridgingRetain([dev newTextureWithDescriptor:mtlDesc]);
 	}
 }
 
-static CFTypeRef newSampler(CFTypeRef devRef, MTLSamplerMinMagFilter minFilter, MTLSamplerMinMagFilter magFilter) {
+static CFTypeRef newSampler(CFTypeRef devRef, MTLSamplerMinMagFilter minFilter, MTLSamplerMinMagFilter magFilter, MTLSamplerMipFilter mipFilter) {
 	@autoreleasepool {
 		id<MTLDevice> dev = (__bridge id<MTLDevice>)devRef;
 		MTLSamplerDescriptor *desc = [MTLSamplerDescriptor new];
 		desc.minFilter = minFilter;
 		desc.magFilter = magFilter;
+		desc.mipFilter = mipFilter;
 		return CFBridgingRetain([dev newSamplerStateWithDescriptor:desc]);
 	}
 }
@@ -405,6 +414,7 @@ type Texture struct {
 	sampler C.CFTypeRef
 	width   int
 	height  int
+	mipmap  bool
 	foreign bool
 }
 
@@ -581,26 +591,33 @@ func (b *Backend) NewTexture(format driver.TextureFormat, width, height int, min
 	if bindings&driver.BufferBindingShaderStorageWrite != 0 {
 		usage |= C.MTLTextureUsageShaderWrite
 	}
-	tex := C.newTexture(b.dev, C.NSUInteger(width), C.NSUInteger(height), mformat, usage)
+	min, mip := samplerFilterFor(minFilter)
+	max, _ := samplerFilterFor(magFilter)
+	mipmap := mip != C.MTLSamplerMipFilterNotMipmapped
+	mipmapped := C.int(0)
+	if mipmap {
+		mipmapped = 1
+	}
+	tex := C.newTexture(b.dev, C.NSUInteger(width), C.NSUInteger(height), mformat, usage, mipmapped)
 	if tex == 0 {
 		return nil, errors.New("metal: [MTLDevice newTextureWithDescriptor:] failed")
 	}
-	min := samplerFilterFor(minFilter)
-	max := samplerFilterFor(magFilter)
-	s := C.newSampler(b.dev, min, max)
+	s := C.newSampler(b.dev, min, max, mip)
 	if s == 0 {
 		C.CFRelease(tex)
 		return nil, errors.New("metal: [MTLDevice newSamplerStateWithDescriptor:] failed")
 	}
-	return &Texture{backend: b, texture: tex, sampler: s, width: width, height: height}, nil
+	return &Texture{backend: b, texture: tex, sampler: s, width: width, height: height, mipmap: mipmap}, nil
 }
 
-func samplerFilterFor(f driver.TextureFilter) C.MTLSamplerMinMagFilter {
+func samplerFilterFor(f driver.TextureFilter) (C.MTLSamplerMinMagFilter, C.MTLSamplerMipFilter) {
 	switch f {
 	case driver.FilterNearest:
-		return C.MTLSamplerMinMagFilterNearest
+		return C.MTLSamplerMinMagFilterNearest, C.MTLSamplerMipFilterNotMipmapped
 	case driver.FilterLinear:
-		return C.MTLSamplerMinMagFilterLinear
+		return C.MTLSamplerMinMagFilterLinear, C.MTLSamplerMipFilterNotMipmapped
+	case driver.FilterLinearMipmapLinear:
+		return C.MTLSamplerMinMagFilterLinear, C.MTLSamplerMipFilterLinear
 	default:
 		panic("invalid texture filter")
 	}
@@ -900,6 +917,9 @@ func (t *Texture) Upload(offset, size image.Point, pixels []byte, stride int) {
 		depth:  1,
 	}
 	C.blitEncCopyBufferToTexture(enc, buf, t.texture, C.NSUInteger(off), C.NSUInteger(dstStride), C.NSUInteger(len(store)), msize, orig)
+	if t.mipmap {
+		C.blitEncGenerateMipmapsForTexture(enc, t.texture)
+	}
 }
 
 func (t *Texture) Release() {

gpu/internal/opengl/opengl.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"image"
+	"math/bits"
 	"runtime"
 	"strings"
 	"time"
@@ -105,6 +106,7 @@ type texture struct {
 	triple   textureTriple
 	width    int
 	height   int
+	mipmap   bool
 	bindings driver.BufferBinding
 	foreign  bool
 }
@@ -695,13 +697,29 @@ func (b *Backend) NewTexture(format driver.TextureFormat, width, height int, min
 		return nil, errors.New("unsupported texture format")
 	}
 	b.BindTexture(0, tex)
-	b.funcs.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, toTexFilter(magFilter))
-	b.funcs.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, toTexFilter(minFilter))
+	min, mipmap := toTexFilter(minFilter)
+	mag, _ := toTexFilter(magFilter)
+	if b.gles && b.glver[0] < 3 {
+		// OpenGL ES 2 only supports mipmaps for power-of-two textures.
+		mipmap = false
+	}
+	tex.mipmap = mipmap
+	b.funcs.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, mag)
+	b.funcs.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, min)
 	b.funcs.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)
 	b.funcs.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)
-	if b.gles && b.glver[0] >= 3 {
+	if mipmap {
+		nmipmaps := 1
+		if mipmap {
+			dim := width
+			if height > dim {
+				dim = height
+			}
+			log2 := 32 - bits.LeadingZeros32(uint32(dim)) - 1
+			nmipmaps = log2 + 1
+		}
 		// Immutable textures are required for BindImageTexture, and can't hurt otherwise.
-		b.funcs.TexStorage2D(gl.TEXTURE_2D, 1, tex.triple.internalFormat, width, height)
+		b.funcs.TexStorage2D(gl.TEXTURE_2D, nmipmaps, tex.triple.internalFormat, width, height)
 	} else {
 		b.funcs.TexImage2D(gl.TEXTURE_2D, 0, tex.triple.internalFormat, width, height, tex.triple.format, tex.triple.typ)
 	}
@@ -1195,12 +1213,14 @@ func (p *pipeline) Release() {
 	*p = pipeline{}
 }
 
-func toTexFilter(f driver.TextureFilter) int {
+func toTexFilter(f driver.TextureFilter) (int, bool) {
 	switch f {
 	case driver.FilterNearest:
-		return gl.NEAREST
+		return gl.NEAREST, false
 	case driver.FilterLinear:
-		return gl.LINEAR
+		return gl.LINEAR, false
+	case driver.FilterLinearMipmapLinear:
+		return gl.LINEAR_MIPMAP_LINEAR, true
 	default:
 		panic("unsupported texture filter")
 	}
@@ -1234,6 +1254,9 @@ func (t *texture) Upload(offset, size image.Point, pixels []byte, stride int) {
 	}
 	t.backend.glstate.pixelStorei(t.backend.funcs, gl.UNPACK_ROW_LENGTH, rowLen)
 	t.backend.funcs.TexSubImage2D(gl.TEXTURE_2D, 0, offset.X, offset.Y, size.X, size.Y, t.triple.format, t.triple.typ, pixels)
+	if t.mipmap {
+		t.backend.funcs.GenerateMipmap(gl.TEXTURE_2D)
+	}
 }
 
 func (t *timer) Begin() {

gpu/internal/vulkan/vulkan.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"image"
+	"math/bits"
 
 	"gioui.org/gpu/internal/driver"
 	"gioui.org/internal/vk"
@@ -75,6 +76,7 @@ type Texture struct {
 	sampler    vk.Sampler
 	fbo        vk.Framebuffer
 	format     vk.Format
+	mipmaps    int
 	layout     vk.ImageLayout
 	passLayout vk.ImageLayout
 	width      int
@@ -155,7 +157,7 @@ func newVulkanDevice(api driver.Vulkan) (driver.Device, error) {
 	if props&reqs == reqs {
 		b.caps |= driver.FeatureFloatRenderTargets
 	}
-	reqs = vk.FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT | vk.FORMAT_FEATURE_SAMPLED_IMAGE_BIT
+	reqs = vk.FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT | vk.FORMAT_FEATURE_SAMPLED_IMAGE_BIT | vk.FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT
 	props = vk.GetPhysicalDeviceFormatProperties(b.physDev, vk.FORMAT_R8G8B8A8_SRGB)
 	if props&reqs == reqs {
 		b.caps |= driver.FeatureSRGB
@@ -292,19 +294,31 @@ func (b *Backend) NewTexture(format driver.TextureFormat, width, height int, min
 		usage |= vk.IMAGE_USAGE_STORAGE_BIT
 	}
 	filterFor := func(f driver.TextureFilter) vk.Filter {
-		switch minFilter {
-		case driver.FilterLinear:
+		switch f {
+		case driver.FilterLinear, driver.FilterLinearMipmapLinear:
 			return vk.FILTER_LINEAR
 		case driver.FilterNearest:
 			return vk.FILTER_NEAREST
 		}
 		panic("unknown filter")
 	}
-	sampler, err := vk.CreateSampler(b.dev, filterFor(minFilter), filterFor(magFilter))
+	mipmapMode := vk.SAMPLER_MIPMAP_MODE_NEAREST
+	mipmap := minFilter == driver.FilterLinearMipmapLinear
+	nmipmaps := 1
+	if mipmap {
+		mipmapMode = vk.SAMPLER_MIPMAP_MODE_LINEAR
+		dim := width
+		if height > dim {
+			dim = height
+		}
+		log2 := 32 - bits.LeadingZeros32(uint32(dim)) - 1
+		nmipmaps = log2 + 1
+	}
+	sampler, err := vk.CreateSampler(b.dev, filterFor(minFilter), filterFor(magFilter), mipmapMode)
 	if err != nil {
 		return nil, mapErr(err)
 	}
-	img, mem, err := vk.CreateImage(b.physDev, b.dev, vkfmt, width, height, usage)
+	img, mem, err := vk.CreateImage(b.physDev, b.dev, vkfmt, width, height, nmipmaps, usage)
 	if err != nil {
 		vk.DestroySampler(b.dev, sampler)
 		return nil, mapErr(err)
@@ -316,7 +330,7 @@ func (b *Backend) NewTexture(format driver.TextureFormat, width, height int, min
 		vk.FreeMemory(b.dev, mem)
 		return nil, mapErr(err)
 	}
-	t := &Texture{backend: b, img: img, mem: mem, view: view, sampler: sampler, layout: vk.IMAGE_LAYOUT_UNDEFINED, passLayout: passLayout, width: width, height: height, format: vkfmt}
+	t := &Texture{backend: b, img: img, mem: mem, view: view, sampler: sampler, layout: vk.IMAGE_LAYOUT_UNDEFINED, passLayout: passLayout, width: width, height: height, format: vkfmt, mipmaps: nmipmaps}
 	if bindings&driver.BufferBindingFramebuffer != 0 {
 		pass, err := vk.CreateRenderPass(b.dev, vkfmt, vk.ATTACHMENT_LOAD_OP_DONT_CARE,
 			vk.IMAGE_LAYOUT_UNDEFINED, vk.IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, nil)
@@ -646,6 +660,40 @@ func (t *Texture) Upload(offset, size image.Point, pixels []byte, stride int) {
 		vk.ACCESS_TRANSFER_WRITE_BIT,
 	)
 	vk.CmdCopyBufferToImage(cmdBuf, stage.buf, t.img, t.layout, op)
+	// Build mipmaps by repeating linear blits.
+	w, h := t.width, t.height
+	for i := 1; i < t.mipmaps; i++ {
+		nw, nh := w/2, h/2
+		if nh < 1 {
+			nh = 1
+		}
+		if nw < 1 {
+			nw = 1
+		}
+		// Transition previous (source) level.
+		b := vk.BuildImageMemoryBarrier(
+			t.img,
+			vk.ACCESS_TRANSFER_WRITE_BIT, vk.ACCESS_TRANSFER_READ_BIT,
+			vk.IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, vk.IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
+			i-1, 1,
+		)
+		vk.CmdPipelineBarrier(cmdBuf, vk.PIPELINE_STAGE_TRANSFER_BIT, vk.PIPELINE_STAGE_TRANSFER_BIT, vk.DEPENDENCY_BY_REGION_BIT, nil, nil, []vk.ImageMemoryBarrier{b})
+		// Blit to this mipmap level.
+		blit := vk.BuildImageBlit(0, 0, 0, 0, w, h, nw, nh, i-1, i)
+		vk.CmdBlitImage(cmdBuf, t.img, vk.IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, t.img, vk.IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, []vk.ImageBlit{blit}, vk.FILTER_LINEAR)
+		w, h = nw, nh
+	}
+	if t.mipmaps > 1 {
+		// Add barrier for last blit.
+		b := vk.BuildImageMemoryBarrier(
+			t.img,
+			vk.ACCESS_TRANSFER_WRITE_BIT, vk.ACCESS_TRANSFER_READ_BIT,
+			vk.IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, vk.IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
+			t.mipmaps-1, 1,
+		)
+		vk.CmdPipelineBarrier(cmdBuf, vk.PIPELINE_STAGE_TRANSFER_BIT, vk.PIPELINE_STAGE_TRANSFER_BIT, vk.DEPENDENCY_BY_REGION_BIT, nil, nil, []vk.ImageMemoryBarrier{b})
+		t.layout = vk.IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL
+	}
 }
 
 func (t *Texture) Release() {
@@ -756,6 +804,7 @@ func (t *Texture) imageBarrier(cmdBuf vk.CommandBuffer, layout vk.ImageLayout, s
 		t.img,
 		t.scope.access, access,
 		t.layout, layout,
+		0, vk.REMAINING_MIP_LEVELS,
 	)
 	vk.CmdPipelineBarrier(cmdBuf, srcStage, stage, vk.DEPENDENCY_BY_REGION_BIT, nil, nil, []vk.ImageMemoryBarrier{b})
 	t.layout = layout