gpu/internal/convertshaders,gpu: represent converted shaders with raw literals

Raw strings with linebreaks are easier to read and produce smaller
diffs.

Signed-off-by: Elias Naur <mail@eliasnaur.com>
This commit is contained in:
Elias Naur
2021-04-12 12:04:53 +02:00
parent 495c690187
commit fbee13a07d
13 changed files with 7086 additions and 474 deletions
-249
View File
@@ -1,249 +0,0 @@
// SPDX-License-Identifier: Unlicense OR MIT
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"os/exec"
"path/filepath"
"sort"
"strings"
"gioui.org/gpu/internal/driver"
)
// GLSLCC is a shader cross-compilation tool.
type GLSLCC struct {
Bin string
IncludeDir string
WorkDir WorkDir
}
func NewGLSLCC() *GLSLCC { return &GLSLCC{Bin: "glslcc"} }
// Metadata contains reflection data about the shader.
type Metadata struct {
Uniforms driver.UniformsReflection
Inputs []driver.InputLocation
Textures []driver.TextureBinding
StorageBuffers []driver.StorageBufferBinding
}
// Convert converts input data to the target shader.
func (glslcc *GLSLCC) Convert(path, variant string, input []byte, lang, profile string) (_ string, _ Metadata, err error) {
base := glslcc.WorkDir.Path(filepath.Base(path), variant, lang, profile)
pathin := base + ".in"
pathout := base + ".out"
reflectout := base + ".json"
if err := glslcc.WorkDir.WriteFile(pathin, input); err != nil {
return "", Metadata{}, fmt.Errorf("unable to write shader to disk: %w", err)
}
var progFlag, progSuffix string
switch filepath.Ext(path) {
case ".vert":
progFlag = "--vert"
progSuffix = "vs"
case ".frag":
progFlag = "--frag"
progSuffix = "fs"
case ".comp":
progFlag = "--compute"
progSuffix = "cs"
default:
return "", Metadata{}, fmt.Errorf("unrecognized shader type: %q", path)
}
cmd := exec.Command(glslcc.Bin,
"--silent",
"--optimize",
"--include-dirs", glslcc.IncludeDir,
"--reflect="+reflectout,
"--output", pathout,
"--lang", lang,
"--profile", profile,
progFlag, pathin,
)
if lang == "hlsl" {
cmd.Args = append(cmd.Args, "--defines=HLSL")
}
output, err := cmd.Output()
if err != nil {
return "", Metadata{}, fmt.Errorf("%s\nfailed to run %v: %w", output, cmd.Args, err)
}
// glslcc modifies the output path
p := strings.IndexByte(pathout, '.')
pathout = pathout[:p] + "_" + progSuffix + pathout[p:]
shader, err := ioutil.ReadFile(pathout)
if err != nil {
return "", Metadata{}, fmt.Errorf("missing shader output %q: %w", pathout, err)
}
reflectdata, err := ioutil.ReadFile(reflectout)
if err != nil {
return "", Metadata{}, fmt.Errorf("missing reflection output %q: %w", pathout, err)
}
metadata, err := glslcc.parseReflection(reflectdata)
if err != nil {
return "", Metadata{}, fmt.Errorf("invalid reflection output %q: %w", reflectout, err)
}
return string(shader), metadata, nil
}
// parseReflection parses glslcc -reflect output.
func (glslcc *GLSLCC) parseReflection(jsonData []byte) (Metadata, error) {
type (
Input struct {
ID int `json:"id"`
Name string `json:"name"`
Location int `json:"location"`
Semantic string `json:"semantic"`
SemanticIndex int `json:"semantic_index"`
Type string `json:"type"`
}
UniformMember struct {
Name string `json:"name"`
Type string `json:"type"`
Offset int `json:"offset"`
Size int `json:"size"`
}
UniformBuffer struct {
ID int `json:"id"`
Name string `json:"name"`
Set int `json:"set"`
Binding int `json:"binding"`
Size int `json:"block_size"`
Members []UniformMember `json:"members"`
}
Texture struct {
ID int `json:"id"`
Name string `json:"name"`
Set int `json:"set"`
Binding int `json:"binding"`
Dimension string `json:"dimension"`
Format string `json:"format"`
}
StorageBuffer struct {
ID int `json:"id"`
Name string `json:"name"`
Set int `json:"set"`
Binding int `json:"binding"`
BlockSize int `json:"block_size"`
}
Shader struct {
Inputs []Input `json:"inputs"`
UniformBuffers []UniformBuffer `json:"uniform_buffers"`
Textures []Texture `json:"textures"`
StorageBuffers []StorageBuffer `json:"storage_buffers"`
}
ReflectMetadata struct {
VS Shader `json:"vs"`
FS Shader `json:"fs"`
CS Shader `json:"cs"`
}
)
var info Metadata
var reflect ReflectMetadata
if err := json.Unmarshal(jsonData, &reflect); err != nil {
return info, fmt.Errorf("parseReflection: %v", err)
}
inputRef := reflect.VS.Inputs
for _, input := range inputRef {
dataType, dataSize, err := parseDataType(input.Type)
if err != nil {
return info, fmt.Errorf("parseReflection: %v", err)
}
info.Inputs = append(info.Inputs, driver.InputLocation{
Name: input.Name,
Location: input.Location,
Semantic: input.Semantic,
SemanticIndex: input.SemanticIndex,
Type: dataType,
Size: dataSize,
})
}
sort.Slice(info.Inputs, func(i, j int) bool {
return info.Inputs[i].Location < info.Inputs[j].Location
})
shaderBlocks := reflect.VS.UniformBuffers
if len(shaderBlocks) == 0 {
shaderBlocks = reflect.FS.UniformBuffers
}
blockOffset := 0
for _, block := range shaderBlocks {
info.Uniforms.Blocks = append(info.Uniforms.Blocks, driver.UniformBlock{
Name: block.Name,
Binding: block.Binding,
})
for _, member := range block.Members {
dataType, size, err := parseDataType(member.Type)
if err != nil {
return info, fmt.Errorf("parseReflection: %v", err)
}
info.Uniforms.Locations = append(info.Uniforms.Locations, driver.UniformLocation{
// Synthetic name generated by glslcc.
Name: fmt.Sprintf("_%d.%s", block.ID, member.Name),
Type: dataType,
Size: size,
Offset: blockOffset + member.Offset,
})
}
blockOffset += block.Size
}
info.Uniforms.Size = blockOffset
textures := reflect.VS.Textures
if len(textures) == 0 {
textures = reflect.FS.Textures
}
for _, texture := range textures {
info.Textures = append(info.Textures, driver.TextureBinding{
Name: texture.Name,
Binding: texture.Binding,
})
}
for _, sb := range reflect.CS.StorageBuffers {
info.StorageBuffers = append(info.StorageBuffers, driver.StorageBufferBinding{
Binding: sb.Binding,
BlockSize: sb.BlockSize,
})
}
return info, nil
}
func parseDataType(t string) (driver.DataType, int, error) {
switch t {
case "float":
return driver.DataTypeFloat, 1, nil
case "float2":
return driver.DataTypeFloat, 2, nil
case "float3":
return driver.DataTypeFloat, 3, nil
case "float4":
return driver.DataTypeFloat, 4, nil
case "int":
return driver.DataTypeInt, 1, nil
case "int2":
return driver.DataTypeInt, 2, nil
case "int3":
return driver.DataTypeInt, 3, nil
case "int4":
return driver.DataTypeInt, 4, nil
default:
return 0, 0, fmt.Errorf("unsupported input data type: %s", t)
}
}
+11 -7
View File
@@ -18,18 +18,22 @@ type GLSLValidator struct {
func NewGLSLValidator() *GLSLValidator { return &GLSLValidator{Bin: "glslangValidator"} }
// ConvertCompute converts glsl compute shader to spirv.
func (glsl *GLSLValidator) ConvertCompute(path string, input []byte) ([]byte, error) {
base := glsl.WorkDir.Path(filepath.Base(path))
// Convert converts a glsl shader to spirv.
func (glsl *GLSLValidator) Convert(path, variant string, hlsl bool, input []byte) ([]byte, error) {
base := glsl.WorkDir.Path(filepath.Base(path), variant)
pathout := base + ".out"
cmd := exec.Command(glsl.Bin,
"-G100", // OpenGL ES 3.1.
"-w", // Suppress warnings.
"-S", "comp",
"--stdin",
"-I"+filepath.Dir(path),
"-V", // OpenGL ES 3.1.
"-w", // Suppress warnings.
"-S", filepath.Ext(path)[1:],
"-o", pathout,
path,
)
if hlsl {
cmd.Args = append(cmd.Args, "-DHLSL")
}
cmd.Stdin = bytes.NewBuffer(input)
out, err := cmd.Output()
+41 -27
View File
@@ -78,7 +78,6 @@ type Converter struct {
packageName string
glslcc *GLSLCC
glslvalidator *GLSLValidator
spirv *SPIRVCross
fxc *FXC
@@ -97,22 +96,19 @@ func NewConverter(workDir WorkDir, packageName, shadersDir string, directCompute
conv.packageName = packageName
conv.glslcc = NewGLSLCC()
conv.glslvalidator = NewGLSLValidator()
conv.spirv = NewSPIRVCross()
conv.fxc = NewFXC()
conv.dxc = NewDXC()
verifyBinaryPath(&conv.glslcc.Bin)
verifyBinaryPath(&conv.glslvalidator.Bin)
verifyBinaryPath(&conv.spirv.Bin)
// we cannot check fxc/dxc, since they may depend on wine
conv.glslcc.IncludeDir = shadersDir
conv.glslcc.WorkDir = workDir.Dir("glslcc")
conv.glslvalidator.WorkDir = workDir.Dir("glslvalidator")
conv.fxc.WorkDir = workDir.Dir("fxc")
conv.dxc.WorkDir = workDir.Dir("dxc")
conv.spirv.WorkDir = workDir.Dir("spirv")
return conv
}
@@ -222,19 +218,19 @@ func (conv *Converter) Run(out io.Writer) error {
fmt.Fprintf(out, "Textures: %#v,\n", src.Textures)
}
if len(src.GLSL100ES) > 0 {
fmt.Fprintf(out, "GLSL100ES: %#v,\n", src.GLSL100ES)
fmt.Fprintf(out, "GLSL100ES: `%s`,\n", src.GLSL100ES)
}
if len(src.GLSL300ES) > 0 {
fmt.Fprintf(out, "GLSL300ES: %#v,\n", src.GLSL300ES)
fmt.Fprintf(out, "GLSL300ES: `%s`,\n", src.GLSL300ES)
}
if len(src.GLSL310ES) > 0 {
fmt.Fprintf(out, "GLSL310ES: %#v,\n", src.GLSL310ES)
fmt.Fprintf(out, "GLSL310ES: `%s`,\n", src.GLSL310ES)
}
if len(src.GLSL130) > 0 {
fmt.Fprintf(out, "GLSL130: %#v,\n", src.GLSL130)
fmt.Fprintf(out, "GLSL130: `%s`,\n", src.GLSL130)
}
if len(src.GLSL150) > 0 {
fmt.Fprintf(out, "GLSL150: %#v,\n", src.GLSL150)
fmt.Fprintf(out, "GLSL150: `%s`,\n", src.GLSL150)
}
if len(src.HLSL) > 0 {
fmt.Fprintf(out, "HLSL: %#v,\n", src.HLSL)
@@ -261,12 +257,12 @@ func (conv *Converter) Shader(shaderPath string) ([]driver.ShaderSources, error)
}
variantArgs := [...]Variant{
{
FetchColorExpr: `_color`,
Header: `layout(binding=0) uniform Color { vec4 _color; };`,
FetchColorExpr: `_color.color`,
Header: `layout(binding=0) uniform Color { vec4 color; } _color;`,
},
{
FetchColorExpr: `mix(_color1, _color2, clamp(vUV.x, 0.0, 1.0))`,
Header: `layout(binding=0) uniform Gradient { vec4 _color1; vec4 _color2; };`,
FetchColorExpr: `mix(_gradient.color1, _gradient.color2, clamp(vUV.x, 0.0, 1.0))`,
Header: `layout(binding=0) uniform Gradient { vec4 color1; vec4 color2; } _gradient;`,
},
{
FetchColorExpr: `texture(tex, vUV)`,
@@ -292,27 +288,26 @@ func (conv *Converter) Shader(shaderPath string) ([]driver.ShaderSources, error)
sources.Name = filepath.Base(shaderPath)
// Ignore error; some shaders are not meant to run in GLSL 1.00.
sources.GLSL100ES, _, _ = conv.glslcc.Convert(shaderPath, variantName, buf.Bytes(), "gles", "100")
sources.GLSL100ES, _, _ = conv.ShaderVariant(shaderPath, variantName, buf.Bytes(), "es", "100")
var metadata Metadata
sources.GLSL300ES, metadata, err = conv.glslcc.Convert(shaderPath, variantName, buf.Bytes(), "gles", "300")
sources.GLSL300ES, metadata, err = conv.ShaderVariant(shaderPath, variantName, buf.Bytes(), "es", "300")
if err != nil {
return nil, fmt.Errorf("failed to convert GLSL300ES:\n%w", err)
}
sources.GLSL130, _, err = conv.glslcc.Convert(shaderPath, variantName, buf.Bytes(), "glsl", "130")
sources.GLSL130, _, err = conv.ShaderVariant(shaderPath, variantName, buf.Bytes(), "glsl", "130")
if err != nil {
return nil, fmt.Errorf("failed to convert GLSL130:\n%w", err)
}
hlsl, _, err := conv.glslcc.Convert(shaderPath, variantName, buf.Bytes(), "hlsl", "40")
hlsl, _, err := conv.ShaderVariant(shaderPath, variantName, buf.Bytes(), "hlsl", "40")
if err != nil {
return nil, fmt.Errorf("failed to convert HLSL:\n%w", err)
}
sources.HLSL, err = conv.fxc.Compile(shaderPath, variantName, []byte(hlsl), "main", "4_0_level_9_1")
if err != nil {
// Attempt shader model 4.0. Only the app/headless
// Attempt shader model 4.0. Only the gpu/headless
// test shaders use features not supported by level
// 9.1.
sources.HLSL, err = conv.fxc.Compile(shaderPath, variantName, []byte(hlsl), "main", "4_0")
@@ -321,14 +316,14 @@ func (conv *Converter) Shader(shaderPath string) ([]driver.ShaderSources, error)
}
}
// OpenGL 3.2 Core only accepts GLSL version 1.50, but is
// otherwise compatible with version 1.30.
sources.GLSL150 = strings.Replace(sources.GLSL130, "#version 130", "#version 150", 1)
sources.GLSL150, _, err = conv.ShaderVariant(shaderPath, variantName, buf.Bytes(), "glsl", "150")
if err != nil {
return nil, fmt.Errorf("failed to convert GLSL150:\n%w", err)
}
sources.Uniforms = metadata.Uniforms
sources.Inputs = metadata.Inputs
sources.Textures = metadata.Textures
sources.StorageBuffers = metadata.StorageBuffers
variants = append(variants, sources)
}
@@ -341,13 +336,32 @@ func (conv *Converter) Shader(shaderPath string) ([]driver.ShaderSources, error)
return variants, nil
}
func (conv *Converter) ShaderVariant(shaderPath, variant string, src []byte, lang, profile string) (string, Metadata, error) {
spirv, err := conv.glslvalidator.Convert(shaderPath, variant, lang == "hlsl", src)
if err != nil {
return "", Metadata{}, fmt.Errorf("failed to generate SPIR-V for %q: %w", shaderPath, err)
}
dst, err := conv.spirv.Convert(shaderPath, variant, spirv, lang, profile)
if err != nil {
return "", Metadata{}, fmt.Errorf("failed to convert shader %q: %w", shaderPath, err)
}
meta, err := conv.spirv.Metadata(shaderPath, variant, spirv)
if err != nil {
return "", Metadata{}, fmt.Errorf("failed to extract metadata for shader %q: %w", shaderPath, err)
}
return dst, meta, nil
}
func (conv *Converter) ComputeShader(shaderPath string) ([]driver.ShaderSources, error) {
shader, err := ioutil.ReadFile(shaderPath)
if err != nil {
return nil, fmt.Errorf("failed to load shader %q: %w", shaderPath, err)
}
spirv, err := conv.glslvalidator.ConvertCompute(shaderPath, shader)
spirv, err := conv.glslvalidator.Convert(shaderPath, "", false, shader)
if err != nil {
return nil, fmt.Errorf("failed to convert compute shader %q: %w", shaderPath, err)
}
@@ -355,14 +369,14 @@ func (conv *Converter) ComputeShader(shaderPath string) ([]driver.ShaderSources,
var sources driver.ShaderSources
sources.Name = filepath.Base(shaderPath)
sources.GLSL310ES, err = conv.spirv.Convert(spirv, "es", "310")
sources.GLSL310ES, err = conv.spirv.Convert(shaderPath, "", spirv, "es", "310")
if err != nil {
return nil, fmt.Errorf("failed to convert es compute shader %q: %w", shaderPath, err)
}
sources.GLSL310ES = unixLineEnding(sources.GLSL310ES)
if conv.directCompute {
hlslSource, err := conv.spirv.Convert(spirv, "hlsl", "50")
hlslSource, err := conv.spirv.Convert(shaderPath, "", spirv, "hlsl", "50")
if err != nil {
return nil, fmt.Errorf("failed to convert hlsl compute shader %q: %w", shaderPath, err)
}
+176 -9
View File
@@ -3,43 +3,210 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"os/exec"
"path/filepath"
"sort"
"strings"
"gioui.org/gpu/internal/driver"
)
// Metadata contains reflection data about a shader.
type Metadata struct {
Uniforms driver.UniformsReflection
Inputs []driver.InputLocation
Textures []driver.TextureBinding
}
// SPIRVCross cross-compiles spirv shaders to es, hlsl and others.
type SPIRVCross struct {
Bin string
Bin string
WorkDir WorkDir
}
func NewSPIRVCross() *SPIRVCross { return &SPIRVCross{Bin: "spirv-cross"} }
// Convert converts compute shader from spirv format to a target format.
func (spirv *SPIRVCross) Convert(input []byte, target, version string) (string, error) {
func (spirv *SPIRVCross) Convert(path, variant string, shader []byte, target, version string) (string, error) {
base := spirv.WorkDir.Path(filepath.Base(path), variant)
if err := spirv.WorkDir.WriteFile(base, shader); err != nil {
return "", fmt.Errorf("unable to write shader to disk: %w", err)
}
var cmd *exec.Cmd
switch target {
case "glsl":
cmd = exec.Command(spirv.Bin,
"--no-es",
"--version", version,
)
case "es":
cmd = exec.Command(spirv.Bin,
"--es",
"--version", version,
"-",
)
case "hlsl":
cmd = exec.Command(spirv.Bin,
"--hlsl",
"--shader-model", version,
"-",
)
default:
return "", fmt.Errorf("unknown target %q", target)
}
cmd.Args = append(cmd.Args, base)
cmd.Stdin = bytes.NewBuffer(input)
out, err := cmd.Output()
out, err := cmd.CombinedOutput()
if err != nil {
return "", fmt.Errorf("failed to run %v: %w", cmd.Args, err)
return "", fmt.Errorf("%s\nfailed to run %v: %w", out, cmd.Args, err)
}
s := string(out)
if target != "hlsl" {
// Strip Windows \r in line endings.
s = unixLineEnding(s)
}
return string(out), nil
return s, nil
}
// Metadata extracts metadata for a SPIR-V shader.
func (spirv *SPIRVCross) Metadata(path, variant string, shader []byte) (Metadata, error) {
base := spirv.WorkDir.Path(filepath.Base(path), variant)
if err := spirv.WorkDir.WriteFile(base, shader); err != nil {
return Metadata{}, fmt.Errorf("unable to write shader to disk: %w", err)
}
cmd := exec.Command(spirv.Bin,
base,
"--reflect",
)
out, err := cmd.Output()
if err != nil {
return Metadata{}, fmt.Errorf("failed to run %v: %w", cmd.Args, err)
}
meta, err := parseMetadata(out)
if err != nil {
return Metadata{}, fmt.Errorf("%s\nfailed to parse metadata: %w", out, err)
}
return meta, nil
}
func parseMetadata(data []byte) (Metadata, error) {
var reflect struct {
Types map[string]struct {
Name string `json:"name"`
Members []struct {
Name string `json:"name"`
Type string `json:"type"`
Offset int `json:"offset"`
} `json:"members"`
} `json:"types"`
Inputs []struct {
Name string `json:"name"`
Type string `json:"type"`
Location int `json:"location"`
} `json:"inputs"`
Textures []struct {
Name string `json:"name"`
Type string `json:"type"`
Set int `json:"set"`
Binding int `json:"binding"`
} `json:"textures"`
UBOs []struct {
Name string `json:"name"`
Type string `json:"type"`
BlockSize int `json:"block_size"`
Set int `json:"set"`
Binding int `json:"binding"`
} `json:"ubos"`
}
if err := json.Unmarshal(data, &reflect); err != nil {
return Metadata{}, fmt.Errorf("failed to parse reflection data: %w", err)
}
var m Metadata
for _, input := range reflect.Inputs {
dataType, dataSize, err := parseDataType(input.Type)
if err != nil {
return Metadata{}, fmt.Errorf("parseReflection: %v", err)
}
m.Inputs = append(m.Inputs, driver.InputLocation{
Name: input.Name,
Location: input.Location,
Semantic: "TEXCOORD",
SemanticIndex: input.Location,
Type: dataType,
Size: dataSize,
})
}
sort.Slice(m.Inputs, func(i, j int) bool {
return m.Inputs[i].Location < m.Inputs[j].Location
})
blockOffset := 0
for _, block := range reflect.UBOs {
m.Uniforms.Blocks = append(m.Uniforms.Blocks, driver.UniformBlock{
Name: block.Name,
Binding: block.Binding,
})
t := reflect.Types[block.Type]
// By convention uniform block variables are named by prepending an underscore
// and converting to lowercase.
blockVar := "_" + strings.ToLower(block.Name)
for _, member := range t.Members {
dataType, size, err := parseDataType(member.Type)
if err != nil {
return Metadata{}, fmt.Errorf("failed to parse reflection data: %v", err)
}
m.Uniforms.Locations = append(m.Uniforms.Locations, driver.UniformLocation{
Name: fmt.Sprintf("%s.%s", blockVar, member.Name),
Type: dataType,
Size: size,
Offset: blockOffset + member.Offset,
})
}
blockOffset += block.BlockSize
}
m.Uniforms.Size = blockOffset
for _, texture := range reflect.Textures {
m.Textures = append(m.Textures, driver.TextureBinding{
Name: texture.Name,
Binding: texture.Binding,
})
}
//return m, fmt.Errorf("not yet!: %+v", reflect)
return m, nil
}
func parseDataType(t string) (driver.DataType, int, error) {
switch t {
case "float":
return driver.DataTypeFloat, 1, nil
case "vec2":
return driver.DataTypeFloat, 2, nil
case "vec3":
return driver.DataTypeFloat, 3, nil
case "vec4":
return driver.DataTypeFloat, 4, nil
case "int":
return driver.DataTypeInt, 1, nil
case "int2":
return driver.DataTypeInt, 2, nil
case "int3":
return driver.DataTypeInt, 3, nil
case "int4":
return driver.DataTypeInt, 4, nil
default:
return 0, 0, fmt.Errorf("unsupported input data type: %s", t)
}
}
+2 -2
View File
@@ -324,7 +324,7 @@ func (b *Backend) NewInputLayout(vertexShader driver.ShaderSources, layout []dri
case 4:
format = d3d11.DXGI_FORMAT_R32G32B32A32_FLOAT
default:
panic("unsupported float data size")
panic("unsupported data size")
}
case driver.DataTypeShort:
switch l.Size {
@@ -333,7 +333,7 @@ func (b *Backend) NewInputLayout(vertexShader driver.ShaderSources, layout []dri
case 2:
format = d3d11.DXGI_FORMAT_R16G16_SINT
default:
panic("unsupported float data size")
panic("unsupported data size")
}
default:
panic("unsupported data type")
+10 -16
View File
@@ -53,22 +53,16 @@ type Device interface {
}
type ShaderSources struct {
Name string
GLSL100ES string
GLSL300ES string
GLSL310ES string
GLSL130 string
GLSL150 string
HLSL []byte
Uniforms UniformsReflection
Inputs []InputLocation
Textures []TextureBinding
StorageBuffers []StorageBufferBinding
}
type StorageBufferBinding struct {
Binding int
BlockSize int
Name string
GLSL100ES string
GLSL300ES string
GLSL310ES string
GLSL130 string
GLSL150 string
HLSL []byte
Uniforms UniformsReflection
Inputs []InputLocation
Textures []TextureBinding
}
type UniformsReflection struct {