// SPDX-License-Identifier: Unlicense OR MIT package main import ( "bytes" "encoding/json" "flag" "fmt" "io" "io/ioutil" "os" "os/exec" "path/filepath" "runtime" "sort" "strings" "text/template" "gioui.org/gpu/backend" ) // This program generates shader variants for // multiple GPU backends (OpenGL ES, Direct3D 11...) // from a single source. var ( packageName = flag.String("package", "", "specify Go package name") shadersDir = flag.String("dir", "shaders", "specify shader directory") directCompute = flag.Bool("directcompute", false, "enable compiling DirectCompute shaders") absShadersDir string ) type shaderArgs struct { FetchColorExpr string Header string } func main() { flag.Parse() if err := generate(); err != nil { fmt.Fprintf(os.Stderr, "generate: %v\n", err) os.Exit(1) } } func generate() error { tmp, err := ioutil.TempDir("", "shader-convert") if err != nil { return err } defer os.RemoveAll(tmp) glslcc, err := exec.LookPath("glslcc") if err != nil { return err } absShadersDir, err = filepath.Abs(*shadersDir) if err != nil { return err } shaders, err := filepath.Glob(filepath.Join(absShadersDir, "*")) if err != nil { return err } var out bytes.Buffer out.WriteString("// Code generated by build.go. DO NOT EDIT.\n\n") fmt.Fprintf(&out, "package %s\n\n", *packageName) fmt.Fprintf(&out, "import %q\n\n", "gioui.org/gpu/backend") out.WriteString("var (\n") for _, shader := range shaders { var err error switch filepath.Ext(shader) { case ".vert", ".frag": err = generateShader(glslcc, tmp, &out, shader) case ".comp": err = generateComputeShader(tmp, &out, shader) default: continue } if err != nil { return err } } out.WriteString(")") if err := ioutil.WriteFile("shaders.go", out.Bytes(), 0644); err != nil { return err } cmd := exec.Command("gofmt", "-s", "-w", "shaders.go") cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() } func generateComputeShader(tmp string, out io.Writer, shader string) error { glslangValidator, err := exec.LookPath("glslangValidator") if err != nil { return err } spirv_cross, err := exec.LookPath("spirv-cross") if err != nil { return err } shaders, err := convertComputeShader(tmp, glslangValidator, spirv_cross, shader) if err != nil { return err } filename := filepath.Base(shader) varname := strings.ReplaceAll(filename, ".", "_") fmt.Fprintf(out, "\tshader_%s = ", varname) fmt.Fprintf(out, "backend.ShaderSources{\n") fmt.Fprintf(out, "Name: %#v,\n", filename) fmt.Fprintf(out, "GLSL310ES: %#v,\n", shaders.glsl310es) fmt.Fprintf(out, "HLSL: %#v,\n", shaders.hlsl) fmt.Fprintf(out, "}") fmt.Fprintf(out, "\n") return nil } func generateShader(glslcc, tmp string, out io.Writer, shader string) error { const nvariants = 3 var variants [nvariants]backend.ShaderSources args := [nvariants]shaderArgs{ { FetchColorExpr: `_color`, Header: `layout(binding=0) uniform Color { vec4 _color; };`, }, { FetchColorExpr: `mix(_color1, _color2, clamp(vUV.x, 0.0, 1.0))`, Header: `layout(binding=0) uniform Gradient { vec4 _color1; vec4 _color2; };`, }, { FetchColorExpr: `texture(tex, vUV)`, Header: `layout(binding=0) uniform sampler2D tex;`, }, } for i := range args { // Ignore error; some shaders are not meant to run in GLSL 1.00. glsl100es, _, _ := convertShader(tmp, glslcc, shader, "gles", "100", &args[i], false) glsl300es, reflect, err := convertShader(tmp, glslcc, shader, "gles", "300", &args[i], false) if err != nil { return err } glsl130, _, err := convertShader(tmp, glslcc, shader, "glsl", "130", &args[i], false) if err != nil { return err } hlsl, _, err := convertShader(tmp, glslcc, shader, "hlsl", "40", &args[i], false) if err != nil { return err } if err := parseReflection(reflect, &variants[i]); err != nil { return err } var hlslProf string switch filepath.Ext(shader) { case ".frag": hlslProf = "ps" case ".vert": hlslProf = "vs" default: return fmt.Errorf("unrecognized shader type %s", shader) } var hlslc []byte hlslc, err = compileHLSL_FXC(tmp, shader, hlsl, "main", hlslProf+"_4_0_level_9_1") if err != nil { // Attempt shader model 4.0. Only the app/headless // test shaders use features not supported by level // 9.1. hlslc, err = compileHLSL_FXC(tmp, shader, hlsl, "main", hlslProf+"_4_0") if err != nil { return err } } // OpenGL 3.2 Core only accepts GLSL version 1.50, but is // otherwise compatible with version 1.30. glsl150 := strings.Replace(glsl130, "#version 130", "#version 150", 1) variants[i].Name = filepath.Base(shader) variants[i].GLSL100ES = glsl100es variants[i].GLSL300ES = glsl300es variants[i].GLSL130 = glsl130 variants[i].GLSL150 = glsl150 variants[i].HLSL = hlslc } name := filepath.Base(shader) name = strings.ReplaceAll(name, ".", "_") fmt.Fprintf(out, "\tshader_%s = ", name) // If the shader don't use the variant arguments, output // only a single version. multiVariant := variants[0].GLSL100ES != variants[1].GLSL100ES if multiVariant { fmt.Fprintf(out, "[...]backend.ShaderSources{\n") } for _, src := range variants { fmt.Fprintf(out, "backend.ShaderSources{\n") fmt.Fprintf(out, "Name: %#v,\n", src.Name) if len(src.Inputs) > 0 { fmt.Fprintf(out, "Inputs: %#v,\n", src.Inputs) } if u := src.Uniforms; len(u.Blocks) > 0 { fmt.Fprintf(out, "Uniforms: backend.UniformsReflection{\n") fmt.Fprintf(out, "Blocks: %#v,\n", u.Blocks) fmt.Fprintf(out, "Locations: %#v,\n", u.Locations) fmt.Fprintf(out, "Size: %d,\n", u.Size) fmt.Fprintf(out, "},\n") } if len(src.Textures) > 0 { fmt.Fprintf(out, "Textures: %#v,\n", src.Textures) } fmt.Fprintf(out, "GLSL100ES: %#v,\n", src.GLSL100ES) fmt.Fprintf(out, "GLSL300ES: %#v,\n", src.GLSL300ES) fmt.Fprintf(out, "GLSL130: %#v,\n", src.GLSL130) fmt.Fprintf(out, "GLSL150: %#v,\n", src.GLSL150) fmt.Fprintf(out, "HLSL: %#v,\n", src.HLSL) fmt.Fprintf(out, "}") if multiVariant { fmt.Fprintf(out, ",") } fmt.Fprintf(out, "\n") if !multiVariant { break } } if multiVariant { fmt.Fprintf(out, "}\n") } return nil } func parseReflection(jsonData []byte, info *backend.ShaderSources) error { type InputReflection struct { ID int `json:"id"` Name string `json:"name"` Location int `json:"location"` Semantic string `json:"semantic"` SemanticIndex int `json:"semantic_index"` Type string `json:"type"` } type UniformMemberReflection struct { Name string `json:"name"` Type string `json:"type"` Offset int `json:"offset"` Size int `json:"size"` } type UniformBufferReflection struct { ID int `json:"id"` Name string `json:"name"` Set int `json:"set"` Binding int `json:"binding"` Size int `json:"block_size"` Members []UniformMemberReflection `json:"members"` } type TextureReflection struct { ID int `json:"id"` Name string `json:"name"` Set int `json:"set"` Binding int `json:"binding"` Dimension string `json:"dimension"` Format string `json:"format"` } type StorageBufferReflection struct { ID int `json:"id"` Name string `json:"name"` Set int `json:"set"` Binding int `json:"binding"` BlockSize int `json:"block_size"` } type shaderReflection struct { Inputs []InputReflection `json:"inputs"` UniformBuffers []UniformBufferReflection `json:"uniform_buffers"` Textures []TextureReflection `json:"textures"` StorageBuffers []StorageBufferReflection `json:"storage_buffers"` } type shaderMetadata struct { VS shaderReflection `json:"vs"` FS shaderReflection `json:"fs"` CS shaderReflection `json:"cs"` } var reflect shaderMetadata if err := json.Unmarshal(jsonData, &reflect); err != nil { return fmt.Errorf("parseReflection: %v", err) } inputRef := reflect.VS.Inputs for _, input := range inputRef { dataType, dataSize, err := parseDataType(input.Type) if err != nil { return fmt.Errorf("parseReflection: %v", err) } info.Inputs = append(info.Inputs, backend.InputLocation{ Name: input.Name, Location: input.Location, Semantic: input.Semantic, SemanticIndex: input.SemanticIndex, Type: dataType, Size: dataSize, }) } sort.Slice(info.Inputs, func(i, j int) bool { return info.Inputs[i].Location < info.Inputs[j].Location }) shaderBlocks := reflect.VS.UniformBuffers if len(shaderBlocks) == 0 { shaderBlocks = reflect.FS.UniformBuffers } blockOffset := 0 for _, block := range shaderBlocks { info.Uniforms.Blocks = append(info.Uniforms.Blocks, backend.UniformBlock{ Name: block.Name, Binding: block.Binding, }) for _, member := range block.Members { dataType, size, err := parseDataType(member.Type) if err != nil { return fmt.Errorf("parseReflection: %v", err) } info.Uniforms.Locations = append(info.Uniforms.Locations, backend.UniformLocation{ // Synthetic name generated by glslcc. Name: fmt.Sprintf("_%d.%s", block.ID, member.Name), Type: dataType, Size: size, Offset: blockOffset + member.Offset, }) } blockOffset += block.Size } info.Uniforms.Size = blockOffset textures := reflect.VS.Textures if len(textures) == 0 { textures = reflect.FS.Textures } for _, texture := range textures { info.Textures = append(info.Textures, backend.TextureBinding{ Name: texture.Name, Binding: texture.Binding, }) } for _, sb := range reflect.CS.StorageBuffers { info.StorageBuffers = append(info.StorageBuffers, backend.StorageBufferBinding{ Binding: sb.Binding, BlockSize: sb.BlockSize, }) } return nil } func parseDataType(t string) (backend.DataType, int, error) { switch t { case "float": return backend.DataTypeFloat, 1, nil case "float2": return backend.DataTypeFloat, 2, nil case "float3": return backend.DataTypeFloat, 3, nil case "float4": return backend.DataTypeFloat, 4, nil case "int": return backend.DataTypeInt, 1, nil case "int2": return backend.DataTypeInt, 2, nil case "int3": return backend.DataTypeInt, 3, nil case "int4": return backend.DataTypeInt, 4, nil default: return 0, 0, fmt.Errorf("unsupported input data type: %s", t) } } func compileHLSL_FXC(tmp, path, src, entry, profile string) ([]byte, error) { base := filepath.Base(path) tmppath := filepath.Join(tmp, base) defer os.Remove(tmppath) if err := ioutil.WriteFile(tmppath, []byte(src), 0644); err != nil { return nil, err } outfile := filepath.Join(tmp, base+".obj") defer os.Remove(outfile) fxcInput := tmppath fxcOutput := outfile var fxc *exec.Cmd if runtime.GOOS == "windows" { fxc = exec.Command("fxc.exe") } else { // Convert paths to wine Windows format. var err error fxcInput, err = windowsPath(fxcInput) if err != nil { return nil, err } fxcOutput, err = windowsPath(fxcOutput) if err != nil { return nil, err } fxc = exec.Command("wine", "fxc.exe") } var stdout, stderr bytes.Buffer fxc.Stderr = &stderr fxc.Stdout = &stdout fxc.Args = append(fxc.Args, "/Fo", fxcOutput, "/T", profile, "/E", entry, fxcInput) if err := fxc.Run(); err != nil { info := "" if runtime.GOOS != "windows" { info = "If the fxc tool cannot be found, set WINEPATH to the Windows path for the Windows SDK.\n" } return nil, fmt.Errorf("%s\n%s\n%s%s: %v", stderr.Bytes(), stdout.Bytes(), info, fxc, err) } return ioutil.ReadFile(outfile) } func compileHLSL_DXC(tmp, path, src, entry, profile string) ([]byte, error) { base := filepath.Base(path) tmppath := filepath.Join(tmp, base) defer os.Remove(tmppath) if err := ioutil.WriteFile(tmppath, []byte(src), 0644); err != nil { return nil, err } outfile := filepath.Join(tmp, base+".obj") defer os.Remove(outfile) dxcInput := tmppath dxcOutput := outfile var dxc *exec.Cmd if runtime.GOOS == "windows" { dxc = exec.Command("dxc.exe") } else { // Convert paths to wine Windows format. var err error dxcInput, err = windowsPath(dxcInput) if err != nil { return nil, err } dxcOutput, err = windowsPath(dxcOutput) if err != nil { return nil, err } dxc = exec.Command("wine", "dxc.exe") } var stdout, stderr bytes.Buffer dxc.Stderr = &stderr dxc.Stdout = &stdout dxc.Args = append(dxc.Args, "-Fo", dxcOutput, "-T", profile, "-Qstrip_reflect", "-E", entry, dxcInput) if err := dxc.Run(); err != nil { info := "" if runtime.GOOS != "windows" { info = "If the dxc tool cannot be found, set WINEPATH to the Windows path for the Windows SDK.\n" } return nil, fmt.Errorf("%s\n%s\n%s%s: %v", stderr.Bytes(), stdout.Bytes(), info, dxc, err) } return ioutil.ReadFile(outfile) } // windowsPath uses the winepath tool to convert a path to Windows format. The returned // path can be used as arguments for Windows command line tools. func windowsPath(path string) (string, error) { var out bytes.Buffer winepath := exec.Command("winepath", "--windows", path) winepath.Stdout = &out if err := winepath.Run(); err != nil { return "", err } return strings.TrimSpace(out.String()), nil } func convertShader(tmp, glslcc, path, lang, profile string, args *shaderArgs, flattenUBOs bool) (string, []byte, error) { shaderTmpl, err := template.ParseFiles(path) if err != nil { return "", nil, err } var buf bytes.Buffer if err := shaderTmpl.Execute(&buf, args); err != nil { return "", nil, err } tmppath := filepath.Join(tmp, filepath.Base(path)) defer os.Remove(tmppath) if err := ioutil.WriteFile(tmppath, buf.Bytes(), 0644); err != nil { return "", nil, err } var progFlag string var progSuffix string switch filepath.Ext(path) { case ".vert": progFlag = "--vert" progSuffix = "vs" case ".frag": progFlag = "--frag" progSuffix = "fs" case ".comp": progFlag = "--compute" progSuffix = "cs" default: return "", nil, fmt.Errorf("unrecognized shader type: %s", path) } cmd := exec.Command(glslcc, "--silent", "--optimize", "--include-dirs", absShadersDir, "--reflect", "--output", filepath.Join(tmp, "shader"), "--lang", lang, "--profile", profile, progFlag, tmppath, ) if lang == "hlsl" { cmd.Args = append(cmd.Args, "--defines=HLSL") } if flattenUBOs { cmd.Args = append(cmd.Args, "--flatten-ubos") } cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { return "", nil, fmt.Errorf("%s: %v", strings.Join(cmd.Args, " "), err) } f, err := os.Open(filepath.Join(tmp, "shader_"+progSuffix)) if err != nil { return "", nil, err } defer f.Close() defer os.Remove(f.Name()) src, err := ioutil.ReadAll(f) if err != nil { return "", nil, err } reflect, err := ioutil.ReadFile(filepath.Join(tmp, "shader_"+progSuffix+".json")) if err != nil { return "", nil, err } return string(src), reflect, nil } type computeShaders struct { glsl310es string hlsl []byte } func convertComputeShader(tmp, glslangValidator, spirv_cross, path string) (computeShaders, error) { spvFile := filepath.Join(tmp, "shader.spv") cmd := exec.Command(glslangValidator, "-G100", // OpenGL ES 3.1. "-w", // Suppress warnings. "-o", spvFile, path, ) defer os.Remove(spvFile) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { return computeShaders{}, fmt.Errorf("%s: %v", strings.Join(cmd.Args, " "), err) } var output computeShaders { cmd = exec.Command(spirv_cross, "--es", "--version", "310", spvFile, ) var shaderDump bytes.Buffer cmd.Stdout = &shaderDump cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { return output, fmt.Errorf("%s: %v", strings.Join(cmd.Args, " "), err) } output.glsl310es = unixLineEnding(shaderDump.String()) } if *directCompute { cmd = exec.Command(spirv_cross, "--hlsl", "--shader-model", "50", spvFile, ) var shaderDump bytes.Buffer cmd.Stdout = &shaderDump cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { return output, fmt.Errorf("%s: %v", strings.Join(cmd.Args, " "), err) } hlslSrc := shaderDump.String() var err error output.hlsl, err = compileHLSL_DXC(tmp, path, hlslSrc, "main", "cs_5_0") if err != nil { return output, err } } return output, nil } func unixLineEnding(s string) string { return strings.ReplaceAll(s, "\r\n", "\n") }