1
0
Fork 0
forked from forgejo/forgejo

Use a general approach to access custom/static/builtin assets (#24022)

The idea is to use a Layered Asset File-system (modules/assetfs/layered.go)

For example: when there are 2 layers: "custom", "builtin", when access
to asset "my/page.tmpl", the Layered Asset File-system will first try to
use "custom" assets, if not found, then use "builtin" assets.

This approach will hugely simplify a lot of code, make them testable.

Other changes:

* Simplify the AssetsHandlerFunc code
* Simplify the `gitea embedded` sub-command code

---------

Co-authored-by: Jason Song <i@wolfogre.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
wxiaoguang 2023-04-12 18:16:45 +08:00 committed by GitHub
parent 42919ccb7c
commit 50a72e7a83
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
36 changed files with 689 additions and 1055 deletions

View file

@ -4,14 +4,10 @@
package templates
import (
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/assetfs"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
)
@ -47,81 +43,30 @@ func BaseVars() Vars {
}
}
func getDirTemplateAssetNames(dir string) []string {
return getDirAssetNames(dir, false)
func AssetFS() *assetfs.LayeredFS {
return assetfs.Layered(CustomAssets(), BuiltinAssets())
}
func getDirAssetNames(dir string, mailer bool) []string {
var tmpls []string
func CustomAssets() *assetfs.Layer {
return assetfs.Local("custom", setting.CustomPath, "templates")
}
if mailer {
dir += filepath.Join(dir, "mail")
}
f, err := os.Stat(dir)
func ListWebTemplateAssetNames(assets *assetfs.LayeredFS) ([]string, error) {
files, err := assets.ListAllFiles(".", true)
if err != nil {
if os.IsNotExist(err) {
return tmpls
}
log.Warn("Unable to check if templates dir %s is a directory. Error: %v", dir, err)
return tmpls
}
if !f.IsDir() {
log.Warn("Templates dir %s is a not directory.", dir)
return tmpls
return nil, err
}
return util.SliceRemoveAllFunc(files, func(file string) bool {
return strings.HasPrefix(file, "mail/") || !strings.HasSuffix(file, ".tmpl")
}), nil
}
files, err := util.StatDir(dir)
func ListMailTemplateAssetNames(assets *assetfs.LayeredFS) ([]string, error) {
files, err := assets.ListAllFiles(".", true)
if err != nil {
log.Warn("Failed to read %s templates dir. %v", dir, err)
return tmpls
return nil, err
}
prefix := "templates/"
if mailer {
prefix += "mail/"
}
for _, filePath := range files {
if !mailer && strings.HasPrefix(filePath, "mail/") {
continue
}
if !strings.HasSuffix(filePath, ".tmpl") {
continue
}
tmpls = append(tmpls, prefix+filePath)
}
return tmpls
}
func walkAssetDir(root string, skipMail bool, callback func(path, name string, d fs.DirEntry, err error) error) error {
mailRoot := filepath.Join(root, "mail")
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
name := path[len(root):]
if len(name) > 0 && name[0] == '/' {
name = name[1:]
}
if err != nil {
if os.IsNotExist(err) {
return callback(path, name, d, err)
}
return err
}
if skipMail && path == mailRoot && d.IsDir() {
return fs.SkipDir
}
if util.CommonSkip(d.Name()) {
if d.IsDir() {
return fs.SkipDir
}
return nil
}
if strings.HasSuffix(d.Name(), ".tmpl") || d.IsDir() {
return callback(path, name, d, err)
}
return nil
}); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("unable to get files for template assets in %s: %w", root, err)
}
return nil
return util.SliceRemoveAllFunc(files, func(file string) bool {
return !strings.HasPrefix(file, "mail/") || !strings.HasSuffix(file, ".tmpl")
}), nil
}

View file

@ -6,76 +6,10 @@
package templates
import (
"io/fs"
"os"
"path/filepath"
"code.gitea.io/gitea/modules/assetfs"
"code.gitea.io/gitea/modules/setting"
)
// GetAsset returns asset content via name
func GetAsset(name string) ([]byte, error) {
bs, err := os.ReadFile(filepath.Join(setting.CustomPath, name))
if err != nil && !os.IsNotExist(err) {
return nil, err
} else if err == nil {
return bs, nil
}
return os.ReadFile(filepath.Join(setting.StaticRootPath, name))
}
// GetAssetFilename returns the filename of the provided asset
func GetAssetFilename(name string) (string, error) {
filename := filepath.Join(setting.CustomPath, name)
_, err := os.Stat(filename)
if err != nil && !os.IsNotExist(err) {
return filename, err
} else if err == nil {
return filename, nil
}
filename = filepath.Join(setting.StaticRootPath, name)
_, err = os.Stat(filename)
return filename, err
}
// walkTemplateFiles calls a callback for each template asset
func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
return err
}
if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// GetTemplateAssetNames returns list of template names
func GetTemplateAssetNames() []string {
tmpls := getDirTemplateAssetNames(filepath.Join(setting.CustomPath, "templates"))
tmpls2 := getDirTemplateAssetNames(filepath.Join(setting.StaticRootPath, "templates"))
return append(tmpls, tmpls2...)
}
func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
return err
}
if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// BuiltinAsset will read the provided asset from the embedded assets
// (This always returns os.ErrNotExist)
func BuiltinAsset(name string) ([]byte, error) {
return nil, os.ErrNotExist
}
// BuiltinAssetNames returns the names of the embedded assets
// (This always returns nil)
func BuiltinAssetNames() []string {
return nil
func BuiltinAssets() *assetfs.Layer {
return assetfs.Local("builtin(static)", setting.StaticRootPath, "templates")
}

View file

@ -21,7 +21,6 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/watcher"
)
var (
@ -66,20 +65,23 @@ func (h *HTMLRender) TemplateLookup(name string) (*template.Template, error) {
}
func (h *HTMLRender) CompileTemplates() error {
dirPrefix := "templates/"
extSuffix := ".tmpl"
tmpls := template.New("")
for _, path := range GetTemplateAssetNames() {
if !strings.HasSuffix(path, extSuffix) {
assets := AssetFS()
files, err := ListWebTemplateAssetNames(assets)
if err != nil {
return nil
}
for _, file := range files {
if !strings.HasSuffix(file, extSuffix) {
continue
}
name := strings.TrimPrefix(path, dirPrefix)
name = strings.TrimSuffix(name, extSuffix)
name := strings.TrimSuffix(file, extSuffix)
tmpl := tmpls.New(filepath.ToSlash(name))
for _, fm := range NewFuncMap() {
tmpl.Funcs(fm)
}
buf, err := GetAsset(path)
buf, err := assets.ReadFile(file)
if err != nil {
return err
}
@ -112,13 +114,10 @@ func HTMLRenderer(ctx context.Context) (context.Context, *HTMLRender) {
log.Fatal("HTMLRenderer error: %v", err)
}
if !setting.IsProd {
watcher.CreateWatcher(ctx, "HTML Templates", &watcher.CreateWatcherOpts{
PathsCallback: walkTemplateFiles,
BetweenCallback: func() {
if err := renderer.CompileTemplates(); err != nil {
log.Error("Template error: %v\n%s", err, log.Stack(2))
}
},
go AssetFS().WatchLocalChanges(ctx, func() {
if err := renderer.CompileTemplates(); err != nil {
log.Error("Template error: %v\n%s", err, log.Stack(2))
}
})
}
return context.WithValue(ctx, rendererKey, renderer), renderer
@ -138,14 +137,8 @@ func handleGenericTemplateError(err error) (string, []interface{}) {
}
templateName, lineNumberStr, message := groups[1], groups[2], groups[3]
filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl")
if assetErr != nil {
return "", nil
}
filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl"))
lineNumber, _ := strconv.Atoi(lineNumberStr)
line := GetLineFromTemplate(templateName, lineNumber, "", -1)
return "PANIC: Unable to compile templates!\n%s in template file %s at line %d:\n\n%s\nStacktrace:\n\n%s", []interface{}{message, filename, lineNumber, log.NewColoredValue(line, log.Reset), log.Stack(2)}
@ -158,16 +151,9 @@ func handleNotDefinedPanicError(err error) (string, []interface{}) {
}
templateName, lineNumberStr, functionName := groups[1], groups[2], groups[3]
functionName, _ = strconv.Unquote(`"` + functionName + `"`)
filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl")
if assetErr != nil {
return "", nil
}
filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl"))
lineNumber, _ := strconv.Atoi(lineNumberStr)
line := GetLineFromTemplate(templateName, lineNumber, functionName, -1)
return "PANIC: Unable to compile templates!\nUndefined function %q in template file %s at line %d:\n\n%s", []interface{}{functionName, filename, lineNumber, log.NewColoredValue(line, log.Reset)}
@ -181,14 +167,8 @@ func handleUnexpected(err error) (string, []interface{}) {
templateName, lineNumberStr, unexpected := groups[1], groups[2], groups[3]
unexpected, _ = strconv.Unquote(`"` + unexpected + `"`)
filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl")
if assetErr != nil {
return "", nil
}
filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl"))
lineNumber, _ := strconv.Atoi(lineNumberStr)
line := GetLineFromTemplate(templateName, lineNumber, unexpected, -1)
return "PANIC: Unable to compile templates!\nUnexpected %q in template file %s at line %d:\n\n%s", []interface{}{unexpected, filename, lineNumber, log.NewColoredValue(line, log.Reset)}
@ -201,14 +181,8 @@ func handleExpectedEnd(err error) (string, []interface{}) {
}
templateName, lineNumberStr, unexpected := groups[1], groups[2], groups[3]
filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl")
if assetErr != nil {
return "", nil
}
filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl"))
lineNumber, _ := strconv.Atoi(lineNumberStr)
line := GetLineFromTemplate(templateName, lineNumber, unexpected, -1)
return "PANIC: Unable to compile templates!\nMissing end with unexpected %q in template file %s at line %d:\n\n%s", []interface{}{unexpected, filename, lineNumber, log.NewColoredValue(line, log.Reset)}
@ -218,7 +192,7 @@ const dashSeparator = "---------------------------------------------------------
// GetLineFromTemplate returns a line from a template with some context
func GetLineFromTemplate(templateName string, targetLineNum int, target string, position int) string {
bs, err := GetAsset("templates/" + templateName + ".tmpl")
bs, err := AssetFS().ReadFile(templateName + ".tmpl")
if err != nil {
return fmt.Sprintf("(unable to read template file: %v)", err)
}

View file

@ -6,15 +6,12 @@ package templates
import (
"context"
"html/template"
"io/fs"
"os"
"strings"
texttmpl "text/template"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/watcher"
)
// mailSubjectTextFuncMap returns functions for injecting to text templates, it's only used for mail subject
@ -62,54 +59,23 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) {
bodyTemplates.Funcs(funcs)
}
assetFS := AssetFS()
refreshTemplates := func() {
for _, assetPath := range BuiltinAssetNames() {
if !strings.HasPrefix(assetPath, "mail/") {
continue
}
if !strings.HasSuffix(assetPath, ".tmpl") {
continue
}
content, err := BuiltinAsset(assetPath)
if err != nil {
log.Warn("Failed to read embedded %s template. %v", assetPath, err)
continue
}
assetName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/")
log.Trace("Adding built-in mailer template for %s", assetName)
buildSubjectBodyTemplate(subjectTemplates,
bodyTemplates,
assetName,
content)
assetPaths, err := ListMailTemplateAssetNames(assetFS)
if err != nil {
log.Error("Failed to list mail templates: %v", err)
return
}
if err := walkMailerTemplates(func(path, name string, d fs.DirEntry, err error) error {
for _, assetPath := range assetPaths {
content, layerName, err := assetFS.ReadLayeredFile(assetPath)
if err != nil {
return err
log.Warn("Failed to read mail template %s by %s: %v", assetPath, layerName, err)
continue
}
if d.IsDir() {
return nil
}
content, err := os.ReadFile(path)
if err != nil {
log.Warn("Failed to read custom %s template. %v", path, err)
return nil
}
assetName := strings.TrimSuffix(name, ".tmpl")
log.Trace("Adding mailer template for %s from %q", assetName, path)
buildSubjectBodyTemplate(subjectTemplates,
bodyTemplates,
assetName,
content)
return nil
}); err != nil && !os.IsNotExist(err) {
log.Warn("Error whilst walking mailer templates directories. %v", err)
tmplName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/")
log.Trace("Adding mail template %s: %s by %s", tmplName, assetPath, layerName)
buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, tmplName, content)
}
}
@ -118,10 +84,7 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) {
if !setting.IsProd {
// Now subjectTemplates and bodyTemplates are both synchronized
// thus it is safe to call refresh from a different goroutine
watcher.CreateWatcher(ctx, "Mailer Templates", &watcher.CreateWatcherOpts{
PathsCallback: walkMailerTemplates,
BetweenCallback: refreshTemplates,
})
go assetFS.WatchLocalChanges(ctx, refreshTemplates)
}
return subjectTemplates, bodyTemplates

View file

@ -6,114 +6,17 @@
package templates
import (
"html/template"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
texttmpl "text/template"
"time"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/assetfs"
"code.gitea.io/gitea/modules/timeutil"
)
var (
subjectTemplates = texttmpl.New("")
bodyTemplates = template.New("")
)
// GlobalModTime provide a global mod time for embedded asset files
func GlobalModTime(filename string) time.Time {
return timeutil.GetExecutableModTime()
}
// GetAssetFilename returns the filename of the provided asset
func GetAssetFilename(name string) (string, error) {
filename := filepath.Join(setting.CustomPath, name)
_, err := os.Stat(filename)
if err != nil && !os.IsNotExist(err) {
return name, err
} else if err == nil {
return filename, nil
}
return "(builtin) " + name, nil
}
// GetAsset get a special asset, only for chi
func GetAsset(name string) ([]byte, error) {
bs, err := os.ReadFile(filepath.Join(setting.CustomPath, name))
if err != nil && !os.IsNotExist(err) {
return nil, err
} else if err == nil {
return bs, nil
}
return BuiltinAsset(strings.TrimPrefix(name, "templates/"))
}
// GetFiles calls a callback for each template asset
func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// GetTemplateAssetNames only for chi
func GetTemplateAssetNames() []string {
realFS := Assets.(vfsgen۰FS)
tmpls := make([]string, 0, len(realFS))
for k := range realFS {
if strings.HasPrefix(k, "/mail/") {
continue
}
tmpls = append(tmpls, "templates/"+k[1:])
}
customDir := path.Join(setting.CustomPath, "templates")
customTmpls := getDirTemplateAssetNames(customDir)
return append(tmpls, customTmpls...)
}
func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// BuiltinAsset reads the provided asset from the builtin embedded assets
func BuiltinAsset(name string) ([]byte, error) {
f, err := Assets.Open("/" + name)
if err != nil {
return nil, err
}
defer f.Close()
return io.ReadAll(f)
}
// BuiltinAssetNames returns the names of the built-in embedded assets
func BuiltinAssetNames() []string {
realFS := Assets.(vfsgen۰FS)
results := make([]string, 0, len(realFS))
for k := range realFS {
results = append(results, k[1:])
}
return results
}
// BuiltinAssetIsDir returns if a provided asset is a directory
func BuiltinAssetIsDir(name string) (bool, error) {
if f, err := Assets.Open("/" + name); err != nil {
return false, err
} else {
defer f.Close()
if fi, err := f.Stat(); err != nil {
return false, err
} else {
return fi.IsDir(), nil
}
}
func BuiltinAssets() *assetfs.Layer {
return assetfs.Bindata("builtin(bindata)", Assets)
}