Better logging (#6038) (#6095)

* Panic don't fatal on create new logger

Fixes #5854

Signed-off-by: Andrew Thornton <art27@cantab.net>

* partial broken

* Update the logging infrastrcture

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Reset the skip levels for Fatal and Error

Signed-off-by: Andrew Thornton <art27@cantab.net>

* broken ncsa

* More log.Error fixes

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Remove nal

* set log-levels to lowercase

* Make console_test test all levels

* switch to lowercased levels

* OK now working

* Fix vetting issues

* Fix lint

* Fix tests

* change default logging to match current gitea

* Improve log testing

Signed-off-by: Andrew Thornton <art27@cantab.net>

* reset error skip levels to 0

* Update documentation and access logger configuration

* Redirect the router log back to gitea if redirect macaron log but also allow setting the log level - i.e. TRACE

* Fix broken level caching

* Refactor the router log

* Add Router logger

* Add colorizing options

* Adjust router colors

* Only create logger if they will be used

* update app.ini.sample

* rename Attribute ColorAttribute

* Change from white to green for function

* Set fatal/error levels

* Restore initial trace logger

* Fix Trace arguments in modules/auth/auth.go

* Properly handle XORMLogger

* Improve admin/config page

* fix fmt

* Add auto-compression of old logs

* Update error log levels

* Remove the unnecessary skip argument from Error, Fatal and Critical

* Add stacktrace support

* Fix tests

* Remove x/sync from vendors?

* Add stderr option to console logger

* Use filepath.ToSlash to protect against Windows in tests

* Remove prefixed underscores from names in colors.go

* Remove not implemented database logger

This was removed from Gogs on 4 Mar 2016 but left in the configuration
since then.

* Ensure that log paths are relative to ROOT_PATH

* use path.Join

* rename jsonConfig to logConfig

* Rename "config" to "jsonConfig" to make it clearer

* Requested changes

* Requested changes: XormLogger

* Try to color the windows terminal

If successful default to colorizing the console logs

* fixup

* Colorize initially too

* update vendor

* Colorize logs on default and remove if this is not a colorizing logger

* Fix documentation

* fix test

* Use go-isatty to detect if on windows we are on msys or cygwin

* Fix spelling mistake

* Add missing vendors

* More changes

* Rationalise the ANSI writer protection

* Adjust colors on advice from @0x5c

* Make Flags a comma separated list

* Move to use the windows constant for ENABLE_VIRTUAL_TERMINAL_PROCESSING

* Ensure matching is done on the non-colored message - to simpify EXPRESSION

routers/routes/routes.go

@@ -5,11 +5,12 @@
 package routes
 
 import (
+	"bytes"
 	"encoding/gob"
-	"fmt"
 	"net/http"
 	"os"
 	"path"
+	"text/template"
 	"time"
 
 	"code.gitea.io/gitea/models"
@@ -46,7 +47,7 @@ import (
 	macaron "gopkg.in/macaron.v1"
 )
 
-func giteaLogger(l *log.LoggerAsWriter) macaron.Handler {
+/*func giteaLogger(l *log.LoggerAsWriter) macaron.Handler {
 	return func(ctx *macaron.Context) {
 		start := time.Now()
 
@@ -57,6 +58,40 @@ func giteaLogger(l *log.LoggerAsWriter) macaron.Handler {
 		rw := ctx.Resp.(macaron.ResponseWriter)
 		l.Log(fmt.Sprintf("[Macaron] Completed %s %s %v %s in %v", ctx.Req.Method, ctx.Req.RequestURI, rw.Status(), http.StatusText(rw.Status()), time.Since(start)))
 	}
+}*/
+
+type routerLoggerOptions struct {
+	Ctx            *macaron.Context
+	Identity       *string
+	Start          *time.Time
+	ResponseWriter *macaron.ResponseWriter
+}
+
+func setupAccessLogger(m *macaron.Macaron) {
+	logger := log.GetLogger("access")
+
+	logTemplate, _ := template.New("log").Parse(setting.AccessLogTemplate)
+	m.Use(func(ctx *macaron.Context) {
+		start := time.Now()
+		ctx.Next()
+		identity := "-"
+		if val, ok := ctx.Data["SignedUserName"]; ok {
+			if stringVal, ok := val.(string); ok && stringVal != "" {
+				identity = stringVal
+			}
+		}
+		rw := ctx.Resp.(macaron.ResponseWriter)
+
+		buf := bytes.NewBuffer([]byte{})
+		logTemplate.Execute(buf, routerLoggerOptions{
+			Ctx:            ctx,
+			Identity:       &identity,
+			Start:          &start,
+			ResponseWriter: &rw,
+		})
+
+		logger.SendLog(log.INFO, "", "", 0, buf.String(), "")
+	})
 }
 
 // NewMacaron initializes Macaron instance.
@@ -64,10 +99,10 @@ func NewMacaron() *macaron.Macaron {
 	gob.Register(&u2f.Challenge{})
 	var m *macaron.Macaron
 	if setting.RedirectMacaronLog {
-		loggerAsWriter := log.NewLoggerAsWriter("INFO")
+		loggerAsWriter := log.NewLoggerAsWriter("INFO", log.GetLogger("macaron"))
 		m = macaron.NewWithLogger(loggerAsWriter)
-		if !setting.DisableRouterLog {
-			m.Use(giteaLogger(loggerAsWriter))
+		if !setting.DisableRouterLog && setting.RouterLogLevel != log.NONE {
+			log.SetupRouterLogger(m, setting.RouterLogLevel)
 		}
 	} else {
 		m = macaron.New()
@@ -75,6 +110,10 @@ func NewMacaron() *macaron.Macaron {
 			m.Use(macaron.Logger())
 		}
 	}
+	// Access Logger is similar to Router Log but more configurable and by default is more like the NCSA Common Log format
+	if setting.EnableAccessLog {
+		setupAccessLogger(m)
+	}
 	m.Use(macaron.Recovery())
 	if setting.EnableGzip {
 		m.Use(gzip.Middleware())
@@ -110,7 +149,7 @@ func NewMacaron() *macaron.Macaron {
 	localeNames, err := options.Dir("locale")
 
 	if err != nil {
-		log.Fatal(4, "Failed to list locale files: %v", err)
+		log.Fatal("Failed to list locale files: %v", err)
 	}
 
 	localFiles := make(map[string][]byte)
@@ -119,7 +158,7 @@ func NewMacaron() *macaron.Macaron {
 		localFiles[name], err = options.Locale(name)
 
 		if err != nil {
-			log.Fatal(4, "Failed to load %s locale file. %v", name, err)
+			log.Fatal("Failed to load %s locale file. %v", name, err)
 		}
 	}