Server-side syntax highlighting for all code (#12047)

* Server-side syntax hilighting for all code

This PR does a few things:

* Remove all traces of highlight.js
* Use chroma library to provide fast syntax hilighting directly on the server
* Provide syntax hilighting for diffs
* Re-style both unified and split diffs views
* Add custom syntax hilighting styling for both regular and arc-green

Fixes #7729
Fixes #10157
Fixes #11825
Fixes #7728
Fixes #3872
Fixes #3682

And perhaps gets closer to #9553

* fix line marker

* fix repo search

* Fix single line select

* properly load settings

* npm uninstall highlight.js

* review suggestion

* code review

* forgot to call function

* fix test

* Apply suggestions from code review

suggestions from @silverwind thanks

Co-authored-by: silverwind <me@silverwind.io>

* code review

* copy/paste error

* Use const for highlight size limit

* Update web_src/less/_repository.less

Co-authored-by: Lauris BH <lauris@nix.lv>

* update size limit to 1MB and other styling tweaks

* fix highlighting for certain diff sections

* fix test

* add worker back as suggested

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Lauris BH <lauris@nix.lv>

vendor/github.com/alecthomas/chroma/lexers/n/nasm.go

@@ -0,0 +1,59 @@
+package n
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// Nasm lexer.
+var Nasm = internal.Register(MustNewLexer(
+	&Config{
+		Name:            "NASM",
+		Aliases:         []string{"nasm"},
+		Filenames:       []string{"*.asm", "*.ASM"},
+		MimeTypes:       []string{"text/x-nasm"},
+		CaseInsensitive: true,
+	},
+	Rules{
+		"root": {
+			{`^\s*%`, CommentPreproc, Push("preproc")},
+			Include("whitespace"),
+			{`[a-z$._?][\w$.?#@~]*:`, NameLabel, nil},
+			{`([a-z$._?][\w$.?#@~]*)(\s+)(equ)`, ByGroups(NameConstant, KeywordDeclaration, KeywordDeclaration), Push("instruction-args")},
+			{`BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|ORG|ALIGN|STRUC|ENDSTRUC|COMMON|CPU|GROUP|UPPERCASE|IMPORT|EXPORT|LIBRARY|MODULE`, Keyword, Push("instruction-args")},
+			{`(?:res|d)[bwdqt]|times`, KeywordDeclaration, Push("instruction-args")},
+			{`[a-z$._?][\w$.?#@~]*`, NameFunction, Push("instruction-args")},
+			{`[\r\n]+`, Text, nil},
+		},
+		"instruction-args": {
+			{"\"(\\\\\"|[^\"\\n])*\"|'(\\\\'|[^'\\n])*'|`(\\\\`|[^`\\n])*`", LiteralString, nil},
+			{`(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)`, LiteralNumberHex, nil},
+			{`[0-7]+q`, LiteralNumberOct, nil},
+			{`[01]+b`, LiteralNumberBin, nil},
+			{`[0-9]+\.e?[0-9]+`, LiteralNumberFloat, nil},
+			{`[0-9]+`, LiteralNumberInteger, nil},
+			Include("punctuation"),
+			{`r[0-9][0-5]?[bwd]|[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]`, NameBuiltin, nil},
+			{`[a-z$._?][\w$.?#@~]*`, NameVariable, nil},
+			{`[\r\n]+`, Text, Pop(1)},
+			Include("whitespace"),
+		},
+		"preproc": {
+			{`[^;\n]+`, CommentPreproc, nil},
+			{`;.*?\n`, CommentSingle, Pop(1)},
+			{`\n`, CommentPreproc, Pop(1)},
+		},
+		"whitespace": {
+			{`\n`, Text, nil},
+			{`[ \t]+`, Text, nil},
+			{`;.*`, CommentSingle, nil},
+		},
+		"punctuation": {
+			{`[,():\[\]]+`, Punctuation, nil},
+			{`[&|^<>+*/%~-]+`, Operator, nil},
+			{`[$]+`, KeywordConstant, nil},
+			{`seg|wrt|strict`, OperatorWord, nil},
+			{`byte|[dq]?word`, KeywordType, nil},
+		},
+	},
+))

vendor/github.com/alecthomas/chroma/lexers/n/newspeak.go

@@ -0,0 +1,55 @@
+package n
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// Newspeak lexer.
+var Newspeak = internal.Register(MustNewLexer(
+	&Config{
+		Name:      "Newspeak",
+		Aliases:   []string{"newspeak"},
+		Filenames: []string{"*.ns2"},
+		MimeTypes: []string{"text/x-newspeak"},
+	},
+	Rules{
+		"root": {
+			{`\b(Newsqueak2)\b`, KeywordDeclaration, nil},
+			{`'[^']*'`, LiteralString, nil},
+			{`\b(class)(\s+)(\w+)(\s*)`, ByGroups(KeywordDeclaration, Text, NameClass, Text), nil},
+			{`\b(mixin|self|super|private|public|protected|nil|true|false)\b`, Keyword, nil},
+			{`(\w+\:)(\s*)([a-zA-Z_]\w+)`, ByGroups(NameFunction, Text, NameVariable), nil},
+			{`(\w+)(\s*)(=)`, ByGroups(NameAttribute, Text, Operator), nil},
+			{`<\w+>`, CommentSpecial, nil},
+			Include("expressionstat"),
+			Include("whitespace"),
+		},
+		"expressionstat": {
+			{`(\d+\.\d*|\.\d+|\d+[fF])[fF]?`, LiteralNumberFloat, nil},
+			{`\d+`, LiteralNumberInteger, nil},
+			{`:\w+`, NameVariable, nil},
+			{`(\w+)(::)`, ByGroups(NameVariable, Operator), nil},
+			{`\w+:`, NameFunction, nil},
+			{`\w+`, NameVariable, nil},
+			{`\(|\)`, Punctuation, nil},
+			{`\[|\]`, Punctuation, nil},
+			{`\{|\}`, Punctuation, nil},
+			{`(\^|\+|\/|~|\*|<|>|=|@|%|\||&|\?|!|,|-|:)`, Operator, nil},
+			{`\.|;`, Punctuation, nil},
+			Include("whitespace"),
+			Include("literals"),
+		},
+		"literals": {
+			{`\$.`, LiteralString, nil},
+			{`'[^']*'`, LiteralString, nil},
+			{`#'[^']*'`, LiteralStringSymbol, nil},
+			{`#\w+:?`, LiteralStringSymbol, nil},
+			{`#(\+|\/|~|\*|<|>|=|@|%|\||&|\?|!|,|-)+`, LiteralStringSymbol, nil},
+		},
+		"whitespace": {
+			{`\s+`, Text, nil},
+			{`"[^"]*"`, Comment, nil},
+		},
+	},
+))

vendor/github.com/alecthomas/chroma/lexers/n/nginx.go

@@ -0,0 +1,47 @@
+package n
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// Nginx Configuration File lexer.
+var Nginx = internal.Register(MustNewLexer(
+	&Config{
+		Name:      "Nginx configuration file",
+		Aliases:   []string{"nginx"},
+		Filenames: []string{"nginx.conf"},
+		MimeTypes: []string{"text/x-nginx-conf"},
+	},
+	Rules{
+		"root": {
+			{`(include)(\s+)([^\s;]+)`, ByGroups(Keyword, Text, Name), nil},
+			{`[^\s;#]+`, Keyword, Push("stmt")},
+			Include("base"),
+		},
+		"block": {
+			{`\}`, Punctuation, Pop(2)},
+			{`[^\s;#]+`, KeywordNamespace, Push("stmt")},
+			Include("base"),
+		},
+		"stmt": {
+			{`\{`, Punctuation, Push("block")},
+			{`;`, Punctuation, Pop(1)},
+			Include("base"),
+		},
+		"base": {
+			{`#.*\n`, CommentSingle, nil},
+			{`on|off`, NameConstant, nil},
+			{`\$[^\s;#()]+`, NameVariable, nil},
+			{`([a-z0-9.-]+)(:)([0-9]+)`, ByGroups(Name, Punctuation, LiteralNumberInteger), nil},
+			{`[a-z-]+/[a-z-+]+`, LiteralString, nil},
+			{`[0-9]+[km]?\b`, LiteralNumberInteger, nil},
+			{`(~)(\s*)([^\s{]+)`, ByGroups(Punctuation, Text, LiteralStringRegex), nil},
+			{`[:=~]`, Punctuation, nil},
+			{`[^\s;#{}$]+`, LiteralString, nil},
+			{`/[^\s;#]*`, Name, nil},
+			{`\s+`, Text, nil},
+			{`[$;]`, Text, nil},
+		},
+	},
+))

vendor/github.com/alecthomas/chroma/lexers/n/nim.go

@@ -0,0 +1,93 @@
+package n
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// Nim lexer.
+var Nim = internal.Register(MustNewLexer(
+	&Config{
+		Name:            "Nim",
+		Aliases:         []string{"nim", "nimrod"},
+		Filenames:       []string{"*.nim", "*.nimrod"},
+		MimeTypes:       []string{"text/x-nim"},
+		CaseInsensitive: true,
+	},
+	Rules{
+		"root": {
+			{`#\[[\s\S]*?\]#`, CommentMultiline, nil},
+			{`##.*$`, LiteralStringDoc, nil},
+			{`#.*$`, Comment, nil},
+			{`[*=><+\-/@$~&%!?|\\\[\]]`, Operator, nil},
+			{"\\.\\.|\\.|,|\\[\\.|\\.\\]|\\{\\.|\\.\\}|\\(\\.|\\.\\)|\\{|\\}|\\(|\\)|:|\\^|`|;", Punctuation, nil},
+			{`(?:[\w]+)"`, LiteralString, Push("rdqs")},
+			{`"""`, LiteralString, Push("tdqs")},
+			{`"`, LiteralString, Push("dqs")},
+			{`'`, LiteralStringChar, Push("chars")},
+			{`(a_?n_?d_?|o_?r_?|n_?o_?t_?|x_?o_?r_?|s_?h_?l_?|s_?h_?r_?|d_?i_?v_?|m_?o_?d_?|i_?n_?|n_?o_?t_?i_?n_?|i_?s_?|i_?s_?n_?o_?t_?)\b`, OperatorWord, nil},
+			{`(p_?r_?o_?c_?\s)(?![(\[\]])`, Keyword, Push("funcname")},
+			{`(a_?d_?d_?r_?|a_?n_?d_?|a_?s_?|a_?s_?m_?|a_?t_?o_?m_?i_?c_?|b_?i_?n_?d_?|b_?l_?o_?c_?k_?|b_?r_?e_?a_?k_?|c_?a_?s_?e_?|c_?a_?s_?t_?|c_?o_?n_?c_?e_?p_?t_?|c_?o_?n_?s_?t_?|c_?o_?n_?t_?i_?n_?u_?e_?|c_?o_?n_?v_?e_?r_?t_?e_?r_?|d_?e_?f_?e_?r_?|d_?i_?s_?c_?a_?r_?d_?|d_?i_?s_?t_?i_?n_?c_?t_?|d_?i_?v_?|d_?o_?|e_?l_?i_?f_?|e_?l_?s_?e_?|e_?n_?d_?|e_?n_?u_?m_?|e_?x_?c_?e_?p_?t_?|e_?x_?p_?o_?r_?t_?|f_?i_?n_?a_?l_?l_?y_?|f_?o_?r_?|f_?u_?n_?c_?|i_?f_?|i_?n_?|y_?i_?e_?l_?d_?|i_?n_?t_?e_?r_?f_?a_?c_?e_?|i_?s_?|i_?s_?n_?o_?t_?|i_?t_?e_?r_?a_?t_?o_?r_?|l_?e_?t_?|m_?a_?c_?r_?o_?|m_?e_?t_?h_?o_?d_?|m_?i_?x_?i_?n_?|m_?o_?d_?|n_?o_?t_?|n_?o_?t_?i_?n_?|o_?b_?j_?e_?c_?t_?|o_?f_?|o_?r_?|o_?u_?t_?|p_?r_?o_?c_?|p_?t_?r_?|r_?a_?i_?s_?e_?|r_?e_?f_?|r_?e_?t_?u_?r_?n_?|s_?h_?a_?r_?e_?d_?|s_?h_?l_?|s_?h_?r_?|s_?t_?a_?t_?i_?c_?|t_?e_?m_?p_?l_?a_?t_?e_?|t_?r_?y_?|t_?u_?p_?l_?e_?|t_?y_?p_?e_?|w_?h_?e_?n_?|w_?h_?i_?l_?e_?|w_?i_?t_?h_?|w_?i_?t_?h_?o_?u_?t_?|x_?o_?r_?)\b`, Keyword, nil},
+			{`(f_?r_?o_?m_?|i_?m_?p_?o_?r_?t_?|i_?n_?c_?l_?u_?d_?e_?)\b`, KeywordNamespace, nil},
+			{`(v_?a_?r)\b`, KeywordDeclaration, nil},
+			{`(i_?n_?t_?|i_?n_?t_?8_?|i_?n_?t_?1_?6_?|i_?n_?t_?3_?2_?|i_?n_?t_?6_?4_?|f_?l_?o_?a_?t_?|f_?l_?o_?a_?t_?3_?2_?|f_?l_?o_?a_?t_?6_?4_?|b_?o_?o_?l_?|c_?h_?a_?r_?|r_?a_?n_?g_?e_?|a_?r_?r_?a_?y_?|s_?e_?q_?|s_?e_?t_?|s_?t_?r_?i_?n_?g_?)\b`, KeywordType, nil},
+			{`(n_?i_?l_?|t_?r_?u_?e_?|f_?a_?l_?s_?e_?)\b`, KeywordPseudo, nil},
+			{`\b_\b`, Name, nil}, // Standalone _ used as discardable variable identifier
+			{`\b((?![_\d])\w)(((?!_)\w)|(_(?!_)\w))*`, Name, nil},
+			{`[0-9][0-9_]*(?=([e.]|\'f(32|64)))`, LiteralNumberFloat, Push("float-suffix", "float-number")},
+			{`0x[a-f0-9][a-f0-9_]*`, LiteralNumberHex, Push("int-suffix")},
+			{`0b[01][01_]*`, LiteralNumberBin, Push("int-suffix")},
+			{`0o[0-7][0-7_]*`, LiteralNumberOct, Push("int-suffix")},
+			{`[0-9][0-9_]*`, LiteralNumberInteger, Push("int-suffix")},
+			{`\s+`, Text, nil},
+			{`.+$`, Error, nil},
+		},
+		"chars": {
+			{`\\([\\abcefnrtvl"\']|x[a-f0-9]{2}|[0-9]{1,3})`, LiteralStringEscape, nil},
+			{`'`, LiteralStringChar, Pop(1)},
+			{`.`, LiteralStringChar, nil},
+		},
+		"strings": {
+			{`(?<!\$)\$(\d+|#|\w+)+`, LiteralStringInterpol, nil},
+			{`[^\\\'"$\n]+`, LiteralString, nil},
+			{`[\'"\\]`, LiteralString, nil},
+			{`\$`, LiteralString, nil},
+		},
+		"dqs": {
+			{`\\([\\abcefnrtvl"\']|\n|x[a-f0-9]{2}|[0-9]{1,3})`, LiteralStringEscape, nil},
+			{`"`, LiteralString, Pop(1)},
+			Include("strings"),
+		},
+		"rdqs": {
+			{`"(?!")`, LiteralString, Pop(1)},
+			{`""`, LiteralStringEscape, nil},
+			Include("strings"),
+		},
+		"tdqs": {
+			{`"""(?!")`, LiteralString, Pop(1)},
+			Include("strings"),
+			Include("nl"),
+		},
+		"funcname": {
+			{`((?![\d_])\w)(((?!_)\w)|(_(?!_)\w))*`, NameFunction, Pop(1)},
+			{"`.+`", NameFunction, Pop(1)},
+		},
+		"nl": {
+			{`\n`, LiteralString, nil},
+		},
+		"float-number": {
+			{`\.(?!\.)[0-9_]*`, LiteralNumberFloat, nil},
+			{`e[+-]?[0-9][0-9_]*`, LiteralNumberFloat, nil},
+			Default(Pop(1)),
+		},
+		"float-suffix": {
+			{`\'f(32|64)`, LiteralNumberFloat, nil},
+			Default(Pop(1)),
+		},
+		"int-suffix": {
+			{`\'i(32|64)`, LiteralNumberIntegerLong, nil},
+			{`\'i(8|16)`, LiteralNumberInteger, nil},
+			Default(Pop(1)),
+		},
+	},
+))

vendor/github.com/alecthomas/chroma/lexers/n/nix.go

@@ -0,0 +1,122 @@
+package n
+
+import (
+	"strings"
+
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// nixb matches right boundary of a nix word. Use it instead of \b.
+const nixb = `(?![a-zA-Z0-9_'-])`
+
+// Nix lexer.
+var Nix = internal.Register(MustNewLexer(
+	&Config{
+		Name:      "Nix",
+		Aliases:   []string{"nixos", "nix"},
+		Filenames: []string{"*.nix"},
+		MimeTypes: []string{"text/x-nix"},
+	},
+	Rules{
+		"root": {
+			Include("keywords"),
+			Include("builtins"),
+			// "./path" and ".float" literals have to be above "." operator
+			Include("literals"),
+			Include("operators"),
+			{`#.*$`, CommentSingle, nil},
+			{`/\*`, CommentMultiline, Push("comment")},
+			{`\(`, Punctuation, Push("paren")},
+			{`\[`, Punctuation, Push("list")},
+			{`"`, StringDouble, Push("qstring")},
+			{`''`, StringSingle, Push("istring")},
+			{`{`, Punctuation, Push("scope")},
+			{`let` + nixb, Keyword, Push("scope")},
+			Include("id"),
+			Include("space"),
+		},
+		"keywords": {
+			{`import` + nixb, KeywordNamespace, nil},
+			{Words(``, nixb, strings.Fields("rec inherit with if then else assert")...), Keyword, nil},
+		},
+		"builtins": {
+			{`throw` + nixb, NameException, nil},
+			{Words(``, nixb, strings.Fields("abort baseNameOf builtins currentTime dependencyClosure derivation dirOf fetchTarball filterSource getAttr getEnv hasAttr isNull map removeAttrs toString toXML")...), NameBuiltin, nil},
+		},
+		"literals": {
+			{Words(``, nixb, strings.Fields("true false null")...), NameConstant, nil},
+			Include("uri"),
+			Include("path"),
+			Include("int"),
+			Include("float"),
+		},
+		"operators": {
+			{` [/-] `, Operator, nil},
+			{`(\.)(\${)`, ByGroups(Operator, StringInterpol), Push("interpol")},
+			{`(\?)(\s*)(\${)`, ByGroups(Operator, Text, StringInterpol), Push("interpol")},
+			{Words(``, ``, strings.Fields("@ . ? ++ + != ! // == && || -> <= < >= > *")...), Operator, nil},
+			{`[;:]`, Punctuation, nil},
+		},
+		"comment": {
+			{`\*/`, CommentMultiline, Pop(1)},
+			{`.|\n`, CommentMultiline, nil},
+		},
+		"paren": {
+			{`\)`, Punctuation, Pop(1)},
+			Include("root"),
+		},
+		"list": {
+			{`\]`, Punctuation, Pop(1)},
+			Include("root"),
+		},
+		"qstring": {
+			{`"`, StringDouble, Pop(1)},
+			{`\${`, StringInterpol, Push("interpol")},
+			{`\\.`, StringEscape, nil},
+			{`.|\n`, StringDouble, nil},
+		},
+		"istring": {
+			{`''\$`, StringEscape, nil},  // "$"
+			{`'''`, StringEscape, nil},   // "''"
+			{`''\\.`, StringEscape, nil}, // "\."
+			{`''`, StringSingle, Pop(1)},
+			{`\${`, StringInterpol, Push("interpol")},
+			// The next rule is important: "$" escapes any symbol except "{"!
+			{`\$.`, StringSingle, nil}, // "$."
+			{`.|\n`, StringSingle, nil},
+		},
+		"scope": {
+			{`}:`, Punctuation, Pop(1)},
+			{`}`, Punctuation, Pop(1)},
+			{`in` + nixb, Keyword, Pop(1)},
+			{`\${`, StringInterpol, Push("interpol")},
+			Include("root"), // "==" has to be above "="
+			{Words(``, ``, strings.Fields("= ? ,")...), Operator, nil},
+		},
+		"interpol": {
+			{`}`, StringInterpol, Pop(1)},
+			Include("root"),
+		},
+		"id": {
+			{`[a-zA-Z_][a-zA-Z0-9_'-]*`, Name, nil},
+		},
+		"uri": {
+			{`[a-zA-Z][a-zA-Z0-9+.-]*:[a-zA-Z0-9%/?:@&=+$,_.!~*'-]+`, StringDoc, nil},
+		},
+		"path": {
+			{`[a-zA-Z0-9._+-]*(/[a-zA-Z0-9._+-]+)+`, StringRegex, nil},
+			{`~(/[a-zA-Z0-9._+-]+)+/?`, StringRegex, nil},
+			{`<[a-zA-Z0-9._+-]+(/[a-zA-Z0-9._+-]+)*>`, StringRegex, nil},
+		},
+		"int": {
+			{`-?[0-9]+` + nixb, NumberInteger, nil},
+		},
+		"float": {
+			{`-?(([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?` + nixb, NumberFloat, nil},
+		},
+		"space": {
+			{`[ \t\r\n]+`, Text, nil},
+		},
+	},
+))