| 1 | //! Syntax highlighting into monochrome CSS classes. |
| 2 | //! |
| 3 | //! The whole file is parsed in one pass (so multi-line strings and comments |
| 4 | //! are right), but the output is split into lines whose `<span>`s are |
| 5 | //! balanced: each line can be put in its own table row. |
| 6 | //! |
| 7 | //! Scopes collapse to a handful of classes (see `syntax.css`): |
| 8 | //! `hl-k` keywords/storage, `hl-d` names/numbers/constants, `hl-s` |
| 9 | //! strings/punctuation/operators, `hl-c` comments, `hl-x` invalid. |
| 10 | |
| 11 | use std::sync::{Arc, OnceLock}; |
| 12 | |
| 13 | use syntect::parsing::{ParseState, Scope, ScopeStack, SyntaxReference, SyntaxSet}; |
| 14 | use syntect::util::LinesWithEndings; |
| 15 | |
| 16 | use super::escape_html; |
| 17 | use crate::cache::{Cache, Key}; |
| 18 | use crate::git::ObjectId; |
| 19 | |
| 20 | static SYNTAXES: OnceLock<SyntaxSet> = OnceLock::new(); |
| 21 | |
| 22 | /// The embedded syntax set (bat's ~200 syntaxes). Loading takes a few tens |
| 23 | /// of milliseconds; call once at startup to keep it off the request path. |
| 24 | pub fn syntaxes() -> &'static SyntaxSet { |
| 25 | SYNTAXES.get_or_init(two_face::syntax::extra_newlines) |
| 26 | } |
| 27 | |
| 28 | /// Pick a syntax from the file name, falling back to the first line |
| 29 | /// (shebangs, modelines). `None` means plain text. |
| 30 | pub fn detect(path: &str, first_line: &str) -> Option<&'static SyntaxReference> { |
| 31 | let set = syntaxes(); |
| 32 | let name = path.rsplit('/').next().unwrap_or(path); |
| 33 | let found = set |
| 34 | .find_syntax_by_extension(name) |
| 35 | .or_else(|| { |
| 36 | let (_, ext) = name.rsplit_once('.')?; |
| 37 | set.find_syntax_by_extension(ext) |
| 38 | .or_else(|| set.find_syntax_by_extension(&ext.to_ascii_lowercase())) |
| 39 | }) |
| 40 | .or_else(|| set.find_syntax_by_first_line(first_line)); |
| 41 | found.filter(|s| s.name != "Plain Text") |
| 42 | } |
| 43 | |
| 44 | /// Syntax for a Markdown code fence info string (`rust`, `sh`, `py`, …). |
| 45 | pub fn for_token(token: &str) -> Option<&'static SyntaxReference> { |
| 46 | let token = token.split([' ', ',', '{']).next()?.trim(); |
| 47 | if token.is_empty() { |
| 48 | return None; |
| 49 | } |
| 50 | syntaxes() |
| 51 | .find_syntax_by_token(token) |
| 52 | .filter(|s| s.name != "Plain Text") |
| 53 | } |
| 54 | |
| 55 | struct Selectors { |
| 56 | comment: Scope, |
| 57 | invalid: Scope, |
| 58 | string: Scope, |
| 59 | punctuation: Scope, |
| 60 | operator: Scope, |
| 61 | keyword: Scope, |
| 62 | storage: Scope, |
| 63 | display: [Scope; 6], |
| 64 | } |
| 65 | |
| 66 | fn selectors() -> &'static Selectors { |
| 67 | static SELECTORS: OnceLock<Selectors> = OnceLock::new(); |
| 68 | SELECTORS.get_or_init(|| { |
| 69 | let s = |name: &str| Scope::new(name).expect("valid scope"); |
| 70 | Selectors { |
| 71 | comment: s("comment"), |
| 72 | invalid: s("invalid"), |
| 73 | string: s("string"), |
| 74 | punctuation: s("punctuation"), |
| 75 | operator: s("keyword.operator"), |
| 76 | keyword: s("keyword"), |
| 77 | storage: s("storage"), |
| 78 | display: [ |
| 79 | s("entity.name"), |
| 80 | s("support.function"), |
| 81 | s("support.type"), |
| 82 | s("support.class"), |
| 83 | s("constant.numeric"), |
| 84 | s("constant.language"), |
| 85 | ], |
| 86 | } |
| 87 | }) |
| 88 | } |
| 89 | |
| 90 | /// Class for text under `stack`: comments win outright, otherwise the |
| 91 | /// innermost scope with a class decides. |
| 92 | fn class_for(stack: &[Scope]) -> Option<&'static str> { |
| 93 | let sel = selectors(); |
| 94 | if stack.iter().any(|s| sel.comment.is_prefix_of(*s)) { |
| 95 | return Some("hl-c"); |
| 96 | } |
| 97 | for &scope in stack.iter().rev() { |
| 98 | if sel.invalid.is_prefix_of(scope) { |
| 99 | return Some("hl-x"); |
| 100 | } |
| 101 | if sel.operator.is_prefix_of(scope) |
| 102 | || sel.punctuation.is_prefix_of(scope) |
| 103 | || sel.string.is_prefix_of(scope) |
| 104 | { |
| 105 | return Some("hl-s"); |
| 106 | } |
| 107 | if sel.keyword.is_prefix_of(scope) || sel.storage.is_prefix_of(scope) { |
| 108 | return Some("hl-k"); |
| 109 | } |
| 110 | if sel.display.iter().any(|d| d.is_prefix_of(scope)) { |
| 111 | return Some("hl-d"); |
| 112 | } |
| 113 | } |
| 114 | None |
| 115 | } |
| 116 | |
| 117 | /// Appends text to a line, opening/closing spans only when the class changes. |
| 118 | struct LineWriter { |
| 119 | out: String, |
| 120 | open: Option<&'static str>, |
| 121 | } |
| 122 | |
| 123 | impl LineWriter { |
| 124 | fn push(&mut self, text: &str, class: Option<&'static str>) { |
| 125 | let text = text.trim_end_matches(['\n', '\r']); |
| 126 | if text.is_empty() { |
| 127 | return; |
| 128 | } |
| 129 | if class != self.open { |
| 130 | if self.open.is_some() { |
| 131 | self.out.push_str("</span>"); |
| 132 | } |
| 133 | if let Some(class) = class { |
| 134 | self.out.push_str("<span class=\""); |
| 135 | self.out.push_str(class); |
| 136 | self.out.push_str("\">"); |
| 137 | } |
| 138 | self.open = class; |
| 139 | } |
| 140 | escape_html(&mut self.out, text); |
| 141 | } |
| 142 | |
| 143 | fn finish(mut self) -> String { |
| 144 | if self.open.is_some() { |
| 145 | self.out.push_str("</span>"); |
| 146 | } |
| 147 | self.out |
| 148 | } |
| 149 | } |
| 150 | |
| 151 | /// Highlight `text` and return one HTML fragment per line. |
| 152 | /// |
| 153 | /// If the grammar fails part-way, the remaining lines are emitted plain. |
| 154 | pub fn lines(text: &str, syntax: &SyntaxReference) -> Vec<String> { |
| 155 | let set = syntaxes(); |
| 156 | let mut state = ParseState::new(syntax); |
| 157 | let mut stack = ScopeStack::new(); |
| 158 | let mut failed = false; |
| 159 | let mut out = Vec::new(); |
| 160 | for line in LinesWithEndings::from(text) { |
| 161 | let ops = if failed { |
| 162 | None |
| 163 | } else { |
| 164 | state.parse_line(line, set).ok() |
| 165 | }; |
| 166 | let Some(ops) = ops else { |
| 167 | failed = true; |
| 168 | out.push(plain_line(line)); |
| 169 | continue; |
| 170 | }; |
| 171 | let mut writer = LineWriter { |
| 172 | out: String::with_capacity(line.len() * 2), |
| 173 | open: None, |
| 174 | }; |
| 175 | let mut pos = 0; |
| 176 | for (at, op) in ops { |
| 177 | let at = at.min(line.len()); |
| 178 | if at > pos { |
| 179 | writer.push(&line[pos..at], class_for(stack.as_slice())); |
| 180 | pos = at; |
| 181 | } |
| 182 | if stack.apply(&op).is_err() { |
| 183 | failed = true; |
| 184 | } |
| 185 | } |
| 186 | writer.push(&line[pos..], class_for(stack.as_slice())); |
| 187 | out.push(writer.finish()); |
| 188 | } |
| 189 | out |
| 190 | } |
| 191 | |
| 192 | /// Lines longer than this (minified code) make syntect crawl; such files |
| 193 | /// are shown plain. |
| 194 | const MAX_LINE_BYTES: usize = 5000; |
| 195 | |
| 196 | /// HTML lines of blob `oid` (the file at `path`, contents `text`), |
| 197 | /// highlighted when a syntax is detected, the blob is at most `max_bytes` |
| 198 | /// and no line is minified-long, plain otherwise. Highlighted lines are cached by blob and |
| 199 | /// shared by every view of it. Also returns the detected language. |
| 200 | pub fn cached_lines( |
| 201 | cache: &Cache, |
| 202 | oid: ObjectId, |
| 203 | path: &str, |
| 204 | text: &str, |
| 205 | max_bytes: u64, |
| 206 | ) -> (Arc<Vec<String>>, Option<&'static str>) { |
| 207 | let first_line = text.lines().next().unwrap_or_default(); |
| 208 | let syntax = detect(path, first_line); |
| 209 | let language = syntax.map(|s| s.name.as_str()); |
| 210 | let lines = match syntax { |
| 211 | Some(syntax) |
| 212 | if text.len() as u64 <= max_bytes |
| 213 | && !text.lines().any(|l| l.len() > MAX_LINE_BYTES) => |
| 214 | { |
| 215 | let key = Key::Highlight { |
| 216 | blob: oid, |
| 217 | syntax: syntax.name.clone(), |
| 218 | }; |
| 219 | cache |
| 220 | .lines(key, || { |
| 221 | Ok::<_, std::convert::Infallible>(lines(text, syntax)) |
| 222 | }) |
| 223 | .unwrap_or_else(|never| match never {}) |
| 224 | } |
| 225 | _ => Arc::new(plain_lines(text)), |
| 226 | }; |
| 227 | (lines, language) |
| 228 | } |
| 229 | |
| 230 | /// Escaped lines without highlighting. |
| 231 | pub fn plain_lines(text: &str) -> Vec<String> { |
| 232 | LinesWithEndings::from(text).map(plain_line).collect() |
| 233 | } |
| 234 | |
| 235 | fn plain_line(line: &str) -> String { |
| 236 | let mut out = String::with_capacity(line.len()); |
| 237 | escape_html(&mut out, line.trim_end_matches(['\n', '\r'])); |
| 238 | out |
| 239 | } |
| 240 | |
| 241 | #[cfg(test)] |
| 242 | mod tests { |
| 243 | use super::*; |
| 244 | |
| 245 | #[test] |
| 246 | fn detects_by_name_extension_and_first_line() { |
| 247 | assert_eq!(detect("src/main.rs", "").unwrap().name, "Rust"); |
| 248 | assert_eq!(detect("Makefile", "").unwrap().name, "Makefile"); |
| 249 | assert_eq!(detect("a/b.PY", "").unwrap().name, "Python"); |
| 250 | assert_eq!( |
| 251 | detect("run", "#!/bin/bash").unwrap().name, |
| 252 | "Bourne Again Shell (bash)" |
| 253 | ); |
| 254 | assert!(detect("notes.txt", "hello").is_none()); |
| 255 | assert_eq!(for_token("rust").unwrap().name, "Rust"); |
| 256 | assert_eq!(for_token("sh").unwrap().name, "Bourne Again Shell (bash)"); |
| 257 | assert!(for_token("").is_none()); |
| 258 | } |
| 259 | |
| 260 | #[test] |
| 261 | fn spans_are_balanced_per_line_across_multiline_comments() { |
| 262 | let src = "fn main() {\n /* a\n b */ let x = \"<s>\";\n}\n"; |
| 263 | let lines = lines(src, detect("x.rs", "").unwrap()); |
| 264 | assert_eq!(lines.len(), 4); |
| 265 | for line in &lines { |
| 266 | assert_eq!( |
| 267 | line.matches("<span").count(), |
| 268 | line.matches("</span>").count(), |
| 269 | "{line}" |
| 270 | ); |
| 271 | } |
| 272 | assert!( |
| 273 | lines[0].contains(r#"<span class="hl-k">fn</span>"#), |
| 274 | "{}", |
| 275 | lines[0] |
| 276 | ); |
| 277 | // The comment's second line is still a comment. |
| 278 | assert!( |
| 279 | lines[2].starts_with(r#"<span class="hl-c"> b */"#), |
| 280 | "{}", |
| 281 | lines[2] |
| 282 | ); |
| 283 | // Escaped string contents. |
| 284 | assert!(lines[2].contains("<s>"), "{}", lines[2]); |
| 285 | assert!(!lines.iter().any(|l| l.contains('\n'))); |
| 286 | } |
| 287 | |
| 288 | #[test] |
| 289 | fn plain_lines_escape() { |
| 290 | assert_eq!(plain_lines("a<b\r\n\nc"), vec!["a<b", "", "c"]); |
| 291 | } |
| 292 | } |