Krolyxon
Branch master

Branches

Tags

Refs →

ferreniumsrcindexlanguages.rs

410 lines · 13.3 KB · Rust

Raw Blame History Download
1//! Language statistics, the way GitHub counts them: bytes per language
2//! over the files of a tree, counting only programming and markup
3//! languages, and leaving out vendored, generated and documentation paths.
4//! `linguist-*` attributes in the root `.gitattributes` override all of
5//! that.
6//!
7//! Results are cached by tree id and computed in the background by
8//! [`warm`], so pages never wait for them.
9
10use std::collections::HashMap;
11use std::sync::{Arc, Mutex, OnceLock};
12use std::time::Duration;
13
14use fancy_regex::Regex;
15use gix::bstr::BStr;
16use gix::glob::wildmatch;
17
18use super::languages_table::{DOCUMENTATION, EXTENSIONS, FILENAMES, LANGUAGES, VENDOR};
19use crate::cache::Cache;
20use crate::git::tree::{self, EntryKind, PathTarget};
21use crate::git::{self, ObjectId};
22use crate::state::AppState;
23
24#[derive(Debug, Clone, Copy, PartialEq, Eq)]
25pub enum Kind {
26 Programming,
27 Markup,
28 Data,
29 Prose,
30}
31
32pub struct Language {
33 pub name: &'static str,
34 pub kind: Kind,
35 /// Parent language, for statistics.
36 pub group: Option<u16>,
37}
38
39/// Bytes per language, largest first.
40#[derive(Debug, Clone, Default, PartialEq, Eq)]
41pub struct Stats {
42 pub languages: Vec<(&'static str, u64)>,
43 pub total: u64,
44}
45
46impl Stats {
47 pub fn weight(&self) -> usize {
48 64 + self.languages.len() * 32
49 }
50
51 pub fn primary(&self) -> Option<&'static str> {
52 self.languages.first().map(|(name, _)| *name)
53 }
54}
55
56/// Language of a path from its file name, then its extensions (longest
57/// first: `x.d.ts` tries `.d.ts`, then `.ts`).
58pub fn detect(path: &str) -> Option<&'static Language> {
59 let name = path.rsplit('/').next().unwrap_or(path);
60 if let Ok(i) = FILENAMES.binary_search_by(|(f, _)| (*f).cmp(name)) {
61 return Some(&LANGUAGES[FILENAMES[i].1 as usize]);
62 }
63 let lower = name.to_ascii_lowercase();
64 // A leading dot is a hidden file, not an extension (`.bashrc`).
65 for (i, _) in lower.match_indices('.').filter(|(i, _)| *i > 0) {
66 let ext = &lower[i..];
67 if let Ok(j) = EXTENSIONS.binary_search_by(|(e, _)| (*e).cmp(ext)) {
68 return Some(&LANGUAGES[EXTENSIONS[j].1 as usize]);
69 }
70 }
71 None
72}
73
74/// Paths of generated files (beyond Linguist's vendor/documentation lists).
75const GENERATED: &[&str] = &[
76 r"\.min\.(js|css|mjs)$",
77 r"-min\.js$",
78 r"\.(js|css)\.map$",
79 r"\.pb\.(go|cc|h)$",
80 r"_pb2(_grpc)?\.py$",
81 r"\.designer\.(cs|vb)$",
82 r"(^|/)(package-lock\.json|yarn\.lock|pnpm-lock\.yaml|Cargo\.lock|composer\.lock|Gemfile\.lock|poetry\.lock|go\.sum)$",
83];
84
85struct Rules {
86 vendor: Regex,
87 documentation: Regex,
88 generated: Regex,
89}
90
91fn alternation(patterns: &[&str]) -> Regex {
92 let joined = patterns
93 .iter()
94 .map(|p| format!("(?:{p})"))
95 .collect::<Vec<_>>()
96 .join("|");
97 Regex::new(&joined).expect("Linguist patterns compile")
98}
99
100fn rules() -> &'static Rules {
101 static RULES: OnceLock<Rules> = OnceLock::new();
102 RULES.get_or_init(|| Rules {
103 vendor: alternation(VENDOR),
104 documentation: alternation(DOCUMENTATION),
105 generated: alternation(GENERATED),
106 })
107}
108
109fn matches(re: &Regex, path: &str) -> bool {
110 re.is_match(path).unwrap_or(false)
111}
112
113#[derive(Debug, Clone, PartialEq, Eq)]
114enum Value {
115 Set,
116 Unset,
117 Is(String),
118}
119
120impl Value {
121 fn truthy(&self) -> bool {
122 match self {
123 Value::Set => true,
124 Value::Unset => false,
125 Value::Is(v) => !matches!(v.as_str(), "false" | "0" | "no"),
126 }
127 }
128}
129
130/// `linguist-*` rules from a `.gitattributes`.
131#[derive(Debug, Default)]
132pub struct Attributes {
133 rules: Vec<(String, Vec<(String, Value)>)>,
134}
135
136impl Attributes {
137 pub fn parse(text: &str) -> Self {
138 let mut rules = Vec::new();
139 for line in text.lines() {
140 let line = line.trim();
141 if line.is_empty() || line.starts_with('#') {
142 continue;
143 }
144 let mut parts = line.split_whitespace();
145 let Some(pattern) = parts.next() else {
146 continue;
147 };
148 let attrs: Vec<(String, Value)> = parts
149 .filter_map(|a| {
150 let (name, value) = if let Some(name) = a.strip_prefix('-') {
151 (name, Value::Unset)
152 } else if let Some(name) = a.strip_prefix('!') {
153 (name, Value::Unset)
154 } else if let Some((name, v)) = a.split_once('=') {
155 (name, Value::Is(v.to_string()))
156 } else {
157 (a, Value::Set)
158 };
159 name.starts_with("linguist-")
160 .then(|| (name.to_string(), value))
161 })
162 .collect();
163 if !attrs.is_empty() {
164 rules.push((pattern.to_string(), attrs));
165 }
166 }
167 Attributes { rules }
168 }
169
170 fn is_empty(&self) -> bool {
171 self.rules.is_empty()
172 }
173
174 /// The last matching value of `attr` for `path`.
175 fn get(&self, path: &str, attr: &str) -> Option<&Value> {
176 let name = path.rsplit('/').next().unwrap_or(path);
177 self.rules
178 .iter()
179 .rev()
180 .filter(|(pattern, _)| {
181 let pattern = pattern.strip_prefix('/').unwrap_or(pattern);
182 if pattern.contains('/') {
183 wildmatch(
184 BStr::new(pattern),
185 BStr::new(path),
186 wildmatch::Mode::NO_MATCH_SLASH_LITERAL,
187 )
188 } else {
189 wildmatch(
190 BStr::new(pattern),
191 BStr::new(name),
192 wildmatch::Mode::empty(),
193 )
194 }
195 })
196 .find_map(|(_, attrs)| {
197 attrs
198 .iter()
199 .rev()
200 .find(|(name, _)| name == attr)
201 .map(|(_, v)| v)
202 })
203 }
204
205 fn flag(&self, path: &str, attr: &str) -> Option<bool> {
206 self.get(path, attr).map(Value::truthy)
207 }
208}
209
210/// Whether a file counts, and as which language.
211fn classify(path: &str, attrs: &Attributes) -> Option<&'static str> {
212 let rules = rules();
213 let excluded =
214 |attr: &str, re: &Regex| attrs.flag(path, attr).unwrap_or_else(|| matches(re, path));
215 if excluded("linguist-vendored", &rules.vendor)
216 || excluded("linguist-generated", &rules.generated)
217 || excluded("linguist-documentation", &rules.documentation)
218 {
219 return None;
220 }
221 let forced = match attrs.get(path, "linguist-language") {
222 Some(Value::Is(name)) => LANGUAGES.iter().find(|l| {
223 l.name.eq_ignore_ascii_case(&name.replace('-', " "))
224 || l.name.eq_ignore_ascii_case(name)
225 }),
226 _ => None,
227 };
228 let language = forced.or_else(|| detect(path))?;
229 let detectable = attrs
230 .flag(path, "linguist-detectable")
231 .unwrap_or(matches!(language.kind, Kind::Programming | Kind::Markup));
232 if !detectable {
233 return None;
234 }
235 let counted = language.group.map_or(language, |g| &LANGUAGES[g as usize]);
236 Some(counted.name)
237}
238
239/// Language statistics of the tree `tree` (with the root `.gitattributes`
240/// of that tree).
241pub fn compute(repo: &git::Handle, tree_id: ObjectId) -> anyhow::Result<Stats> {
242 let attrs = match tree::list(repo, tree_id)?
243 .iter()
244 .find(|e| e.name == ".gitattributes" && e.kind.is_blob())
245 {
246 Some(entry) => Attributes::parse(&String::from_utf8_lossy(&tree::blob(repo, entry.oid)?)),
247 None => Attributes::default(),
248 };
249 let rules = rules();
250 // Directories can be skipped wholesale unless attributes might bring
251 // something inside back.
252 let prune = attrs.is_empty();
253 let mut bytes: HashMap<&'static str, u64> = HashMap::new();
254 tree::walk(
255 repo,
256 tree_id,
257 &mut |dir| !(prune && (matches(&rules.vendor, dir) || matches(&rules.documentation, dir))),
258 &mut |path, kind, size| {
259 if kind == EntryKind::Symlink {
260 return;
261 }
262 if let Some(name) = classify(path, &attrs) {
263 *bytes.entry(name).or_default() += size;
264 }
265 },
266 )?;
267 let mut languages: Vec<(&'static str, u64)> =
268 bytes.into_iter().filter(|(_, n)| *n > 0).collect();
269 languages.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0)));
270 let total = languages.iter().map(|(_, n)| n).sum();
271 Ok(Stats { languages, total })
272}
273
274/// Background state: wake-ups and each repository's main language (for
275/// the index page).
276#[derive(Default)]
277pub struct Warmer {
278 pub wake: tokio::sync::Notify,
279 primary: Mutex<HashMap<String, &'static str>>,
280}
281
282impl Warmer {
283 pub fn primary(&self, repo: &str) -> Option<&'static str> {
284 self.primary
285 .lock()
286 .unwrap_or_else(|e| e.into_inner())
287 .get(repo)
288 .copied()
289 }
290
291 fn set_primary(&self, repo: &str, language: Option<&'static str>) {
292 let mut map = self.primary.lock().unwrap_or_else(|e| e.into_inner());
293 match language {
294 Some(l) => {
295 map.insert(repo.to_string(), l);
296 }
297 None => {
298 map.remove(repo);
299 }
300 }
301 }
302}
303
304/// Root tree of a commit.
305pub fn root_tree(repo: &git::Handle, commit: ObjectId) -> anyhow::Result<ObjectId> {
306 match tree::lookup(repo, commit, b"")? {
307 Some(PathTarget::Tree(id)) => Ok(id),
308 _ => anyhow::bail!("commit has no tree"),
309 }
310}
311
312/// Cached statistics of `tree`, without computing them.
313pub fn cached(cache: &Cache, tree: ObjectId) -> Option<Arc<Stats>> {
314 cache.peek_languages(tree)
315}
316
317/// How often the warmer looks for repositories without statistics, besides
318/// being woken by page views that found none.
319const WARM_INTERVAL: Duration = Duration::from_secs(60);
320
321/// Compute missing statistics for every repository's default branch, one
322/// repository at a time, forever.
323pub async fn warm(state: Arc<AppState>) {
324 loop {
325 let index = state.index.load();
326 for repo in index.repos.values() {
327 let Some(head) = repo.meta.head else {
328 state.warmer.set_primary(&repo.name, None);
329 continue;
330 };
331 let repo = repo.clone();
332 let result = state
333 .blocking(move |state| -> anyhow::Result<()> {
334 let git = git::handle(&repo.gix);
335 let tree = root_tree(&git, head)?;
336 let stats = match state.cache.peek_languages(tree) {
337 Some(stats) => stats,
338 None => {
339 let start = std::time::Instant::now();
340 let stats = state.cache.languages(tree, || compute(&git, tree))?;
341 tracing::debug!(
342 "{}: language stats in {:?}",
343 repo.name,
344 start.elapsed()
345 );
346 stats
347 }
348 };
349 state.warmer.set_primary(&repo.name, stats.primary());
350 Ok(())
351 })
352 .await;
353 match result {
354 Ok(Ok(())) => {}
355 Ok(Err(e)) | Err(e) => tracing::warn!("language stats: {e:#}"),
356 }
357 }
358 tokio::select! {
359 _ = state.warmer.wake.notified() => {}
360 _ = tokio::time::sleep(WARM_INTERVAL) => {}
361 }
362 }
363}
364
365#[cfg(test)]
366mod tests {
367 use super::*;
368
369 #[test]
370 fn detection() {
371 let name = |p: &str| detect(p).map(|l| l.name);
372 assert_eq!(name("src/main.rs"), Some("Rust"));
373 assert_eq!(name("Makefile"), Some("Makefile"));
374 assert_eq!(name("a/b/Dockerfile"), Some("Dockerfile"));
375 assert_eq!(name("x.PY"), Some("Python"));
376 assert_eq!(name("types.d.ts"), Some("TypeScript"));
377 assert_eq!(name("lib.h"), Some("C"));
378 assert_eq!(name(".bashrc"), Some("Shell"));
379 assert_eq!(name("LICENSE"), Some("Text"));
380 assert_eq!(name("no-extension"), None);
381 }
382
383 #[test]
384 fn exclusions_and_attributes() {
385 let none = Attributes::default();
386 assert_eq!(classify("src/main.rs", &none), Some("Rust"));
387 assert_eq!(classify("vendor/lib.rs", &none), None);
388 assert_eq!(classify("node_modules/x/index.js", &none), None);
389 assert_eq!(classify("docs/conf.py", &none), None);
390 assert_eq!(classify("app.min.js", &none), None);
391 assert_eq!(classify("config.json", &none), None, "data");
392 assert_eq!(classify("README.md", &none), None, "prose");
393
394 let attrs = Attributes::parse(
395 "# comment\n*.json linguist-detectable\nvendor/** -linguist-vendored\n\
396 gen/*.rs linguist-generated=true\n*.inc linguist-language=C++\n",
397 );
398 assert_eq!(classify("config.json", &attrs), Some("JSON"));
399 assert_eq!(classify("vendor/lib.rs", &attrs), Some("Rust"));
400 assert_eq!(classify("gen/x.rs", &attrs), None);
401 assert_eq!(classify("a/b.inc", &attrs), Some("C++"));
402 }
403
404 #[test]
405 fn all_linguist_patterns_compile() {
406 let r = rules();
407 assert!(matches(&r.vendor, "third_party/x.c"));
408 assert!(matches(&r.documentation, "Documentation/index.rst"));
409 }
410}