Krolyxon
Commit a44bdf7

Branches

Tags

Refs →

remagesrcthumbs.rs

619 lines · 20.2 KB · Rust

Raw Blame History Download
1//! freedesktop thumbnail cache: lookup, validation, generation and atomic writes.
2//!
3//! Entries live in `$XDG_CACHE_HOME/thumbnails/<bucket>/<md5(uri)>.png`, validated by their
4//! `Thumb::URI` and `Thumb::MTime` text chunks. The cache is shared with file managers, so
5//! the URI must be spelled exactly as GLib spells it.
6
7use std::fs::{self, File};
8use std::io::{BufReader, BufWriter, Write};
9use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
10use std::path::{Component, Path, PathBuf};
11use std::time::UNIX_EPOCH;
12
13use anyhow::{Context, anyhow, bail};
14use fast_image_resize as fir;
15use image::metadata::Orientation;
16use md5::{Digest, Md5};
17
18use crate::decode::{self, DecodedImage};
19
20/// The freedesktop size buckets, smallest first.
21pub const BUCKETS: [(&str, u32); 4] = [
22 ("normal", 128),
23 ("large", 256),
24 ("x-large", 512),
25 ("xx-large", 1024),
26];
27
28/// The folder failed entries go in, under `fail/`.
29const APP: &str = "remage";
30
31/// A thumbnail ready for the atlas: straight-alpha RGBA8, at most `px` on its longer side.
32pub struct Thumb {
33 pub size: [u32; 2],
34 pub pixels: Vec<u8>,
35}
36
37impl std::fmt::Debug for Thumb {
38 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
39 f.debug_struct("Thumb").field("size", &self.size).finish()
40 }
41}
42
43#[derive(Debug, thiserror::Error)]
44pub enum ThumbError {
45 /// A `fail/remage` entry says this file can't be thumbnailed; don't retry.
46 #[error("thumbnailing failed before")]
47 KnownFailure,
48 #[error("{0:#}")]
49 Failed(anyhow::Error),
50}
51
52/// How a job uses the shared cache.
53#[derive(Clone, Debug)]
54pub struct CacheOptions {
55 /// `$XDG_CACHE_HOME/thumbnails`; `None` disables the cache entirely.
56 pub root: Option<PathBuf>,
57 /// Read but never write (`-p`).
58 pub private: bool,
59}
60
61impl CacheOptions {
62 pub fn new(enabled: bool, private: bool) -> Self {
63 Self {
64 root: enabled
65 .then(|| dirs::cache_dir().map(|d| d.join("thumbnails")))
66 .flatten(),
67 private,
68 }
69 }
70}
71
72/// The smallest bucket at least `px` wide, else the largest.
73pub fn bucket_for(px: u32) -> (&'static str, u32) {
74 BUCKETS
75 .iter()
76 .copied()
77 .find(|(_, s)| *s >= px)
78 .unwrap_or(BUCKETS[BUCKETS.len() - 1])
79}
80
81/// Make `path` absolute and drop `.` and `..` without resolving symlinks, as GLib does.
82pub fn normalize(path: &Path) -> PathBuf {
83 let abs = std::path::absolute(path).unwrap_or_else(|_| path.to_path_buf());
84 let mut out = PathBuf::new();
85 for c in abs.components() {
86 match c {
87 Component::CurDir => {}
88 Component::ParentDir => {
89 out.pop();
90 }
91 other => out.push(other),
92 }
93 }
94 out
95}
96
97/// `file://` URI of an absolute path, escaped like GLib's `g_filename_to_uri`.
98pub fn file_uri(path: &Path) -> String {
99 use std::os::unix::ffi::OsStrExt;
100 const HEX: &[u8; 16] = b"0123456789ABCDEF";
101 let mut uri = String::from("file://");
102 for &b in path.as_os_str().as_bytes() {
103 let keep = b.is_ascii_alphanumeric() || b"-._~!$&'()*+,;=:@/".contains(&b);
104 if keep {
105 uri.push(b as char);
106 } else {
107 uri.push('%');
108 uri.push(HEX[(b >> 4) as usize] as char);
109 uri.push(HEX[(b & 15) as usize] as char);
110 }
111 }
112 uri
113}
114
115/// The cache file name for a URI: its MD5 in lowercase hex, plus `.png`.
116pub fn cache_name(uri: &str) -> String {
117 let digest = Md5::digest(uri.as_bytes());
118 let mut s = String::with_capacity(36);
119 for b in digest {
120 s.push_str(&format!("{b:02x}"));
121 }
122 s.push_str(".png");
123 s
124}
125
126/// Seconds since the epoch, as `Thumb::MTime` stores it.
127fn mtime_secs(meta: &fs::Metadata) -> Option<u64> {
128 meta.modified()
129 .ok()?
130 .duration_since(UNIX_EPOCH)
131 .ok()
132 .map(|d| d.as_secs())
133}
134
135/// PNG text chunks as keyword–text pairs.
136type Text = Vec<(String, String)>;
137
138/// Text chunks of a PNG, and its pixels as RGBA8 if `pixels` is set.
139fn read_png(path: &Path, pixels: bool) -> anyhow::Result<(Text, Option<Thumb>)> {
140 let file = File::open(path)?;
141 let mut dec = png::Decoder::new(BufReader::new(file));
142 dec.set_transformations(png::Transformations::normalize_to_color8());
143 let mut reader = dec.read_info()?;
144 let mut thumb = None;
145 if pixels {
146 let mut buf = vec![0; reader.output_buffer_size().context("thumbnail too large")?];
147 let out = reader.next_frame(&mut buf)?;
148 buf.truncate(out.buffer_size());
149 let rgba = to_rgba(buf, out.color_type).context("unsupported thumbnail colour type")?;
150 thumb = Some(Thumb {
151 size: [out.width, out.height],
152 pixels: rgba,
153 });
154 // Text chunks after the image data.
155 let _ = reader.finish();
156 }
157 let info = reader.info();
158 let mut text: Text = info
159 .uncompressed_latin1_text
160 .iter()
161 .map(|c| (c.keyword.clone(), c.text.clone()))
162 .collect();
163 text.extend(
164 info.utf8_text
165 .iter()
166 .filter_map(|c| Some((c.keyword.clone(), c.get_text().ok()?))),
167 );
168 text.extend(
169 info.compressed_latin1_text
170 .iter()
171 .filter_map(|c| Some((c.keyword.clone(), c.get_text().ok()?))),
172 );
173 Ok((text, thumb))
174}
175
176fn to_rgba(buf: Vec<u8>, color: png::ColorType) -> Option<Vec<u8>> {
177 use png::ColorType::*;
178 Some(match color {
179 Rgba => buf,
180 Rgb => buf
181 .chunks_exact(3)
182 .flat_map(|p| [p[0], p[1], p[2], 255])
183 .collect(),
184 Grayscale => buf.iter().flat_map(|&g| [g, g, g, 255]).collect(),
185 GrayscaleAlpha => buf
186 .chunks_exact(2)
187 .flat_map(|p| [p[0], p[0], p[0], p[1]])
188 .collect(),
189 Indexed => return None,
190 })
191}
192
193/// Whether the text chunks describe `uri` at `mtime`. Some writers (ffmpegthumbnailer) store
194/// a bare path as the URI; that counts too.
195fn is_valid(text: &[(String, String)], uri: &str, mtime: u64) -> bool {
196 let get = |k: &str| text.iter().find(|(key, _)| key == k).map(|(_, v)| v.trim());
197 // Some writers store fractional seconds.
198 let mtime_ok = get("Thumb::MTime")
199 .and_then(|m| m.parse::<f64>().ok())
200 .is_some_and(|m| m.trunc() as u64 == mtime);
201 let path = uri.strip_prefix("file://").map(|p| {
202 let bytes = percent_decode(p);
203 String::from_utf8_lossy(&bytes).into_owned()
204 });
205 let uri_ok = get("Thumb::URI").is_some_and(|u| u == uri || Some(u) == path.as_deref());
206 uri_ok && mtime_ok
207}
208
209fn percent_decode(s: &str) -> Vec<u8> {
210 let b = s.as_bytes();
211 let mut out = Vec::with_capacity(b.len());
212 let mut i = 0;
213 while i < b.len() {
214 let hex = |c: u8| (c as char).to_digit(16);
215 if b[i] == b'%'
216 && let (Some(h), Some(l)) = (
217 b.get(i + 1).and_then(|&c| hex(c)),
218 b.get(i + 2).and_then(|&c| hex(c)),
219 )
220 {
221 out.push((h * 16 + l) as u8);
222 i += 3;
223 } else {
224 out.push(b[i]);
225 i += 1;
226 }
227 }
228 out
229}
230
231/// Write a PNG atomically (temporary file, then rename) with mode 0600.
232fn write_png(path: &Path, thumb: &Thumb, text: &[(&str, String)]) -> anyhow::Result<()> {
233 let dir = path.parent().context("cache path has no parent")?;
234 fs::DirBuilder::new()
235 .recursive(true)
236 .mode(0o700)
237 .create(dir)?;
238 let name = path.file_name().context("cache path has no name")?;
239 let tmp = dir.join(format!(
240 ".{}.remage-{}.tmp",
241 name.to_string_lossy(),
242 std::process::id()
243 ));
244 let result = (|| -> anyhow::Result<()> {
245 let file = fs::OpenOptions::new()
246 .write(true)
247 .create(true)
248 .truncate(true)
249 .mode(0o600)
250 .open(&tmp)?;
251 let mut w = BufWriter::new(file);
252 let mut enc = png::Encoder::new(&mut w, thumb.size[0], thumb.size[1]);
253 enc.set_color(png::ColorType::Rgba);
254 enc.set_depth(png::BitDepth::Eight);
255 enc.set_compression(png::Compression::Fast);
256 for (k, v) in text {
257 enc.add_text_chunk(k.to_string(), v.clone())?;
258 }
259 let mut writer = enc.write_header()?;
260 writer.write_image_data(&thumb.pixels)?;
261 writer.finish()?;
262 w.flush()?;
263 drop(w);
264 fs::rename(&tmp, path)?;
265 Ok(())
266 })();
267 if result.is_err() {
268 let _ = fs::remove_file(&tmp);
269 }
270 result
271}
272
273/// Shrink `src` so its longer side is at most `max` (never enlarging).
274pub fn resize(src: Thumb, max: u32) -> anyhow::Result<Thumb> {
275 let [w, h] = src.size;
276 if w <= max && h <= max {
277 return Ok(src);
278 }
279 let k = max as f64 / w.max(h) as f64;
280 let nw = ((w as f64 * k).round() as u32).clamp(1, max);
281 let nh = ((h as f64 * k).round() as u32).clamp(1, max);
282 let mut img = fir::images::Image::from_vec_u8(w, h, src.pixels, fir::PixelType::U8x4)
283 .context("resize source")?;
284 // Resample in premultiplied alpha so transparent edges don't bleed dark fringes.
285 let mul = fir::MulDiv::default();
286 mul.multiply_alpha_inplace(&mut img)
287 .context("premultiply")?;
288 let mut dst = fir::images::Image::new(nw, nh, fir::PixelType::U8x4);
289 let opts = fir::ResizeOptions::new()
290 .resize_alg(fir::ResizeAlg::Convolution(fir::FilterType::Lanczos3))
291 .use_alpha(false);
292 fir::Resizer::new()
293 .resize(&img, &mut dst, &opts)
294 .context("resize")?;
295 mul.divide_alpha_inplace(&mut dst)
296 .context("unpremultiply")?;
297 Ok(Thumb {
298 size: [nw, nh],
299 pixels: dst.into_vec(),
300 })
301}
302
303/// Load a thumbnail of `path` at most `px` on its longer side: from the shared cache when a
304/// valid entry is big enough, otherwise generated (and written back unless private).
305pub fn load(path: &Path, px: u32, cache: &CacheOptions) -> Result<Thumb, ThumbError> {
306 let fail = |e: anyhow::Error| ThumbError::Failed(e);
307 let meta = fs::metadata(path).map_err(|e| fail(e.into()))?;
308 let mtime = mtime_secs(&meta).unwrap_or(0);
309 let abs = normalize(path);
310 let uri = file_uri(&abs);
311 let name = cache_name(&uri);
312 let (bucket, bucket_px) = bucket_for(px);
313
314 if let Some(root) = &cache.root {
315 // Our bucket, or any larger one.
316 for (dir, _) in BUCKETS.iter().filter(|(_, s)| *s >= bucket_px) {
317 let p = root.join(dir).join(&name);
318 if let Ok((text, Some(t))) = read_png(&p, true)
319 && is_valid(&text, &uri, mtime)
320 {
321 return resize(t, px).map_err(fail);
322 }
323 }
324 // Only trust failures this version recorded: a later one may decode more formats.
325 let failed = root.join("fail").join(APP).join(&name);
326 if let Ok((text, _)) = read_png(&failed, false)
327 && is_valid(&text, &uri, mtime)
328 && text
329 .iter()
330 .any(|(k, v)| k == "Software" && *v == software())
331 {
332 return Err(ThumbError::KnownFailure);
333 }
334 }
335
336 let generated = match generate(path, bucket_px) {
337 Ok(t) => t,
338 Err(e) => {
339 if let Some(root) = cache.root.as_ref().filter(|_| !cache.private) {
340 let failed = root.join("fail").join(APP).join(&name);
341 let empty = Thumb {
342 size: [1, 1],
343 pixels: vec![0; 4],
344 };
345 let text = [
346 ("Thumb::URI", uri.clone()),
347 ("Thumb::MTime", mtime.to_string()),
348 ("Software", software()),
349 ];
350 if let Err(e) = write_png(&failed, &empty, &text) {
351 log::debug!("{}: {e:#}", failed.display());
352 }
353 }
354 return Err(fail(e));
355 }
356 };
357
358 // Images no bigger than the bucket gain nothing from a cache entry.
359 let worth_caching = generated.source_size.iter().any(|&s| s > bucket_px);
360 let thumb = Thumb {
361 size: generated.size,
362 pixels: generated.pixels,
363 };
364 if let Some(root) = cache.root.as_ref().filter(|_| !cache.private)
365 && worth_caching
366 && !abs.starts_with(root)
367 {
368 let dest = root.join(bucket).join(&name);
369 let [sw, sh] = generated.source_size;
370 let text = [
371 ("Thumb::URI", uri.clone()),
372 ("Thumb::MTime", mtime.to_string()),
373 ("Thumb::Size", meta.len().to_string()),
374 ("Thumb::Image::Width", sw.to_string()),
375 ("Thumb::Image::Height", sh.to_string()),
376 ("Software", software()),
377 ];
378 if let Err(e) = write_png(&dest, &thumb, &text) {
379 log::debug!("{}: {e:#}", dest.display());
380 }
381 }
382 resize(thumb, px).map_err(fail)
383}
384
385fn software() -> String {
386 format!("remage {}", env!("CARGO_PKG_VERSION"))
387}
388
389/// A freshly made thumbnail at most `max` on its longer side.
390struct Generated {
391 size: [u32; 2],
392 pixels: Vec<u8>,
393 /// The original's upright size.
394 source_size: [u32; 2],
395}
396
397fn generate(path: &Path, max: u32) -> anyhow::Result<Generated> {
398 // JPEG: decode at 1/2, 1/4 or 1/8 straight from the DCT.
399 if let Some(g) = jpeg_scaled(path, max)? {
400 return Ok(g);
401 }
402 // Scalable and previewed sources (SVG, RAW) only need the bucket's size.
403 let opts = decode::Options {
404 target: max,
405 ..decode::Options::default()
406 };
407 let img = decode::decode(path, &opts)?;
408 let source_size = img.source_size;
409 let DecodedImage {
410 size, mut frames, ..
411 } = img;
412 let pixels = frames.swap_remove(0).pixels;
413 let t = resize(Thumb { size, pixels }, max)?;
414 Ok(Generated {
415 size: t.size,
416 pixels: t.pixels,
417 source_size,
418 })
419}
420
421/// DCT-scaled JPEG decode, or `None` when the file isn't a plain 8-bit RGB/grey JPEG.
422fn jpeg_scaled(path: &Path, max: u32) -> anyhow::Result<Option<Generated>> {
423 let mut magic = [0u8; 3];
424 {
425 use std::io::Read;
426 let mut f = File::open(path)?;
427 if f.read_exact(&mut magic).is_err() || magic != [0xFF, 0xD8, 0xFF] {
428 return Ok(None);
429 }
430 }
431 let mut dec = jpeg_decoder::Decoder::new(BufReader::new(File::open(path)?));
432 dec.read_info()?;
433 let info = dec.info().ok_or_else(|| anyhow!("no JPEG header"))?;
434 use jpeg_decoder::PixelFormat::*;
435 if !matches!(info.pixel_format, L8 | RGB24) {
436 return Ok(None);
437 }
438 let (w, h) = (info.width as u32, info.height as u32);
439 let k = (max as f64 / w.max(h) as f64).min(1.0);
440 let req = |v: u32| ((v as f64 * k).ceil() as u32).clamp(1, u16::MAX as u32) as u16;
441 let (sw, sh) = dec.scale(req(w), req(h))?;
442 let data = dec.decode()?;
443 let pixels: Vec<u8> = match info.pixel_format {
444 L8 => data.iter().flat_map(|&g| [g, g, g, 255]).collect(),
445 _ => data
446 .chunks_exact(3)
447 .flat_map(|p| [p[0], p[1], p[2], 255])
448 .collect(),
449 };
450 let (sw, sh) = (sw as u32, sh as u32);
451 if pixels.len() != (sw * sh * 4) as usize {
452 bail!("JPEG decoded to an unexpected size");
453 }
454 let orientation = dec
455 .exif_data()
456 .and_then(Orientation::from_exif_chunk)
457 .unwrap_or(Orientation::NoTransforms);
458 let mut img = image::DynamicImage::ImageRgba8(
459 image::RgbaImage::from_raw(sw, sh, pixels).context("JPEG buffer size")?,
460 );
461 img.apply_orientation(orientation);
462 let source_size = if img.width() == sw { [w, h] } else { [h, w] };
463 let rgba = img.into_rgba8();
464 let size = [rgba.width(), rgba.height()];
465 let t = resize(
466 Thumb {
467 size,
468 pixels: rgba.into_raw(),
469 },
470 max,
471 )?;
472 Ok(Some(Generated {
473 size: t.size,
474 pixels: t.pixels,
475 source_size,
476 }))
477}
478
479#[cfg(test)]
480mod tests {
481 use super::*;
482
483 fn tmp(name: &str) -> PathBuf {
484 let dir = std::env::temp_dir().join(format!("remage-thumbs-{}-{name}", std::process::id()));
485 let _ = fs::remove_dir_all(&dir);
486 fs::create_dir_all(&dir).unwrap();
487 dir
488 }
489
490 #[test]
491 fn uris_are_escaped_like_glib() {
492 assert_eq!(
493 file_uri(Path::new("/home/u/a b/ü#?%.png")),
494 "file:///home/u/a%20b/%C3%BC%23%3F%25.png"
495 );
496 assert_eq!(
497 file_uri(Path::new("/x/(1)!$&'*+,;=:@~-_.png")),
498 "file:///x/(1)!$&'*+,;=:@~-_.png"
499 );
500 }
501
502 #[test]
503 fn names_are_md5_of_the_uri() {
504 // The spec's own example.
505 assert_eq!(
506 cache_name("file:///home/jens/photos/me.png"),
507 "c6ee772d9e49320e97ec29a7eb5b1697.png"
508 );
509 }
510
511 #[test]
512 fn normalize_is_lexical() {
513 assert_eq!(
514 normalize(Path::new("/a/./b/../c.png")),
515 Path::new("/a/c.png")
516 );
517 assert!(normalize(Path::new("x.png")).is_absolute());
518 }
519
520 #[test]
521 fn buckets() {
522 assert_eq!(bucket_for(96), ("normal", 128));
523 assert_eq!(bucket_for(128), ("normal", 128));
524 assert_eq!(bucket_for(160), ("large", 256));
525 assert_eq!(bucket_for(320), ("x-large", 512));
526 assert_eq!(bucket_for(2000), ("xx-large", 1024));
527 }
528
529 #[test]
530 fn generates_writes_and_rereads() {
531 let dir = tmp("cache");
532 let img = dir.join("pic.png");
533 image::RgbaImage::from_fn(600, 300, |x, _| image::Rgba([(x % 256) as u8, 0, 0, 255]))
534 .save(&img)
535 .unwrap();
536 let cache = CacheOptions {
537 root: Some(dir.join("thumbnails")),
538 private: false,
539 };
540 let t = load(&img, 160, &cache).unwrap();
541 assert_eq!(t.size, [160, 80]);
542 let entry = dir
543 .join("thumbnails/large")
544 .join(cache_name(&file_uri(&normalize(&img))));
545 let (text, stored) = read_png(&entry, true).unwrap();
546 assert_eq!(stored.unwrap().size, [256, 128]);
547 assert!(
548 text.iter()
549 .any(|(k, v)| k == "Thumb::Image::Width" && v == "600")
550 );
551 {
552 use std::os::unix::fs::PermissionsExt;
553 let mode = fs::metadata(&entry).unwrap().permissions().mode();
554 assert_eq!(mode & 0o777, 0o600);
555 }
556 // A second load reads the entry (same result), a smaller size downscales it.
557 assert_eq!(load(&img, 160, &cache).unwrap().size, [160, 80]);
558 assert_eq!(load(&img, 96, &cache).unwrap().size, [96, 48]);
559 // A stale entry (different mtime) is ignored and replaced.
560 let text = vec![
561 ("Thumb::URI".to_string(), file_uri(&normalize(&img))),
562 ("Thumb::MTime".to_string(), "1".to_string()),
563 ];
564 let meta = fs::metadata(&img).unwrap();
565 assert!(!is_valid(&text, &text[0].1, mtime_secs(&meta).unwrap()));
566 assert!(is_valid(&text, &text[0].1, 1));
567 // A bare path as the URI is accepted too.
568 let bare = vec![
569 ("Thumb::URI".to_string(), "/a b/c.png".to_string()),
570 ("Thumb::MTime".to_string(), "1".to_string()),
571 ];
572 assert!(is_valid(&bare, "file:///a%20b/c.png", 1));
573 fs::remove_dir_all(&dir).unwrap();
574 }
575
576 #[test]
577 fn failures_are_remembered_and_private_writes_nothing() {
578 let dir = tmp("fail");
579 let bad = dir.join("bad.png");
580 fs::write(&bad, b"not a png").unwrap();
581 let cache = CacheOptions {
582 root: Some(dir.join("thumbnails")),
583 private: true,
584 };
585 assert!(matches!(
586 load(&bad, 128, &cache),
587 Err(ThumbError::Failed(_))
588 ));
589 assert!(!dir.join("thumbnails").exists());
590 let cache = CacheOptions {
591 private: false,
592 ..cache
593 };
594 assert!(matches!(
595 load(&bad, 128, &cache),
596 Err(ThumbError::Failed(_))
597 ));
598 assert!(matches!(
599 load(&bad, 128, &cache),
600 Err(ThumbError::KnownFailure)
601 ));
602 fs::remove_dir_all(&dir).unwrap();
603 }
604
605 #[test]
606 fn jpeg_is_scaled_and_oriented() {
607 let dir = tmp("jpeg");
608 let p = dir.join("a.jpg");
609 image::RgbImage::from_pixel(800, 400, image::Rgb([10, 200, 30]))
610 .save(&p)
611 .unwrap();
612 let g = jpeg_scaled(&p, 128).unwrap().unwrap();
613 assert_eq!(g.source_size, [800, 400]);
614 assert_eq!(g.size, [128, 64]);
615 let px = &g.pixels[..4];
616 assert!(px[1] > 150 && px[0] < 60, "{px:?}");
617 fs::remove_dir_all(&dir).unwrap();
618 }
619}