Krolyxon
Commit 74a9b56

Branches

Tags

Refs →

remagesrcthumbs.rs

625 lines · 20.4 KB · Rust

Raw Blame History Download
1//! freedesktop thumbnail cache: lookup, validation, generation and atomic writes.
2//!
3//! Entries live in `$XDG_CACHE_HOME/thumbnails/<bucket>/<md5(uri)>.png`, validated by their
4//! `Thumb::URI` and `Thumb::MTime` text chunks. The cache is shared with file managers, so
5//! the URI must be spelled exactly as GLib spells it.
6
7use std::fs::{self, File};
8use std::io::{BufReader, BufWriter, Write};
9use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
10use std::path::{Component, Path, PathBuf};
11use std::time::UNIX_EPOCH;
12
13use anyhow::{Context, anyhow, bail};
14use fast_image_resize as fir;
15use image::metadata::Orientation;
16use md5::{Digest, Md5};
17
18use crate::decode::{self, DecodedImage};
19
20/// The freedesktop size buckets, smallest first.
21pub const BUCKETS: [(&str, u32); 4] = [
22 ("normal", 128),
23 ("large", 256),
24 ("x-large", 512),
25 ("xx-large", 1024),
26];
27
28/// The folder failed entries go in, under `fail/`.
29const APP: &str = "remage";
30
31/// A thumbnail ready for the atlas: straight-alpha RGBA8, at most `px` on its longer side.
32pub struct Thumb {
33 pub size: [u32; 2],
34 pub pixels: Vec<u8>,
35}
36
37impl std::fmt::Debug for Thumb {
38 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
39 f.debug_struct("Thumb").field("size", &self.size).finish()
40 }
41}
42
43#[derive(Debug, thiserror::Error)]
44pub enum ThumbError {
45 /// A `fail/remage` entry says this file can't be thumbnailed; don't retry.
46 #[error("thumbnailing failed before")]
47 KnownFailure,
48 #[error("{0:#}")]
49 Failed(anyhow::Error),
50}
51
52/// How a job uses the shared cache.
53#[derive(Clone, Debug)]
54pub struct CacheOptions {
55 /// `$XDG_CACHE_HOME/thumbnails`; `None` disables the cache entirely.
56 pub root: Option<PathBuf>,
57 /// Read but never write (`-p`).
58 pub private: bool,
59}
60
61impl CacheOptions {
62 pub fn new(enabled: bool, private: bool) -> Self {
63 Self {
64 root: enabled
65 .then(|| dirs::cache_dir().map(|d| d.join("thumbnails")))
66 .flatten(),
67 private,
68 }
69 }
70}
71
72/// The smallest bucket at least `px` wide, else the largest.
73pub fn bucket_for(px: u32) -> (&'static str, u32) {
74 BUCKETS
75 .iter()
76 .copied()
77 .find(|(_, s)| *s >= px)
78 .unwrap_or(BUCKETS[BUCKETS.len() - 1])
79}
80
81/// Make `path` absolute and drop `.` and `..` without resolving symlinks, as GLib does.
82pub fn normalize(path: &Path) -> PathBuf {
83 let abs = std::path::absolute(path).unwrap_or_else(|_| path.to_path_buf());
84 let mut out = PathBuf::new();
85 for c in abs.components() {
86 match c {
87 Component::CurDir => {}
88 Component::ParentDir => {
89 out.pop();
90 }
91 other => out.push(other),
92 }
93 }
94 out
95}
96
97/// `file://` URI of an absolute path, escaped like GLib's `g_filename_to_uri`.
98pub fn file_uri(path: &Path) -> String {
99 use std::os::unix::ffi::OsStrExt;
100 const HEX: &[u8; 16] = b"0123456789ABCDEF";
101 let mut uri = String::from("file://");
102 for &b in path.as_os_str().as_bytes() {
103 let keep = b.is_ascii_alphanumeric() || b"-._~!$&'()*+,;=:@/".contains(&b);
104 if keep {
105 uri.push(b as char);
106 } else {
107 uri.push('%');
108 uri.push(HEX[(b >> 4) as usize] as char);
109 uri.push(HEX[(b & 15) as usize] as char);
110 }
111 }
112 uri
113}
114
115/// The cache file name for a URI: its MD5 in lowercase hex, plus `.png`.
116pub fn cache_name(uri: &str) -> String {
117 let digest = Md5::digest(uri.as_bytes());
118 let mut s = String::with_capacity(36);
119 for b in digest {
120 s.push_str(&format!("{b:02x}"));
121 }
122 s.push_str(".png");
123 s
124}
125
126/// Seconds since the epoch, as `Thumb::MTime` stores it.
127fn mtime_secs(meta: &fs::Metadata) -> Option<u64> {
128 meta.modified()
129 .ok()?
130 .duration_since(UNIX_EPOCH)
131 .ok()
132 .map(|d| d.as_secs())
133}
134
135/// PNG text chunks as keyword–text pairs.
136type Text = Vec<(String, String)>;
137
138/// Text chunks of a PNG, and its pixels as RGBA8 if `pixels` is set.
139fn read_png(path: &Path, pixels: bool) -> anyhow::Result<(Text, Option<Thumb>)> {
140 let file = File::open(path)?;
141 let mut dec = png::Decoder::new(BufReader::new(file));
142 dec.set_transformations(png::Transformations::normalize_to_color8());
143 let mut reader = dec.read_info()?;
144 let mut thumb = None;
145 if pixels {
146 let mut buf = vec![0; reader.output_buffer_size().context("thumbnail too large")?];
147 let out = reader.next_frame(&mut buf)?;
148 buf.truncate(out.buffer_size());
149 let rgba = to_rgba(buf, out.color_type).context("unsupported thumbnail colour type")?;
150 thumb = Some(Thumb {
151 size: [out.width, out.height],
152 pixels: rgba,
153 });
154 // Text chunks after the image data.
155 let _ = reader.finish();
156 }
157 let info = reader.info();
158 let mut text: Text = info
159 .uncompressed_latin1_text
160 .iter()
161 .map(|c| (c.keyword.clone(), c.text.clone()))
162 .collect();
163 text.extend(
164 info.utf8_text
165 .iter()
166 .filter_map(|c| Some((c.keyword.clone(), c.get_text().ok()?))),
167 );
168 text.extend(
169 info.compressed_latin1_text
170 .iter()
171 .filter_map(|c| Some((c.keyword.clone(), c.get_text().ok()?))),
172 );
173 Ok((text, thumb))
174}
175
176fn to_rgba(buf: Vec<u8>, color: png::ColorType) -> Option<Vec<u8>> {
177 use png::ColorType::*;
178 Some(match color {
179 Rgba => buf,
180 Rgb => buf
181 .as_chunks::<3>()
182 .0
183 .iter()
184 .flat_map(|p| [p[0], p[1], p[2], 255])
185 .collect(),
186 Grayscale => buf.iter().flat_map(|&g| [g, g, g, 255]).collect(),
187 GrayscaleAlpha => buf
188 .as_chunks::<2>()
189 .0
190 .iter()
191 .flat_map(|p| [p[0], p[0], p[0], p[1]])
192 .collect(),
193 Indexed => return None,
194 })
195}
196
197/// Whether the text chunks describe `uri` at `mtime`. Some writers (ffmpegthumbnailer) store
198/// a bare path as the URI; that counts too.
199fn is_valid(text: &[(String, String)], uri: &str, mtime: u64) -> bool {
200 let get = |k: &str| text.iter().find(|(key, _)| key == k).map(|(_, v)| v.trim());
201 // Some writers store fractional seconds.
202 let mtime_ok = get("Thumb::MTime")
203 .and_then(|m| m.parse::<f64>().ok())
204 .is_some_and(|m| m.trunc() as u64 == mtime);
205 let path = uri.strip_prefix("file://").map(|p| {
206 let bytes = percent_decode(p);
207 String::from_utf8_lossy(&bytes).into_owned()
208 });
209 let uri_ok = get("Thumb::URI").is_some_and(|u| u == uri || Some(u) == path.as_deref());
210 uri_ok && mtime_ok
211}
212
213fn percent_decode(s: &str) -> Vec<u8> {
214 let b = s.as_bytes();
215 let mut out = Vec::with_capacity(b.len());
216 let mut i = 0;
217 while i < b.len() {
218 let hex = |c: u8| (c as char).to_digit(16);
219 if b[i] == b'%'
220 && let (Some(h), Some(l)) = (
221 b.get(i + 1).and_then(|&c| hex(c)),
222 b.get(i + 2).and_then(|&c| hex(c)),
223 )
224 {
225 out.push((h * 16 + l) as u8);
226 i += 3;
227 } else {
228 out.push(b[i]);
229 i += 1;
230 }
231 }
232 out
233}
234
235/// Write a PNG atomically (temporary file, then rename) with mode 0600.
236fn write_png(path: &Path, thumb: &Thumb, text: &[(&str, String)]) -> anyhow::Result<()> {
237 let dir = path.parent().context("cache path has no parent")?;
238 fs::DirBuilder::new()
239 .recursive(true)
240 .mode(0o700)
241 .create(dir)?;
242 let name = path.file_name().context("cache path has no name")?;
243 let tmp = dir.join(format!(
244 ".{}.remage-{}.tmp",
245 name.to_string_lossy(),
246 std::process::id()
247 ));
248 let result = (|| -> anyhow::Result<()> {
249 let file = fs::OpenOptions::new()
250 .write(true)
251 .create(true)
252 .truncate(true)
253 .mode(0o600)
254 .open(&tmp)?;
255 let mut w = BufWriter::new(file);
256 let mut enc = png::Encoder::new(&mut w, thumb.size[0], thumb.size[1]);
257 enc.set_color(png::ColorType::Rgba);
258 enc.set_depth(png::BitDepth::Eight);
259 enc.set_compression(png::Compression::Fast);
260 for (k, v) in text {
261 enc.add_text_chunk(k.to_string(), v.clone())?;
262 }
263 let mut writer = enc.write_header()?;
264 writer.write_image_data(&thumb.pixels)?;
265 writer.finish()?;
266 w.flush()?;
267 drop(w);
268 fs::rename(&tmp, path)?;
269 Ok(())
270 })();
271 if result.is_err() {
272 let _ = fs::remove_file(&tmp);
273 }
274 result
275}
276
277/// Shrink `src` so its longer side is at most `max` (never enlarging).
278pub fn resize(src: Thumb, max: u32) -> anyhow::Result<Thumb> {
279 let [w, h] = src.size;
280 if w <= max && h <= max {
281 return Ok(src);
282 }
283 let k = max as f64 / w.max(h) as f64;
284 let nw = ((w as f64 * k).round() as u32).clamp(1, max);
285 let nh = ((h as f64 * k).round() as u32).clamp(1, max);
286 let mut img = fir::images::Image::from_vec_u8(w, h, src.pixels, fir::PixelType::U8x4)
287 .context("resize source")?;
288 // Resample in premultiplied alpha so transparent edges don't bleed dark fringes.
289 let mul = fir::MulDiv::default();
290 mul.multiply_alpha_inplace(&mut img)
291 .context("premultiply")?;
292 let mut dst = fir::images::Image::new(nw, nh, fir::PixelType::U8x4);
293 let opts = fir::ResizeOptions::new()
294 .resize_alg(fir::ResizeAlg::Convolution(fir::FilterType::Lanczos3))
295 .use_alpha(false);
296 fir::Resizer::new()
297 .resize(&img, &mut dst, &opts)
298 .context("resize")?;
299 mul.divide_alpha_inplace(&mut dst)
300 .context("unpremultiply")?;
301 Ok(Thumb {
302 size: [nw, nh],
303 pixels: dst.into_vec(),
304 })
305}
306
307/// Load a thumbnail of `path` at most `px` on its longer side: from the shared cache when a
308/// valid entry is big enough, otherwise generated (and written back unless private).
309pub fn load(path: &Path, px: u32, cache: &CacheOptions) -> Result<Thumb, ThumbError> {
310 let fail = |e: anyhow::Error| ThumbError::Failed(e);
311 let meta = fs::metadata(path).map_err(|e| fail(e.into()))?;
312 let mtime = mtime_secs(&meta).unwrap_or(0);
313 let abs = normalize(path);
314 let uri = file_uri(&abs);
315 let name = cache_name(&uri);
316 let (bucket, bucket_px) = bucket_for(px);
317
318 if let Some(root) = &cache.root {
319 // Our bucket, or any larger one.
320 for (dir, _) in BUCKETS.iter().filter(|(_, s)| *s >= bucket_px) {
321 let p = root.join(dir).join(&name);
322 if let Ok((text, Some(t))) = read_png(&p, true)
323 && is_valid(&text, &uri, mtime)
324 {
325 return resize(t, px).map_err(fail);
326 }
327 }
328 // Only trust failures this version recorded: a later one may decode more formats.
329 let failed = root.join("fail").join(APP).join(&name);
330 if let Ok((text, _)) = read_png(&failed, false)
331 && is_valid(&text, &uri, mtime)
332 && text
333 .iter()
334 .any(|(k, v)| k == "Software" && *v == software())
335 {
336 return Err(ThumbError::KnownFailure);
337 }
338 }
339
340 let generated = match generate(path, bucket_px) {
341 Ok(t) => t,
342 Err(e) => {
343 if let Some(root) = cache.root.as_ref().filter(|_| !cache.private) {
344 let failed = root.join("fail").join(APP).join(&name);
345 let empty = Thumb {
346 size: [1, 1],
347 pixels: vec![0; 4],
348 };
349 let text = [
350 ("Thumb::URI", uri.clone()),
351 ("Thumb::MTime", mtime.to_string()),
352 ("Software", software()),
353 ];
354 if let Err(e) = write_png(&failed, &empty, &text) {
355 log::debug!("{}: {e:#}", failed.display());
356 }
357 }
358 return Err(fail(e));
359 }
360 };
361
362 // Images no bigger than the bucket gain nothing from a cache entry.
363 let worth_caching = generated.source_size.iter().any(|&s| s > bucket_px);
364 let thumb = Thumb {
365 size: generated.size,
366 pixels: generated.pixels,
367 };
368 if let Some(root) = cache.root.as_ref().filter(|_| !cache.private)
369 && worth_caching
370 && !abs.starts_with(root)
371 {
372 let dest = root.join(bucket).join(&name);
373 let [sw, sh] = generated.source_size;
374 let text = [
375 ("Thumb::URI", uri.clone()),
376 ("Thumb::MTime", mtime.to_string()),
377 ("Thumb::Size", meta.len().to_string()),
378 ("Thumb::Image::Width", sw.to_string()),
379 ("Thumb::Image::Height", sh.to_string()),
380 ("Software", software()),
381 ];
382 if let Err(e) = write_png(&dest, &thumb, &text) {
383 log::debug!("{}: {e:#}", dest.display());
384 }
385 }
386 resize(thumb, px).map_err(fail)
387}
388
389fn software() -> String {
390 format!("remage {}", env!("CARGO_PKG_VERSION"))
391}
392
393/// A freshly made thumbnail at most `max` on its longer side.
394struct Generated {
395 size: [u32; 2],
396 pixels: Vec<u8>,
397 /// The original's upright size.
398 source_size: [u32; 2],
399}
400
401fn generate(path: &Path, max: u32) -> anyhow::Result<Generated> {
402 // JPEG: decode at 1/2, 1/4 or 1/8 straight from the DCT.
403 if let Some(g) = jpeg_scaled(path, max)? {
404 return Ok(g);
405 }
406 // Scalable and previewed sources (SVG, RAW) only need the bucket's size.
407 let opts = decode::Options {
408 target: max,
409 ..decode::Options::default()
410 };
411 let img = decode::decode(path, &opts)?;
412 let source_size = img.source_size;
413 let DecodedImage {
414 size, mut frames, ..
415 } = img;
416 let pixels = frames.swap_remove(0).pixels;
417 let t = resize(Thumb { size, pixels }, max)?;
418 Ok(Generated {
419 size: t.size,
420 pixels: t.pixels,
421 source_size,
422 })
423}
424
425/// DCT-scaled JPEG decode, or `None` when the file isn't a plain 8-bit RGB/grey JPEG.
426fn jpeg_scaled(path: &Path, max: u32) -> anyhow::Result<Option<Generated>> {
427 let mut magic = [0u8; 3];
428 {
429 use std::io::Read;
430 let mut f = File::open(path)?;
431 if f.read_exact(&mut magic).is_err() || magic != [0xFF, 0xD8, 0xFF] {
432 return Ok(None);
433 }
434 }
435 let mut dec = jpeg_decoder::Decoder::new(BufReader::new(File::open(path)?));
436 dec.read_info()?;
437 let info = dec.info().ok_or_else(|| anyhow!("no JPEG header"))?;
438 use jpeg_decoder::PixelFormat::*;
439 if !matches!(info.pixel_format, L8 | RGB24) {
440 return Ok(None);
441 }
442 let (w, h) = (info.width as u32, info.height as u32);
443 let k = (max as f64 / w.max(h) as f64).min(1.0);
444 let req = |v: u32| ((v as f64 * k).ceil() as u32).clamp(1, u16::MAX as u32) as u16;
445 let (sw, sh) = dec.scale(req(w), req(h))?;
446 let data = dec.decode()?;
447 let pixels: Vec<u8> = match info.pixel_format {
448 L8 => data.iter().flat_map(|&g| [g, g, g, 255]).collect(),
449 _ => data
450 .as_chunks::<3>()
451 .0
452 .iter()
453 .flat_map(|p| [p[0], p[1], p[2], 255])
454 .collect(),
455 };
456 let (sw, sh) = (sw as u32, sh as u32);
457 if pixels.len() != (sw * sh * 4) as usize {
458 bail!("JPEG decoded to an unexpected size");
459 }
460 let orientation = dec
461 .exif_data()
462 .and_then(Orientation::from_exif_chunk)
463 .unwrap_or(Orientation::NoTransforms);
464 let mut img = image::DynamicImage::ImageRgba8(
465 image::RgbaImage::from_raw(sw, sh, pixels).context("JPEG buffer size")?,
466 );
467 img.apply_orientation(orientation);
468 let source_size = if img.width() == sw { [w, h] } else { [h, w] };
469 let rgba = img.into_rgba8();
470 let size = [rgba.width(), rgba.height()];
471 let t = resize(
472 Thumb {
473 size,
474 pixels: rgba.into_raw(),
475 },
476 max,
477 )?;
478 Ok(Some(Generated {
479 size: t.size,
480 pixels: t.pixels,
481 source_size,
482 }))
483}
484
485#[cfg(test)]
486mod tests {
487 use super::*;
488
489 fn tmp(name: &str) -> PathBuf {
490 let dir = std::env::temp_dir().join(format!("remage-thumbs-{}-{name}", std::process::id()));
491 let _ = fs::remove_dir_all(&dir);
492 fs::create_dir_all(&dir).unwrap();
493 dir
494 }
495
496 #[test]
497 fn uris_are_escaped_like_glib() {
498 assert_eq!(
499 file_uri(Path::new("/home/u/a b/ü#?%.png")),
500 "file:///home/u/a%20b/%C3%BC%23%3F%25.png"
501 );
502 assert_eq!(
503 file_uri(Path::new("/x/(1)!$&'*+,;=:@~-_.png")),
504 "file:///x/(1)!$&'*+,;=:@~-_.png"
505 );
506 }
507
508 #[test]
509 fn names_are_md5_of_the_uri() {
510 // The spec's own example.
511 assert_eq!(
512 cache_name("file:///home/jens/photos/me.png"),
513 "c6ee772d9e49320e97ec29a7eb5b1697.png"
514 );
515 }
516
517 #[test]
518 fn normalize_is_lexical() {
519 assert_eq!(
520 normalize(Path::new("/a/./b/../c.png")),
521 Path::new("/a/c.png")
522 );
523 assert!(normalize(Path::new("x.png")).is_absolute());
524 }
525
526 #[test]
527 fn buckets() {
528 assert_eq!(bucket_for(96), ("normal", 128));
529 assert_eq!(bucket_for(128), ("normal", 128));
530 assert_eq!(bucket_for(160), ("large", 256));
531 assert_eq!(bucket_for(320), ("x-large", 512));
532 assert_eq!(bucket_for(2000), ("xx-large", 1024));
533 }
534
535 #[test]
536 fn generates_writes_and_rereads() {
537 let dir = tmp("cache");
538 let img = dir.join("pic.png");
539 image::RgbaImage::from_fn(600, 300, |x, _| image::Rgba([(x % 256) as u8, 0, 0, 255]))
540 .save(&img)
541 .unwrap();
542 let cache = CacheOptions {
543 root: Some(dir.join("thumbnails")),
544 private: false,
545 };
546 let t = load(&img, 160, &cache).unwrap();
547 assert_eq!(t.size, [160, 80]);
548 let entry = dir
549 .join("thumbnails/large")
550 .join(cache_name(&file_uri(&normalize(&img))));
551 let (text, stored) = read_png(&entry, true).unwrap();
552 assert_eq!(stored.unwrap().size, [256, 128]);
553 assert!(
554 text.iter()
555 .any(|(k, v)| k == "Thumb::Image::Width" && v == "600")
556 );
557 {
558 use std::os::unix::fs::PermissionsExt;
559 let mode = fs::metadata(&entry).unwrap().permissions().mode();
560 assert_eq!(mode & 0o777, 0o600);
561 }
562 // A second load reads the entry (same result), a smaller size downscales it.
563 assert_eq!(load(&img, 160, &cache).unwrap().size, [160, 80]);
564 assert_eq!(load(&img, 96, &cache).unwrap().size, [96, 48]);
565 // A stale entry (different mtime) is ignored and replaced.
566 let text = vec![
567 ("Thumb::URI".to_string(), file_uri(&normalize(&img))),
568 ("Thumb::MTime".to_string(), "1".to_string()),
569 ];
570 let meta = fs::metadata(&img).unwrap();
571 assert!(!is_valid(&text, &text[0].1, mtime_secs(&meta).unwrap()));
572 assert!(is_valid(&text, &text[0].1, 1));
573 // A bare path as the URI is accepted too.
574 let bare = vec![
575 ("Thumb::URI".to_string(), "/a b/c.png".to_string()),
576 ("Thumb::MTime".to_string(), "1".to_string()),
577 ];
578 assert!(is_valid(&bare, "file:///a%20b/c.png", 1));
579 fs::remove_dir_all(&dir).unwrap();
580 }
581
582 #[test]
583 fn failures_are_remembered_and_private_writes_nothing() {
584 let dir = tmp("fail");
585 let bad = dir.join("bad.png");
586 fs::write(&bad, b"not a png").unwrap();
587 let cache = CacheOptions {
588 root: Some(dir.join("thumbnails")),
589 private: true,
590 };
591 assert!(matches!(
592 load(&bad, 128, &cache),
593 Err(ThumbError::Failed(_))
594 ));
595 assert!(!dir.join("thumbnails").exists());
596 let cache = CacheOptions {
597 private: false,
598 ..cache
599 };
600 assert!(matches!(
601 load(&bad, 128, &cache),
602 Err(ThumbError::Failed(_))
603 ));
604 assert!(matches!(
605 load(&bad, 128, &cache),
606 Err(ThumbError::KnownFailure)
607 ));
608 fs::remove_dir_all(&dir).unwrap();
609 }
610
611 #[test]
612 fn jpeg_is_scaled_and_oriented() {
613 let dir = tmp("jpeg");
614 let p = dir.join("a.jpg");
615 image::RgbImage::from_pixel(800, 400, image::Rgb([10, 200, 30]))
616 .save(&p)
617 .unwrap();
618 let g = jpeg_scaled(&p, 128).unwrap().unwrap();
619 assert_eq!(g.source_size, [800, 400]);
620 assert_eq!(g.size, [128, 64]);
621 let px = &g.pixels[..4];
622 assert!(px[1] > 150 && px[0] < 60, "{px:?}");
623 fs::remove_dir_all(&dir).unwrap();
624 }
625}