Krolyxon
Branch master

Branches

Tags

Refs →

remagesrcrenderimage.rs

538 lines · 18.5 KB · Rust

Raw Blame History Download
1//! Image textures: incremental row uploads, GPU mipmaps, linear/nearest sampling, and tiles
2//! for images larger than the GPU's texture limit.
3
4use bytemuck::{Pod, Zeroable};
5use wgpu::util::DeviceExt;
6
7use crate::decode::{DecodedImage, color};
8use crate::state::view::Affine;
9
10const FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba8UnormSrgb;
11
12#[repr(C)]
13#[derive(Clone, Copy, Pod, Zeroable)]
14struct Uniforms {
15 m: [f32; 4],
16 t: [f32; 2],
17 viewport: [f32; 2],
18 bg: [f32; 4],
19 dot: [f32; 4],
20 params: [f32; 4],
21 /// Gamma, brightness, contrast, and 1 when any differs from neutral.
22 adjust: [f32; 4],
23}
24
25/// How images are painted: the area behind transparent pixels, and display adjustments.
26#[derive(Clone, Copy, Debug)]
27pub struct AlphaStyle {
28 /// Gamma, brightness, contrast.
29 pub adjust: [f32; 3],
30 /// 0 solid, 1 dots, 2 checker.
31 pub mode: u32,
32 pub bg: [f32; 4],
33 pub dot: [f32; 4],
34 pub cell: f32,
35 pub radius: f32,
36}
37
38/// One texture of an image: all of it, or a tile of a huge one.
39struct Tile {
40 texture: wgpu::Texture,
41 bind_group: wgpu::BindGroup,
42 uniforms: wgpu::Buffer,
43 origin: [u32; 2],
44 size: [u32; 2],
45}
46
47pub struct GpuImage {
48 tiles: Vec<Tile>,
49 size: [u32; 2],
50 /// Where the stored pixels land in the upright image (see [`color::orient_uv`]).
51 orient: ([f32; 4], [f32; 2]),
52 /// The frame these textures hold (or are being filled with).
53 frame: usize,
54 /// Upload progress: the tile being filled and its rows done.
55 tile: usize,
56 rows_done: u32,
57 ready: bool,
58}
59
60impl GpuImage {
61 pub fn is_ready(&self) -> bool {
62 self.ready
63 }
64
65 pub fn frame(&self) -> usize {
66 self.frame
67 }
68
69 /// Whether these textures can hold `img`'s frames and draw them the right way up.
70 pub fn fits(&self, img: &DecodedImage) -> bool {
71 self.size == img.size && self.orient == color::orient_uv(img.orientation)
72 }
73}
74
75/// Split `size` into tiles no larger than `max` on either side.
76pub fn tiles(size: [u32; 2], max: u32) -> Vec<([u32; 2], [u32; 2])> {
77 let max = max.max(1);
78 let mut out = Vec::new();
79 for y in (0..size[1].max(1)).step_by(max as usize) {
80 for x in (0..size[0].max(1)).step_by(max as usize) {
81 out.push(([x, y], [(size[0] - x).min(max), (size[1] - y).min(max)]));
82 }
83 }
84 out
85}
86
87/// `affine` places the upright image; the result places the pixels as stored.
88fn turned(affine: Affine, (om, ot): ([f32; 4], [f32; 2])) -> Affine {
89 let m = affine.m;
90 Affine {
91 m: [
92 m[0] * om[0] + m[1] * om[2],
93 m[0] * om[1] + m[1] * om[3],
94 m[2] * om[0] + m[3] * om[2],
95 m[2] * om[1] + m[3] * om[3],
96 ],
97 t: [
98 affine.t[0] + m[0] * ot[0] + m[1] * ot[1],
99 affine.t[1] + m[2] * ot[0] + m[3] * ot[1],
100 ],
101 }
102}
103
104pub struct ImagePipeline {
105 pipeline: wgpu::RenderPipeline,
106 layout: wgpu::BindGroupLayout,
107 linear: wgpu::BindGroup,
108 nearest: wgpu::BindGroup,
109 mip_pipeline: wgpu::RenderPipeline,
110 mip_layout: wgpu::BindGroupLayout,
111 mip_sampler: wgpu::Sampler,
112}
113
114fn sampler_group(
115 device: &wgpu::Device,
116 layout: &wgpu::BindGroupLayout,
117 filter: wgpu::FilterMode,
118) -> wgpu::BindGroup {
119 let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
120 label: Some("image sampler"),
121 mag_filter: filter,
122 min_filter: filter,
123 mipmap_filter: wgpu::MipmapFilterMode::Linear,
124 ..Default::default()
125 });
126 device.create_bind_group(&wgpu::BindGroupDescriptor {
127 label: Some("image sampler"),
128 layout,
129 entries: &[wgpu::BindGroupEntry {
130 binding: 0,
131 resource: wgpu::BindingResource::Sampler(&sampler),
132 }],
133 })
134}
135
136fn texture_entry(binding: u32) -> wgpu::BindGroupLayoutEntry {
137 wgpu::BindGroupLayoutEntry {
138 binding,
139 visibility: wgpu::ShaderStages::FRAGMENT,
140 ty: wgpu::BindingType::Texture {
141 sample_type: wgpu::TextureSampleType::Float { filterable: true },
142 view_dimension: wgpu::TextureViewDimension::D2,
143 multisampled: false,
144 },
145 count: None,
146 }
147}
148
149fn sampler_entry(binding: u32) -> wgpu::BindGroupLayoutEntry {
150 wgpu::BindGroupLayoutEntry {
151 binding,
152 visibility: wgpu::ShaderStages::FRAGMENT,
153 ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
154 count: None,
155 }
156}
157
158impl ImagePipeline {
159 pub fn new(device: &wgpu::Device, target: wgpu::TextureFormat) -> Self {
160 let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
161 label: Some("image"),
162 entries: &[
163 wgpu::BindGroupLayoutEntry {
164 binding: 0,
165 visibility: wgpu::ShaderStages::VERTEX_FRAGMENT,
166 ty: wgpu::BindingType::Buffer {
167 ty: wgpu::BufferBindingType::Uniform,
168 has_dynamic_offset: false,
169 min_binding_size: None,
170 },
171 count: None,
172 },
173 texture_entry(1),
174 ],
175 });
176 let sampler_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
177 label: Some("image sampler"),
178 entries: &[sampler_entry(0)],
179 });
180 let shader = device.create_shader_module(wgpu::include_wgsl!("shaders/image.wgsl"));
181 let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
182 label: Some("image"),
183 bind_group_layouts: &[Some(&layout), Some(&sampler_layout)],
184 immediate_size: 0,
185 });
186 let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
187 label: Some("image"),
188 layout: Some(&pipeline_layout),
189 vertex: wgpu::VertexState {
190 module: &shader,
191 entry_point: Some("vs"),
192 compilation_options: Default::default(),
193 buffers: &[],
194 },
195 primitive: wgpu::PrimitiveState::default(),
196 depth_stencil: None,
197 multisample: wgpu::MultisampleState::default(),
198 fragment: Some(wgpu::FragmentState {
199 module: &shader,
200 entry_point: Some("fs"),
201 compilation_options: Default::default(),
202 targets: &[Some(wgpu::ColorTargetState {
203 format: target,
204 blend: Some(wgpu::BlendState::ALPHA_BLENDING),
205 write_mask: wgpu::ColorWrites::ALL,
206 })],
207 }),
208 multiview_mask: None,
209 cache: None,
210 });
211
212 let mip_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
213 label: Some("mip"),
214 entries: &[texture_entry(0), sampler_entry(1)],
215 });
216 let blit = device.create_shader_module(wgpu::include_wgsl!("shaders/blit.wgsl"));
217 let mip_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
218 label: Some("mip"),
219 bind_group_layouts: &[Some(&mip_layout)],
220 immediate_size: 0,
221 });
222 let mip_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
223 label: Some("mip"),
224 layout: Some(&mip_pipeline_layout),
225 vertex: wgpu::VertexState {
226 module: &blit,
227 entry_point: Some("vs"),
228 compilation_options: Default::default(),
229 buffers: &[],
230 },
231 primitive: wgpu::PrimitiveState::default(),
232 depth_stencil: None,
233 multisample: wgpu::MultisampleState::default(),
234 fragment: Some(wgpu::FragmentState {
235 module: &blit,
236 entry_point: Some("fs"),
237 compilation_options: Default::default(),
238 targets: &[Some(FORMAT.into())],
239 }),
240 multiview_mask: None,
241 cache: None,
242 });
243 let mip_sampler = device.create_sampler(&wgpu::SamplerDescriptor {
244 label: Some("mip"),
245 mag_filter: wgpu::FilterMode::Linear,
246 min_filter: wgpu::FilterMode::Linear,
247 ..Default::default()
248 });
249
250 Self {
251 pipeline,
252 linear: sampler_group(device, &sampler_layout, wgpu::FilterMode::Linear),
253 nearest: sampler_group(device, &sampler_layout, wgpu::FilterMode::Nearest),
254 layout,
255 mip_pipeline,
256 mip_layout,
257 mip_sampler,
258 }
259 }
260
261 /// Allocate textures for `img`, tiled at `max_dim`; fill them with [`ImagePipeline::upload`].
262 pub fn create(&self, device: &wgpu::Device, img: &DecodedImage, max_dim: u32) -> GpuImage {
263 let tiles = tiles(img.size, max_dim)
264 .into_iter()
265 .map(|(origin, size)| self.tile(device, origin, size))
266 .collect::<Vec<_>>();
267 if tiles.len() > 1 {
268 log::debug!(
269 "{}×{} image in {} tiles",
270 img.size[0],
271 img.size[1],
272 tiles.len()
273 );
274 }
275 GpuImage {
276 tiles,
277 size: img.size,
278 orient: color::orient_uv(img.orientation),
279 frame: 0,
280 tile: 0,
281 rows_done: 0,
282 ready: false,
283 }
284 }
285
286 fn tile(&self, device: &wgpu::Device, origin: [u32; 2], [w, h]: [u32; 2]) -> Tile {
287 let mips = 32 - w.max(h).max(1).leading_zeros();
288 let texture = device.create_texture(&wgpu::TextureDescriptor {
289 label: Some("image"),
290 size: wgpu::Extent3d {
291 width: w,
292 height: h,
293 depth_or_array_layers: 1,
294 },
295 mip_level_count: mips,
296 sample_count: 1,
297 dimension: wgpu::TextureDimension::D2,
298 format: FORMAT,
299 usage: wgpu::TextureUsages::TEXTURE_BINDING
300 | wgpu::TextureUsages::COPY_DST
301 | wgpu::TextureUsages::RENDER_ATTACHMENT,
302 view_formats: &[],
303 });
304 let uniforms = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
305 label: Some("image uniforms"),
306 contents: bytemuck::bytes_of(&Uniforms::zeroed()),
307 usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
308 });
309 let view = texture.create_view(&wgpu::TextureViewDescriptor::default());
310 let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
311 label: Some("image"),
312 layout: &self.layout,
313 entries: &[
314 wgpu::BindGroupEntry {
315 binding: 0,
316 resource: uniforms.as_entire_binding(),
317 },
318 wgpu::BindGroupEntry {
319 binding: 1,
320 resource: wgpu::BindingResource::TextureView(&view),
321 },
322 ],
323 });
324 Tile {
325 texture,
326 bind_group,
327 uniforms,
328 origin,
329 size: [w, h],
330 }
331 }
332
333 /// Start filling the textures with another frame; finish with [`ImagePipeline::upload`].
334 pub fn restart(&self, gpu: &mut GpuImage, frame: usize) {
335 gpu.frame = frame;
336 gpu.tile = 0;
337 gpu.rows_done = 0;
338 gpu.ready = false;
339 }
340
341 /// Upload up to `budget` bytes of rows. Returns true once the image is complete.
342 pub fn upload(
343 &self,
344 device: &wgpu::Device,
345 queue: &wgpu::Queue,
346 gpu: &mut GpuImage,
347 img: &DecodedImage,
348 budget: usize,
349 ) -> bool {
350 let pixels = img.frame(gpu.frame);
351 let stride = img.size[0] as usize * 4;
352 let mut left = budget;
353 while !gpu.ready {
354 let tile = &gpu.tiles[gpu.tile];
355 let [w, h] = tile.size;
356 let row_bytes = w as usize * 4;
357 let rows = ((left / row_bytes.max(1)) as u32).clamp(1, h - gpu.rows_done);
358 let y = tile.origin[1] + gpu.rows_done;
359 let start = y as usize * stride + tile.origin[0] as usize * 4;
360 let end = start + (rows as usize - 1) * stride + row_bytes;
361 queue.write_texture(
362 wgpu::TexelCopyTextureInfo {
363 texture: &tile.texture,
364 mip_level: 0,
365 origin: wgpu::Origin3d {
366 x: 0,
367 y: gpu.rows_done,
368 z: 0,
369 },
370 aspect: wgpu::TextureAspect::All,
371 },
372 &pixels[start..end],
373 wgpu::TexelCopyBufferLayout {
374 offset: 0,
375 bytes_per_row: Some(stride as u32),
376 rows_per_image: Some(rows),
377 },
378 wgpu::Extent3d {
379 width: w,
380 height: rows,
381 depth_or_array_layers: 1,
382 },
383 );
384 gpu.rows_done += rows;
385 left = left.saturating_sub(rows as usize * row_bytes);
386 if gpu.rows_done == h {
387 self.generate_mips(device, queue, &gpu.tiles[gpu.tile].texture);
388 gpu.tile += 1;
389 gpu.rows_done = 0;
390 gpu.ready = gpu.tile == gpu.tiles.len();
391 }
392 if left == 0 {
393 break;
394 }
395 }
396 if !gpu.ready {
397 queue.submit([]);
398 }
399 gpu.ready
400 }
401
402 fn generate_mips(&self, device: &wgpu::Device, queue: &wgpu::Queue, texture: &wgpu::Texture) {
403 let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
404 label: Some("mips"),
405 });
406 let level_view = |level| {
407 texture.create_view(&wgpu::TextureViewDescriptor {
408 base_mip_level: level,
409 mip_level_count: Some(1),
410 ..Default::default()
411 })
412 };
413 for level in 1..texture.mip_level_count() {
414 let src = level_view(level - 1);
415 let dst = level_view(level);
416 let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
417 label: Some("mip"),
418 layout: &self.mip_layout,
419 entries: &[
420 wgpu::BindGroupEntry {
421 binding: 0,
422 resource: wgpu::BindingResource::TextureView(&src),
423 },
424 wgpu::BindGroupEntry {
425 binding: 1,
426 resource: wgpu::BindingResource::Sampler(&self.mip_sampler),
427 },
428 ],
429 });
430 let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
431 label: Some("mip"),
432 color_attachments: &[Some(wgpu::RenderPassColorAttachment {
433 view: &dst,
434 depth_slice: None,
435 resolve_target: None,
436 ops: wgpu::Operations {
437 load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT),
438 store: wgpu::StoreOp::Store,
439 },
440 })],
441 depth_stencil_attachment: None,
442 timestamp_writes: None,
443 occlusion_query_set: None,
444 multiview_mask: None,
445 });
446 pass.set_pipeline(&self.mip_pipeline);
447 pass.set_bind_group(0, &bind_group, &[]);
448 pass.draw(0..3, 0..1);
449 }
450 queue.submit([encoder.finish()]);
451 }
452
453 /// Write this frame's placement; call before the render pass begins.
454 pub fn prepare(
455 &self,
456 queue: &wgpu::Queue,
457 gpu: &GpuImage,
458 affine: Affine,
459 viewport: [f32; 2],
460 alpha: f32,
461 style: &AlphaStyle,
462 ) {
463 let adjust = style.adjust;
464 let on = adjust != [1.0, 0.0, 1.0];
465 let [iw, ih] = [gpu.size[0] as f32, gpu.size[1] as f32];
466 let affine = turned(affine, gpu.orient);
467 let m = affine.m;
468 for tile in &gpu.tiles {
469 // The tile's corner and extent in the image's 0..1 coordinates.
470 let (u0, v0) = (tile.origin[0] as f32 / iw, tile.origin[1] as f32 / ih);
471 let (du, dv) = (tile.size[0] as f32 / iw, tile.size[1] as f32 / ih);
472 let u = Uniforms {
473 m: [m[0] * du, m[1] * dv, m[2] * du, m[3] * dv],
474 t: [
475 affine.t[0] + m[0] * u0 + m[1] * v0,
476 affine.t[1] + m[2] * u0 + m[3] * v0,
477 ],
478 viewport,
479 bg: style.bg,
480 dot: style.dot,
481 params: [alpha, style.mode as f32, style.cell, style.radius],
482 adjust: [adjust[0], adjust[1], adjust[2], on as u32 as f32],
483 };
484 queue.write_buffer(&tile.uniforms, 0, bytemuck::bytes_of(&u));
485 }
486 }
487
488 pub fn draw(&self, pass: &mut wgpu::RenderPass<'_>, gpu: &GpuImage, nearest: bool) {
489 pass.set_pipeline(&self.pipeline);
490 pass.set_bind_group(1, if nearest { &self.nearest } else { &self.linear }, &[]);
491 for tile in &gpu.tiles {
492 pass.set_bind_group(0, &tile.bind_group, &[]);
493 pass.draw(0..6, 0..1);
494 }
495 }
496}
497
498#[cfg(test)]
499mod tests {
500 use super::*;
501
502 #[test]
503 fn stored_pixels_are_placed_upright() {
504 use image::metadata::Orientation;
505 // An upright 200×100 image at (10, 20).
506 let upright = Affine {
507 m: [200.0, 0.0, 0.0, 100.0],
508 t: [10.0, 20.0],
509 };
510 let at = |a: Affine, u: f32, v: f32| {
511 [
512 a.m[0] * u + a.m[1] * v + a.t[0],
513 a.m[2] * u + a.m[3] * v + a.t[1],
514 ]
515 };
516 assert_eq!(
517 turned(upright, color::orient_uv(Orientation::NoTransforms)),
518 upright
519 );
520 // Stored on its side, to be turned clockwise: its top-left corner ends up top right
521 // and its top-right corner bottom right.
522 let a = turned(upright, color::orient_uv(Orientation::Rotate90));
523 assert_eq!(at(a, 0.0, 0.0), [210.0, 20.0]);
524 assert_eq!(at(a, 1.0, 0.0), [210.0, 120.0]);
525 assert_eq!(at(a, 0.0, 1.0), [10.0, 20.0]);
526 }
527
528 #[test]
529 fn tiling_covers_the_image() {
530 assert_eq!(tiles([10, 5], 16), [([0, 0], [10, 5])]);
531 let t = tiles([10, 5], 4);
532 assert_eq!(t.len(), 6);
533 assert_eq!(t[2], ([8, 0], [2, 4]));
534 assert_eq!(t[5], ([8, 4], [2, 1]));
535 let area: u32 = t.iter().map(|(_, s)| s[0] * s[1]).sum();
536 assert_eq!(area, 50);
537 }
538}