tyiu/notedeck
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

integrate gifs

Browse Source 
Signed-off-by: kernelkind <kernelkind@gmail.com>
This commit is contained in:
kernelkind
2025-02-20 15:08:13 -05:00
parent d1c7a5a239
commit 490dedfaf1
8 changed files with 318 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
crates/notedeck_columns/src/images.rs
View File

@@ -1,17 +1,28 @@
use egui::{pos2, Color32, ColorImage, Rect, Sense, SizeHint};
use image::codecs::gif::GifDecoder;
use image::imageops::FilterType;
use image::AnimationDecoder;
use image::DynamicImage;
use image::FlatSamples;
use image::Frame;
use notedeck::Animation;
use notedeck::ImageFrame;
use notedeck::MediaCache;
use notedeck::MediaCacheType;
use notedeck::Result;
use notedeck::TextureFrame;
use notedeck::TexturedImage;
use poll_promise::Promise;
use std::collections::VecDeque;
use std::io::Cursor;
use std::path;
use std::path::PathBuf;
use std::sync::mpsc;
use std::sync::mpsc::SyncSender;
use std::thread;
use std::time::Duration;
use tokio::fs;
//pub type ImageCacheKey = String;
//pub type ImageCacheValue = Promise<Result<TextureHandle>>;
//pub type MediaCache = HashMap<String, ImageCacheValue>;
// NOTE(jb55): chatgpt wrote this because I was too dumb to
pub fn aspect_fill(
ui: &mut egui::Ui,
@@ -103,7 +114,7 @@ pub fn round_image(image: &mut ColorImage) {
}
}
fn process_pfp_bitmap(imgtyp: ImageType, image: &mut image::DynamicImage) -> ColorImage {
fn process_pfp_bitmap(imgtyp: ImageType, mut image: image::DynamicImage) -> ColorImage {
#[cfg(feature = "profiling")]
puffin::profile_function!();
@@ -126,10 +137,10 @@ fn process_pfp_bitmap(imgtyp: ImageType, image: &mut image::DynamicImage) -> Col
if image.width() > smaller {
let excess = image.width() - smaller;
*image = image.crop_imm(excess / 2, 0, image.width() - excess, image.height());
image = image.crop_imm(excess / 2, 0, image.width() - excess, image.height());
} else if image.height() > smaller {
let excess = image.height() - smaller;
*image = image.crop_imm(0, excess / 2, image.width(), image.height() - excess);
image = image.crop_imm(0, excess / 2, image.width(), image.height() - excess);
}
let image = image.resize(size, size, FilterType::CatmullRom); // DynamicImage
let image_buffer = image.into_rgba8(); // RgbaImage (ImageBuffer)
@@ -167,8 +178,8 @@ fn parse_img_response(response: ehttp::Response, imgtyp: ImageType) -> Result<Co
} else if content_type.starts_with("image/") {
#[cfg(feature = "profiling")]
puffin::profile_scope!("load_from_memory");
let mut dyn_image = image::load_from_memory(&response.bytes)?;
Ok(process_pfp_bitmap(imgtyp, &mut dyn_image))
let dyn_image = image::load_from_memory(&response.bytes)?;
Ok(process_pfp_bitmap(imgtyp, dyn_image))
} else {
Err(format!("Expected image, found content-type {:?}", content_type).into())
}
@@ -178,32 +189,162 @@ fn fetch_img_from_disk(
ctx: &egui::Context,
url: &str,
path: &path::Path,
cache_type: MediaCacheType,
) -> Promise<Result<TexturedImage>> {
let ctx = ctx.clone();
let url = url.to_owned();
let path = path.to_owned();
Promise::spawn_async(async move {
let data = fs::read(path).await?;
let image_buffer = image::load_from_memory(&data).map_err(notedeck::Error::Image)?;
match cache_type {
MediaCacheType::Image => {
let data = fs::read(path).await?;
let image_buffer =
image::load_from_memory(&data).map_err(notedeck::Error::Image)?;
// TODO: remove unwrap here
let flat_samples = image_buffer.as_flat_samples_u8().unwrap();
let img = ColorImage::from_rgba_unmultiplied(
[
image_buffer.width() as usize,
image_buffer.height() as usize,
],
flat_samples.as_slice(),
);
Ok(TexturedImage::Static(ctx.load_texture(
&url,
img,
Default::default(),
)))
let img = buffer_to_color_image(
image_buffer.as_flat_samples_u8(),
image_buffer.width(),
image_buffer.height(),
);
Ok(TexturedImage::Static(ctx.load_texture(
&url,
img,
Default::default(),
)))
}
MediaCacheType::Gif => {
let gif_bytes = fs::read(path.clone()).await?; // Read entire file into a Vec<u8>
generate_gif(ctx, url, &path, gif_bytes, false, |i| {
buffer_to_color_image(i.as_flat_samples_u8(), i.width(), i.height())
})
}
}
})
}
fn generate_gif(
ctx: egui::Context,
url: String,
path: &path::Path,
data: Vec<u8>,
write_to_disk: bool,
process_to_egui: impl Fn(DynamicImage) -> ColorImage + Send + Copy + 'static,
) -> Result<TexturedImage> {
let decoder = {
let reader = Cursor::new(data.as_slice());
GifDecoder::new(reader)?
};
let (tex_input, tex_output) = mpsc::sync_channel(4);
let (maybe_encoder_input, maybe_encoder_output) = if write_to_disk {
let (inp, out) = mpsc::sync_channel(4);
(Some(inp), Some(out))
} else {
(None, None)
};
let mut frames: VecDeque<Frame> = decoder
.into_frames()
.collect::<std::result::Result<VecDeque<_>, image::ImageError>>()
.map_err(|e| notedeck::Error::Generic(e.to_string()))?;
let first_frame = frames.pop_front().map(|frame| {
generate_animation_frame(
&ctx,
&url,
0,
frame,
maybe_encoder_input.as_ref(),
process_to_egui,
)
});
let cur_url = url.clone();
thread::spawn(move || {
for (index, frame) in frames.into_iter().enumerate() {
let texture_frame = generate_animation_frame(
&ctx,
&cur_url,
index,
frame,
maybe_encoder_input.as_ref(),
process_to_egui,
);
if tex_input.send(texture_frame).is_err() {
tracing::error!("AnimationTextureFrame mpsc stopped abruptly");
break;
}
}
});
if let Some(encoder_output) = maybe_encoder_output {
let path = path.to_owned();
thread::spawn(move || {
let mut imgs = Vec::new();
while let Ok(img) = encoder_output.recv() {
imgs.push(img);
}
if let Err(e) = MediaCache::write_gif(&path, &url, imgs) {
tracing::error!("Could not write gif to disk: {e}");
}
});
}
first_frame.map_or_else(
|| {
Err(notedeck::Error::Generic(
"first frame not found for gif".to_owned(),
))
},
|first_frame| {
Ok(TexturedImage::Animated(Animation {
other_frames: Default::default(),
receiver: Some(tex_output),
first_frame,
}))
},
)
}
fn generate_animation_frame(
ctx: &egui::Context,
url: &str,
index: usize,
frame: image::Frame,
maybe_encoder_input: Option<&SyncSender<ImageFrame>>,
process_to_egui: impl Fn(DynamicImage) -> ColorImage + Send + 'static,
) -> TextureFrame {
let delay = Duration::from(frame.delay());
let img = DynamicImage::ImageRgba8(frame.into_buffer());
let color_img = process_to_egui(img);
if let Some(sender) = maybe_encoder_input {
if let Err(e) = sender.send(ImageFrame {
delay,
image: color_img.clone(),
}) {
tracing::error!("ImageFrame mpsc unexpectedly closed: {e}");
}
}
TextureFrame {
delay,
texture: ctx.load_texture(format!("{}{}", url, index), color_img, Default::default()),
}
}
fn buffer_to_color_image(
samples: Option<FlatSamples<&[u8]>>,
width: u32,
height: u32,
) -> ColorImage {
// TODO(jb55): remove unwrap here
let flat_samples = samples.unwrap();
ColorImage::from_rgba_unmultiplied([width as usize, height as usize], flat_samples.as_slice())
}
pub fn fetch_binary_from_disk(path: PathBuf) -> Result<Vec<u8>> {
std::fs::read(path).map_err(|e| notedeck::Error::Generic(e.to_string()))
}
@@ -222,14 +363,15 @@ pub fn fetch_img(
ctx: &egui::Context,
url: &str,
imgtyp: ImageType,
cache_type: MediaCacheType,
) -> Promise<Result<TexturedImage>> {
let key = MediaCache::key(url);
let path = img_cache.cache_dir.join(key);
if path.exists() {
fetch_img_from_disk(ctx, url, &path)
fetch_img_from_disk(ctx, url, &path, cache_type)
} else {
fetch_img_from_net(&img_cache.cache_dir, ctx, url, imgtyp)
fetch_img_from_net(&img_cache.cache_dir, ctx, url, imgtyp, cache_type)
}
// TODO: fetch image from local cache
@@ -240,6 +382,7 @@ fn fetch_img_from_net(
ctx: &egui::Context,
url: &str,
imgtyp: ImageType,
cache_type: MediaCacheType,
) -> Promise<Result<TexturedImage>> {
let (sender, promise) = Promise::new();
let request = ehttp::Request::get(url);
@@ -247,17 +390,35 @@ fn fetch_img_from_net(
let cloned_url = url.to_owned();
let cache_path = cache_path.to_owned();
ehttp::fetch(request, move |response| {
let handle = response
.map_err(notedeck::Error::Generic)
.and_then(|resp| parse_img_response(resp, imgtyp))
.map(|img| {
let texture_handle = ctx.load_texture(&cloned_url, img.clone(), Default::default());
let handle = response.map_err(notedeck::Error::Generic).and_then(|resp| {
match cache_type {
MediaCacheType::Image => {
let img = parse_img_response(resp, imgtyp);
img.map(|img| {
let texture_handle =
ctx.load_texture(&cloned_url, img.clone(), Default::default());
// write to disk
std::thread::spawn(move || MediaCache::write(&cache_path, &cloned_url, img));
// write to disk
std::thread::spawn(move || {
MediaCache::write(&cache_path, &cloned_url, img)
});
TexturedImage::Static(texture_handle)
});
TexturedImage::Static(texture_handle)
})
}
MediaCacheType::Gif => {
let gif_bytes = resp.bytes;
generate_gif(
ctx.clone(),
cloned_url,
&cache_path,
gif_bytes,
true,
move |img| process_pfp_bitmap(imgtyp, img),
)
}
}
});
sender.send(handle); // send the results back to the UI thread.
ctx.request_repaint();

25
crates/notedeck_columns/src/ui/images.rs
View File

@@ -1,44 +1,54 @@
use notedeck::{Images, MediaCache, TexturedImage};
use notedeck::{GifStateMap, Images, MediaCache, MediaCacheType, TexturedImage};
use crate::images::ImageType;
use super::ProfilePic;
#[allow(clippy::too_many_arguments)]
pub fn render_images(
ui: &mut egui::Ui,
images: &mut Images,
url: &str,
img_type: ImageType,
cache_type: MediaCacheType,
show_waiting: impl FnOnce(&mut egui::Ui),
show_error: impl FnOnce(&mut egui::Ui, String),
show_success: impl FnOnce(&mut egui::Ui, &str, &mut TexturedImage),
show_success: impl FnOnce(&mut egui::Ui, &str, &mut TexturedImage, &mut GifStateMap),
) -> egui::Response {
let cache = &mut images.static_imgs;
let cache = match cache_type {
MediaCacheType::Image => &mut images.static_imgs,
MediaCacheType::Gif => &mut images.gifs,
};
render_media_cache(
ui,
cache,
&mut images.gif_states,
url,
img_type,
cache_type,
show_waiting,
show_error,
show_success,
)
}
pub fn render_media_cache(
#[allow(clippy::too_many_arguments)]
fn render_media_cache(
ui: &mut egui::Ui,
cache: &mut MediaCache,
gif_states: &mut GifStateMap,
url: &str,
img_type: ImageType,
cache_type: MediaCacheType,
show_waiting: impl FnOnce(&mut egui::Ui),
show_error: impl FnOnce(&mut egui::Ui, String),
show_success: impl FnOnce(&mut egui::Ui, &str, &mut TexturedImage),
show_success: impl FnOnce(&mut egui::Ui, &str, &mut TexturedImage, &mut GifStateMap),
) -> egui::Response {
let m_cached_promise = cache.map().get(url);
if m_cached_promise.is_none() {
let res = crate::images::fetch_img(cache, ui.ctx(), url, img_type);
let res = crate::images::fetch_img(cache, ui.ctx(), url, img_type, cache_type.clone());
cache.map_mut().insert(url.to_owned(), res);
}
@@ -53,11 +63,12 @@ pub fn render_media_cache(
ui.ctx(),
ProfilePic::no_pfp_url(),
ImageType::Profile(128),
cache_type,
);
cache.map_mut().insert(url.to_owned(), no_pfp);
show_error(ui, err)
}
Some(Ok(renderable_media)) => show_success(ui, url, renderable_media),
Some(Ok(renderable_media)) => show_success(ui, url, renderable_media, gif_states),
}
})
.response

25
crates/notedeck_columns/src/ui/note/contents.rs
View File

@@ -1,3 +1,4 @@
use crate::gif::{handle_repaint, retrieve_latest_texture};
use crate::ui::images::render_images;
use crate::ui::{
self,
@@ -8,7 +9,7 @@ use egui::{Color32, Hyperlink, Image, RichText};
use nostrdb::{BlockType, Mention, Ndb, Note, NoteKey, Transaction};
use tracing::warn;
use notedeck::{supported_mime_hosted_at_url, Images, NoteCache};
use notedeck::{supported_mime_hosted_at_url, Images, MediaCacheType, NoteCache};
pub struct NoteContents<'a> {
ndb: &'a Ndb,
@@ -142,7 +143,7 @@ fn render_note_contents(
puffin::profile_function!();
let selectable = options.has_selectable_text();
let mut images: Vec<String> = vec![];
let mut images: Vec<(String, MediaCacheType)> = vec![];
let mut note_action: Option<NoteAction> = None;
let mut inline_note: Option<(&[u8; 32], &str)> = None;
let hide_media = options.has_hide_media();
@@ -211,8 +212,10 @@ fn render_note_contents(
if !hide_media {
let url = block.as_str().to_string();
if supported_mime_hosted_at_url(&mut img_cache.urls, &url) {
images.push(url);
if let Some(cache_type) =
supported_mime_hosted_at_url(&mut img_cache.urls, &url)
{
images.push((url, cache_type));
}
} else {
#[cfg(feature = "profiling")]
@@ -280,7 +283,7 @@ fn rot13(input: &str) -> String {
fn image_carousel(
ui: &mut egui::Ui,
img_cache: &mut Images,
images: Vec<String>,
images: Vec<(String, MediaCacheType)>,
carousel_id: egui::Id,
) {
// let's make sure everything is within our area
@@ -294,25 +297,31 @@ fn image_carousel(
.id_salt(carousel_id)
.show(ui, |ui| {
ui.horizontal(|ui| {
for image in images {
for (image, cache_type) in images {
render_images(
ui,
img_cache,
&image,
ImageType::Content(width.round() as u32, height.round() as u32),
cache_type,
|ui| {
ui.allocate_space(egui::vec2(spinsz, spinsz));
},
|ui, _| {
ui.allocate_space(egui::vec2(spinsz, spinsz));
},
|ui, url, renderable_media| {
|ui, url, renderable_media, gifs| {
let texture = handle_repaint(
ui,
retrieve_latest_texture(&image, gifs, renderable_media),
);
let img_resp = ui.add(
Image::new(notedeck::get_texture(renderable_media))
Image::new(texture)
.max_height(height)
.rounding(5.0)
.fit_to_original_size(1.0),
);
img_resp.context_menu(|ui| {
if ui.button("Copy Link").clicked() {
ui.ctx().copy_text(url.to_owned());

9
crates/notedeck_columns/src/ui/note/post.rs
View File

@@ -1,4 +1,5 @@
use crate::draft::{Draft, Drafts, MentionHint};
use crate::gif::{handle_repaint, retrieve_latest_texture};
use crate::media_upload::{nostrbuild_nip96_upload, MediaPath};
use crate::post::{downcast_post_buffer, MentionType, NewPost};
use crate::profile::get_display_name;
@@ -13,7 +14,7 @@ use egui::{vec2, Frame, Layout, Margin, Pos2, ScrollArea, Sense, TextBuffer};
use enostr::{FilledKeypair, FullKeypair, NoteId, Pubkey, RelayPool};
use nostrdb::{Ndb, Transaction};
use notedeck::{get_texture, Images, NoteCache};
use notedeck::{Images, NoteCache};
use tracing::error;
use super::contents::render_note_preview;
@@ -390,6 +391,7 @@ impl<'a> PostView<'a> {
self.img_cache,
&media.url,
crate::images::ImageType::Content(width, height),
notedeck::MediaCacheType::Image, // TODO(kernelkind): support gifs in PostView
|ui| {
ui.spinner();
},
@@ -397,7 +399,7 @@ impl<'a> PostView<'a> {
self.draft.upload_errors.push(e.to_string());
error!("{e}");
},
|ui, _, renderable_media| {
|ui, url, renderable_media, gifs| {
let media_size = vec2(width as f32, height as f32);
let max_size = vec2(300.0, 300.0);
let size = if media_size.x > max_size.x || media_size.y > max_size.y {
@@ -406,7 +408,8 @@ impl<'a> PostView<'a> {
media_size
};
let texture_handle = get_texture(renderable_media);
let texture_handle =
handle_repaint(ui, retrieve_latest_texture(url, gifs, renderable_media));
let img_resp = ui.add(
egui::Image::new(texture_handle)
.max_size(size)

12
crates/notedeck_columns/src/ui/profile/picture.rs
View File

@@ -1,3 +1,4 @@
use crate::gif::{handle_repaint, retrieve_latest_texture};
use crate::images::ImageType;
use crate::ui::images::render_images;
use crate::ui::{Preview, PreviewConfig};
@@ -5,7 +6,7 @@ use egui::{vec2, Sense, Stroke, TextureHandle};
use nostrdb::{Ndb, Transaction};
use tracing::info;
use notedeck::{AppContext, Images};
use notedeck::{supported_mime_hosted_at_url, AppContext, Images};
pub struct ProfilePic<'cache, 'url> {
cache: &'cache mut Images,
@@ -92,19 +93,24 @@ fn render_pfp(
// We will want to downsample these so it's not blurry on hi res displays
let img_size = 128u32;
let cache_type = supported_mime_hosted_at_url(&mut img_cache.urls, url)
.unwrap_or(notedeck::MediaCacheType::Image);
render_images(
ui,
img_cache,
url,
ImageType::Profile(img_size),
cache_type,
|ui| {
paint_circle(ui, ui_size, border);
},
|ui, _| {
paint_circle(ui, ui_size, border);
},
|ui, _, renderable_media| {
let texture_handle = notedeck::get_texture(renderable_media);
|ui, url, renderable_media, gifs| {
let texture_handle =
handle_repaint(ui, retrieve_latest_texture(url, gifs, renderable_media));
pfp_image(ui, texture_handle, ui_size, border);
},
)