From 06417ff69e772f24ffd7fb2b025f879463d8c51f Mon Sep 17 00:00:00 2001 From: kieran Date: Tue, 7 Jan 2025 12:00:46 +0000 Subject: [PATCH 1/3] image-cache: use sha256 hash of url for key --- Cargo.lock | 1 + Cargo.toml | 1 + crates/notedeck/Cargo.toml | 1 + crates/notedeck/src/imgcache.rs | 15 +++++++++++++-- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 64d7c7ea..eeefe65b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2505,6 +2505,7 @@ dependencies = [ "security-framework", "serde", "serde_json", + "sha2", "strum", "strum_macros", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 20b1c77b..60cf1f1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ url = "2.5.2" urlencoding = "2.1.3" uuid = { version = "1.10.0", features = ["v4"] } security-framework = "2.11.0" +sha2 = "0.10.8" [profile.small] inherits = 'release' diff --git a/crates/notedeck/Cargo.toml b/crates/notedeck/Cargo.toml index 31bdfba2..b8138641 100644 --- a/crates/notedeck/Cargo.toml +++ b/crates/notedeck/Cargo.toml @@ -22,6 +22,7 @@ serde = { workspace = true } hex = { workspace = true } thiserror = { workspace = true } puffin = { workspace = true, optional = true } +sha2 = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/notedeck/src/imgcache.rs b/crates/notedeck/src/imgcache.rs index 4a78fc00..00f6a1f1 100644 --- a/crates/notedeck/src/imgcache.rs +++ b/crates/notedeck/src/imgcache.rs @@ -5,9 +5,12 @@ use poll_promise::Promise; use egui::ColorImage; use std::collections::HashMap; -use std::fs::File; +use std::fs::{create_dir_all, File}; +use hex::ToHex; +use sha2::Digest; use std::path; +use std::path::PathBuf; pub type ImageCacheValue = Promise>; pub type ImageCacheMap = HashMap; @@ -46,6 +49,9 @@ impl ImageCache { pub fn write(cache_dir: &path::Path, url: &str, data: ColorImage) -> Result<()> { let file_path = cache_dir.join(Self::key(url)); + if let Some(p) = file_path.parent() { + create_dir_all(p)?; + } let file = File::options() .write(true) .create(true) @@ -64,7 +70,12 @@ impl ImageCache { } pub fn key(url: &str) -> String { - base32::encode(base32::Alphabet::Crockford, url.as_bytes()) + let k: String = sha2::Sha256::digest(url.as_bytes()).encode_hex(); + PathBuf::from(&k[0..2]) + .join(&k[2..4]) + .join(k) + .to_string_lossy() + .to_string() } pub fn map(&self) -> &ImageCacheMap { From eaa9b3ae4cc1131361bb9699db674d11282f2fd9 Mon Sep 17 00:00:00 2001 From: kieran Date: Thu, 9 Jan 2025 20:40:35 +0000 Subject: [PATCH 2/3] image-cache: migrate storage --- crates/notedeck/src/imgcache.rs | 36 +++++++++++++++++++++++++++++++ crates/notedeck_chrome/src/app.rs | 3 +++ 2 files changed, 39 insertions(+) diff --git a/crates/notedeck/src/imgcache.rs b/crates/notedeck/src/imgcache.rs index 00f6a1f1..432a9fe3 100644 --- a/crates/notedeck/src/imgcache.rs +++ b/crates/notedeck/src/imgcache.rs @@ -11,6 +11,7 @@ use hex::ToHex; use sha2::Digest; use std::path; use std::path::PathBuf; +use tracing::warn; pub type ImageCacheValue = Promise>; pub type ImageCacheMap = HashMap; @@ -78,6 +79,41 @@ impl ImageCache { .to_string() } + /// Migrate from base32 encoded url to sha256 url + sub-dir structure + pub fn migrate_v0(&self) -> Result<()> { + for file in std::fs::read_dir(&self.cache_dir)? { + if let Ok(file) = file { + if !file.path().is_file() { + continue; + } + let old_filename = file.file_name().to_string_lossy().to_string(); + let old_url = if let Some(u) = + base32::decode(base32::Alphabet::Crockford, &old_filename) + .and_then(|s| String::from_utf8(s).ok()) + { + u + } else { + warn!("Invalid base32 filename: {}", &old_filename); + continue; + }; + let new_path = self.cache_dir.join(Self::key(&old_url)); + if let Some(p) = new_path.parent() { + create_dir_all(p)?; + } + + if let Err(e) = std::fs::rename(file.path(), &new_path) { + warn!( + "Failed to migrate file from {} to {}: {:?}", + file.path().display(), + new_path.display(), + e + ); + } + } + } + Ok(()) + } + pub fn map(&self) -> &ImageCacheMap { &self.url_imgs } diff --git a/crates/notedeck_chrome/src/app.rs b/crates/notedeck_chrome/src/app.rs index 6cc6435b..6551eb67 100644 --- a/crates/notedeck_chrome/src/app.rs +++ b/crates/notedeck_chrome/src/app.rs @@ -207,6 +207,9 @@ impl Notedeck { let tabs = Tabs::new(None); let app_rect_handler = AppSizeHandler::new(&path); + // migrate + img_cache.migrate_v0().expect("img-cache migration"); + Self { ndb, img_cache, From 1a744d8e3b75e2f54b6b5a4ee7eb7898f33025d7 Mon Sep 17 00:00:00 2001 From: kieran Date: Fri, 10 Jan 2025 09:31:04 +0000 Subject: [PATCH 3/3] image-cache: remove expect --- crates/notedeck/src/imgcache.rs | 56 +++++++++++++++++-------------- crates/notedeck_chrome/src/app.rs | 4 ++- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/crates/notedeck/src/imgcache.rs b/crates/notedeck/src/imgcache.rs index 432a9fe3..9cbe31a1 100644 --- a/crates/notedeck/src/imgcache.rs +++ b/crates/notedeck/src/imgcache.rs @@ -82,33 +82,37 @@ impl ImageCache { /// Migrate from base32 encoded url to sha256 url + sub-dir structure pub fn migrate_v0(&self) -> Result<()> { for file in std::fs::read_dir(&self.cache_dir)? { - if let Ok(file) = file { - if !file.path().is_file() { - continue; - } - let old_filename = file.file_name().to_string_lossy().to_string(); - let old_url = if let Some(u) = - base32::decode(base32::Alphabet::Crockford, &old_filename) - .and_then(|s| String::from_utf8(s).ok()) - { - u - } else { - warn!("Invalid base32 filename: {}", &old_filename); - continue; - }; - let new_path = self.cache_dir.join(Self::key(&old_url)); - if let Some(p) = new_path.parent() { - create_dir_all(p)?; - } + let file = if let Ok(f) = file { + f + } else { + // not sure how this could fail, skip entry + continue; + }; + if !file.path().is_file() { + continue; + } + let old_filename = file.file_name().to_string_lossy().to_string(); + let old_url = if let Some(u) = + base32::decode(base32::Alphabet::Crockford, &old_filename) + .and_then(|s| String::from_utf8(s).ok()) + { + u + } else { + warn!("Invalid base32 filename: {}", &old_filename); + continue; + }; + let new_path = self.cache_dir.join(Self::key(&old_url)); + if let Some(p) = new_path.parent() { + create_dir_all(p)?; + } - if let Err(e) = std::fs::rename(file.path(), &new_path) { - warn!( - "Failed to migrate file from {} to {}: {:?}", - file.path().display(), - new_path.display(), - e - ); - } + if let Err(e) = std::fs::rename(file.path(), &new_path) { + warn!( + "Failed to migrate file from {} to {}: {:?}", + file.path().display(), + new_path.display(), + e + ); } } Ok(()) diff --git a/crates/notedeck_chrome/src/app.rs b/crates/notedeck_chrome/src/app.rs index 6551eb67..c420d0d3 100644 --- a/crates/notedeck_chrome/src/app.rs +++ b/crates/notedeck_chrome/src/app.rs @@ -208,7 +208,9 @@ impl Notedeck { let app_rect_handler = AppSizeHandler::new(&path); // migrate - img_cache.migrate_v0().expect("img-cache migration"); + if let Err(e) = img_cache.migrate_v0() { + error!("error migrating image cache: {e}"); + } Self { ndb,