Add Fluent-based localization manager and add script to export source strings for translations

Changelog-Added: Added Fluent-based localization manager and added script to export source strings for translations
Signed-off-by: Terry Yiu <git@tyiu.xyz>
This commit is contained in:
2025-06-26 20:56:32 -04:00
committed by William Casarin
parent 80820a52d2
commit d07c3e9135
9 changed files with 1894 additions and 105 deletions

View File

@@ -39,6 +39,13 @@ bech32 = { workspace = true }
lightning-invoice = { workspace = true }
secp256k1 = { workspace = true }
hashbrown = { workspace = true }
fluent = { workspace = true }
fluent-resmgr = { workspace = true }
fluent-langneg = { workspace = true }
unic-langid = { workspace = true }
once_cell = { workspace = true }
md5 = { workspace = true }
regex = "1"
[dev-dependencies]
tempfile = { workspace = true }

View File

@@ -1,4 +1,5 @@
use crate::account::FALLBACK_PUBKEY;
use crate::i18n::{LocalizationContext, LocalizationManager};
use crate::persist::{AppSizeHandler, ZoomHandler};
use crate::wallet::GlobalWallet;
use crate::zaps::Zaps;
@@ -17,6 +18,7 @@ use std::cell::RefCell;
use std::collections::BTreeSet;
use std::path::Path;
use std::rc::Rc;
use std::sync::Arc;
use tracing::{error, info};
pub enum AppAction {
@@ -48,6 +50,7 @@ pub struct Notedeck {
zaps: Zaps,
frame_history: FrameHistory,
job_pool: JobPool,
i18n: LocalizationContext,
}
/// Our chrome, which is basically nothing
@@ -227,6 +230,21 @@ impl Notedeck {
let zaps = Zaps::default();
let job_pool = JobPool::default();
// Initialize localization
let i18n_resource_dir = Path::new("assets/translations");
let localization_manager = Arc::new(
LocalizationManager::new(&i18n_resource_dir).unwrap_or_else(|e| {
error!("Failed to initialize localization manager: {}", e);
// Create a fallback manager with a temporary directory
LocalizationManager::new(&std::env::temp_dir().join("notedeck_i18n_fallback"))
.expect("Failed to create fallback localization manager")
}),
);
let i18n = LocalizationContext::new(localization_manager);
// Initialize global i18n context
crate::i18n::init_global_i18n(i18n.clone());
Self {
ndb,
img_cache,
@@ -246,6 +264,7 @@ impl Notedeck {
clipboard: Clipboard::new(None),
zaps,
job_pool,
i18n,
}
}
@@ -270,6 +289,7 @@ impl Notedeck {
zaps: &mut self.zaps,
frame_history: &mut self.frame_history,
job_pool: &mut self.job_pool,
i18n: &self.i18n,
}
}

View File

@@ -1,6 +1,7 @@
use crate::{
account::accounts::Accounts, frame_history::FrameHistory, wallet::GlobalWallet, zaps::Zaps,
Args, DataPath, Images, JobPool, NoteCache, ThemeHandler, UnknownIds,
account::accounts::Accounts, frame_history::FrameHistory, i18n::LocalizationContext,
wallet::GlobalWallet, zaps::Zaps, Args, DataPath, Images, JobPool, NoteCache, ThemeHandler,
UnknownIds,
};
use egui_winit::clipboard::Clipboard;
@@ -24,4 +25,5 @@ pub struct AppContext<'a> {
pub zaps: &'a mut Zaps,
pub frame_history: &'a mut FrameHistory,
pub job_pool: &'a mut JobPool,
pub i18n: &'a LocalizationContext,
}

View File

@@ -0,0 +1,766 @@
use fluent::FluentArgs;
use fluent::{FluentBundle, FluentResource};
use fluent_langneg::negotiate_languages;
use std::collections::HashMap;
use std::path::Path;
use std::sync::{Arc, RwLock};
use unic_langid::LanguageIdentifier;
/// Manages localization resources and provides localized strings
pub struct LocalizationManager {
/// Current locale
current_locale: RwLock<LanguageIdentifier>,
/// Available locales
available_locales: Vec<LanguageIdentifier>,
/// Fallback locale
fallback_locale: LanguageIdentifier,
/// Resource directory path
resource_dir: std::path::PathBuf,
/// Cached parsed FluentResource per locale
resource_cache: RwLock<HashMap<LanguageIdentifier, Arc<FluentResource>>>,
/// Cached string results per locale (only for strings without arguments)
string_cache: RwLock<HashMap<LanguageIdentifier, HashMap<String, String>>>,
}
impl LocalizationManager {
/// Creates a new LocalizationManager with the specified resource directory
pub fn new(resource_dir: &Path) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
// Default to English (US)
let default_locale: LanguageIdentifier = "en-US"
.parse()
.map_err(|e| format!("Locale parse error: {e:?}"))?;
let fallback_locale = default_locale.clone();
// Check if pseudolocale is enabled via environment variable
let enable_pseudolocale = std::env::var("NOTEDECK_PSEUDOLOCALE").is_ok();
// Build available locales list
let mut available_locales = vec![default_locale.clone()];
// Add en-XA if pseudolocale is enabled
if enable_pseudolocale {
let pseudolocale: LanguageIdentifier = "en-XA"
.parse()
.map_err(|e| format!("Pseudolocale parse error: {e:?}"))?;
available_locales.push(pseudolocale);
tracing::info!(
"Pseudolocale (en-XA) enabled via NOTEDECK_PSEUDOLOCALE environment variable"
);
}
Ok(Self {
current_locale: RwLock::new(default_locale),
available_locales,
fallback_locale,
resource_dir: resource_dir.to_path_buf(),
resource_cache: RwLock::new(HashMap::new()),
string_cache: RwLock::new(HashMap::new()),
})
}
/// Gets a localized string by its ID
pub fn get_string(&self, id: &str) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
tracing::debug!(
"Getting string '{}' for locale '{}'",
id,
self.get_current_locale()?
);
let result = self.get_string_with_args(id, None);
if let Err(ref e) = result {
tracing::error!("Failed to get string '{}': {}", id, e);
}
result
}
/// Loads and caches a parsed FluentResource for the given locale
fn load_resource_for_locale(
&self,
locale: &LanguageIdentifier,
) -> Result<Arc<FluentResource>, Box<dyn std::error::Error + Send + Sync>> {
// Construct the path using the stored resource directory
let expected_path = self.resource_dir.join(format!("{}/main.ftl", locale));
// Try to open the file directly
if let Err(e) = std::fs::File::open(&expected_path) {
tracing::error!(
"Direct file open failed: {} ({})",
expected_path.display(),
e
);
return Err(format!("Failed to open FTL file: {}", e).into());
}
// Load the FTL file directly instead of using ResourceManager
let ftl_string = std::fs::read_to_string(&expected_path)
.map_err(|e| format!("Failed to read FTL file: {}", e))?;
// Parse the FTL content
let resource = FluentResource::try_new(ftl_string)
.map_err(|e| format!("Failed to parse FTL content: {:?}", e))?;
tracing::debug!(
"Loaded and cached parsed FluentResource for locale: {}",
locale
);
Ok(Arc::new(resource))
}
/// Gets cached parsed FluentResource for the current locale, loading it if necessary
fn get_cached_resource(
&self,
) -> Result<Arc<FluentResource>, Box<dyn std::error::Error + Send + Sync>> {
let locale = self
.current_locale
.read()
.map_err(|e| format!("Lock error: {e}"))?;
// Try to get from cache first
{
let cache = self
.resource_cache
.read()
.map_err(|e| format!("Cache lock error: {e}"))?;
if let Some(resource) = cache.get(&locale) {
tracing::debug!("Using cached parsed FluentResource for locale: {}", locale);
return Ok(resource.clone());
}
}
// Not in cache, load and cache it
let resource = self.load_resource_for_locale(&locale)?;
// Store in cache
{
let mut cache = self
.resource_cache
.write()
.map_err(|e| format!("Cache lock error: {e}"))?;
cache.insert(locale.clone(), resource.clone());
tracing::debug!("Cached parsed FluentResource for locale: {}", locale);
}
Ok(resource)
}
/// Gets cached string result, or formats it and caches the result
fn get_cached_string(
&self,
id: &str,
args: Option<&FluentArgs>,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
let locale = self
.current_locale
.read()
.map_err(|e| format!("Lock error: {e}"))?;
// Only cache simple strings without arguments
// For strings with arguments, we can't cache the final result since args may vary
if args.is_none() {
// Try to get from string cache first
{
let cache = self
.string_cache
.read()
.map_err(|e| format!("String cache lock error: {e}"))?;
if let Some(locale_cache) = cache.get(&locale) {
if let Some(cached_string) = locale_cache.get(id) {
tracing::debug!(
"Using cached string result for '{}' in locale: {}",
id,
locale
);
return Ok(cached_string.clone());
}
}
}
}
// Not in cache or has arguments, format it using cached resource
let resource = self.get_cached_resource()?;
// Create a bundle for this request (not cached due to thread-safety issues)
let mut bundle = FluentBundle::new(vec![locale.clone()]);
bundle
.add_resource(resource.as_ref())
.map_err(|e| format!("Failed to add resource to bundle: {:?}", e))?;
let message = bundle
.get_message(id)
.ok_or_else(|| format!("Message not found: {}", id))?;
let pattern = message
.value()
.ok_or_else(|| format!("Message has no value: {}", id))?;
// Format the message
let mut errors = Vec::new();
let result = bundle.format_pattern(pattern, args, &mut errors);
if !errors.is_empty() {
tracing::warn!("Localization errors for {}: {:?}", id, errors);
}
let result_string = result.into_owned();
// Only cache simple strings without arguments
// This prevents caching issues when the same message ID is used with different arguments
if args.is_none() {
let mut cache = self
.string_cache
.write()
.map_err(|e| format!("String cache lock error: {e}"))?;
let locale_cache = cache.entry(locale.clone()).or_insert_with(HashMap::new);
locale_cache.insert(id.to_string(), result_string.clone());
tracing::debug!("Cached string result for '{}' in locale: {}", id, locale);
} else {
tracing::debug!("Not caching string '{}' due to arguments", id);
}
Ok(result_string)
}
/// Gets a localized string by its ID with optional arguments
pub fn get_string_with_args(
&self,
id: &str,
args: Option<&FluentArgs>,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
self.get_cached_string(id, args)
}
/// Sets the current locale
pub fn set_locale(
&self,
locale: LanguageIdentifier,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
tracing::info!("Attempting to set locale to: {}", locale);
tracing::info!("Available locales: {:?}", self.available_locales);
// Validate that the locale is available
if !self.available_locales.contains(&locale) {
tracing::error!(
"Locale {} is not available. Available locales: {:?}",
locale,
self.available_locales
);
return Err(format!("Locale {} is not available", locale).into());
}
let mut current = self
.current_locale
.write()
.map_err(|e| format!("Lock error: {e}"))?;
tracing::info!("Switching locale from {} to {}", *current, locale);
*current = locale.clone();
tracing::info!("Successfully set locale to: {}", locale);
// Clear caches when locale changes since they are locale-specific
let mut string_cache = self
.string_cache
.write()
.map_err(|e| format!("String cache lock error: {e}"))?;
string_cache.clear();
tracing::debug!("String cache cleared due to locale change");
Ok(())
}
/// Clears the parsed FluentResource cache (useful for development when FTL files change)
pub fn clear_cache(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let mut cache = self
.resource_cache
.write()
.map_err(|e| format!("Cache lock error: {e}"))?;
cache.clear();
tracing::info!("Parsed FluentResource cache cleared");
let mut string_cache = self
.string_cache
.write()
.map_err(|e| format!("String cache lock error: {e}"))?;
string_cache.clear();
tracing::info!("String result cache cleared");
Ok(())
}
/// Gets the current locale
pub fn get_current_locale(
&self,
) -> Result<LanguageIdentifier, Box<dyn std::error::Error + Send + Sync>> {
let current = self
.current_locale
.read()
.map_err(|e| format!("Lock error: {e}"))?;
Ok(current.clone())
}
/// Gets all available locales
pub fn get_available_locales(&self) -> &[LanguageIdentifier] {
&self.available_locales
}
/// Gets the fallback locale
pub fn get_fallback_locale(&self) -> &LanguageIdentifier {
&self.fallback_locale
}
/// Gets cache statistics for monitoring performance
pub fn get_cache_stats(&self) -> Result<CacheStats, Box<dyn std::error::Error + Send + Sync>> {
let resource_cache = self
.resource_cache
.read()
.map_err(|e| format!("Cache lock error: {e}"))?;
let string_cache = self
.string_cache
.read()
.map_err(|e| format!("String cache lock error: {e}"))?;
let mut total_strings = 0;
for locale_cache in string_cache.values() {
total_strings += locale_cache.len();
}
Ok(CacheStats {
resource_cache_size: resource_cache.len(),
string_cache_size: total_strings,
cached_locales: resource_cache.keys().cloned().collect(),
})
}
/// Limits the string cache size to prevent memory growth
pub fn limit_string_cache_size(
&self,
max_strings_per_locale: usize,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let mut string_cache = self
.string_cache
.write()
.map_err(|e| format!("String cache lock error: {e}"))?;
for locale_cache in string_cache.values_mut() {
if locale_cache.len() > max_strings_per_locale {
// Remove oldest entries (simple approach: just clear and let it rebuild)
// In a more sophisticated implementation, you might use an LRU cache
locale_cache.clear();
tracing::debug!("Cleared string cache for locale due to size limit");
}
}
Ok(())
}
/// Negotiates the best locale from a list of preferred locales
pub fn negotiate_locale(&self, preferred: &[LanguageIdentifier]) -> LanguageIdentifier {
let available = self.available_locales.clone();
let negotiated = negotiate_languages(
preferred,
&available,
Some(&self.fallback_locale),
fluent_langneg::NegotiationStrategy::Filtering,
);
negotiated
.first()
.map_or(self.fallback_locale.clone(), |v| (*v).clone())
}
}
/// Context for sharing localization across the application
#[derive(Clone)]
pub struct LocalizationContext {
/// The localization manager
manager: Arc<LocalizationManager>,
}
impl LocalizationContext {
/// Creates a new LocalizationContext
pub fn new(manager: Arc<LocalizationManager>) -> Self {
let context = Self { manager };
// Auto-switch to pseudolocale if environment variable is set
if std::env::var("NOTEDECK_PSEUDOLOCALE").is_ok() {
tracing::info!("NOTEDECK_PSEUDOLOCALE environment variable detected");
if let Ok(pseudolocale) = "en-XA".parse::<LanguageIdentifier>() {
tracing::info!("Attempting to switch to pseudolocale: {}", pseudolocale);
if let Err(e) = context.set_locale(pseudolocale) {
tracing::warn!("Failed to switch to pseudolocale: {}", e);
} else {
tracing::info!("Automatically switched to pseudolocale (en-XA)");
}
} else {
tracing::error!("Failed to parse en-XA as LanguageIdentifier");
}
} else {
tracing::info!("NOTEDECK_PSEUDOLOCALE environment variable not set");
}
context
}
/// Gets a localized string by its ID
pub fn get_string(&self, id: &str) -> Option<String> {
self.manager.get_string(id).ok()
}
/// Gets a localized string by its ID with optional arguments
pub fn get_string_with_args(&self, id: &str, args: Option<&FluentArgs>) -> String {
self.manager
.get_string_with_args(id, args)
.unwrap_or_else(|_| format!("[MISSING: {}]", id))
}
/// Sets the current locale
pub fn set_locale(
&self,
locale: LanguageIdentifier,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
self.manager.set_locale(locale)
}
/// Gets the current locale
pub fn get_current_locale(
&self,
) -> Result<LanguageIdentifier, Box<dyn std::error::Error + Send + Sync>> {
self.manager.get_current_locale()
}
/// Clears the resource cache (useful for development when FTL files change)
pub fn clear_cache(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
self.manager.clear_cache()
}
/// Gets the underlying manager
pub fn manager(&self) -> &Arc<LocalizationManager> {
&self.manager
}
}
/// Trait for objects that can be localized
pub trait Localizable {
/// Gets a localized string by its ID
fn get_localized_string(&self, id: &str) -> String;
/// Gets a localized string by its ID with optional arguments
fn get_localized_string_with_args(&self, id: &str, args: Option<&FluentArgs>) -> String;
}
impl Localizable for LocalizationContext {
fn get_localized_string(&self, id: &str) -> String {
self.get_string(id)
.unwrap_or_else(|| format!("[MISSING: {}]", id))
}
fn get_localized_string_with_args(&self, id: &str, args: Option<&FluentArgs>) -> String {
self.get_string_with_args(id, args)
}
}
/// Statistics about cache usage
#[derive(Debug, Clone)]
pub struct CacheStats {
pub resource_cache_size: usize,
pub string_cache_size: usize,
pub cached_locales: Vec<LanguageIdentifier>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_localization_manager_creation() {
let temp_dir = std::env::temp_dir().join("notedeck_i18n_test");
std::fs::create_dir_all(&temp_dir).unwrap();
let manager = LocalizationManager::new(&temp_dir);
assert!(manager.is_ok());
// Cleanup
std::fs::remove_dir_all(&temp_dir).unwrap();
}
#[test]
fn test_locale_management() {
let temp_dir = std::env::temp_dir().join("notedeck_i18n_test2");
std::fs::create_dir_all(&temp_dir).unwrap();
let manager = LocalizationManager::new(&temp_dir).unwrap();
// Test default locale
let current = manager.get_current_locale().unwrap();
assert_eq!(current.to_string(), "en-US");
// Test available locales
let available = manager.get_available_locales();
assert_eq!(available.len(), 1);
assert_eq!(available[0].to_string(), "en-US");
// Cleanup
std::fs::remove_dir_all(&temp_dir).unwrap();
}
#[test]
fn test_ftl_caching() {
let temp_dir = std::env::temp_dir().join("notedeck_i18n_test3");
std::fs::create_dir_all(&temp_dir).unwrap();
// Create a test FTL file
let en_us_dir = temp_dir.join("en-US");
std::fs::create_dir_all(&en_us_dir).unwrap();
let ftl_content = "test_key = Test Value\nanother_key = Another Value";
std::fs::write(en_us_dir.join("main.ftl"), ftl_content).unwrap();
let manager = LocalizationManager::new(&temp_dir).unwrap();
// First call should load and cache the FTL content
let result1 = manager.get_string("test_key");
assert!(result1.is_ok());
assert_eq!(result1.as_ref().unwrap(), "Test Value");
// Second call should use cached FTL content
let result2 = manager.get_string("test_key");
assert!(result2.is_ok());
assert_eq!(result2.unwrap(), "Test Value");
// Test another key from the same FTL content
let result3 = manager.get_string("another_key");
assert!(result3.is_ok());
assert_eq!(result3.unwrap(), "Another Value");
// Cleanup
std::fs::remove_dir_all(&temp_dir).unwrap();
}
#[test]
fn test_cache_clearing() {
let temp_dir = std::env::temp_dir().join("notedeck_i18n_test4");
std::fs::create_dir_all(&temp_dir).unwrap();
// Create a test FTL file
let en_us_dir = temp_dir.join("en-US");
std::fs::create_dir_all(&en_us_dir).unwrap();
let ftl_content = "test_key = Test Value";
std::fs::write(en_us_dir.join("main.ftl"), ftl_content).unwrap();
let manager = LocalizationManager::new(&temp_dir).unwrap();
// Load and cache the FTL content
let result1 = manager.get_string("test_key");
assert!(result1.is_ok());
// Clear the cache
let clear_result = manager.clear_cache();
assert!(clear_result.is_ok());
// Should still work after clearing cache (will reload)
let result2 = manager.get_string("test_key");
assert!(result2.is_ok());
assert_eq!(result2.unwrap(), "Test Value");
// Cleanup
std::fs::remove_dir_all(&temp_dir).unwrap();
}
#[test]
fn test_context_caching() {
let temp_dir = std::env::temp_dir().join("notedeck_i18n_test5");
std::fs::create_dir_all(&temp_dir).unwrap();
// Create a test FTL file
let en_us_dir = temp_dir.join("en-US");
std::fs::create_dir_all(&en_us_dir).unwrap();
let ftl_content = "test_key = Test Value";
std::fs::write(en_us_dir.join("main.ftl"), ftl_content).unwrap();
let manager = Arc::new(LocalizationManager::new(&temp_dir).unwrap());
let context = LocalizationContext::new(manager);
// Debug: check what the normalized key should be
let normalized_key = crate::i18n::normalize_ftl_key("test_key", None);
println!("Normalized key: '{}'", normalized_key);
// First call should load and cache the FTL content
let result1 = context.get_string("test_key");
println!("First result: {:?}", result1);
assert!(result1.is_some());
assert_eq!(result1.unwrap(), "Test Value");
// Second call should use cached FTL content
let result2 = context.get_string("test_key");
assert!(result2.is_some());
assert_eq!(result2.unwrap(), "Test Value");
// Test cache clearing through context
let clear_result = context.clear_cache();
assert!(clear_result.is_ok());
// Should still work after clearing cache
let result3 = context.get_string("test_key");
assert!(result3.is_some());
assert_eq!(result3.unwrap(), "Test Value");
// Cleanup
std::fs::remove_dir_all(&temp_dir).unwrap();
}
#[test]
fn test_bundle_caching() {
let temp_dir = std::env::temp_dir().join("notedeck_i18n_test6");
std::fs::create_dir_all(&temp_dir).unwrap();
// Create a test FTL file
let en_us_dir = temp_dir.join("en-US");
std::fs::create_dir_all(&en_us_dir).unwrap();
let ftl_content = "test_key = Test Value\nanother_key = Another Value";
std::fs::write(en_us_dir.join("main.ftl"), ftl_content).unwrap();
let manager = LocalizationManager::new(&temp_dir).unwrap();
// First call should create bundle and cache the resource
let result1 = manager.get_string("test_key");
assert!(result1.is_ok());
assert_eq!(result1.unwrap(), "Test Value");
// Second call should use cached resource but create new bundle
let result2 = manager.get_string("another_key");
assert!(result2.is_ok());
assert_eq!(result2.unwrap(), "Another Value");
// Check cache stats
let stats = manager.get_cache_stats().unwrap();
assert_eq!(stats.resource_cache_size, 1);
assert_eq!(stats.string_cache_size, 2); // Both strings should be cached
// Cleanup
std::fs::remove_dir_all(&temp_dir).unwrap();
}
#[test]
fn test_string_caching() {
let temp_dir = std::env::temp_dir().join("notedeck_i18n_test7");
std::fs::create_dir_all(&temp_dir).unwrap();
// Create a test FTL file
let en_us_dir = temp_dir.join("en-US");
std::fs::create_dir_all(&en_us_dir).unwrap();
let ftl_content = "test_key = Test Value";
std::fs::write(en_us_dir.join("main.ftl"), ftl_content).unwrap();
let manager = LocalizationManager::new(&temp_dir).unwrap();
// First call should format and cache the string
let result1 = manager.get_string("test_key");
assert!(result1.is_ok());
assert_eq!(result1.unwrap(), "Test Value");
// Second call should use cached string
let result2 = manager.get_string("test_key");
assert!(result2.is_ok());
assert_eq!(result2.unwrap(), "Test Value");
// Check cache stats
let stats = manager.get_cache_stats().unwrap();
assert_eq!(stats.string_cache_size, 1);
// Cleanup
std::fs::remove_dir_all(&temp_dir).unwrap();
}
#[test]
fn test_cache_clearing_on_locale_change() {
let temp_dir = std::env::temp_dir().join("notedeck_i18n_test8");
std::fs::create_dir_all(&temp_dir).unwrap();
// Create test FTL files for two locales
let en_us_dir = temp_dir.join("en-US");
std::fs::create_dir_all(&en_us_dir).unwrap();
std::fs::write(en_us_dir.join("main.ftl"), "test_key = Test Value").unwrap();
let en_xa_dir = temp_dir.join("en-XA");
std::fs::create_dir_all(&en_xa_dir).unwrap();
std::fs::write(en_xa_dir.join("main.ftl"), "test_key = Test Value XA").unwrap();
// Enable pseudolocale for this test
std::env::set_var("NOTEDECK_PSEUDOLOCALE", "1");
let manager = LocalizationManager::new(&temp_dir).unwrap();
// Load some strings in en-US
let result1 = manager.get_string("test_key");
assert!(result1.is_ok());
// Check that caches are populated
let stats1 = manager.get_cache_stats().unwrap();
assert!(stats1.resource_cache_size > 0);
assert!(stats1.string_cache_size > 0);
// Switch to en-XA
let en_xa: LanguageIdentifier = "en-XA".parse().unwrap();
manager.set_locale(en_xa).unwrap();
// Check that string cache is cleared (resource cache remains for both locales)
let stats2 = manager.get_cache_stats().unwrap();
assert_eq!(stats2.string_cache_size, 0);
// Cleanup
std::env::remove_var("NOTEDECK_PSEUDOLOCALE");
std::fs::remove_dir_all(&temp_dir).unwrap();
}
#[test]
fn test_string_caching_with_arguments() {
let temp_dir = std::env::temp_dir().join("notedeck_i18n_test9");
std::fs::create_dir_all(&temp_dir).unwrap();
// Create a test FTL file with a message that takes arguments
let en_us_dir = temp_dir.join("en-US");
std::fs::create_dir_all(&en_us_dir).unwrap();
let ftl_content = "welcome_message = Welcome {$name}!";
std::fs::write(en_us_dir.join("main.ftl"), ftl_content).unwrap();
let manager = LocalizationManager::new(&temp_dir).unwrap();
// First call with arguments should not be cached
let mut args = fluent::FluentArgs::new();
args.set("name", "Alice");
let result1 = manager.get_string_with_args("welcome_message", Some(&args));
assert!(result1.is_ok());
// Note: Fluent may add bidirectional text control characters, so we check contains
let result1_str = result1.unwrap();
assert!(result1_str.contains("Alice"));
// Check that it's not in the string cache
let stats1 = manager.get_cache_stats().unwrap();
assert_eq!(stats1.string_cache_size, 0);
// Second call with different arguments should work correctly
let mut args2 = fluent::FluentArgs::new();
args2.set("name", "Bob");
let result2 = manager.get_string_with_args("welcome_message", Some(&args2));
assert!(result2.is_ok());
let result2_str = result2.unwrap();
assert!(result2_str.contains("Bob"));
// Check that it's still not in the string cache
let stats2 = manager.get_cache_stats().unwrap();
assert_eq!(stats2.string_cache_size, 0);
// Test a simple string without arguments - should be cached
let ftl_content_simple = "simple_message = Hello World";
std::fs::write(en_us_dir.join("main.ftl"), ftl_content_simple).unwrap();
// Clear cache to start fresh
manager.clear_cache().unwrap();
let result3 = manager.get_string("simple_message");
assert!(result3.is_ok());
assert_eq!(result3.unwrap(), "Hello World");
// Check that simple string is cached
let stats3 = manager.get_cache_stats().unwrap();
assert_eq!(stats3.string_cache_size, 1);
// Cleanup
std::fs::remove_dir_all(&temp_dir).unwrap();
}
}

View File

@@ -0,0 +1,222 @@
//! Internationalization (i18n) module for Notedeck
//!
//! This module provides localization support using fluent and fluent-resmgr.
//! It handles loading translation files, managing locales, and providing
//! localized strings throughout the application.
pub mod manager;
pub use manager::CacheStats;
pub use manager::LocalizationContext;
pub use manager::LocalizationManager;
/// Re-export commonly used types for convenience
pub use fluent::FluentArgs;
pub use fluent::FluentValue;
pub use unic_langid::LanguageIdentifier;
use md5;
use once_cell::sync::OnceCell;
use regex::Regex;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::Mutex;
use tracing::info;
/// Global localization manager for easy access from anywhere
static GLOBAL_I18N: OnceCell<Arc<LocalizationManager>> = OnceCell::new();
/// Cache for normalized FTL keys to avoid repeated normalization
static NORMALIZED_KEY_CACHE: OnceCell<Mutex<HashMap<String, String>>> = OnceCell::new();
/// Initialize the global localization context
pub fn init_global_i18n(context: LocalizationContext) {
info!("Initializing global i18n context");
let _ = GLOBAL_I18N.set(context.manager().clone());
// Initialize the normalized key cache
let _ = NORMALIZED_KEY_CACHE.set(Mutex::new(HashMap::new()));
info!("Global i18n context initialized successfully");
}
/// Get the global localization manager
pub fn get_global_i18n() -> Option<Arc<LocalizationManager>> {
GLOBAL_I18N.get().cloned()
}
fn simple_hash(s: &str) -> String {
let digest = md5::compute(s.as_bytes());
// Take the first 2 bytes and convert to 4 hex characters
format!("{:02x}{:02x}", digest[0], digest[1])
}
pub fn normalize_ftl_key(key: &str, comment: Option<&str>) -> String {
// Try to get from cache first
let cache_key = if let Some(comment) = comment {
format!("{}:{}", key, comment)
} else {
key.to_string()
};
if let Some(cache) = NORMALIZED_KEY_CACHE.get() {
if let Ok(cache) = cache.lock() {
if let Some(cached) = cache.get(&cache_key) {
return cached.clone();
}
}
}
// Replace each invalid character with exactly one underscore
// This matches the behavior of the Python extraction script
let re = Regex::new(r"[^a-zA-Z0-9_-]").unwrap();
let mut result = re.replace_all(key, "_").to_string();
// Remove leading/trailing underscores
result = result.trim_matches('_').to_string();
// Ensure the key starts with a letter (Fluent requirement)
if !(result.len() > 0 && result.chars().next().unwrap().is_ascii_alphabetic()) {
result = format!("k_{}", result);
}
// If we have a comment, append a hash of it to reduce collisions
if let Some(comment) = comment {
let hash_str = format!("_{}", simple_hash(comment));
result.push_str(&hash_str);
}
// Cache the result
if let Some(cache) = NORMALIZED_KEY_CACHE.get() {
if let Ok(mut cache) = cache.lock() {
cache.insert(cache_key, result.clone());
}
}
tracing::debug!(
"normalize_ftl_key: original='{}', comment='{:?}', final='{}'",
key,
comment,
result
);
result
}
/// Macro for getting localized strings with format-like syntax
///
/// Syntax: tr!("message", comment)
/// tr!("message with {param}", comment, param="value")
/// tr!("message with {first} and {second}", comment, first="value1", second="value2")
///
/// The first argument is the source message (like format!).
/// The second argument is always the comment to provide context for translators.
/// If `{name}` placeholders are found, there must be corresponding named arguments after the comment.
/// All placeholders must be named and start with a letter (a-zA-Z).
#[macro_export]
macro_rules! tr {
// Simple case: just message and comment
($message:expr, $comment:expr) => {
{
let norm_key = $crate::i18n::normalize_ftl_key($message, Some($comment));
if let Some(i18n) = $crate::i18n::get_global_i18n() {
let result = i18n.get_string(&norm_key);
match result {
Ok(ref s) if s != $message => s.clone(),
_ => {
tracing::warn!("FALLBACK: Using key '{}' as string (not found in FTL)", $message);
$message.to_string()
}
}
} else {
tracing::warn!("FALLBACK: Global i18n not initialized, using key '{}' as string", $message);
$message.to_string()
}
}
};
// Case with named parameters: message, comment, param=value, ...
($message:expr, $comment:expr, $($param:ident = $value:expr),*) => {
{
let norm_key = $crate::i18n::normalize_ftl_key($message, Some($comment));
if let Some(i18n) = $crate::i18n::get_global_i18n() {
let mut args = $crate::i18n::FluentArgs::new();
$(
args.set(stringify!($param), $value);
)*
match i18n.get_string_with_args(&norm_key, Some(&args)) {
Ok(s) => s,
Err(_) => {
// Fallback: replace placeholders with values
let mut result = $message.to_string();
$(
result = result.replace(&format!("{{{}}}", stringify!($param)), &$value.to_string());
)*
result
}
}
} else {
// Fallback: replace placeholders with values
let mut result = $message.to_string();
$(
result = result.replace(&format!("{{{}}}", stringify!($param)), &$value.to_string());
)*
result
}
}
};
}
/// Macro for getting localized pluralized strings with count and named arguments
///
/// Syntax: tr_plural!(one, other, comment, count, param1=..., param2=...)
/// - one: Message for the singular ("one") plural rule
/// - other: Message for the "other" plural rule
/// - comment: Context for translators
/// - count: The count value
/// - named arguments: Any additional named parameters for interpolation
#[macro_export]
macro_rules! tr_plural {
// With named parameters
($one:expr, $other:expr, $comment:expr, $count:expr, $($param:ident = $value:expr),*) => {{
let norm_key = $crate::i18n::normalize_ftl_key($other, Some($comment));
if let Some(i18n) = $crate::i18n::get_global_i18n() {
let mut args = $crate::i18n::FluentArgs::new();
args.set("count", $count);
$(args.set(stringify!($param), $value);)*
match i18n.get_string_with_args(&norm_key, Some(&args)) {
Ok(s) => s,
Err(_) => {
// Fallback: use simple pluralization
if $count == 1 {
let mut result = $one.to_string();
$(result = result.replace(&format!("{{{}}}", stringify!($param)), &$value.to_string());)*
result = result.replace("{count}", &$count.to_string());
result
} else {
let mut result = $other.to_string();
$(result = result.replace(&format!("{{{}}}", stringify!($param)), &$value.to_string());)*
result = result.replace("{count}", &$count.to_string());
result
}
}
}
} else {
// Fallback: use simple pluralization
if $count == 1 {
let mut result = $one.to_string();
$(result = result.replace(&format!("{{{}}}", stringify!($param)), &$value.to_string());)*
result = result.replace("{count}", &$count.to_string());
result
} else {
let mut result = $other.to_string();
$(result = result.replace(&format!("{{{}}}", stringify!($param)), &$value.to_string());)*
result = result.replace("{count}", &$count.to_string());
result
}
}
}};
// Without named parameters
($one:expr, $other:expr, $comment:expr, $count:expr) => {{
$crate::tr_plural!($one, $other, $comment, $count, )
}};
}

View File

@@ -9,6 +9,7 @@ mod error;
pub mod filter;
pub mod fonts;
mod frame_history;
pub mod i18n;
mod imgcache;
mod job_pool;
mod muted;
@@ -44,6 +45,11 @@ pub use context::AppContext;
pub use error::{show_one_error_message, Error, FilterError, ZapError};
pub use filter::{FilterState, FilterStates, UnifiedSubscription};
pub use fonts::NamedFontFamily;
pub use i18n::manager::Localizable;
pub use i18n::{
CacheStats, FluentArgs, FluentValue, LanguageIdentifier, LocalizationContext,
LocalizationManager,
};
pub use imgcache::{
Animation, GifState, GifStateMap, ImageFrame, Images, LoadableTextureState, MediaCache,
MediaCacheType, TextureFrame, TextureState, TexturedImage, TexturesCache,
@@ -83,3 +89,5 @@ pub use enostr;
pub use nostrdb;
pub use zaps::Zaps;
pub use crate::i18n::{get_global_i18n, init_global_i18n};