tyiu/notedeck
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

switch to TimelineCache

Browse Source 
Signed-off-by: William Casarin <jb55@jb55.com>
This commit is contained in:
William Casarin
2025-01-19 12:42:41 -08:00
parent e52ba5937f
commit 4b542c0a74
19 changed files with 720 additions and 673 deletions
Download Patch File Download Diff File Expand all files Collapse all files

276
crates/notedeck_columns/src/timeline/cache.rs Normal file
View File

@@ -0,0 +1,276 @@
use crate::{
actionbar::TimelineOpenResult,
multi_subscriber::MultiSubscriber,
profile::Profile,
thread::Thread,
//subscriptions::SubRefs,
timeline::{PubkeySource, Timeline},
};
use notedeck::{NoteCache, NoteRef, RootNoteId, RootNoteIdBuf};
use enostr::{Pubkey, PubkeyRef, RelayPool};
use nostrdb::{Filter, FilterBuilder, Ndb, Transaction};
use std::collections::HashMap;
use tracing::{debug, info, warn};
#[derive(Default)]
pub struct TimelineCache {
pub threads: HashMap<RootNoteIdBuf, Thread>,
pub profiles: HashMap<Pubkey, Profile>,
}
pub enum Vitality<'a, M> {
Fresh(&'a mut M),
Stale(&'a mut M),
}
impl<'a, M> Vitality<'a, M> {
pub fn get_ptr(self) -> &'a mut M {
match self {
Self::Fresh(ptr) => ptr,
Self::Stale(ptr) => ptr,
}
}
pub fn is_stale(&self) -> bool {
match self {
Self::Fresh(_ptr) => false,
Self::Stale(_ptr) => true,
}
}
}
#[derive(Hash, Debug, Copy, Clone)]
pub enum TimelineCacheKey<'a> {
Profile(PubkeyRef<'a>),
Thread(RootNoteId<'a>),
}
impl<'a> TimelineCacheKey<'a> {
pub fn profile(pubkey: PubkeyRef<'a>) -> Self {
Self::Profile(pubkey)
}
pub fn thread(root_id: RootNoteId<'a>) -> Self {
Self::Thread(root_id)
}
pub fn bytes(&self) -> &[u8; 32] {
match self {
Self::Profile(pk) => pk.bytes(),
Self::Thread(root_id) => root_id.bytes(),
}
}
/// The filters used to update our timeline cache
pub fn filters_raw(&self) -> Vec<FilterBuilder> {
match self {
TimelineCacheKey::Thread(root_id) => Thread::filters_raw(*root_id),
TimelineCacheKey::Profile(pubkey) => vec![Filter::new()
.authors([pubkey.bytes()])
.kinds([1])
.limit(notedeck::filter::default_limit())],
}
}
pub fn filters_since(&self, since: u64) -> Vec<Filter> {
self.filters_raw()
.into_iter()
.map(|fb| fb.since(since).build())
.collect()
}
pub fn filters(&self) -> Vec<Filter> {
self.filters_raw()
.into_iter()
.map(|mut fb| fb.build())
.collect()
}
}
impl TimelineCache {
fn contains_key(&self, key: TimelineCacheKey<'_>) -> bool {
match key {
TimelineCacheKey::Profile(pubkey) => self.profiles.contains_key(pubkey.bytes()),
TimelineCacheKey::Thread(root_id) => self.threads.contains_key(root_id.bytes()),
}
}
fn get_expected_mut(&mut self, key: TimelineCacheKey<'_>) -> &mut Timeline {
match key {
TimelineCacheKey::Profile(pubkey) => self
.profiles
.get_mut(pubkey.bytes())
.map(|p| &mut p.timeline),
TimelineCacheKey::Thread(root_id) => self
.threads
.get_mut(root_id.bytes())
.map(|t| &mut t.timeline),
}
.expect("expected notes in timline cache")
}
/// Insert a new profile or thread into the cache, based on the TimelineCacheKey
#[allow(clippy::too_many_arguments)]
fn insert_new(
&mut self,
id: TimelineCacheKey<'_>,
txn: &Transaction,
ndb: &Ndb,
notes: &[NoteRef],
note_cache: &mut NoteCache,
filters: Vec<Filter>,
) {
match id {
TimelineCacheKey::Profile(pubkey) => {
let mut profile = Profile::new(PubkeySource::Explicit(pubkey.to_owned()), filters);
// insert initial notes into timeline
profile.timeline.insert_new(txn, ndb, note_cache, notes);
self.profiles.insert(pubkey.to_owned(), profile);
}
TimelineCacheKey::Thread(root_id) => {
let mut thread = Thread::new(root_id.to_owned());
thread.timeline.insert_new(txn, ndb, note_cache, notes);
self.threads.insert(root_id.to_owned(), thread);
}
}
}
/// Get and/or update the notes associated with this timeline
pub fn notes<'a>(
&'a mut self,
ndb: &Ndb,
note_cache: &mut NoteCache,
txn: &Transaction,
id: TimelineCacheKey<'a>,
) -> Vitality<'a, Timeline> {
// we can't use the naive hashmap entry API here because lookups
// require a copy, wait until we have a raw entry api. We could
// also use hashbrown?
if self.contains_key(id) {
return Vitality::Stale(self.get_expected_mut(id));
}
let filters = id.filters();
let notes = if let Ok(results) = ndb.query(txn, &filters, 1000) {
results
.into_iter()
.map(NoteRef::from_query_result)
.collect()
} else {
debug!("got no results from TimelineCache lookup for {:?}", id);
vec![]
};
if notes.is_empty() {
warn!("NotesHolder query returned 0 notes? ")
} else {
info!("found NotesHolder with {} notes", notes.len());
}
self.insert_new(id, txn, ndb, &notes, note_cache, filters);
Vitality::Fresh(self.get_expected_mut(id))
}
pub fn subscription(
&mut self,
id: TimelineCacheKey<'_>,
) -> Option<&mut Option<MultiSubscriber>> {
match id {
TimelineCacheKey::Profile(pubkey) => self
.profiles
.get_mut(pubkey.bytes())
.map(|p| &mut p.subscription),
TimelineCacheKey::Thread(root_id) => self
.threads
.get_mut(root_id.bytes())
.map(|t| &mut t.subscription),
}
}
pub fn open<'a>(
&mut self,
ndb: &Ndb,
note_cache: &mut NoteCache,
txn: &Transaction,
pool: &mut RelayPool,
id: TimelineCacheKey<'a>,
) -> Option<TimelineOpenResult<'a>> {
let result = match self.notes(ndb, note_cache, txn, id) {
Vitality::Stale(timeline) => {
// The timeline cache is stale, let's update it
let notes = find_new_notes(timeline.all_or_any_notes(), id, txn, ndb);
let cached_timeline_result = if notes.is_empty() {
None
} else {
let new_notes = notes.iter().map(|n| n.key).collect();
Some(TimelineOpenResult::new_notes(new_notes, id))
};
// we can't insert and update the VirtualList now, because we
// are already borrowing it mutably. Let's pass it as a
// result instead
//
// holder.get_view().insert(&notes); <-- no
cached_timeline_result
}
Vitality::Fresh(_timeline) => None,
};
let sub_id = if let Some(sub) = self.subscription(id) {
if let Some(multi_subscriber) = sub {
multi_subscriber.subscribe(ndb, pool);
multi_subscriber.sub.as_ref().map(|s| s.local)
} else {
let mut multi_sub = MultiSubscriber::new(id.filters());
multi_sub.subscribe(ndb, pool);
let sub_id = multi_sub.sub.as_ref().map(|s| s.local);
*sub = Some(multi_sub);
sub_id
}
} else {
None
};
let timeline = self.get_expected_mut(id);
if let Some(sub_id) = sub_id {
timeline.subscription = Some(sub_id);
}
// TODO: We have subscription ids tracked in different places. Fix this
result
}
}
/// Look for new thread notes since our last fetch
fn find_new_notes(
notes: &[NoteRef],
id: TimelineCacheKey<'_>,
txn: &Transaction,
ndb: &Ndb,
) -> Vec<NoteRef> {
if notes.is_empty() {
return vec![];
}
let last_note = notes[0];
let filters = id.filters_since(last_note.created_at + 1);
if let Ok(results) = ndb.query(txn, &filters, 1000) {
debug!("got {} results from NotesHolder update", results.len());
results
.into_iter()
.map(NoteRef::from_query_result)
.collect()
} else {
debug!("got no results from NotesHolder update",);
vec![]
}
}

14
crates/notedeck_columns/src/timeline/kind.rs
View File

@@ -2,7 +2,7 @@ use crate::error::Error;
use crate::timeline::{Timeline, TimelineTab};
use enostr::{Filter, Pubkey};
use nostrdb::{Ndb, Transaction};
use notedeck::{filter::default_limit, FilterError, FilterState};
use notedeck::{filter::default_limit, FilterError, FilterState, RootNoteIdBuf};
use serde::{Deserialize, Serialize};
use std::{borrow::Cow, fmt::Display};
use tracing::{error, warn};
@@ -58,6 +58,9 @@ pub enum TimelineKind {
Profile(PubkeySource),
/// This could be any note id, doesn't need to be the root id
Thread(RootNoteIdBuf),
Universe,
/// Generic filter
@@ -75,6 +78,7 @@ impl Display for TimelineKind {
TimelineKind::Profile(_) => f.write_str("Profile"),
TimelineKind::Universe => f.write_str("Universe"),
TimelineKind::Hashtag(_) => f.write_str("Hashtag"),
TimelineKind::Thread(_) => f.write_str("Thread"),
}
}
}
@@ -88,6 +92,7 @@ impl TimelineKind {
TimelineKind::Universe => None,
TimelineKind::Generic => None,
TimelineKind::Hashtag(_ht) => None,
TimelineKind::Thread(_ht) => None,
}
}
@@ -103,6 +108,10 @@ impl TimelineKind {
TimelineKind::Profile(pk)
}
pub fn thread(root_id: RootNoteIdBuf) -> Self {
TimelineKind::Thread(root_id)
}
pub fn is_notifications(&self) -> bool {
matches!(self, TimelineKind::Notifications(_))
}
@@ -122,6 +131,8 @@ impl TimelineKind {
TimelineTab::no_replies(),
)),
TimelineKind::Thread(root_id) => Some(Timeline::thread(root_id)),
TimelineKind::Generic => {
warn!("you can't convert a TimelineKind::Generic to a Timeline");
None
@@ -213,6 +224,7 @@ impl TimelineKind {
},
TimelineKind::Notifications(_pubkey_source) => ColumnTitle::simple("Notifications"),
TimelineKind::Profile(_pubkey_source) => ColumnTitle::needs_db(self),
TimelineKind::Thread(_root_id) => ColumnTitle::simple("Thread"),
TimelineKind::Universe => ColumnTitle::simple("Universe"),
TimelineKind::Generic => ColumnTitle::simple("Custom"),
TimelineKind::Hashtag(hashtag) => ColumnTitle::formatted(hashtag.to_string()),

209
crates/notedeck_columns/src/timeline/mod.rs
View File

@@ -3,11 +3,13 @@ use crate::{
decks::DecksCache,
error::Error,
subscriptions::{self, SubKind, Subscriptions},
thread::Thread,
Result,
};
use notedeck::{
filter, CachedNote, FilterError, FilterState, FilterStates, NoteCache, NoteRef, UnknownIds,
filter, CachedNote, FilterError, FilterState, FilterStates, NoteCache, NoteRef, RootNoteIdBuf,
UnknownIds,
};
use std::fmt;
@@ -15,16 +17,18 @@ use std::sync::atomic::{AtomicU32, Ordering};
use egui_virtual_list::VirtualList;
use enostr::{PoolRelay, Pubkey, RelayPool};
use nostrdb::{Filter, Ndb, Note, Subscription, Transaction};
use nostrdb::{Filter, Ndb, Note, NoteKey, Subscription, Transaction};
use std::cell::RefCell;
use std::hash::Hash;
use std::rc::Rc;
use tracing::{debug, error, info, warn};
pub mod cache;
pub mod kind;
pub mod route;
pub use cache::{TimelineCache, TimelineCacheKey};
pub use kind::{ColumnTitle, PubkeySource, TimelineKind};
pub use route::TimelineRoute;
@@ -123,7 +127,7 @@ impl TimelineTab {
}
}
pub fn insert(&mut self, new_refs: &[NoteRef], reversed: bool) {
fn insert(&mut self, new_refs: &[NoteRef], reversed: bool) {
if new_refs.is_empty() {
return;
}
@@ -189,7 +193,6 @@ pub struct Timeline {
pub views: Vec<TimelineTab>,
pub selected_view: usize,
/// Our nostrdb subscription
pub subscription: Option<Subscription>,
}
@@ -210,6 +213,18 @@ impl Timeline {
))
}
pub fn thread(note_id: RootNoteIdBuf) -> Self {
let filter = Thread::filters_raw(note_id.borrow())
.iter_mut()
.map(|fb| fb.build())
.collect();
Timeline::new(
TimelineKind::Thread(note_id),
FilterState::ready(filter),
TimelineTab::only_notes_and_replies(),
)
}
pub fn hashtag(hashtag: String) -> Self {
let filter = Filter::new()
.kinds([1])
@@ -280,18 +295,107 @@ impl Timeline {
self.views.iter_mut().find(|tab| tab.filter == view)
}
pub fn poll_notes_into_view(
timeline_idx: usize,
mut timelines: Vec<&mut Timeline>,
/// Initial insert of notes into a timeline. Subsequent inserts should
/// just use the insert function
pub fn insert_new(
&mut self,
txn: &Transaction,
ndb: &Ndb,
note_cache: &mut NoteCache,
notes: &[NoteRef],
) {
let filters = {
let views = &self.views;
let filters: Vec<fn(&CachedNote, &Note) -> bool> =
views.iter().map(|v| v.filter.filter()).collect();
filters
};
for note_ref in notes {
for (view, filter) in filters.iter().enumerate() {
if let Ok(note) = ndb.get_note_by_key(txn, note_ref.key) {
if filter(
note_cache.cached_note_or_insert_mut(note_ref.key, &note),
&note,
) {
self.views[view].notes.push(*note_ref)
}
}
}
}
}
/// The main function used for inserting notes into timelines. Handles
/// inserting into multiple views if we have them. All timeline note
/// insertions should use this function.
pub fn insert(
&mut self,
new_note_ids: &[NoteKey],
ndb: &Ndb,
txn: &Transaction,
unknown_ids: &mut UnknownIds,
note_cache: &mut NoteCache,
reversed: bool,
) -> Result<()> {
let timeline = timelines
.get_mut(timeline_idx)
.ok_or(Error::TimelineNotFound)?;
let sub = timeline
let mut new_refs: Vec<(Note, NoteRef)> = Vec::with_capacity(new_note_ids.len());
for key in new_note_ids {
let note = if let Ok(note) = ndb.get_note_by_key(txn, *key) {
note
} else {
error!("hit race condition in poll_notes_into_view: https://github.com/damus-io/nostrdb/issues/35 note {:?} was not added to timeline", key);
continue;
};
// Ensure that unknown ids are captured when inserting notes
// into the timeline
UnknownIds::update_from_note(txn, ndb, unknown_ids, note_cache, &note);
let created_at = note.created_at();
new_refs.push((
note,
NoteRef {
key: *key,
created_at,
},
));
}
for view in &mut self.views {
match view.filter {
ViewFilter::NotesAndReplies => {
let refs: Vec<NoteRef> = new_refs.iter().map(|(_note, nr)| *nr).collect();
view.insert(&refs, reversed);
}
ViewFilter::Notes => {
let mut filtered_refs = Vec::with_capacity(new_refs.len());
for (note, nr) in &new_refs {
let cached_note = note_cache.cached_note_or_insert(nr.key, note);
if ViewFilter::filter_notes(cached_note, note) {
filtered_refs.push(*nr);
}
}
view.insert(&filtered_refs, reversed);
}
}
}
Ok(())
}
pub fn poll_notes_into_view(
&mut self,
ndb: &Ndb,
txn: &Transaction,
unknown_ids: &mut UnknownIds,
note_cache: &mut NoteCache,
reversed: bool,
) -> Result<()> {
let sub = self
.subscription
.ok_or(Error::App(notedeck::Error::no_active_sub()))?;
@@ -302,55 +406,7 @@ impl Timeline {
debug!("{} new notes! {:?}", new_note_ids.len(), new_note_ids);
}
let mut new_refs: Vec<(Note, NoteRef)> = Vec::with_capacity(new_note_ids.len());
for key in new_note_ids {
let note = if let Ok(note) = ndb.get_note_by_key(txn, key) {
note
} else {
error!("hit race condition in poll_notes_into_view: https://github.com/damus-io/nostrdb/issues/35 note {:?} was not added to timeline", key);
continue;
};
UnknownIds::update_from_note(txn, ndb, unknown_ids, note_cache, &note);
let created_at = note.created_at();
new_refs.push((note, NoteRef { key, created_at }));
}
// We're assuming reverse-chronological here (timelines). This
// flag ensures we trigger the items_inserted_at_start
// optimization in VirtualList. We need this flag because we can
// insert notes into chronological order sometimes, and this
// optimization doesn't make sense in those situations.
let reversed = false;
// ViewFilter::NotesAndReplies
if let Some(view) = timeline.view_mut(ViewFilter::NotesAndReplies) {
let refs: Vec<NoteRef> = new_refs.iter().map(|(_note, nr)| *nr).collect();
view.insert(&refs, reversed);
}
//
// handle the filtered case (ViewFilter::Notes, no replies)
//
// TODO(jb55): this is mostly just copied from above, let's just use a loop
// I initially tried this but ran into borrow checker issues
if let Some(view) = timeline.view_mut(ViewFilter::Notes) {
let mut filtered_refs = Vec::with_capacity(new_refs.len());
for (note, nr) in &new_refs {
let cached_note = note_cache.cached_note_or_insert(nr.key, note);
if ViewFilter::filter_notes(cached_note, note) {
filtered_refs.push(*nr);
}
}
view.insert(&filtered_refs, reversed);
}
Ok(())
self.insert(&new_note_ids, ndb, txn, unknown_ids, note_cache, reversed)
}
}
@@ -550,45 +606,18 @@ fn setup_initial_timeline(
timeline.subscription, timeline.filter
);
let lim = filters[0].limit().unwrap_or(filter::default_limit()) as i32;
let notes = ndb
let notes: Vec<NoteRef> = ndb
.query(&txn, filters, lim)?
.into_iter()
.map(NoteRef::from_query_result)
.collect();
copy_notes_into_timeline(timeline, &txn, ndb, note_cache, notes);
timeline.insert_new(&txn, ndb, note_cache, &notes);
Ok(())
}
pub fn copy_notes_into_timeline(
timeline: &mut Timeline,
txn: &Transaction,
ndb: &Ndb,
note_cache: &mut NoteCache,
notes: Vec<NoteRef>,
) {
let filters = {
let views = &timeline.views;
let filters: Vec<fn(&CachedNote, &Note) -> bool> =
views.iter().map(|v| v.filter.filter()).collect();
filters
};
for note_ref in notes {
for (view, filter) in filters.iter().enumerate() {
if let Ok(note) = ndb.get_note_by_key(txn, note_ref.key) {
if filter(
note_cache.cached_note_or_insert_mut(note_ref.key, &note),
&note,
) {
timeline.views[view].notes.push(note_ref)
}
}
}
}
}
pub fn setup_initial_nostrdb_subs(
ndb: &Ndb,
note_cache: &mut NoteCache,

20
crates/notedeck_columns/src/timeline/route.rs
View File

@@ -2,10 +2,8 @@ use crate::{
column::Columns,
draft::Drafts,
nav::RenderNavAction,
notes_holder::NotesHolderStorage,
profile::{Profile, ProfileAction},
thread::Thread,
timeline::{TimelineId, TimelineKind},
profile::ProfileAction,
timeline::{TimelineCache, TimelineId, TimelineKind},
ui::{
self,
note::{NoteOptions, QuoteRepostView},
@@ -34,8 +32,7 @@ pub fn render_timeline_route(
img_cache: &mut ImageCache,
unknown_ids: &mut UnknownIds,
note_cache: &mut NoteCache,
threads: &mut NotesHolderStorage<Thread>,
profiles: &mut NotesHolderStorage<Profile>,
timeline_cache: &mut TimelineCache,
accounts: &mut Accounts,
route: TimelineRoute,
col: usize,
@@ -71,7 +68,7 @@ pub fn render_timeline_route(
}
TimelineRoute::Thread(id) => ui::ThreadView::new(
threads,
timeline_cache,
ndb,
note_cache,
unknown_ids,
@@ -121,9 +118,10 @@ pub fn render_timeline_route(
&pubkey,
accounts,
ndb,
profiles,
timeline_cache,
img_cache,
note_cache,
unknown_ids,
col,
ui,
&accounts.mutefun(),
@@ -160,9 +158,10 @@ pub fn render_profile_route(
pubkey: &Pubkey,
accounts: &Accounts,
ndb: &Ndb,
profiles: &mut NotesHolderStorage<Profile>,
timeline_cache: &mut TimelineCache,
img_cache: &mut ImageCache,
note_cache: &mut NoteCache,
unknown_ids: &mut UnknownIds,
col: usize,
ui: &mut egui::Ui,
is_muted: &MuteFun,
@@ -171,10 +170,11 @@ pub fn render_profile_route(
pubkey,
accounts,
col,
profiles,
timeline_cache,
ndb,
note_cache,
img_cache,
unknown_ids,
is_muted,
NoteOptions::default(),
)