introduce HybridFilter

This introduces a new filter construct called HybridFilter. This allows
filters to have different remote filter than local ones. For example,
adding kind0 to the remote for keeping profiles up to date on your
timeline, but only subscribing to kind1 locally.

Only home/contact filters use this feature for now.

Fixes: https://github.com/damus-io/notedeck/issues/995
Signed-off-by: William Casarin <jb55@jb55.com>

crates/notedeck/src/contacts.rs

@@ -0,0 +1,24 @@
+use crate::{
+    filter::{self, HybridFilter},
+    Error,
+};
+use nostrdb::{Filter, Note};
+
+pub fn contacts_filter(pk: &[u8; 32]) -> Filter {
+    Filter::new().authors([pk]).kinds([3]).limit(1).build()
+}
+
+/// Contact filters have an additional kind0 in the remote filter so it can fetch profiles as well
+/// we don't need this in the local filter since we only care about the kind1 results
+pub fn hybrid_contacts_filter(
+    note: &Note,
+    add_pk: Option<&[u8; 32]>,
+    with_hashtags: bool,
+) -> Result<HybridFilter, Error> {
+    let local = filter::filter_from_tags(&note, add_pk, with_hashtags)?
+        .into_filter([1], filter::default_limit());
+    let remote = filter::filter_from_tags(&note, add_pk, with_hashtags)?
+        .into_filter([1, 0], filter::default_remote_limit());
+
+    Ok(HybridFilter::split(local, remote))
+}

crates/notedeck/src/filter.rs

@@ -55,7 +55,7 @@ impl FilterStates {
         None
     }
 
-    pub fn get_any_ready(&self) -> Option<&Vec<Filter>> {
+    pub fn get_any_ready(&self) -> Option<&HybridFilter> {
         if let FilterState::Ready(fs) = &self.initial_state {
             Some(fs)
         } else {
@@ -95,7 +95,7 @@ pub enum FilterState {
     NeedsRemote,
     FetchingRemote(FetchingRemoteType),
     GotRemote(GotRemoteType),
-    Ready(Vec<Filter>),
+    Ready(HybridFilter),
     Broken(FilterError),
 }
 
@@ -132,6 +132,17 @@ impl FilterState {
 
     /// The filter is ready
     pub fn ready(filter: Vec<Filter>) -> Self {
+        Self::Ready(HybridFilter::unsplit(filter))
+    }
+
+    /// The filter is ready, but we have a different local filter from
+    /// our remote one
+    pub fn ready_split(local: Vec<Filter>, remote: Vec<Filter>) -> Self {
+        Self::Ready(HybridFilter::split(local, remote))
+    }
+
+    /// Our hybrid filter is ready (either split or unsplit)
+    pub fn ready_hybrid(filter: HybridFilter) -> Self {
         Self::Ready(filter)
     }
 
@@ -195,6 +206,49 @@ pub struct FilteredTags {
     pub hashtags: Option<FilterBuilder>,
 }
 
+/// The local and remote filter are related but slightly different
+#[derive(Debug, Clone)]
+pub struct SplitFilter {
+    pub local: Vec<Filter>,
+    pub remote: Vec<Filter>,
+}
+
+/// Either a [`SplitFilter`] or a regular unsplit filter,. Split filters
+/// have different remote and local filters but are tracked together.
+#[derive(Debug, Clone)]
+pub enum HybridFilter {
+    Split(SplitFilter),
+    Unsplit(Vec<Filter>),
+}
+
+impl HybridFilter {
+    pub fn unsplit(filter: Vec<Filter>) -> Self {
+        HybridFilter::Unsplit(filter)
+    }
+
+    pub fn split(local: Vec<Filter>, remote: Vec<Filter>) -> Self {
+        HybridFilter::Split(SplitFilter { local, remote })
+    }
+
+    pub fn local(&self) -> &[Filter] {
+        match self {
+            Self::Split(split) => &split.local,
+
+            // local as the same as remote in unsplit
+            Self::Unsplit(local) => &local,
+        }
+    }
+
+    pub fn remote(&self) -> &[Filter] {
+        match self {
+            Self::Split(split) => &split.remote,
+
+            // local as the same as remote in unsplit
+            Self::Unsplit(remote) => remote,
+        }
+    }
+}
+
 impl FilteredTags {
     pub fn into_follow_filter(self) -> Vec<Filter> {
         self.into_filter([1], default_limit())

crates/notedeck/src/lib.rs

@@ -2,6 +2,7 @@ pub mod abbrev;
 mod account;
 mod app;
 mod args;
+pub mod contacts;
 mod context;
 pub mod debouncer;
 mod error;

crates/notedeck_columns/src/multi_subscriber.rs

@@ -2,7 +2,7 @@ use egui_nav::ReturnType;
 use enostr::{Filter, NoteId, RelayPool};
 use hashbrown::HashMap;
 use nostrdb::{Ndb, Subscription};
-use notedeck::UnifiedSubscription;
+use notedeck::{filter::HybridFilter, UnifiedSubscription};
 use uuid::Uuid;
 
 use crate::{subscriptions, timeline::ThreadSelection};
@@ -113,7 +113,11 @@ impl ThreadSubs {
                     };
 
                     if scope.root_id.bytes() != id.root_id.bytes() {
-                        tracing::error!("Somehow the current scope's root is not equal to the selected note's root. scope's root: {:?}, thread's root: {:?}", scope.root_id.hex(), id.root_id.bytes());
+                        tracing::error!(
+                            "Somehow the current scope's root is not equal to the selected note's root. scope's root: {:?}, thread's root: {:?}",
+                            scope.root_id.hex(),
+                            id.root_id.bytes()
+                        );
                     }
 
                     if ndb_unsub(ndb, cur_sub.sub, id) {
@@ -132,7 +136,11 @@ impl ThreadSubs {
                 };
 
                 if scope.root_id.bytes() != id.root_id.bytes() {
-                    tracing::error!("Somehow the current scope's root is not equal to the selected note's root. scope's root: {:?}, thread's root: {:?}", scope.root_id.hex(), id.root_id.bytes());
+                    tracing::error!(
+                        "Somehow the current scope's root is not equal to the selected note's root. scope's root: {:?}, thread's root: {:?}",
+                        scope.root_id.hex(),
+                        id.root_id.bytes()
+                    );
                 }
                 for sub in scope.stack {
                     if ndb_unsub(ndb, sub.sub, id) {
@@ -266,7 +274,7 @@ fn local_sub_new_scope(
 
 #[derive(Debug)]
 pub struct TimelineSub {
-    filter: Option<Vec<Filter>>,
+    filter: Option<HybridFilter>,
     state: SubState,
 }
 
@@ -299,11 +307,11 @@ impl Default for TimelineSub {
 }
 
 impl TimelineSub {
-    pub fn try_add_local(&mut self, ndb: &Ndb, filter: &[Filter]) {
+    pub fn try_add_local(&mut self, ndb: &Ndb, filter: &HybridFilter) {
         let before = self.state.clone();
         match &mut self.state {
             SubState::NoSub { dependers } => {
-                let Some(sub) = ndb_sub(ndb, filter, "") else {
+                let Some(sub) = ndb_sub(ndb, filter.local(), "") else {
                     return;
                 };
 
@@ -318,7 +326,7 @@ impl TimelineSub {
                 dependers: _,
             } => {}
             SubState::RemoteOnly { remote, dependers } => {
-                let Some(local) = ndb_sub(ndb, filter, "") else {
+                let Some(local) = ndb_sub(ndb, filter.local(), "") else {
                     return;
                 };
                 self.state = SubState::Unified {
@@ -375,12 +383,12 @@ impl TimelineSub {
         );
     }
 
-    pub fn try_add_remote(&mut self, pool: &mut RelayPool, filter: &Vec<Filter>) {
+    pub fn try_add_remote(&mut self, pool: &mut RelayPool, filter: &HybridFilter) {
         let before = self.state.clone();
         match &mut self.state {
             SubState::NoSub { dependers } => {
                 let subid = subscriptions::new_sub_id();
-                pool.subscribe(subid.clone(), filter.clone());
+                pool.subscribe(subid.clone(), filter.remote().to_vec());
                 self.filter = Some(filter.to_owned());
                 self.state = SubState::RemoteOnly {
                     remote: subid,
@@ -389,7 +397,7 @@ impl TimelineSub {
             }
             SubState::LocalOnly { local, dependers } => {
                 let subid = subscriptions::new_sub_id();
-                pool.subscribe(subid.clone(), filter.clone());
+                pool.subscribe(subid.clone(), filter.remote().to_vec());
                 self.filter = Some(filter.to_owned());
                 self.state = SubState::Unified {
                     unified: UnifiedSubscription {
@@ -519,7 +527,7 @@ impl TimelineSub {
         );
     }
 
-    pub fn get_filter(&self) -> Option<&Vec<Filter>> {
+    pub fn get_filter(&self) -> Option<&HybridFilter> {
         self.filter.as_ref()
     }
 

crates/notedeck_columns/src/timeline/cache.rs

@@ -129,7 +129,7 @@ impl TimelineCache {
         }
 
         let notes = if let FilterState::Ready(filters) = id.filters(txn, ndb) {
-            if let Ok(results) = ndb.query(txn, &filters, 1000) {
+            if let Ok(results) = ndb.query(txn, filters.local(), 1000) {
                 results
                     .into_iter()
                     .map(NoteRef::from_query_result)
@@ -171,7 +171,7 @@ impl TimelineCache {
                 // The timeline cache is stale, let's update it
                 let notes = find_new_notes(
                     timeline.all_or_any_notes(),
-                    timeline.subscription.get_filter()?,
+                    timeline.subscription.get_filter()?.local(),
                     txn,
                     ndb,
                 );

crates/notedeck_columns/src/timeline/kind.rs

@@ -4,10 +4,10 @@ use crate::timeline::{Timeline, TimelineTab};
 use enostr::{Filter, NoteId, Pubkey};
 use nostrdb::{Ndb, Transaction};
 use notedeck::{
+    contacts::{contacts_filter, hybrid_contacts_filter},
     filter::{self, default_limit},
     FilterError, FilterState, NoteCache, RootIdError, RootNoteIdBuf,
 };
-use notedeck_ui::contacts::contacts_filter;
 use serde::{Deserialize, Serialize};
 use std::hash::{Hash, Hasher};
 use std::{borrow::Cow, fmt::Display};
@@ -651,7 +651,7 @@ fn contact_filter_state(txn: &Transaction, ndb: &Ndb, pk: &Pubkey) -> FilterStat
         FilterState::needs_remote()
     } else {
         let with_hashtags = false;
-        match filter::filter_from_tags(&results[0].note, Some(pk.bytes()), with_hashtags) {
+        match hybrid_contacts_filter(&results[0].note, Some(pk.bytes()), with_hashtags) {
             Err(notedeck::Error::Filter(FilterError::EmptyContactList)) => {
                 FilterState::needs_remote()
             }
@@ -659,7 +659,7 @@ fn contact_filter_state(txn: &Transaction, ndb: &Ndb, pk: &Pubkey) -> FilterStat
                 error!("Error getting contact filter state: {err}");
                 FilterState::Broken(FilterError::EmptyContactList)
             }
-            Ok(filter) => FilterState::ready(filter.into_follow_filter()),
+            Ok(filter) => FilterState::ready_hybrid(filter),
         }
     }
 }

crates/notedeck_columns/src/timeline/mod.rs

@@ -7,8 +7,10 @@ use crate::{
 };
 
 use notedeck::{
-    filter, Accounts, CachedNote, ContactState, FilterError, FilterState, FilterStates, NoteCache,
-    NoteRef, UnknownIds,
+    contacts::hybrid_contacts_filter,
+    filter::{self, HybridFilter},
+    Accounts, CachedNote, ContactState, FilterError, FilterState, FilterStates, NoteCache, NoteRef,
+    UnknownIds,
 };
 
 use egui_virtual_list::VirtualList;
@@ -205,12 +207,12 @@ impl Timeline {
     /// Create a timeline from a contact list
     pub fn contact_list(contact_list: &Note, pubkey: &[u8; 32]) -> Result<Self> {
         let with_hashtags = false;
-        let filter = filter::filter_from_tags(contact_list, Some(pubkey), with_hashtags)?
-            .into_follow_filter();
+        let add_pk = Some(pubkey);
+        let filter = hybrid_contacts_filter(contact_list, add_pk, with_hashtags)?;
 
         Ok(Timeline::new(
             TimelineKind::contact_list(Pubkey::new(*pubkey)),
-            FilterState::ready(filter),
+            FilterState::ready_hybrid(filter),
             TimelineTab::full_tabs(),
         ))
     }
@@ -346,7 +348,10 @@ impl Timeline {
             let note = if let Ok(note) = ndb.get_note_by_key(txn, *key) {
                 note
             } else {
-                error!("hit race condition in poll_notes_into_view: https://github.com/damus-io/nostrdb/issues/35 note {:?} was not added to timeline", key);
+                error!(
+                    "hit race condition in poll_notes_into_view: https://github.com/damus-io/nostrdb/issues/35 note {:?} was not added to timeline",
+                    key
+                );
                 continue;
             };
 
@@ -537,7 +542,7 @@ pub fn send_initial_timeline_filter(
 
         FilterState::Ready(filter) => {
             let filter = filter.to_owned();
-            let new_filters: Vec<Filter> = filter.into_iter().map(|f| {
+            let new_filters: Vec<Filter> = filter.remote().to_owned().into_iter().map(|f| {
                 // limit the size of remote filters
                 let default_limit = filter::default_remote_limit();
                 let mut lim = f.limit().unwrap_or(default_limit);
@@ -611,7 +616,7 @@ fn setup_initial_timeline(
     txn: &Transaction,
     timeline: &mut Timeline,
     note_cache: &mut NoteCache,
-    filters: &[Filter],
+    filters: &HybridFilter,
 ) -> Result<()> {
     // some timelines are one-shot and a refreshed, like last_per_pubkey algo feed
     if timeline.kind.should_subscribe_locally() {
@@ -624,12 +629,12 @@ fn setup_initial_timeline(
     );
 
     let mut lim = 0i32;
-    for filter in filters {
+    for filter in filters.local() {
         lim += filter.limit().unwrap_or(1) as i32;
     }
 
     let notes: Vec<NoteRef> = ndb
-        .query(txn, filters, lim)?
+        .query(txn, filters.local(), lim)?
         .into_iter()
         .map(NoteRef::from_query_result)
         .collect();
@@ -728,7 +733,8 @@ pub fn is_timeline_ready(
         let txn = Transaction::new(ndb).expect("txn");
         let note = ndb.get_note_by_key(&txn, note_key).expect("note");
         let add_pk = timeline.kind.pubkey().map(|pk| pk.bytes());
-        filter::filter_from_tags(&note, add_pk, with_hashtags).map(|f| f.into_follow_filter())
+
+        hybrid_contacts_filter(&note, add_pk, with_hashtags).map_err(Into::into)
     };
 
     // TODO: into_follow_filter is hardcoded to contact lists, let's generalize
@@ -755,7 +761,7 @@ pub fn is_timeline_ready(
             setup_initial_timeline(ndb, &txn, timeline, note_cache, &filter).expect("setup init");
             timeline
                 .filter
-                .set_relay_state(relay_id, FilterState::ready(filter.clone()));
+                .set_relay_state(relay_id, FilterState::ready_hybrid(filter.clone()));
 
             //let ck = &timeline.kind;
             //let subid = damus.gen_subid(&SubKind::Column(ck.clone()));

crates/notedeck_ui/src/contacts.rs

@@ -1,5 +0,0 @@
-use nostrdb::Filter;
-
-pub fn contacts_filter(pk: &[u8; 32]) -> Filter {
-    Filter::new().authors([pk]).kinds([3]).limit(1).build()
-}

crates/notedeck_ui/src/lib.rs

@@ -3,7 +3,6 @@ pub mod app_images;
 pub mod blur;
 pub mod colors;
 pub mod constants;
-pub mod contacts;
 pub mod context_menu;
 pub mod gif;
 pub mod icons;