tyiu/damus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add trie-based user search cache to replace non-performant linear scans

Browse Source 
Changelog-Added: Speed up user search
Tested-by: William Casarin <jb55@jb55.com>
Fixes: #1219
Closes: #1342
This commit is contained in:
Terry Yiu
2023-07-01 14:42:36 -04:00
committed by William Casarin
parent 4b7444f338
commit 6e964f71ff
13 changed files with 519 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
damus/Models/DamusState.swift
View File

@@ -31,6 +31,7 @@ struct DamusState {
let muted_threads: MutedThreadsManager
let wallet: WalletModel
let nav: NavigationCoordinator
let user_search_cache: UserSearchCache
@discardableResult
func add_zap(zap: Zapping) -> Bool {
@@ -58,5 +59,6 @@ struct DamusState {
}
static var empty: DamusState {
return DamusState.init(pool: RelayPool(), keypair: Keypair(pubkey: "", privkey: ""), likes: EventCounter(our_pubkey: ""), boosts: EventCounter(our_pubkey: ""), contacts: Contacts(our_pubkey: ""), profiles: Profiles(), dms: DirectMessagesModel(our_pubkey: ""), previews: PreviewCache(), zaps: Zaps(our_pubkey: ""), lnurls: LNUrls(), settings: UserSettingsStore(), relay_filters: RelayFilters(our_pubkey: ""), relay_metadata: RelayMetadatas(), drafts: Drafts(), events: EventCache(), bookmarks: BookmarksManager(pubkey: ""), postbox: PostBox(pool: RelayPool()), bootstrap_relays: [], replies: ReplyCounter(our_pubkey: ""), muted_threads: MutedThreadsManager(keypair: Keypair(pubkey: "", privkey: nil)), wallet: WalletModel(settings: UserSettingsStore()), nav: NavigationCoordinator()) }
let user_search_cache = UserSearchCache()
return DamusState.init(pool: RelayPool(), keypair: Keypair(pubkey: "", privkey: ""), likes: EventCounter(our_pubkey: ""), boosts: EventCounter(our_pubkey: ""), contacts: Contacts(our_pubkey: ""), profiles: Profiles(user_search_cache: user_search_cache), dms: DirectMessagesModel(our_pubkey: ""), previews: PreviewCache(), zaps: Zaps(our_pubkey: ""), lnurls: LNUrls(), settings: UserSettingsStore(), relay_filters: RelayFilters(our_pubkey: ""), relay_metadata: RelayMetadatas(), drafts: Drafts(), events: EventCache(), bookmarks: BookmarksManager(pubkey: ""), postbox: PostBox(pool: RelayPool()), bootstrap_relays: [], replies: ReplyCounter(our_pubkey: ""), muted_threads: MutedThreadsManager(keypair: Keypair(pubkey: "", privkey: nil)), wallet: WalletModel(settings: UserSettingsStore()), nav: NavigationCoordinator(), user_search_cache: user_search_cache) }
}

11
damus/Models/HomeModel.swift
View File

@@ -612,7 +612,8 @@ func add_contact_if_friend(contacts: Contacts, ev: NostrEvent) {
contacts.add_friend_contact(ev)
}
func load_our_contacts(contacts: Contacts, m_old_ev: NostrEvent?, ev: NostrEvent) {
func load_our_contacts(state: DamusState, m_old_ev: NostrEvent?, ev: NostrEvent) {
let contacts = state.contacts
var new_pks = Set<String>()
// our contacts
for tag in ev.tags {
@@ -641,6 +642,8 @@ func load_our_contacts(contacts: Contacts, m_old_ev: NostrEvent?, ev: NostrEvent
contacts.remove_friend(pk)
}
}
state.user_search_cache.updateOwnContactsPetnames(id: contacts.our_pubkey, oldEvent: m_old_ev, newEvent: ev)
}
@@ -701,7 +704,9 @@ func process_metadata_profile(our_pubkey: String, profiles: Profiles, profile: P
}
var old_nip05: String? = nil
if let mprof = profiles.lookup_with_timestamp(id: ev.pubkey) {
let mprof = profiles.lookup_with_timestamp(id: ev.pubkey)
if let mprof {
old_nip05 = mprof.profile.nip05
if mprof.event.created_at > ev.created_at {
// skip if we already have an newer profile
@@ -810,7 +815,7 @@ func load_our_stuff(state: DamusState, ev: NostrEvent) {
let m_old_ev = state.contacts.event
state.contacts.event = ev
load_our_contacts(contacts: state.contacts, m_old_ev: m_old_ev, ev: ev)
load_our_contacts(state: state, m_old_ev: m_old_ev, ev: ev)
load_our_relays(state: state, m_old_ev: m_old_ev, ev: ev)
}

129
damus/Models/Trie.swift Normal file
View File

@@ -0,0 +1,129 @@
//
// Trie.swift
// damus
//
// Created by Terry Yiu on 6/26/23.
//
import Foundation
/// Tree data structure of all the substring permutations of a collection of strings optimized for searching for values of type V.
///
/// Each node in the tree can have child nodes.
/// Each node represents a single character in substrings, and each of its child nodes represent the subsequent character in those substrings.
///
/// A node that has no children mean that there are no substrings with any additional characters beyond the branch of letters leading up to that node.
///
/// A node that has values mean that there are strings that end in the character represented by the node and contain the substring represented by the branch of letters leading up to that node.
///
/// https://en.wikipedia.org/wiki/Trie
class Trie<V: Hashable> {
private var children: [Character : Trie] = [:]
/// Separate exact matches from strict substrings so that exact matches appear first in returned results.
private var exactMatchValues = Set<V>()
private var substringMatchValues = Set<V>()
private var parent: Trie? = nil
}
extension Trie {
var hasChildren: Bool {
return !self.children.isEmpty
}
var hasValues: Bool {
return !self.exactMatchValues.isEmpty || !self.substringMatchValues.isEmpty
}
/// Finds the branch that matches the specified key and returns the values from all of its descendant nodes.
func find(key: String) -> [V] {
var currentNode = self
// Find branch with matching prefix.
for char in key {
if let child = currentNode.children[char] {
currentNode = child
} else {
return []
}
}
// Perform breadth-first search from matching branch and collect values from all descendants.
var exactMatches = Set<V>()
var substringMatches = Set<V>()
var queue = [currentNode]
while !queue.isEmpty {
let node = queue.removeFirst()
exactMatches.formUnion(node.exactMatchValues)
substringMatches.formUnion(node.substringMatchValues)
queue.append(contentsOf: node.children.values)
}
return Array(exactMatches) + substringMatches
}
/// Inserts value of type V into this trie for the specified key. This function stores all substring endings of the key, not only the key itself.
/// Runtime performance is O(n^2) and storage cost is O(n), where n is the number of characters in the key.
func insert(key: String, value: V) {
// Create root branches for each character of the key to enable substring searches instead of only just prefix searches.
// Hence the nested loop.
for i in 0..<key.count {
var currentNode = self
// Find branch with matching prefix.
for char in key[key.index(key.startIndex, offsetBy: i)...] {
if let child = currentNode.children[char] {
currentNode = child
} else {
let child = Trie()
child.parent = currentNode
currentNode.children[char] = child
currentNode = child
}
}
if i == 0 {
currentNode.exactMatchValues.insert(value)
} else {
currentNode.substringMatchValues.insert(value)
}
}
}
/// Removes value of type V from this trie for the specified key.
func remove(key: String, value: V) {
for i in 0..<key.count {
var currentNode = self
var foundLeafNode = true
// Find branch with matching prefix.
for j in i..<key.count {
let char = key[key.index(key.startIndex, offsetBy: j)]
if let child = currentNode.children[char] {
currentNode = child
} else {
foundLeafNode = false
break
}
}
if foundLeafNode {
currentNode.exactMatchValues.remove(value)
currentNode.substringMatchValues.remove(value)
// Clean up the tree if this leaf node no longer holds values or children.
for j in (i..<key.count).reversed() {
if let parent = currentNode.parent, !currentNode.hasValues && !currentNode.hasChildren {
currentNode = parent
let char = key[key.index(key.startIndex, offsetBy: j)]
currentNode.children.removeValue(forKey: char)
}
}
}
}
}
}

107
damus/Models/UserSearchCache.swift Normal file
View File

@@ -0,0 +1,107 @@
//
// UserSearchCache.swift
// damus
//
// Created by Terry Yiu on 6/27/23.
//
import Foundation
/// Cache of searchable users by name, display_name, NIP-05 identifier, or own contact list petname.
/// Optimized for fast searches of substrings by using a Trie.
/// Optimal for performing user searches that could be initiated by typing quickly on a keyboard into a text input field.
class UserSearchCache {
private let trie = Trie<String>()
func search(key: String) -> [String] {
let results = trie.find(key: key)
return results
}
/// Computes the differences between an old profile, if it exists, and a new profile, and updates the user search cache accordingly.
func updateProfile(id: String, profiles: Profiles, oldProfile: Profile?, newProfile: Profile) {
// Remove searchable keys tied to the old profile if they differ from the new profile
// to keep the trie clean without empty nodes while avoiding excessive graph searching.
if let oldProfile {
if let oldName = oldProfile.name, newProfile.name?.caseInsensitiveCompare(oldName) != .orderedSame {
trie.remove(key: oldName.lowercased(), value: id)
}
if let oldDisplayName = oldProfile.display_name, newProfile.display_name?.caseInsensitiveCompare(oldDisplayName) != .orderedSame {
trie.remove(key: oldDisplayName.lowercased(), value: id)
}
if let oldNip05 = oldProfile.nip05, newProfile.nip05?.caseInsensitiveCompare(oldNip05) != .orderedSame {
trie.remove(key: oldNip05.lowercased(), value: id)
}
}
addProfile(id: id, profiles: profiles, profile: newProfile)
}
/// Adds a profile to the user search cache.
private func addProfile(id: String, profiles: Profiles, profile: Profile) {
// Searchable by name.
if let name = profile.name {
trie.insert(key: name.lowercased(), value: id)
}
// Searchable by display name.
if let displayName = profile.display_name {
trie.insert(key: displayName.lowercased(), value: id)
}
// Searchable by NIP-05 identifier.
if let nip05 = profiles.is_validated(id) {
trie.insert(key: "\(nip05.username.lowercased())@\(nip05.host.lowercased())", value: id)
}
}
/// Computes the diffences between an old contacts event and a new contacts event for our own user, and updates the search cache accordingly.
func updateOwnContactsPetnames(id: String, oldEvent: NostrEvent?, newEvent: NostrEvent) {
guard newEvent.known_kind == .contacts && newEvent.pubkey == id else {
return
}
var petnames: [String: String] = [:]
// Gets all petnames from our new contacts list.
newEvent.tags.forEach { tag in
guard tag.count >= 4 && tag[0] == "p" else {
return
}
let pubkey = tag[1]
let petname = tag[3]
petnames[pubkey] = petname
}
// Compute the diff with the old contacts list, if it exists,
// mark the ones that are the same to not be removed from the user search cache,
// and remove the old ones that are different from the user search cache.
if let oldEvent, oldEvent.known_kind == .contacts && oldEvent.pubkey == id {
oldEvent.tags.forEach { tag in
guard tag.count >= 4 && tag[0] == "p" else {
return
}
let pubkey = tag[1]
let oldPetname = tag[3]
if let newPetname = petnames[pubkey] {
if newPetname.caseInsensitiveCompare(oldPetname) == .orderedSame {
petnames.removeValue(forKey: pubkey)
} else {
trie.remove(key: oldPetname, value: pubkey)
}
} else {
trie.remove(key: oldPetname, value: pubkey)
}
}
}
// Add the new petnames to the user search cache.
for (pubkey, petname) in petnames {
trie.insert(key: petname, value: pubkey)
}
}
}