nostrdb: add profiles to nostrdb

This adds profiles to nostrdb

- Remove in-memory Profiles caches, nostrdb is as fast as an in-memory cache
- Remove ProfileDatabase and just use nostrdb directly

Changelog-Changed: Use nostrdb for profiles

damus/Nostr/CoreData/PersistedProfile.swift

@@ -1,39 +0,0 @@
-//
-//  PersistedProfile.swift
-//  damus
-//
-//  Created by Bryan Montz on 4/30/23.
-//
-
-import Foundation
-import CoreData
-
-@objc(PersistedProfile)
-final class PersistedProfile: NSManagedObject {
-    @NSManaged var id: String?
-    @NSManaged var name: String?
-    @NSManaged var display_name: String?
-    @NSManaged var about: String?
-    @NSManaged var picture: String?
-    @NSManaged var banner: String?
-    @NSManaged var website: String?
-    @NSManaged var lud06: String?
-    @NSManaged var lud16: String?
-    @NSManaged var nip05: String?
-    @NSManaged var damus_donation: Int16
-    @NSManaged var last_update: Date?       // The date that the profile was last updated by the user
-    @NSManaged var network_pull_date: Date? // The date we got this profile from a relay (for staleness checking)
-    
-    func copyValues(from profile: Profile) {
-        name = profile.name
-        display_name = profile.display_name
-        about = profile.about
-        picture = profile.picture
-        banner = profile.banner
-        website = profile.website
-        lud06 = profile.lud06
-        lud16 = profile.lud16
-        nip05 = profile.nip05
-        damus_donation = profile.damus_donation != nil ? Int16(profile.damus_donation!) : 0
-    }
-}

damus/Nostr/Nostr.swift

@@ -7,6 +7,114 @@
 
 import Foundation
 
+typealias Profile = NdbProfile
+//typealias ProfileRecord = NdbProfileRecord
+
+class ProfileRecord {
+    let data: NdbProfileRecord
+
+    init(data: NdbProfileRecord) {
+        self.data = data
+    }
+
+    var profile: Profile? { return data.profile }
+    var receivedAt: UInt64 { data.receivedAt }
+    var noteKey: UInt64 { data.noteKey }
+
+    private var _lnurl: String? = nil
+    var lnurl: String? {
+        if let _lnurl {
+            return _lnurl
+        }
+        
+        guard let profile = data.profile,
+              let addr = profile.lud16 ?? profile.lud06 else {
+            return nil;
+        }
+        
+        if addr.contains("@") {
+            // this is a heavy op and is used a lot in views, cache it!
+            let addr = lnaddress_to_lnurl(addr);
+            self._lnurl = addr
+            return addr
+        }
+        
+        if !addr.lowercased().hasPrefix("lnurl") {
+            return nil
+        }
+        
+        return addr;
+    }
+    
+}
+
+extension NdbProfile {
+    var display_name: String? {
+        return displayName
+    }
+
+    static func displayName(profile: Profile?, pubkey: Pubkey) -> DisplayName {
+        return parse_display_name(profile: profile, pubkey: pubkey)
+    }
+
+    var damus_donation: Int? {
+        return Int(damusDonation)
+    }
+
+    var damus_donation_v2: Int {
+        return Int(damusDonationV2)
+    }
+
+    var website_url: URL? {
+        if self.website?.trimmingCharacters(in: .whitespacesAndNewlines) == "" {
+            return nil
+        }
+        return self.website.flatMap { url in
+            let trim = url.trimmingCharacters(in: .whitespacesAndNewlines)
+            if !(trim.hasPrefix("http://") || trim.hasPrefix("https://")) {
+                return URL(string: "https://" + trim)
+            }
+            return URL(string: trim)
+        }
+    }
+
+    init(name: String? = nil, display_name: String? = nil, about: String? = nil, picture: String? = nil, banner: String? = nil, website: String? = nil, lud06: String? = nil, lud16: String? = nil, nip05: String? = nil, damus_donation: Int? = nil, reactions: Bool = true) {
+
+        var fbb = FlatBufferBuilder()
+
+        let name_off = fbb.create(string: name)
+        let display_name_off = fbb.create(string: display_name)
+        let about_off = fbb.create(string: about)
+        let picture_off = fbb.create(string: picture)
+        let banner_off = fbb.create(string: banner)
+        let website_off = fbb.create(string: website)
+        let lud06_off = fbb.create(string: lud06)
+        let lud16_off = fbb.create(string: lud16)
+        let nip05_off = fbb.create(string: nip05)
+
+        let profile_data = NdbProfile.createNdbProfile(&fbb,
+                                    nameOffset: name_off,
+                                    websiteOffset: website_off,
+                                    aboutOffset: about_off,
+                                    lud16Offset: lud16_off,
+                                    bannerOffset: banner_off,
+                                    displayNameOffset: display_name_off,
+                                    reactions: reactions,
+                                    pictureOffset: picture_off,
+                                    nip05Offset: nip05_off,
+                                    damusDonation: 0,
+                                    damusDonationV2: damus_donation.map({ Int32($0) }) ?? 0,
+                                    lud06Offset: lud06_off)
+
+        fbb.finish(offset: profile_data)
+
+        var buf = ByteBuffer(bytes: fbb.sizedByteArray)
+        let profile: Profile = try! getCheckedRoot(byteBuffer: &buf)
+        self = profile
+    }
+}
+
+/*
 class Profile: Codable {
     var value: [String: AnyCodable]
     
@@ -24,19 +132,6 @@ class Profile: Codable {
         self.damus_donation = damus_donation
     }
     
-    convenience init(persisted_profile: PersistedProfile) {
-        self.init(name: persisted_profile.name,
-                  display_name: persisted_profile.display_name,
-                  about: persisted_profile.about,
-                  picture: persisted_profile.picture,
-                  banner: persisted_profile.banner,
-                  website: persisted_profile.website,
-                  lud06: persisted_profile.lud06,
-                  lud16: persisted_profile.lud16,
-                  nip05: persisted_profile.nip05,
-                  damus_donation: Int(persisted_profile.damus_donation))
-    }
-    
     private func str(_ str: String) -> String? {
         return get_val(str)
     }
@@ -200,6 +295,7 @@ class Profile: Codable {
         return parse_display_name(profile: profile, pubkey: pubkey)
     }
 }
+*/
 
 func make_test_profile() -> Profile {
     return Profile(name: "jb55", display_name: "Will", about: "Its a me", picture: "https://cdn.jb55.com/img/red-me.jpg", banner: "https://pbs.twimg.com/profile_banners/9918032/1531711830/600x200",  website: "jb55.com", lud06: "jb55@jb55.com", lud16: nil, nip05: "jb55@jb55.com", damus_donation: 1)
@@ -222,3 +318,4 @@ func lnaddress_to_lnurl(_ lnaddr: String) -> String? {
     
     return bech32_encode(hrp: "lnurl", Array(dat))
 }
+

damus/Nostr/ProfileDatabase.swift

@@ -1,181 +0,0 @@
-//
-//  ProfileDatabase.swift
-//  damus
-//
-//  Created by Bryan Montz on 4/30/23.
-//
-
-import Foundation
-import CoreData
-
-enum ProfileDatabaseError: Error {
-    case missing_context
-    case outdated_input
-}
-
-final class ProfileDatabase {
-    
-    private let entity_name = "PersistedProfile"
-    private var persistent_container: NSPersistentContainer?
-    private var background_context: NSManagedObjectContext?
-    private let cache_url: URL
-    
-    /// This queue is used to synchronize access to the network_pull_date_cache dictionary, which
-    /// prevents data races from crashing the app.
-    private var queue = DispatchQueue(label: "io.damus.profile_db",
-                                      qos: .userInteractive,
-                                      attributes: .concurrent)
-    private var network_pull_date_cache = [Pubkey: Date]()
-
-    init(cache_url: URL = ProfileDatabase.profile_cache_url) {
-        self.cache_url = cache_url
-        set_up()
-    }
-    
-    private static var profile_cache_url: URL {
-        (FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.appendingPathComponent("profiles"))!
-    }
-    
-    private var persistent_store_description: NSPersistentStoreDescription {
-        let description = NSPersistentStoreDescription(url: cache_url)
-        description.type = NSSQLiteStoreType
-        description.setOption(true as NSNumber, forKey: NSMigratePersistentStoresAutomaticallyOption)
-        description.setOption(true as NSNumber, forKey: NSInferMappingModelAutomaticallyOption)
-        description.setOption(true as NSNumber, forKey: NSSQLiteManualVacuumOption)
-        return description
-    }
-    
-    private var object_model: NSManagedObjectModel? {
-        guard let url = Bundle.main.url(forResource: "Damus", withExtension: "momd") else {
-            return nil
-        }
-        return NSManagedObjectModel(contentsOf: url)
-    }
-    
-    private func set_up() {
-        guard let object_model else {
-            print("⚠️ Warning: ProfileDatabase failed to load its object model")
-            return
-        }
-        
-        persistent_container = NSPersistentContainer(name: "Damus", managedObjectModel: object_model)
-        persistent_container?.persistentStoreDescriptions = [persistent_store_description]
-        persistent_container?.loadPersistentStores { _, error in
-            if let error {
-                print("WARNING: ProfileDatabase failed to load: \(error)")
-            }
-        }
-        
-        persistent_container?.viewContext.automaticallyMergesChangesFromParent = true
-        persistent_container?.viewContext.mergePolicy = NSMergePolicy(merge: .mergeByPropertyObjectTrumpMergePolicyType)
-        
-        background_context = persistent_container?.newBackgroundContext()
-        background_context?.mergePolicy = NSMergePolicy(merge: .mergeByPropertyObjectTrumpMergePolicyType)
-    }
-    
-    private func get_persisted(id: Pubkey, context: NSManagedObjectContext) -> PersistedProfile? {
-        let request = NSFetchRequest<PersistedProfile>(entityName: entity_name)
-        request.predicate = NSPredicate(format: "id == %@", id.hex())
-        request.fetchLimit = 1
-        return try? context.fetch(request).first
-    }
-    
-    func get_network_pull_date(id: Pubkey) -> Date? {
-        var pull_date: Date?
-        queue.sync {
-            pull_date = network_pull_date_cache[id]
-        }
-        if let pull_date {
-            return pull_date
-        }
-        
-        let request = NSFetchRequest<PersistedProfile>(entityName: entity_name)
-        request.predicate = NSPredicate(format: "id == %@", id.hex())
-        request.fetchLimit = 1
-        request.propertiesToFetch = ["network_pull_date"]
-        guard let profile = try? persistent_container?.viewContext.fetch(request).first else {
-            return nil
-        }
-        
-        queue.async(flags: .barrier) {
-            self.network_pull_date_cache[id] = profile.network_pull_date
-        }
-        return profile.network_pull_date
-    }
-    
-    // MARK: - Public
-    
-    /// Updates or inserts a new Profile into the local database. Rejects profiles whose update date
-    /// is older than one we already have. Database writes occur on a background context for best performance.
-    /// - Parameters:
-    ///   - id: Profile id (pubkey)
-    ///   - profile: Profile object to be stored
-    ///   - last_update: Date that the Profile was updated
-    func upsert(id: Pubkey, profile: Profile, last_update: Date) async throws {
-        guard let context = background_context else {
-            throw ProfileDatabaseError.missing_context
-        }
-        
-        try await context.perform {
-            var persisted_profile: PersistedProfile?
-            if let profile = self.get_persisted(id: id, context: context) {
-                if let existing_last_update = profile.last_update, last_update < existing_last_update {
-                    throw ProfileDatabaseError.outdated_input
-                } else {
-                    persisted_profile = profile
-                }
-            } else {
-                persisted_profile = NSEntityDescription.insertNewObject(forEntityName: self.entity_name, into: context) as? PersistedProfile
-                persisted_profile?.id = id.hex()
-            }
-            persisted_profile?.copyValues(from: profile)
-            persisted_profile?.last_update = last_update
-            
-            let pull_date = Date.now
-            persisted_profile?.network_pull_date = pull_date
-            self.queue.async(flags: .barrier) {
-                self.network_pull_date_cache[id] = pull_date
-            }
-            
-            try context.save()
-        }
-    }
-    
-    func get(id: Pubkey) -> Profile? {
-        guard let container = persistent_container,
-              let profile = get_persisted(id: id, context: container.viewContext) else {
-            return nil
-        }
-        return Profile(persisted_profile: profile)
-    }
-    
-    var count: Int {
-        let request = NSFetchRequest<PersistedProfile>(entityName: entity_name)
-        let count = try? persistent_container?.viewContext.count(for: request)
-        return count ?? 0
-    }
-    
-    func remove_all_profiles() throws {
-        guard let context = background_context, let container = persistent_container else {
-            throw ProfileDatabaseError.missing_context
-        }
-        
-        queue.async(flags: .barrier) {
-            self.network_pull_date_cache.removeAll()
-        }
-        
-        let request = NSFetchRequest<NSFetchRequestResult>(entityName: entity_name)
-        let batch_delete_request = NSBatchDeleteRequest(fetchRequest: request)
-        batch_delete_request.resultType = .resultTypeObjectIDs
-        
-        let result = try container.persistentStoreCoordinator.execute(batch_delete_request, with: context) as! NSBatchDeleteResult
-        
-        // NSBatchDeleteRequest is an NSPersistentStoreRequest, which operates on disk. So now we'll manually update our in-memory context.
-        if let object_ids = result.result as? [NSManagedObjectID] {
-            let changes: [AnyHashable: Any] = [
-                NSDeletedObjectsKey: object_ids
-            ]
-            NSManagedObjectContext.mergeChanges(fromRemoteContextSave: changes, into: [context])
-        }
-    }
-}

damus/Nostr/Profiles.swift

@@ -15,72 +15,52 @@ class ValidationModel: ObservableObject {
     }
 }
 
-class ProfileDataModel: ObservableObject {
-    @Published var profile: TimestampedProfile?
-
-    init() {
-        self.profile = nil
-    }
-}
-
 class ProfileData {
     var status: UserStatusModel
-    var profile_model: ProfileDataModel
     var validation_model: ValidationModel
     var zapper: Pubkey?
 
     init() {
         status = .init()
-        profile_model = .init()
         validation_model = .init()
         zapper = nil
     }
 }
 
 class Profiles {
-    
-    static let db_freshness_threshold: TimeInterval = 24 * 60 * 60
-    
-    /// This queue is used to synchronize access to the profiles dictionary, which
-    /// prevents data races from crashing the app.
-    private var profiles_queue = DispatchQueue(label: "io.damus.profiles",
-                                      qos: .userInteractive,
-                                      attributes: .concurrent)
+    private var ndb: Ndb
 
-    private var validated_queue = DispatchQueue(label: "io.damus.profiles.validated",
-                                                  qos: .userInteractive,
-                                                  attributes: .concurrent)
-    
+    static let db_freshness_threshold: TimeInterval = 24 * 60 * 60
+
+    @MainActor
     private var profiles: [Pubkey: ProfileData] = [:]
 
+    @MainActor
     var nip05_pubkey: [String: Pubkey] = [:]
 
-    private let database = ProfileDatabase()
-
     let user_search_cache: UserSearchCache
 
-    init(user_search_cache: UserSearchCache) {
+    init(user_search_cache: UserSearchCache, ndb: Ndb) {
         self.user_search_cache = user_search_cache
+        self.ndb = ndb
     }
-    
+
+    @MainActor
     func is_validated(_ pk: Pubkey) -> NIP05? {
-        validated_queue.sync {
-            self.profile_data(pk).validation_model.validated
-        }
+        self.profile_data(pk).validation_model.validated
     }
 
+    @MainActor
     func invalidate_nip05(_ pk: Pubkey) {
-        validated_queue.async(flags: .barrier) {
-            self.profile_data(pk).validation_model.validated = nil
-        }
+        self.profile_data(pk).validation_model.validated = nil
     }
 
+    @MainActor
     func set_validated(_ pk: Pubkey, nip05: NIP05?) {
-        validated_queue.async(flags: .barrier) {
-            self.profile_data(pk).validation_model.validated = nip05
-        }
+        self.profile_data(pk).validation_model.validated = nip05
     }
-    
+
+    @MainActor
     func profile_data(_ pubkey: Pubkey) -> ProfileData {
         guard let data = profiles[pubkey] else {
             let data = ProfileData()
@@ -91,60 +71,28 @@ class Profiles {
         return data
     }
 
+    @MainActor
     func lookup_zapper(pubkey: Pubkey) -> Pubkey? {
         profile_data(pubkey).zapper
     }
-    
-    func add(id: Pubkey, profile: TimestampedProfile) {
-        profiles_queue.async(flags: .barrier) {
-            let old_timestamped_profile = self.profile_data(id).profile_model.profile
-            self.profile_data(id).profile_model.profile = profile
-            self.user_search_cache.updateProfile(id: id, profiles: self, oldProfile: old_timestamped_profile?.profile, newProfile: profile.profile)
-        }
-        
-        Task {
-            do {
-                try await database.upsert(id: id, profile: profile.profile, last_update: Date(timeIntervalSince1970: TimeInterval(profile.timestamp)))
-            } catch {
-                print("⚠️ Warning: Profiles failed to save a profile: \(error)")
-            }
-        }
+
+    func lookup_with_timestamp(_ pubkey: Pubkey) -> ProfileRecord? {
+        return ndb.lookup_profile(pubkey)
     }
-    
+
     func lookup(id: Pubkey) -> Profile? {
-        var profile: Profile?
-        profiles_queue.sync {
-            profile = self.profile_data(id).profile_model.profile?.profile
-        }
-        return profile ?? database.get(id: id)
+        return ndb.lookup_profile(id)?.profile
     }
-    
-    func lookup_with_timestamp(id: Pubkey) -> TimestampedProfile? {
-        profiles_queue.sync {
-            return self.profile_data(id).profile_model.profile
-        }
-    }
-    
+
     func has_fresh_profile(id: Pubkey) -> Bool {
         var profile: Profile?
-        profiles_queue.sync {
-            profile = self.profile_data(id).profile_model.profile?.profile
-        }
-        if profile != nil {
-            return true
-        }
-        // check memory first
-        return false
-
-        // then disk
-        guard let pull_date = database.get_network_pull_date(id: id) else {
-            return false
-        }
-        return Date.now.timeIntervalSince(pull_date) < Profiles.db_freshness_threshold
+        guard let profile = lookup_with_timestamp(id) else { return false }
+        return Date.now.timeIntervalSince(Date(timeIntervalSince1970: Double(profile.receivedAt))) < Profiles.db_freshness_threshold
     }
 }
 
 
+@MainActor
 func invalidate_zapper_cache(pubkey: Pubkey, profiles: Profiles, lnurl: LNUrls) {
     profiles.profile_data(pubkey).zapper = nil
     lnurl.endpoints.removeValue(forKey: pubkey)

damus/Nostr/RelayConnection.swift

@@ -56,12 +56,17 @@ final class RelayConnection: ObservableObject {
     private var subscriptionToken: AnyCancellable?
     
     private var handleEvent: (NostrConnectionEvent) -> ()
+    private var processEvent: (WebSocketEvent) -> ()
     private let url: RelayURL
     var log: RelayLog?
 
-    init(url: RelayURL, handleEvent: @escaping (NostrConnectionEvent) -> ()) {
+    init(url: RelayURL,
+         handleEvent: @escaping (NostrConnectionEvent) -> (),
+         processEvent: @escaping (WebSocketEvent) -> ())
+    {
         self.url = url
         self.handleEvent = handleEvent
+        self.processEvent = processEvent
     }
     
     func ping() {
@@ -138,6 +143,7 @@ final class RelayConnection: ObservableObject {
     }
     
     private func receive(event: WebSocketEvent) {
+        processEvent(event)
         switch event {
         case .connected:
             DispatchQueue.main.async {

damus/Nostr/RelayPool.swift

@@ -30,12 +30,15 @@ class RelayPool {
     var request_queue: [QueuedRequest] = []
     var seen: Set<SeenEvent> = Set()
     var counts: [String: UInt64] = [:]
-    
+    var ndb: Ndb
+
     private let network_monitor = NWPathMonitor()
     private let network_monitor_queue = DispatchQueue(label: "io.damus.network_monitor")
     private var last_network_status: NWPath.Status = .unsatisfied
 
-    init() {
+    init(ndb: Ndb) {
+        self.ndb = ndb
+
         network_monitor.pathUpdateHandler = { [weak self] path in
             if (path.status == .satisfied || path.status == .requiresConnection) && self?.last_network_status != path.status {
                 DispatchQueue.main.async {
@@ -110,9 +113,15 @@ class RelayPool {
         if get_relay(relay_id) != nil {
             throw RelayError.RelayAlreadyExists
         }
-        let conn = RelayConnection(url: url) { event in
+        let conn = RelayConnection(url: url, handleEvent: { event in
             self.handle_event(relay_id: relay_id, event: event)
-        }
+        }, processEvent: { wsev in
+            guard case .message(let msg) = wsev,
+                  case .string(let str) = msg
+            else { return }
+
+            self.ndb.process_event(str)
+        })
         let relay = Relay(descriptor: desc, connection: conn)
         self.relays.append(relay)
     }