From 7983157c38ab54aa39861c8b00f553ca0ec210cf Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Fri, 12 May 2023 07:15:30 -0500 Subject: [PATCH 01/18] add CoreData model for Profile --- damus.xcodeproj/project.pbxproj | 25 +++++++++++++++++++ .../Damus.xcdatamodel/contents | 16 ++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 damus/Nostr/CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index b7c8de47..f874ff59 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -259,6 +259,7 @@ 50088DA129E8271A008A1FDF /* WebSocket.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50088DA029E8271A008A1FDF /* WebSocket.swift */; }; 501F8C802A0220E1001AFC1D /* KeychainStorage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C7F2A0220E1001AFC1D /* KeychainStorage.swift */; }; 501F8C822A0224EB001AFC1D /* KeychainStorageTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C812A0224EB001AFC1D /* KeychainStorageTests.swift */; }; + 501F8C5829FF5FC5001AFC1D /* Damus.xcdatamodeld in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5629FF5FC5001AFC1D /* Damus.xcdatamodeld */; }; 50A50A8D29A09E1C00C01BE7 /* RequestTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */; }; 50B5685329F97CB400A23243 /* CredentialHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50B5685229F97CB400A23243 /* CredentialHandler.swift */; }; 5C42E78C29DB76D90086AAC1 /* EmptyUserSearchView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C42E78B29DB76D90086AAC1 /* EmptyUserSearchView.swift */; }; @@ -682,6 +683,7 @@ 50088DA029E8271A008A1FDF /* WebSocket.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebSocket.swift; sourceTree = ""; }; 501F8C7F2A0220E1001AFC1D /* KeychainStorage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeychainStorage.swift; sourceTree = ""; }; 501F8C812A0224EB001AFC1D /* KeychainStorageTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeychainStorageTests.swift; sourceTree = ""; }; + 501F8C5729FF5FC5001AFC1D /* Damus.xcdatamodel */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcdatamodel; path = Damus.xcdatamodel; sourceTree = ""; }; 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RequestTests.swift; sourceTree = ""; }; 50B5685229F97CB400A23243 /* CredentialHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CredentialHandler.swift; sourceTree = ""; }; 5C42E78B29DB76D90086AAC1 /* EmptyUserSearchView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EmptyUserSearchView.swift; sourceTree = ""; }; @@ -985,6 +987,7 @@ 4C75EFAB28049CC80006080F /* Nostr */ = { isa = PBXGroup; children = ( + 501F8C5329FF5EE2001AFC1D /* CoreData */, 4CE6DF1527F8DEBF00C66700 /* RelayConnection.swift */, 4C75EFA527FF87A20006080F /* Nostr.swift */, 4C75EFAE28049D340006080F /* NostrFilter.swift */, @@ -1358,6 +1361,14 @@ path = Images; sourceTree = ""; }; + 501F8C5329FF5EE2001AFC1D /* CoreData */ = { + isa = PBXGroup; + children = ( + 501F8C5629FF5FC5001AFC1D /* Damus.xcdatamodeld */, + ); + path = CoreData; + sourceTree = ""; + }; 7C0F392D29B57C8F0039859C /* Extensions */ = { isa = PBXGroup; children = ( @@ -1727,6 +1738,7 @@ 4C1A9A2729DDE31900516EAC /* TranslationSettingsView.swift in Sources */, 4CB8838629656C8B00DC99E7 /* NIP05.swift in Sources */, 4CF0ABD82981980C00D66079 /* Lists.swift in Sources */, + 501F8C5829FF5FC5001AFC1D /* Damus.xcdatamodeld in Sources */, 4C30AC8029A6A53F00E2BD5A /* ProfilePicturesView.swift in Sources */, 4C8EC52529D1FA6C0085D9A8 /* DamusColors.swift in Sources */, 4C5C7E6A284EDE2E00A22DF5 /* SearchResultsView.swift in Sources */, @@ -2329,6 +2341,19 @@ productName = secp256k1; }; /* End XCSwiftPackageProductDependency section */ + +/* Begin XCVersionGroup section */ + 501F8C5629FF5FC5001AFC1D /* Damus.xcdatamodeld */ = { + isa = XCVersionGroup; + children = ( + 501F8C5729FF5FC5001AFC1D /* Damus.xcdatamodel */, + ); + currentVersion = 501F8C5729FF5FC5001AFC1D /* Damus.xcdatamodel */; + path = Damus.xcdatamodeld; + sourceTree = ""; + versionGroupType = wrapper.xcdatamodel; + }; +/* End XCVersionGroup section */ }; rootObject = 4CE6DEDB27F7A08100C66700 /* Project object */; } diff --git a/damus/Nostr/CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents b/damus/Nostr/CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents new file mode 100644 index 00000000..57a6fe21 --- /dev/null +++ b/damus/Nostr/CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file From 76c57af5486f63e776cd01631d18c4a74462cc80 Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Fri, 12 May 2023 07:18:12 -0500 Subject: [PATCH 02/18] add managed object class to match new Profile CoreData model --- damus.xcodeproj/project.pbxproj | 4 ++++ damus/Nostr/CoreData/PersistedProfile.swift | 24 +++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 damus/Nostr/CoreData/PersistedProfile.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index f874ff59..581fd570 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -259,6 +259,7 @@ 50088DA129E8271A008A1FDF /* WebSocket.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50088DA029E8271A008A1FDF /* WebSocket.swift */; }; 501F8C802A0220E1001AFC1D /* KeychainStorage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C7F2A0220E1001AFC1D /* KeychainStorage.swift */; }; 501F8C822A0224EB001AFC1D /* KeychainStorageTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C812A0224EB001AFC1D /* KeychainStorageTests.swift */; }; + 501F8C5529FF5EF6001AFC1D /* PersistedProfile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5429FF5EF6001AFC1D /* PersistedProfile.swift */; }; 501F8C5829FF5FC5001AFC1D /* Damus.xcdatamodeld in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5629FF5FC5001AFC1D /* Damus.xcdatamodeld */; }; 50A50A8D29A09E1C00C01BE7 /* RequestTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */; }; 50B5685329F97CB400A23243 /* CredentialHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50B5685229F97CB400A23243 /* CredentialHandler.swift */; }; @@ -683,6 +684,7 @@ 50088DA029E8271A008A1FDF /* WebSocket.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebSocket.swift; sourceTree = ""; }; 501F8C7F2A0220E1001AFC1D /* KeychainStorage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeychainStorage.swift; sourceTree = ""; }; 501F8C812A0224EB001AFC1D /* KeychainStorageTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeychainStorageTests.swift; sourceTree = ""; }; + 501F8C5429FF5EF6001AFC1D /* PersistedProfile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PersistedProfile.swift; sourceTree = ""; }; 501F8C5729FF5FC5001AFC1D /* Damus.xcdatamodel */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcdatamodel; path = Damus.xcdatamodel; sourceTree = ""; }; 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RequestTests.swift; sourceTree = ""; }; 50B5685229F97CB400A23243 /* CredentialHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CredentialHandler.swift; sourceTree = ""; }; @@ -1364,6 +1366,7 @@ 501F8C5329FF5EE2001AFC1D /* CoreData */ = { isa = PBXGroup; children = ( + 501F8C5429FF5EF6001AFC1D /* PersistedProfile.swift */, 501F8C5629FF5FC5001AFC1D /* Damus.xcdatamodeld */, ); path = CoreData; @@ -1793,6 +1796,7 @@ F7908E92298B0F0700AB113A /* RelayDetailView.swift in Sources */, 4C3A1D332960DB0500558C0F /* Markdown.swift in Sources */, 4CE879552996BAB900F758CC /* RelayPaidDetail.swift in Sources */, + 501F8C5529FF5EF6001AFC1D /* PersistedProfile.swift in Sources */, 4CF0ABD42980996B00D66079 /* Report.swift in Sources */, 4C06670B28FDE64700038D2A /* damus.c in Sources */, 4C1A9A2529DDDF2600516EAC /* ZapSettingsView.swift in Sources */, diff --git a/damus/Nostr/CoreData/PersistedProfile.swift b/damus/Nostr/CoreData/PersistedProfile.swift new file mode 100644 index 00000000..a836d2f9 --- /dev/null +++ b/damus/Nostr/CoreData/PersistedProfile.swift @@ -0,0 +1,24 @@ +// +// PersistedProfile.swift +// damus +// +// Created by Bryan Montz on 4/30/23. +// + +import Foundation +import CoreData + +@objc(PersistedProfile) +final class PersistedProfile: NSManagedObject { + @NSManaged var id: String? + @NSManaged var name: String? + @NSManaged var display_name: String? + @NSManaged var about: String? + @NSManaged var picture: String? + @NSManaged var banner: String? + @NSManaged var website: String? + @NSManaged var lud06: String? + @NSManaged var lud16: String? + @NSManaged var nip05: String? + @NSManaged var last_update: Date? +} From 7027b7016ca84ccfb759361853d80cde4f00d57b Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Fri, 12 May 2023 07:19:43 -0500 Subject: [PATCH 03/18] add two-way translation between existing Profile class and new PersistedProfile CoreData class --- damus/Nostr/CoreData/PersistedProfile.swift | 12 ++++++++++++ damus/Nostr/Nostr.swift | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/damus/Nostr/CoreData/PersistedProfile.swift b/damus/Nostr/CoreData/PersistedProfile.swift index a836d2f9..4d8a037a 100644 --- a/damus/Nostr/CoreData/PersistedProfile.swift +++ b/damus/Nostr/CoreData/PersistedProfile.swift @@ -21,4 +21,16 @@ final class PersistedProfile: NSManagedObject { @NSManaged var lud16: String? @NSManaged var nip05: String? @NSManaged var last_update: Date? + + func copyValues(from profile: Profile) { + name = profile.name + display_name = profile.display_name + about = profile.about + picture = profile.picture + banner = profile.banner + website = profile.website + lud06 = profile.lud06 + lud16 = profile.lud16 + nip05 = profile.nip05 + } } diff --git a/damus/Nostr/Nostr.swift b/damus/Nostr/Nostr.swift index bcb7d775..81077c5e 100644 --- a/damus/Nostr/Nostr.swift +++ b/damus/Nostr/Nostr.swift @@ -23,6 +23,18 @@ class Profile: Codable { self.nip05 = nip05 } + convenience init(persisted_profile: PersistedProfile) { + self.init(name: persisted_profile.name, + display_name: persisted_profile.display_name, + about: persisted_profile.about, + picture: persisted_profile.picture, + banner: persisted_profile.banner, + website: persisted_profile.website, + lud06: persisted_profile.lud06, + lud16: persisted_profile.lud16, + nip05: persisted_profile.nip05) + } + private func str(_ str: String) -> String? { return get_val(str) } From 4646f0e23c1937cb1dc6e0a58b15377344195e3b Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Fri, 12 May 2023 07:21:25 -0500 Subject: [PATCH 04/18] add ProfileDatabase class to read and write profiles to disk --- damus.xcodeproj/project.pbxproj | 4 ++ damus/Nostr/ProfileDatabase.swift | 110 ++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 damus/Nostr/ProfileDatabase.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 581fd570..fb547596 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -261,6 +261,7 @@ 501F8C822A0224EB001AFC1D /* KeychainStorageTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C812A0224EB001AFC1D /* KeychainStorageTests.swift */; }; 501F8C5529FF5EF6001AFC1D /* PersistedProfile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5429FF5EF6001AFC1D /* PersistedProfile.swift */; }; 501F8C5829FF5FC5001AFC1D /* Damus.xcdatamodeld in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5629FF5FC5001AFC1D /* Damus.xcdatamodeld */; }; + 501F8C5A29FF70F5001AFC1D /* ProfileDatabase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5929FF70F5001AFC1D /* ProfileDatabase.swift */; }; 50A50A8D29A09E1C00C01BE7 /* RequestTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */; }; 50B5685329F97CB400A23243 /* CredentialHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50B5685229F97CB400A23243 /* CredentialHandler.swift */; }; 5C42E78C29DB76D90086AAC1 /* EmptyUserSearchView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C42E78B29DB76D90086AAC1 /* EmptyUserSearchView.swift */; }; @@ -686,6 +687,7 @@ 501F8C812A0224EB001AFC1D /* KeychainStorageTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeychainStorageTests.swift; sourceTree = ""; }; 501F8C5429FF5EF6001AFC1D /* PersistedProfile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PersistedProfile.swift; sourceTree = ""; }; 501F8C5729FF5FC5001AFC1D /* Damus.xcdatamodel */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcdatamodel; path = Damus.xcdatamodel; sourceTree = ""; }; + 501F8C5929FF70F5001AFC1D /* ProfileDatabase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileDatabase.swift; sourceTree = ""; }; 50A50A8C29A09E1C00C01BE7 /* RequestTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RequestTests.swift; sourceTree = ""; }; 50B5685229F97CB400A23243 /* CredentialHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CredentialHandler.swift; sourceTree = ""; }; 5C42E78B29DB76D90086AAC1 /* EmptyUserSearchView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EmptyUserSearchView.swift; sourceTree = ""; }; @@ -1000,6 +1002,7 @@ 4C75EFBA2804A34C0006080F /* ProofOfWork.swift */, 4CEE2AEC2805B22500AB5EEF /* NostrRequest.swift */, 4CACA9DB280C38C000D9BBE8 /* Profiles.swift */, + 501F8C5929FF70F5001AFC1D /* ProfileDatabase.swift */, 4C3BEFD32819DE8F00B3DE84 /* NostrKind.swift */, 4C363A8F28247A1D006E126D /* NostrLink.swift */, 50088DA029E8271A008A1FDF /* WebSocket.swift */, @@ -1651,6 +1654,7 @@ 4CD348EF29C3659D00497EB2 /* ImageUploadModel.swift in Sources */, 4C7D096E2A0AEA0400943473 /* ScannerCoordinator.swift in Sources */, 4C3BEFDC281DCE6100B3DE84 /* Liked.swift in Sources */, + 501F8C5A29FF70F5001AFC1D /* ProfileDatabase.swift in Sources */, 4CF0ABE7298444FD00D66079 /* MutedEventView.swift in Sources */, 9C83F89329A937B900136C08 /* TextViewWrapper.swift in Sources */, 4CF0ABE12981A83900D66079 /* MutelistView.swift in Sources */, diff --git a/damus/Nostr/ProfileDatabase.swift b/damus/Nostr/ProfileDatabase.swift new file mode 100644 index 00000000..6da247bd --- /dev/null +++ b/damus/Nostr/ProfileDatabase.swift @@ -0,0 +1,110 @@ +// +// ProfileDatabase.swift +// damus +// +// Created by Bryan Montz on 4/30/23. +// + +import Foundation +import CoreData + +enum ProfileDatabaseError: Error { + case missing_context + case outdated_input +} + +final class ProfileDatabase { + + private let entity_name = "PersistedProfile" + private var persistent_container: NSPersistentContainer? + private var background_context: NSManagedObjectContext? + + init() { + set_up() + } + + private var profile_cache_url: URL { + (FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.appendingPathComponent("profiles"))! + } + + private var persistent_store_description: NSPersistentStoreDescription { + let description = NSPersistentStoreDescription(url: profile_cache_url) + description.type = NSSQLiteStoreType + description.setOption(true as NSNumber, forKey: NSMigratePersistentStoresAutomaticallyOption) + description.setOption(true as NSNumber, forKey: NSInferMappingModelAutomaticallyOption) + description.setOption(true as NSNumber, forKey: NSSQLiteManualVacuumOption) + return description + } + + private var object_model: NSManagedObjectModel? { + guard let url = Bundle.main.url(forResource: "Damus", withExtension: "momd") else { + return nil + } + return NSManagedObjectModel(contentsOf: url) + } + + private func set_up() { + guard let object_model else { + print("⚠️ Warning: ProfileDatabase failed to load its object model") + return + } + + persistent_container = NSPersistentContainer(name: "Damus", managedObjectModel: object_model) + persistent_container?.persistentStoreDescriptions = [persistent_store_description] + persistent_container?.loadPersistentStores { _, error in + if let error { + print("WARNING: ProfileDatabase failed to load: \(error)") + } + } + + persistent_container?.viewContext.automaticallyMergesChangesFromParent = true + persistent_container?.viewContext.mergePolicy = NSMergePolicy(merge: .mergeByPropertyObjectTrumpMergePolicyType) + + background_context = persistent_container?.newBackgroundContext() + background_context?.mergePolicy = NSMergePolicy(merge: .mergeByPropertyObjectTrumpMergePolicyType) + } + + private func get_persisted(id: String) -> PersistedProfile? { + let request = NSFetchRequest(entityName: entity_name) + request.predicate = NSPredicate(format: "id == %@", id) + request.fetchLimit = 1 + return try? persistent_container?.viewContext.fetch(request).first + } + + // MARK: - Public + + /// Updates or inserts a new Profile into the local database. Rejects profiles whose update date + /// is older than one we already have. Database writes occur on a background context for best performance. + /// - Parameters: + /// - id: Profile id (pubkey) + /// - profile: Profile object to be stored + /// - last_update: Date that the Profile was updated + func upsert(id: String, profile: Profile, last_update: Date) throws { + guard let context = background_context else { + throw ProfileDatabaseError.missing_context + } + + var persisted_profile: PersistedProfile? + if let profile = get_persisted(id: id) { + if let existing_last_update = profile.last_update, last_update < existing_last_update { + throw ProfileDatabaseError.outdated_input + } else { + persisted_profile = profile + } + } else { + persisted_profile = NSEntityDescription.insertNewObject(forEntityName: entity_name, into: context) as? PersistedProfile + persisted_profile?.id = id + } + persisted_profile?.copyValues(from: profile) + persisted_profile?.last_update = last_update + + try context.save() + } + + func get(id: String) -> Profile? { + guard let profile = get_persisted(id: id) else { + return nil + } + return Profile(persisted_profile: profile) + } +} From 3b0bb48dd43c45b105f957914450c33e9eea2584 Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Fri, 12 May 2023 07:21:59 -0500 Subject: [PATCH 05/18] integrate ProfileDatabase with existing Profiles caching class --- damus/Nostr/Profiles.swift | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/damus/Nostr/Profiles.swift b/damus/Nostr/Profiles.swift index fa3f06d3..6cfa6272 100644 --- a/damus/Nostr/Profiles.swift +++ b/damus/Nostr/Profiles.swift @@ -6,8 +6,6 @@ // import Foundation -import UIKit - class Profiles { @@ -22,28 +20,34 @@ class Profiles { var nip05_pubkey: [String: String] = [:] var zappers: [String: String] = [:] + private let database = ProfileDatabase() + func is_validated(_ pk: String) -> NIP05? { - return validated[pk] + validated[pk] } func lookup_zapper(pubkey: String) -> String? { - if let zapper = zappers[pubkey] { - return zapper - } - - return nil + zappers[pubkey] } func add(id: String, profile: TimestampedProfile) { queue.async(flags: .barrier) { self.profiles[id] = profile } + + do { + try database.upsert(id: id, profile: profile.profile, last_update: Date(timeIntervalSince1970: TimeInterval(profile.timestamp))) + } catch { + print("⚠️ Warning: Profiles failed to save a profile: \(error)") + } } func lookup(id: String) -> Profile? { + var profile: Profile? queue.sync { - return profiles[id]?.profile + profile = profiles[id]?.profile } + return profile ?? database.get(id: id) } func lookup_with_timestamp(id: String) -> TimestampedProfile? { From b5ae7df795ddcc2b4b1da02ec2bc8bdb99a063e1 Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Sat, 13 May 2023 09:17:41 -0500 Subject: [PATCH 06/18] add ability to change ProfileDatabase's storage URL --- damus/Nostr/ProfileDatabase.swift | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/damus/Nostr/ProfileDatabase.swift b/damus/Nostr/ProfileDatabase.swift index 6da247bd..47490f6d 100644 --- a/damus/Nostr/ProfileDatabase.swift +++ b/damus/Nostr/ProfileDatabase.swift @@ -18,17 +18,19 @@ final class ProfileDatabase { private let entity_name = "PersistedProfile" private var persistent_container: NSPersistentContainer? private var background_context: NSManagedObjectContext? + private let cache_url: URL - init() { + init(cache_url: URL = ProfileDatabase.profile_cache_url) { + self.cache_url = cache_url set_up() } - private var profile_cache_url: URL { + private static var profile_cache_url: URL { (FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.appendingPathComponent("profiles"))! } private var persistent_store_description: NSPersistentStoreDescription { - let description = NSPersistentStoreDescription(url: profile_cache_url) + let description = NSPersistentStoreDescription(url: cache_url) description.type = NSSQLiteStoreType description.setOption(true as NSNumber, forKey: NSMigratePersistentStoresAutomaticallyOption) description.setOption(true as NSNumber, forKey: NSInferMappingModelAutomaticallyOption) From 31d327a0857b91d8fc13bf48a7f1bee24a6d0506 Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Sat, 13 May 2023 09:18:23 -0500 Subject: [PATCH 07/18] add ability to retrieve count of cached Profiles --- damus/Nostr/ProfileDatabase.swift | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/damus/Nostr/ProfileDatabase.swift b/damus/Nostr/ProfileDatabase.swift index 47490f6d..845e0ec3 100644 --- a/damus/Nostr/ProfileDatabase.swift +++ b/damus/Nostr/ProfileDatabase.swift @@ -109,4 +109,10 @@ final class ProfileDatabase { } return Profile(persisted_profile: profile) } + + var count: Int { + let request = NSFetchRequest(entityName: entity_name) + let count = try? persistent_container?.viewContext.count(for: request) + return count ?? 0 + } } From 6172347455b5e1b654ee60ff93496dac26ec55ca Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Sat, 13 May 2023 09:18:47 -0500 Subject: [PATCH 08/18] add ability to remove all cached Profiles in ProfileDatabase --- damus/Nostr/ProfileDatabase.swift | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/damus/Nostr/ProfileDatabase.swift b/damus/Nostr/ProfileDatabase.swift index 845e0ec3..66df42f8 100644 --- a/damus/Nostr/ProfileDatabase.swift +++ b/damus/Nostr/ProfileDatabase.swift @@ -115,4 +115,24 @@ final class ProfileDatabase { let count = try? persistent_container?.viewContext.count(for: request) return count ?? 0 } + + func remove_all_profiles() throws { + guard let context = background_context, let container = persistent_container else { + throw ProfileDatabaseError.missing_context + } + + let request = NSFetchRequest(entityName: entity_name) + let batch_delete_request = NSBatchDeleteRequest(fetchRequest: request) + batch_delete_request.resultType = .resultTypeObjectIDs + + let result = try container.persistentStoreCoordinator.execute(batch_delete_request, with: context) as! NSBatchDeleteResult + + // NSBatchDeleteRequest is an NSPersistentStoreRequest, which operates on disk. So now we'll manually update our in-memory context. + if let object_ids = result.result as? [NSManagedObjectID] { + let changes: [AnyHashable: Any] = [ + NSDeletedObjectsKey: object_ids + ] + NSManagedObjectContext.mergeChanges(fromRemoteContextSave: changes, into: [context]) + } + } } From 32431096f5666562f42e2aa1b74c06a2e961eaeb Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Sat, 13 May 2023 09:20:01 -0500 Subject: [PATCH 09/18] add tests for ProfileDatabase --- damus.xcodeproj/project.pbxproj | 4 + damusTests/ProfileDatabaseTests.swift | 117 ++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 damusTests/ProfileDatabaseTests.swift diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index fb547596..91ddb701 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -259,6 +259,7 @@ 50088DA129E8271A008A1FDF /* WebSocket.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50088DA029E8271A008A1FDF /* WebSocket.swift */; }; 501F8C802A0220E1001AFC1D /* KeychainStorage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C7F2A0220E1001AFC1D /* KeychainStorage.swift */; }; 501F8C822A0224EB001AFC1D /* KeychainStorageTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C812A0224EB001AFC1D /* KeychainStorageTests.swift */; }; + 5019CADD2A0FB0A9000069E1 /* ProfileDatabaseTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5019CADC2A0FB0A9000069E1 /* ProfileDatabaseTests.swift */; }; 501F8C5529FF5EF6001AFC1D /* PersistedProfile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5429FF5EF6001AFC1D /* PersistedProfile.swift */; }; 501F8C5829FF5FC5001AFC1D /* Damus.xcdatamodeld in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5629FF5FC5001AFC1D /* Damus.xcdatamodeld */; }; 501F8C5A29FF70F5001AFC1D /* ProfileDatabase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5929FF70F5001AFC1D /* ProfileDatabase.swift */; }; @@ -685,6 +686,7 @@ 50088DA029E8271A008A1FDF /* WebSocket.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebSocket.swift; sourceTree = ""; }; 501F8C7F2A0220E1001AFC1D /* KeychainStorage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeychainStorage.swift; sourceTree = ""; }; 501F8C812A0224EB001AFC1D /* KeychainStorageTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeychainStorageTests.swift; sourceTree = ""; }; + 5019CADC2A0FB0A9000069E1 /* ProfileDatabaseTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileDatabaseTests.swift; sourceTree = ""; }; 501F8C5429FF5EF6001AFC1D /* PersistedProfile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PersistedProfile.swift; sourceTree = ""; }; 501F8C5729FF5FC5001AFC1D /* Damus.xcdatamodel */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcdatamodel; path = Damus.xcdatamodel; sourceTree = ""; }; 501F8C5929FF70F5001AFC1D /* ProfileDatabase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileDatabase.swift; sourceTree = ""; }; @@ -1275,6 +1277,7 @@ 4CB883AD2976FA9300DC99E7 /* FormatTests.swift */, 3A3040EC29A5CB86008A0F29 /* ReplyDescriptionTests.swift */, 3A3040EE29A8FEE9008A0F29 /* EventDetailBarTests.swift */, + 5019CADC2A0FB0A9000069E1 /* ProfileDatabaseTests.swift */, 3A3040F229A91366008A0F29 /* ProfileViewTests.swift */, 3A30410029AB12AA008A0F29 /* EventGroupViewTests.swift */, 4C8D00D329E3C5D40036AF10 /* NIP19Tests.swift */, @@ -1833,6 +1836,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 5019CADD2A0FB0A9000069E1 /* ProfileDatabaseTests.swift in Sources */, 3A3040ED29A5CB86008A0F29 /* ReplyDescriptionTests.swift in Sources */, 4C8D00D429E3C5D40036AF10 /* NIP19Tests.swift in Sources */, 3A30410129AB12AA008A0F29 /* EventGroupViewTests.swift in Sources */, diff --git a/damusTests/ProfileDatabaseTests.swift b/damusTests/ProfileDatabaseTests.swift new file mode 100644 index 00000000..31fe4b1b --- /dev/null +++ b/damusTests/ProfileDatabaseTests.swift @@ -0,0 +1,117 @@ +// +// ProfileDatabaseTests.swift +// damusTests +// +// Created by Bryan Montz on 5/13/23. +// + +import XCTest +@testable import damus + +class ProfileDatabaseTests: XCTestCase { + + static let cache_url = (FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.appendingPathComponent("test-profiles"))! + let database = ProfileDatabase(cache_url: ProfileDatabaseTests.cache_url) + + override func tearDownWithError() throws { + // This method is called after the invocation of each test method in the class. + try database.remove_all_profiles() + } + + var test_profile: Profile { + Profile(name: "test-name", + display_name: "test-display-name", + about: "test-about", + picture: "test-picture", + banner: "test-banner", + website: "test-website", + lud06: "test-lud06", + lud16: "test-lud16", + nip05: "test-nip05") + } + + func testStoreAndRetrieveProfile() throws { + let id = "test-id" + + let profile = test_profile + + // make sure it's not there yet + XCTAssertNil(database.get(id: id)) + + // store the profile + try database.upsert(id: id, profile: profile, last_update: .now) + + // read the profile out of the database + let retrievedProfile = try XCTUnwrap(database.get(id: id)) + + XCTAssertEqual(profile.name, retrievedProfile.name) + XCTAssertEqual(profile.display_name, retrievedProfile.display_name) + XCTAssertEqual(profile.about, retrievedProfile.about) + XCTAssertEqual(profile.picture, retrievedProfile.picture) + XCTAssertEqual(profile.banner, retrievedProfile.banner) + XCTAssertEqual(profile.website, retrievedProfile.website) + XCTAssertEqual(profile.lud06, retrievedProfile.lud06) + XCTAssertEqual(profile.lud16, retrievedProfile.lud16) + XCTAssertEqual(profile.nip05, retrievedProfile.nip05) + } + + func testRejectOutdatedProfile() throws { + let id = "test-id" + + // store a profile + let profile = test_profile + let profile_last_updated = Date.now + try database.upsert(id: id, profile: profile, last_update: profile_last_updated) + + // try to store a profile with the same id but the last_update date is older than the previously stored profile + let outdatedProfile = test_profile + let outdated_last_updated = profile_last_updated.addingTimeInterval(-60) + + XCTAssertThrowsError(try database.upsert(id: id, profile: outdatedProfile, last_update: outdated_last_updated)) { error in + XCTAssertEqual(error as? ProfileDatabaseError, ProfileDatabaseError.outdated_input) + } + } + + func testUpdateExistingProfile() throws { + let id = "test-id" + + // store a profile + let profile = test_profile + let profile_last_update = Date.now + try database.upsert(id: id, profile: profile, last_update: profile_last_update) + + // update the same profile + let updated_profile = test_profile + updated_profile.nip05 = "updated-nip05" + let updated_profile_last_update = profile_last_update.addingTimeInterval(60) + try database.upsert(id: id, profile: updated_profile, last_update: updated_profile_last_update) + + // retrieve the profile and make sure it was updated + let retrieved_profile = database.get(id: id) + XCTAssertEqual(retrieved_profile?.nip05, "updated-nip05") + } + + func testStoreMultipleAndRemoveAllProfiles() throws { + XCTAssertEqual(database.count, 0) + + // store a profile + let id = "test-id" + let profile = test_profile + let profile_last_update = Date.now + try database.upsert(id: id, profile: profile, last_update: profile_last_update) + + XCTAssertEqual(database.count, 1) + + // store another profile + let id2 = "test-id-2" + let profile2 = test_profile + let profile_last_update2 = Date.now + try database.upsert(id: id2, profile: profile2, last_update: profile_last_update2) + + XCTAssertEqual(database.count, 2) + + try database.remove_all_profiles() + + XCTAssertEqual(database.count, 0) + } +} From bc315dd571574af83cc4ad9b5f8307d08c40a6fa Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Mon, 15 May 2023 07:33:49 -0500 Subject: [PATCH 10/18] remove identical function --- damus/Models/FollowersModel.swift | 2 +- damus/Models/SearchHomeModel.swift | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/damus/Models/FollowersModel.swift b/damus/Models/FollowersModel.swift index bbc83b29..d28fa0b6 100644 --- a/damus/Models/FollowersModel.swift +++ b/damus/Models/FollowersModel.swift @@ -57,7 +57,7 @@ class FollowersModel: ObservableObject { func load_profiles(relay_id: String) { var filter = NostrFilter.filter_profiles - let authors = find_profiles_to_fetch_pk(profiles: damus_state.profiles, event_pubkeys: contacts ?? []) + let authors = find_profiles_to_fetch_from_keys(profiles: damus_state.profiles, pks: contacts ?? []) if authors.isEmpty { return } diff --git a/damus/Models/SearchHomeModel.swift b/damus/Models/SearchHomeModel.swift index 221f78fb..3b8c4642 100644 --- a/damus/Models/SearchHomeModel.swift +++ b/damus/Models/SearchHomeModel.swift @@ -90,20 +90,6 @@ class SearchHomeModel: ObservableObject { } } -func find_profiles_to_fetch_pk(profiles: Profiles, event_pubkeys: [String]) -> [String] { - var pubkeys = Set() - - for pk in event_pubkeys { - if profiles.lookup(id: pk) != nil { - continue - } - - pubkeys.insert(pk) - } - - return Array(pubkeys) -} - func find_profiles_to_fetch(profiles: Profiles, load: PubkeysToLoad, cache: EventCache) -> [String] { switch load { case .from_events(let events): From 3f7b0a4d6e1aeabad45aa64d4aba619e374e9ccd Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Mon, 15 May 2023 08:29:06 -0500 Subject: [PATCH 11/18] add network pull date to PersistedProfile model for staleness checking --- .../CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents | 3 ++- damus/Nostr/CoreData/PersistedProfile.swift | 3 ++- damus/Nostr/ProfileDatabase.swift | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/damus/Nostr/CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents b/damus/Nostr/CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents index 57a6fe21..0f985f18 100644 --- a/damus/Nostr/CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents +++ b/damus/Nostr/CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents @@ -1,5 +1,5 @@ - + @@ -9,6 +9,7 @@ + diff --git a/damus/Nostr/CoreData/PersistedProfile.swift b/damus/Nostr/CoreData/PersistedProfile.swift index 4d8a037a..6a1b0790 100644 --- a/damus/Nostr/CoreData/PersistedProfile.swift +++ b/damus/Nostr/CoreData/PersistedProfile.swift @@ -20,7 +20,8 @@ final class PersistedProfile: NSManagedObject { @NSManaged var lud06: String? @NSManaged var lud16: String? @NSManaged var nip05: String? - @NSManaged var last_update: Date? + @NSManaged var last_update: Date? // The date that the profile was last updated by the user + @NSManaged var network_pull_date: Date? // The date we got this profile from a relay (for staleness checking) func copyValues(from profile: Profile) { name = profile.name diff --git a/damus/Nostr/ProfileDatabase.swift b/damus/Nostr/ProfileDatabase.swift index 66df42f8..c16491a1 100644 --- a/damus/Nostr/ProfileDatabase.swift +++ b/damus/Nostr/ProfileDatabase.swift @@ -99,6 +99,7 @@ final class ProfileDatabase { } persisted_profile?.copyValues(from: profile) persisted_profile?.last_update = last_update + persisted_profile?.network_pull_date = Date.now try context.save() } From d58a1e0ba35a84b488c6a0afa5618d2fc1c384fd Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Mon, 15 May 2023 08:31:49 -0500 Subject: [PATCH 12/18] add ability to check the freshness of a PersistedProfile --- damus/Nostr/ProfileDatabase.swift | 2 +- damus/Nostr/Profiles.swift | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/damus/Nostr/ProfileDatabase.swift b/damus/Nostr/ProfileDatabase.swift index c16491a1..5ee1b316 100644 --- a/damus/Nostr/ProfileDatabase.swift +++ b/damus/Nostr/ProfileDatabase.swift @@ -66,7 +66,7 @@ final class ProfileDatabase { background_context?.mergePolicy = NSMergePolicy(merge: .mergeByPropertyObjectTrumpMergePolicyType) } - private func get_persisted(id: String) -> PersistedProfile? { + func get_persisted(id: String) -> PersistedProfile? { let request = NSFetchRequest(entityName: entity_name) request.predicate = NSPredicate(format: "id == %@", id) request.fetchLimit = 1 diff --git a/damus/Nostr/Profiles.swift b/damus/Nostr/Profiles.swift index 6cfa6272..d13f1117 100644 --- a/damus/Nostr/Profiles.swift +++ b/damus/Nostr/Profiles.swift @@ -9,6 +9,8 @@ import Foundation class Profiles { + static let db_freshness_threshold: TimeInterval = 24 * 60 * 60 + /// This queue is used to synchronize access to the profiles dictionary, which /// prevents data races from crashing the app. private var queue = DispatchQueue(label: "io.damus.profiles", @@ -55,4 +57,22 @@ class Profiles { return profiles[id] } } + + func has_fresh_profile(id: String) -> Bool { + // check memory first + var profile: Profile? + queue.sync { + profile = profiles[id]?.profile + } + if profile != nil { + return true + } + + // then disk + guard let persisted_profile = database.get_persisted(id: id), + let pull_date = persisted_profile.network_pull_date else { + return false + } + return Date.now.timeIntervalSince(pull_date) < Profiles.db_freshness_threshold + } } From 91113fbc6d9cb9c694b2b766a4fc16246dacd1f8 Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Mon, 15 May 2023 08:33:35 -0500 Subject: [PATCH 13/18] allow models to fetch profiles when they get stale --- damus/Models/FollowingModel.swift | 2 +- damus/Models/SearchHomeModel.swift | 16 +++------------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/damus/Models/FollowingModel.swift b/damus/Models/FollowingModel.swift index 900a2867..4638dd26 100644 --- a/damus/Models/FollowingModel.swift +++ b/damus/Models/FollowingModel.swift @@ -24,7 +24,7 @@ class FollowingModel { var f = NostrFilter.filter_kinds([NostrKind.metadata.rawValue]) f.authors = self.contacts.reduce(into: Array()) { acc, pk in // don't fetch profiles we already have - if damus_state.profiles.lookup(id: pk) != nil { + if damus_state.profiles.has_fresh_profile(id: pk) { return } acc.append(pk) diff --git a/damus/Models/SearchHomeModel.swift b/damus/Models/SearchHomeModel.swift index 3b8c4642..0d6af0cf 100644 --- a/damus/Models/SearchHomeModel.swift +++ b/damus/Models/SearchHomeModel.swift @@ -100,17 +100,7 @@ func find_profiles_to_fetch(profiles: Profiles, load: PubkeysToLoad, cache: Even } func find_profiles_to_fetch_from_keys(profiles: Profiles, pks: [String]) -> [String] { - var pubkeys = Set() - - for pk in pks { - if profiles.lookup(id: pk) != nil { - continue - } - - pubkeys.insert(pk) - } - - return Array(pubkeys) + Array(Set(pks.filter { pk in !profiles.has_fresh_profile(id: pk) })) } func find_profiles_to_fetch_from_events(profiles: Profiles, events: [NostrEvent], cache: EventCache) -> [String] { @@ -118,11 +108,11 @@ func find_profiles_to_fetch_from_events(profiles: Profiles, events: [NostrEvent] for ev in events { // lookup profiles from boosted events - if ev.known_kind == .boost, let bev = ev.get_inner_event(cache: cache), profiles.lookup(id: bev.pubkey) == nil { + if ev.known_kind == .boost, let bev = ev.get_inner_event(cache: cache), !profiles.has_fresh_profile(id: bev.pubkey) { pubkeys.insert(bev.pubkey) } - if profiles.lookup(id: ev.pubkey) == nil { + if !profiles.has_fresh_profile(id: ev.pubkey) { pubkeys.insert(ev.pubkey) } } From 7259641e269a361e328f7cd4ce4411156f741f81 Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Thu, 25 May 2023 08:20:41 -0500 Subject: [PATCH 14/18] speed optimizations: cache network_pull_date in memory and ensure writes on background queue --- damus/Nostr/ProfileDatabase.swift | 62 ++++++++++++++++++++++--------- damus/Nostr/Profiles.swift | 3 +- 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/damus/Nostr/ProfileDatabase.swift b/damus/Nostr/ProfileDatabase.swift index 5ee1b316..31a4fb43 100644 --- a/damus/Nostr/ProfileDatabase.swift +++ b/damus/Nostr/ProfileDatabase.swift @@ -19,6 +19,7 @@ final class ProfileDatabase { private var persistent_container: NSPersistentContainer? private var background_context: NSManagedObjectContext? private let cache_url: URL + private var network_pull_date_cache = [String: Date]() init(cache_url: URL = ProfileDatabase.profile_cache_url) { self.cache_url = cache_url @@ -66,11 +67,28 @@ final class ProfileDatabase { background_context?.mergePolicy = NSMergePolicy(merge: .mergeByPropertyObjectTrumpMergePolicyType) } - func get_persisted(id: String) -> PersistedProfile? { + private func get_persisted(id: String, context: NSManagedObjectContext) -> PersistedProfile? { let request = NSFetchRequest(entityName: entity_name) request.predicate = NSPredicate(format: "id == %@", id) request.fetchLimit = 1 - return try? persistent_container?.viewContext.fetch(request).first + return try? context.fetch(request).first + } + + func get_network_pull_date(id: String) -> Date? { + if let pull_date = network_pull_date_cache[id] { + return pull_date + } + + let request = NSFetchRequest(entityName: entity_name) + request.predicate = NSPredicate(format: "id == %@", id) + request.fetchLimit = 1 + request.propertiesToFetch = ["network_pull_date"] + guard let profile = try? persistent_container?.viewContext.fetch(request).first else { + return nil + } + + network_pull_date_cache[id] = profile.network_pull_date + return profile.network_pull_date } // MARK: - Public @@ -86,26 +104,34 @@ final class ProfileDatabase { throw ProfileDatabaseError.missing_context } - var persisted_profile: PersistedProfile? - if let profile = get_persisted(id: id) { - if let existing_last_update = profile.last_update, last_update < existing_last_update { - throw ProfileDatabaseError.outdated_input - } else { - persisted_profile = profile + Task { + try await context.perform { + var persisted_profile: PersistedProfile? + if let profile = self.get_persisted(id: id, context: context) { + if let existing_last_update = profile.last_update, last_update < existing_last_update { + throw ProfileDatabaseError.outdated_input + } else { + persisted_profile = profile + } + } else { + persisted_profile = NSEntityDescription.insertNewObject(forEntityName: self.entity_name, into: context) as? PersistedProfile + persisted_profile?.id = id + } + persisted_profile?.copyValues(from: profile) + persisted_profile?.last_update = last_update + + let pull_date = Date.now + persisted_profile?.network_pull_date = pull_date + self.network_pull_date_cache[id] = pull_date + + try context.save() } - } else { - persisted_profile = NSEntityDescription.insertNewObject(forEntityName: entity_name, into: context) as? PersistedProfile - persisted_profile?.id = id } - persisted_profile?.copyValues(from: profile) - persisted_profile?.last_update = last_update - persisted_profile?.network_pull_date = Date.now - - try context.save() } func get(id: String) -> Profile? { - guard let profile = get_persisted(id: id) else { + guard let container = persistent_container, + let profile = get_persisted(id: id, context: container.viewContext) else { return nil } return Profile(persisted_profile: profile) @@ -122,6 +148,8 @@ final class ProfileDatabase { throw ProfileDatabaseError.missing_context } + network_pull_date_cache.removeAll() + let request = NSFetchRequest(entityName: entity_name) let batch_delete_request = NSBatchDeleteRequest(fetchRequest: request) batch_delete_request.resultType = .resultTypeObjectIDs diff --git a/damus/Nostr/Profiles.swift b/damus/Nostr/Profiles.swift index d13f1117..f0aec4bb 100644 --- a/damus/Nostr/Profiles.swift +++ b/damus/Nostr/Profiles.swift @@ -69,8 +69,7 @@ class Profiles { } // then disk - guard let persisted_profile = database.get_persisted(id: id), - let pull_date = persisted_profile.network_pull_date else { + guard let pull_date = database.get_network_pull_date(id: id) else { return false } return Date.now.timeIntervalSince(pull_date) < Profiles.db_freshness_threshold From 61ff7da2aebddf9e6823491c70e1f131e7bedb20 Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Thu, 25 May 2023 09:09:50 -0500 Subject: [PATCH 15/18] make database write async at the call site and update tests --- damus/Nostr/ProfileDatabase.swift | 38 +++++++++++++-------------- damus/Nostr/Profiles.swift | 10 ++++--- damusTests/ProfileDatabaseTests.swift | 29 +++++++++++--------- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/damus/Nostr/ProfileDatabase.swift b/damus/Nostr/ProfileDatabase.swift index 31a4fb43..9fbd2c9b 100644 --- a/damus/Nostr/ProfileDatabase.swift +++ b/damus/Nostr/ProfileDatabase.swift @@ -99,33 +99,31 @@ final class ProfileDatabase { /// - id: Profile id (pubkey) /// - profile: Profile object to be stored /// - last_update: Date that the Profile was updated - func upsert(id: String, profile: Profile, last_update: Date) throws { + func upsert(id: String, profile: Profile, last_update: Date) async throws { guard let context = background_context else { throw ProfileDatabaseError.missing_context } - Task { - try await context.perform { - var persisted_profile: PersistedProfile? - if let profile = self.get_persisted(id: id, context: context) { - if let existing_last_update = profile.last_update, last_update < existing_last_update { - throw ProfileDatabaseError.outdated_input - } else { - persisted_profile = profile - } + try await context.perform { + var persisted_profile: PersistedProfile? + if let profile = self.get_persisted(id: id, context: context) { + if let existing_last_update = profile.last_update, last_update < existing_last_update { + throw ProfileDatabaseError.outdated_input } else { - persisted_profile = NSEntityDescription.insertNewObject(forEntityName: self.entity_name, into: context) as? PersistedProfile - persisted_profile?.id = id + persisted_profile = profile } - persisted_profile?.copyValues(from: profile) - persisted_profile?.last_update = last_update - - let pull_date = Date.now - persisted_profile?.network_pull_date = pull_date - self.network_pull_date_cache[id] = pull_date - - try context.save() + } else { + persisted_profile = NSEntityDescription.insertNewObject(forEntityName: self.entity_name, into: context) as? PersistedProfile + persisted_profile?.id = id } + persisted_profile?.copyValues(from: profile) + persisted_profile?.last_update = last_update + + let pull_date = Date.now + persisted_profile?.network_pull_date = pull_date + self.network_pull_date_cache[id] = pull_date + + try context.save() } } diff --git a/damus/Nostr/Profiles.swift b/damus/Nostr/Profiles.swift index f0aec4bb..05f914db 100644 --- a/damus/Nostr/Profiles.swift +++ b/damus/Nostr/Profiles.swift @@ -37,10 +37,12 @@ class Profiles { self.profiles[id] = profile } - do { - try database.upsert(id: id, profile: profile.profile, last_update: Date(timeIntervalSince1970: TimeInterval(profile.timestamp))) - } catch { - print("⚠️ Warning: Profiles failed to save a profile: \(error)") + Task { + do { + try await database.upsert(id: id, profile: profile.profile, last_update: Date(timeIntervalSince1970: TimeInterval(profile.timestamp))) + } catch { + print("⚠️ Warning: Profiles failed to save a profile: \(error)") + } } } diff --git a/damusTests/ProfileDatabaseTests.swift b/damusTests/ProfileDatabaseTests.swift index 31fe4b1b..89a2c189 100644 --- a/damusTests/ProfileDatabaseTests.swift +++ b/damusTests/ProfileDatabaseTests.swift @@ -30,7 +30,7 @@ class ProfileDatabaseTests: XCTestCase { nip05: "test-nip05") } - func testStoreAndRetrieveProfile() throws { + func testStoreAndRetrieveProfile() async throws { let id = "test-id" let profile = test_profile @@ -39,7 +39,7 @@ class ProfileDatabaseTests: XCTestCase { XCTAssertNil(database.get(id: id)) // store the profile - try database.upsert(id: id, profile: profile, last_update: .now) + try await database.upsert(id: id, profile: profile, last_update: .now) // read the profile out of the database let retrievedProfile = try XCTUnwrap(database.get(id: id)) @@ -55,50 +55,55 @@ class ProfileDatabaseTests: XCTestCase { XCTAssertEqual(profile.nip05, retrievedProfile.nip05) } - func testRejectOutdatedProfile() throws { + func testRejectOutdatedProfile() async throws { let id = "test-id" // store a profile let profile = test_profile let profile_last_updated = Date.now - try database.upsert(id: id, profile: profile, last_update: profile_last_updated) + try await database.upsert(id: id, profile: profile, last_update: profile_last_updated) // try to store a profile with the same id but the last_update date is older than the previously stored profile let outdatedProfile = test_profile let outdated_last_updated = profile_last_updated.addingTimeInterval(-60) - XCTAssertThrowsError(try database.upsert(id: id, profile: outdatedProfile, last_update: outdated_last_updated)) { error in - XCTAssertEqual(error as? ProfileDatabaseError, ProfileDatabaseError.outdated_input) + do { + try await database.upsert(id: id, profile: outdatedProfile, last_update: outdated_last_updated) + XCTFail("expected to throw error") + } catch let error as ProfileDatabaseError { + XCTAssertEqual(error, ProfileDatabaseError.outdated_input) + } catch { + XCTFail("not the expected error") } } - func testUpdateExistingProfile() throws { + func testUpdateExistingProfile() async throws { let id = "test-id" // store a profile let profile = test_profile let profile_last_update = Date.now - try database.upsert(id: id, profile: profile, last_update: profile_last_update) + try await database.upsert(id: id, profile: profile, last_update: profile_last_update) // update the same profile let updated_profile = test_profile updated_profile.nip05 = "updated-nip05" let updated_profile_last_update = profile_last_update.addingTimeInterval(60) - try database.upsert(id: id, profile: updated_profile, last_update: updated_profile_last_update) + try await database.upsert(id: id, profile: updated_profile, last_update: updated_profile_last_update) // retrieve the profile and make sure it was updated let retrieved_profile = database.get(id: id) XCTAssertEqual(retrieved_profile?.nip05, "updated-nip05") } - func testStoreMultipleAndRemoveAllProfiles() throws { + func testStoreMultipleAndRemoveAllProfiles() async throws { XCTAssertEqual(database.count, 0) // store a profile let id = "test-id" let profile = test_profile let profile_last_update = Date.now - try database.upsert(id: id, profile: profile, last_update: profile_last_update) + try await database.upsert(id: id, profile: profile, last_update: profile_last_update) XCTAssertEqual(database.count, 1) @@ -106,7 +111,7 @@ class ProfileDatabaseTests: XCTestCase { let id2 = "test-id-2" let profile2 = test_profile let profile_last_update2 = Date.now - try database.upsert(id: id2, profile: profile2, last_update: profile_last_update2) + try await database.upsert(id: id2, profile: profile2, last_update: profile_last_update2) XCTAssertEqual(database.count, 2) From 7e963c902510ea548fe5a638fd2c256f84b42995 Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Fri, 26 May 2023 06:38:20 -0500 Subject: [PATCH 16/18] synchronize access to network_pull_date_cache with a queue, writes with barrier flag --- damus/Nostr/ProfileDatabase.swift | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/damus/Nostr/ProfileDatabase.swift b/damus/Nostr/ProfileDatabase.swift index 9fbd2c9b..0e5ec17d 100644 --- a/damus/Nostr/ProfileDatabase.swift +++ b/damus/Nostr/ProfileDatabase.swift @@ -19,6 +19,12 @@ final class ProfileDatabase { private var persistent_container: NSPersistentContainer? private var background_context: NSManagedObjectContext? private let cache_url: URL + + /// This queue is used to synchronize access to the network_pull_date_cache dictionary, which + /// prevents data races from crashing the app. + private var queue = DispatchQueue(label: "io.damus.profile_db", + qos: .userInteractive, + attributes: .concurrent) private var network_pull_date_cache = [String: Date]() init(cache_url: URL = ProfileDatabase.profile_cache_url) { @@ -75,7 +81,11 @@ final class ProfileDatabase { } func get_network_pull_date(id: String) -> Date? { - if let pull_date = network_pull_date_cache[id] { + var pull_date: Date? + queue.sync { + pull_date = network_pull_date_cache[id] + } + if let pull_date { return pull_date } @@ -87,7 +97,9 @@ final class ProfileDatabase { return nil } - network_pull_date_cache[id] = profile.network_pull_date + queue.async(flags: .barrier) { + self.network_pull_date_cache[id] = profile.network_pull_date + } return profile.network_pull_date } @@ -121,7 +133,9 @@ final class ProfileDatabase { let pull_date = Date.now persisted_profile?.network_pull_date = pull_date - self.network_pull_date_cache[id] = pull_date + self.queue.async(flags: .barrier) { + self.network_pull_date_cache[id] = pull_date + } try context.save() } @@ -146,7 +160,9 @@ final class ProfileDatabase { throw ProfileDatabaseError.missing_context } - network_pull_date_cache.removeAll() + queue.async(flags: .barrier) { + self.network_pull_date_cache.removeAll() + } let request = NSFetchRequest(entityName: entity_name) let batch_delete_request = NSBatchDeleteRequest(fetchRequest: request) From e10dc932336db7f5112fc7efab3b8dae29cb3ccd Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Fri, 26 May 2023 07:02:14 -0500 Subject: [PATCH 17/18] add damus_donation to profile data model and update tests --- .../CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents | 3 ++- damus/Nostr/CoreData/PersistedProfile.swift | 2 ++ damus/Nostr/Nostr.swift | 3 ++- damusTests/ProfileDatabaseTests.swift | 4 +++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/damus/Nostr/CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents b/damus/Nostr/CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents index 0f985f18..dbdcac1f 100644 --- a/damus/Nostr/CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents +++ b/damus/Nostr/CoreData/Damus.xcdatamodeld/Damus.xcdatamodel/contents @@ -1,8 +1,9 @@ - + + diff --git a/damus/Nostr/CoreData/PersistedProfile.swift b/damus/Nostr/CoreData/PersistedProfile.swift index 6a1b0790..460b3f4a 100644 --- a/damus/Nostr/CoreData/PersistedProfile.swift +++ b/damus/Nostr/CoreData/PersistedProfile.swift @@ -20,6 +20,7 @@ final class PersistedProfile: NSManagedObject { @NSManaged var lud06: String? @NSManaged var lud16: String? @NSManaged var nip05: String? + @NSManaged var damus_donation: Int16 @NSManaged var last_update: Date? // The date that the profile was last updated by the user @NSManaged var network_pull_date: Date? // The date we got this profile from a relay (for staleness checking) @@ -33,5 +34,6 @@ final class PersistedProfile: NSManagedObject { lud06 = profile.lud06 lud16 = profile.lud16 nip05 = profile.nip05 + damus_donation = profile.damus_donation != nil ? Int16(profile.damus_donation!) : 0 } } diff --git a/damus/Nostr/Nostr.swift b/damus/Nostr/Nostr.swift index b1cba94b..d9c10ec4 100644 --- a/damus/Nostr/Nostr.swift +++ b/damus/Nostr/Nostr.swift @@ -33,7 +33,8 @@ class Profile: Codable { website: persisted_profile.website, lud06: persisted_profile.lud06, lud16: persisted_profile.lud16, - nip05: persisted_profile.nip05) + nip05: persisted_profile.nip05, + damus_donation: Int(persisted_profile.damus_donation)) } private func str(_ str: String) -> String? { diff --git a/damusTests/ProfileDatabaseTests.swift b/damusTests/ProfileDatabaseTests.swift index 89a2c189..d07afb40 100644 --- a/damusTests/ProfileDatabaseTests.swift +++ b/damusTests/ProfileDatabaseTests.swift @@ -27,7 +27,8 @@ class ProfileDatabaseTests: XCTestCase { website: "test-website", lud06: "test-lud06", lud16: "test-lud16", - nip05: "test-nip05") + nip05: "test-nip05", + damus_donation: 100) } func testStoreAndRetrieveProfile() async throws { @@ -53,6 +54,7 @@ class ProfileDatabaseTests: XCTestCase { XCTAssertEqual(profile.lud06, retrievedProfile.lud06) XCTAssertEqual(profile.lud16, retrievedProfile.lud16) XCTAssertEqual(profile.nip05, retrievedProfile.nip05) + XCTAssertEqual(profile.damus_donation, retrievedProfile.damus_donation) } func testRejectOutdatedProfile() async throws { From 61303f49ad31f5113b98954e637ca64df0f7ee9c Mon Sep 17 00:00:00 2001 From: Bryan Montz Date: Fri, 26 May 2023 07:32:18 -0500 Subject: [PATCH 18/18] fix zap test --- damusTests/ZapTests.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/damusTests/ZapTests.swift b/damusTests/ZapTests.swift index 4199e7dd..7d4c27ea 100644 --- a/damusTests/ZapTests.swift +++ b/damusTests/ZapTests.swift @@ -11,7 +11,8 @@ import XCTest final class ZapTests: XCTestCase { override func setUpWithError() throws { - // Put setup code here. This method is called before the invocation of each test method in the class. + let db = ProfileDatabase() + try db.remove_all_profiles() } override func tearDownWithError() throws {