diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj index 7129ae7e..cfbe371f 100644 --- a/damus.xcodeproj/project.pbxproj +++ b/damus.xcodeproj/project.pbxproj @@ -1907,6 +1907,11 @@ D7DF58322DFCF18D00E9AD28 /* SendPaymentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7DF58312DFCF18800E9AD28 /* SendPaymentView.swift */; }; D7DF58332DFCF18D00E9AD28 /* SendPaymentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7DF58312DFCF18800E9AD28 /* SendPaymentView.swift */; }; D7DF58342DFCF18D00E9AD28 /* SendPaymentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7DF58312DFCF18800E9AD28 /* SendPaymentView.swift */; }; + D7E14B8E2F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E14B8D2F68B0C900BF84B7 /* Ndb+Compaction.swift */; }; + D7E14B8F2F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E14B8D2F68B0C900BF84B7 /* Ndb+Compaction.swift */; }; + D7E14B902F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E14B8D2F68B0C900BF84B7 /* Ndb+Compaction.swift */; }; + D7E14B912F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E14B8D2F68B0C900BF84B7 /* Ndb+Compaction.swift */; }; + D7E14B932F68B22D00BF84B7 /* NdbCompactionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E14B922F68B22D00BF84B7 /* NdbCompactionTests.swift */; }; D7E5B2D32EA0188200CF47AC /* StreamPipelineDiagnostics.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E5B2D22EA0187B00CF47AC /* StreamPipelineDiagnostics.swift */; }; D7E5B2D42EA0188200CF47AC /* StreamPipelineDiagnostics.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E5B2D22EA0187B00CF47AC /* StreamPipelineDiagnostics.swift */; }; D7E5B2D52EA0188200CF47AC /* StreamPipelineDiagnostics.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E5B2D22EA0187B00CF47AC /* StreamPipelineDiagnostics.swift */; }; @@ -2920,6 +2925,8 @@ D7DB93092D69485A00DA1EE5 /* NIP65.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NIP65.swift; sourceTree = ""; }; D7DEEF2E2A8C021E00E0C99F /* NostrEventTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NostrEventTests.swift; sourceTree = ""; }; D7DF58312DFCF18800E9AD28 /* SendPaymentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SendPaymentView.swift; sourceTree = ""; }; + D7E14B8D2F68B0C900BF84B7 /* Ndb+Compaction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Ndb+Compaction.swift"; sourceTree = ""; }; + D7E14B922F68B22D00BF84B7 /* NdbCompactionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NdbCompactionTests.swift; sourceTree = ""; }; D7E5B2D22EA0187B00CF47AC /* StreamPipelineDiagnostics.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StreamPipelineDiagnostics.swift; sourceTree = ""; }; D7EB00AF2CD59C8300660C07 /* PresentFullScreenItemNotify.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PresentFullScreenItemNotify.swift; sourceTree = ""; }; D7EBF8BA2E5901F7004EAE29 /* NostrNetworkManagerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NostrNetworkManagerTests.swift; sourceTree = ""; }; @@ -3402,6 +3409,7 @@ 4C9054862A6AEB4500811EEC /* nostrdb */ = { isa = PBXGroup; children = ( + D7E14B8D2F68B0C900BF84B7 /* Ndb+Compaction.swift */, D75154BE2EC5910600BF2CB2 /* NdbUseLock.swift */, D74EC84E2E1856AF0091DC51 /* NonCopyableLinkedList.swift */, D733F9E42D92C75C00317B11 /* UnownedNdbNote.swift */, @@ -3918,6 +3926,7 @@ 4CE6DEF627F7A08200C66700 /* damusTests */ = { isa = PBXGroup; children = ( + D7E14B922F68B22D00BF84B7 /* NdbCompactionTests.swift */, D774A5CB2F4F9679006A4D64 /* StorageStatsManagerTests.swift */, D77A96BE2F3131BE00CC3246 /* RelayHintsTests.swift */, D776BE432F233012002DA1C9 /* EntityPreloaderTests.swift */, @@ -5983,6 +5992,7 @@ 4C8D00CA29DF80350036AF10 /* TruncatedText.swift in Sources */, D5C1AFC62E5DFF700092F72F /* ContactCardManagerMock.swift in Sources */, 4C4DD3DB2A6CA7E8005B4E85 /* ContentParsing.swift in Sources */, + D7E14B912F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */, F71694F22A67314D001F4053 /* SuggestedUserView.swift in Sources */, 5C8F97172EB45FD7009399B1 /* LiveChatView.swift in Sources */, 4C9BB83429C12D9900FC4E37 /* EventProfileName.swift in Sources */, @@ -6427,6 +6437,7 @@ 501F8C822A0224EB001AFC1D /* KeychainStorageTests.swift in Sources */, D72E127A2BEEEED000F4F781 /* NostrFilterTests.swift in Sources */, B5B4D1432B37D47600844320 /* NdbExtensions.swift in Sources */, + D7E14B932F68B22D00BF84B7 /* NdbCompactionTests.swift in Sources */, 3ACBCB78295FE5C70037388A /* TimeAgoTests.swift in Sources */, D795356B2EBD28A800AACF98 /* AppLifecycleHandlingTests.swift in Sources */, D72A2D072AD9C1FB002AFF62 /* MockProfiles.swift in Sources */, @@ -6619,6 +6630,7 @@ 82D6FB0F2CD99F7900C925F4 /* DamusLogoGradient.swift in Sources */, 82D6FB102CD99F7900C925F4 /* DamusBackground.swift in Sources */, 82D6FB112CD99F7900C925F4 /* DamusLightGradient.swift in Sources */, + D7E14B902F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */, 5C4FA8042DCAF80E00CE658C /* FollowPackTimeline.swift in Sources */, 82D6FB132CD99F7900C925F4 /* Shimmer.swift in Sources */, 82D6FB142CD99F7900C925F4 /* EndBlock.swift in Sources */, @@ -7377,6 +7389,7 @@ D73E5F152C6A97F4007EB227 /* RelayNipList.swift in Sources */, D73E5F162C6A97F4007EB227 /* RelayView.swift in Sources */, D73E5F172C6A97F4007EB227 /* RelayConfigView.swift in Sources */, + D7E14B8F2F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */, D73E5F182C6A97F4007EB227 /* RelayDetailView.swift in Sources */, D73E5F192C6A97F4007EB227 /* RelayToggle.swift in Sources */, D73E5F1A2C6A97F4007EB227 /* RelayStatusView.swift in Sources */, @@ -7635,6 +7648,7 @@ 4CBB6F7A2B7311AA000477A4 /* bech32_util.c in Sources */, 4CBB6F712B731184000477A4 /* bolt11.c in Sources */, D74EC84F2E1856B70091DC51 /* NonCopyableLinkedList.swift in Sources */, + D7E14B8E2F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */, 4CBB6F702B731179000477A4 /* invoice.c in Sources */, 4CBB6F6F2B73116B000477A4 /* content_parser.c in Sources */, 4CBB6F6E2B731113000477A4 /* block.c in Sources */, diff --git a/damus/ContentView.swift b/damus/ContentView.swift index e27f4bd1..af9d2677 100644 --- a/damus/ContentView.swift +++ b/damus/ContentView.swift @@ -140,6 +140,18 @@ struct ContentView: View { // connect retry timer let timer = Timer.publish(every: 1, on: .main, in: .common).autoconnect() + init(keypair: Keypair, appDelegate: AppDelegate?) { + // Compact the database if requested from the previous session. + // This runs before opening the main Ndb instance so that it works on an idle database. + // This also gets run here instead of `connect` because we should anticipate this to add a few seconds of delay in worst case scenarios. + // If we were to add this in the `connect` function, parallel functions that depend on `damus_state!` could cause crashes in the app. + // By placing this here, we only delay the splash screen a bit + Ndb.compact_if_needed() + + self.keypair = keypair + self.appDelegate = appDelegate + } + func navIsAtRoot() -> Bool { return navigationCoordinator.isAtRoot() } @@ -1168,4 +1180,5 @@ func logout(_ state: DamusState?) { state?.close() notify(.logout) -} \ No newline at end of file +} + diff --git a/damus/Features/Settings/Views/StorageSettingsView.swift b/damus/Features/Settings/Views/StorageSettingsView.swift index c2a03afc..c9c4697d 100644 --- a/damus/Features/Settings/Views/StorageSettingsView.swift +++ b/damus/Features/Settings/Views/StorageSettingsView.swift @@ -18,6 +18,12 @@ fileprivate enum CacheClearingState { case cleared } +/// A simple type to keep track of the compact scheduling state +fileprivate enum CompactSchedulingState { + case not_scheduled + case scheduled +} + /// Storage category for display in list and chart struct StorageCategory: Identifiable { let id: String @@ -46,6 +52,8 @@ struct StorageSettingsView: View { @State private var isPreparingExport: Bool = false @State fileprivate var cache_clearing_state: CacheClearingState = .not_cleared @State var showing_cache_clear_alert: Bool = false + @State fileprivate var compact_scheduling_state: CompactSchedulingState = .not_scheduled + @State var showing_compact_alert: Bool = false /// Storage categories with cumulative ranges for angle selection (iOS 17+) private var categoryRanges: [(category: String, range: Range)] { @@ -164,6 +172,7 @@ struct StorageSettingsView: View { // Clear Cache Section Section { self.ClearCacheButton + self.CompactDatabaseButton } } @@ -215,6 +224,10 @@ struct StorageSettingsView: View { if stats == nil { loadStorageStats() } + // Reflect any previously scheduled compaction in the button state. + if UserDefaults.standard.bool(forKey: Ndb.compact_on_next_launch_key) { + compact_scheduling_state = .scheduled + } } .onReceive(handle_notify(.switched_timeline)) { _ in dismiss() @@ -338,6 +351,38 @@ struct StorageSettingsView: View { secondaryButton: .cancel()) } } + + /// Compact database button view with confirmation dialog. + /// + /// Schedules a one-time database compaction to run on the next app launch. The user + /// is informed that the app will need to restart to complete the operation. + var CompactDatabaseButton: some View { + Button(action: { self.showing_compact_alert = true }, label: { + HStack(spacing: 6) { + switch compact_scheduling_state { + case .not_scheduled: + Text("Compact Database", comment: "Button to compact the NostrDB database on next launch.") + case .scheduled: + Image(systemName: "checkmark.circle.fill") + .foregroundColor(.green) + .accessibilityHidden(true) + Text("Compaction scheduled. Restart app to continue.", comment: "Message indicating that a database compaction has been scheduled for the next app launch.") + } + } + }) + .disabled(self.compact_scheduling_state != .not_scheduled) + .alert(isPresented: $showing_compact_alert) { + Alert( + title: Text("Compact Database", comment: "Confirmation dialog title for database compaction"), + message: Text("This will reclaim unused space in the database. The app will need to restart to complete the operation. Proceed?", comment: "Message explaining what database compaction does and that a restart is required."), + primaryButton: .default(Text("OK", comment: "Button label indicating user wants to proceed.")) { + Ndb.set_compact_on_next_launch() + compact_scheduling_state = .scheduled + }, + secondaryButton: .cancel() + ) + } + } } /// Pie chart displaying storage usage distribution (iOS 17+) diff --git a/damusTests/NdbCompactionTests.swift b/damusTests/NdbCompactionTests.swift new file mode 100644 index 00000000..3a266b12 --- /dev/null +++ b/damusTests/NdbCompactionTests.swift @@ -0,0 +1,132 @@ +// +// NdbCompactionTests.swift +// damus +// + +import XCTest +@testable import damus + +final class NdbCompactionTests: XCTestCase { + + var testDirectory: URL! + + override func setUp() async throws { + try await super.setUp() + testDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent("NdbCompactionTests-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: testDirectory, withIntermediateDirectories: true) + // Ensure the flag is cleared before each test. + UserDefaults.standard.set(false, forKey: Ndb.compact_on_next_launch_key) + } + + override func tearDown() async throws { + if let testDirectory { + try? FileManager.default.removeItem(at: testDirectory) + } + // Leave the flag cleared after each test. + UserDefaults.standard.set(false, forKey: Ndb.compact_on_next_launch_key) + try await super.tearDown() + } + + // MARK: - set_compact_on_next_launch + + func testSetCompactOnNextLaunch_setsUserDefaultsFlag() { + // Given: the flag is not set + XCTAssertFalse(UserDefaults.standard.bool(forKey: Ndb.compact_on_next_launch_key)) + + // When + Ndb.set_compact_on_next_launch() + + // Then + XCTAssertTrue( + UserDefaults.standard.bool(forKey: Ndb.compact_on_next_launch_key), + "set_compact_on_next_launch() should set the UserDefaults flag to true" + ) + } + + // MARK: - compact_if_needed: flag not set + + func testCompactIfNeeded_doesNothingWhenFlagNotSet() { + // Given: the flag is false (set in setUp) + let dbPath = testDirectory.path + + // When + Ndb.compact_if_needed(db_path: dbPath) + + // Then: no temp directory was created, no files were touched + let tempPath = "\(dbPath)/ndb_compact_temp" + XCTAssertFalse( + FileManager.default.fileExists(atPath: tempPath), + "compact_if_needed should not create a temp directory when the flag is not set" + ) + XCTAssertFalse( + UserDefaults.standard.bool(forKey: Ndb.compact_on_next_launch_key), + "Flag should remain false when compact_if_needed does nothing" + ) + } + + // MARK: - compact_if_needed: no database present + + func testCompactIfNeeded_clearsFlagWhenNoDatabaseExists() { + // Given: flag is set, but no data.mdb exists in the directory + Ndb.set_compact_on_next_launch() + let emptyPath = testDirectory.appendingPathComponent("empty_db").path + try? FileManager.default.createDirectory(atPath: emptyPath, withIntermediateDirectories: true) + + // When + Ndb.compact_if_needed(db_path: emptyPath) + + // Then: flag is cleared, no temp directory left over + XCTAssertFalse( + UserDefaults.standard.bool(forKey: Ndb.compact_on_next_launch_key), + "compact_if_needed should clear the flag even when no database file exists" + ) + let tempPath = "\(emptyPath)/ndb_compact_temp" + XCTAssertFalse( + FileManager.default.fileExists(atPath: tempPath), + "No temp directory should remain after compact_if_needed with no DB" + ) + } + + // MARK: - compact_if_needed: full round-trip + + func testCompactIfNeeded_compactsAndClearsFlagWithRealDatabase() { + // Given: a real Ndb database and the compact flag is set + let dbPath = testDirectory.appendingPathComponent("real_db").path + try? FileManager.default.createDirectory(atPath: dbPath, withIntermediateDirectories: true) + + guard let ndb = Ndb(path: dbPath) else { + XCTFail("Could not open Ndb at \(dbPath)") + return + } + ndb.close() + + let originalSize = (try? FileManager.default.attributesOfItem(atPath: "\(dbPath)/data.mdb")[.size] as? Int) ?? 0 + XCTAssertGreaterThan(originalSize, 0, "A freshly created database should have a non-zero size") + + Ndb.set_compact_on_next_launch() + + // When + Ndb.compact_if_needed(db_path: dbPath) + + // Then: the flag is cleared + XCTAssertFalse( + UserDefaults.standard.bool(forKey: Ndb.compact_on_next_launch_key), + "compact_if_needed should clear the flag after a successful compaction" + ) + + // The compacted database must be a valid, openable Ndb + guard let compactedNdb = Ndb(path: dbPath) else { + XCTFail("Could not open Ndb after compaction — database may be corrupt") + return + } + compactedNdb.close() + + // No leftover temp directory + let tempPath = "\(dbPath)/ndb_compact_temp" + XCTAssertFalse( + FileManager.default.fileExists(atPath: tempPath), + "Temp directory should be cleaned up after successful compaction" + ) + } +} diff --git a/nostrdb/Ndb+Compaction.swift b/nostrdb/Ndb+Compaction.swift new file mode 100644 index 00000000..9e7bfb40 --- /dev/null +++ b/nostrdb/Ndb+Compaction.swift @@ -0,0 +1,132 @@ +// +// Ndb+Compaction.swift +// damus +// + +import Foundation + +extension Ndb { + /// Makes a compacted copy of the database in a separate directory. + /// + /// This uses `mdb_env_copy2` with `MDB_CP_COMPACT` (flag = 0x01), which omits free pages + /// and sequentially renumbers all pages in the output, reducing database file size. + /// - Parameter path: The directory path where the compacted database will be written. + /// The directory must already exist and be empty. + func compact(to path: String) throws { + enum CompactError: Error { + case mdbOperationError(errno: Int32) + } + + try withNdb({ + try path.withCString({ pathCString in + let rc = ndb_snapshot(self.ndb.ndb, pathCString, Self.MDB_CP_COMPACT) + guard rc == 0 else { + throw CompactError.mdbOperationError(errno: rc) + } + }) + }) + } + + /// LMDB compact-copy flag. Passed to `ndb_snapshot` / `mdb_env_copy2` to produce a + /// compacted (free-page-omitting) database copy. Mirrors `MDB_CP_COMPACT = 0x01` from lmdb.h. + private static let MDB_CP_COMPACT: UInt32 = 1 + + /// Name of the temporary subdirectory created during an in-place compaction. + private static let compactTempDirName = "ndb_compact_temp" + + /// The `UserDefaults` key used to signal that the database should be compacted on the next app launch. + static let compact_on_next_launch_key = "ndb_compact_on_next_launch" + + /// Requests that the database be compacted the next time the app launches. + /// + /// Call this to schedule a one-time compaction. The flag is cleared automatically after + /// a successful compaction in `compact_if_needed()`. + static func set_compact_on_next_launch() { + UserDefaults.standard.set(true, forKey: compact_on_next_launch_key) + } + + /// Compacts the NostrDB database files if the compact-on-next-launch flag is set. + /// + /// This is intended to be called once during app startup **before** `Ndb` is opened for + /// normal use. The algorithm is: + /// 1. Open a temporary `Ndb` instance at the same path to access the LMDB environment. + /// 2. Write a compacted copy of the database to a sibling temp directory. + /// 3. Close the temporary `Ndb` instance. + /// 4. Atomically replace the original `data.mdb` with the compacted copy. + /// 5. Remove the temp directory. + /// 6. Clear the flag so compaction does not run again on the following launch. + /// + /// - Parameter db_path: Override the database directory path. Pass `nil` (default) to use + /// `Ndb.db_path`. Mainly useful for testing. + static func compact_if_needed(db_path: String? = nil) { + guard UserDefaults.standard.bool(forKey: compact_on_next_launch_key) else { return } + + guard let path = db_path ?? Self.db_path else { + Log.error("compact_if_needed: could not determine db path", for: .storage) + return + } + + guard db_file_exists(path: path) else { + // No database file present yet; nothing to compact — just clear the flag. + UserDefaults.standard.set(false, forKey: compact_on_next_launch_key) + return + } + + Log.info("Compacting NostrDB on startup…", for: .storage) + + let tempPath = "\(path)/\(compactTempDirName)" + + // Clean up any leftover temp directory from a previously failed attempt. + try? FileManager.default.removeItem(atPath: tempPath) + + do { + try FileManager.default.createDirectory(atPath: tempPath, withIntermediateDirectories: true) + } catch { + Log.error("compact_if_needed: failed to create temp dir: %@", for: .storage, String(describing: error)) + return + } + + // Open a temporary Ndb instance just to drive the compaction. + guard let tempNdb = Ndb(path: path) else { + Log.error("compact_if_needed: failed to open ndb for compaction", for: .storage) + try? FileManager.default.removeItem(atPath: tempPath) + return + } + // Ensure the temporary Ndb is closed regardless of how this function exits. + defer { tempNdb.close() } + + do { + try tempNdb.compact(to: tempPath) + } catch { + Log.error("compact_if_needed: compaction failed: %@", for: .storage, String(describing: error)) + try? FileManager.default.removeItem(atPath: tempPath) + return + } + + tempNdb.close() + + // Atomically replace the original data.mdb with the compacted copy. + let originalDataMdb = URL(fileURLWithPath: "\(path)/\(main_db_file_name)") + let compactedDataMdb = URL(fileURLWithPath: "\(tempPath)/\(main_db_file_name)") + + do { + _ = try FileManager.default.replaceItemAt( + originalDataMdb, + withItemAt: compactedDataMdb, + backupItemName: nil, + options: [.usingNewMetadataOnly] + ) + Log.info("NostrDB compacted successfully", for: .storage) + } catch { + Log.error("compact_if_needed: failed to replace db file: %@", for: .storage, String(describing: error)) + try? FileManager.default.removeItem(atPath: tempPath) + return + } + + // Clean up the temp directory (any remaining files such as lock.mdb). + try? FileManager.default.removeItem(atPath: tempPath) + + // Clear the flag so we don't compact again on the next launch. + UserDefaults.standard.set(false, forKey: compact_on_next_launch_key) + } +} diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift index cd5f920b..8759513e 100644 --- a/nostrdb/Ndb.swift +++ b/nostrdb/Ndb.swift @@ -241,7 +241,7 @@ class Ndb { } } - private static func db_file_exists(path: String) -> Bool { + static func db_file_exists(path: String) -> Bool { return FileManager.default.fileExists(atPath: "\(path)/\(Self.main_db_file_name)") } @@ -324,7 +324,7 @@ class Ndb { }) }) } - + // MARK: Thread safety mechanisms // Use these for all externally accessible methods that interact with the nostrdb database to prevent race conditions with app lifecycle events (i.e. NostrDB opening and closing)