Merge pull request #3686 from danieldaquino/gh-3680

feat: implement LMDB compact solution for storage optimization

damus.xcodeproj/project.pbxproj

@@ -1907,6 +1907,11 @@
 		D7DF58322DFCF18D00E9AD28 /* SendPaymentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7DF58312DFCF18800E9AD28 /* SendPaymentView.swift */; };
 		D7DF58332DFCF18D00E9AD28 /* SendPaymentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7DF58312DFCF18800E9AD28 /* SendPaymentView.swift */; };
 		D7DF58342DFCF18D00E9AD28 /* SendPaymentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7DF58312DFCF18800E9AD28 /* SendPaymentView.swift */; };
+		D7E14B8E2F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E14B8D2F68B0C900BF84B7 /* Ndb+Compaction.swift */; };
+		D7E14B8F2F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E14B8D2F68B0C900BF84B7 /* Ndb+Compaction.swift */; };
+		D7E14B902F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E14B8D2F68B0C900BF84B7 /* Ndb+Compaction.swift */; };
+		D7E14B912F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E14B8D2F68B0C900BF84B7 /* Ndb+Compaction.swift */; };
+		D7E14B932F68B22D00BF84B7 /* NdbCompactionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E14B922F68B22D00BF84B7 /* NdbCompactionTests.swift */; };
 		D7E5B2D32EA0188200CF47AC /* StreamPipelineDiagnostics.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E5B2D22EA0187B00CF47AC /* StreamPipelineDiagnostics.swift */; };
 		D7E5B2D42EA0188200CF47AC /* StreamPipelineDiagnostics.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E5B2D22EA0187B00CF47AC /* StreamPipelineDiagnostics.swift */; };
 		D7E5B2D52EA0188200CF47AC /* StreamPipelineDiagnostics.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7E5B2D22EA0187B00CF47AC /* StreamPipelineDiagnostics.swift */; };
@@ -2920,6 +2925,8 @@
 		D7DB93092D69485A00DA1EE5 /* NIP65.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NIP65.swift; sourceTree = "<group>"; };
 		D7DEEF2E2A8C021E00E0C99F /* NostrEventTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NostrEventTests.swift; sourceTree = "<group>"; };
 		D7DF58312DFCF18800E9AD28 /* SendPaymentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SendPaymentView.swift; sourceTree = "<group>"; };
+		D7E14B8D2F68B0C900BF84B7 /* Ndb+Compaction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Ndb+Compaction.swift"; sourceTree = "<group>"; };
+		D7E14B922F68B22D00BF84B7 /* NdbCompactionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NdbCompactionTests.swift; sourceTree = "<group>"; };
 		D7E5B2D22EA0187B00CF47AC /* StreamPipelineDiagnostics.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StreamPipelineDiagnostics.swift; sourceTree = "<group>"; };
 		D7EB00AF2CD59C8300660C07 /* PresentFullScreenItemNotify.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PresentFullScreenItemNotify.swift; sourceTree = "<group>"; };
 		D7EBF8BA2E5901F7004EAE29 /* NostrNetworkManagerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NostrNetworkManagerTests.swift; sourceTree = "<group>"; };
@@ -3402,6 +3409,7 @@
 		4C9054862A6AEB4500811EEC /* nostrdb */ = {
 			isa = PBXGroup;
 			children = (
+				D7E14B8D2F68B0C900BF84B7 /* Ndb+Compaction.swift */,
 				D75154BE2EC5910600BF2CB2 /* NdbUseLock.swift */,
 				D74EC84E2E1856AF0091DC51 /* NonCopyableLinkedList.swift */,
 				D733F9E42D92C75C00317B11 /* UnownedNdbNote.swift */,
@@ -3918,6 +3926,7 @@
 		4CE6DEF627F7A08200C66700 /* damusTests */ = {
 			isa = PBXGroup;
 			children = (
+				D7E14B922F68B22D00BF84B7 /* NdbCompactionTests.swift */,
 				D774A5CB2F4F9679006A4D64 /* StorageStatsManagerTests.swift */,
 				D77A96BE2F3131BE00CC3246 /* RelayHintsTests.swift */,
 				D776BE432F233012002DA1C9 /* EntityPreloaderTests.swift */,
@@ -5983,6 +5992,7 @@
 				4C8D00CA29DF80350036AF10 /* TruncatedText.swift in Sources */,
 				D5C1AFC62E5DFF700092F72F /* ContactCardManagerMock.swift in Sources */,
 				4C4DD3DB2A6CA7E8005B4E85 /* ContentParsing.swift in Sources */,
+				D7E14B912F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */,
 				F71694F22A67314D001F4053 /* SuggestedUserView.swift in Sources */,
 				5C8F97172EB45FD7009399B1 /* LiveChatView.swift in Sources */,
 				4C9BB83429C12D9900FC4E37 /* EventProfileName.swift in Sources */,
@@ -6427,6 +6437,7 @@
 				501F8C822A0224EB001AFC1D /* KeychainStorageTests.swift in Sources */,
 				D72E127A2BEEEED000F4F781 /* NostrFilterTests.swift in Sources */,
 				B5B4D1432B37D47600844320 /* NdbExtensions.swift in Sources */,
+				D7E14B932F68B22D00BF84B7 /* NdbCompactionTests.swift in Sources */,
 				3ACBCB78295FE5C70037388A /* TimeAgoTests.swift in Sources */,
 				D795356B2EBD28A800AACF98 /* AppLifecycleHandlingTests.swift in Sources */,
 				D72A2D072AD9C1FB002AFF62 /* MockProfiles.swift in Sources */,
@@ -6619,6 +6630,7 @@
 				82D6FB0F2CD99F7900C925F4 /* DamusLogoGradient.swift in Sources */,
 				82D6FB102CD99F7900C925F4 /* DamusBackground.swift in Sources */,
 				82D6FB112CD99F7900C925F4 /* DamusLightGradient.swift in Sources */,
+				D7E14B902F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */,
 				5C4FA8042DCAF80E00CE658C /* FollowPackTimeline.swift in Sources */,
 				82D6FB132CD99F7900C925F4 /* Shimmer.swift in Sources */,
 				82D6FB142CD99F7900C925F4 /* EndBlock.swift in Sources */,
@@ -7377,6 +7389,7 @@
 				D73E5F152C6A97F4007EB227 /* RelayNipList.swift in Sources */,
 				D73E5F162C6A97F4007EB227 /* RelayView.swift in Sources */,
 				D73E5F172C6A97F4007EB227 /* RelayConfigView.swift in Sources */,
+				D7E14B8F2F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */,
 				D73E5F182C6A97F4007EB227 /* RelayDetailView.swift in Sources */,
 				D73E5F192C6A97F4007EB227 /* RelayToggle.swift in Sources */,
 				D73E5F1A2C6A97F4007EB227 /* RelayStatusView.swift in Sources */,
@@ -7635,6 +7648,7 @@
 				4CBB6F7A2B7311AA000477A4 /* bech32_util.c in Sources */,
 				4CBB6F712B731184000477A4 /* bolt11.c in Sources */,
 				D74EC84F2E1856B70091DC51 /* NonCopyableLinkedList.swift in Sources */,
+				D7E14B8E2F68B0D000BF84B7 /* Ndb+Compaction.swift in Sources */,
 				4CBB6F702B731179000477A4 /* invoice.c in Sources */,
 				4CBB6F6F2B73116B000477A4 /* content_parser.c in Sources */,
 				4CBB6F6E2B731113000477A4 /* block.c in Sources */,

damus/ContentView.swift

@@ -140,6 +140,18 @@ struct ContentView: View {
     // connect retry timer
     let timer = Timer.publish(every: 1, on: .main, in: .common).autoconnect()
     
+    init(keypair: Keypair, appDelegate: AppDelegate?) {
+        // Compact the database if requested from the previous session.
+        // This runs before opening the main Ndb instance so that it works on an idle database.
+        // This also gets run here instead of `connect` because we should anticipate this to add a few seconds of delay in worst case scenarios.
+        // If we were to add this in the `connect` function, parallel functions that depend on `damus_state!` could cause crashes in the app.
+        // By placing this here, we only delay the splash screen a bit
+        Ndb.compact_if_needed()
+        
+        self.keypair = keypair
+        self.appDelegate = appDelegate
+    }
+    
     func navIsAtRoot() -> Bool {
         return navigationCoordinator.isAtRoot()
     }
@@ -1168,4 +1180,5 @@ func logout(_ state: DamusState?)
 {
     state?.close()
     notify(.logout)
-}
+}
+

damus/Features/Settings/Views/StorageSettingsView.swift

@@ -18,6 +18,12 @@ fileprivate enum CacheClearingState {
     case cleared
 }
 
+/// A simple type to keep track of the compact scheduling state
+fileprivate enum CompactSchedulingState {
+    case not_scheduled
+    case scheduled
+}
+
 /// Storage category for display in list and chart
 struct StorageCategory: Identifiable {
     let id: String
@@ -46,6 +52,8 @@ struct StorageSettingsView: View {
     @State private var isPreparingExport: Bool = false
     @State fileprivate var cache_clearing_state: CacheClearingState = .not_cleared
     @State var showing_cache_clear_alert: Bool = false
+    @State fileprivate var compact_scheduling_state: CompactSchedulingState = .not_scheduled
+    @State var showing_compact_alert: Bool = false
     
     /// Storage categories with cumulative ranges for angle selection (iOS 17+)
     private var categoryRanges: [(category: String, range: Range<Double>)] {
@@ -164,6 +172,7 @@ struct StorageSettingsView: View {
                 // Clear Cache Section
                 Section {
                     self.ClearCacheButton
+                    self.CompactDatabaseButton
                 }
             }
             
@@ -215,6 +224,10 @@ struct StorageSettingsView: View {
             if stats == nil {
                 loadStorageStats()
             }
+            // Reflect any previously scheduled compaction in the button state.
+            if UserDefaults.standard.bool(forKey: Ndb.compact_on_next_launch_key) {
+                compact_scheduling_state = .scheduled
+            }
         }
         .onReceive(handle_notify(.switched_timeline)) { _ in
             dismiss()
@@ -338,6 +351,38 @@ struct StorageSettingsView: View {
                   secondaryButton: .cancel())
         }
     }
+
+    /// Compact database button view with confirmation dialog.
+    ///
+    /// Schedules a one-time database compaction to run on the next app launch.  The user
+    /// is informed that the app will need to restart to complete the operation.
+    var CompactDatabaseButton: some View {
+        Button(action: { self.showing_compact_alert = true }, label: {
+            HStack(spacing: 6) {
+                switch compact_scheduling_state {
+                    case .not_scheduled:
+                        Text("Compact Database", comment: "Button to compact the NostrDB database on next launch.")
+                    case .scheduled:
+                        Image(systemName: "checkmark.circle.fill")
+                            .foregroundColor(.green)
+                            .accessibilityHidden(true)
+                        Text("Compaction scheduled. Restart app to continue.", comment: "Message indicating that a database compaction has been scheduled for the next app launch.")
+                }
+            }
+        })
+        .disabled(self.compact_scheduling_state != .not_scheduled)
+        .alert(isPresented: $showing_compact_alert) {
+            Alert(
+                title: Text("Compact Database", comment: "Confirmation dialog title for database compaction"),
+                message: Text("This will reclaim unused space in the database. The app will need to restart to complete the operation. Proceed?", comment: "Message explaining what database compaction does and that a restart is required."),
+                primaryButton: .default(Text("OK", comment: "Button label indicating user wants to proceed.")) {
+                    Ndb.set_compact_on_next_launch()
+                    compact_scheduling_state = .scheduled
+                },
+                secondaryButton: .cancel()
+            )
+        }
+    }
 }
 
 /// Pie chart displaying storage usage distribution (iOS 17+)

damusTests/NdbCompactionTests.swift

@@ -0,0 +1,132 @@
+//
+//  NdbCompactionTests.swift
+//  damus
+//
+
+import XCTest
+@testable import damus
+
+final class NdbCompactionTests: XCTestCase {
+
+    var testDirectory: URL!
+
+    override func setUp() async throws {
+        try await super.setUp()
+        testDirectory = FileManager.default.temporaryDirectory
+            .appendingPathComponent("NdbCompactionTests-\(UUID().uuidString)")
+        try FileManager.default.createDirectory(at: testDirectory, withIntermediateDirectories: true)
+        // Ensure the flag is cleared before each test.
+        UserDefaults.standard.set(false, forKey: Ndb.compact_on_next_launch_key)
+    }
+
+    override func tearDown() async throws {
+        if let testDirectory {
+            try? FileManager.default.removeItem(at: testDirectory)
+        }
+        // Leave the flag cleared after each test.
+        UserDefaults.standard.set(false, forKey: Ndb.compact_on_next_launch_key)
+        try await super.tearDown()
+    }
+
+    // MARK: - set_compact_on_next_launch
+
+    func testSetCompactOnNextLaunch_setsUserDefaultsFlag() {
+        // Given: the flag is not set
+        XCTAssertFalse(UserDefaults.standard.bool(forKey: Ndb.compact_on_next_launch_key))
+
+        // When
+        Ndb.set_compact_on_next_launch()
+
+        // Then
+        XCTAssertTrue(
+            UserDefaults.standard.bool(forKey: Ndb.compact_on_next_launch_key),
+            "set_compact_on_next_launch() should set the UserDefaults flag to true"
+        )
+    }
+
+    // MARK: - compact_if_needed: flag not set
+
+    func testCompactIfNeeded_doesNothingWhenFlagNotSet() {
+        // Given: the flag is false (set in setUp)
+        let dbPath = testDirectory.path
+
+        // When
+        Ndb.compact_if_needed(db_path: dbPath)
+
+        // Then: no temp directory was created, no files were touched
+        let tempPath = "\(dbPath)/ndb_compact_temp"
+        XCTAssertFalse(
+            FileManager.default.fileExists(atPath: tempPath),
+            "compact_if_needed should not create a temp directory when the flag is not set"
+        )
+        XCTAssertFalse(
+            UserDefaults.standard.bool(forKey: Ndb.compact_on_next_launch_key),
+            "Flag should remain false when compact_if_needed does nothing"
+        )
+    }
+
+    // MARK: - compact_if_needed: no database present
+
+    func testCompactIfNeeded_clearsFlagWhenNoDatabaseExists() {
+        // Given: flag is set, but no data.mdb exists in the directory
+        Ndb.set_compact_on_next_launch()
+        let emptyPath = testDirectory.appendingPathComponent("empty_db").path
+        try? FileManager.default.createDirectory(atPath: emptyPath, withIntermediateDirectories: true)
+
+        // When
+        Ndb.compact_if_needed(db_path: emptyPath)
+
+        // Then: flag is cleared, no temp directory left over
+        XCTAssertFalse(
+            UserDefaults.standard.bool(forKey: Ndb.compact_on_next_launch_key),
+            "compact_if_needed should clear the flag even when no database file exists"
+        )
+        let tempPath = "\(emptyPath)/ndb_compact_temp"
+        XCTAssertFalse(
+            FileManager.default.fileExists(atPath: tempPath),
+            "No temp directory should remain after compact_if_needed with no DB"
+        )
+    }
+
+    // MARK: - compact_if_needed: full round-trip
+
+    func testCompactIfNeeded_compactsAndClearsFlagWithRealDatabase() {
+        // Given: a real Ndb database and the compact flag is set
+        let dbPath = testDirectory.appendingPathComponent("real_db").path
+        try? FileManager.default.createDirectory(atPath: dbPath, withIntermediateDirectories: true)
+
+        guard let ndb = Ndb(path: dbPath) else {
+            XCTFail("Could not open Ndb at \(dbPath)")
+            return
+        }
+        ndb.close()
+
+        let originalSize = (try? FileManager.default.attributesOfItem(atPath: "\(dbPath)/data.mdb")[.size] as? Int) ?? 0
+        XCTAssertGreaterThan(originalSize, 0, "A freshly created database should have a non-zero size")
+
+        Ndb.set_compact_on_next_launch()
+
+        // When
+        Ndb.compact_if_needed(db_path: dbPath)
+
+        // Then: the flag is cleared
+        XCTAssertFalse(
+            UserDefaults.standard.bool(forKey: Ndb.compact_on_next_launch_key),
+            "compact_if_needed should clear the flag after a successful compaction"
+        )
+
+        // The compacted database must be a valid, openable Ndb
+        guard let compactedNdb = Ndb(path: dbPath) else {
+            XCTFail("Could not open Ndb after compaction — database may be corrupt")
+            return
+        }
+        compactedNdb.close()
+
+        // No leftover temp directory
+        let tempPath = "\(dbPath)/ndb_compact_temp"
+        XCTAssertFalse(
+            FileManager.default.fileExists(atPath: tempPath),
+            "Temp directory should be cleaned up after successful compaction"
+        )
+    }
+}

nostrdb/Ndb+Compaction.swift

@@ -0,0 +1,132 @@
+//
+//  Ndb+Compaction.swift
+//  damus
+//
+
+import Foundation
+
+extension Ndb {
+    /// Makes a compacted copy of the database in a separate directory.
+    ///
+    /// This uses `mdb_env_copy2` with `MDB_CP_COMPACT` (flag = 0x01), which omits free pages
+    /// and sequentially renumbers all pages in the output, reducing database file size.
+    /// - Parameter path: The directory path where the compacted database will be written.
+    ///                   The directory must already exist and be empty.
+    func compact(to path: String) throws {
+        enum CompactError: Error {
+            case mdbOperationError(errno: Int32)
+        }
+
+        try withNdb({
+            try path.withCString({ pathCString in
+                let rc = ndb_snapshot(self.ndb.ndb, pathCString, Self.MDB_CP_COMPACT)
+                guard rc == 0 else {
+                    throw CompactError.mdbOperationError(errno: rc)
+                }
+            })
+        })
+    }
+
+    /// LMDB compact-copy flag.  Passed to `ndb_snapshot` / `mdb_env_copy2` to produce a
+    /// compacted (free-page-omitting) database copy.  Mirrors `MDB_CP_COMPACT = 0x01` from lmdb.h.
+    private static let MDB_CP_COMPACT: UInt32 = 1
+
+    /// Name of the temporary subdirectory created during an in-place compaction.
+    private static let compactTempDirName = "ndb_compact_temp"
+
+    /// The `UserDefaults` key used to signal that the database should be compacted on the next app launch.
+    static let compact_on_next_launch_key = "ndb_compact_on_next_launch"
+
+    /// Requests that the database be compacted the next time the app launches.
+    ///
+    /// Call this to schedule a one-time compaction. The flag is cleared automatically after
+    /// a successful compaction in `compact_if_needed()`.
+    static func set_compact_on_next_launch() {
+        UserDefaults.standard.set(true, forKey: compact_on_next_launch_key)
+    }
+
+    /// Compacts the NostrDB database files if the compact-on-next-launch flag is set.
+    ///
+    /// This is intended to be called once during app startup **before** `Ndb` is opened for
+    /// normal use.  The algorithm is:
+    ///   1. Open a temporary `Ndb` instance at the same path to access the LMDB environment.
+    ///   2. Write a compacted copy of the database to a sibling temp directory.
+    ///   3. Close the temporary `Ndb` instance.
+    ///   4. Atomically replace the original `data.mdb` with the compacted copy.
+    ///   5. Remove the temp directory.
+    ///   6. Clear the flag so compaction does not run again on the following launch.
+    ///
+    /// - Parameter db_path: Override the database directory path.  Pass `nil` (default) to use
+    ///   `Ndb.db_path`.  Mainly useful for testing.
+    static func compact_if_needed(db_path: String? = nil) {
+        guard UserDefaults.standard.bool(forKey: compact_on_next_launch_key) else { return }
+
+        guard let path = db_path ?? Self.db_path else {
+            Log.error("compact_if_needed: could not determine db path", for: .storage)
+            return
+        }
+
+        guard db_file_exists(path: path) else {
+            // No database file present yet; nothing to compact — just clear the flag.
+            UserDefaults.standard.set(false, forKey: compact_on_next_launch_key)
+            return
+        }
+
+        Log.info("Compacting NostrDB on startup…", for: .storage)
+
+        let tempPath = "\(path)/\(compactTempDirName)"
+
+        // Clean up any leftover temp directory from a previously failed attempt.
+        try? FileManager.default.removeItem(atPath: tempPath)
+
+        do {
+            try FileManager.default.createDirectory(atPath: tempPath, withIntermediateDirectories: true)
+        } catch {
+            Log.error("compact_if_needed: failed to create temp dir: %@", for: .storage, String(describing: error))
+            return
+        }
+
+        // Open a temporary Ndb instance just to drive the compaction.
+        guard let tempNdb = Ndb(path: path) else {
+            Log.error("compact_if_needed: failed to open ndb for compaction", for: .storage)
+            try? FileManager.default.removeItem(atPath: tempPath)
+            return
+        }
+        // Ensure the temporary Ndb is closed regardless of how this function exits.
+        defer { tempNdb.close() }
+
+        do {
+            try tempNdb.compact(to: tempPath)
+        } catch {
+            Log.error("compact_if_needed: compaction failed: %@", for: .storage, String(describing: error))
+            try? FileManager.default.removeItem(atPath: tempPath)
+            return
+        }
+        
+        tempNdb.close()
+
+        // Atomically replace the original data.mdb with the compacted copy.
+        let originalDataMdb = URL(fileURLWithPath: "\(path)/\(main_db_file_name)")
+        let compactedDataMdb = URL(fileURLWithPath: "\(tempPath)/\(main_db_file_name)")
+
+        do {
+            _ = try FileManager.default.replaceItemAt(
+                originalDataMdb,
+                withItemAt: compactedDataMdb,
+                backupItemName: nil,
+                options: [.usingNewMetadataOnly]
+            )
+            Log.info("NostrDB compacted successfully", for: .storage)
+        } catch {
+            Log.error("compact_if_needed: failed to replace db file: %@", for: .storage, String(describing: error))
+            try? FileManager.default.removeItem(atPath: tempPath)
+            return
+        }
+
+        // Clean up the temp directory (any remaining files such as lock.mdb).
+        try? FileManager.default.removeItem(atPath: tempPath)
+
+        // Clear the flag so we don't compact again on the next launch.
+        UserDefaults.standard.set(false, forKey: compact_on_next_launch_key)
+    }
+}

nostrdb/Ndb.swift

@@ -241,7 +241,7 @@ class Ndb {
         }
     }
     
-    private static func db_file_exists(path: String) -> Bool {
+    static func db_file_exists(path: String) -> Bool {
         return FileManager.default.fileExists(atPath: "\(path)/\(Self.main_db_file_name)")
     }
     
@@ -324,7 +324,7 @@ class Ndb {
             })
         })
     }
-    
+
     // MARK: Thread safety mechanisms
     // Use these for all externally accessible methods that interact with the nostrdb database to prevent race conditions with app lifecycle events (i.e. NostrDB opening and closing)