Add support for multilingual hashtags

Changelog-Added: Add support for multilingual hashtags
Reviewed-by: William Casarin <jb55@jb55.com>
Closes: https://github.com/damus-io/damus/issues/949

damus-c/cursor.h

@@ -432,7 +432,7 @@ static inline int is_whitespace(char c) {
 }
 
 static inline int is_boundary(char c) {
-    return !isalnum(c);
+    return is_whitespace(c) || ispunct(c);
 }
 
 static inline int is_invalid_url_ending(char c) {

damus.xcodeproj/project.pbxproj

@@ -53,6 +53,7 @@
 		4C198DF229F88C6B004C165C /* BlurHashDecode.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C198DEE29F88C6B004C165C /* BlurHashDecode.swift */; };
 		4C198DF529F88D2E004C165C /* ImageMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C198DF429F88D2E004C165C /* ImageMetadata.swift */; };
 		4C19AE512A5CEF7C00C90DB7 /* NostrScript.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C19AE4C2A5CEF7C00C90DB7 /* NostrScript.swift */; };
+		4C19AE552A5D977400C90DB7 /* HashtagTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C19AE542A5D977400C90DB7 /* HashtagTests.swift */; };
 		4C1A9A1A29DCA17E00516EAC /* ReplyCounter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C1A9A1929DCA17E00516EAC /* ReplyCounter.swift */; };
 		4C1A9A1D29DDCF9B00516EAC /* NotificationSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C1A9A1C29DDCF9B00516EAC /* NotificationSettingsView.swift */; };
 		4C1A9A1F29DDD24B00516EAC /* AppearanceSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C1A9A1E29DDD24B00516EAC /* AppearanceSettingsView.swift */; };
@@ -495,6 +496,7 @@
 		4C19AE4B2A5CEF7C00C90DB7 /* primal.ts */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.typescript; path = primal.ts; sourceTree = "<group>"; };
 		4C19AE4C2A5CEF7C00C90DB7 /* NostrScript.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NostrScript.swift; sourceTree = "<group>"; };
 		4C19AE502A5CEF7C00C90DB7 /* nostr.ts */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.typescript; path = nostr.ts; sourceTree = "<group>"; };
+		4C19AE542A5D977400C90DB7 /* HashtagTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HashtagTests.swift; sourceTree = "<group>"; };
 		4C1A9A1929DCA17E00516EAC /* ReplyCounter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ReplyCounter.swift; sourceTree = "<group>"; };
 		4C1A9A1C29DDCF9B00516EAC /* NotificationSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NotificationSettingsView.swift; sourceTree = "<group>"; };
 		4C1A9A1E29DDD24B00516EAC /* AppearanceSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppearanceSettingsView.swift; sourceTree = "<group>"; };
@@ -1510,6 +1512,7 @@
 				3A5E47C62A4A76C800C0D090 /* TrieTests.swift */,
 				3A90B1822A4EA3C600000D94 /* UserSearchCacheTests.swift */,
 				4C4F14A62A2A61A30045A0B9 /* NostrScriptTests.swift */,
+				4C19AE542A5D977400C90DB7 /* HashtagTests.swift */,
 			);
 			path = damusTests;
 			sourceTree = "<group>";
@@ -2114,6 +2117,7 @@
 			files = (
 				5019CADD2A0FB0A9000069E1 /* ProfileDatabaseTests.swift in Sources */,
 				3A90B1832A4EA3C600000D94 /* UserSearchCacheTests.swift in Sources */,
+				4C19AE552A5D977400C90DB7 /* HashtagTests.swift in Sources */,
 				3A3040ED29A5CB86008A0F29 /* ReplyDescriptionTests.swift in Sources */,
 				4C8D00D429E3C5D40036AF10 /* NIP19Tests.swift in Sources */,
 				3A30410129AB12AA008A0F29 /* EventGroupViewTests.swift in Sources */,
@@ -2405,7 +2409,7 @@
 				CLANG_ENABLE_MODULES = YES;
 				CODE_SIGN_ENTITLEMENTS = damus/damus.entitlements;
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 1;
+				CURRENT_PROJECT_VERSION = 2;
 				DEVELOPMENT_ASSET_PATHS = "\"damus/Preview Content\"";
 				DEVELOPMENT_TEAM = XK7H4JAB3D;
 				ENABLE_PREVIEWS = YES;
@@ -2454,7 +2458,7 @@
 				CLANG_ENABLE_MODULES = YES;
 				CODE_SIGN_ENTITLEMENTS = damus/damus.entitlements;
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 1;
+				CURRENT_PROJECT_VERSION = 2;
 				DEVELOPMENT_ASSET_PATHS = "\"damus/Preview Content\"";
 				DEVELOPMENT_TEAM = XK7H4JAB3D;
 				ENABLE_PREVIEWS = YES;

damus/Models/Mentions.swift

@@ -415,7 +415,7 @@ func parse_while(_ p: Parser, match: (Character) -> Bool) -> String? {
 }
 
 func is_hashtag_char(_ c: Character) -> Bool {
-    return c.isLetter || c.isNumber
+    return (c.isLetter || c.isNumber || c.isASCII) && (!c.isPunctuation && !c.isWhitespace)
 }
 
 func prev_char(_ p: Parser, n: Int) -> Character? {

damus/Nostr/NostrLink.swift

@@ -116,9 +116,13 @@ func decode_nostr_uri(_ s: String) -> NostrLink? {
         return decode_universal_link(s)
     }
 
-    var uri = s.replacingOccurrences(of: "nostr://", with: "")
+    var uri = s
+    uri = uri.replacingOccurrences(of: "nostr://", with: "")
     uri = uri.replacingOccurrences(of: "nostr:", with: "")
 
+    // Fix for non-latin characters resulting in second colon being encoded
+    uri = uri.replacingOccurrences(of: "damus:t%3A", with: "t:")
+    
     uri = uri.replacingOccurrences(of: "damus://", with: "")
     uri = uri.replacingOccurrences(of: "damus:", with: "")
     

damusTests/HashtagTests.swift

@@ -0,0 +1,79 @@
+//
+//  HashtagTests.swift
+//  damusTests
+//
+//  Created by William Casarin on 2023-07-11.
+//
+
+import XCTest
+@testable import damus
+
+final class HashtagTests: XCTestCase {
+
+    override func setUpWithError() throws {
+        // Put setup code here. This method is called before the invocation of each test method in the class.
+    }
+
+    override func tearDownWithError() throws {
+        // Put teardown code here. This method is called after the invocation of each test method in the class.
+    }
+
+    func testParseHashtag() {
+        let parsed = parse_mentions(content: "some hashtag #bitcoin derp", tags: []).blocks
+        
+        XCTAssertNotNil(parsed)
+        XCTAssertEqual(parsed.count, 3)
+        XCTAssertEqual(parsed[0].is_text, "some hashtag ")
+        XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
+        XCTAssertEqual(parsed[2].is_text, " derp")
+    }
+    
+    func testHashtagWithComma() {
+        let parsed = parse_mentions(content: "some hashtag #bitcoin, cool", tags: []).blocks
+        
+        XCTAssertNotNil(parsed)
+        XCTAssertEqual(parsed.count, 3)
+        XCTAssertEqual(parsed[0].is_text, "some hashtag ")
+        XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
+        XCTAssertEqual(parsed[2].is_text, ", cool")
+    }
+    
+    func testHashtagWithEmoji() {
+        let parsed = parse_mentions(content: "some hashtag #bitcoin☕️ cool", tags: []).blocks
+        
+        XCTAssertNotNil(parsed)
+        XCTAssertEqual(parsed.count, 3)
+        XCTAssertEqual(parsed[0].is_text, "some hashtag ")
+        XCTAssertEqual(parsed[1].is_hashtag, "bitcoin☕️")
+        XCTAssertEqual(parsed[2].is_text, " cool")
+    }
+    
+    func testHashtagWithAccents() {
+        let parsed = parse_mentions(content: "hello from #türkiye", tags: []).blocks
+        
+        XCTAssertNotNil(parsed)
+        XCTAssertEqual(parsed.count, 2)
+        XCTAssertEqual(parsed[0].is_text, "hello from ")
+        XCTAssertEqual(parsed[1].is_hashtag, "türkiye")
+    }
+
+    func testHashtagWithNonLatinCharacters() {
+        let parsed = parse_mentions(content: "this is a #시험 hope it works", tags: []).blocks
+        
+        XCTAssertNotNil(parsed)
+        XCTAssertEqual(parsed.count, 3)
+        XCTAssertEqual(parsed[0].is_text, "this is a ")
+        XCTAssertEqual(parsed[1].is_hashtag, "시험")
+        XCTAssertEqual(parsed[2].is_text, " hope it works")
+    }
+    
+    func testParseHashtagEnd() {
+        let parsed = parse_mentions(content: "some hashtag #bitcoin", tags: []).blocks
+        
+        XCTAssertNotNil(parsed)
+        XCTAssertEqual(parsed.count, 2)
+        XCTAssertEqual(parsed[0].is_text, "some hashtag ")
+        XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
+    }
+    
+}

damusTests/damusTests.swift

@@ -176,46 +176,6 @@ class damusTests: XCTestCase {
         XCTAssertEqual(ev.tags[1][1], "bitcoin")
         
     }
-    
-    func testParseHashtag() {
-        let parsed = parse_mentions(content: "some hashtag #bitcoin derp", tags: []).blocks
-        
-        XCTAssertNotNil(parsed)
-        XCTAssertEqual(parsed.count, 3)
-        XCTAssertEqual(parsed[0].is_text, "some hashtag ")
-        XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
-        XCTAssertEqual(parsed[2].is_text, " derp")
-    }
-    
-    func testHashtagWithComma() {
-        let parsed = parse_mentions(content: "some hashtag #bitcoin, cool", tags: []).blocks
-        
-        XCTAssertNotNil(parsed)
-        XCTAssertEqual(parsed.count, 3)
-        XCTAssertEqual(parsed[0].is_text, "some hashtag ")
-        XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
-        XCTAssertEqual(parsed[2].is_text, ", cool")
-    }
-    
-    func testHashtagWithEmoji() {
-        let parsed = parse_mentions(content: "some hashtag #bitcoin☕️ cool", tags: []).blocks
-        
-        XCTAssertNotNil(parsed)
-        XCTAssertEqual(parsed.count, 3)
-        XCTAssertEqual(parsed[0].is_text, "some hashtag ")
-        XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
-        XCTAssertEqual(parsed[2].is_text, "☕️ cool")
-    }
-    
-    func testParseHashtagEnd() {
-        let parsed = parse_mentions(content: "some hashtag #bitcoin", tags: []).blocks
-        
-        XCTAssertNotNil(parsed)
-        XCTAssertEqual(parsed.count, 2)
-        XCTAssertEqual(parsed[0].is_text, "some hashtag ")
-        XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
-    }
-    
     func testParseMentionOnlyText() {
         let parsed = parse_mentions(content: "there is no mention here", tags: [["e", "event_id"]]).blocks