tyiu/damus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for multilingual hashtags

Browse Source 
Changelog-Added: Add support for multilingual hashtags
Reviewed-by: William Casarin <jb55@jb55.com>
Closes: https://github.com/damus-io/damus/issues/949
This commit is contained in:
cr0bar
2023-07-11 07:21:56 -07:00
committed by William Casarin
parent aae97c5cb7 031c7823ae
commit dc21b6139c
6 changed files with 92 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
damus-c/cursor.h
View File

@@ -432,7 +432,7 @@ static inline int is_whitespace(char c) {
}
static inline int is_boundary(char c) {
return !isalnum(c);
return is_whitespace(c) || ispunct(c);
}
static inline int is_invalid_url_ending(char c) {

8
damus.xcodeproj/project.pbxproj
View File

@@ -53,6 +53,7 @@
4C198DF229F88C6B004C165C /* BlurHashDecode.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C198DEE29F88C6B004C165C /* BlurHashDecode.swift */; };
4C198DF529F88D2E004C165C /* ImageMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C198DF429F88D2E004C165C /* ImageMetadata.swift */; };
4C19AE512A5CEF7C00C90DB7 /* NostrScript.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C19AE4C2A5CEF7C00C90DB7 /* NostrScript.swift */; };
4C19AE552A5D977400C90DB7 /* HashtagTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C19AE542A5D977400C90DB7 /* HashtagTests.swift */; };
4C1A9A1A29DCA17E00516EAC /* ReplyCounter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C1A9A1929DCA17E00516EAC /* ReplyCounter.swift */; };
4C1A9A1D29DDCF9B00516EAC /* NotificationSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C1A9A1C29DDCF9B00516EAC /* NotificationSettingsView.swift */; };
4C1A9A1F29DDD24B00516EAC /* AppearanceSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C1A9A1E29DDD24B00516EAC /* AppearanceSettingsView.swift */; };
@@ -495,6 +496,7 @@
4C19AE4B2A5CEF7C00C90DB7 /* primal.ts */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.typescript; path = primal.ts; sourceTree = "<group>"; };
4C19AE4C2A5CEF7C00C90DB7 /* NostrScript.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NostrScript.swift; sourceTree = "<group>"; };
4C19AE502A5CEF7C00C90DB7 /* nostr.ts */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.typescript; path = nostr.ts; sourceTree = "<group>"; };
4C19AE542A5D977400C90DB7 /* HashtagTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HashtagTests.swift; sourceTree = "<group>"; };
4C1A9A1929DCA17E00516EAC /* ReplyCounter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ReplyCounter.swift; sourceTree = "<group>"; };
4C1A9A1C29DDCF9B00516EAC /* NotificationSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NotificationSettingsView.swift; sourceTree = "<group>"; };
4C1A9A1E29DDD24B00516EAC /* AppearanceSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppearanceSettingsView.swift; sourceTree = "<group>"; };
@@ -1510,6 +1512,7 @@
3A5E47C62A4A76C800C0D090 /* TrieTests.swift */,
3A90B1822A4EA3C600000D94 /* UserSearchCacheTests.swift */,
4C4F14A62A2A61A30045A0B9 /* NostrScriptTests.swift */,
4C19AE542A5D977400C90DB7 /* HashtagTests.swift */,
);
path = damusTests;
sourceTree = "<group>";
@@ -2114,6 +2117,7 @@
files = (
5019CADD2A0FB0A9000069E1 /* ProfileDatabaseTests.swift in Sources */,
3A90B1832A4EA3C600000D94 /* UserSearchCacheTests.swift in Sources */,
4C19AE552A5D977400C90DB7 /* HashtagTests.swift in Sources */,
3A3040ED29A5CB86008A0F29 /* ReplyDescriptionTests.swift in Sources */,
4C8D00D429E3C5D40036AF10 /* NIP19Tests.swift in Sources */,
3A30410129AB12AA008A0F29 /* EventGroupViewTests.swift in Sources */,
@@ -2405,7 +2409,7 @@
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = damus/damus.entitlements;
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
CURRENT_PROJECT_VERSION = 2;
DEVELOPMENT_ASSET_PATHS = "\"damus/Preview Content\"";
DEVELOPMENT_TEAM = XK7H4JAB3D;
ENABLE_PREVIEWS = YES;
@@ -2454,7 +2458,7 @@
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = damus/damus.entitlements;
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
CURRENT_PROJECT_VERSION = 2;
DEVELOPMENT_ASSET_PATHS = "\"damus/Preview Content\"";
DEVELOPMENT_TEAM = XK7H4JAB3D;
ENABLE_PREVIEWS = YES;

2
damus/Models/Mentions.swift
View File

@@ -415,7 +415,7 @@ func parse_while(_ p: Parser, match: (Character) -> Bool) -> String? {
}
func is_hashtag_char(_ c: Character) -> Bool {
return c.isLetter || c.isNumber
return (c.isLetter || c.isNumber || c.isASCII) && (!c.isPunctuation && !c.isWhitespace)
}
func prev_char(_ p: Parser, n: Int) -> Character? {

6
damus/Nostr/NostrLink.swift
View File

@@ -116,8 +116,12 @@ func decode_nostr_uri(_ s: String) -> NostrLink? {
return decode_universal_link(s)
}
var uri = s.replacingOccurrences(of: "nostr://", with: "")
var uri = s
uri = uri.replacingOccurrences(of: "nostr://", with: "")
uri = uri.replacingOccurrences(of: "nostr:", with: "")
// Fix for non-latin characters resulting in second colon being encoded
uri = uri.replacingOccurrences(of: "damus:t%3A", with: "t:")
uri = uri.replacingOccurrences(of: "damus://", with: "")
uri = uri.replacingOccurrences(of: "damus:", with: "")

79
damusTests/HashtagTests.swift Normal file
View File

@@ -0,0 +1,79 @@
//
// HashtagTests.swift
// damusTests
//
// Created by William Casarin on 2023-07-11.
//
import XCTest
@testable import damus
final class HashtagTests: XCTestCase {
override func setUpWithError() throws {
// Put setup code here. This method is called before the invocation of each test method in the class.
}
override func tearDownWithError() throws {
// Put teardown code here. This method is called after the invocation of each test method in the class.
}
func testParseHashtag() {
let parsed = parse_mentions(content: "some hashtag #bitcoin derp", tags: []).blocks
XCTAssertNotNil(parsed)
XCTAssertEqual(parsed.count, 3)
XCTAssertEqual(parsed[0].is_text, "some hashtag ")
XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
XCTAssertEqual(parsed[2].is_text, " derp")
}
func testHashtagWithComma() {
let parsed = parse_mentions(content: "some hashtag #bitcoin, cool", tags: []).blocks
XCTAssertNotNil(parsed)
XCTAssertEqual(parsed.count, 3)
XCTAssertEqual(parsed[0].is_text, "some hashtag ")
XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
XCTAssertEqual(parsed[2].is_text, ", cool")
}
func testHashtagWithEmoji() {
let parsed = parse_mentions(content: "some hashtag #bitcoin☕️ cool", tags: []).blocks
XCTAssertNotNil(parsed)
XCTAssertEqual(parsed.count, 3)
XCTAssertEqual(parsed[0].is_text, "some hashtag ")
XCTAssertEqual(parsed[1].is_hashtag, "bitcoin☕️")
XCTAssertEqual(parsed[2].is_text, " cool")
}
func testHashtagWithAccents() {
let parsed = parse_mentions(content: "hello from #türkiye", tags: []).blocks
XCTAssertNotNil(parsed)
XCTAssertEqual(parsed.count, 2)
XCTAssertEqual(parsed[0].is_text, "hello from ")
XCTAssertEqual(parsed[1].is_hashtag, "türkiye")
}
func testHashtagWithNonLatinCharacters() {
let parsed = parse_mentions(content: "this is a #시험 hope it works", tags: []).blocks
XCTAssertNotNil(parsed)
XCTAssertEqual(parsed.count, 3)
XCTAssertEqual(parsed[0].is_text, "this is a ")
XCTAssertEqual(parsed[1].is_hashtag, "시험")
XCTAssertEqual(parsed[2].is_text, " hope it works")
}
func testParseHashtagEnd() {
let parsed = parse_mentions(content: "some hashtag #bitcoin", tags: []).blocks
XCTAssertNotNil(parsed)
XCTAssertEqual(parsed.count, 2)
XCTAssertEqual(parsed[0].is_text, "some hashtag ")
XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
}
}

40
damusTests/damusTests.swift
View File

@@ -176,46 +176,6 @@ class damusTests: XCTestCase {
XCTAssertEqual(ev.tags[1][1], "bitcoin")
}
func testParseHashtag() {
let parsed = parse_mentions(content: "some hashtag #bitcoin derp", tags: []).blocks
XCTAssertNotNil(parsed)
XCTAssertEqual(parsed.count, 3)
XCTAssertEqual(parsed[0].is_text, "some hashtag ")
XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
XCTAssertEqual(parsed[2].is_text, " derp")
}
func testHashtagWithComma() {
let parsed = parse_mentions(content: "some hashtag #bitcoin, cool", tags: []).blocks
XCTAssertNotNil(parsed)
XCTAssertEqual(parsed.count, 3)
XCTAssertEqual(parsed[0].is_text, "some hashtag ")
XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
XCTAssertEqual(parsed[2].is_text, ", cool")
}
func testHashtagWithEmoji() {
let parsed = parse_mentions(content: "some hashtag #bitcoin☕️ cool", tags: []).blocks
XCTAssertNotNil(parsed)
XCTAssertEqual(parsed.count, 3)
XCTAssertEqual(parsed[0].is_text, "some hashtag ")
XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
XCTAssertEqual(parsed[2].is_text, "☕️ cool")
}
func testParseHashtagEnd() {
let parsed = parse_mentions(content: "some hashtag #bitcoin", tags: []).blocks
XCTAssertNotNil(parsed)
XCTAssertEqual(parsed.count, 2)
XCTAssertEqual(parsed[0].is_text, "some hashtag ")
XCTAssertEqual(parsed[1].is_hashtag, "bitcoin")
}
func testParseMentionOnlyText() {
let parsed = parse_mentions(content: "there is no mention here", tags: [["e", "event_id"]]).blocks