Add support for iOS 15 and unicode release v13 (#1)

Sources/EmojiSourceKit/EmojiDownloader.swift

@@ -0,0 +1,125 @@
+//
+//  File.swift
+//  
+//
+//  Created by Niklas Amslgruber on 10.06.23.
+//
+
+import Foundation
+import ArgumentParser
+import EmojiKit
+
+struct EmojiDownloader: ParsableCommand, AsyncParsableCommand {
+
+    static let configuration: CommandConfiguration = CommandConfiguration(
+        commandName: "download",
+        abstract: "Downloads a list of all available emojis and their counts from unicode.rog for the respective unicode version"
+    )
+
+    @Argument var path: String
+    @Option(name: .shortAndLong) var version: EmojiManager.Version = .v15
+
+    private func getPath() -> String {
+        #if DEBUG
+        var url = URL(filePath: #file)
+        url = url.deletingLastPathComponent().deletingLastPathComponent()
+        url.append(path: "EmojiKitLibrary/Resources")
+
+        return url.absoluteString
+        #else
+        return path
+        #endif
+    }
+
+    func run() async throws {
+        print("⚙️", "Starting to download all emojis for version \(version.rawValue) from unicode.org...\n")
+
+        guard let emojiListURL = await getTemporaryURLForEmojiList(version: version), let emojiCountsURL = await getTemporaryURLForEmojiCounts(version: version) else {
+            print("⚠️", "Could not get content from unicode.org. Either the emoji list or the emoji count file is not available.\n")
+            return
+        }
+
+        print("🎉", "Successfully retrieved temporary URLs for version \(version.rawValue).\n")
+
+        print("⚙️", "Starting to parse content...\n")
+
+        let parser = UnicodeParser()
+
+        do {
+            let emojisByCategory: [EmojiCategory] = try await parser.parseEmojiList(for: emojiListURL)
+
+            let emojiCounts: [EmojiCategory.Name: Int] = parser.parseCountHTML(for: emojiCountsURL)
+
+            for category in emojisByCategory {
+                assert(emojiCounts[category.name] == category.values.count)
+            }
+
+            print("🎉", "Successfully parsed emojis and matched counts to the count file.\n")
+
+            save(data: emojisByCategory, for: version)
+
+            print("🎉", "Successfully saved emojis to file.\n")
+
+        } catch {
+            print("⚠️", "Could not parse emoji lists or emoji counts. Process failed with: \(error).\n")
+        }
+    }
+
+    func getTemporaryURLForEmojiList(version: EmojiManager.Version) async -> URL? {
+        return await load(urlString: "https://unicode.org/Public/emoji/\(version.versionIdentifier)/emoji-test.txt")
+    }
+
+    func getTemporaryURLForEmojiCounts(version: EmojiManager.Version) async -> URL? {
+        return await load(urlString: "https://www.unicode.org/emoji/charts-\(version.versionIdentifier)/emoji-counts.html")
+    }
+
+    private func load(urlString: String) async -> URL? {
+        guard let url = URL(string: urlString) else {
+            return nil
+        }
+        let session = URLSession(configuration: .default)
+
+        do {
+            let (tmpFileURL, response) = try await session.download(from: url)
+
+            guard let statusCode = (response as? HTTPURLResponse)?.statusCode, statusCode == 200 else {
+                print("⚠️", "Failed with a non 200 HTTP status")
+                return nil
+            }
+            return tmpFileURL
+        } catch {
+            print("⚠️", error)
+            return nil
+        }
+    }
+
+    private func save(data: [EmojiCategory], for: EmojiManager.Version) {
+        let directory = getPath()
+
+        let encoder = JSONEncoder()
+        encoder.outputFormatting = .prettyPrinted
+
+        guard let result = try? encoder.encode(data) else {
+            print("⚠️", "Couldn't encode emoji categories.")
+            return
+        }
+
+        var filePath = URL(filePath: directory)
+        filePath.append(path: "\(version.fileName).json")
+        let jsonString = String(data: result, encoding: .utf8)
+
+        print("⚙️", "Saving emojis to file \(filePath.absoluteString)...\n")
+
+        if FileManager.default.fileExists(atPath: filePath.absoluteString) == false {
+            FileManager.default.createFile(atPath: filePath.absoluteString, contents: nil)
+        }
+
+        do {
+            try jsonString?.write(to: filePath, atomically: true, encoding: .utf8)
+        } catch {
+            print("⚠️", error)
+        }
+    }
+}
+
+extension EmojiManager.Version: ExpressibleByArgument {}

Sources/EmojiSourceKit/EmojiScripts.swift

@@ -0,0 +1,21 @@
+//
+//  EmojiScripts.swift
+//  
+//
+//  Created by Niklas Amslgruber on 12.06.23.
+//
+
+import Foundation
+import ArgumentParser
+
+@main
+struct EmojiScripts: ParsableCommand, AsyncParsableCommand {
+
+    static let configuration: CommandConfiguration = CommandConfiguration(
+        commandName: "emojis",
+        abstract: "Manage Emojis from Unicode",
+        subcommands: [
+            EmojiDownloader.self,
+        ]
+    )
+}

Sources/EmojiSourceKit/StringHelper.swift

@@ -0,0 +1,33 @@
+//
+//  StringHelper.swift
+//  
+//
+//  Created by Niklas Amslgruber on 10.06.23.
+//
+
+import Foundation
+
+extension String {
+
+    func trim() -> String {
+        return self.trimmingCharacters(in: .whitespacesAndNewlines)
+    }
+
+    func asEmoji() -> String? {
+        guard let unicodeNumber = Int(self, radix: 16), let unicode = Unicode.Scalar(unicodeNumber) else {
+            return nil
+        }
+        return String(unicode)
+    }
+}
+
+extension String.SubSequence {
+
+    func trim() -> String {
+        return String(self).trim()
+    }
+
+    func asEmoji() -> String? {
+        return String(self).asEmoji()
+    }
+}

Sources/EmojiSourceKit/UnicodeParser.swift

@@ -0,0 +1,147 @@
+//
+//  UnicodeParser.swift
+//  
+//
+//  Created by Niklas Amslgruber on 12.06.23.
+//
+
+import Foundation
+import SwiftSoup
+import EmojiKit
+
+class UnicodeParser {
+
+    enum Tags: String {
+        case comment = "#"
+        case group = "# group:"
+        case unqualified
+        case minimallyQualified = "minimally-qualified"
+    }
+
+    func parseEmojiList(for fileUrl: URL) async throws -> [EmojiCategory] {
+        let handle = try FileHandle(forReadingFrom: fileUrl)
+        var currentGroup: EmojiCategory.Name = .activities
+        var emojisByGroup: [EmojiCategory.Name: [String]] = [:]
+
+        for try await line in handle.bytes.lines {
+
+            /// Skip comments, but keep groups
+            if isLine(line, ofType: .comment), isLine(line, ofType: .group) == false {
+                continue
+            }
+
+            /// Get current group
+            if isLine(line, ofType: .group) {
+                let name = line.split(separator: ":")
+                let categoryName = name.last?.trim() ?? ""
+                guard let category = EmojiCategory.Name(rawValue: categoryName) else {
+                    continue
+                }
+                currentGroup = category
+                emojisByGroup[category] = []
+            }
+
+            /// Split line into list of entries
+            let lineComponents = line.split(separator: ";")
+
+            /// Get hex-string from compenents
+            guard let hexString = lineComponents.map({ $0.trim() }).first else {
+                continue
+            }
+
+            /// Check if category exists
+            guard lineComponents.count > 1 else {
+                continue
+            }
+
+            let category = lineComponents[1].trim()
+
+            /// Remove `unqualified` or `minimally-qualified` entries
+            guard (isLine(category, ofType: .unqualified) || isLine(category, ofType: .minimallyQualified)) == false else {
+                continue
+            }
+
+            let hexComponents = hexString.split(separator: " ")
+
+            /// Check for multi-hex emojis
+            if hexComponents.count > 1 {
+                let multiHexEmoji = hexComponents.compactMap({ $0.asEmoji() }).joined()
+
+
+                if multiHexEmoji.isEmpty == false {
+                    emojisByGroup[currentGroup]?.append(multiHexEmoji)
+                }
+            } else {
+                if let unicode = hexString.asEmoji(), unicode.isEmpty == false {
+                    emojisByGroup[currentGroup]?.append(unicode)
+                }
+            }
+        }
+        try handle.close()
+
+        var result: [EmojiCategory] = []
+
+        for category in EmojiCategory.Name.allCases {
+            result.append(EmojiCategory(name: category, values: emojisByGroup[category] ?? []))
+        }
+        return result
+    }
+
+    func parseCountHTML(for url: URL) -> [EmojiCategory.Name: Int] {
+        do {
+            let html = try String(contentsOf: url)
+            let doc: Document = try SwiftSoup.parse(html)
+
+            guard let table = try doc.select("table").first() else {
+                return [:]
+            }
+
+            let rows: Elements = try table.select("tbody tr")
+            let categories = rows.first
+            let totals = rows.last
+
+            guard let categories, let totals, let categoryEntries = try? categories.select("th"), let countEntries = try? totals.select("th") else {
+                return [:]
+            }
+
+            var categoryNames: [EmojiCategory.Name] = []
+            var countNumbers: [Int] = []
+
+            for categoryElement in categoryEntries {
+                if categoryElement == categoryEntries.first || categoryElement == categoryEntries.last {
+                    continue
+                }
+                guard let text = try? categoryElement.text(), let category = EmojiCategory.Name(rawValue: text) else {
+                    continue
+                }
+                categoryNames.append(category)
+            }
+
+            for countElement in countEntries {
+                if countElement == countEntries.first || countElement == countEntries.last {
+                    continue
+                }
+                guard let text = try? countElement.text(), let number = Int(text) else {
+                    continue
+                }
+                countNumbers.append(number)
+            }
+
+            var result: [EmojiCategory.Name: Int] = [:]
+
+            for (index, categoryName) in categoryNames.enumerated() {
+                result[categoryName] = countNumbers[index]
+            }
+
+            return result
+        } catch {
+            print("Error parsing HTML: \(error)")
+        }
+
+        return [:]
+    }
+
+    private func isLine(_ line: String, ofType type: Tags) -> Bool {
+        return line.starts(with: type.rawValue)
+    }
+}