From 7ff5452c344736f022427cc86f920ad418dbc504 Mon Sep 17 00:00:00 2001 From: Terry Yiu <963907+tyiu@users.noreply.github.com> Date: Sun, 9 Jun 2024 00:43:24 -0400 Subject: [PATCH] Allow graceful failures when downloading files, limit set of supported locales, and do not pretty print JSON to save on file size --- Sources/EmojiSourceKit/EmojiDownloader.swift | 339 +++++++++---------- 1 file changed, 169 insertions(+), 170 deletions(-) diff --git a/Sources/EmojiSourceKit/EmojiDownloader.swift b/Sources/EmojiSourceKit/EmojiDownloader.swift index 832a34a..e5d3380 100644 --- a/Sources/EmojiSourceKit/EmojiDownloader.swift +++ b/Sources/EmojiSourceKit/EmojiDownloader.swift @@ -52,29 +52,29 @@ struct EmojiDownloader: ParsableCommand, AsyncParsableCommand { var allCLDRAnnotations = [String: Emoji]() for locale in EmojiDownloader.supportedLocales { guard let cldrAnnotationsURL = await getURLForCLDRAnnotations(locale: locale) else { - return + continue } guard let cldrAnnotationsDerivedURL = await getURLForCLDRAnnotationsDerived(locale: locale) else { - return + continue } - print("Trying CLDR data at \(cldrAnnotationsURL)\n") + print("Trying CLDR data at \(cldrAnnotationsURL) for locale \(locale)\n") let cldrAnnotationsHandle = try FileHandle(forReadingFrom: cldrAnnotationsURL) guard let cldrAnnotationsData = try cldrAnnotationsHandle.readToEnd() else { - print("⚠️", "Could not read CLDR annotations data.\n") - return + print("⚠️", "Could not read CLDR annotations data for locale \(locale).\n") + continue } let cldrAnnotationsMap = emojisMap(data: cldrAnnotationsData, locale: locale) ?? [:] - print("Trying CLDR data at \(cldrAnnotationsDerivedURL)\n") + print("Trying CLDR data at \(cldrAnnotationsDerivedURL) for locale \(locale)\n") let cldrAnnotationsDerivedHandle = try FileHandle(forReadingFrom: cldrAnnotationsDerivedURL) guard let cldrAnnotationsDerivedData = try cldrAnnotationsDerivedHandle.readToEnd() else { - print("⚠️", "Could not read CLDR annotations derived data.\n") - return + print("⚠️", "Could not read CLDR annotations derived data for locale \(locale).\n") + continue } let cldrAnnotationsDerivedMap = emojisMap(data: cldrAnnotationsDerivedData, locale: locale) ?? [:] @@ -119,172 +119,172 @@ struct EmojiDownloader: ParsableCommand, AsyncParsableCommand { if parser.parse() { return handler.emojisMap } else { - print("Failed to parse XML\n") + print("Failed to parse XML for locale \(locale)\n") return nil } } static let supportedLocales = [ - "af", - "am", - "ar", - "ar_SA", - "as", - "ast", - "az", - "be", - "bew", - "bg", - "bgn", - "bn", - "br", - "bs", - "ca", - "ccp", - "ceb", - "chr", - "ckb", - "cs", - "cv", - "cy", - "da", - "de", - "de_CH", - "doi", - "dsb", - "el", +// "af", +// "am", +// "ar", +// "ar_SA", +// "as", +// "ast", +// "az", +// "be", +// "bew", +// "bg", +// "bgn", +// "bn", +// "br", +// "bs", +// "ca", +// "ccp", +// "ceb", +// "chr", +// "ckb", +// "cs", +// "cv", +// "cy", +// "da", +// "de", +// "de_CH", +// "doi", +// "dsb", +// "el", "en", - "en_001", - "en_AU", - "en_CA", - "en_GB", - "en_IN", - "es", - "es_419", - "es_MX", - "es_US", - "et", - "eu", - "fa", - "ff", - "ff_Adlm", - "fi", - "fil", - "fo", - "fr", - "fr_CA", - "ga", - "gd", - "gl", - "gu", - "ha", - "ha_NE", - "he", - "hi", - "hi_Latn", - "hr", - "hsb", - "hu", - "hy", - "ia", - "id", - "ig", - "is", - "it", - "ja", - "jv", - "ka", - "kab", - "kk", - "kl", - "km", - "kn", - "ko", - "kok", - "ku", - "ky", - "lb", - "lij", - "lo", - "lt", - "lv", - "mai", - "mi", - "mk", - "ml", - "mn", - "mni", - "mr", - "ms", - "mt", - "my", - "nb", - "ne", - "nl", - "nn", - "no", - "nso", - "oc", - "om", - "or", - "pa", - "pa_Arab", - "pcm", - "pl", - "ps", - "pt", - "pt_PT", - "qu", - "quc", - "rhg", - "rm", - "ro", - "root", - "ru", - "rw", - "sa", - "sat", - "sc", - "sd", - "si", - "sk", - "sl", - "so", - "sq", - "sr", - "sr_Cyrl", - "sr_Cyrl_BA", - "sr_Latn", - "sr_Latn_BA", - "su", - "sv", - "sw", - "sw_KE", - "ta", - "te", - "tg", - "th", - "ti", - "tk", - "tn", - "to", - "tr", - "tt", - "ug", - "uk", - "ur", - "uz", - "vi", - "wo", - "xh", - "yo", - "yo_BJ", - "yue", - "yue_Hans", - "zh", - "zh_Hant", - "zh_Hant_HK", - "zu" +// "en_001", +// "en_AU", +// "en_CA", +// "en_GB", +// "en_IN", +// "es", +// "es_419", +// "es_MX", +// "es_US", +// "et", +// "eu", +// "fa", +// "ff", +// "ff_Adlm", +// "fi", +// "fil", +// "fo", +// "fr", +// "fr_CA", +// "ga", +// "gd", +// "gl", +// "gu", +// "ha", +// "ha_NE", +// "he", +// "hi", +// "hi_Latn", +// "hr", +// "hsb", +// "hu", +// "hy", +// "ia", +// "id", +// "ig", +// "is", +// "it", +// "ja", +// "jv", +// "ka", +// "kab", +// "kk", +// "kl", +// "km", +// "kn", +// "ko", +// "kok", +// "ku", +// "ky", +// "lb", +// "lij", +// "lo", +// "lt", +// "lv", +// "mai", +// "mi", +// "mk", +// "ml", +// "mn", +// "mni", +// "mr", +// "ms", +// "mt", +// "my", +// "nb", +// "ne", +// "nl", +// "nn", +// "no", +// "nso", +// "oc", +// "om", +// "or", +// "pa", +// "pa_Arab", +// "pcm", +// "pl", +// "ps", +// "pt", +// "pt_PT", +// "qu", +// "quc", +// "rhg", +// "rm", +// "ro", +// "root", +// "ru", +// "rw", +// "sa", +// "sat", +// "sc", +// "sd", +// "si", +// "sk", +// "sl", +// "so", +// "sq", +// "sr", +// "sr_Cyrl", +// "sr_Cyrl_BA", +// "sr_Latn", +// "sr_Latn_BA", +// "su", +// "sv", +// "sw", +// "sw_KE", +// "ta", +// "te", +// "tg", +// "th", +// "ti", +// "tk", +// "tn", +// "to", +// "tr", +// "tt", +// "ug", +// "uk", +// "ur", +// "uz", +// "vi", +// "wo", +// "xh", +// "yo", +// "yo_BJ", +// "yue", +// "yue_Hans", +// "zh", +// "zh_Hant", +// "zh_Hant_HK", +// "zu" ] func getURLForCLDRAnnotations(locale: String) async -> URL? { @@ -317,7 +317,7 @@ struct EmojiDownloader: ParsableCommand, AsyncParsableCommand { let (tmpFileURL, response) = try await session.download(from: url) guard let statusCode = (response as? HTTPURLResponse)?.statusCode, statusCode == 200 else { - print("⚠️", "Failed with a non 200 HTTP status") + print("⚠️", "Failed with a non 200 HTTP status on \(urlString)") return nil } return tmpFileURL @@ -331,7 +331,6 @@ struct EmojiDownloader: ParsableCommand, AsyncParsableCommand { let directory = getPath() let encoder = JSONEncoder() - encoder.outputFormatting = .prettyPrinted guard let result = try? encoder.encode(data) else { print("⚠️", "Couldn't encode emoji categories.")