tyiu/EmojiKit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

Allow graceful failures when downloading files, limit set of supported locales, and do not pretty print JSON to save on file size

Browse Source
This commit is contained in:
Terry Yiu
2024-06-09 00:43:24 -04:00
parent 07a4eb2216
commit 7ff5452c34
1 changed files with 169 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

339
Sources/EmojiSourceKit/EmojiDownloader.swift
View File

@@ -52,29 +52,29 @@ struct EmojiDownloader: ParsableCommand, AsyncParsableCommand {
var allCLDRAnnotations = [String: Emoji]() var allCLDRAnnotations = [String: Emoji]()
for locale in EmojiDownloader.supportedLocales { for locale in EmojiDownloader.supportedLocales {
guard let cldrAnnotationsURL = await getURLForCLDRAnnotations(locale: locale) else { guard let cldrAnnotationsURL = await getURLForCLDRAnnotations(locale: locale) else {
return continue
} }
guard let cldrAnnotationsDerivedURL = await getURLForCLDRAnnotationsDerived(locale: locale) else { guard let cldrAnnotationsDerivedURL = await getURLForCLDRAnnotationsDerived(locale: locale) else {
return continue
} }
print("Trying CLDR data at \(cldrAnnotationsURL)\n") print("Trying CLDR data at \(cldrAnnotationsURL) for locale \(locale)\n")
let cldrAnnotationsHandle = try FileHandle(forReadingFrom: cldrAnnotationsURL) let cldrAnnotationsHandle = try FileHandle(forReadingFrom: cldrAnnotationsURL)
guard let cldrAnnotationsData = try cldrAnnotationsHandle.readToEnd() else { guard let cldrAnnotationsData = try cldrAnnotationsHandle.readToEnd() else {
print("⚠️", "Could not read CLDR annotations data.\n") print("⚠️", "Could not read CLDR annotations data for locale \(locale).\n")
return continue
} }
let cldrAnnotationsMap = emojisMap(data: cldrAnnotationsData, locale: locale) ?? [:] let cldrAnnotationsMap = emojisMap(data: cldrAnnotationsData, locale: locale) ?? [:]
print("Trying CLDR data at \(cldrAnnotationsDerivedURL)\n") print("Trying CLDR data at \(cldrAnnotationsDerivedURL) for locale \(locale)\n")
let cldrAnnotationsDerivedHandle = try FileHandle(forReadingFrom: cldrAnnotationsDerivedURL) let cldrAnnotationsDerivedHandle = try FileHandle(forReadingFrom: cldrAnnotationsDerivedURL)
guard let cldrAnnotationsDerivedData = try cldrAnnotationsDerivedHandle.readToEnd() else { guard let cldrAnnotationsDerivedData = try cldrAnnotationsDerivedHandle.readToEnd() else {
print("⚠️", "Could not read CLDR annotations derived data.\n") print("⚠️", "Could not read CLDR annotations derived data for locale \(locale).\n")
return continue
} }
let cldrAnnotationsDerivedMap = emojisMap(data: cldrAnnotationsDerivedData, locale: locale) ?? [:] let cldrAnnotationsDerivedMap = emojisMap(data: cldrAnnotationsDerivedData, locale: locale) ?? [:]
@@ -119,172 +119,172 @@ struct EmojiDownloader: ParsableCommand, AsyncParsableCommand {
if parser.parse() { if parser.parse() {
return handler.emojisMap return handler.emojisMap
} else { } else {
print("Failed to parse XML\n") print("Failed to parse XML for locale \(locale)\n")
return nil return nil
} }
} }
static let supportedLocales = [ static let supportedLocales = [
"af", // "af",
"am", // "am",
"ar", // "ar",
"ar_SA", // "ar_SA",
"as", // "as",
"ast", // "ast",
"az", // "az",
"be", // "be",
"bew", // "bew",
"bg", // "bg",
"bgn", // "bgn",
"bn", // "bn",
"br", // "br",
"bs", // "bs",
"ca", // "ca",
"ccp", // "ccp",
"ceb", // "ceb",
"chr", // "chr",
"ckb", // "ckb",
"cs", // "cs",
"cv", // "cv",
"cy", // "cy",
"da", // "da",
"de", // "de",
"de_CH", // "de_CH",
"doi", // "doi",
"dsb", // "dsb",
"el", // "el",
"en", "en",
"en_001", // "en_001",
"en_AU", // "en_AU",
"en_CA", // "en_CA",
"en_GB", // "en_GB",
"en_IN", // "en_IN",
"es", // "es",
"es_419", // "es_419",
"es_MX", // "es_MX",
"es_US", // "es_US",
"et", // "et",
"eu", // "eu",
"fa", // "fa",
"ff", // "ff",
"ff_Adlm", // "ff_Adlm",
"fi", // "fi",
"fil", // "fil",
"fo", // "fo",
"fr", // "fr",
"fr_CA", // "fr_CA",
"ga", // "ga",
"gd", // "gd",
"gl", // "gl",
"gu", // "gu",
"ha", // "ha",
"ha_NE", // "ha_NE",
"he", // "he",
"hi", // "hi",
"hi_Latn", // "hi_Latn",
"hr", // "hr",
"hsb", // "hsb",
"hu", // "hu",
"hy", // "hy",
"ia", // "ia",
"id", // "id",
"ig", // "ig",
"is", // "is",
"it", // "it",
"ja", // "ja",
"jv", // "jv",
"ka", // "ka",
"kab", // "kab",
"kk", // "kk",
"kl", // "kl",
"km", // "km",
"kn", // "kn",
"ko", // "ko",
"kok", // "kok",
"ku", // "ku",
"ky", // "ky",
"lb", // "lb",
"lij", // "lij",
"lo", // "lo",
"lt", // "lt",
"lv", // "lv",
"mai", // "mai",
"mi", // "mi",
"mk", // "mk",
"ml", // "ml",
"mn", // "mn",
"mni", // "mni",
"mr", // "mr",
"ms", // "ms",
"mt", // "mt",
"my", // "my",
"nb", // "nb",
"ne", // "ne",
"nl", // "nl",
"nn", // "nn",
"no", // "no",
"nso", // "nso",
"oc", // "oc",
"om", // "om",
"or", // "or",
"pa", // "pa",
"pa_Arab", // "pa_Arab",
"pcm", // "pcm",
"pl", // "pl",
"ps", // "ps",
"pt", // "pt",
"pt_PT", // "pt_PT",
"qu", // "qu",
"quc", // "quc",
"rhg", // "rhg",
"rm", // "rm",
"ro", // "ro",
"root", // "root",
"ru", // "ru",
"rw", // "rw",
"sa", // "sa",
"sat", // "sat",
"sc", // "sc",
"sd", // "sd",
"si", // "si",
"sk", // "sk",
"sl", // "sl",
"so", // "so",
"sq", // "sq",
"sr", // "sr",
"sr_Cyrl", // "sr_Cyrl",
"sr_Cyrl_BA", // "sr_Cyrl_BA",
"sr_Latn", // "sr_Latn",
"sr_Latn_BA", // "sr_Latn_BA",
"su", // "su",
"sv", // "sv",
"sw", // "sw",
"sw_KE", // "sw_KE",
"ta", // "ta",
"te", // "te",
"tg", // "tg",
"th", // "th",
"ti", // "ti",
"tk", // "tk",
"tn", // "tn",
"to", // "to",
"tr", // "tr",
"tt", // "tt",
"ug", // "ug",
"uk", // "uk",
"ur", // "ur",
"uz", // "uz",
"vi", // "vi",
"wo", // "wo",
"xh", // "xh",
"yo", // "yo",
"yo_BJ", // "yo_BJ",
"yue", // "yue",
"yue_Hans", // "yue_Hans",
"zh", // "zh",
"zh_Hant", // "zh_Hant",
"zh_Hant_HK", // "zh_Hant_HK",
"zu" // "zu"
] ]
func getURLForCLDRAnnotations(locale: String) async -> URL? { func getURLForCLDRAnnotations(locale: String) async -> URL? {
@@ -317,7 +317,7 @@ struct EmojiDownloader: ParsableCommand, AsyncParsableCommand {
let (tmpFileURL, response) = try await session.download(from: url) let (tmpFileURL, response) = try await session.download(from: url)
guard let statusCode = (response as? HTTPURLResponse)?.statusCode, statusCode == 200 else { guard let statusCode = (response as? HTTPURLResponse)?.statusCode, statusCode == 200 else {
print("⚠️", "Failed with a non 200 HTTP status") print("⚠️", "Failed with a non 200 HTTP status on \(urlString)")
return nil return nil
} }
return tmpFileURL return tmpFileURL
@@ -331,7 +331,6 @@ struct EmojiDownloader: ParsableCommand, AsyncParsableCommand {
let directory = getPath() let directory = getPath()
let encoder = JSONEncoder() let encoder = JSONEncoder()
encoder.outputFormatting = .prettyPrinted
guard let result = try? encoder.encode(data) else { guard let result = try? encoder.encode(data) else {
print("⚠️", "Couldn't encode emoji categories.") print("⚠️", "Couldn't encode emoji categories.")