1mod error;
2mod raw;
3
4use error::ResourceErr;
5
6#[cfg(any(feature = "bin", feature = "stat", test))]
7pub mod analysis;
8
9use fluent_bundle::{FluentResource, bundle::FluentBundle};
10use intl_memoizer::concurrent::IntlLangMemoizer;
11use unic_langid::LanguageIdentifier;
12
13use hashbrown::HashMap;
14use serde::{Deserialize, Serialize};
15use std::{borrow::Cow, io};
16
17use assets::{
18 AssetExt, AssetHandle, AssetReadGuard, ReloadWatcher, SharedString, source::DirEntry,
19};
20use common_assets as assets;
21use common_i18n::{Content, LocalizationArg};
22use tracing::warn;
23
24pub use fluent::{FluentValue, fluent_args};
26pub use fluent_bundle::FluentArgs;
27
28pub const REFERENCE_LANG: &str = "en";
31
32#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
34pub struct LanguageMetadata {
35 pub language_name: String,
37
38 pub language_identifier: String,
45}
46
47#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
49pub struct Font {
50 pub asset_key: String,
52
53 scale_ratio: f32,
55}
56
57impl Font {
58 #[must_use]
60 pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 }
61}
62
63pub type Fonts = HashMap<String, Font>;
65
66struct Language {
68 pub(crate) bundle: FluentBundle<FluentResource, IntlLangMemoizer>,
70
71 pub(crate) fonts: Fonts,
73 pub(crate) metadata: LanguageMetadata,
74}
75
76impl Language {
77 fn try_msg<'a>(&'a self, key: &str, args: Option<&'a FluentArgs>) -> Option<Cow<'a, str>> {
78 let bundle = &self.bundle;
79 let msg = bundle.get_message(key)?;
80 let mut errs = Vec::new();
81 let msg = bundle.format_pattern(msg.value()?, args, &mut errs);
82 for err in errs {
83 tracing::error!("err: {err} for {key}");
84 }
85
86 Some(msg)
87 }
88
89 fn try_attr<'a>(
90 &'a self,
91 key: &str,
92 attr: &str,
93 args: Option<&'a FluentArgs>,
94 ) -> Option<Cow<'a, str>> {
95 let bundle = &self.bundle;
96 let msg = bundle.get_message(key)?;
97 let attr = msg.get_attribute(attr)?;
98 let attr = attr.value();
99
100 let mut errs = Vec::new();
101 let msg = bundle.format_pattern(attr, args, &mut errs);
102 for err in errs {
103 tracing::error!("err: {err} for {key}");
104 }
105
106 Some(msg)
107 }
108
109 fn try_variation<'a>(
112 &'a self,
113 key: &str,
114 seed: u16,
115 args: Option<&'a FluentArgs>,
116 ) -> Option<Cow<'a, str>> {
117 let bundle = &self.bundle;
118 let msg = bundle.get_message(key)?;
119
120 let mut errs = Vec::new();
121
122 let attrs: Vec<_> = msg.attributes().collect();
123 let msg = if !attrs.is_empty() {
124 let idx = usize::from(seed) % attrs.len();
125 let variation = &attrs[idx];
141 bundle.format_pattern(variation.value(), args, &mut errs)
142 } else {
143 bundle.format_pattern(msg.value()?, args, &mut errs)
145 };
146
147 for err in errs {
148 tracing::error!("err: {err} for {key}");
149 }
150
151 Some(msg)
152 }
153}
154impl assets::Compound for Language {
155 fn load(cache: assets::AnyCache, path: &SharedString) -> Result<Self, assets::BoxedError> {
156 let manifest = cache
157 .load::<raw::Manifest>(&[path, ".", "_manifest"].concat())?
158 .cloned();
159 let raw::Manifest {
160 mut fonts,
161 metadata,
162 } = manifest;
163
164 let lang_id: LanguageIdentifier = metadata.language_identifier.parse()?;
165 let mut bundle = FluentBundle::new_concurrent(vec![lang_id]);
166
167 for id in cache.load_rec_dir::<raw::Resource>(path)?.read().ids() {
169 match cache.load(id) {
170 Ok(handle) => {
171 let source: &raw::Resource = &handle.read();
172 let src = source.src.clone();
173
174 let resource = FluentResource::try_new(src).map_err(|(_ast, errs)| {
175 ResourceErr::parsing_error(errs, id.to_string(), &source.src)
176 })?;
177
178 bundle
179 .add_resource(resource)
180 .map_err(|e| ResourceErr::BundleError(format!("{e:?}")))?;
181 },
182 Err(err) => {
183 warn!("Unable to load asset {id}, error={err:?}");
185 },
186 }
187 }
188
189 bundle.set_use_isolating(false);
193
194 fonts.entry("universal".to_owned()).or_insert(Font {
201 asset_key: "voxygen.font.GoNotoCurrent".to_owned(),
202 scale_ratio: 1.0,
203 });
204
205 Ok(Self {
206 bundle,
207 fonts,
208 metadata,
209 })
210 }
211}
212
213#[derive(Copy, Clone)]
216pub struct LocalizationHandle {
217 active: AssetHandle<Language>,
218 watcher: ReloadWatcher,
219 fallback: Option<AssetHandle<Language>>,
220 pub use_english_fallback: bool,
221}
222
223pub type Localization = LocalizationGuard;
226
227pub struct LocalizationGuard {
230 active: AssetReadGuard<Language>,
231 fallback: Option<AssetReadGuard<Language>>,
232}
233
234impl LocalizationGuard {
235 pub fn try_fallback_msg(&self, key: &str) -> Option<Cow<str>> {
237 self.fallback.as_ref().and_then(|fb| fb.try_msg(key, None))
238 }
239
240 pub fn try_msg(&self, key: &str) -> Option<Cow<str>> {
245 self.active
246 .try_msg(key, None)
247 .or_else(|| self.try_fallback_msg(key))
248 }
249
250 pub fn get_msg(&self, key: &str) -> Cow<str> {
257 self.try_msg(key)
262 .unwrap_or_else(|| Cow::Owned(key.to_owned()))
263 }
264
265 pub fn try_msg_ctx<'a>(&'a self, key: &str, args: &'a FluentArgs) -> Option<Cow<'static, str>> {
270 self.active
276 .try_msg(key, Some(args))
277 .or_else(|| {
278 self.fallback
279 .as_ref()
280 .and_then(|fb| fb.try_msg(key, Some(args)))
281 })
282 .map(|res| Cow::Owned(res.into_owned()))
283 }
284
285 pub fn get_msg_ctx<'a>(&'a self, key: &str, args: &'a FluentArgs) -> Cow<'static, str> {
292 self.try_msg_ctx(key, args)
293 .unwrap_or_else(|| Cow::Owned(key.to_owned()))
294 }
295
296 pub fn try_variation(&self, key: &str, seed: u16) -> Option<Cow<str>> {
304 self.active.try_variation(key, seed, None).or_else(|| {
305 self.fallback
306 .as_ref()
307 .and_then(|fb| fb.try_variation(key, seed, None))
308 })
309 }
310
311 pub fn get_variation(&self, key: &str, seed: u16) -> Cow<str> {
321 self.try_variation(key, seed)
322 .unwrap_or_else(|| Cow::Owned(key.to_owned()))
323 }
324
325 pub fn try_variation_ctx<'a>(
334 &'a self,
335 key: &str,
336 seed: u16,
337 args: &'a FluentArgs,
338 ) -> Option<Cow<'a, str>> {
339 self.active
340 .try_variation(key, seed, Some(args))
341 .or_else(|| {
342 self.fallback
343 .as_ref()
344 .and_then(|fb| fb.try_variation(key, seed, Some(args)))
345 })
346 }
347
348 fn get_content_for_lang(lang: &Language, content: &Content) -> Result<String, String> {
356 match content {
357 Content::Plain(text) => Ok(text.clone()),
358 Content::Key(key) => lang
359 .try_msg(key, None)
360 .map(Cow::into_owned)
361 .ok_or_else(|| key.to_string()),
362 Content::Attr(key, attr) => lang
363 .try_attr(key, attr, None)
364 .map(Cow::into_owned)
365 .ok_or_else(|| format!("{key}.{attr}")),
366 Content::Localized { key, seed, args } => {
367 let mut is_arg_failure = false;
369
370 let mut fargs = FluentArgs::new();
371 for (k, arg) in args {
372 let arg_val = match arg {
373 LocalizationArg::Content(content) => {
374 let arg_res = Self::get_content_for_lang(lang, content)
375 .unwrap_or_else(|broken_text| {
376 is_arg_failure = true;
377 broken_text
378 })
379 .into();
380
381 FluentValue::String(arg_res)
382 },
383 LocalizationArg::Nat(n) => FluentValue::from(n),
384 };
385 fargs.set(k, arg_val);
386 }
387
388 lang.try_variation(key, *seed, Some(&fargs))
389 .map(Cow::into_owned)
390 .ok_or_else(|| key.clone())
391 .and_then(|text| if is_arg_failure { Err(text) } else { Ok(text) })
392 },
393 }
394 }
395
396 pub fn get_content(&self, content: &Content) -> String {
433 match Self::get_content_for_lang(&self.active, content) {
434 Ok(text) => text,
435 Err(broken_text) => self
440 .fallback
441 .as_ref()
442 .and_then(|fb| Self::get_content_for_lang(fb, content).ok())
443 .unwrap_or(broken_text),
444 }
445 }
446
447 pub fn get_content_fallback(&self, content: &Content) -> String {
448 self.fallback
449 .as_ref()
450 .map(|fb| Self::get_content_for_lang(fb, content))
451 .transpose()
452 .map(|msg| msg.unwrap_or_default())
453 .unwrap_or_else(|e| e)
454 }
455
456 pub fn get_variation_ctx<'a>(
467 &'a self,
468 key: &str,
469 seed: u16,
470 args: &'a FluentArgs,
471 ) -> Cow<'a, str> {
472 self.try_variation_ctx(key, seed, args)
473 .unwrap_or_else(|| Cow::Owned(key.to_owned()))
474 }
475
476 pub fn try_attr(&self, key: &str, attr: &str) -> Option<Cow<str>> {
481 self.active.try_attr(key, attr, None).or_else(|| {
482 self.fallback
483 .as_ref()
484 .and_then(|fb| fb.try_attr(key, attr, None))
485 })
486 }
487
488 pub fn get_attr(&self, key: &str, attr: &str) -> Cow<str> {
495 self.try_attr(key, attr)
496 .unwrap_or_else(|| Cow::Owned(format!("{key}.{attr}")))
497 }
498
499 pub fn try_attr_ctx(
504 &self,
505 key: &str,
506 attr: &str,
507 args: &FluentArgs,
508 ) -> Option<Cow<'static, str>> {
509 self.active
513 .try_attr(key, attr, Some(args))
514 .or_else(|| {
515 self.fallback
516 .as_ref()
517 .and_then(|fb| fb.try_attr(key, attr, Some(args)))
518 })
519 .map(|res| Cow::Owned(res.into_owned()))
520 }
521
522 pub fn get_attr_ctx(&self, key: &str, attr: &str, args: &FluentArgs) -> Cow<'static, str> {
529 self.try_attr_ctx(key, attr, args)
530 .unwrap_or_else(|| Cow::Owned(format!("{key}.{attr}")))
531 }
532
533 #[must_use]
534 pub fn fonts(&self) -> &Fonts { &self.active.fonts }
535
536 #[must_use]
537 pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata }
538}
539
540impl LocalizationHandle {
541 pub fn set_english_fallback(&mut self, use_english_fallback: bool) {
542 self.use_english_fallback = use_english_fallback;
543 }
544
545 #[must_use]
546 pub fn read(&self) -> LocalizationGuard {
547 LocalizationGuard {
548 active: self.active.read(),
549 fallback: if self.use_english_fallback {
550 self.fallback.map(|f| f.read())
551 } else {
552 None
553 },
554 }
555 }
556
557 pub fn load(specifier: &str) -> Result<Self, assets::Error> {
560 let default_key = ["voxygen.i18n.", REFERENCE_LANG].concat();
561 let language_key = ["voxygen.i18n.", specifier].concat();
562 let is_default = language_key == default_key;
563 let active = Language::load(&language_key)?;
564 Ok(Self {
565 active,
566 watcher: active.reload_watcher(),
567 fallback: if is_default {
568 None
569 } else {
570 Some(Language::load(&default_key)?)
571 },
572 use_english_fallback: false,
573 })
574 }
575
576 #[must_use]
577 pub fn load_expect(specifier: &str) -> Self {
578 Self::load(specifier).expect("Can't load language files")
579 }
580
581 pub fn reloaded(&mut self) -> bool { self.watcher.reloaded() }
582}
583
584struct FindManifests;
585
586impl assets::DirLoadable for FindManifests {
587 fn select_ids(
588 cache: assets::AnyCache,
589 specifier: &SharedString,
590 ) -> io::Result<Vec<SharedString>> {
591 use assets::Source;
592
593 let mut specifiers = Vec::new();
594
595 let source = cache.raw_source();
596 source.read_dir(specifier, &mut |entry| {
597 if let DirEntry::Directory(spec) = entry {
598 let manifest_spec = [spec, ".", "_manifest"].concat();
599
600 if source.exists(DirEntry::File(&manifest_spec, "ron")) {
601 specifiers.push(manifest_spec.into());
602 }
603 }
604 })?;
605
606 Ok(specifiers)
607 }
608}
609
610#[derive(Clone, Debug)]
611struct LocalizationList(Vec<LanguageMetadata>);
612
613impl assets::Compound for LocalizationList {
614 fn load(cache: assets::AnyCache, specifier: &SharedString) -> Result<Self, assets::BoxedError> {
615 let languages = assets::load_rec_dir::<FindManifests>(specifier)
617 .unwrap_or_else(|e| panic!("Failed to get manifests from {}: {:?}", specifier, e))
618 .read()
619 .ids()
620 .filter_map(|spec| cache.load::<raw::Manifest>(spec).ok())
621 .map(|localization| localization.read().metadata.clone())
622 .collect();
623
624 Ok(LocalizationList(languages))
625 }
626}
627
628#[must_use]
630pub fn list_localizations() -> Vec<LanguageMetadata> {
631 let LocalizationList(list) = LocalizationList::load_expect_cloned("voxygen.i18n");
632 list
633}
634
635#[cfg(test)]
636mod tests {
637 use super::*;
638
639 #[test]
640 fn check_localization_list() {
642 let list = list_localizations();
643 assert!(!list.is_empty());
644 }
645
646 #[test]
647 fn validate_reference_language() { let _ = LocalizationHandle::load_expect(REFERENCE_LANG); }
649
650 #[test]
651 fn validate_all_localizations() {
653 let list = list_localizations();
654 for meta in list {
655 let _ = LocalizationHandle::load_expect(&meta.language_identifier);
656 }
657 }
658
659 #[test]
660 fn test_strict_all_localizations() {
661 use analysis::{Language, ReferenceLanguage};
662 use assets::find_root;
663
664 let root = find_root().unwrap();
665 let i18n_directory = root.join("assets/voxygen/i18n");
666 let reference = ReferenceLanguage::at(&i18n_directory.join(REFERENCE_LANG));
667
668 let list = list_localizations();
669
670 for meta in list {
671 let code = meta.language_identifier;
672 let lang = Language {
673 code: code.clone(),
674 path: i18n_directory.join(code.clone()),
675 };
676 reference.compare_with(&lang);
679 }
680 }
681}