veloren_client_i18n/
lib.rs

1mod error;
2mod raw;
3
4use error::ResourceErr;
5
6#[cfg(any(feature = "bin", feature = "stat", test))]
7pub mod analysis;
8
9use fluent_bundle::{FluentResource, bundle::FluentBundle};
10use intl_memoizer::concurrent::IntlLangMemoizer;
11use unic_langid::LanguageIdentifier;
12
13use hashbrown::HashMap;
14use serde::{Deserialize, Serialize};
15use std::{borrow::Cow, io};
16
17use assets::{
18    AssetExt, AssetHandle, AssetReadGuard, ReloadWatcher, SharedString, source::DirEntry,
19};
20use common_assets as assets;
21use common_i18n::{Content, LocalizationArg};
22use tracing::warn;
23
24// Re-export for argument creation
25pub use fluent::{FluentValue, fluent_args};
26pub use fluent_bundle::FluentArgs;
27
28/// The reference language, aka the more up-to-date localization data.
29/// Also the default language at first startup.
30pub const REFERENCE_LANG: &str = "en";
31
32/// How a language can be described
33#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
34pub struct LanguageMetadata {
35    /// A human friendly language name (e.g. "English (US)")
36    pub language_name: String,
37
38    /// A short text identifier for this language (e.g. "en_US")
39    ///
40    /// On the opposite of `language_name` that can change freely,
41    /// `language_identifier` value shall be stable in time as it
42    /// is used by setting components to store the language
43    /// selected by the user.
44    pub language_identifier: String,
45}
46
47/// Store font metadata
48#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
49pub struct Font {
50    /// Key to retrieve the font in the asset system
51    pub asset_key: String,
52
53    /// Scale ratio to resize the UI text dynamically
54    scale_ratio: f32,
55}
56
57impl Font {
58    /// Scale input size to final UI size
59    #[must_use]
60    pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 }
61}
62
63/// Store font metadata
64pub type Fonts = HashMap<String, Font>;
65
66/// Store internationalization data
67struct Language {
68    /// The bundle storing all localized texts
69    pub(crate) bundle: FluentBundle<FluentResource, IntlLangMemoizer>,
70
71    /// Font configuration is stored here
72    pub(crate) fonts: Fonts,
73    pub(crate) metadata: LanguageMetadata,
74}
75
76impl Language {
77    fn try_msg<'a>(&'a self, key: &str, args: Option<&'a FluentArgs>) -> Option<Cow<'a, str>> {
78        let bundle = &self.bundle;
79        let msg = bundle.get_message(key)?;
80        let mut errs = Vec::new();
81        let msg = bundle.format_pattern(msg.value()?, args, &mut errs);
82        for err in errs {
83            tracing::error!("err: {err} for {key}");
84        }
85
86        Some(msg)
87    }
88
89    fn try_attr<'a>(
90        &'a self,
91        key: &str,
92        attr: &str,
93        args: Option<&'a FluentArgs>,
94    ) -> Option<Cow<'a, str>> {
95        let bundle = &self.bundle;
96        let msg = bundle.get_message(key)?;
97        let attr = msg.get_attribute(attr)?;
98        let attr = attr.value();
99
100        let mut errs = Vec::new();
101        let msg = bundle.format_pattern(attr, args, &mut errs);
102        for err in errs {
103            tracing::error!("err: {err} for {key}");
104        }
105
106        Some(msg)
107    }
108
109    /// NOTE: Exists for legacy reasons, avoid.
110    // Read more in the issue on get_variation at Gitlab
111    fn try_variation<'a>(
112        &'a self,
113        key: &str,
114        seed: u16,
115        args: Option<&'a FluentArgs>,
116    ) -> Option<Cow<'a, str>> {
117        let bundle = &self.bundle;
118        let msg = bundle.get_message(key)?;
119
120        let mut errs = Vec::new();
121
122        let attrs: Vec<_> = msg.attributes().collect();
123        let msg = if !attrs.is_empty() {
124            let idx = usize::from(seed) % attrs.len();
125            // unwrap is ok here, because idx is bound to attrs.len()
126            // by using modulo operator.
127            //
128            // For example:
129            // (I)
130            // * attributes = [.x = 5, .y = 7, z. = 4]
131            // * len = 3
132            // * seed can be 12, 50, 1
133            // 12 % 3 = 0, attrs.skip(0) => first element
134            // 50 % 3 = 2, attrs.skip(2) => third element
135            // 1 % 3 = 1, attrs.skip(1) => second element
136            // (II)
137            // * attributes = []
138            // * len = 0
139            // * no matter what seed is, we return None in code above
140            let variation = &attrs[idx];
141            bundle.format_pattern(variation.value(), args, &mut errs)
142        } else {
143            // Fall back to single message if there are no attributes
144            bundle.format_pattern(msg.value()?, args, &mut errs)
145        };
146
147        for err in errs {
148            tracing::error!("err: {err} for {key}");
149        }
150
151        Some(msg)
152    }
153}
154impl assets::Compound for Language {
155    fn load(cache: assets::AnyCache, path: &SharedString) -> Result<Self, assets::BoxedError> {
156        let manifest = cache
157            .load::<raw::Manifest>(&[path, ".", "_manifest"].concat())?
158            .cloned();
159        let raw::Manifest {
160            mut fonts,
161            metadata,
162        } = manifest;
163
164        let lang_id: LanguageIdentifier = metadata.language_identifier.parse()?;
165        let mut bundle = FluentBundle::new_concurrent(vec![lang_id]);
166
167        // Here go dragons
168        for id in cache.load_rec_dir::<raw::Resource>(path)?.read().ids() {
169            match cache.load(id) {
170                Ok(handle) => {
171                    let source: &raw::Resource = &handle.read();
172                    let src = source.src.clone();
173
174                    let resource = FluentResource::try_new(src).map_err(|(_ast, errs)| {
175                        ResourceErr::parsing_error(errs, id.to_string(), &source.src)
176                    })?;
177
178                    bundle
179                        .add_resource(resource)
180                        .map_err(|e| ResourceErr::BundleError(format!("{e:?}")))?;
181                },
182                Err(err) => {
183                    // TODO: shouldn't we just panic here?
184                    warn!("Unable to load asset {id}, error={err:?}");
185                },
186            }
187        }
188
189        // NOTE:
190        // Basically a hack, but conrod can't use isolation marks yet.
191        // Veloren Issue 1649
192        bundle.set_use_isolating(false);
193
194        // Add a universal fallback-ish font, that's supposed to cover all
195        // languages.
196        // Use it for language menu, chat, etc.
197        //
198        // At the moment, covers all languages except Korean, so Korean uses
199        // different font here.
200        fonts.entry("universal".to_owned()).or_insert(Font {
201            asset_key: "voxygen.font.GoNotoCurrent".to_owned(),
202            scale_ratio: 1.0,
203        });
204
205        Ok(Self {
206            bundle,
207            fonts,
208            metadata,
209        })
210    }
211}
212
213/// The central data structure to handle localization in Veloren
214// inherit Copy + Clone from AssetHandle (what?)
215#[derive(Copy, Clone)]
216pub struct LocalizationHandle {
217    active: AssetHandle<Language>,
218    watcher: ReloadWatcher,
219    fallback: Option<AssetHandle<Language>>,
220    pub use_english_fallback: bool,
221}
222
223/// Read [`LocalizationGuard`]
224// arbitrary choice to minimize changing all of veloren
225pub type Localization = LocalizationGuard;
226
227/// RAII guard returned from [`LocalizationHandle::read()`], resembles
228/// [`AssetGuard`]
229pub struct LocalizationGuard {
230    active: AssetReadGuard<Language>,
231    fallback: Option<AssetReadGuard<Language>>,
232}
233
234impl LocalizationGuard {
235    /// Get a localized text from the given key in the fallback language.
236    pub fn try_fallback_msg(&self, key: &str) -> Option<Cow<str>> {
237        self.fallback.as_ref().and_then(|fb| fb.try_msg(key, None))
238    }
239
240    /// Get a localized text from the given key
241    ///
242    /// First lookup is done in the active language, second in
243    /// the fallback (if present).
244    pub fn try_msg(&self, key: &str) -> Option<Cow<str>> {
245        self.active
246            .try_msg(key, None)
247            .or_else(|| self.try_fallback_msg(key))
248    }
249
250    /// Get a localized text from the given key
251    ///
252    /// First lookup is done in the active language, second in
253    /// the fallback (if present).
254    /// If the key is not present in the localization object
255    /// then the key itself is returned.
256    pub fn get_msg(&self, key: &str) -> Cow<str> {
257        // NOTE: we clone the key if translation was missing
258        // We could use borrowed version, but it would mean that
259        // `key`, `self`, and result should have the same lifetime.
260        // Which would make it way more awkward to use with runtime generated keys.
261        self.try_msg(key)
262            .unwrap_or_else(|| Cow::Owned(key.to_owned()))
263    }
264
265    /// Get a localized text from the given key using given arguments
266    ///
267    /// First lookup is done in the active language, second in
268    /// the fallback (if present).
269    pub fn try_msg_ctx<'a>(&'a self, key: &str, args: &'a FluentArgs) -> Option<Cow<'static, str>> {
270        // NOTE: as after using args we get our result owned (because you need
271        // to clone pattern during forming value from args), this conversion
272        // to Cow::Owned is no-op.
273        // We could use String here, but using Cow everywhere in i18n API is
274        // prefered for consistency.
275        self.active
276            .try_msg(key, Some(args))
277            .or_else(|| {
278                self.fallback
279                    .as_ref()
280                    .and_then(|fb| fb.try_msg(key, Some(args)))
281            })
282            .map(|res| Cow::Owned(res.into_owned()))
283    }
284
285    /// Get a localized text from the given key using given arguments
286    ///
287    /// First lookup is done in the active language, second in
288    /// the fallback (if present).
289    /// If the key is not present in the localization object
290    /// then the key itself is returned.
291    pub fn get_msg_ctx<'a>(&'a self, key: &str, args: &'a FluentArgs) -> Cow<'static, str> {
292        self.try_msg_ctx(key, args)
293            .unwrap_or_else(|| Cow::Owned(key.to_owned()))
294    }
295
296    /// NOTE: Exists for legacy reasons, avoid.
297    ///
298    /// Get a localized text from the variation of given key
299    ///
300    /// First lookup is done in the active language, second in
301    /// the fallback (if present).
302    // Read more in the issue on get_variation at Gitlab
303    pub fn try_variation(&self, key: &str, seed: u16) -> Option<Cow<str>> {
304        self.active.try_variation(key, seed, None).or_else(|| {
305            self.fallback
306                .as_ref()
307                .and_then(|fb| fb.try_variation(key, seed, None))
308        })
309    }
310
311    /// NOTE: Exists for legacy reasons, avoid.
312    ///
313    /// Get a localized text from the variation of given key
314    ///
315    /// First lookup is done in the active language, second in
316    /// the fallback (if present).
317    /// If the key is not present in the localization object
318    /// then the key itself is returned.
319    // Read more in the issue on get_variation at Gitlab
320    pub fn get_variation(&self, key: &str, seed: u16) -> Cow<str> {
321        self.try_variation(key, seed)
322            .unwrap_or_else(|| Cow::Owned(key.to_owned()))
323    }
324
325    /// NOTE: Exists for legacy reasons, avoid.
326    ///
327    /// Get a localized text from the variation of given key with given
328    /// arguments
329    ///
330    /// First lookup is done in the active language, second in
331    /// the fallback (if present).
332    // Read more in the issue on get_variation at Gitlab
333    pub fn try_variation_ctx<'a>(
334        &'a self,
335        key: &str,
336        seed: u16,
337        args: &'a FluentArgs,
338    ) -> Option<Cow<'a, str>> {
339        self.active
340            .try_variation(key, seed, Some(args))
341            .or_else(|| {
342                self.fallback
343                    .as_ref()
344                    .and_then(|fb| fb.try_variation(key, seed, Some(args)))
345            })
346    }
347
348    // Function to localize content for given language.
349    //
350    // Returns Ok(localized_text) if found no errors.
351    // Returns Err(broken_text) on failure.
352    //
353    // broken_text will have i18n keys in it, just i18n key if it was instant miss
354    // or text with missed keys inlined if it was missed down the chain.
355    fn get_content_for_lang(lang: &Language, content: &Content) -> Result<String, String> {
356        match content {
357            Content::Plain(text) => Ok(text.clone()),
358            Content::Key(key) => lang
359                .try_msg(key, None)
360                .map(Cow::into_owned)
361                .ok_or_else(|| key.to_string()),
362            Content::Attr(key, attr) => lang
363                .try_attr(key, attr, None)
364                .map(Cow::into_owned)
365                .ok_or_else(|| format!("{key}.{attr}")),
366            Content::Localized { key, seed, args } => {
367                // flag to detect failure down the chain
368                let mut is_arg_failure = false;
369
370                let mut fargs = FluentArgs::new();
371                for (k, arg) in args {
372                    let arg_val = match arg {
373                        LocalizationArg::Content(content) => {
374                            let arg_res = Self::get_content_for_lang(lang, content)
375                                .unwrap_or_else(|broken_text| {
376                                    is_arg_failure = true;
377                                    broken_text
378                                })
379                                .into();
380
381                            FluentValue::String(arg_res)
382                        },
383                        LocalizationArg::Nat(n) => FluentValue::from(n),
384                    };
385                    fargs.set(k, arg_val);
386                }
387
388                lang.try_variation(key, *seed, Some(&fargs))
389                    .map(Cow::into_owned)
390                    .ok_or_else(|| key.clone())
391                    .and_then(|text| if is_arg_failure { Err(text) } else { Ok(text) })
392            },
393        }
394    }
395
396    /// Tries its best to localize compound message.
397    ///
398    /// # Example
399    /// ```text
400    /// Content::Localized { "npc-speech-tell_site", seed, {
401    ///     "dir" => Content::Localized("npc-speech-dir_north", seed, {})
402    ///     "dist" => Content::Localized("npc-speech-dist_very_far", seed, {})
403    ///     "site" => Content::Plain(site)
404    /// }}
405    /// ```
406    /// ```fluent
407    /// npc-speech-tell_site =
408    ///    .a0 = Have you visited { $site }? It's just { $dir } of here!
409    ///    .a1 = You should visit { $site } some time.
410    ///    .a2 = If you travel { $dist } to the { $dir }, you can get to { $site }.
411    ///    .a3 = To the { $dir } you'll find { $site }, it's { $dist }.
412    ///
413    /// npc-speech-dir_north = north
414    /// # ... other keys
415    ///
416    /// npc-speech-dist_very_far = very far away
417    /// # ... other keys
418    /// ```
419    ///
420    /// 1) Because content we want is localized itself and has arguments, we
421    ///    iterate over them and localize, recursively. Having that, we localize
422    ///    our content.
423    /// 2) Now there is a chance that some of args have missing internalization.
424    ///    In that case, we insert arg name as placeholder and mark it as
425    ///    broken. Then we repeat *whole* procedure on fallback language if we
426    ///    have it.
427    /// 3) Otherwise, return result from (1).
428    // NOTE: it's important that we only use one language at the time, because
429    // otherwise we will get partially-translated message.
430    //
431    // TODO: return Cow<str>?
432    pub fn get_content(&self, content: &Content) -> String {
433        match Self::get_content_for_lang(&self.active, content) {
434            Ok(text) => text,
435            // If localisation or some part of it failed, repeat with fallback.
436            // If it did fail as well, it's probably because fallback was disabled,
437            // so we don't have better option other than returning broken text
438            // we produced earlier.
439            Err(broken_text) => self
440                .fallback
441                .as_ref()
442                .and_then(|fb| Self::get_content_for_lang(fb, content).ok())
443                .unwrap_or(broken_text),
444        }
445    }
446
447    pub fn get_content_fallback(&self, content: &Content) -> String {
448        self.fallback
449            .as_ref()
450            .map(|fb| Self::get_content_for_lang(fb, content))
451            .transpose()
452            .map(|msg| msg.unwrap_or_default())
453            .unwrap_or_else(|e| e)
454    }
455
456    /// NOTE: Exists for legacy reasons, avoid.
457    ///
458    /// Get a localized text from the variation of given key with given
459    /// arguments
460    ///
461    /// First lookup is done in the active language, second in
462    /// the fallback (if present).
463    /// If the key is not present in the localization object
464    /// then the key itself is returned.
465    // Read more in the issue on get_variation at Gitlab
466    pub fn get_variation_ctx<'a>(
467        &'a self,
468        key: &str,
469        seed: u16,
470        args: &'a FluentArgs,
471    ) -> Cow<'a, str> {
472        self.try_variation_ctx(key, seed, args)
473            .unwrap_or_else(|| Cow::Owned(key.to_owned()))
474    }
475
476    /// Get a localized text from the given key by given attribute
477    ///
478    /// First lookup is done in the active language, second in
479    /// the fallback (if present).
480    pub fn try_attr(&self, key: &str, attr: &str) -> Option<Cow<str>> {
481        self.active.try_attr(key, attr, None).or_else(|| {
482            self.fallback
483                .as_ref()
484                .and_then(|fb| fb.try_attr(key, attr, None))
485        })
486    }
487
488    /// Get a localized text from the given key by given attribute
489    ///
490    /// First lookup is done in the active language, second in
491    /// the fallback (if present).
492    /// If the key is not present in the localization object
493    /// then the key itself is returned.
494    pub fn get_attr(&self, key: &str, attr: &str) -> Cow<str> {
495        self.try_attr(key, attr)
496            .unwrap_or_else(|| Cow::Owned(format!("{key}.{attr}")))
497    }
498
499    /// Get a localized text from the given key by given attribute and arguments
500    ///
501    /// First lookup is done in the active language, second in
502    /// the fallback (if present).
503    pub fn try_attr_ctx(
504        &self,
505        key: &str,
506        attr: &str,
507        args: &FluentArgs,
508    ) -> Option<Cow<'static, str>> {
509        // NOTE: we explicitly Own result, because in 99.999% cases it got
510        // owned during formatting of arguments, hence it's a no-op, but makes
511        // using this function much easier
512        self.active
513            .try_attr(key, attr, Some(args))
514            .or_else(|| {
515                self.fallback
516                    .as_ref()
517                    .and_then(|fb| fb.try_attr(key, attr, Some(args)))
518            })
519            .map(|res| Cow::Owned(res.into_owned()))
520    }
521
522    /// Get a localized text from the given key by given attribute and arguments
523    ///
524    /// First lookup is done in the active language, second in
525    /// the fallback (if present).
526    /// If the key is not present in the localization object
527    /// then the key itself is returned.
528    pub fn get_attr_ctx(&self, key: &str, attr: &str, args: &FluentArgs) -> Cow<'static, str> {
529        self.try_attr_ctx(key, attr, args)
530            .unwrap_or_else(|| Cow::Owned(format!("{key}.{attr}")))
531    }
532
533    #[must_use]
534    pub fn fonts(&self) -> &Fonts { &self.active.fonts }
535
536    #[must_use]
537    pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata }
538}
539
540impl LocalizationHandle {
541    pub fn set_english_fallback(&mut self, use_english_fallback: bool) {
542        self.use_english_fallback = use_english_fallback;
543    }
544
545    #[must_use]
546    pub fn read(&self) -> LocalizationGuard {
547        LocalizationGuard {
548            active: self.active.read(),
549            fallback: if self.use_english_fallback {
550                self.fallback.map(|f| f.read())
551            } else {
552                None
553            },
554        }
555    }
556
557    /// # Errors
558    /// Returns error if active of fallback language can't be loaded
559    pub fn load(specifier: &str) -> Result<Self, assets::Error> {
560        let default_key = ["voxygen.i18n.", REFERENCE_LANG].concat();
561        let language_key = ["voxygen.i18n.", specifier].concat();
562        let is_default = language_key == default_key;
563        let active = Language::load(&language_key)?;
564        Ok(Self {
565            active,
566            watcher: active.reload_watcher(),
567            fallback: if is_default {
568                None
569            } else {
570                Some(Language::load(&default_key)?)
571            },
572            use_english_fallback: false,
573        })
574    }
575
576    #[must_use]
577    pub fn load_expect(specifier: &str) -> Self {
578        Self::load(specifier).expect("Can't load language files")
579    }
580
581    pub fn reloaded(&mut self) -> bool { self.watcher.reloaded() }
582}
583
584struct FindManifests;
585
586impl assets::DirLoadable for FindManifests {
587    fn select_ids(
588        cache: assets::AnyCache,
589        specifier: &SharedString,
590    ) -> io::Result<Vec<SharedString>> {
591        use assets::Source;
592
593        let mut specifiers = Vec::new();
594
595        let source = cache.raw_source();
596        source.read_dir(specifier, &mut |entry| {
597            if let DirEntry::Directory(spec) = entry {
598                let manifest_spec = [spec, ".", "_manifest"].concat();
599
600                if source.exists(DirEntry::File(&manifest_spec, "ron")) {
601                    specifiers.push(manifest_spec.into());
602                }
603            }
604        })?;
605
606        Ok(specifiers)
607    }
608}
609
610#[derive(Clone, Debug)]
611struct LocalizationList(Vec<LanguageMetadata>);
612
613impl assets::Compound for LocalizationList {
614    fn load(cache: assets::AnyCache, specifier: &SharedString) -> Result<Self, assets::BoxedError> {
615        // List language directories
616        let languages = assets::load_rec_dir::<FindManifests>(specifier)
617            .unwrap_or_else(|e| panic!("Failed to get manifests from {}: {:?}", specifier, e))
618            .read()
619            .ids()
620            .filter_map(|spec| cache.load::<raw::Manifest>(spec).ok())
621            .map(|localization| localization.read().metadata.clone())
622            .collect();
623
624        Ok(LocalizationList(languages))
625    }
626}
627
628/// Load all the available languages located in the voxygen asset directory
629#[must_use]
630pub fn list_localizations() -> Vec<LanguageMetadata> {
631    let LocalizationList(list) = LocalizationList::load_expect_cloned("voxygen.i18n");
632    list
633}
634
635#[cfg(test)]
636mod tests {
637    use super::*;
638
639    #[test]
640    // Test that localization list is loaded (not empty)
641    fn check_localization_list() {
642        let list = list_localizations();
643        assert!(!list.is_empty());
644    }
645
646    #[test]
647    // Test that reference language can be loaded
648    fn validate_reference_language() { let _ = LocalizationHandle::load_expect(REFERENCE_LANG); }
649
650    #[test]
651    // Test to verify that all languages are valid and loadable
652    fn validate_all_localizations() {
653        let list = list_localizations();
654        for meta in list {
655            let _ = LocalizationHandle::load_expect(&meta.language_identifier);
656        }
657    }
658
659    #[test]
660    fn test_strict_all_localizations() {
661        use analysis::{Language, ReferenceLanguage};
662        use assets::find_root;
663
664        let root = find_root().unwrap();
665        let i18n_directory = root.join("assets/voxygen/i18n");
666        let reference = ReferenceLanguage::at(&i18n_directory.join(REFERENCE_LANG));
667
668        let list = list_localizations();
669
670        for meta in list {
671            let code = meta.language_identifier;
672            let lang = Language {
673                code: code.clone(),
674                path: i18n_directory.join(code.clone()),
675            };
676            // TODO: somewhere here should go check that all needed
677            // versions are given
678            reference.compare_with(&lang);
679        }
680    }
681}