Add basic hardcoded Any-Hex transliterators

experimental/transliteration/src/transliterator/hardcoded.rs

@@ -0,0 +1,86 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+//! This module defines implementations for code-based transliterators that are part of
+//! transform rules.
+
+use crate::transliterator::replaceable::{Forward, Replaceable, Utf8Matcher};
+
+/// A transliterator that replaces every character with its `case`-case hexadecimal representation,
+/// 0-padded to `min_length`, and surrounded by `prefix` and `suffix`.
+#[derive(Debug)]
+pub(super) struct HexTransliterator {
+    prefix: &'static str,
+    suffix: &'static str,
+    min_length: u8,
+    case: Case,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum Case {
+    Upper,
+    Lower,
+}
+
+impl HexTransliterator {
+    pub(super) fn new(
+        prefix: &'static str,
+        suffix: &'static str,
+        min_length: u8,
+        case: Case,
+    ) -> Self {
+        Self {
+            prefix,
+            suffix,
+            min_length,
+            case,
+        }
+    }
+
+    pub(super) fn transliterate(&self, mut rep: Replaceable) {
+        while !rep.is_finished() {
+            let mut matcher = rep.start_match();
+            // Thought: ok this fully specified path is annoying, maybe a separate API surface is
+            // better for Forward vs Reverse matching.
+            let c = Utf8Matcher::<Forward>::next_char(&matcher);
+            // there must always be a char, because we just checked that `rep` is not finished yet.
+            let c = c.unwrap();
+            Utf8Matcher::<Forward>::match_and_consume_char(&mut matcher, c);
+            let mut dest = matcher.finish_match();
+
+            let c_u32 = c as u32;
+            // rounding-up division by 4
+            let length = (u32::BITS - c_u32.leading_zeros() + 3) / 4;
+            let padding = self.min_length.saturating_sub(length as u8);
+            dest.apply_size_hint(
+                self.prefix.len() + padding as usize + length as usize + self.suffix.len(),
+            );
+
+            dest.push_str(self.prefix);
+            for _ in 0..padding {
+                dest.push_str("0");
+            }
+            let mut remaining_c = c_u32;
+            // temporary buffer because forward iteration through a u32's bytes is easier and
+            // we need the reverse order
+            let mut buf = [0; 6];
+            for slot in buf.iter_mut() {
+                if c_u32 == 0 {
+                    break;
+                }
+                *slot = match remaining_c & 0xF {
+                    x @ 0x0..=0x9 => b'0' + x as u8,
+                    x @ 0xA..=0xF if self.case == Case::Lower => b'a' + (x - 0xA) as u8,
+                    x => b'A' + (x - 0xA) as u8,
+                };
+                remaining_c >>= 4;
+            }
+            // only `length` hex digits are actually from the char
+            for c in buf[..length as usize].iter().rev() {
+                dest.push(*c as char);
+            }
+            dest.push_str(self.suffix);
+        }
+    }
+}

experimental/transliteration/src/transliterator/mod.rs

@@ -2,11 +2,13 @@
 // called LICENSE at the top level of the ICU4X source tree
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
+mod hardcoded;
 #[allow(clippy::indexing_slicing, clippy::unwrap_used)] // TODO(#3958): Remove.
 mod replaceable;
 
 use crate::provider::{FunctionCall, Rule, RuleULE, SimpleId, VarTable};
 use crate::provider::{RuleBasedTransliterator, Segment, TransliteratorRulesV1Marker};
+use crate::transliterator::hardcoded::Case;
 use crate::TransliteratorError;
 use alloc::boxed::Box;
 use alloc::string::{String, ToString};
@@ -123,6 +125,7 @@ enum InternalTransliterator {
     RuleBased(DataPayload<TransliteratorRulesV1Marker>),
     Composing(ComposingTransliterator),
     Decomposing(DecomposingTransliterator),
+    Hex(hardcoded::HexTransliterator),
     Null,
     Remove,
     Dyn(Box<dyn CustomTransliterator>),
@@ -135,6 +138,7 @@ impl InternalTransliterator {
             // TODO(#3910): internal hardcoded transliterators
             Self::Composing(t) => t.transliterate(rep, env),
             Self::Decomposing(t) => t.transliterate(rep, env),
+            Self::Hex(t) => t.transliterate(rep),
             Self::Null => (),
             Self::Remove => rep.replace_modifiable_with_str(""),
             Self::Dyn(custom) => {
@@ -392,6 +396,21 @@ impl Transliterator {
             )),
             "any-null" => Ok(InternalTransliterator::Null),
             "any-remove" => Ok(InternalTransliterator::Remove),
+            "any-hex-unicode" => Ok(InternalTransliterator::Hex(
+                hardcoded::HexTransliterator::new("U+", "", 4, Case::Upper),
+            )),
+            "any-hex-rust" => Ok(InternalTransliterator::Hex(
+                hardcoded::HexTransliterator::new("\\u{", "}", 2, Case::Lower),
+            )),
+            "any-hex-xml" => Ok(InternalTransliterator::Hex(
+                hardcoded::HexTransliterator::new("&#x", ";", 1, Case::Upper),
+            )),
+            "any-hex-perl" => Ok(InternalTransliterator::Hex(
+                hardcoded::HexTransliterator::new("\\x{", "}", 1, Case::Upper),
+            )),
+            "any-hex-plain" => Ok(InternalTransliterator::Hex(
+                hardcoded::HexTransliterator::new("", "", 4, Case::Upper),
+            )),
             s => Err(DataError::custom("unavailable transliterator")
                 .with_debug_context(s)
                 .into()),
@@ -1364,4 +1383,22 @@ mod tests {
         let output = "aa";
         assert_eq!(t.transliterate(input.to_string()), output);
     }
+
+    #[test]
+    fn test_hex_rust() {
+        let t = Transliterator::try_new("und-t-und-s0-test-d0-test-m0-hexrust".parse().unwrap())
+            .unwrap();
+        let input = "\0äa\u{10FFFF}❤!";
+        let output = r"\u{00}\u{e4}\u{61}\u{10ffff}\u{2764}\u{21}";
+        assert_eq!(t.transliterate(input.to_string()), output);
+    }
+
+    #[test]
+    fn test_hex_unicode() {
+        let t = Transliterator::try_new("und-t-und-s0-test-d0-test-m0-hexuni".parse().unwrap())
+            .unwrap();
+        let input = "\0äa\u{10FFFF}❤!";
+        let output = "U+0000U+00E4U+0061U+10FFFFU+2764U+0021";
+        assert_eq!(t.transliterate(input.to_string()), output);
+    }
 }

experimental/transliteration/src/transliterator/replaceable.rs

@@ -427,6 +427,10 @@ impl<'a, 'b> RepMatcher<'a, 'b, true> {
 
 // we can only finish matching the key once
 impl<'a, 'b> RepMatcher<'a, 'b, false> {
+    pub(super) fn finish_match(self) -> Insertable<'a, 'b> {
+        Insertable::from_matcher(self.finish_key())
+    }
+
     pub(super) fn finish_key(self) -> RepMatcher<'a, 'b, true> {
         RepMatcher {
             rep: self.rep,

provider/datagen/tests/data/postcard/fingerprints.csv

@@ -2057,6 +2057,8 @@ transliterator/rules@1, und+und-t-und-Beng-d0-intindic, 2621B, 24a04df29d08559d
 transliterator/rules@1, und+und-t-und-Latn-d0-ascii, 27110B, c66743617e3238ff
 transliterator/rules@1, und+und-t-und-d0-test-m0-cursfilt-s0-test, 93B, ac67e05bc986cd23
 transliterator/rules@1, und+und-t-und-d0-test-m0-emtymach-s0-test, 105B, 12b65cade4ce4468
+transliterator/rules@1, und+und-t-und-d0-test-m0-hexrust-s0-test, 98B, b8802989a6bfec0f
+transliterator/rules@1, und+und-t-und-d0-test-m0-hexuni-s0-test, 104B, 4335c71013bd81d
 transliterator/rules@1, und+und-t-und-d0-test-m0-niels-s0-test, 1800B, 6a560a4143a4b60c
 transliterator/rules@1, und+und-t-und-d0-test-m0-rectesta-s0-test, 370B, af652bcb33e1038b
 transliterator/rules@1, und+und-t-und-d0-test-m0-rectestr-s0-test, 281B, 51be7571fd233bd6