feat: support use pinyin filter (#45821)

relate: https://github.com/milvus-io/milvus/issues/45811
design doc:
https://github.com/milvus-io/milvus-design-docs/blob/main/design_docs/20260209-pinyin_filter.md

Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>

internal/core/thirdparty/tantivy/tantivy-binding/Cargo.lock

@@ -3113,6 +3113,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pinyin"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16f2611cd06a1ac239a0cea4521de9eb068a6ca110324ee00631aa68daa74fc0"
+
 [[package]]
 name = "pkg-config"
 version = "0.3.32"
@@ -4028,6 +4034,7 @@ dependencies = [
  "log",
  "md5",
  "once_cell",
+ "pinyin",
  "prost",
  "rand 0.7.3",
  "rand 0.9.2",

internal/core/thirdparty/tantivy/tantivy-binding/Cargo.toml

@@ -49,6 +49,7 @@ prost = "0.13.5"
 once_cell = "1.20.3"
 unicode-general-category = "1.0.0"
 unicode_categories = "0.1.1"
+pinyin = "0.10"
 
 # lindera dependencies for fetch and prepare dictionary online.
 tokio = { version = "1.45.0", features = [

internal/core/thirdparty/tantivy/tantivy-binding/src/analyzer/filter/filter.rs

@@ -1,8 +1,10 @@
 use serde_json as json;
 use tantivy::tokenizer::*;
 
+use super::util::*;
 use super::{
-    CnAlphaNumOnlyFilter, CnCharOnlyFilter, RegexFilter, RemovePunctFilter, SynonymFilter,
+    CnAlphaNumOnlyFilter, CnCharOnlyFilter, PinyinFilter, RegexFilter, RemovePunctFilter,
+    SynonymFilter,
 };
 use crate::analyzer::options::FileResourcePathHelper;
 use crate::error::{Result, TantivyBindingError};
@@ -21,6 +23,7 @@ pub(crate) enum SystemFilter {
     Stemmer(Stemmer),
     Regex(RegexFilter),
     Synonym(SynonymFilter),
+    Pinyin(PinyinFilter),
 }
 
 pub(crate) trait FilterBuilder {
@@ -47,6 +50,7 @@ impl SystemFilter {
             Self::RemovePunct(filter) => builder.filter(filter).dynamic(),
             Self::Regex(filter) => builder.filter(filter).dynamic(),
             Self::Synonym(filter) => builder.filter(filter).dynamic(),
+            Self::Pinyin(filter) => builder.filter(filter).dynamic(),
             Self::Invalid => builder,
         }
     }
@@ -108,6 +112,7 @@ impl From<&str> for SystemFilter {
             "cncharonly" => Self::CnCharOnly(CnCharOnlyFilter),
             "cnalphanumonly" => Self::CnAlphaNumOnly(CnAlphaNumOnlyFilter),
             "removepunct" => Self::RemovePunct(RemovePunctFilter),
+            "pinyin" => Self::Pinyin(PinyinFilter::default()),
             _ => Self::Invalid,
         }
     }
@@ -135,6 +140,7 @@ pub fn create_filter(
                 "synonym" => {
                     SynonymFilter::from_json(params, helper).map(|f| SystemFilter::Synonym(f))
                 }
+                "pinyin" => PinyinFilter::from_json(params).map(|f| SystemFilter::Pinyin(f)),
                 other => Err(TantivyBindingError::InternalError(format!(
                     "unsupport filter type: {}",
                     other

internal/core/thirdparty/tantivy/tantivy-binding/src/analyzer/filter/mod.rs

@@ -1,6 +1,7 @@
 mod cn_char_filter;
 mod decompounder_filter;
 mod filter;
+mod pinyin_filter;
 mod regex_filter;
 mod remove_punct_filter;
 mod stemmer_filter;
@@ -10,6 +11,7 @@ mod synonym_filter;
 mod util;
 
 pub(crate) use cn_char_filter::{CnAlphaNumOnlyFilter, CnCharOnlyFilter};
+use pinyin_filter::PinyinFilter;
 use regex_filter::RegexFilter;
 use remove_punct_filter::RemovePunctFilter;
 use synonym_filter::SynonymFilter;

internal/core/thirdparty/tantivy/tantivy-binding/src/analyzer/filter/pinyin_filter.rs

@@ -0,0 +1,312 @@
+use crate::error::{Result, TantivyBindingError};
+use pinyin::ToPinyin;
+use serde_json as json;
+use tantivy::tokenizer::{Token, TokenFilter, TokenStream, Tokenizer};
+
+#[derive(Clone)]
+pub struct PinyinOptions {
+    // Whether to keep the original token
+    keep_original: bool,
+    // Whether to keep full pinyin tokens for each chinese character
+    keep_full_pinyin: bool,
+    // Whether to keep joined full pinyin token for the whole chinese word
+    keep_joined_full_pinyin: bool,
+    // Whether to keep separate first letter tokens for whole chinese word
+    keep_separate_first_letter: bool,
+}
+
+impl PinyinOptions {
+    pub fn default() -> Self {
+        Self {
+            keep_original: true,
+            keep_full_pinyin: true,
+            keep_joined_full_pinyin: false,
+            keep_separate_first_letter: false,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct PinyinFilter {
+    options: PinyinOptions,
+}
+
+impl Default for PinyinFilter {
+    fn default() -> Self {
+        Self {
+            options: PinyinOptions::default(),
+        }
+    }
+}
+
+impl PinyinFilter {
+    pub fn from_json(value: &json::Map<String, json::Value>) -> Result<PinyinFilter> {
+        let mut options = PinyinOptions::default();
+
+        if let Some(v) = value.get("keep_original") {
+            options.keep_original = v.as_bool().ok_or_else(|| {
+                TantivyBindingError::InternalError(
+                    "keep_original must be a boolean value".to_string(),
+                )
+            })?;
+        }
+
+        if let Some(v) = value.get("keep_full_pinyin") {
+            options.keep_full_pinyin = v.as_bool().ok_or_else(|| {
+                TantivyBindingError::InternalError(
+                    "keep_full_pinyin must be a boolean value".to_string(),
+                )
+            })?;
+        }
+
+        if let Some(v) = value.get("keep_joined_full_pinyin") {
+            options.keep_joined_full_pinyin = v.as_bool().ok_or_else(|| {
+                TantivyBindingError::InternalError(
+                    "keep_joined_full_pinyin must be a boolean value".to_string(),
+                )
+            })?;
+        }
+
+        if let Some(v) = value.get("keep_separate_first_letter") {
+            options.keep_separate_first_letter = v.as_bool().ok_or_else(|| {
+                TantivyBindingError::InternalError(
+                    "keep_separate_first_letter must be a boolean value".to_string(),
+                )
+            })?;
+        }
+
+        Ok(Self { options })
+    }
+}
+
+impl TokenFilter for PinyinFilter {
+    type Tokenizer<T: Tokenizer> = PinyinFilterWrapper<T>;
+
+    fn transform<T: Tokenizer>(self, tokenizer: T) -> PinyinFilterWrapper<T> {
+        PinyinFilterWrapper {
+            inner: tokenizer,
+            options: self.options,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct PinyinFilterWrapper<T> {
+    inner: T,
+    options: PinyinOptions,
+}
+
+impl<T: Tokenizer> Tokenizer for PinyinFilterWrapper<T> {
+    type TokenStream<'a> = PinyinFilterStream<T::TokenStream<'a>>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        PinyinFilterStream {
+            options: self.options.clone(),
+            cache: Vec::new(),
+            index: 0,
+            tail: self.inner.token_stream(text),
+        }
+    }
+}
+
+pub struct PinyinFilterStream<T> {
+    options: PinyinOptions,
+    cache: Vec<Token>,
+    index: usize,
+    tail: T,
+}
+
+impl<T> PinyinFilterStream<T> {
+    pub fn cache_clean(&mut self) {
+        self.cache.clear();
+        self.index = 0;
+    }
+
+    fn has_cached_token(&self) -> bool {
+        self.index <= self.cache.len()
+    }
+}
+
+impl<T: TokenStream> TokenStream for PinyinFilterStream<T> {
+    fn advance(&mut self) -> bool {
+        self.index += 1;
+        while !self.has_cached_token() {
+            if !self.tail.advance() {
+                return false;
+            }
+            self.cache_clean();
+            self.index += 1;
+
+            if self.options.keep_original {
+                self.cache.push(self.tail.token().clone());
+            }
+
+            let mut join_pinyin = String::new();
+            let mut first_letter = String::new();
+            for (index, char) in self
+                .tail
+                .token()
+                .text
+                .as_str()
+                .to_pinyin()
+                .flatten()
+                .enumerate()
+            {
+                if self.options.keep_full_pinyin {
+                    let mut start_position = self.tail.token().position;
+                    if index <= self.tail.token().position_length {
+                        start_position = start_position + index;
+                    }
+                    self.cache.push(Token {
+                        text: char.plain().to_string(),
+                        offset_from: self.tail.token().offset_from,
+                        offset_to: self.tail.token().offset_to,
+                        position: start_position,
+                        position_length: 1,
+                    })
+                }
+
+                if self.options.keep_joined_full_pinyin {
+                    join_pinyin.push_str(char.plain());
+                }
+
+                if self.options.keep_separate_first_letter {
+                    first_letter.push_str(char.first_letter());
+                }
+            }
+            if self.options.keep_joined_full_pinyin && !join_pinyin.is_empty() {
+                self.cache.push(Token {
+                    text: join_pinyin,
+                    offset_from: self.tail.token().offset_from,
+                    offset_to: self.tail.token().offset_to,
+                    position: self.tail.token().position,
+                    position_length: self.tail.token().position_length,
+                })
+            }
+
+            if self.options.keep_separate_first_letter && !first_letter.is_empty() {
+                self.cache.push(Token {
+                    text: first_letter,
+                    offset_from: self.tail.token().offset_from,
+                    offset_to: self.tail.token().offset_to,
+                    position: self.tail.token().position,
+                    position_length: self.tail.token().position_length,
+                })
+            }
+        }
+
+        return true;
+    }
+
+    fn token(&self) -> &Token {
+        &self.cache[self.index - 1]
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        &mut self.cache[self.index - 1]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::analyzer::analyzer::create_analyzer;
+
+    fn is_subset<T: PartialEq>(subset: &[T], superset: &[T]) -> bool {
+        subset.iter().all(|item| superset.contains(item))
+    }
+
+    #[test]
+    fn test_pinyin_filter_with_joined_full_pinyin() {
+        let params = r#"{
+            "tokenizer": "jieba",
+            "filter": [{
+                "type": "pinyin",
+                "keep_original": true,
+                "keep_full_pinyin": false,
+                "keep_joined_full_pinyin": true,
+                "keep_separate_first_letter": false
+            }]
+        }"#;
+
+        let tokenizer = create_analyzer(&params.to_string(), "");
+        assert!(tokenizer.is_ok(), "error: {}", tokenizer.err().unwrap());
+
+        let mut bining = tokenizer.unwrap();
+        let mut stream = bining.token_stream("中文测试");
+
+        let mut results = Vec::<String>::new();
+        while stream.advance() {
+            let token = stream.token();
+            results.push(token.text.clone());
+        }
+
+        let expected: Vec<String> = vec!["zhongwen".to_string(), "ceshi".to_string()];
+        print!("test tokens :{:?}\n", results);
+        assert!(is_subset(&expected, &results));
+    }
+
+    #[test]
+    fn test_pinyin_filter_with_full_pinyin() {
+        let params = r#"{
+            "tokenizer": "jieba",
+            "filter": [{
+                "type": "pinyin",
+                "keep_original": true,
+                "keep_full_pinyin": true,
+                "keep_joined_full_pinyin": false,
+                "keep_separate_first_letter": false
+            }]
+        }"#;
+
+        let tokenizer = create_analyzer(&params.to_string(), "");
+        assert!(tokenizer.is_ok(), "error: {}", tokenizer.err().unwrap());
+
+        let mut bining = tokenizer.unwrap();
+        let mut stream = bining.token_stream("中文测试");
+
+        let mut results = Vec::<String>::new();
+        while stream.advance() {
+            let token = stream.token();
+            results.push(token.text.clone());
+        }
+
+        let expected: Vec<String> = vec![
+            "zhong".to_string(),
+            "wen".to_string(),
+            "ce".to_string(),
+            "shi".to_string(),
+        ];
+        println!("test tokens :{:?}", results);
+        assert!(is_subset(&expected, &results));
+    }
+
+    #[test]
+    fn test_pinyin_filter_with_first_letter() {
+        let params = r#"{
+            "tokenizer": "jieba",
+            "filter": [{
+                "type": "pinyin",
+                "keep_original": true,
+                "keep_full_pinyin": false,
+                "keep_joined_full_pinyin": false,
+                "keep_separate_first_letter": true
+            }]
+        }"#;
+
+        let tokenizer = create_analyzer(&params.to_string(), "");
+        assert!(tokenizer.is_ok(), "error: {}", tokenizer.err().unwrap());
+
+        let mut bining = tokenizer.unwrap();
+        let mut stream = bining.token_stream("中文测试");
+
+        let mut results = Vec::<String>::new();
+        while stream.advance() {
+            let token = stream.token();
+            results.push(token.text.clone());
+        }
+
+        let expected: Vec<String> = vec!["zw".to_string(), "cs".to_string()];
+        print!("test tokens :{:?}\n", results);
+        assert!(is_subset(&expected, &results));
+    }
+}