mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 10:15:43 +00:00
enhance: use adapter analyzer for tantivy index writer v5 (#46963)
relate: https://github.com/milvus-io/milvus/issues/46962 Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>
This commit is contained in:
aoiasd
+102
@@ -0,0 +1,102 @@
|
||||
use crate::analyzer::create_analyzer as create_new_analyzer;
|
||||
use crate::error::Result;
|
||||
use tantivy::tokenizer::{BoxTokenStream as NewBoxTokenStream, TextAnalyzer as NewAnalyzer};
|
||||
use tantivy_5::tokenizer::{TextAnalyzer, Token, TokenStream, Tokenizer};
|
||||
|
||||
struct AdapterTokenStream<'a> {
|
||||
token_stream: NewBoxTokenStream<'a>,
|
||||
token: Token,
|
||||
}
|
||||
|
||||
impl<'a> AdapterTokenStream<'a> {
|
||||
fn new(token_stream: NewBoxTokenStream<'a>) -> Self {
|
||||
Self {
|
||||
token_stream,
|
||||
token: Token::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> TokenStream for AdapterTokenStream<'a> {
|
||||
fn advance(&mut self) -> bool {
|
||||
if self.token_stream.advance() {
|
||||
let new_token = self.token_stream.token();
|
||||
self.token = Token {
|
||||
offset_from: new_token.offset_from,
|
||||
offset_to: new_token.offset_to,
|
||||
position: new_token.position,
|
||||
text: new_token.text.clone(),
|
||||
position_length: new_token.position_length,
|
||||
};
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
fn token(&self) -> &Token {
|
||||
&self.token
|
||||
}
|
||||
|
||||
fn token_mut(&mut self) -> &mut Token {
|
||||
&mut self.token
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct AdapterAnalyzer {
|
||||
tokenizer: NewAnalyzer,
|
||||
}
|
||||
|
||||
impl AdapterAnalyzer {
|
||||
fn new(tokenizer: NewAnalyzer) -> Self {
|
||||
Self { tokenizer }
|
||||
}
|
||||
}
|
||||
|
||||
impl Tokenizer for AdapterAnalyzer {
|
||||
type TokenStream<'a> = AdapterTokenStream<'a>;
|
||||
fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
|
||||
AdapterTokenStream::new(self.tokenizer.token_stream(text))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn create_analyzer(params: &str) -> Result<TextAnalyzer> {
|
||||
let tokenizer = create_new_analyzer(params, "")?;
|
||||
Ok(TextAnalyzer::builder(AdapterAnalyzer::new(tokenizer)).build())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::create_analyzer;
|
||||
|
||||
#[test]
|
||||
fn test_standard_analyzer() {
|
||||
let params = r#"{
|
||||
"type": "standard",
|
||||
"stop_words": ["_english_"]
|
||||
}"#;
|
||||
|
||||
let tokenizer = create_analyzer(¶ms.to_string());
|
||||
assert!(tokenizer.is_ok(), "error: {}", tokenizer.err().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chinese_analyzer() {
|
||||
let params = r#"{
|
||||
"type": "chinese"
|
||||
}"#;
|
||||
|
||||
let tokenizer = create_analyzer(¶ms.to_string());
|
||||
assert!(tokenizer.is_ok(), "error: {}", tokenizer.err().unwrap());
|
||||
let mut bining = tokenizer.unwrap();
|
||||
let mut stream = bining.token_stream("系统安全;,'';lxyz密码");
|
||||
|
||||
let mut results = Vec::<String>::new();
|
||||
while stream.advance() {
|
||||
let token = stream.token();
|
||||
results.push(token.text.clone());
|
||||
}
|
||||
|
||||
print!("test tokens :{:?}\n", results)
|
||||
}
|
||||
}
|
||||
-288
@@ -1,288 +0,0 @@
|
||||
use serde_json as json;
|
||||
use std::collections::HashMap;
|
||||
use tantivy_5::tokenizer::*;
|
||||
|
||||
use crate::error::{Result, TantivyBindingError};
|
||||
|
||||
use super::{
|
||||
build_in_analyzer::{chinese_analyzer, english_analyzer},
|
||||
filter::SystemFilter,
|
||||
standard_analyzer,
|
||||
tokenizers::get_builder_with_tokenizer,
|
||||
util::{get_stop_words_list, get_string_list},
|
||||
};
|
||||
|
||||
struct AnalyzerBuilder<'a> {
|
||||
filters: HashMap<String, SystemFilter>,
|
||||
params: &'a json::Map<String, json::Value>,
|
||||
}
|
||||
|
||||
impl AnalyzerBuilder<'_> {
|
||||
fn new(params: &json::Map<String, json::Value>) -> AnalyzerBuilder {
|
||||
AnalyzerBuilder {
|
||||
filters: HashMap::new(),
|
||||
params: params,
|
||||
}
|
||||
}
|
||||
|
||||
fn get_tokenizer_params(&self) -> Result<&json::Value> {
|
||||
let tokenizer = self.params.get("tokenizer");
|
||||
if tokenizer.is_none() {
|
||||
return Err(TantivyBindingError::InternalError(format!(
|
||||
"tokenizer name or type must be set"
|
||||
)));
|
||||
}
|
||||
let value = tokenizer.unwrap();
|
||||
if value.is_object() || value.is_string() {
|
||||
return Ok(tokenizer.unwrap());
|
||||
}
|
||||
|
||||
Err(TantivyBindingError::InternalError(
|
||||
"tokenizer name should be string or dict".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
fn add_custom_filter(
|
||||
&mut self,
|
||||
name: &String,
|
||||
params: &json::Map<String, json::Value>,
|
||||
) -> Result<()> {
|
||||
match SystemFilter::try_from(params) {
|
||||
Ok(filter) => {
|
||||
self.filters.insert(name.to_string(), filter);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
fn add_custom_filters(&mut self, params: &json::Map<String, json::Value>) -> Result<()> {
|
||||
for (name, value) in params {
|
||||
if !value.is_object() {
|
||||
continue;
|
||||
}
|
||||
self.add_custom_filter(name, value.as_object().unwrap())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn build_filter(
|
||||
&mut self,
|
||||
mut builder: TextAnalyzerBuilder,
|
||||
params: &json::Value,
|
||||
) -> Result<TextAnalyzerBuilder> {
|
||||
if !params.is_array() {
|
||||
return Err(TantivyBindingError::InternalError(
|
||||
"filter params should be array".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let filters = params.as_array().unwrap();
|
||||
|
||||
for filter in filters {
|
||||
if filter.is_string() {
|
||||
let filter_name = filter.as_str().unwrap();
|
||||
let customize = self.filters.remove(filter_name);
|
||||
if !customize.is_none() {
|
||||
builder = customize.unwrap().transform(builder);
|
||||
continue;
|
||||
}
|
||||
|
||||
// check if filter was system filter
|
||||
let system = SystemFilter::from(filter_name);
|
||||
match system {
|
||||
SystemFilter::Invalid => {
|
||||
return Err(TantivyBindingError::InternalError(format!(
|
||||
"build analyzer failed, filter not found :{}",
|
||||
filter_name
|
||||
)))
|
||||
}
|
||||
other => {
|
||||
builder = other.transform(builder);
|
||||
}
|
||||
}
|
||||
} else if filter.is_object() {
|
||||
let filter = SystemFilter::try_from(filter.as_object().unwrap())?;
|
||||
builder = filter.transform(builder);
|
||||
}
|
||||
}
|
||||
Ok(builder)
|
||||
}
|
||||
|
||||
fn build_option(&mut self, mut builder: TextAnalyzerBuilder) -> Result<TextAnalyzerBuilder> {
|
||||
for (key, value) in self.params {
|
||||
match key.as_str() {
|
||||
"tokenizer" => {}
|
||||
"filter" => {
|
||||
// build with filter if filter param exist
|
||||
builder = self.build_filter(builder, value)?;
|
||||
}
|
||||
other => {
|
||||
return Err(TantivyBindingError::InternalError(format!(
|
||||
"unknown analyzer option key: {}",
|
||||
other
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(builder)
|
||||
}
|
||||
|
||||
fn get_stop_words_option(&self) -> Result<Vec<String>> {
|
||||
let value = self.params.get("stop_words");
|
||||
match value {
|
||||
Some(value) => {
|
||||
let str_list = get_string_list(value, "filter stop_words")?;
|
||||
Ok(get_stop_words_list(str_list))
|
||||
}
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_template(self, type_: &str) -> Result<TextAnalyzer> {
|
||||
match type_ {
|
||||
"standard" => Ok(standard_analyzer(self.get_stop_words_option()?)),
|
||||
"chinese" => Ok(chinese_analyzer(self.get_stop_words_option()?)),
|
||||
"english" => Ok(english_analyzer(self.get_stop_words_option()?)),
|
||||
other_ => Err(TantivyBindingError::InternalError(format!(
|
||||
"unknown build-in analyzer type: {}",
|
||||
other_
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn build(mut self) -> Result<TextAnalyzer> {
|
||||
// build base build-in analyzer
|
||||
match self.params.get("type") {
|
||||
Some(type_) => {
|
||||
if !type_.is_string() {
|
||||
return Err(TantivyBindingError::InternalError(format!(
|
||||
"analyzer type should be string"
|
||||
)));
|
||||
}
|
||||
return self.build_template(type_.as_str().unwrap());
|
||||
}
|
||||
None => {}
|
||||
};
|
||||
|
||||
//build custom analyzer
|
||||
let tokenizer_params = self.get_tokenizer_params()?;
|
||||
let mut builder = get_builder_with_tokenizer(&tokenizer_params)?;
|
||||
|
||||
// build with option
|
||||
builder = self.build_option(builder)?;
|
||||
Ok(builder.build())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn create_analyzer_with_filter(params: &String) -> Result<TextAnalyzer> {
|
||||
match json::from_str::<json::Value>(¶ms) {
|
||||
Ok(value) => {
|
||||
if value.is_null() {
|
||||
return Ok(standard_analyzer(vec![]));
|
||||
}
|
||||
if !value.is_object() {
|
||||
return Err(TantivyBindingError::InternalError(
|
||||
"tokenizer params should be a json map".to_string(),
|
||||
));
|
||||
}
|
||||
let json_params = value.as_object().unwrap();
|
||||
|
||||
// create builder
|
||||
let analyzer_params = json_params.get("analyzer");
|
||||
if analyzer_params.is_none() {
|
||||
return Ok(standard_analyzer(vec![]));
|
||||
}
|
||||
if !analyzer_params.unwrap().is_object() {
|
||||
return Err(TantivyBindingError::InternalError(
|
||||
"analyzer params should be a json map".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let builder_params = analyzer_params.unwrap().as_object().unwrap();
|
||||
if builder_params.is_empty() {
|
||||
return Ok(standard_analyzer(vec![]));
|
||||
}
|
||||
|
||||
let mut builder = AnalyzerBuilder::new(builder_params);
|
||||
|
||||
// build custom filter
|
||||
let filter_params = json_params.get("filter");
|
||||
if !filter_params.is_none() && filter_params.unwrap().is_object() {
|
||||
builder.add_custom_filters(filter_params.unwrap().as_object().unwrap())?;
|
||||
}
|
||||
|
||||
// build analyzer
|
||||
builder.build()
|
||||
}
|
||||
Err(err) => Err(err.into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn create_analyzer(params: &str) -> Result<TextAnalyzer> {
|
||||
if params.len() == 0 {
|
||||
return Ok(standard_analyzer(vec![]));
|
||||
}
|
||||
create_analyzer_with_filter(&format!("{{\"analyzer\":{}}}", params))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_standard_analyzer() {
|
||||
let params = r#"{
|
||||
"type": "standard",
|
||||
"stop_words": ["_english_"]
|
||||
}"#;
|
||||
|
||||
let tokenizer = create_analyzer(¶ms.to_string());
|
||||
assert!(tokenizer.is_ok(), "error: {}", tokenizer.err().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chinese_analyzer() {
|
||||
let params = r#"{
|
||||
"type": "chinese"
|
||||
}"#;
|
||||
|
||||
let tokenizer = create_analyzer(¶ms.to_string());
|
||||
assert!(tokenizer.is_ok(), "error: {}", tokenizer.err().unwrap());
|
||||
let mut bining = tokenizer.unwrap();
|
||||
let mut stream = bining.token_stream("系统安全;,'';lxyz密码");
|
||||
|
||||
let mut results = Vec::<String>::new();
|
||||
while stream.advance() {
|
||||
let token = stream.token();
|
||||
results.push(token.text.clone());
|
||||
}
|
||||
|
||||
print!("test tokens :{:?}\n", results)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_lindera_analyzer() {
|
||||
let params = r#"{
|
||||
"tokenizer": {
|
||||
"type": "lindera",
|
||||
"dict_kind": "ipadic"
|
||||
}
|
||||
}"#;
|
||||
|
||||
let tokenizer = create_analyzer(¶ms.to_string());
|
||||
assert!(tokenizer.is_ok(), "error: {}", tokenizer.err().unwrap());
|
||||
|
||||
let mut bining = tokenizer.unwrap();
|
||||
let mut stream =
|
||||
bining.token_stream("東京スカイツリーの最寄り駅はとうきょうスカイツリー駅です");
|
||||
|
||||
let mut results = Vec::<String>::new();
|
||||
while stream.advance() {
|
||||
let token = stream.token();
|
||||
results.push(token.text.clone());
|
||||
}
|
||||
|
||||
print!("test tokens :{:?}\n", results)
|
||||
}
|
||||
}
|
||||
Vendored
-40
@@ -1,40 +0,0 @@
|
||||
use tantivy_5::tokenizer::*;
|
||||
|
||||
use super::filter::*;
|
||||
use super::stop_words;
|
||||
use super::tokenizers::*;
|
||||
|
||||
// default build-in analyzer
|
||||
pub(crate) fn standard_analyzer(stop_words: Vec<String>) -> TextAnalyzer {
|
||||
let builder = standard_builder().filter(LowerCaser);
|
||||
|
||||
if stop_words.len() > 0 {
|
||||
return builder.filter(StopWordFilter::remove(stop_words)).build();
|
||||
}
|
||||
|
||||
builder.build()
|
||||
}
|
||||
|
||||
pub fn chinese_analyzer(stop_words: Vec<String>) -> TextAnalyzer {
|
||||
let builder = jieba_builder().filter(CnAlphaNumOnlyFilter);
|
||||
if stop_words.len() > 0 {
|
||||
return builder.filter(StopWordFilter::remove(stop_words)).build();
|
||||
}
|
||||
|
||||
builder.build()
|
||||
}
|
||||
|
||||
pub fn english_analyzer(stop_words: Vec<String>) -> TextAnalyzer {
|
||||
let builder = standard_builder()
|
||||
.filter(LowerCaser)
|
||||
.filter(Stemmer::new(Language::English))
|
||||
.filter(StopWordFilter::remove(
|
||||
stop_words::ENGLISH.iter().map(|&word| word.to_owned()),
|
||||
));
|
||||
|
||||
if stop_words.len() > 0 {
|
||||
return builder.filter(StopWordFilter::remove(stop_words)).build();
|
||||
}
|
||||
|
||||
builder.build()
|
||||
}
|
||||
-286
@@ -1,286 +0,0 @@
|
||||
use serde_json as json;
|
||||
use tantivy_5::tokenizer::*;
|
||||
|
||||
use super::util::*;
|
||||
use crate::error::{Result, TantivyBindingError};
|
||||
|
||||
pub(crate) enum SystemFilter {
|
||||
Invalid,
|
||||
LowerCase(LowerCaser),
|
||||
AsciiFolding(AsciiFoldingFilter),
|
||||
AlphaNumOnly(AlphaNumOnlyFilter),
|
||||
CnCharOnly(CnCharOnlyFilter),
|
||||
CnAlphaNumOnly(CnAlphaNumOnlyFilter),
|
||||
Length(RemoveLongFilter),
|
||||
Stop(StopWordFilter),
|
||||
Decompounder(SplitCompoundWords),
|
||||
Stemmer(Stemmer),
|
||||
}
|
||||
|
||||
impl SystemFilter {
|
||||
pub(crate) fn transform(self, builder: TextAnalyzerBuilder) -> TextAnalyzerBuilder {
|
||||
match self {
|
||||
Self::LowerCase(filter) => builder.filter(filter).dynamic(),
|
||||
Self::AsciiFolding(filter) => builder.filter(filter).dynamic(),
|
||||
Self::AlphaNumOnly(filter) => builder.filter(filter).dynamic(),
|
||||
Self::CnCharOnly(filter) => builder.filter(filter).dynamic(),
|
||||
Self::CnAlphaNumOnly(filter) => builder.filter(filter).dynamic(),
|
||||
Self::Length(filter) => builder.filter(filter).dynamic(),
|
||||
Self::Stop(filter) => builder.filter(filter).dynamic(),
|
||||
Self::Decompounder(filter) => builder.filter(filter).dynamic(),
|
||||
Self::Stemmer(filter) => builder.filter(filter).dynamic(),
|
||||
Self::Invalid => builder,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// create length filter from params
|
||||
// {
|
||||
// "type": "length",
|
||||
// "max": 10, // length
|
||||
// }
|
||||
// TODO support min length
|
||||
fn get_length_filter(params: &json::Map<String, json::Value>) -> Result<SystemFilter> {
|
||||
let limit_str = params.get("max");
|
||||
if limit_str.is_none() || !limit_str.unwrap().is_u64() {
|
||||
return Err(TantivyBindingError::InternalError(
|
||||
"lenth max param was none or not uint".to_string(),
|
||||
));
|
||||
}
|
||||
let limit = limit_str.unwrap().as_u64().unwrap() as usize;
|
||||
Ok(SystemFilter::Length(RemoveLongFilter::limit(limit + 1)))
|
||||
}
|
||||
|
||||
fn get_stop_words_filter(params: &json::Map<String, json::Value>) -> Result<SystemFilter> {
|
||||
let value = params.get("stop_words");
|
||||
if value.is_none() {
|
||||
return Err(TantivyBindingError::InternalError(
|
||||
"stop filter stop_words can't be empty".to_string(),
|
||||
));
|
||||
}
|
||||
let str_list = get_string_list(value.unwrap(), "stop_words filter")?;
|
||||
Ok(SystemFilter::Stop(StopWordFilter::remove(
|
||||
get_stop_words_list(str_list),
|
||||
)))
|
||||
}
|
||||
|
||||
fn get_decompounder_filter(params: &json::Map<String, json::Value>) -> Result<SystemFilter> {
|
||||
let value = params.get("word_list");
|
||||
if value.is_none() || !value.unwrap().is_array() {
|
||||
return Err(TantivyBindingError::InternalError(
|
||||
"decompounder word list should be array".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let stop_words = value.unwrap().as_array().unwrap();
|
||||
let mut str_list = Vec::<String>::new();
|
||||
for element in stop_words {
|
||||
match element.as_str() {
|
||||
Some(word) => str_list.push(word.to_string()),
|
||||
_ => {
|
||||
return Err(TantivyBindingError::InternalError(
|
||||
"decompounder word list item should be string".to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match SplitCompoundWords::from_dictionary(str_list) {
|
||||
Ok(f) => Ok(SystemFilter::Decompounder(f)),
|
||||
Err(e) => Err(TantivyBindingError::InternalError(format!(
|
||||
"create decompounder failed: {}",
|
||||
e
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_stemmer_filter(params: &json::Map<String, json::Value>) -> Result<SystemFilter> {
|
||||
let value = params.get("language");
|
||||
if value.is_none() || !value.unwrap().is_string() {
|
||||
return Err(TantivyBindingError::InternalError(
|
||||
"stemmer language field should be string".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
match value.unwrap().as_str().unwrap().into_language() {
|
||||
Ok(language) => Ok(SystemFilter::Stemmer(Stemmer::new(language))),
|
||||
Err(e) => Err(TantivyBindingError::InternalError(format!(
|
||||
"create stemmer failed : {}",
|
||||
e
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
trait LanguageParser {
|
||||
fn into_language(self) -> Result<Language>;
|
||||
}
|
||||
|
||||
impl LanguageParser for &str {
|
||||
fn into_language(self) -> Result<Language> {
|
||||
match self.to_lowercase().as_str() {
|
||||
"arabic" => Ok(Language::Arabic),
|
||||
"arabig" => Ok(Language::Arabic),
|
||||
"danish" => Ok(Language::Danish),
|
||||
"dutch" => Ok(Language::Dutch),
|
||||
"english" => Ok(Language::English),
|
||||
"finnish" => Ok(Language::Finnish),
|
||||
"french" => Ok(Language::French),
|
||||
"german" => Ok(Language::German),
|
||||
"greek" => Ok(Language::Greek),
|
||||
"hungarian" => Ok(Language::Hungarian),
|
||||
"italian" => Ok(Language::Italian),
|
||||
"norwegian" => Ok(Language::Norwegian),
|
||||
"portuguese" => Ok(Language::Portuguese),
|
||||
"romanian" => Ok(Language::Romanian),
|
||||
"russian" => Ok(Language::Russian),
|
||||
"spanish" => Ok(Language::Spanish),
|
||||
"swedish" => Ok(Language::Swedish),
|
||||
"tamil" => Ok(Language::Tamil),
|
||||
"turkish" => Ok(Language::Turkish),
|
||||
other => Err(TantivyBindingError::InternalError(format!(
|
||||
"unsupport language: {}",
|
||||
other
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for SystemFilter {
|
||||
fn from(value: &str) -> Self {
|
||||
match value {
|
||||
"lowercase" => Self::LowerCase(LowerCaser),
|
||||
"asciifolding" => Self::AsciiFolding(AsciiFoldingFilter),
|
||||
"alphanumonly" => Self::AlphaNumOnly(AlphaNumOnlyFilter),
|
||||
"cncharonly" => Self::CnCharOnly(CnCharOnlyFilter),
|
||||
"cnalphanumonly" => Self::CnAlphaNumOnly(CnAlphaNumOnlyFilter),
|
||||
_ => Self::Invalid,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<&json::Map<String, json::Value>> for SystemFilter {
|
||||
type Error = TantivyBindingError;
|
||||
|
||||
fn try_from(params: &json::Map<String, json::Value>) -> Result<Self> {
|
||||
match params.get(&"type".to_string()) {
|
||||
Some(value) => {
|
||||
if !value.is_string() {
|
||||
return Err(TantivyBindingError::InternalError(
|
||||
"filter type should be string".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
match value.as_str().unwrap() {
|
||||
"length" => get_length_filter(params),
|
||||
"stop" => get_stop_words_filter(params),
|
||||
"decompounder" => get_decompounder_filter(params),
|
||||
"stemmer" => get_stemmer_filter(params),
|
||||
other => Err(TantivyBindingError::InternalError(format!(
|
||||
"unsupport filter type: {}",
|
||||
other
|
||||
))),
|
||||
}
|
||||
}
|
||||
None => Err(TantivyBindingError::InternalError(
|
||||
"no type field in filter params".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CnCharOnlyFilter;
|
||||
|
||||
pub struct CnCharOnlyFilterStream<T> {
|
||||
regex: regex::Regex,
|
||||
tail: T,
|
||||
}
|
||||
|
||||
impl TokenFilter for CnCharOnlyFilter {
|
||||
type Tokenizer<T: Tokenizer> = CnCharOnlyFilterWrapper<T>;
|
||||
|
||||
fn transform<T: Tokenizer>(self, tokenizer: T) -> CnCharOnlyFilterWrapper<T> {
|
||||
CnCharOnlyFilterWrapper(tokenizer)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CnCharOnlyFilterWrapper<T>(T);
|
||||
|
||||
impl<T: Tokenizer> Tokenizer for CnCharOnlyFilterWrapper<T> {
|
||||
type TokenStream<'a> = CnCharOnlyFilterStream<T::TokenStream<'a>>;
|
||||
|
||||
fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
|
||||
CnCharOnlyFilterStream {
|
||||
regex: regex::Regex::new("\\p{Han}+").unwrap(),
|
||||
tail: self.0.token_stream(text),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: TokenStream> TokenStream for CnCharOnlyFilterStream<T> {
|
||||
fn advance(&mut self) -> bool {
|
||||
while self.tail.advance() {
|
||||
if self.regex.is_match(&self.tail.token().text) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
fn token(&self) -> &Token {
|
||||
self.tail.token()
|
||||
}
|
||||
|
||||
fn token_mut(&mut self) -> &mut Token {
|
||||
self.tail.token_mut()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CnAlphaNumOnlyFilter;
|
||||
|
||||
pub struct CnAlphaNumOnlyFilterStream<T> {
|
||||
regex: regex::Regex,
|
||||
tail: T,
|
||||
}
|
||||
|
||||
impl TokenFilter for CnAlphaNumOnlyFilter {
|
||||
type Tokenizer<T: Tokenizer> = CnAlphaNumOnlyFilterWrapper<T>;
|
||||
|
||||
fn transform<T: Tokenizer>(self, tokenizer: T) -> CnAlphaNumOnlyFilterWrapper<T> {
|
||||
CnAlphaNumOnlyFilterWrapper(tokenizer)
|
||||
}
|
||||
}
|
||||
#[derive(Clone)]
|
||||
pub struct CnAlphaNumOnlyFilterWrapper<T>(T);
|
||||
|
||||
impl<T: Tokenizer> Tokenizer for CnAlphaNumOnlyFilterWrapper<T> {
|
||||
type TokenStream<'a> = CnAlphaNumOnlyFilterStream<T::TokenStream<'a>>;
|
||||
|
||||
fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
|
||||
CnAlphaNumOnlyFilterStream {
|
||||
regex: regex::Regex::new(r"[\p{Han}a-zA-Z0-9]+").unwrap(),
|
||||
tail: self.0.token_stream(text),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: TokenStream> TokenStream for CnAlphaNumOnlyFilterStream<T> {
|
||||
fn advance(&mut self) -> bool {
|
||||
while self.tail.advance() {
|
||||
if self.regex.is_match(&self.tail.token().text) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
fn token(&self) -> &Token {
|
||||
self.tail.token()
|
||||
}
|
||||
|
||||
fn token_mut(&mut self) -> &mut Token {
|
||||
self.tail.token_mut()
|
||||
}
|
||||
}
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
//! This is totally copied from src/analyzer
|
||||
|
||||
mod analyzer;
|
||||
mod build_in_analyzer;
|
||||
mod filter;
|
||||
mod stop_words;
|
||||
mod tokenizers;
|
||||
mod util;
|
||||
|
||||
pub(crate) use self::analyzer::create_analyzer;
|
||||
pub(crate) use self::build_in_analyzer::standard_analyzer;
|
||||
Vendored
-5
@@ -1,5 +0,0 @@
|
||||
pub const ENGLISH: &[&str] = &[
|
||||
"a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it",
|
||||
"no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these",
|
||||
"they", "this", "to", "was", "will", "with",
|
||||
];
|
||||
-83
@@ -1,83 +0,0 @@
|
||||
use jieba_rs;
|
||||
use lazy_static::lazy_static;
|
||||
use tantivy_5::tokenizer::{Token, TokenStream, Tokenizer};
|
||||
|
||||
lazy_static! {
|
||||
static ref JIEBA: jieba_rs::Jieba = jieba_rs::Jieba::new();
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Clone)]
|
||||
pub enum JiebaMode {
|
||||
Exact,
|
||||
Search,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct JiebaTokenizer {
|
||||
mode: JiebaMode,
|
||||
hmm: bool,
|
||||
}
|
||||
|
||||
pub struct JiebaTokenStream {
|
||||
tokens: Vec<Token>,
|
||||
index: usize,
|
||||
}
|
||||
|
||||
impl TokenStream for JiebaTokenStream {
|
||||
fn advance(&mut self) -> bool {
|
||||
if self.index < self.tokens.len() {
|
||||
self.index += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
fn token(&self) -> &Token {
|
||||
&self.tokens[self.index - 1]
|
||||
}
|
||||
|
||||
fn token_mut(&mut self) -> &mut Token {
|
||||
&mut self.tokens[self.index - 1]
|
||||
}
|
||||
}
|
||||
|
||||
impl JiebaTokenizer {
|
||||
pub fn new() -> JiebaTokenizer {
|
||||
JiebaTokenizer {
|
||||
mode: JiebaMode::Search,
|
||||
hmm: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn tokenize(&self, text: &str) -> Vec<Token> {
|
||||
let mut indices = text.char_indices().collect::<Vec<_>>();
|
||||
indices.push((text.len(), '\0'));
|
||||
let ori_tokens = match self.mode {
|
||||
JiebaMode::Exact => JIEBA.tokenize(text, jieba_rs::TokenizeMode::Default, self.hmm),
|
||||
JiebaMode::Search => JIEBA.tokenize(text, jieba_rs::TokenizeMode::Search, self.hmm),
|
||||
};
|
||||
|
||||
let mut tokens = Vec::with_capacity(ori_tokens.len());
|
||||
for token in ori_tokens {
|
||||
tokens.push(Token {
|
||||
offset_from: indices[token.start].0,
|
||||
offset_to: indices[token.end].0,
|
||||
position: token.start,
|
||||
text: String::from(&text[(indices[token.start].0)..(indices[token.end].0)]),
|
||||
position_length: token.end - token.start,
|
||||
});
|
||||
}
|
||||
tokens
|
||||
}
|
||||
}
|
||||
|
||||
impl Tokenizer for JiebaTokenizer {
|
||||
type TokenStream<'a> = JiebaTokenStream;
|
||||
|
||||
fn token_stream(&mut self, text: &str) -> JiebaTokenStream {
|
||||
let tokens = self.tokenize(text);
|
||||
JiebaTokenStream { tokens, index: 0 }
|
||||
}
|
||||
}
|
||||
-150
@@ -1,150 +0,0 @@
|
||||
use core::result::Result::Err;
|
||||
|
||||
use lindera::dictionary::{load_dictionary_from_kind, DictionaryKind};
|
||||
use lindera::mode::Mode;
|
||||
use lindera::segmenter::Segmenter;
|
||||
use lindera::token::Token as LToken;
|
||||
use lindera::tokenizer::Tokenizer as LTokenizer;
|
||||
use tantivy_5::tokenizer::{Token, TokenStream, Tokenizer};
|
||||
|
||||
use crate::error::{Result, TantivyBindingError};
|
||||
use serde_json as json;
|
||||
|
||||
pub struct LinderaTokenStream<'a> {
|
||||
pub tokens: Vec<LToken<'a>>,
|
||||
pub token: &'a mut Token,
|
||||
}
|
||||
|
||||
impl<'a> TokenStream for LinderaTokenStream<'a> {
|
||||
fn advance(&mut self) -> bool {
|
||||
if self.tokens.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let token = self.tokens.remove(0);
|
||||
self.token.text = token.text.to_string();
|
||||
self.token.offset_from = token.byte_start;
|
||||
self.token.offset_to = token.byte_end;
|
||||
self.token.position = token.position;
|
||||
self.token.position_length = token.position_length;
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
fn token(&self) -> &Token {
|
||||
self.token
|
||||
}
|
||||
|
||||
fn token_mut(&mut self) -> &mut Token {
|
||||
self.token
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct LinderaTokenizer {
|
||||
tokenizer: LTokenizer,
|
||||
token: Token,
|
||||
}
|
||||
|
||||
impl LinderaTokenizer {
|
||||
/// Create a new `LinderaTokenizer`.
|
||||
/// This function will create a new `LinderaTokenizer` with settings from the YAML file specified in the `LINDERA_CONFIG_PATH` environment variable.
|
||||
pub fn from_json(params: &json::Map<String, json::Value>) -> Result<LinderaTokenizer> {
|
||||
let kind = fetch_lindera_kind(params)?;
|
||||
let dictionary = load_dictionary_from_kind(kind);
|
||||
if dictionary.is_err() {
|
||||
return Err(TantivyBindingError::InvalidArgument(format!(
|
||||
"lindera tokenizer with invalid dict_kind"
|
||||
)));
|
||||
}
|
||||
let segmenter = Segmenter::new(Mode::Normal, dictionary.unwrap(), None);
|
||||
Ok(LinderaTokenizer::from_segmenter(segmenter))
|
||||
}
|
||||
|
||||
/// Create a new `LinderaTokenizer`.
|
||||
/// This function will create a new `LinderaTokenizer` with the specified `lindera::segmenter::Segmenter`.
|
||||
pub fn from_segmenter(segmenter: lindera::segmenter::Segmenter) -> LinderaTokenizer {
|
||||
LinderaTokenizer {
|
||||
tokenizer: LTokenizer::new(segmenter),
|
||||
token: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Tokenizer for LinderaTokenizer {
|
||||
type TokenStream<'a> = LinderaTokenStream<'a>;
|
||||
|
||||
fn token_stream<'a>(&'a mut self, text: &'a str) -> LinderaTokenStream<'a> {
|
||||
self.token.reset();
|
||||
LinderaTokenStream {
|
||||
tokens: self.tokenizer.tokenize(text).unwrap(),
|
||||
token: &mut self.token,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait DictionaryKindParser {
|
||||
fn into_dict_kind(self) -> Result<DictionaryKind>;
|
||||
}
|
||||
|
||||
impl DictionaryKindParser for &str {
|
||||
fn into_dict_kind(self) -> Result<DictionaryKind> {
|
||||
match self {
|
||||
"ipadic" => Ok(DictionaryKind::IPADIC),
|
||||
"ipadic-neologd" => Ok(DictionaryKind::IPADICNEologd),
|
||||
"unidic" => Ok(DictionaryKind::UniDic),
|
||||
"ko-dic" => Ok(DictionaryKind::KoDic),
|
||||
"cc-cedict" => Ok(DictionaryKind::CcCedict),
|
||||
other => Err(TantivyBindingError::InvalidArgument(format!(
|
||||
"unsupported lindera dict type: {}",
|
||||
other
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn fetch_lindera_kind(params: &json::Map<String, json::Value>) -> Result<DictionaryKind> {
|
||||
match params.get("dict_kind") {
|
||||
Some(val) => {
|
||||
if !val.is_string() {
|
||||
return Err(TantivyBindingError::InvalidArgument(
|
||||
"lindera tokenizer dict kind should be string".to_string(),
|
||||
));
|
||||
}
|
||||
val.as_str().unwrap().into_dict_kind()
|
||||
}
|
||||
_ => Err(TantivyBindingError::InvalidArgument(
|
||||
"lindera tokenizer dict_kind must be set".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn test_lindera_tokenizer() {
|
||||
let params = r#"{
|
||||
"type": "lindera",
|
||||
"dict_kind": "ipadic"
|
||||
}"#;
|
||||
let json_param = json::from_str::<json::Map<String, json::Value>>(¶ms);
|
||||
assert!(json_param.is_ok());
|
||||
|
||||
let tokenizer = LinderaTokenizer::from_json(&json_param.unwrap());
|
||||
assert!(tokenizer.is_ok(), "error: {}", tokenizer.err().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "lindera-cc-cedict")]
|
||||
fn test_lindera_tokenizer_cc() {
|
||||
let params = r#"{
|
||||
"type": "lindera",
|
||||
"dict_kind": "cc-cedict"
|
||||
}"#;
|
||||
let json_param = json::from_str::<json::Map<String, json::Value>>(¶ms);
|
||||
assert!(json_param.is_ok());
|
||||
|
||||
let tokenizer = LinderaTokenizer::from_json(&json_param.unwrap());
|
||||
assert!(tokenizer.is_ok(), "error: {}", tokenizer.err().unwrap());
|
||||
}
|
||||
}
|
||||
Vendored
-5
@@ -1,5 +0,0 @@
|
||||
mod jieba_tokenizer;
|
||||
mod lindera_tokenizer;
|
||||
mod tokenizer;
|
||||
|
||||
pub(crate) use self::tokenizer::*;
|
||||
-74
@@ -1,74 +0,0 @@
|
||||
use log::warn;
|
||||
use serde_json as json;
|
||||
use tantivy_5::tokenizer::*;
|
||||
use tantivy_5::tokenizer::{TextAnalyzer, TextAnalyzerBuilder};
|
||||
|
||||
use crate::error::{Result, TantivyBindingError};
|
||||
|
||||
use super::jieba_tokenizer::JiebaTokenizer;
|
||||
use super::lindera_tokenizer::LinderaTokenizer;
|
||||
|
||||
pub fn standard_builder() -> TextAnalyzerBuilder {
|
||||
TextAnalyzer::builder(SimpleTokenizer::default()).dynamic()
|
||||
}
|
||||
|
||||
pub fn whitespace_builder() -> TextAnalyzerBuilder {
|
||||
TextAnalyzer::builder(WhitespaceTokenizer::default()).dynamic()
|
||||
}
|
||||
|
||||
pub fn jieba_builder() -> TextAnalyzerBuilder {
|
||||
TextAnalyzer::builder(JiebaTokenizer::new()).dynamic()
|
||||
}
|
||||
|
||||
pub fn lindera_builder(
|
||||
params: Option<&json::Map<String, json::Value>>,
|
||||
) -> Result<TextAnalyzerBuilder> {
|
||||
if params.is_none() {
|
||||
return Err(TantivyBindingError::InvalidArgument(
|
||||
"lindera tokenizer must be customized".to_string(),
|
||||
));
|
||||
}
|
||||
let tokenizer = LinderaTokenizer::from_json(params.unwrap())?;
|
||||
Ok(TextAnalyzer::builder(tokenizer).dynamic())
|
||||
}
|
||||
|
||||
pub fn get_builder_with_tokenizer(params: &json::Value) -> Result<TextAnalyzerBuilder> {
|
||||
let name;
|
||||
let params_map;
|
||||
if params.is_string() {
|
||||
name = params.as_str().unwrap();
|
||||
params_map = None;
|
||||
} else {
|
||||
let m = params.as_object().unwrap();
|
||||
match m.get("type") {
|
||||
Some(val) => {
|
||||
if !val.is_string() {
|
||||
return Err(TantivyBindingError::InvalidArgument(
|
||||
"tokenizer type should be string".to_string(),
|
||||
));
|
||||
}
|
||||
name = val.as_str().unwrap();
|
||||
}
|
||||
_ => {
|
||||
return Err(TantivyBindingError::InvalidArgument(
|
||||
"customized tokenizer must set type".to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
params_map = Some(m);
|
||||
}
|
||||
|
||||
match name {
|
||||
"standard" => Ok(standard_builder()),
|
||||
"whitespace" => Ok(whitespace_builder()),
|
||||
"jieba" => Ok(jieba_builder()),
|
||||
"lindera" => lindera_builder(params_map),
|
||||
other => {
|
||||
warn!("unsupported tokenizer: {}", other);
|
||||
Err(TantivyBindingError::InvalidArgument(format!(
|
||||
"unsupported tokenizer: {}",
|
||||
other
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
-45
@@ -1,45 +0,0 @@
|
||||
use serde_json as json;
|
||||
|
||||
use super::stop_words;
|
||||
use crate::error::{Result, TantivyBindingError};
|
||||
|
||||
pub(crate) fn get_string_list(value: &json::Value, label: &str) -> Result<Vec<String>> {
|
||||
if !value.is_array() {
|
||||
return Err(TantivyBindingError::InternalError(
|
||||
format!("{} should be array", label).to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let stop_words = value.as_array().unwrap();
|
||||
let mut str_list = Vec::<String>::new();
|
||||
for element in stop_words {
|
||||
match element.as_str() {
|
||||
Some(word) => str_list.push(word.to_string()),
|
||||
_ => {
|
||||
return Err(TantivyBindingError::InternalError(
|
||||
format!("{} list item should be string", label).to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(str_list)
|
||||
}
|
||||
|
||||
pub(crate) fn get_stop_words_list(str_list: Vec<String>) -> Vec<String> {
|
||||
let mut stop_words = Vec::new();
|
||||
for str in str_list {
|
||||
if str.len() > 0 && str.chars().nth(0).unwrap() == '_' {
|
||||
match str.as_str() {
|
||||
"_english_" => {
|
||||
for word in stop_words::ENGLISH {
|
||||
stop_words.push(word.to_string());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
_other => {}
|
||||
}
|
||||
}
|
||||
stop_words.push(str);
|
||||
}
|
||||
stop_words
|
||||
}
|
||||
Reference in New Issue
Block a user