package cn.edu.bjut.textmining.util;

import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.seg.common.Term;

import java.util.ArrayList;
import java.util.List;
import java.util.Locale;

/** HanLP portable 1.8.6分词；保留含字母、汉字或数字的词元。 */
public final class ChineseTokenizer {
    private ChineseTokenizer() {}

    public static List<String> tokenize(String text) {
        List<String> tokens = new ArrayList<String>();
        for (Term term : HanLP.segment(text)) {
            String word = term.word.toLowerCase(Locale.ROOT).trim();
            if (word.matches(".*[\\p{L}\\p{N}].*")) {
                tokens.add(word);
            }
        }
        return tokens;
    }
}