zh:courses:textmining2026:ch03
差别
这里会显示出您选择的修订版和当前版本之间的差别。
| 两侧同时换到之前的修订记录 前一修订版 | |||
|
zh:courses:textmining2026:ch03 [2026/09/28 17:24] pzczxs [UTF-8文件工具类] |
zh:courses:textmining2026:ch03 [2026/09/28 17:25] (当前版本) pzczxs [UTF-8文件工具类] |
||
|---|---|---|---|
| 行 80: | 行 80: | ||
| } | } | ||
| } | } | ||
| - | } | ||
| - | </file> | ||
| - | |||
| - | <file java ChineseTokenizer.java> | ||
| - | package cn.edu.bjut.textmining.util; | ||
| - | |||
| - | import com.hankcs.hanlp.HanLP; | ||
| - | import com.hankcs.hanlp.seg.common.Term; | ||
| - | |||
| - | import java.util.ArrayList; | ||
| - | import java.util.List; | ||
| - | import java.util.Locale; | ||
| - | |||
| - | /** HanLP portable 1.8.6分词；保留含字母、汉字或数字的词元。 */ | ||
| - | public final class ChineseTokenizer { | ||
| - | private ChineseTokenizer() {} | ||
| - | |||
| - | public static List<String> tokenize(String text) { | ||
| - | List<String> tokens = new ArrayList<String>(); | ||
| - | for (Term term : HanLP.segment(text)) { | ||
| - | String word = term.word.toLowerCase(Locale.ROOT).trim(); | ||
| - | if (word.matches(".*[\\p{L}\\p{N}].*")) { | ||
| - | tokens.add(word); | ||
| - | } | ||
| - | } | ||
| - | return tokens; | ||
| - | } | ||
| } | } | ||
| </file> | </file> | ||
zh/courses/textmining2026/ch03.1790587483.txt.gz · 最后更改: 2026/09/28 17:24 由 pzczxs