差别

这里会显示出您选择的修订版和当前版本之间的差别。

--- zh:courses:textmining2026:ch02 [2026/09/06 15:08]
pzczxs [词干提取]
+++ zh:courses:textmining2026:ch02 [2026/09/07 14:39] (当前版本)
pzczxs [第三方库依赖]
@@ 行 1: / 行 1: @@
 ====== 第二章：Java文本处理基础 ======
 ===== 课件 =====
-下载：Java文本处理基础
+下载：{{ :zh:courses:textmining2026:ch02.pptx |Java文本处理基础}}
 ==== 第三方库依赖 ====
@@ 行 45: / 行 45: @@
 	<artifactId>lucene-analyzers-common</artifactId>
 	<version>8.11.2</version>
+</dependency>
+<dependency>
+	<groupId>com.hankcs</groupId>
+	<artifactId>hanlp</artifactId>
+	<version>portable-1.8.4</version>
 </dependency>
 </code>
@@ 行 216: / 行 221: @@
 ===== 句子切分 =====
+<file java SentenceSplitDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import java.text.BreakIterator;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+public class SentenceSplitDemo {
+    private static final String DOT_PLACEHOLDER = "<DOT>";
+    private static final Pattern COMMON_TITLE = Pattern.compile(
+            "\\b(Mr|Mrs|Ms|Dr|Prof|Sr|Jr)\\.", Pattern.CASE_INSENSITIVE);
+    public static void main(String[] args) {
+        String text = "Mr. Smith likes Java. Text mining is useful!";
+        System.out.println(splitSentences(text));
+    }
+    public static List<String> splitSentences(String text) {
+        List<String> sentences = new ArrayList<String>();
+        if (text == null || text.trim().length() == 0) {
+            return sentences;
+        }
+        String protectedText = protectCommonTitles(text);
+        BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.US);
+        iterator.setText(protectedText);
+        int start = iterator.first();
+        int end = iterator.next();
+        while (end != BreakIterator.DONE) {
+            String sentence = protectedText.substring(start, end)
+                    .replace(DOT_PLACEHOLDER, ".")
+                    .trim();
+            if (sentence.length() > 0) {
+                sentences.add(sentence);
+            }
+            start = end;
+            end = iterator.next();
+        }
+        return sentences;
+    }
+    private static String protectCommonTitles(String text) {
+        Matcher matcher = COMMON_TITLE.matcher(text);
+        StringBuffer buffer = new StringBuffer();
+        while (matcher.find()) {
+            matcher.appendReplacement(
+                    buffer,
+                    Matcher.quoteReplacement(matcher.group(1) + DOT_PLACEHOLDER));
+        }
+        matcher.appendTail(buffer);
+        return buffer.toString();
+    }
+}
+</file>
+===== 分词 =====
 <file java OpenNlpSimpleTokenizerDemo.java>
 package cn.edu.bjut.textmining.chapter2;
@@ 行 230: / 行 293: @@
         for (String token : tokens) {
             System.out.println(token);
+        }
+    }
+}
+</file>
+<file java HanlpSegmentDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import com.hankcs.hanlp.HanLP;
+import com.hankcs.hanlp.seg.common.Term;
+import java.util.ArrayList;
+import java.util.List;
+/**
+ * HanLP 中文分词示例。
+ * 中文文本没有天然的词间空格，成熟分词工具会结合词典与统计模型判断词语边界。
+ * 本示例与 ChineseTextPreprocessingDemo 的小词典最长优先分词使用同一条校园通知，
+ * 便于对照两类方法的切分差异。
+ */
+public class HanlpSegmentDemo {
+    public static void main(String[] args) {
+        String sentence =
+                "学习委员提醒：明天下午的《高等数学》习题课改到三教302，"
+                + "请带教材、笔、作业本和草稿纸！";
+        List<Term> terms = HanLP.segment(sentence);
+        System.out.println("原句：" + sentence);
+        List<String> words = new ArrayList<String>();
+        for (Term term : terms) {
+            words.add(term.word);
+        }
+        System.out.println("词元序列：");
+        System.out.println(words);
+        System.out.println("词元与词性：");
+        for (Term term : terms) {
+            System.out.println(term.word + "/" + term.nature);
         }
     }
@@ 行 285: / 行 388: @@
     public static void main(String[] args) {
         try {
-            File dictionaryFile = FileUtil.path("data", "chapter2", "en-lemmatizer.dict").toFile();
+            File dictionaryFile = FileUtil.path(
+                    "data", "chapter2", "en-lemmatizer.dict").toFile();
             DictionaryLemmatizer lemmatizer = new DictionaryLemmatizer(dictionaryFile);
-            Path sampleFile = FileUtil.path("data", "chapter2", "lemmatization-samples.tsv");
+            Path sampleFile = FileUtil.path(
+                    "data", "chapter2", "lemmatization-samples.tsv");
             List<String> lines = FileUtil.readNonEmptyUtf8Lines(sampleFile);
             List<String> tokenList = new ArrayList<String>();
@@ 行 316: / 行 421: @@
 </file>
+===== 停用词过滤 =====
+<file java StopWordsDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import cn.edu.bjut.textmining.util.FileUtil;
+import cn.edu.bjut.textmining.util.WordListUtil;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+public class StopWordsDemo {
+    public static void main(String[] args) {
+        try {
+            List<String> words = Arrays.asList("this", "is", "text", "mining",
+            		"in", "java", "book");
+            Path stopwordFile = FileUtil.path(
+                    "data", "chapter2", "basic-english-stopwords.txt");
+            Set<String> stopWords = WordListUtil.readLowerCaseSet(stopwordFile);
+            List<String> keywords = new ArrayList<String>();
+            for (String word : words) {
+                if (!stopWords.contains(word)) {
+                    keywords.add(word);
+                }
+            }
+            System.out.println(keywords);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
+</file>
+~~DISCUSSION~~

硕风徐徐

用户工具

站点工具

差别

页面工具