差别

这里会显示出您选择的修订版和当前版本之间的差别。

--- zh:courses:textmining2026:ch02 [2026/09/06 14:52]
pzczxs [句子切分]
+++ zh:courses:textmining2026:ch02 [2026/09/07 14:39] (当前版本)
pzczxs [第三方库依赖]
@@ 行 1: / 行 1: @@
 ====== 第二章：Java文本处理基础 ======
 ===== 课件 =====
-下载：Java文本处理基础
+下载：{{ :zh:courses:textmining2026:ch02.pptx |Java文本处理基础}}
 ==== 第三方库依赖 ====
@@ 行 45: / 行 45: @@
 	<artifactId>lucene-analyzers-common</artifactId>
 	<version>8.11.2</version>
+</dependency>
+<dependency>
+	<groupId>com.hankcs</groupId>
+	<artifactId>hanlp</artifactId>
+	<version>portable-1.8.4</version>
 </dependency>
 </code>
@@ 行 216: / 行 221: @@
 ===== 句子切分 =====
+<file java SentenceSplitDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import java.text.BreakIterator;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+public class SentenceSplitDemo {
+    private static final String DOT_PLACEHOLDER = "<DOT>";
+    private static final Pattern COMMON_TITLE = Pattern.compile(
+            "\\b(Mr|Mrs|Ms|Dr|Prof|Sr|Jr)\\.", Pattern.CASE_INSENSITIVE);
+    public static void main(String[] args) {
+        String text = "Mr. Smith likes Java. Text mining is useful!";
+        System.out.println(splitSentences(text));
+    }
+    public static List<String> splitSentences(String text) {
+        List<String> sentences = new ArrayList<String>();
+        if (text == null || text.trim().length() == 0) {
+            return sentences;
+        }
+        String protectedText = protectCommonTitles(text);
+        BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.US);
+        iterator.setText(protectedText);
+        int start = iterator.first();
+        int end = iterator.next();
+        while (end != BreakIterator.DONE) {
+            String sentence = protectedText.substring(start, end)
+                    .replace(DOT_PLACEHOLDER, ".")
+                    .trim();
+            if (sentence.length() > 0) {
+                sentences.add(sentence);
+            }
+            start = end;
+            end = iterator.next();
+        }
+        return sentences;
+    }
+    private static String protectCommonTitles(String text) {
+        Matcher matcher = COMMON_TITLE.matcher(text);
+        StringBuffer buffer = new StringBuffer();
+        while (matcher.find()) {
+            matcher.appendReplacement(
+                    buffer,
+                    Matcher.quoteReplacement(matcher.group(1) + DOT_PLACEHOLDER));
+        }
+        matcher.appendTail(buffer);
+        return buffer.toString();
+    }
+}
+</file>
+===== 分词 =====
 <file java OpenNlpSimpleTokenizerDemo.java>
 package cn.edu.bjut.textmining.chapter2;
@@ 行 230: / 行 293: @@
         for (String token : tokens) {
             System.out.println(token);
+        }
+    }
+}
+</file>
+<file java HanlpSegmentDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import com.hankcs.hanlp.HanLP;
+import com.hankcs.hanlp.seg.common.Term;
+import java.util.ArrayList;
+import java.util.List;
+/**
+ * HanLP 中文分词示例。
+ * 中文文本没有天然的词间空格，成熟分词工具会结合词典与统计模型判断词语边界。
+ * 本示例与 ChineseTextPreprocessingDemo 的小词典最长优先分词使用同一条校园通知，
+ * 便于对照两类方法的切分差异。
+ */
+public class HanlpSegmentDemo {
+    public static void main(String[] args) {
+        String sentence =
+                "学习委员提醒：明天下午的《高等数学》习题课改到三教302，"
+                + "请带教材、笔、作业本和草稿纸！";
+        List<Term> terms = HanLP.segment(sentence);
+        System.out.println("原句：" + sentence);
+        List<String> words = new ArrayList<String>();
+        for (Term term : terms) {
+            words.add(term.word);
+        }
+        System.out.println("词元序列：");
+        System.out.println(words);
+        System.out.println("词元与词性：");
+        for (Term term : terms) {
+            System.out.println(term.word + "/" + term.nature);
         }
     }
@@ 行 236: / 行 339: @@
 ===== 词干提取 =====
-<file java OpenNlpPorterStemmerDemo.java>
+<file java PorterStemmerDemo.java>
 package cn.edu.bjut.textmining.chapter2;
-import opennlp.tools.stemmer.PorterStemmer;
+import cn.edu.bjut.textmining.util.FileUtil;
+import org.tartarus.snowball.ext.PorterStemmer;
-public class OpenNlpPorterStemmerDemo {
+import java.nio.file.Path;
-	public static String stem(String token) {
+import java.util.List;
-		PorterStemmer stemmer = new PorterStemmer();
-		return stemmer.stem(token);
-	}
-	public static void main(String[] args) {
+/** 使用 Lucene analyzers-common 中附带的 Snowball PorterStemmer。 */
-		String[] words = { "running", "jumps", "better", "happily" };
+public class PorterStemmerDemo {
+    public static void main(String[] args) {
+        try {
+            Path sampleFile = FileUtil.path("data", "chapter2", "stemming-samples.txt");
+            List<String> words = FileUtil.readNonEmptyUtf8Lines(sampleFile);
+            for (String word : words) {
+                System.out.println(word + " -> " + stem(word));
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
-		for (String word : words) {
+    public static String stem(String word) {
-			String stem = stem(word);
+        PorterStemmer stemmer = new PorterStemmer();
-			System.out.println(word + "-->" + stem);
+        stemmer.setCurrent(word);
-		}
+        stemmer.stem();
-	}
+        return stemmer.getCurrent();
+    }
 }
 </file>
+===== 词形还原 =====
+<file java OpenNlpLemmatizerDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import cn.edu.bjut.textmining.util.FileUtil;
+import opennlp.tools.lemmatizer.DictionaryLemmatizer;
+import java.io.File;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+/** 使用 OpenNLP 根据“词形 + 词性”查询词元。 */
+public class OpenNlpLemmatizerDemo {
+    public static void main(String[] args) {
+        try {
+            File dictionaryFile = FileUtil.path(
+                    "data", "chapter2", "en-lemmatizer.dict").toFile();
+            DictionaryLemmatizer lemmatizer = new DictionaryLemmatizer(dictionaryFile);
+            Path sampleFile = FileUtil.path(
+                    "data", "chapter2", "lemmatization-samples.tsv");
+            List<String> lines = FileUtil.readNonEmptyUtf8Lines(sampleFile);
+            List<String> tokenList = new ArrayList<String>();
+            List<String> tagList = new ArrayList<String>();
+            for (String line : lines) {
+                String[] parts = line.split("\\t");
+                if (parts.length != 2) {
+                    throw new IllegalArgumentException("词形还原样本格式错误：" + line);
+                }
+                tokenList.add(parts[0]);
+                tagList.add(parts[1]);
+            }
+            String[] tokens = tokenList.toArray(new String[tokenList.size()]);
+            String[] tags = tagList.toArray(new String[tagList.size()]);
+            String[] lemmas = lemmatizer.lemmatize(tokens, tags);
+            System.out.println("词形: " + Arrays.toString(tokens));
+            System.out.println("词性: " + Arrays.toString(tags));
+            System.out.println("词元: " + Arrays.toString(lemmas));
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
+</file>
+===== 停用词过滤 =====
+<file java StopWordsDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import cn.edu.bjut.textmining.util.FileUtil;
+import cn.edu.bjut.textmining.util.WordListUtil;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+public class StopWordsDemo {
+    public static void main(String[] args) {
+        try {
+            List<String> words = Arrays.asList("this", "is", "text", "mining",
+            		"in", "java", "book");
+            Path stopwordFile = FileUtil.path(
+                    "data", "chapter2", "basic-english-stopwords.txt");
+            Set<String> stopWords = WordListUtil.readLowerCaseSet(stopwordFile);
+            List<String> keywords = new ArrayList<String>();
+            for (String word : words) {
+                if (!stopWords.contains(word)) {
+                    keywords.add(word);
+                }
+            }
+            System.out.println(keywords);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
+</file>
+~~DISCUSSION~~

硕风徐徐

用户工具

站点工具

差别

页面工具