差别

这里会显示出您选择的修订版和当前版本之间的差别。

--- zh:courses:textmining2026:ch02 [2026/09/06 11:38]
pzczxs [Java类库中的文本处理工具]
+++ zh:courses:textmining2026:ch02 [2026/09/07 14:39] (当前版本)
pzczxs [第三方库依赖]
@@ 行 1: / 行 1: @@
 ====== 第二章：Java文本处理基础 ======
 ===== 课件 =====
-下载：Java文本处理基础
+下载：{{ :zh:courses:textmining2026:ch02.pptx |Java文本处理基础}}
 ==== 第三方库依赖 ====
@@ 行 45: / 行 45: @@
 	<artifactId>lucene-analyzers-common</artifactId>
 	<version>8.11.2</version>
+</dependency>
+<dependency>
+	<groupId>com.hankcs</groupId>
+	<artifactId>hanlp</artifactId>
+	<version>portable-1.8.4</version>
 </dependency>
 </code>
@@ 行 124: / 行 129: @@
 }
 </file>
+<file java StringUtilsSplitDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import org.apache.commons.lang3.StringUtils;
+import java.util.Arrays;
+/** 展示 StringUtils 对 null、连续空白和首尾空白的安全处理。 */
+public class StringUtilsSplitDemo {
+    public static void main(String[] args) {
+        String text = "  Java   text\tmining  ";
+        String[] tokens = StringUtils.split(text);
+        String safeNull = StringUtils.defaultString(null);
+        String[] emptyTokens = StringUtils.split(safeNull);
+        System.out.println(Arrays.toString(tokens));
+        System.out.println("null 转为空串后是否没有词项: "
+                + (emptyTokens == null || emptyTokens.length == 0));
+    }
+}
+</file>
+===== 文件读写与编码处理 =====
+<file java FileReadDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import cn.edu.bjut.textmining.util.FileUtil;
+import java.io.BufferedReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+public class FileReadDemo {
+    public static void main(String[] args) {
+        Path path = FileUtil.path("data", "chapter2", "utf8-text-reading-sample.txt");
+        try (BufferedReader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
+            for ( String line; (line = reader.readLine()) != null; ) {
+                System.out.println(line);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
+</file>
+<file java MappedFileDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import cn.edu.bjut.textmining.util.FileUtil;
+import java.io.RandomAccessFile;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+public class MappedFileDemo {
+    private static final int MAP_BYTES = 2048;
+    private static final int PREVIEW_CHARS = 220;
+    public static void main(String[] args) {
+        Path path = FileUtil.path("data", "chapter2", "tang-poems-classics.txt");
+        try (RandomAccessFile file = new RandomAccessFile(path.toFile(), "r");
+             FileChannel channel = file.getChannel()) {
+            long fileSize = channel.size();
+            long mapSize = Math.min(fileSize, MAP_BYTES);
+            MappedByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, mapSize);
+            String mappedText = StandardCharsets.UTF_8.decode(buffer).toString();
+            int previewEnd = Math.min(mappedText.length(), PREVIEW_CHARS);
+            int lastLineBreak = mappedText.lastIndexOf('\n', previewEnd);
+            if (lastLineBreak > 0) {
+                previewEnd = lastLineBreak;
+            }
+            System.out.println("文件大小：" + fileSize + " 字节");
+            System.out.println("本次映射：" + mapSize + " 字节");
+            System.out.println("文件头部内容：");
+            System.out.println(mappedText.substring(0, previewEnd));
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
+</file>
+===== 句子切分 =====
+<file java SentenceSplitDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import java.text.BreakIterator;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+public class SentenceSplitDemo {
+    private static final String DOT_PLACEHOLDER = "<DOT>";
+    private static final Pattern COMMON_TITLE = Pattern.compile(
+            "\\b(Mr|Mrs|Ms|Dr|Prof|Sr|Jr)\\.", Pattern.CASE_INSENSITIVE);
+    public static void main(String[] args) {
+        String text = "Mr. Smith likes Java. Text mining is useful!";
+        System.out.println(splitSentences(text));
+    }
+    public static List<String> splitSentences(String text) {
+        List<String> sentences = new ArrayList<String>();
+        if (text == null || text.trim().length() == 0) {
+            return sentences;
+        }
+        String protectedText = protectCommonTitles(text);
+        BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.US);
+        iterator.setText(protectedText);
+        int start = iterator.first();
+        int end = iterator.next();
+        while (end != BreakIterator.DONE) {
+            String sentence = protectedText.substring(start, end)
+                    .replace(DOT_PLACEHOLDER, ".")
+                    .trim();
+            if (sentence.length() > 0) {
+                sentences.add(sentence);
+            }
+            start = end;
+            end = iterator.next();
+        }
+        return sentences;
+    }
+    private static String protectCommonTitles(String text) {
+        Matcher matcher = COMMON_TITLE.matcher(text);
+        StringBuffer buffer = new StringBuffer();
+        while (matcher.find()) {
+            matcher.appendReplacement(
+                    buffer,
+                    Matcher.quoteReplacement(matcher.group(1) + DOT_PLACEHOLDER));
+        }
+        matcher.appendTail(buffer);
+        return buffer.toString();
+    }
+}
+</file>
+===== 分词 =====
+<file java OpenNlpSimpleTokenizerDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import opennlp.tools.tokenize.SimpleTokenizer;
+public class OpenNlpSimpleTokenizerDemo {
+    public static void main(String[] args) {
+        String text = "Text analysis and text mining are amazing!";
+        SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
+        System.out.println("词元：");
+        String[] tokens = tokenizer.tokenize(text);
+        for (String token : tokens) {
+            System.out.println(token);
+        }
+    }
+}
+</file>
+<file java HanlpSegmentDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import com.hankcs.hanlp.HanLP;
+import com.hankcs.hanlp.seg.common.Term;
+import java.util.ArrayList;
+import java.util.List;
+/**
+ * HanLP 中文分词示例。
+ * 中文文本没有天然的词间空格，成熟分词工具会结合词典与统计模型判断词语边界。
+ * 本示例与 ChineseTextPreprocessingDemo 的小词典最长优先分词使用同一条校园通知，
+ * 便于对照两类方法的切分差异。
+ */
+public class HanlpSegmentDemo {
+    public static void main(String[] args) {
+        String sentence =
+                "学习委员提醒：明天下午的《高等数学》习题课改到三教302，"
+                + "请带教材、笔、作业本和草稿纸！";
+        List<Term> terms = HanLP.segment(sentence);
+        System.out.println("原句：" + sentence);
+        List<String> words = new ArrayList<String>();
+        for (Term term : terms) {
+            words.add(term.word);
+        }
+        System.out.println("词元序列：");
+        System.out.println(words);
+        System.out.println("词元与词性：");
+        for (Term term : terms) {
+            System.out.println(term.word + "/" + term.nature);
+        }
+    }
+}
+</file>
+===== 词干提取 =====
+<file java PorterStemmerDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import cn.edu.bjut.textmining.util.FileUtil;
+import org.tartarus.snowball.ext.PorterStemmer;
+import java.nio.file.Path;
+import java.util.List;
+/** 使用 Lucene analyzers-common 中附带的 Snowball PorterStemmer。 */
+public class PorterStemmerDemo {
+    public static void main(String[] args) {
+        try {
+            Path sampleFile = FileUtil.path("data", "chapter2", "stemming-samples.txt");
+            List<String> words = FileUtil.readNonEmptyUtf8Lines(sampleFile);
+            for (String word : words) {
+                System.out.println(word + " -> " + stem(word));
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+    public static String stem(String word) {
+        PorterStemmer stemmer = new PorterStemmer();
+        stemmer.setCurrent(word);
+        stemmer.stem();
+        return stemmer.getCurrent();
+    }
+}
+</file>
+===== 词形还原 =====
+<file java OpenNlpLemmatizerDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import cn.edu.bjut.textmining.util.FileUtil;
+import opennlp.tools.lemmatizer.DictionaryLemmatizer;
+import java.io.File;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+/** 使用 OpenNLP 根据“词形 + 词性”查询词元。 */
+public class OpenNlpLemmatizerDemo {
+    public static void main(String[] args) {
+        try {
+            File dictionaryFile = FileUtil.path(
+                    "data", "chapter2", "en-lemmatizer.dict").toFile();
+            DictionaryLemmatizer lemmatizer = new DictionaryLemmatizer(dictionaryFile);
+            Path sampleFile = FileUtil.path(
+                    "data", "chapter2", "lemmatization-samples.tsv");
+            List<String> lines = FileUtil.readNonEmptyUtf8Lines(sampleFile);
+            List<String> tokenList = new ArrayList<String>();
+            List<String> tagList = new ArrayList<String>();
+            for (String line : lines) {
+                String[] parts = line.split("\\t");
+                if (parts.length != 2) {
+                    throw new IllegalArgumentException("词形还原样本格式错误：" + line);
+                }
+                tokenList.add(parts[0]);
+                tagList.add(parts[1]);
+            }
+            String[] tokens = tokenList.toArray(new String[tokenList.size()]);
+            String[] tags = tagList.toArray(new String[tagList.size()]);
+            String[] lemmas = lemmatizer.lemmatize(tokens, tags);
+            System.out.println("词形: " + Arrays.toString(tokens));
+            System.out.println("词性: " + Arrays.toString(tags));
+            System.out.println("词元: " + Arrays.toString(lemmas));
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
+</file>
+===== 停用词过滤 =====
+<file java StopWordsDemo.java>
+package cn.edu.bjut.textmining.chapter2;
+import cn.edu.bjut.textmining.util.FileUtil;
+import cn.edu.bjut.textmining.util.WordListUtil;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+public class StopWordsDemo {
+    public static void main(String[] args) {
+        try {
+            List<String> words = Arrays.asList("this", "is", "text", "mining",
+            		"in", "java", "book");
+            Path stopwordFile = FileUtil.path(
+                    "data", "chapter2", "basic-english-stopwords.txt");
+            Set<String> stopWords = WordListUtil.readLowerCaseSet(stopwordFile);
+            List<String> keywords = new ArrayList<String>();
+            for (String word : words) {
+                if (!stopWords.contains(word)) {
+                    keywords.add(word);
+                }
+            }
+            System.out.println(keywords);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
+</file>
+~~DISCUSSION~~

硕风徐徐

用户工具

站点工具

差别

页面工具