硕风徐徐

用户工具

站点工具

您的足迹: • ch02

侧边栏

文本分析与文本挖掘

返回课堂教学

zh:courses:textmining2026:ch02

这是本文档旧的修订版！

目录

第二章：Java文本处理基础

课件

下载：Java文本处理基础

第三方库依赖

<dependency>
	<groupId>junit</groupId>
	<artifactId>junit</artifactId>
	<version>3.8.1</version>
	<scope>test</scope>
</dependency>
<dependency>
	<groupId>org.slf4j</groupId>
	<artifactId>slf4j-simple</artifactId>
	<version>2.0.18</version>
</dependency>
<dependency>
	<groupId>org.apache.opennlp</groupId>
	<artifactId>opennlp-tools</artifactId>
	<version>2.3.0</version>
</dependency>
<dependency>
	<groupId>org.apache.commons</groupId>
	<artifactId>commons-lang3</artifactId>
	<version>3.12.0</version>
</dependency>
<dependency>
	<groupId>commons-io</groupId>
	<artifactId>commons-io</artifactId>
	<version>2.11.0</version>
</dependency>
<dependency>
	<groupId>org.jsoup</groupId>
	<artifactId>jsoup</artifactId>
	<version>1.22.2</version>
</dependency>
<dependency>
	<groupId>org.apache.lucene</groupId>
	<artifactId>lucene-core</artifactId>
	<version>8.11.2</version>
</dependency>
<dependency>
	<groupId>org.apache.lucene</groupId>
	<artifactId>lucene-analyzers-common</artifactId>
	<version>8.11.2</version>
</dependency>

String与StringBuilder类对比

StringBuilderDemo.java
package cn.edu.bjut.textmining.chapter2;
 
import java.util.Arrays;
import java.util.List;
 
public class StringBuilderDemo {
    public static void main(String[] args) {
        List<String> lines = Arrays.asList("Text", "mining", "with", "Java");
 
        String content = "";
        for (String line : lines) {
            content += line + " "; 
        }
        System.out.println(content.trim());
 
        StringBuilder buffer = new StringBuilder();
        for (String line : lines) {
            buffer.append(line).append(" ");
        }
        String finalContent = buffer.toString().trim();
        System.out.println(finalContent);
 
        String joinedContent = String.join(" ", lines); 
        System.out.println(joinedContent);
    }
}

正则表达式

RegexDemo.java
package cn.edu.bjut.textmining.chapter2;
 
import java.util.regex.Matcher;
import java.util.regex.Pattern;
 
public class RegexDemo {
    public static void main(String[] args) {
        String rawText = "联系人: 李四, 电话: 138-1234-5678, 邮箱: lisi@company.com; "
                + "联系人: 张三, 电话: 159-0000-1111, 邮箱: zhangsan@university.edu.cn";
 
        extractInfo("手机号", "\\d{3}-\\d{4}-\\d{4}", rawText);
        extractInfo("邮箱", "[\\w\\.-]+@[\\w\\.-]+\\.[a-zA-Z]{2,}", rawText);
    }
 
    private static void extractInfo(String type, String regex, String content) {
        System.out.println("--- 提取 " + type + " ---");
        Pattern pattern = Pattern.compile(regex);
        Matcher matcher = pattern.matcher(content);
        while (matcher.find()) {
            System.out.println("找到: " + matcher.group()
                    + " (位置: " + matcher.start() + "-" + matcher.end() + ")");
        }
    }
}

Java类库中的文本处理工具

TokenizerDemo.java
package cn.edu.bjut.textmining.chapter2;
 
import java.util.StringTokenizer;
 
public class TokenizerDemo {
    public static void main(String[] args) {
        String text = "Text analysis and text mining are amazing!";
        StringTokenizer tokenizer = new StringTokenizer(text);
 
        System.out.println("词元：");
        while (tokenizer.hasMoreTokens()) {
            System.out.println(tokenizer.nextToken());
        }
    }
}
StringUtilsSplitDemo.java
package cn.edu.bjut.textmining.chapter2;
 
import org.apache.commons.lang3.StringUtils;
 
import java.util.Arrays;
 
/** 展示 StringUtils 对 null、连续空白和首尾空白的安全处理。 */
public class StringUtilsSplitDemo {
    public static void main(String[] args) {
        String text = "  Java   text\tmining  ";
        String[] tokens = StringUtils.split(text);
        String safeNull = StringUtils.defaultString(null);
        String[] emptyTokens = StringUtils.split(safeNull);
 
        System.out.println(Arrays.toString(tokens));
        System.out.println("null 转为空串后是否没有词项: "
                + (emptyTokens == null || emptyTokens.length == 0));
    }
}

文件读写与编码处理

FileReadDemo.java
package cn.edu.bjut.textmining.chapter2;
 
import cn.edu.bjut.textmining.util.FileUtil;
 
import java.io.BufferedReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
 
public class FileReadDemo {
    public static void main(String[] args) {
        Path path = FileUtil.path("data", "chapter2", "utf8-text-reading-sample.txt");
        try (BufferedReader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
            for ( String line; (line = reader.readLine()) != null; ) {
                System.out.println(line);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
MappedFileDemo.java
package cn.edu.bjut.textmining.chapter2;
 
import cn.edu.bjut.textmining.util.FileUtil;
 
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
 
public class MappedFileDemo {
    private static final int MAP_BYTES = 2048;
    private static final int PREVIEW_CHARS = 220;
 
    public static void main(String[] args) {
        Path path = FileUtil.path("data", "chapter2", "tang-poems-classics.txt");
 
        try (RandomAccessFile file = new RandomAccessFile(path.toFile(), "r");
             FileChannel channel = file.getChannel()) {
 
            long fileSize = channel.size();
            long mapSize = Math.min(fileSize, MAP_BYTES);
            MappedByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, mapSize);
 
            String mappedText = StandardCharsets.UTF_8.decode(buffer).toString();
            int previewEnd = Math.min(mappedText.length(), PREVIEW_CHARS);
            int lastLineBreak = mappedText.lastIndexOf('\n', previewEnd);
            if (lastLineBreak > 0) {
                previewEnd = lastLineBreak;
            }
 
            System.out.println("文件大小：" + fileSize + " 字节");
            System.out.println("本次映射：" + mapSize + " 字节");
            System.out.println("文件头部内容：");
            System.out.println(mappedText.substring(0, previewEnd));
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

句子切分

SentenceSplitDemo.java
package cn.edu.bjut.textmining.chapter2;
 
import java.text.BreakIterator;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
 
public class SentenceSplitDemo {
    private static final String DOT_PLACEHOLDER = "<DOT>";
    private static final Pattern COMMON_TITLE = Pattern.compile(
            "\\b(Mr|Mrs|Ms|Dr|Prof|Sr|Jr)\\.", Pattern.CASE_INSENSITIVE);
 
    public static void main(String[] args) {
        String text = "Mr. Smith likes Java. Text mining is useful!";
        System.out.println(splitSentences(text));
    }
 
    public static List<String> splitSentences(String text) {
        List<String> sentences = new ArrayList<String>();
        if (text == null || text.trim().length() == 0) {
            return sentences;
        }
 
        String protectedText = protectCommonTitles(text);
        BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.US);
        iterator.setText(protectedText);
        int start = iterator.first();
        int end = iterator.next();
        while (end != BreakIterator.DONE) {
            String sentence = protectedText.substring(start, end)
                    .replace(DOT_PLACEHOLDER, ".")
                    .trim();
            if (sentence.length() > 0) {
                sentences.add(sentence);
            }
            start = end;
            end = iterator.next();
        }
        return sentences;
    }
 
    private static String protectCommonTitles(String text) {
        Matcher matcher = COMMON_TITLE.matcher(text);
        StringBuffer buffer = new StringBuffer();
        while (matcher.find()) {
            matcher.appendReplacement(
                    buffer,
                    Matcher.quoteReplacement(matcher.group(1) + DOT_PLACEHOLDER));
        }
        matcher.appendTail(buffer);
        return buffer.toString();
    }
}

分词

OpenNlpSimpleTokenizerDemo.java
package cn.edu.bjut.textmining.chapter2;
 
import opennlp.tools.tokenize.SimpleTokenizer;
 
public class OpenNlpSimpleTokenizerDemo {
    public static void main(String[] args) {
        String text = "Text analysis and text mining are amazing!";
        SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
 
        System.out.println("词元：");
        String[] tokens = tokenizer.tokenize(text);
        for (String token : tokens) {
            System.out.println(token);
        }
    }
}

词干提取

PorterStemmerDemo.java
package cn.edu.bjut.textmining.chapter2;
 
import cn.edu.bjut.textmining.util.FileUtil;
import org.tartarus.snowball.ext.PorterStemmer;
 
import java.nio.file.Path;
import java.util.List;
 
/** 使用 Lucene analyzers-common 中附带的 Snowball PorterStemmer。 */
public class PorterStemmerDemo {
    public static void main(String[] args) {
        try {
            Path sampleFile = FileUtil.path("data", "chapter2", "stemming-samples.txt");
            List<String> words = FileUtil.readNonEmptyUtf8Lines(sampleFile);
            for (String word : words) {
                System.out.println(word + " -> " + stem(word));
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
 
    public static String stem(String word) {
        PorterStemmer stemmer = new PorterStemmer();
        stemmer.setCurrent(word);
        stemmer.stem();
        return stemmer.getCurrent();
    }
}

词形还原

OpenNlpLemmatizerDemo.java
package cn.edu.bjut.textmining.chapter2;
 
import cn.edu.bjut.textmining.util.FileUtil;
import opennlp.tools.lemmatizer.DictionaryLemmatizer;
 
import java.io.File;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
 
/** 使用 OpenNLP 根据“词形 + 词性”查询词元。 */
public class OpenNlpLemmatizerDemo {
    public static void main(String[] args) {
        try {
            File dictionaryFile = FileUtil.path(
                    "data", "chapter2", "en-lemmatizer.dict").toFile();
            DictionaryLemmatizer lemmatizer = new DictionaryLemmatizer(dictionaryFile);
 
            Path sampleFile = FileUtil.path(
                    "data", "chapter2", "lemmatization-samples.tsv");
            List<String> lines = FileUtil.readNonEmptyUtf8Lines(sampleFile);
            List<String> tokenList = new ArrayList<String>();
            List<String> tagList = new ArrayList<String>();
            for (String line : lines) {
                String[] parts = line.split("\\t");
                if (parts.length != 2) {
                    throw new IllegalArgumentException("词形还原样本格式错误：" + line);
                }
                tokenList.add(parts[0]);
                tagList.add(parts[1]);
            }
 
            String[] tokens = tokenList.toArray(new String[tokenList.size()]);
            String[] tags = tagList.toArray(new String[tagList.size()]);
            String[] lemmas = lemmatizer.lemmatize(tokens, tags);
 
            System.out.println("词形: " + Arrays.toString(tokens));
            System.out.println("词性: " + Arrays.toString(tags));
            System.out.println("词元: " + Arrays.toString(lemmas));
 
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

停用词过滤

StopWordsDemo.java
package cn.edu.bjut.textmining.chapter2;
 
import cn.edu.bjut.textmining.util.FileUtil;
import cn.edu.bjut.textmining.util.WordListUtil;
 
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
 
public class StopWordsDemo {
    public static void main(String[] args) {
        try {
            List<String> words = Arrays.asList("this", "is", "text", "mining", 
            		"in", "java", "book");
            Path stopwordFile = FileUtil.path(
                    "data", "chapter2", "basic-english-stopwords.txt");
            Set<String> stopWords = WordListUtil.readLowerCaseSet(stopwordFile);
 
            List<String> keywords = new ArrayList<String>();
            for (String word : words) {
                if (!stopWords.contains(word)) {
                    keywords.add(word);
                }
            }
            System.out.println(keywords);
 
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

评论

请输入您的评论. 可以使用维基语法:
102​ +7​ =
 
zh/courses/textmining2026/ch02.1788680338.txt.gz · 最后更改: 2026/09/06 15:38 由 pzczxs

页面工具

网站备案号：京ICP备18021658号