下载：Java文本处理基础

<dependency>
	<groupId>junit</groupId>
	<artifactId>junit</artifactId>
	<version>3.8.1</version>
	<scope>test</scope>
</dependency>
<dependency>
	<groupId>org.slf4j</groupId>
	<artifactId>slf4j-simple</artifactId>
	<version>2.0.18</version>
</dependency>
<dependency>
	<groupId>org.apache.opennlp</groupId>
	<artifactId>opennlp-tools</artifactId>
	<version>2.3.0</version>
</dependency>
<dependency>
	<groupId>org.apache.commons</groupId>
	<artifactId>commons-lang3</artifactId>
	<version>3.12.0</version>
</dependency>
<dependency>
	<groupId>commons-io</groupId>
	<artifactId>commons-io</artifactId>
	<version>2.11.0</version>
</dependency>
<dependency>
	<groupId>org.jsoup</groupId>
	<artifactId>jsoup</artifactId>
	<version>1.22.2</version>
</dependency>
<dependency>
	<groupId>org.apache.lucene</groupId>
	<artifactId>lucene-core</artifactId>
	<version>8.11.2</version>
</dependency>
<dependency>
	<groupId>org.apache.lucene</groupId>
	<artifactId>lucene-analyzers-common</artifactId>
	<version>8.11.2</version>
</dependency>

StringBuilderDemo.java

package cn.edu.bjut.textmining.chapter2;
 
import java.util.Arrays;
import java.util.List;
 
public class StringBuilderDemo {
    public static void main(String[] args) {
        List<String> lines = Arrays.asList("Text", "mining", "with", "Java");
 
        String content = "";
        for (String line : lines) {
            content += line + " "; 
        }
        System.out.println(content.trim());
 
        StringBuilder buffer = new StringBuilder();
        for (String line : lines) {
            buffer.append(line).append(" ");
        }
        String finalContent = buffer.toString().trim();
        System.out.println(finalContent);
 
        String joinedContent = String.join(" ", lines); 
        System.out.println(joinedContent);
    }
}

RegexDemo.java

package cn.edu.bjut.textmining.chapter2;
 
import java.util.regex.Matcher;
import java.util.regex.Pattern;
 
public class RegexDemo {
    public static void main(String[] args) {
        String rawText = "联系人: 李四, 电话: 138-1234-5678, 邮箱: lisi@company.com; "
                + "联系人: 张三, 电话: 159-0000-1111, 邮箱: zhangsan@university.edu.cn";
 
        extractInfo("手机号", "\\d{3}-\\d{4}-\\d{4}", rawText);
        extractInfo("邮箱", "[\\w\\.-]+@[\\w\\.-]+\\.[a-zA-Z]{2,}", rawText);
    }
 
    private static void extractInfo(String type, String regex, String content) {
        System.out.println("--- 提取 " + type + " ---");
        Pattern pattern = Pattern.compile(regex);
        Matcher matcher = pattern.matcher(content);
        while (matcher.find()) {
            System.out.println("找到: " + matcher.group()
                    + " (位置: " + matcher.start() + "-" + matcher.end() + ")");
        }
    }
}

TokenizerDemo.java

package cn.edu.bjut.textmining.chapter2;
 
import java.util.StringTokenizer;
 
public class TokenizerDemo {
    public static void main(String[] args) {
        String text = "Text analysis and text mining are amazing!";
        StringTokenizer tokenizer = new StringTokenizer(text);
 
        System.out.println("词元：");
        while (tokenizer.hasMoreTokens()) {
            System.out.println(tokenizer.nextToken());
        }
    }
}

StringUtilsSplitDemo.java

package cn.edu.bjut.textmining.chapter2;
 
import org.apache.commons.lang3.StringUtils;
 
import java.util.Arrays;
 
/** 展示 StringUtils 对 null、连续空白和首尾空白的安全处理。 */
public class StringUtilsSplitDemo {
    public static void main(String[] args) {
        String text = "  Java   text\tmining  ";
        String[] tokens = StringUtils.split(text);
        String safeNull = StringUtils.defaultString(null);
        String[] emptyTokens = StringUtils.split(safeNull);
 
        System.out.println(Arrays.toString(tokens));
        System.out.println("null 转为空串后是否没有词项: "
                + (emptyTokens == null || emptyTokens.length == 0));
    }
}

FileReadDemo.java

package cn.edu.bjut.textmining.chapter2;
 
import cn.edu.bjut.textmining.util.FileUtil;
 
import java.io.BufferedReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
 
public class FileReadDemo {
    public static void main(String[] args) {
        Path path = FileUtil.path("data", "chapter2", "utf8-text-reading-sample.txt");
        try (BufferedReader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
            for ( String line; (line = reader.readLine()) != null; ) {
                System.out.println(line);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

MappedFileDemo.java

package cn.edu.bjut.textmining.chapter2;
 
import cn.edu.bjut.textmining.util.FileUtil;
 
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
 
public class MappedFileDemo {
    private static final int MAP_BYTES = 2048;
    private static final int PREVIEW_CHARS = 220;
 
    public static void main(String[] args) {
        Path path = FileUtil.path("data", "chapter2", "tang-poems-classics.txt");
 
        try (RandomAccessFile file = new RandomAccessFile(path.toFile(), "r");
             FileChannel channel = file.getChannel()) {
 
            long fileSize = channel.size();
            long mapSize = Math.min(fileSize, MAP_BYTES);
            MappedByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, mapSize);
 
            String mappedText = StandardCharsets.UTF_8.decode(buffer).toString();
            int previewEnd = Math.min(mappedText.length(), PREVIEW_CHARS);
            int lastLineBreak = mappedText.lastIndexOf('\n', previewEnd);
            if (lastLineBreak > 0) {
                previewEnd = lastLineBreak;
            }
 
            System.out.println("文件大小：" + fileSize + " 字节");
            System.out.println("本次映射：" + mapSize + " 字节");
            System.out.println("文件头部内容：");
            System.out.println(mappedText.substring(0, previewEnd));
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}