硕风徐徐

用户工具

站点工具

您的足迹: • ch03

侧边栏

文本分析与文本挖掘

返回课堂教学

zh:courses:textmining2026:ch03

这是本文档旧的修订版！

目录

第三章 文本表示

课件

下载：文本表示

UTF-8文件工具类

FileUtil.java
package cn.edu.bjut.textmining.util;
 
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
 
/**
 * UTF-8 文件读写工具类。
 */
public final class FileUtil {
	public static Path path(String first, String... more) {
		return Paths.get(first, more);
	}
 
	public static List<String> readUtf8Lines(Path path) {
		return readLines(path, StandardCharsets.UTF_8);
	}
 
	public static List<String> readNonEmptyUtf8Lines(Path path) {
		List<String> lines = readUtf8Lines(path);
		List<String> result = new ArrayList<String>();
		for (String line : lines) {
			if (line != null && line.trim().length() > 0) {
				result.add(line);
			}
		}
		return result;
	}
 
	public static List<String> readLines(Path path, Charset charset) {
		List<String> result = new ArrayList<String>();
		try (BufferedReader reader = Files.newBufferedReader(path, charset)) {
			for (String line; (line = reader.readLine()) != null;) {
				result.add(line);
			}
		} catch (IOException e) {
			throw new UncheckedIOException("读取文件失败：" + path.toAbsolutePath(), e);
		}
		return result;
	}
 
	public static String readUtf8Text(Path path) {
		List<String> lines = readUtf8Lines(path);
		StringBuilder builder = new StringBuilder();
		for (int i = 0; i < lines.size(); i++) {
			builder.append(lines.get(i));
			if (i < lines.size() - 1) {
				builder.append(System.lineSeparator());
			}
		}
		return builder.toString();
	}
 
	public static void writeUtf8Text(Path path, String content) {
		try {
			Path parent = path.getParent();
			if (parent != null) {
				Files.createDirectories(parent);
			}
			try (BufferedWriter writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) {
				writer.write(content);
			}
		} catch (IOException e) {
			throw new UncheckedIOException("写入文件失败：" + path.toAbsolutePath(), e);
		}
	}
}

基于Lucene的TF-IDF计算

第三方库依赖（POM.xml）

<dependency>
    <groupId>org.apache.lucene</groupId>
    <artifactId>lucene-core</artifactId>
    <version>8.11.2</version>
</dependency>
<dependency>
    <groupId>org.apache.lucene</groupId>
    <artifactId>lucene-analyzers-common</artifactId>
    <version>8.11.2</version>
</dependency>

源代码

LuceneTfIdfDemo.java
package cn.edu.bjut.textmining.chapter3;
 
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiTerms;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
 
import java.io.IOException;
import java.io.UncheckedIOException;
 
public class LuceneTfIdfDemo {
	public static void main(String[] args) {
		try (Directory directory = new ByteBuffersDirectory(); 
				Analyzer analyzer = new StandardAnalyzer()) {
			String doc1 = "Java Lucene is powerful.";
			String doc2 = "Lucene is a search library.";
 
			IndexWriterConfig config = new IndexWriterConfig(analyzer);
			try (IndexWriter writer = new IndexWriter(directory, config)) {
				addDocument(writer, doc1);
				addDocument(writer, doc2);
			}
 
			try (IndexReader reader = DirectoryReader.open(directory)) {
				int totalDocs = reader.numDocs();
				String targetTerm = "lucene";
				int docId = 0;
				Term term = new Term("content", targetTerm);
				long docFreq = reader.docFreq(term);
				double idf = Math.log((double) totalDocs / (docFreq + 1)) + 1.0;
 
				int tf = 0;
				PostingsEnum postings = MultiTerms.getTermPostingsEnum(reader, 
						"content", new BytesRef(targetTerm));
				if (postings != null && postings.advance(docId) == docId) {
					tf = postings.freq();
				}
 
				System.out.println("Total Documents: " + totalDocs);
				System.out.println("Term: " + targetTerm);
				System.out.println("TF (Frequency in doc0): " + tf);
				System.out.println("DocFreq (Docs containing term): " + docFreq);
				System.out.printf("IDF Value: %.4f%n", idf);
				System.out.printf("TF-IDF Score: %.4f%n", tf * idf);
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
 
	private static void addDocument(IndexWriter writer, String content) {
		try {
			Document document = new Document();
			document.add(new TextField("content", content, Field.Store.YES));
			writer.addDocument(document);
		} catch (IOException e) {
			throw new UncheckedIOException("写入 Lucene 文档失败", e);
		}
	}
}

训练word2Vec模型

第三方库依赖（POM.xml）

<dependency>
    <groupId>org.deeplearning4j</groupId>
    <artifactId>deeplearning4j-core</artifactId>
    <version>1.0.0-M2.1</version>
</dependency>
<dependency>
    <groupId>org.deeplearning4j</groupId>
    <artifactId>deeplearning4j-nlp</artifactId>
    <version>1.0.0-M2.1</version>
</dependency>
<!-- 必须指定后端，这里使用CPU版本，避免CUDA环境配置的麻烦 -->
<dependency>
    <groupId>org.nd4j</groupId>
    <artifactId>nd4j-native-platform</artifactId>
    <version>1.0.0-M2.1</version>
</dependency>

源代码

WordEmbeddingDemo.java
package cn.edu.bjut.textmining.chapter3;
 
import cn.edu.bjut.textmining.util.FileUtil;
import cn.edu.bjut.textmining.util.WordListUtil;
 
import org.deeplearning4j.models.embeddings.learning.impl.elements.CBOW;
import org.deeplearning4j.models.word2vec.VocabWord;
import org.deeplearning4j.models.word2vec.Word2Vec;
import org.deeplearning4j.text.sentenceiterator.BasicLineIterator;
import org.deeplearning4j.text.sentenceiterator.SentenceIterator;
import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor;
import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;
import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
 
import java.nio.file.Path;
import java.util.Collection;
import java.util.List;
import java.util.Set;
 
/**
 * Word2Vec 词嵌入示例。
 * 语料为 SIGHAN 2005 中文分词评测公开发布的《人民日报》1998 年 1 月语料
 * （北京大学标注版本），语料本身已按词切分。
 * 程序先过滤标点、数字和停用词，得到每行一句的清洗语料，
 * 再用 DeepLearning4J 训练 CBOW 词向量，观察词向量中的语义关系。
 */
public class WordEmbeddingDemo {
    public static void main(String[] args) {
        try {
            Path rawCorpus = FileUtil.path(
                    "data", "chapter3", "sighan2005-pku-training.txt");
            Path stopwordFile = FileUtil.path(
                    "data", "chapter3", "chinese-stopwords-hit.txt");
            Path cleanCorpus = FileUtil.path(
                    "output", "chapter3", "pku-clean.txt");
            Set<String> stopWords = WordListUtil.readLowerCaseSet(stopwordFile);
            int sentenceCount = prepareCorpus(rawCorpus, cleanCorpus, stopWords);
 
            SentenceIterator iterator = new BasicLineIterator(cleanCorpus.toFile());
            // finish() 释放迭代器持有的文件资源。
            try (AutoCloseable iteratorResource = iterator::finish) {
                TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory();
                tokenizerFactory.setTokenPreProcessor(new CommonPreprocessor());
 
                Word2Vec vectorModel = new Word2Vec.Builder()
                        .minWordFrequency(5)
                        .epochs(5)
                        .iterations(1)
                        .layerSize(100)
                        .seed(42)
                        .windowSize(5)
                        .workers(1)
                        .negativeSample(10)
                        .elementsLearningAlgorithm(new CBOW<VocabWord>())
                        .iterate(iterator)
                        .tokenizerFactory(tokenizerFactory)
                        .build();
 
                System.out.println("语料来源: 《人民日报》1998年1月（SIGHAN 2005 公开切分语料）");
                System.out.println("停用词文件: " + stopwordFile);
                System.out.println("清洗后句行数: " + sentenceCount);
                System.out.println("正在训练 Word2Vec 模型...");
                vectorModel.fit();
 
                String targetWord = "足球";
                if (!vectorModel.hasWord(targetWord)) {
                    throw new IllegalStateException("语料词表中没有“足球”");
                }
 
                double[] vector = vectorModel.getWordVector(targetWord);
                System.out.println("“足球”的向量前 5 维:");
                for (int i = 0; i < Math.min(5, vector.length); i++) {
                    System.out.printf("%.4f ", vector[i]);
                }
                System.out.println("...");
 
                Collection<String> nearest = vectorModel.wordsNearest(targetWord, 5);
                System.out.println("与“足球”最相似的词: " + nearest);
                printSimilarity(vectorModel, "足球", "篮球");
                printSimilarity(vectorModel, "足球", "排球");
                printSimilarity(vectorModel, "足球", "经济");
            }
 
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
 
    /**
     * 语料清洗：原语料已按词切分，逐行过滤非纯汉字词
     * （标点、全角数字、英文字母等）和停用词，写入每行一句的清洗语料。
     */
    private static int prepareCorpus(
            Path source, Path target, Set<String> stopWords) {
        List<String> lines = FileUtil.readUtf8Lines(source);
        StringBuilder builder = new StringBuilder();
        int sentenceCount = 0;
        for (String line : lines) {
            StringBuilder cleaned = new StringBuilder();
            for (String token : line.trim().split("\\s+")) {
                if (!token.matches("[\\u4e00-\\u9fff]+")) {
                    continue;
                }
                if (stopWords.contains(token)) {
                    continue;
                }
                if (cleaned.length() > 0) {
                    cleaned.append(' ');
                }
                cleaned.append(token);
            }
            if (cleaned.length() > 0) {
                builder.append(cleaned).append(System.lineSeparator());
                sentenceCount++;
            }
        }
        FileUtil.writeUtf8Text(target, builder.toString());
        return sentenceCount;
    }
 
    private static void printSimilarity(
            Word2Vec model, String left, String right) {
        if (model.hasWord(left) && model.hasWord(right)) {
            System.out.printf("“%s”与“%s”的余弦相似度: %.4f%n",
                    left, right, model.similarity(left, right));
        } else {
            System.out.println("跳过相似度计算，词表中缺少: " + right);
        }
    }
}

BERT预训练语言模型

第三方库依赖（POM.xml）

<dependency>
    <groupId>ai.djl</groupId>
    <artifactId>api</artifactId>
    <version>0.28.0</version>
</dependency>
<dependency>
    <groupId>ai.djl.pytorch</groupId>
    <artifactId>pytorch-engine</artifactId>
    <version>0.28.0</version>
</dependency>
<dependency>
    <groupId>ai.djl.pytorch</groupId>
    <artifactId>pytorch-model-zoo</artifactId>
    <version>0.28.0</version>
</dependency>
<dependency>
    <groupId>ai.djl.huggingface</groupId>
    <artifactId>tokenizers</artifactId>
    <version>0.28.0</version>
</dependency>

源代码

BertEmbeddingDemo.java
package cn.edu.bjut.textmining.chapter3;
 
import ai.djl.Application;
import ai.djl.huggingface.translator.TextEmbeddingTranslatorFactory;
import ai.djl.inference.Predictor;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
 
import java.util.Arrays;
 
public class BertEmbeddingDemo {
    public static void main(String[] args) {
        try {
            String text1 = "He went to the bank to deposit money.";
            String text2 = "He walked along the river bank.";
 
            Criteria<String, float[]> criteria = Criteria.builder()
                    .optApplication(Application.NLP.TEXT_EMBEDDING)
                    .setTypes(String.class, float[].class)
                    .optModelUrls("djl://ai.djl.huggingface.pytorch/"
                            + "sentence-transformers/all-MiniLM-L6-v2")
                    .optEngine("PyTorch")
                    .optTranslatorFactory(new TextEmbeddingTranslatorFactory())
                    .optProgress(new ProgressBar())
                    .build();
 
            try (ZooModel<String, float[]> model = criteria.loadModel();
                 Predictor<String, float[]> predictor = model.newPredictor()) {
                float[] embedding1 = predictor.predict(text1);
                float[] embedding2 = predictor.predict(text2);
 
                System.out.println("句子1: " + text1);
                System.out.println("向量维度: " + embedding1.length);
                System.out.println("前5位: "
                        + Arrays.toString(Arrays.copyOf(embedding1, 5)));
                System.out.println("句子2: " + text2);
                System.out.println("向量维度: " + embedding2.length);
                System.out.println("前5位: "
                        + Arrays.toString(Arrays.copyOf(embedding2, 5)));
                System.out.printf("余弦相似度: %.4f%n",
                        cosineSimilarity(embedding1, embedding2));
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
 
    private static double cosineSimilarity(float[] vectorA, float[] vectorB) {
        double dotProduct = 0.0;
        double normA = 0.0;
        double normB = 0.0;
        for (int i = 0; i < vectorA.length; i++) {
            dotProduct += vectorA[i] * vectorB[i];
            normA += vectorA[i] * vectorA[i];
            normB += vectorB[i] * vectorB[i];
        }
        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
    }
}

上机作业

  1. 给定一段文本（中文、英文均可），实现特征选择方法（熵、相对熵、$\chi^2$统计量和互信息），并完成特征选择。

评论

请输入您的评论. 可以使用维基语法:
172 -9 = 
 
zh/courses/textmining2026/ch03.1789372955.txt.gz · 最后更改: 2026/09/14 16:02 由 pzczxs

页面工具

网站备案号：京ICP备18021658号