第三章：文本表示

课件

UTF-8文件工具类

package cn.edu.bjut.textmining.util;
 
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
 
/**
 * UTF-8 文件读写工具类。
 */
public final class FileUtil {
	public static Path path(String first, String... more) {
		return Paths.get(first, more);
	}
 
	public static List<String> readUtf8Lines(Path path) {
		return readLines(path, StandardCharsets.UTF_8);
	}
 
	public static List<String> readNonEmptyUtf8Lines(Path path) {
		List<String> lines = readUtf8Lines(path);
		List<String> result = new ArrayList<String>();
		for (String line : lines) {
			if (line != null && line.trim().length() > 0) {
				result.add(line);
			}
		}
		return result;
	}
 
	public static List<String> readLines(Path path, Charset charset) {
		List<String> result = new ArrayList<String>();
		try (BufferedReader reader = Files.newBufferedReader(path, charset)) {
			for (String line; (line = reader.readLine()) != null;) {
				result.add(line);
			}
		} catch (IOException e) {
			throw new UncheckedIOException("读取文件失败：" + path.toAbsolutePath(), e);
		}
		return result;
	}
 
	public static String readUtf8Text(Path path) {
		List<String> lines = readUtf8Lines(path);
		StringBuilder builder = new StringBuilder();
		for (int i = 0; i < lines.size(); i++) {
			builder.append(lines.get(i));
			if (i < lines.size() - 1) {
				builder.append(System.lineSeparator());
			}
		}
		return builder.toString();
	}
 
	public static void writeUtf8Text(Path path, String content) {
		try {
			Path parent = path.getParent();
			if (parent != null) {
				Files.createDirectories(parent);
			}
			try (BufferedWriter writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) {
				writer.write(content);
			}
		} catch (IOException e) {
			throw new UncheckedIOException("写入文件失败：" + path.toAbsolutePath(), e);
		}
	}
}

基于Lucene的TF-IDF计算

第三方库依赖（POM.xml）

<dependency>
    <groupId>org.apache.lucene</groupId>
    <artifactId>lucene-core</artifactId>
    <version>8.11.2</version>
</dependency>
<dependency>
    <groupId>org.apache.lucene</groupId>
    <artifactId>lucene-analyzers-common</artifactId>
    <version>8.11.2</version>
</dependency>

源代码

LuceneTfIdfDemo.java

package cn.edu.bjut.textmining.chapter3;
 
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiTerms;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
 
import java.io.IOException;
import java.io.UncheckedIOException;
 
public class LuceneTfIdfDemo {
	public static void main(String[] args) {
		try (Directory directory = new ByteBuffersDirectory(); 
				Analyzer analyzer = new StandardAnalyzer()) {
			String doc1 = "Java Lucene is powerful.";
			String doc2 = "Lucene is a search library.";
 
			IndexWriterConfig config = new IndexWriterConfig(analyzer);
			try (IndexWriter writer = new IndexWriter(directory, config)) {
				addDocument(writer, doc1);
				addDocument(writer, doc2);
			}
 
			try (IndexReader reader = DirectoryReader.open(directory)) {
				int totalDocs = reader.numDocs();
				String targetTerm = "lucene";
				int docId = 0;
				Term term = new Term("content", targetTerm);
				long docFreq = reader.docFreq(term);
				double idf = Math.log((double) totalDocs / (docFreq + 1)) + 1.0;
 
				int tf = 0;
				PostingsEnum postings = MultiTerms.getTermPostingsEnum(reader, 
						"content", new BytesRef(targetTerm));
				if (postings != null && postings.advance(docId) == docId) {
					tf = postings.freq();
				}
 
				System.out.println("Total Documents: " + totalDocs);
				System.out.println("Term: " + targetTerm);
				System.out.println("TF (Frequency in doc0): " + tf);
				System.out.println("DocFreq (Docs containing term): " + docFreq);
				System.out.printf("IDF Value: %.4f%n", idf);
				System.out.printf("TF-IDF Score: %.4f%n", tf * idf);
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
 
	private static void addDocument(IndexWriter writer, String content) {
		try {
			Document document = new Document();
			document.add(new TextField("content", content, Field.Store.YES));
			writer.addDocument(document);
		} catch (IOException e) {
			throw new UncheckedIOException("写入 Lucene 文档失败", e);
		}
	}
}

训练word2Vec模型

第三方库依赖（POM.xml）

<dependency>
    <groupId>org.deeplearning4j</groupId>
    <artifactId>deeplearning4j-core</artifactId>
    <version>1.0.0-M2.1</version>
</dependency>
<dependency>
    <groupId>org.deeplearning4j</groupId>
    <artifactId>deeplearning4j-nlp</artifactId>
    <version>1.0.0-M2.1</version>
</dependency>
<!-- 必须指定后端，这里使用CPU版本，避免CUDA环境配置的麻烦 -->
<dependency>
    <groupId>org.nd4j</groupId>
    <artifactId>nd4j-native-platform</artifactId>
    <version>1.0.0-M2.1</version>
</dependency>

源代码

WordEmbeddingDemo.java

package cn.edu.bjut.textmining.chapter3;
 
import cn.edu.bjut.textmining.util.FileUtil;
import cn.edu.bjut.textmining.util.WordListUtil;
 
import org.deeplearning4j.models.embeddings.learning.impl.elements.CBOW;
import org.deeplearning4j.models.word2vec.VocabWord;
import org.deeplearning4j.models.word2vec.Word2Vec;
import org.deeplearning4j.text.sentenceiterator.BasicLineIterator;
import org.deeplearning4j.text.sentenceiterator.SentenceIterator;
import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor;
import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;
import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
 
import java.nio.file.Path;
import java.util.Collection;
import java.util.List;
import java.util.Set;
 
/**
 * Word2Vec 词嵌入示例。
 * 语料为 SIGHAN 2005 中文分词评测公开发布的《人民日报》1998 年 1 月语料
 * （北京大学标注版本），语料本身已按词切分。
 * 程序先过滤标点、数字和停用词，得到每行一句的清洗语料，
 * 再用 DeepLearning4J 训练 CBOW 词向量，观察词向量中的语义关系。
 */
public class WordEmbeddingDemo {
    public static void main(String[] args) {
        try {
            Path rawCorpus = FileUtil.path(
                    "data", "chapter3", "sighan2005-pku-training.txt");
            Path stopwordFile = FileUtil.path(
                    "data", "chapter3", "chinese-stopwords-hit.txt");
            Path cleanCorpus = FileUtil.path(
                    "output", "chapter3", "pku-clean.txt");
            Set<String> stopWords = WordListUtil.readLowerCaseSet(stopwordFile);
            int sentenceCount = prepareCorpus(rawCorpus, cleanCorpus, stopWords);
 
            SentenceIterator iterator = new BasicLineIterator(cleanCorpus.toFile());
            // finish() 释放迭代器持有的文件资源。
            try (AutoCloseable iteratorResource = iterator::finish) {
                TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory();
                tokenizerFactory.setTokenPreProcessor(new CommonPreprocessor());
 
                Word2Vec vectorModel = new Word2Vec.Builder()
                        .minWordFrequency(5)
                        .epochs(5)
                        .iterations(1)
                        .layerSize(100)
                        .seed(42)
                        .windowSize(5)
                        .workers(1)
                        .negativeSample(10)
                        .elementsLearningAlgorithm(new CBOW<VocabWord>())
                        .iterate(iterator)
                        .tokenizerFactory(tokenizerFactory)
                        .build();
 
                System.out.println("语料来源: 《人民日报》1998年1月（SIGHAN 2005 公开切分语料）");
                System.out.println("停用词文件: " + stopwordFile);
                System.out.println("清洗后句行数: " + sentenceCount);
                System.out.println("正在训练 Word2Vec 模型...");
                vectorModel.fit();
 
                String targetWord = "足球";
                if (!vectorModel.hasWord(targetWord)) {
                    throw new IllegalStateException("语料词表中没有“足球”");
                }
 
                double[] vector = vectorModel.getWordVector(targetWord);
                System.out.println("“足球”的向量前 5 维:");
                for (int i = 0; i < Math.min(5, vector.length); i++) {
                    System.out.printf("%.4f ", vector[i]);
                }
                System.out.println("...");
 
                Collection<String> nearest = vectorModel.wordsNearest(targetWord, 5);
                System.out.println("与“足球”最相似的词: " + nearest);
                printSimilarity(vectorModel, "足球", "篮球");
                printSimilarity(vectorModel, "足球", "排球");
                printSimilarity(vectorModel, "足球", "经济");
            }
 
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
 
    /**
     * 语料清洗：原语料已按词切分，逐行过滤非纯汉字词
     * （标点、全角数字、英文字母等）和停用词，写入每行一句的清洗语料。
     */
    private static int prepareCorpus(
            Path source, Path target, Set<String> stopWords) {
        List<String> lines = FileUtil.readUtf8Lines(source);
        StringBuilder builder = new StringBuilder();
        int sentenceCount = 0;
        for (String line : lines) {
            StringBuilder cleaned = new StringBuilder();
            for (String token : line.trim().split("\\s+")) {
                if (!token.matches("[\\u4e00-\\u9fff]+")) {
                    continue;
                }
                if (stopWords.contains(token)) {
                    continue;
                }
                if (cleaned.length() > 0) {
                    cleaned.append(' ');
                }
                cleaned.append(token);
            }
            if (cleaned.length() > 0) {
                builder.append(cleaned).append(System.lineSeparator());
                sentenceCount++;
            }
        }
        FileUtil.writeUtf8Text(target, builder.toString());
        return sentenceCount;
    }
 
    private static void printSimilarity(
            Word2Vec model, String left, String right) {
        if (model.hasWord(left) && model.hasWord(right)) {
            System.out.printf("“%s”与“%s”的余弦相似度: %.4f%n",
                    left, right, model.similarity(left, right));
        } else {
            System.out.println("跳过相似度计算，词表中缺少: " + right);
        }
    }
}

BERT预训练语言模型

第三方库依赖（POM.xml）

<dependency>
    <groupId>ai.djl</groupId>
    <artifactId>api</artifactId>
    <version>0.28.0</version>
</dependency>
<dependency>
    <groupId>ai.djl.pytorch</groupId>
    <artifactId>pytorch-engine</artifactId>
    <version>0.28.0</version>
</dependency>
<dependency>
    <groupId>ai.djl.pytorch</groupId>
    <artifactId>pytorch-model-zoo</artifactId>
    <version>0.28.0</version>
</dependency>
<dependency>
    <groupId>ai.djl.huggingface</groupId>
    <artifactId>tokenizers</artifactId>
    <version>0.28.0</version>
</dependency>

源代码

BertEmbeddingDemo.java

package cn.edu.bjut.textmining.chapter3;
 
import ai.djl.Application;
import ai.djl.huggingface.translator.TextEmbeddingTranslatorFactory;
import ai.djl.inference.Predictor;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
 
import java.util.Arrays;
 
public class BertEmbeddingDemo {
    public static void main(String[] args) {
        try {
            String text1 = "He went to the bank to deposit money.";
            String text2 = "He walked along the river bank.";
 
            Criteria<String, float[]> criteria = Criteria.builder()
                    .optApplication(Application.NLP.TEXT_EMBEDDING)
                    .setTypes(String.class, float[].class)
                    .optModelUrls("djl://ai.djl.huggingface.pytorch/"
                            + "sentence-transformers/all-MiniLM-L6-v2")
                    .optEngine("PyTorch")
                    .optTranslatorFactory(new TextEmbeddingTranslatorFactory())
                    .optProgress(new ProgressBar())
                    .build();
 
            try (ZooModel<String, float[]> model = criteria.loadModel();
                 Predictor<String, float[]> predictor = model.newPredictor()) {
                float[] embedding1 = predictor.predict(text1);
                float[] embedding2 = predictor.predict(text2);
 
                System.out.println("句子1: " + text1);
                System.out.println("向量维度: " + embedding1.length);
                System.out.println("前5位: "
                        + Arrays.toString(Arrays.copyOf(embedding1, 5)));
                System.out.println("句子2: " + text2);
                System.out.println("向量维度: " + embedding2.length);
                System.out.println("前5位: "
                        + Arrays.toString(Arrays.copyOf(embedding2, 5)));
                System.out.printf("余弦相似度: %.4f%n",
                        cosineSimilarity(embedding1, embedding2));
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
 
    private static double cosineSimilarity(float[] vectorA, float[] vectorB) {
        double dotProduct = 0.0;
        double normA = 0.0;
        double normB = 0.0;
        for (int i = 0; i < vectorA.length; i++) {
            dotProduct += vectorA[i] * vectorB[i];
            normA += vectorA[i] * vectorA[i];
            normB += vectorB[i] * vectorB[i];
        }
        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
    }
}

向量模型

OpenAICompatibleEmbeddingClient.java

package cn.edu.bjut.textmining.chapter3;
 
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
 
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.List;
 
/** 通过 OpenAI compatible API 获取文本向量。 */
public final class OpenAICompatibleEmbeddingClient {
    // 配置直接写在这里，无需设置环境变量。地址末尾不要添加 /embeddings。
    private static final String BASE_URL =
            "https://llm-8olx338nm0p2vj5q.cn-beijing.maas.aliyuncs.com/compatible-mode/v1";
 
    // 这是占位用的假密钥；运行前在自己的电脑上替换为真实密钥。
    private static final String API_KEY = "sk-ws-H.PHL" ;
 
    // qwen3.8-flash 是文本生成模型，无法用于本类的 /embeddings 向量请求。
    // 获取文本向量时，须填写服务支持的向量模型，如 qwen3.7-text-embedding，
    // 并将 BASE_URL 和 API_KEY 一起换成有权调用该向量服务的配置。
    // 官方模型列表：https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
    private static final String MODEL = "qwen3.7-text-embedding";
 
    private final String baseUrl;
    private final String apiKey;
    private final String model;
 
    public OpenAICompatibleEmbeddingClient(String baseUrl, String apiKey, String model) {
        this.baseUrl = requireText(baseUrl, "BASE_URL").replaceAll("/+$", "");
        this.apiKey = requireText(apiKey, "API_KEY");
        this.model = requireText(model, "MODEL");
        URI uri = URI.create(this.baseUrl);
        if (!("https".equals(uri.getScheme()) || "http".equals(uri.getScheme()))
                || uri.getHost() == null || uri.getQuery() != null
                || uri.getFragment() != null || uri.getUserInfo() != null) {
            throw new IllegalArgumentException("BASE_URL 必须是 HTTP(S) API 根地址，不含查询参数");
        }
    }
 
    /**
     * 供 EmbeddingApiDemo 调用。此方法直接使用上方常量，不读取环境变量。
     */
    public static OpenAICompatibleEmbeddingClient fromEnvironment() {
        return new OpenAICompatibleEmbeddingClient(BASE_URL, API_KEY, MODEL);
    }
 
    public String getModel() {
        return model;
    }
 
    /** 返回原始 JSON，调用者可将其保存后检查向量和请求信息。 */
    public String embed(List<String> texts) {
        if (texts == null || texts.isEmpty()) {
            throw new IllegalArgumentException("输入文本不能为空");
        }
        JsonArray input = new JsonArray();
        for (String text : texts) {
            input.add(requireText(text, "输入文本"));
        }
        JsonObject request = new JsonObject();
        request.addProperty("model", model);
        request.add("input", input);
        request.addProperty("encoding_format", "float");
        byte[] body = request.toString().getBytes(StandardCharsets.UTF_8);
        try {
            HttpURLConnection connection = (HttpURLConnection)
                    URI.create(baseUrl + "/embeddings").toURL().openConnection();
            // disconnect() 也在离开代码块时执行，包含请求失败的情况。
            try (AutoCloseable connectionResource = connection::disconnect) {
                connection.setRequestMethod("POST");
                connection.setConnectTimeout(15000);
                connection.setReadTimeout(60000);
                connection.setInstanceFollowRedirects(false);
                connection.setDoOutput(true);
                connection.setRequestProperty("Authorization", "Bearer " + apiKey);
                connection.setRequestProperty("Content-Type", "application/json; charset=UTF-8");
                connection.setRequestProperty("Accept", "application/json");
                connection.setFixedLengthStreamingMode(body.length);
                try (OutputStream output = connection.getOutputStream()) {
                    output.write(body);
                }
                int status = connection.getResponseCode();
                boolean success = status >= 200 && status < 300;
                String response = readResponse(success
                        ? connection.getInputStream() : connection.getErrorStream());
                if (!success) {
                    throw new IllegalStateException("向量 API 返回 HTTP " + status
                            + "：" + response.replace(apiKey, "[已隐藏密钥]"));
                }
                return response;
            }
        } catch (IOException e) {
            throw new UncheckedIOException("请求向量 API 失败", e);
        } catch (RuntimeException e) {
            throw e;
        } catch (Exception e) {
            throw new IllegalStateException("关闭 HTTP 连接失败", e);
        }
    }
 
    /** index 对应输入位置；响应数组的排列顺序不一定等于输入顺序。 */
    public static double[][] parseEmbeddings(String response, int inputCount) {
        if (inputCount <= 0) {
            throw new IllegalArgumentException("输入数量必须为正整数");
        }
        try {
            JsonArray data = JsonParser.parseString(response)
                    .getAsJsonObject().getAsJsonArray("data");
            if (data == null || data.size() != inputCount) {
                throw new IllegalArgumentException("返回向量数量与输入数量不一致");
            }
            double[][] vectors = new double[inputCount][];
            int dimension = -1;
            for (JsonElement element : data) {
                JsonObject item = element.getAsJsonObject();
                int index = item.get("index").getAsBigDecimal().intValueExact();
                if (index < 0 || index >= inputCount || vectors[index] != null) {
                    throw new IllegalArgumentException("响应 index 越界或重复：" + index);
                }
                JsonArray embedding = item.getAsJsonArray("embedding");
                if (embedding == null || embedding.size() == 0) {
                    throw new IllegalArgumentException("返回了空向量");
                }
                if (dimension == -1) {
                    dimension = embedding.size();
                }
                if (embedding.size() != dimension) {
                    throw new IllegalArgumentException("返回向量的维数不一致");
                }
                vectors[index] = new double[dimension];
                double norm = 0.0;
                for (int j = 0; j < dimension; j++) {
                    double value = embedding.get(j).getAsDouble();
                    if (!Double.isFinite(value)) {
                        throw new IllegalArgumentException("向量中存在非有限数值");
                    }
                    vectors[index][j] = value;
                    norm = Math.hypot(norm, value);
                }
                if (norm == 0.0 || !Double.isFinite(norm)) {
                    throw new IllegalArgumentException("向量范数必须是有限正数");
                }
            }
            for (double[] vector : vectors) {
                if (vector == null) {
                    throw new IllegalArgumentException("响应缺少输入对应的向量");
                }
            }
            return vectors;
        } catch (RuntimeException e) {
            throw new IllegalArgumentException("无法解析有效的 embeddings 响应", e);
        }
    }
 
    private static String readResponse(InputStream stream) {
        if (stream == null) {
            return "";
        }
        try (InputStream input = stream;
             ByteArrayOutputStream buffer = new ByteArrayOutputStream()) {
            byte[] bytes = new byte[4096];
            for (int count; (count = input.read(bytes)) != -1; ) {
                buffer.write(bytes, 0, count);
            }
            return new String(buffer.toByteArray(), StandardCharsets.UTF_8);
        } catch (IOException e) {
            throw new UncheckedIOException("读取向量 API 响应失败", e);
        }
    }
 
    private static String requireText(String value, String name) {
        if (value == null || value.trim().isEmpty()) {
            throw new IllegalArgumentException("请设置非空的 " + name);
        }
        return value.trim();
    }
}

上机作业

给定一段文本（中文、英文均可），实现特征选择方法（熵、相对熵、$\chi^2$统计量和互信息），并完成特征选择。
利用第三方库，完成GloVe词嵌入的训练。

目录

第三章：文本表示

课件

UTF-8文件工具类

基于Lucene的TF-IDF计算

第三方库依赖（POM.xml）

源代码

训练word2Vec模型

第三方库依赖（POM.xml）

源代码

BERT预训练语言模型

第三方库依赖（POM.xml）

源代码

向量模型

上机作业