package cn.edu.bjut.textmining.chapter5.support; import java.io.InputStream; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; import java.util.Locale; import cn.edu.bjut.textmining.chapter5.KMeansEngine; /** 将原始记录、特征矩阵行与聚类结果连接起来,不改变特征或聚类计算。 */ public final class DocumentRegistry { public static final String SMALL_CONFIG = "ch5-char12-rawtf-smoothidf-l2-v1"; public static final String LARGE_CONFIG = "ch5-char23-filter-df2-60pct-top5000-rawtf-smoothidf-l2-v1"; private DocumentRegistry() {} /** clusterByInput 使用算法内部的零基簇号;-1 表示零向量未进入聚类。 */ public static void write(String prefix, List records, TextFeatures.Result features, int[] clusterByInput, String config) { if (records.size() != features.vectors.length || records.size() != clusterByInput.length) { throw new IllegalArgumentException("原始记录、向量行和聚类状态必须一一对应"); } Path source = ProjectPaths.data("tnews_public/train.json"); List original = TNewsCorpusLoader.readAll(source); String sourceHash = sha256(source); List rows = new ArrayList(); rows.add("local_id\tsource_id\tsource_physical_line\tsource_file\tsource_file_sha256" + "\ttext_sha256\ttext\tpreprocess_version\tvector_row_0based\tfeature_dimension" + "\tstatus\tcluster_id\treference_label"); for (int i = 0; i < records.size(); i++) { TNewsCorpusLoader.NewsRecord record = records.get(i); int line = record.getSourceLine(); if (line < 1 || line > original.size() || !record.getTitle().equals(original.get(line - 1).getTitle()) || !record.getLabel().equals(original.get(line - 1).getLabel())) { throw new IllegalStateException("清单与原始文件不符:source_line=" + line); } boolean zero = KMeansEngine.squaredNorm(features.vectors[i]) == 0.0; if ((clusterByInput[i] < 0) != zero) { throw new IllegalArgumentException("只有零向量可以没有聚类结果,向量行=" + i); } String sourceId = String.format(Locale.ROOT, "tnews-train-%06d", line); String localId = ("news_sample".equals(prefix) ? "N" : "selection-") + (i + 1); rows.add(localId + "\t" + sourceId + "\t" + line + "\tdata/chapter5/tnews_public/train.json\t" + sourceHash + "\t" + sha256(record.getTitle().getBytes(StandardCharsets.UTF_8)) + "\t" + tsv(record.getTitle()) + "\t" + config + "\t" + i + "\t" + features.vocabulary.size() + "\t" + (zero ? "zero_vector" : "clustered") + "\t" + (zero ? "" : Integer.toString(clusterByInput[i] + 1)) + "\t" + record.getLabel()); } ResultWriter.lines(prefix + "_document_registry.tsv", rows); } private static String tsv(String value) { // JSON 风格反斜杠转义,避免原文制表符/换行破坏列结构。 return value.replace("\\", "\\\\").replace("\t", "\\t") .replace("\r", "\\r").replace("\n", "\\n"); } private static MessageDigest digest() { try { return MessageDigest.getInstance("SHA-256"); } catch (NoSuchAlgorithmException e) { throw new IllegalStateException("JDK 缺少 SHA-256", e); } } public static String sha256(Path path) { MessageDigest digest = digest(); try (InputStream in = Files.newInputStream(path)) { byte[] buffer = new byte[8192]; int count; while ((count = in.read(buffer)) != -1) { digest.update(buffer, 0, count); } return hex(digest.digest()); } catch (IOException e) { throw new UncheckedIOException("无法校验来源文件:" + path, e); } } private static String sha256(byte[] bytes) { return hex(digest().digest(bytes)); } private static String hex(byte[] bytes) { StringBuilder result = new StringBuilder(); for (byte b : bytes) result.append(String.format(Locale.ROOT, "%02x", b & 0xff)); return result.toString(); } }