差别

这里会显示出您选择的修订版和当前版本之间的差别。

--- zh:courses:textmining2026:ch04 [2026/09/28 17:42]
pzczxs [源代码]
+++ zh:courses:textmining2026:ch04 [2026/10/13 09:51] (当前版本)
pzczxs [上机作业]
@@ 行 3: / 行 3: @@
 下载：{{ :zh:courses:textmining2026:ch04.pptx |文本分类}}
+===== 数据集 =====
+下载：{{ :zh:courses:textmining2026:chapter4.zip |}}
 ===== 工具类 =====
 ==== 第三方依赖 ====
@@ 行 90: / 行 92: @@
 </file>
 ===== 支持类 =====
 <file java EvaluationSupport.java>
 package cn.edu.bjut.textmining.chapter4.support;
@@ 行 507: / 行 508: @@
     public double idf(int index) {
         return idf[index];
+    }
+}
+</file>
+<file java NewsClassifier.java>
+package cn.edu.bjut.textmining.chapter4.support;
+import java.util.List;
+/** 体育为正类；分数可为类别概率、近邻分布或SVM间隔。 */
+public interface NewsClassifier {
+    void fit(List<NewsRecord> training);
+    double score(NewsRecord record);
+    double threshold();
+    String name();
+    default String predict(NewsRecord record) {
+        return score(record) >= threshold() ? "sports" : "finance";
     }
 }
@@ 行 605: / 行 627: @@
 </file>
-<file code OpenNlpNaiveBayesDemo.java>
+<file java OpenNlpNaiveBayesDemo.java>
 package cn.edu.bjut.textmining.chapter4;
@@ 行 628: / 行 650: @@
 }
 </file>
+===== kNN分类器 =====
+==== 第三方依赖 ====
+<code xml>
+<dependency>
+    <groupId>nz.ac.waikato.cms.weka</groupId>
+    <artifactId>weka-stable</artifactId>
+    <version>3.8.6</version>
+</dependency>
+</code>
+==== 源代码 ====
+<file java WekaNewsClassifier.java>
+package cn.edu.bjut.textmining.chapter4;
+import weka.classifiers.lazy.IBk;
+import weka.core.Attribute;
+import weka.core.EuclideanDistance;
+import weka.core.Instance;
+import weka.core.Instances;
+import weka.core.SparseInstance;
+import weka.core.neighboursearch.LinearNNSearch;
+import java.util.ArrayList;
+import java.util.List;
+import cn.edu.bjut.textmining.chapter4.support.NewsClassifier;
+import cn.edu.bjut.textmining.chapter4.support.NewsRecord;
+import cn.edu.bjut.textmining.chapter4.support.TfidfVectorizer;
+/** 高维TF-IDF采用线性扫描近邻；关闭逐列极差归一化。 */
+public final class WekaNewsClassifier implements NewsClassifier {
+    public static final int K = 5;
+    private final TfidfVectorizer vectorizer = new TfidfVectorizer();
+    private final IBk classifier = new IBk(K);
+    private Instances header;
+    private double sportsPrior;
+    public void fit(List<NewsRecord> training) {
+        vectorizer.fit(training);
+        int sports = 0;
+        for (NewsRecord r : training) {
+            if (r.isSports()) {
+                sports++;
+            }
+        }
+        sportsPrior = sports / (double) training.size();
+        ArrayList<Attribute> attrs = new ArrayList<Attribute>();
+        for (String term : vectorizer.terms()) {
+            attrs.add(new Attribute(term));
+        }
+        ArrayList<String> labels = new ArrayList<String>();
+        labels.add("finance");
+        labels.add("sports");
+        attrs.add(new Attribute("class", labels));
+        header = new Instances("Tnews", attrs, training.size());
+        header.setClassIndex(attrs.size() - 1);
+        for (NewsRecord r : training) {
+            // 零向量没有方向，不能作为单位TF-IDF的语义近邻。
+            if (TfidfVectorizer.isZero(vectorizer.transform(r))) {
+                continue;
+            }
+            Instance x = instance(r);
+            x.setValue(header.classIndex(), r.label);
+            header.add(x);
+        }
+        EuclideanDistance distance = new EuclideanDistance();
+        distance.setDontNormalize(true);
+        LinearNNSearch search = new LinearNNSearch();
+        try {
+            search.setDistanceFunction(distance);
+            classifier.setNearestNeighbourSearchAlgorithm(search);
+            classifier.buildClassifier(header);
+            System.out.println(
+                    "kNN可用训练向量："
+                            + header.numInstances()
+                            + "；排除零向量："
+                            + (training.size() - header.numInstances()));
+        } catch (Exception e) {
+            throw new IllegalStateException("Weka模型训练失败", e);
+        }
+    }
+    private Instance instance(NewsRecord r) {
+        double[] features = vectorizer.transform(r);
+        double[] values = new double[features.length + 1];
+        System.arraycopy(features, 0, values, 0, features.length);
+        values[features.length] = Double.NaN;
+        Instance x = new SparseInstance(1.0, values);
+        x.setDataset(header);
+        return x;
+    }
+    public double score(NewsRecord r) {
+        // 该标题未命中训练词表时，退回训练集类别先验。
+        if (TfidfVectorizer.isZero(vectorizer.transform(r))) {
+            return sportsPrior;
+        }
+        try {
+            return classifier.distributionForInstance(instance(r))[1];
+        } catch (Exception e) {
+            throw new IllegalStateException("Weka预测失败", e);
+        }
+    }
+    public double threshold() {
+        return 0.5;
+    }
+    public String name() {
+        return "Weka-kNN";
+    }
+}
+</file>
+<file java KnnWekaDemo.java>
+package cn.edu.bjut.textmining.chapter4;
+import cn.edu.bjut.textmining.chapter4.support.EvaluationSupport;
+import cn.edu.bjut.textmining.chapter4.support.NewsClassifier;
+import cn.edu.bjut.textmining.chapter4.support.NewsExperiment;
+/** Weka kNN：训练集拟合TF-IDF，然后以k=5近邻分类。 */
+public final class KnnWekaDemo {
+    public static void main(String[] args) {
+        try {
+            NewsExperiment experiment = NewsExperiment.load();
+            experiment.printSummary();
+            NewsClassifier model = new WekaNewsClassifier();
+            model.fit(experiment.training);
+            EvaluationSupport.showExamples(model, experiment);
+            EvaluationSupport.evaluate(model, experiment);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
+</file>
+===== 上机作业 =====
+  *利用第三方库libsvm，利用SVM方法完成新闻文本数据的分类。
+<code xml>
+<dependency>
+    <groupId>tw.edu.ntu.csie</groupId>
+    <artifactId>libsvm</artifactId>
+    <version>3.31</version>
+</dependency>
+</code>
+  *借助第三方库完成ROC和PR曲线的绘制。
 ~~DISCUSSION~~

硕风徐徐

用户工具

站点工具

差别

页面工具