差别

这里会显示出您选择的修订版和当前版本之间的差别。

--- zh:courses:textmining2026:ch04 [2026/09/28 17:38]
pzczxs [源代码]
+++ zh:courses:textmining2026:ch04 [2026/10/13 09:51] (当前版本)
pzczxs [上机作业]
@@ 行 3: / 行 3: @@
 下载：{{ :zh:courses:textmining2026:ch04.pptx |文本分类}}
+===== 数据集 =====
+下载：{{ :zh:courses:textmining2026:chapter4.zip |}}
 ===== 工具类 =====
 ==== 第三方依赖 ====
@@ 行 90: / 行 92: @@
 </file>
 ===== 支持类 =====
-==== 第三方依赖 ====
-==== 源代码 ====
 <file java EvaluationSupport.java>
 package cn.edu.bjut.textmining.chapter4.support;
@@ 行 508: / 行 508: @@
     public double idf(int index) {
         return idf[index];
+    }
+}
+</file>
+<file java NewsClassifier.java>
+package cn.edu.bjut.textmining.chapter4.support;
+import java.util.List;
+/** 体育为正类；分数可为类别概率、近邻分布或SVM间隔。 */
+public interface NewsClassifier {
+    void fit(List<NewsRecord> training);
+    double score(NewsRecord record);
+    double threshold();
+    String name();
+    default String predict(NewsRecord record) {
+        return score(record) >= threshold() ? "sports" : "finance";
     }
 }
 </file>
 ===== 朴素贝叶斯分类器 =====
+==== 第三方依赖 ====
+<code xml>
+<dependency>
+	<groupId>org.apache.opennlp</groupId>
+	<artifactId>opennlp-tools</artifactId>
+	<version>2.3.0</version>
+</dependency>
+</code>
+==== 源代码 ====
+<file java OpenNlpNewsClassifier.java>
+package cn.edu.bjut.textmining.chapter4;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import cn.edu.bjut.textmining.chapter4.support.NewsClassifier;
+import cn.edu.bjut.textmining.chapter4.support.NewsRecord;
+import cn.edu.bjut.textmining.chapter4.support.TfidfVectorizer;
+import opennlp.tools.doccat.DoccatFactory;
+import opennlp.tools.doccat.DoccatModel;
+import opennlp.tools.doccat.DocumentCategorizerME;
+import opennlp.tools.doccat.DocumentSample;
+import opennlp.tools.ml.AbstractTrainer;
+import opennlp.tools.ml.naivebayes.NaiveBayesTrainer;
+import opennlp.tools.util.CollectionObjectStream;
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.TrainingParameters;
+/** OpenNLP朴素贝叶斯；重复词元保留，训练词表由训练数据建立。 */
+public final class OpenNlpNewsClassifier implements NewsClassifier {
+    private DocumentCategorizerME categorizer;
+    private int sportsIndex;
+    private final Set<String> vocabulary = new HashSet<String>();
+    private double sportsPrior;
+    public void fit(List<NewsRecord> training) {
+        TfidfVectorizer vectorizer = new TfidfVectorizer();
+        vectorizer.fit(training);
+        vocabulary.clear();
+        vocabulary.addAll(vectorizer.terms());
+        int sports = 0;
+        for (NewsRecord r : training) {
+            if (r.isSports()) {
+                sports++;
+            }
+        }
+        sportsPrior = sports / (double) training.size();
+        List<DocumentSample> samples = new ArrayList<DocumentSample>();
+        for (NewsRecord r : training) {
+            samples.add(new DocumentSample(r.label, filteredTokens(r)));
+        }
+        TrainingParameters p = new TrainingParameters();
+        p.put(TrainingParameters.CUTOFF_PARAM, 1);
+        p.put(AbstractTrainer.ALGORITHM_PARAM, NaiveBayesTrainer.NAIVE_BAYES_VALUE);
+        try (ObjectStream<DocumentSample> stream =
+                new CollectionObjectStream<DocumentSample>(samples)) {
+            DoccatModel model = DocumentCategorizerME.train("zh", stream, p, new DoccatFactory());
+            categorizer = new DocumentCategorizerME(model);
+            sportsIndex = categorizer.getIndex("sports");
+        } catch (Exception e) {
+            throw new IllegalStateException("OpenNLP模型训练失败", e);
+        }
+    }
+    public double score(NewsRecord r) {
+        if (categorizer == null) {
+            throw new IllegalStateException("请先训练模型");
+        }
+        String[] tokens = filteredTokens(r);
+        return tokens.length == 0 ? sportsPrior : categorizer.categorize(tokens)[sportsIndex];
+    }
+    private String[] filteredTokens(NewsRecord record) {
+        List<String> words = new ArrayList<String>();
+        for (String word : record.tokens) {
+            if (vocabulary.contains(word)) {
+                words.add(word);
+            }
+        }
+        return words.toArray(new String[0]);
+    }
+    public double threshold() {
+        return 0.5;
+    }
+    public String name() {
+        return "OpenNLP-NB";
+    }
+}
+</file>
+<file java OpenNlpNaiveBayesDemo.java>
+package cn.edu.bjut.textmining.chapter4;
+import cn.edu.bjut.textmining.chapter4.support.EvaluationSupport;
+import cn.edu.bjut.textmining.chapter4.support.NewsClassifier;
+import cn.edu.bjut.textmining.chapter4.support.NewsExperiment;
+/** OpenNLP朴素贝叶斯：中文词元到类别得分。 */
+public final class OpenNlpNaiveBayesDemo {
+    public static void main(String[] args) {
+        try {
+            NewsExperiment experiment = NewsExperiment.load();
+            experiment.printSummary();
+            NewsClassifier model = new OpenNlpNewsClassifier();
+            model.fit(experiment.training);
+            EvaluationSupport.showExamples(model, experiment);
+            EvaluationSupport.evaluate(model, experiment);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
+</file>
+===== kNN分类器 =====
+==== 第三方依赖 ====
+<code xml>
+<dependency>
+    <groupId>nz.ac.waikato.cms.weka</groupId>
+    <artifactId>weka-stable</artifactId>
+    <version>3.8.6</version>
+</dependency>
+</code>
+==== 源代码 ====
+<file java WekaNewsClassifier.java>
+package cn.edu.bjut.textmining.chapter4;
+import weka.classifiers.lazy.IBk;
+import weka.core.Attribute;
+import weka.core.EuclideanDistance;
+import weka.core.Instance;
+import weka.core.Instances;
+import weka.core.SparseInstance;
+import weka.core.neighboursearch.LinearNNSearch;
+import java.util.ArrayList;
+import java.util.List;
+import cn.edu.bjut.textmining.chapter4.support.NewsClassifier;
+import cn.edu.bjut.textmining.chapter4.support.NewsRecord;
+import cn.edu.bjut.textmining.chapter4.support.TfidfVectorizer;
+/** 高维TF-IDF采用线性扫描近邻；关闭逐列极差归一化。 */
+public final class WekaNewsClassifier implements NewsClassifier {
+    public static final int K = 5;
+    private final TfidfVectorizer vectorizer = new TfidfVectorizer();
+    private final IBk classifier = new IBk(K);
+    private Instances header;
+    private double sportsPrior;
+    public void fit(List<NewsRecord> training) {
+        vectorizer.fit(training);
+        int sports = 0;
+        for (NewsRecord r : training) {
+            if (r.isSports()) {
+                sports++;
+            }
+        }
+        sportsPrior = sports / (double) training.size();
+        ArrayList<Attribute> attrs = new ArrayList<Attribute>();
+        for (String term : vectorizer.terms()) {
+            attrs.add(new Attribute(term));
+        }
+        ArrayList<String> labels = new ArrayList<String>();
+        labels.add("finance");
+        labels.add("sports");
+        attrs.add(new Attribute("class", labels));
+        header = new Instances("Tnews", attrs, training.size());
+        header.setClassIndex(attrs.size() - 1);
+        for (NewsRecord r : training) {
+            // 零向量没有方向，不能作为单位TF-IDF的语义近邻。
+            if (TfidfVectorizer.isZero(vectorizer.transform(r))) {
+                continue;
+            }
+            Instance x = instance(r);
+            x.setValue(header.classIndex(), r.label);
+            header.add(x);
+        }
+        EuclideanDistance distance = new EuclideanDistance();
+        distance.setDontNormalize(true);
+        LinearNNSearch search = new LinearNNSearch();
+        try {
+            search.setDistanceFunction(distance);
+            classifier.setNearestNeighbourSearchAlgorithm(search);
+            classifier.buildClassifier(header);
+            System.out.println(
+                    "kNN可用训练向量："
+                            + header.numInstances()
+                            + "；排除零向量："
+                            + (training.size() - header.numInstances()));
+        } catch (Exception e) {
+            throw new IllegalStateException("Weka模型训练失败", e);
+        }
+    }
+    private Instance instance(NewsRecord r) {
+        double[] features = vectorizer.transform(r);
+        double[] values = new double[features.length + 1];
+        System.arraycopy(features, 0, values, 0, features.length);
+        values[features.length] = Double.NaN;
+        Instance x = new SparseInstance(1.0, values);
+        x.setDataset(header);
+        return x;
+    }
+    public double score(NewsRecord r) {
+        // 该标题未命中训练词表时，退回训练集类别先验。
+        if (TfidfVectorizer.isZero(vectorizer.transform(r))) {
+            return sportsPrior;
+        }
+        try {
+            return classifier.distributionForInstance(instance(r))[1];
+        } catch (Exception e) {
+            throw new IllegalStateException("Weka预测失败", e);
+        }
+    }
+    public double threshold() {
+        return 0.5;
+    }
+    public String name() {
+        return "Weka-kNN";
+    }
+}
+</file>
+<file java KnnWekaDemo.java>
+package cn.edu.bjut.textmining.chapter4;
+import cn.edu.bjut.textmining.chapter4.support.EvaluationSupport;
+import cn.edu.bjut.textmining.chapter4.support.NewsClassifier;
+import cn.edu.bjut.textmining.chapter4.support.NewsExperiment;
+/** Weka kNN：训练集拟合TF-IDF，然后以k=5近邻分类。 */
+public final class KnnWekaDemo {
+    public static void main(String[] args) {
+        try {
+            NewsExperiment experiment = NewsExperiment.load();
+            experiment.printSummary();
+            NewsClassifier model = new WekaNewsClassifier();
+            model.fit(experiment.training);
+            EvaluationSupport.showExamples(model, experiment);
+            EvaluationSupport.evaluate(model, experiment);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
+</file>
+===== 上机作业 =====
+  *利用第三方库libsvm，利用SVM方法完成新闻文本数据的分类。
+<code xml>
+<dependency>
+    <groupId>tw.edu.ntu.csie</groupId>
+    <artifactId>libsvm</artifactId>
+    <version>3.31</version>
+</dependency>
+</code>
+  *借助第三方库完成ROC和PR曲线的绘制。
 ~~DISCUSSION~~

硕风徐徐

用户工具

站点工具

差别

页面工具