下载：文本分类

下载：chapter4.zip

<dependency>
	<groupId>com.hankcs</groupId>
	<artifactId>hanlp</artifactId>
	<version>portable-1.8.4</version>
</dependency>

Chapter4Files.java

package cn.edu.bjut.textmining.util;
 
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
 
/** UTF-8文件读写；数据路径相对于项目根目录。 */
public final class Chapter4Files {
    private Chapter4Files() {}
 
    public static Path data(String name) {
        return Paths.get("data", "chapter4", name);
    }
 
    public static Path output(String name) {
        return Paths.get("output", "chapter4", name);
    }
 
    public static List<String> read(Path path) {
        List<String> lines = new ArrayList<String>();
        try (BufferedReader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
            for (String line; (line = reader.readLine()) != null; ) {
                lines.add(line);
            }
        } catch (IOException e) {
            throw new UncheckedIOException("读取文件失败：" + path.toAbsolutePath(), e);
        }
        return lines;
    }
 
    public static void write(Path path, String text) {
        try {
            Files.createDirectories(path.getParent());
            BufferedWriter writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8); 
            writer.write(text);
        } catch (IOException e) {
            throw new UncheckedIOException("保存文件失败：" + path.toAbsolutePath(), e);
        }
    }
}

ChineseTokenizer.java

package cn.edu.bjut.textmining.util;
 
import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.seg.common.Term;
 
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
 
/** HanLP portable 1.8.6分词；保留含字母、汉字或数字的词元。 */
public final class ChineseTokenizer {
    private ChineseTokenizer() {}
 
    public static List<String> tokenize(String text) {
        List<String> tokens = new ArrayList<String>();
        for (Term term : HanLP.segment(text)) {
            String word = term.word.toLowerCase(Locale.ROOT).trim();
            if (word.matches(".*[\\p{L}\\p{N}].*")) {
                tokens.add(word);
            }
        }
        return tokens;
    }
}

EvaluationSupport.java

package cn.edu.bjut.textmining.chapter4.support;
 
import cn.edu.bjut.textmining.util.Chapter4Files;
 
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
 
/** 混淆矩阵与按平均秩计算的ROC AUC；不要求分数是概率。 */
public final class EvaluationSupport {
    private EvaluationSupport() {}
 
    public static final class Scored {
        public final NewsRecord record;
        public final double score;
 
        public Scored(NewsRecord record, double score) {
            if (!Double.isFinite(score)) {
                throw new IllegalArgumentException("预测分数须为有限数");
            }
            this.record = record;
            this.score = score;
        }
    }
 
    public static final class Metrics {
        public int tp, fp, fn, tn;
        public double accuracy, precision, recall, f1, auc;
 
        public String row(String name) {
            return String.format(
                    Locale.ROOT,
                    "%s\t%d\t%d\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\n",
                    name,
                    tp,
                    fp,
                    fn,
                    tn,
                    accuracy,
                    precision,
                    recall,
                    f1,
                    auc);
        }
    }
 
    public static List<Scored> score(NewsClassifier model, List<NewsRecord> test) {
        List<Scored> results = new ArrayList<Scored>();
        for (NewsRecord r : test) {
            results.add(new Scored(r, model.score(r)));
        }
        return results;
    }
 
    public static Metrics calculate(List<Scored> scores, double threshold) {
        if (scores.isEmpty()) {
            throw new IllegalArgumentException("测试集不能为空");
        }
        Metrics m = new Metrics();
        for (Scored s : scores) {
            if (s.record.isSports()) {
                if (s.score >= threshold) {
                    m.tp++;
                } else {
                    m.fn++;
                }
            } else {
                if (s.score >= threshold) {
                    m.fp++;
                } else {
                    m.tn++;
                }
            }
        }
        m.accuracy = (m.tp + m.tn) / (double) scores.size();
        m.precision = divide(m.tp, m.tp + m.fp);
        m.recall = divide(m.tp, m.tp + m.fn);
        m.f1 = divide(2.0 * m.precision * m.recall, m.precision + m.recall);
        m.auc = auc(scores);
        return m;
    }
 
    private static double divide(double a, double b) {
        return b == 0.0 ? 0.0 : a / b;
    }
 
    private static double auc(List<Scored> scores) {
        List<Scored> sorted = new ArrayList<Scored>(scores);
        Collections.sort(
                sorted,
                new Comparator<Scored>() {
                    public int compare(Scored a, Scored b) {
                        return Double.compare(a.score, b.score);
                    }
                });
        int positive = 0, negative = 0, i = 0;
        double rankSum = 0;
        while (i < sorted.size()) {
            int end = i + 1;
            while (end < sorted.size()
                    && Double.compare(sorted.get(i).score, sorted.get(end).score) == 0) {
                end++;
            }
            double rank = (i + 1 + end) / 2.0;
            for (int j = i; j < end; j++) {
                if (sorted.get(j).record.isSports()) {
                    positive++;
                    rankSum += rank;
                } else {
                    negative++;
                }
            }
            i = end;
        }
        if (positive == 0 || negative == 0) {
            return Double.NaN;
        }
        return (rankSum - positive * (positive + 1) / 2.0) / (positive * (double) negative);
    }
 
    public static void saveScores(String name, List<Scored> scores, double threshold) {
        StringBuilder out = new StringBuilder("id\tlabel\tscore\tprediction\ttext\n");
        for (Scored s : scores) {
            out.append(s.record.id)
                    .append('\t')
                    .append(s.record.label)
                    .append('\t')
                    .append(s.score)
                    .append('\t')
                    .append(s.score >= threshold ? "sports" : "finance")
                    .append('\t')
                    .append(s.record.text)
                    .append('\n');
        }
        Chapter4Files.write(Chapter4Files.output("predictions-" + name + ".tsv"), out.toString());
    }
 
    public static Metrics evaluate(NewsClassifier model, NewsExperiment e) {
        List<Scored> scored = score(model, e.test);
        Metrics m = calculate(scored, model.threshold());
        System.out.printf(
                Locale.ROOT,
                "%s: Accuracy=%.2f%%, Precision=%.2f%%, Recall=%.2f%%, F1=%.2f%%, AUC=%.4f%n",
                model.name(),
                100 * m.accuracy,
                100 * m.precision,
                100 * m.recall,
                100 * m.f1,
                m.auc);
        System.out.printf("TP=%d, FP=%d, FN=%d, TN=%d%n", m.tp, m.fp, m.fn, m.tn);
        saveScores(model.name(), scored, model.threshold());
        return m;
    }
 
    public static void showExamples(NewsClassifier model, NewsExperiment e) {
        for (int i = 0; i < e.examples.size(); i++) {
            NewsRecord r = e.examples.get(i);
            double score = model.score(r);
            System.out.println("N" + (i + 1) + " " + r.id + "：" + r.text);
            System.out.println("分词：" + r.tokens);
            System.out.printf(
                    Locale.ROOT,
                    "真实=%s，预测=%s，%s=%.6f%n",
                    r.chineseLabel(),
                    score >= model.threshold() ? "体育" : "财经",
                    model instanceof LibSvmNewsClassifier ? "体育方向决策值" : "体育类别得分",
                    score);
        }
    }
}

NewsRecord.java

package cn.edu.bjut.textmining.chapter4.support;
 
import java.util.Collections;
import java.util.List;
 
/** 一条公开新闻标题及其来源信息、分词结果。 */
public final class NewsRecord {
	public final String id;
	public final String label;
	public final String text;
	public final String sourceSplit;
	public final int sourceLine;
	public final List<String> tokens;
 
	public NewsRecord(String id, String label, String text, String sourceSplit, int sourceLine, List<String> tokens) {
		this.id = id;
		this.label = label;
		this.text = text;
		this.sourceSplit = sourceSplit;
		this.sourceLine = sourceLine;
		this.tokens = Collections.unmodifiableList(tokens);
	}
 
	public boolean isSports() {
		return "sports".equals(label);
	}
 
	public String chineseLabel() {
		return isSports() ? "体育" : "财经";
	}
}

NewsExperiment.java

package cn.edu.bjut.textmining.chapter4.support;
 
import cn.edu.bjut.textmining.util.Chapter4Files;
import cn.edu.bjut.textmining.util.ChineseTokenizer;
 
import java.text.Normalizer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
 
/** 各入口共用同一训练集、测试集和四条观察标题。 */
public final class NewsExperiment {
    public final List<NewsRecord> training;
    public final List<NewsRecord> test;
    public final List<NewsRecord> examples;
 
    private NewsExperiment(
            List<NewsRecord> training, List<NewsRecord> test, List<NewsRecord> examples) {
        this.training = Collections.unmodifiableList(training);
        this.test = Collections.unmodifiableList(test);
        this.examples = Collections.unmodifiableList(examples);
    }
 
    public static NewsExperiment load() {
        List<NewsRecord> training = read("train.tsv");
        List<NewsRecord> test = read("test.tsv");
        Set<String> seen = new HashSet<String>();
        for (NewsRecord r : training) {
            if (!seen.add(key(r.text))) {
                throw new IllegalArgumentException("训练集含重复标题");
            }
        }
        for (NewsRecord r : test) {
            if (!seen.add(key(r.text))) {
                throw new IllegalArgumentException("训练/测试集有重复标题");
            }
        }
        List<NewsRecord> examples = new ArrayList<NewsRecord>();
        for (String line : Chapter4Files.read(Chapter4Files.data("examples.tsv"))) {
            if (line.trim().isEmpty()
                    || line.equals("id")
                    || line.startsWith("id\t")
                    || line.startsWith("#")) {
                continue;
            }
            String id = line.split("\t", -1)[0];
            NewsRecord found = null;
            for (NewsRecord r : test) {
                if (r.id.equals(id)) {
                    found = r;
                    break;
                }
            }
            if (found == null) {
                throw new IllegalArgumentException("观察标题不在测试集中：" + id);
            }
            examples.add(found);
        }
        if (examples.size() != 4) {
            throw new IllegalArgumentException("观察标题应为4条");
        }
        return new NewsExperiment(training, test, examples);
    }
 
    private static List<NewsRecord> read(String name) {
        List<NewsRecord> records = new ArrayList<NewsRecord>();
        for (String line : Chapter4Files.read(Chapter4Files.data(name))) {
            if (line.startsWith("id\t") || line.trim().isEmpty()) {
                continue;
            }
            String[] fields = line.split("\t", -1);
            if (fields.length != 5
                    || !("sports".equals(fields[1]) || "finance".equals(fields[1]))) {
                throw new IllegalArgumentException("数据格式错误：" + name);
            }
            List<String> tokens = ChineseTokenizer.tokenize(fields[2]);
            if (tokens.isEmpty()) {
                throw new IllegalArgumentException("标题分词为空：" + fields[0]);
            }
            records.add(
                    new NewsRecord(
                            fields[0],
                            fields[1],
                            fields[2],
                            fields[3],
                            Integer.parseInt(fields[4]),
                            tokens));
        }
        if (records.isEmpty()) {
            throw new IllegalArgumentException("数据集不能为空：" + name);
        }
        return records;
    }
 
    private static String key(String text) {
        return Normalizer.normalize(text, Normalizer.Form.NFKC)
                .replaceAll("[\\s\\p{Z}]+", "")
                .toLowerCase(Locale.ROOT);
    }
 
    public void printSummary() {
        System.out.println("数据：CLUE TNEWS体育/财经新闻标题");
        System.out.println("训练集：" + training.size() + "；测试集：" + test.size());
        System.out.println("体育为正类，财经为负类；四条观察标题均来自测试集。");
    }
}

TfidfVectorizer.java

package cn.edu.bjut.textmining.chapter4.support;
 
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
 
/** 只从训练集学习词表与IDF，随后将任何标题映射到相同坐标系。 */
public final class TfidfVectorizer {
    public static final int MIN_DF = 2;
    public static final int MAX_FEATURES = 2000;
    private final List<String> terms = new ArrayList<String>();
    private final Map<String, Integer> indexes = new HashMap<String, Integer>();
    private double[] idf;
 
    public void fit(List<NewsRecord> records) {
        final Map<String, Integer> df = new HashMap<String, Integer>();
        for (NewsRecord record : records) {
            for (String token : new HashSet<String>(record.tokens)) {
                df.put(token, df.containsKey(token) ? df.get(token) + 1 : 1);
            }
        }
        terms.clear();
        indexes.clear();
        for (String term : df.keySet()) {
            if (df.get(term) >= MIN_DF) {
                terms.add(term);
            }
        }
        Collections.sort(
                terms,
                new Comparator<String>() {
                    public int compare(String a, String b) {
                        int c = Integer.compare(df.get(b), df.get(a));
                        return c != 0 ? c : a.compareTo(b);
                    }
                });
        if (terms.size() > MAX_FEATURES) {
            terms.subList(MAX_FEATURES, terms.size()).clear();
        }
        if (terms.isEmpty()) {
            throw new IllegalArgumentException("训练词表为空");
        }
        idf = new double[terms.size()];
        for (int i = 0; i < terms.size(); i++) {
            indexes.put(terms.get(i), i);
            idf[i] = Math.log((records.size() + 1.0) / (df.get(terms.get(i)) + 1.0)) + 1.0;
        }
    }
 
    public double[] transform(NewsRecord record) {
        if (idf == null) {
            throw new IllegalStateException("请先拟合训练集");
        }
        double[] vector = new double[terms.size()];
        for (String token : record.tokens) {
            Integer i = indexes.get(token);
            if (i != null) {
                vector[i]++;
            }
        }
        double squaredNorm = 0.0;
        for (int i = 0; i < vector.length; i++) {
            vector[i] *= idf[i];
            squaredNorm += vector[i] * vector[i];
        }
        if (squaredNorm > 0.0) {
            double norm = Math.sqrt(squaredNorm);
            for (int i = 0; i < vector.length; i++) {
                vector[i] /= norm;
            }
        }
        return vector;
    }
 
    /** 未命中训练词表时，向量没有可用于距离比较的词项证据。 */
    public static boolean isZero(double[] vector) {
        for (double value : vector) {
            if (value != 0.0) {
                return false;
            }
        }
        return true;
    }
 
    public List<String> terms() {
        return Collections.unmodifiableList(terms);
    }
 
    public double idf(int index) {
        return idf[index];
    }
}

NewsClassifier.java

package cn.edu.bjut.textmining.chapter4.support;
 
import java.util.List;
 
/** 体育为正类；分数可为类别概率、近邻分布或SVM间隔。 */
public interface NewsClassifier {
    void fit(List<NewsRecord> training);
 
    double score(NewsRecord record);
 
    double threshold();
 
    String name();
 
    default String predict(NewsRecord record) {
        return score(record) >= threshold() ? "sports" : "finance";
    }
}

<dependency>
	<groupId>org.apache.opennlp</groupId>
	<artifactId>opennlp-tools</artifactId>
	<version>2.3.0</version>
</dependency>

OpenNlpNewsClassifier.java

package cn.edu.bjut.textmining.chapter4;
 
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
 
import cn.edu.bjut.textmining.chapter4.support.NewsClassifier;
import cn.edu.bjut.textmining.chapter4.support.NewsRecord;
import cn.edu.bjut.textmining.chapter4.support.TfidfVectorizer;
import opennlp.tools.doccat.DoccatFactory;
import opennlp.tools.doccat.DoccatModel;
import opennlp.tools.doccat.DocumentCategorizerME;
import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.ml.AbstractTrainer;
import opennlp.tools.ml.naivebayes.NaiveBayesTrainer;
import opennlp.tools.util.CollectionObjectStream;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
 
/** OpenNLP朴素贝叶斯；重复词元保留，训练词表由训练数据建立。 */
public final class OpenNlpNewsClassifier implements NewsClassifier {
    private DocumentCategorizerME categorizer;
    private int sportsIndex;
    private final Set<String> vocabulary = new HashSet<String>();
    private double sportsPrior;
 
    public void fit(List<NewsRecord> training) {
        TfidfVectorizer vectorizer = new TfidfVectorizer();
        vectorizer.fit(training);
        vocabulary.clear();
        vocabulary.addAll(vectorizer.terms());
        int sports = 0;
        for (NewsRecord r : training) {
            if (r.isSports()) {
                sports++;
            }
        }
        sportsPrior = sports / (double) training.size();
        List<DocumentSample> samples = new ArrayList<DocumentSample>();
        for (NewsRecord r : training) {
            samples.add(new DocumentSample(r.label, filteredTokens(r)));
        }
        TrainingParameters p = new TrainingParameters();
        p.put(TrainingParameters.CUTOFF_PARAM, 1);
        p.put(AbstractTrainer.ALGORITHM_PARAM, NaiveBayesTrainer.NAIVE_BAYES_VALUE);
        try (ObjectStream<DocumentSample> stream =
                new CollectionObjectStream<DocumentSample>(samples)) {
            DoccatModel model = DocumentCategorizerME.train("zh", stream, p, new DoccatFactory());
            categorizer = new DocumentCategorizerME(model);
            sportsIndex = categorizer.getIndex("sports");
        } catch (Exception e) {
            throw new IllegalStateException("OpenNLP模型训练失败", e);
        }
    }
 
    public double score(NewsRecord r) {
        if (categorizer == null) {
            throw new IllegalStateException("请先训练模型");
        }
        String[] tokens = filteredTokens(r);
        return tokens.length == 0 ? sportsPrior : categorizer.categorize(tokens)[sportsIndex];
    }
 
    private String[] filteredTokens(NewsRecord record) {
        List<String> words = new ArrayList<String>();
        for (String word : record.tokens) {
            if (vocabulary.contains(word)) {
                words.add(word);
            }
        }
        return words.toArray(new String[0]);
    }
 
    public double threshold() {
        return 0.5;
    }
 
    public String name() {
        return "OpenNLP-NB";
    }
}

OpenNlpNaiveBayesDemo.java

package cn.edu.bjut.textmining.chapter4;
 
import cn.edu.bjut.textmining.chapter4.support.EvaluationSupport;
import cn.edu.bjut.textmining.chapter4.support.NewsClassifier;
import cn.edu.bjut.textmining.chapter4.support.NewsExperiment;
 
/** OpenNLP朴素贝叶斯：中文词元到类别得分。 */
public final class OpenNlpNaiveBayesDemo {
    public static void main(String[] args) {
        try {
            NewsExperiment experiment = NewsExperiment.load();
            experiment.printSummary();
            NewsClassifier model = new OpenNlpNewsClassifier();
            model.fit(experiment.training);
            EvaluationSupport.showExamples(model, experiment);
            EvaluationSupport.evaluate(model, experiment);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

<dependency>
    <groupId>nz.ac.waikato.cms.weka</groupId>
    <artifactId>weka-stable</artifactId>
    <version>3.8.6</version>
</dependency>

WekaNewsClassifier.java

package cn.edu.bjut.textmining.chapter4;
 
import weka.classifiers.lazy.IBk;
import weka.core.Attribute;
import weka.core.EuclideanDistance;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.SparseInstance;
import weka.core.neighboursearch.LinearNNSearch;
 
import java.util.ArrayList;
import java.util.List;
 
import cn.edu.bjut.textmining.chapter4.support.NewsClassifier;
import cn.edu.bjut.textmining.chapter4.support.NewsRecord;
import cn.edu.bjut.textmining.chapter4.support.TfidfVectorizer;
 
/** 高维TF-IDF采用线性扫描近邻；关闭逐列极差归一化。 */
public final class WekaNewsClassifier implements NewsClassifier {
    public static final int K = 5;
    private final TfidfVectorizer vectorizer = new TfidfVectorizer();
    private final IBk classifier = new IBk(K);
    private Instances header;
    private double sportsPrior;
 
    public void fit(List<NewsRecord> training) {
        vectorizer.fit(training);
        int sports = 0;
        for (NewsRecord r : training) {
            if (r.isSports()) {
                sports++;
            }
        }
        sportsPrior = sports / (double) training.size();
        ArrayList<Attribute> attrs = new ArrayList<Attribute>();
        for (String term : vectorizer.terms()) {
            attrs.add(new Attribute(term));
        }
        ArrayList<String> labels = new ArrayList<String>();
        labels.add("finance");
        labels.add("sports");
        attrs.add(new Attribute("class", labels));
        header = new Instances("Tnews", attrs, training.size());
        header.setClassIndex(attrs.size() - 1);
        for (NewsRecord r : training) {
            // 零向量没有方向，不能作为单位TF-IDF的语义近邻。
            if (TfidfVectorizer.isZero(vectorizer.transform(r))) {
                continue;
            }
            Instance x = instance(r);
            x.setValue(header.classIndex(), r.label);
            header.add(x);
        }
        EuclideanDistance distance = new EuclideanDistance();
        distance.setDontNormalize(true);
        LinearNNSearch search = new LinearNNSearch();
        try {
            search.setDistanceFunction(distance);
            classifier.setNearestNeighbourSearchAlgorithm(search);
            classifier.buildClassifier(header);
            System.out.println(
                    "kNN可用训练向量："
                            + header.numInstances()
                            + "；排除零向量："
                            + (training.size() - header.numInstances()));
        } catch (Exception e) {
            throw new IllegalStateException("Weka模型训练失败", e);
        }
    }
 
    private Instance instance(NewsRecord r) {
        double[] features = vectorizer.transform(r);
        double[] values = new double[features.length + 1];
        System.arraycopy(features, 0, values, 0, features.length);
        values[features.length] = Double.NaN;
        Instance x = new SparseInstance(1.0, values);
        x.setDataset(header);
        return x;
    }
 
    public double score(NewsRecord r) {
        // 该标题未命中训练词表时，退回训练集类别先验。
        if (TfidfVectorizer.isZero(vectorizer.transform(r))) {
            return sportsPrior;
        }
        try {
            return classifier.distributionForInstance(instance(r))[1];
        } catch (Exception e) {
            throw new IllegalStateException("Weka预测失败", e);
        }
    }
 
    public double threshold() {
        return 0.5;
    }
 
    public String name() {
        return "Weka-kNN";
    }
}

KnnWekaDemo.java

package cn.edu.bjut.textmining.chapter4;
 
import cn.edu.bjut.textmining.chapter4.support.EvaluationSupport;
import cn.edu.bjut.textmining.chapter4.support.NewsClassifier;
import cn.edu.bjut.textmining.chapter4.support.NewsExperiment;
 
/** Weka kNN：训练集拟合TF-IDF，然后以k=5近邻分类。 */
public final class KnnWekaDemo {
    public static void main(String[] args) {
        try {
            NewsExperiment experiment = NewsExperiment.load();
            experiment.printSummary();
            NewsClassifier model = new WekaNewsClassifier();
            model.fit(experiment.training);
            EvaluationSupport.showExamples(model, experiment);
            EvaluationSupport.evaluate(model, experiment);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

1. 利用第三方库libsvm，利用SVM方法完成新闻文本数据的分类。

<dependency>
    <groupId>tw.edu.ntu.csie</groupId>
    <artifactId>libsvm</artifactId>
    <version>3.31</version>
</dependency>

1. 借助第三方库完成ROC和PR曲线的绘制。