下载：文本表示

FileUtil.java

package cn.edu.bjut.textmining.util;
 
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
 
/**
 * UTF-8 文件读写工具类。
 */
public final class FileUtil {
	public static Path path(String first, String... more) {
		return Paths.get(first, more);
	}
 
	public static List<String> readUtf8Lines(Path path) {
		return readLines(path, StandardCharsets.UTF_8);
	}
 
	public static List<String> readNonEmptyUtf8Lines(Path path) {
		List<String> lines = readUtf8Lines(path);
		List<String> result = new ArrayList<String>();
		for (String line : lines) {
			if (line != null && line.trim().length() > 0) {
				result.add(line);
			}
		}
		return result;
	}
 
	public static List<String> readLines(Path path, Charset charset) {
		List<String> result = new ArrayList<String>();
		try (BufferedReader reader = Files.newBufferedReader(path, charset)) {
			for (String line; (line = reader.readLine()) != null;) {
				result.add(line);
			}
		} catch (IOException e) {
			throw new UncheckedIOException("读取文件失败：" + path.toAbsolutePath(), e);
		}
		return result;
	}
 
	public static String readUtf8Text(Path path) {
		List<String> lines = readUtf8Lines(path);
		StringBuilder builder = new StringBuilder();
		for (int i = 0; i < lines.size(); i++) {
			builder.append(lines.get(i));
			if (i < lines.size() - 1) {
				builder.append(System.lineSeparator());
			}
		}
		return builder.toString();
	}
 
	public static void writeUtf8Text(Path path, String content) {
		try {
			Path parent = path.getParent();
			if (parent != null) {
				Files.createDirectories(parent);
			}
			try (BufferedWriter writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) {
				writer.write(content);
			}
		} catch (IOException e) {
			throw new UncheckedIOException("写入文件失败：" + path.toAbsolutePath(), e);
		}
	}
}

LuceneTfIdfDemo.java

package cn.edu.bjut.textmining.chapter3;
 
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiTerms;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
 
import java.io.IOException;
import java.io.UncheckedIOException;
 
public class LuceneTfIdfDemo {
	public static void main(String[] args) {
		try (Directory directory = new ByteBuffersDirectory(); 
				Analyzer analyzer = new StandardAnalyzer()) {
			String doc1 = "Java Lucene is powerful.";
			String doc2 = "Lucene is a search library.";
 
			IndexWriterConfig config = new IndexWriterConfig(analyzer);
			try (IndexWriter writer = new IndexWriter(directory, config)) {
				addDocument(writer, doc1);
				addDocument(writer, doc2);
			}
 
			try (IndexReader reader = DirectoryReader.open(directory)) {
				int totalDocs = reader.numDocs();
				String targetTerm = "lucene";
				int docId = 0;
				Term term = new Term("content", targetTerm);
				long docFreq = reader.docFreq(term);
				double idf = Math.log((double) totalDocs / (docFreq + 1)) + 1.0;
 
				int tf = 0;
				PostingsEnum postings = MultiTerms.getTermPostingsEnum(reader, 
						"content", new BytesRef(targetTerm));
				if (postings != null && postings.advance(docId) == docId) {
					tf = postings.freq();
				}
 
				System.out.println("Total Documents: " + totalDocs);
				System.out.println("Term: " + targetTerm);
				System.out.println("TF (Frequency in doc0): " + tf);
				System.out.println("DocFreq (Docs containing term): " + docFreq);
				System.out.printf("IDF Value: %.4f%n", idf);
				System.out.printf("TF-IDF Score: %.4f%n", tf * idf);
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
 
	private static void addDocument(IndexWriter writer, String content) {
		try {
			Document document = new Document();
			document.add(new TextField("content", content, Field.Store.YES));
			writer.addDocument(document);
		} catch (IOException e) {
			throw new UncheckedIOException("写入 Lucene 文档失败", e);
		}
	}
}