数字内容安全之文本信息检索

文本信息检索

实验要求

针对“语料库.txt”文件，实现基于布尔模型的检索系统和基于TF-IDF的检索系统。（每一行看做是一个文档）

基于布尔模型的检索系统

输入 以布尔表达式形式输入，一次检索单词数量不超过三个（AA and BB or CC ），返回所有的击中的结果（超过10项的只显示前10项）

基于TF-IDF的检索系统

输入不超过8个字的短语，系统首先自动进行分词，按照TF-IDF的值求和排序返回前10项结果。

实验过程

由于两次实验都需要用将 语料库 的每一行看作一个文本，故一开始便将语料库按行分类存储到 text 集合和 listLine 集合中：

//表示语料库中第i行文本下的分词结果，即：第i行文本中，string 词语在该行本文出现的次数 private Map<Integer, Map<String, Integer>> text = new HashMap<>(); //表示语料库第i行的文本内容 private List<String> listLine = new ArrayList<>();

该部分代码与实验三FMM和BMM读取部分代码类似，故省略。

基于布尔模型的检索系统

由于输入的为汉字和 and or not 代表逻辑关系的表达式，而Java支持脚本语言，故使用 JavaSE6 中自带了JavaScript语言的脚本引擎 ScriptEngine：

// 字符串转条件表达式 ScriptEngineManager manager = new ScriptEngineManager(); ScriptEngine engine = manager.getEngineByExtension("js"); // 输入input文本格式处理：如输入 ***中国 and 的 not 共产党*** String[] words = input.trim().split(" +"); String[] word3 = new String[3]; for (int i = 0; i < words.length; i+=2) { //将输入的单词存入 word3 数组中 word3[i/2] = words[i]; //将输入的 words[i] 替换为 --> 判断word文本行里是否包含words[i]这一单词 words[i] = "word.containsKey(word3[" + i/2 + "])"; } //将words数组用空格隔开 input = String.join(" ", words); //将用户输入的逻辑 替换为 --> 计算机可以识别的逻辑符号 input = input.replaceAll(" and ", " && "); input = input.replaceAll(" or ", " || "); input = input.replaceAll(" not ", " && ! "); //脚本语言的绑定 engine.put("word3", word3);

然后对每行文本进行执行，符合条件表达式即输出文本：

Boolean result = (Boolean) engine.eval(input);// 字符串转条件表达式 if (result) { System.out.println(listLine.get(i));//符合就输出相应文本 num ++; } if (num == 10) break;//输出前十行

输出结果：

基于TF-IDF的检索系统

1、读取实验三对语料库的分词结果output.txt，利用BMM对输入的数据进行分词：

for (int j = line.length(); j >= 0 ; j--) { for(int i=0; i<j; i++) { String string = line.substring(i, j); if (yuliaoku.containsKey(string) && !string.equals(" ")) { inputWords.add(string); j -= string.length()-1; break; } else if (i == j-1 && !string.equals(" ")) { inputWords.add(string); j -= string.length()-1; } } }

2、获取TF值：

/*********获取TF值---第i个网页下，分词结果的第j个词语的TF值********/ Map<Integer, Map<Integer, Double>> wordTF = new HashMap<>(); for (int i = 0; i < listLine.size(); i++) { //获取第i个网页下的分词结果 Map<String, Integer> map = text.get(i); //存储分词结果的第j个词语的TF值 Map map1 = new HashMap(); for (int j = 0; j < inputWords.size(); j++) { int num = map.containsKey(inputWords.get(j)) ? map.get(inputWords.get(j)) : 0; double aaa = (double)num / map.size(); map1.put(j, aaa); wordTF.put(i, map1); } }

3、获取IDF值 ------ 网页总数量 / 当前词语出现的网页数量，再取log：

double[] wordIDF = new double[inputWords.size()]; for (int i = 0; i < inputWords.size(); i++) { int idf = 0; for (int j = 0; j < listLine.size(); j++) { if (listLine.get(j).contains(inputWords.get(i))) { idf ++; } } if (idf == 0) { System.out.println( idf + "在所有网页中未出现，无法进行 网页数/词频 运算！"); System.exit(-1); } double aaa = listLine.size() / idf; wordIDF[i] = Math.log(aaa); }

4、对每个部分求和：

//得到每个部分的TF-IDF值 double result = 0.0; for (int j = 0; j < inputWords.size(); j++) { Double map = wordTF.get(i).get(j); result += map * wordIDF[j]; }

5、排序输出：

List<Map.Entry<Integer, Double>> entryList = new ArrayList<>(tf_idf.entrySet()); Collections.sort(entryList, new Comparator<Map.Entry<Integer, Double>>() { @Override public int compare(Map.Entry<Integer, Double> o1, Map.Entry<Integer, Double> o2) { return o2.getValue().compareTo(o1.getValue()); } }); for (int i = 0; i < 10; i++) { int a = entryList.get(i).getKey(); System.out.println("文本为：" + listLine.get(a)); System.out.println("TF-IDF值为：" + entryList.get(i).getValue()); System.out.println("---------------------------------------------------------------------------------------------------------");//分隔符 }

输出结果：

实验完整代码

package com.tuxiangchuli.java; import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; import java.io.*; import java.util.*; public class BooleanSearch { private Map<Integer, Map<String, Integer>> text = new HashMap<>(); //行数 private List<String> listLine = new ArrayList<>(); private Map<Integer, Integer> textNum = new HashMap<>(); private Map<String, Integer> yuliaoku = new HashMap<>(); private void readOutput() throws IOException { String fileName = "D:\\output.txt"; //FileInputStream字节输入流；InputStreamReader将字节流转化为字符流；BufferedReader缓冲方式读取 BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(fileName), "UTF-8")); String line; while ((line = br.readLine()) != null) { String[] input = line.trim().split(" +"); if (input.length == 2) continue; yuliaoku.put(input[0], Integer.valueOf(input[2])); } br.close(); } private void readText() throws IOException { /********读取语料库每一行,存到List中********/ String fileName = "D:\\yuliaoku.txt"; //FileInputStream字节输入流；InputStreamReader将字节流转化为字符流；BufferedReader缓冲方式读取 BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(fileName), "UTF-8")); String line; while ((line = br.readLine()) != null) { listLine.add(line); } br.close(); for (int i = 0; i < listLine.size(); i++) { Map<String, Integer> words = new HashMap<>(); String[] arrays = listLine.get(i).trim().split(" +");//每个文本分词并统计其词频 textNum.put(i, arrays.length); for (String s : arrays) { if (s.indexOf("19980") != -1) continue; String[] endString = s.split("/"); s = endString[0]; int count = words.containsKey(s) ? words.get(s) : 0; words.put(s, count + 1); } text.put(i, words); } } private void tf_IDFSearch(String line) throws IOException { readOutput(); List<String> inputWords = new ArrayList<>(); //BMM分词 for (int j = line.length(); j >= 0 ; j--) { for(int i=0; i<j; i++) { String string = line.substring(i, j); if (yuliaoku.containsKey(string) && !string.equals(" ")) { inputWords.add(string); j -= string.length()-1; break; } else if (i == j-1 && !string.equals(" ")) { inputWords.add(string); j -= string.length()-1; } } } /*********获取TF值---第i个网页，第j个词的TF值********/ Map<Integer, Map<Integer, Double>> wordTF = new HashMap<>(); for (int i = 0; i < listLine.size(); i++) { Map<String, Integer> map = text.get(i); Map map1 = new HashMap(); for (int j = 0; j < inputWords.size(); j++) { int num = map.containsKey(inputWords.get(j)) ? map.get(inputWords.get(j)) : 0; double aaa = (double)num / map.size(); map1.put(j, aaa); wordTF.put(i, map1); } } /**********获取IDF值**********/ System.out.println("------------------------------------------"); System.out.println("输入的分词：" + inputWords); System.out.println("------------------------------------------"); double[] wordIDF = new double[inputWords.size()]; for (int i = 0; i < inputWords.size(); i++) { int idf = 0; for (int j = 0; j < listLine.size(); j++) { if (listLine.get(j).contains(inputWords.get(i))) { idf ++; } } if (idf == 0) { System.out.println( idf + "在所有网页中未出现，无法进行 网页数/词频 运算！"); System.exit(-1); } double aaa = listLine.size() / idf; wordIDF[i] = Math.log(aaa); } /************第i个网页获得的TF-IDF计算如下：******/ Map<Integer, Double> tf_idf = new HashMap<>(); for (int i = 0; i < listLine.size(); i++) { double result = 0.0; for (int j = 0; j < inputWords.size(); j++) { Double map = wordTF.get(i).get(j); result += map * wordIDF[j]; } tf_idf.put(i, result); } List<Map.Entry<Integer, Double>> entryList = new ArrayList<>(tf_idf.entrySet()); Collections.sort(entryList, new Comparator<Map.Entry<Integer, Double>>() { @Override public int compare(Map.Entry<Integer, Double> o1, Map.Entry<Integer, Double> o2) { return o2.getValue().compareTo(o1.getValue()); } }); for (int i = 0; i < 10; i++) { int a = entryList.get(i).getKey(); System.out.println("文本为：" + listLine.get(a)); System.out.println("TF-IDF值为：" + entryList.get(i).getValue()); System.out.println("---------------------------------------------------------------------------------------------------------"); } } private void boolSearch(String input) throws ScriptException { // 字符串转条件表达式 ScriptEngineManager manager = new ScriptEngineManager(); ScriptEngine engine = manager.getEngineByExtension("js"); String[] words = input.trim().split(" +"); String[] word3 = new String[3]; for (int i = 0; i < words.length; i+=2) { word3[i/2] = words[i]; words[i] = "word.containsKey(word3[" + i/2 + "])"; } input = String.join(" ", words); input = input.replaceAll(" and ", " && "); input = input.replaceAll(" or ", " || "); input = input.replaceAll(" not ", " && ! "); engine.put("word3", word3); int num = 0; for (int i=0; i<text.size(); i++) { Map<String, Integer> word; word = text.get(i); engine.put("word", word); Boolean result = (Boolean) engine.eval(input);// 字符串转条件表达式 if (result) { //符合就输出相应文本 System.out.println(listLine.get(i)); num ++; } if (num == 10) break; } } public static void main(String[] args) throws IOException, ScriptException { BooleanSearch booleanSearch = new BooleanSearch(); booleanSearch.readText(); Scanner scanner = new Scanner(System.in); String input; while (scanner.hasNextLine()) { input = scanner.nextLine(); input.getBytes("UTF-8"); /*****布尔检索*******/ //booleanSearch.boolSearch(input);//需要用到撤销注释 /*****基于TF-IDF的检索系统*******/ booleanSearch.tf_IDFSearch(input); } } }