MongoDB 全文搜索解决方案(lucene+IKAnalyzer)

mongodb 解决 全文搜索是个不小的问题

可以用 正则匹配 但是效率很低 往往到大数据量的搜索的时候就会出现 查询超时等现象

当然也可以用官方的做法(在mongodb的文档类型中加字段,存分词结果,

然后从该字段中匹配) 但是我尝试了 效率比原先的好像还要低
 

后来我尝试了 lucene+IKAnalyzer 发现效率有所提升啊

原理:lucene 把大文本的数据 利用分词器 在新建的索引文件中建立索引

取数据的时候从索引文件中取

取出mongodb 中的数据进行 索引的创建

package sample3;    import java.io.File;    import org.apache.lucene.analysis.Analyzer;  import org.apache.lucene.document.Document;  import org.apache.lucene.document.Field;  import org.apache.lucene.index.IndexWriter;  import org.apache.lucene.index.IndexWriter.MaxFieldLength;  import org.apache.lucene.store.Directory;  import org.apache.lucene.store.FSDirectory;  import org.wltea.analyzer.lucene.IKAnalyzer;    import com.mongodb.DB;  import com.mongodb.DBCollection;  import com.mongodb.DBCursor;  import com.mongodb.Mongo;    /**   * 创建索引   * @author  zhanghaijun   *   */  public class Demo1 {   public static void main(String[] args) throws Exception {    //先在数据库中拿到要创建索引的数据     Mongo mongo = new Mongo();    DB db = mongo.getDB("zhang");    DBCollection msg = db.getCollection("test3");    DBCursor cursor = msg.find();    //是否重新创建索引文件，false：在原有的基础上追加    boolean create = true;    //创建索引    Directory directory = FSDirectory.open(new File("E:\\lucene\\index"));    Analyzer analyzer = new IKAnalyzer();//IK中文分词器     IndexWriter indexWriter = new IndexWriter(directory,analyzer,MaxFieldLength.LIMITED);    boolean exist = cursor.hasNext();    while(exist){     //System.out.println(cursor.next().get("text").toString());     Document doc = new Document();     Field fieldText = new Field("text",cursor.next().get("text").toString(),Field.Store.YES,          Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);     doc.add(fieldText);     indexWriter.addDocument(doc);     exist = cursor.hasNext();    }    cursor = null;    //optimize()方法是对索引进行优化       indexWriter.optimize();            //最后关闭索引       indexWriter.close();   }  }

数据的查找(直接从索引文件中查找)

package sample3;    import java.io.File;    import org.apache.lucene.document.Document;  import org.apache.lucene.index.IndexReader;  import org.apache.lucene.search.IndexSearcher;  import org.apache.lucene.search.Query;  import org.apache.lucene.search.ScoreDoc;  import org.apache.lucene.search.TopDocs;  import org.apache.lucene.store.FSDirectory;  import org.wltea.analyzer.lucene.IKAnalyzer;  import org.wltea.analyzer.lucene.IKQueryParser;  import org.wltea.analyzer.lucene.IKSimilarity;    /**   * 查找索引   */  public class Demo2 {   public static void main(String[] args) throws Exception {    // onlysearching, so read-only=true    long starttime = System.currentTimeMillis();    IndexReader reader =IndexReader.open(FSDirectory.open(new File("E:\\lucene\\index")),true);     IndexSearcher searcher = new IndexSearcher(reader);    searcher.setSimilarity(new IKSimilarity());   //在索引器中使用IKSimilarity相似度评估器     //String[] keys = {"4","testtest"};      //关键字数组    //String[] fields = {"id","title"};  //搜索的字段    //BooleanClause.Occur[] flags = {BooleanClause.Occur.MUST,BooleanClause.Occur.MUST};    //BooleanClause.Occur[]数组,它表示多个条件之间的关系     //使用 IKQueryParser类提供的parseMultiField方法构建多字段多条件查询    //Query query = IKQueryParser.parseMultiField(fields,keys, flags);     //IKQueryParser多个字段搜索      Query query =IKQueryParser.parse("text","上海人");  //IK搜索单个字段           IKAnalyzer analyzer = new IKAnalyzer();          //Query query =MultiFieldQueryParser.parse(Version.LUCENE_CURRENT, keys, fields, flags,analyzer);   //用MultiFieldQueryParser得到query对象          System.out.println("query:"+query.toString()); //查询条件        /*TopScoreDocCollector topCollector = TopScoreDocCollector.create(searcher.maxDoc(), false);    searcher.search(query,topCollector);        ScoreDoc[] docs = topCollector.topDocs(3).scoreDocs;    System.out.println(docs.length);*/              /**       *得到TopDocs对象之后,可以获取它的成员变量totalHits和scoreDocs.这两个成员变量的访问权限是public的,所以可以直接访问       */       TopDocs topDocs = searcher.search(query,1000001);       Integer count = topDocs.totalHits;       ScoreDoc[] scoreDocs = topDocs.scoreDocs;       for(int i = 0;i<count;i++){           ScoreDoc scoreDoc = scoreDocs[i];           Document document = searcher.doc(scoreDoc.doc);           document.get("text");       }       System.out.println("查找数据量:"+count);       long endtime = System.currentTimeMillis();       System.out.println(endtime-starttime);    reader.close(); //关闭索引      }  }

//直接查找的代码:

package sample3;    import java.net.UnknownHostException;    import com.mongodb.BasicDBObject;  import com.mongodb.DB;  import com.mongodb.DBCollection;  import com.mongodb.DBCursor;  import com.mongodb.DBObject;  import com.mongodb.Mongo;  import com.mongodb.MongoException;    public class Demo3 {   public static void main(String[] args) throws Exception{    Mongo mongo = new Mongo();    DB db = mongo.getDB("zhang");    DBCollection dbc = db.getCollection("test3");    DBObject basicdb = new BasicDBObject();    basicdb.put("$regex","上海人");    basicdb.put("$options","");    long startTime = System.currentTimeMillis();    DBCursor cursor = dbc.find(new BasicDBObject("text",basicdb));       int j =0;    while(cursor.hasNext()){        cursor.next().get("text");        j++;    }    System.out.println("查找数据量"+j);    long endTime = System.currentTimeMillis();    System.out.println("未优化前:"+(endTime-startTime));   }  }

测试结果

       从索引中找数据  

优化后的:  query:text:上海人 text:上海  查找数据量:1000001  查找用时:2917

      直接从mongodb 正则匹配的

查找数据量1000000  未优化前:6573

恩 速度还是有不少提高的

还有更好的解决方案吗 求指导哦