Solr4.8.0源码分析(6)之非排序查询

            1   /**
  2    * getDocList version that uses+populates query and filter caches.

              3
               * In the event of a timeout, the cache is not populated.

              4
            */
              5
            private
            void getDocListC(QueryResult qr, QueryCommand cmd) throws IOException {
  6     DocListAndSet out = new DocListAndSet();
  7    qr.setDocListAndSet(out);
  8     QueryResultKey key=null;
  9int maxDocRequested = cmd.getOffset() + cmd.getLen(); //当有偏移的查询产生，Solr首先会获取cmd.getOffset()+cmd.getLen()个的doc id然后　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　//再根据偏移量获取子集，所以maxDocRequested是实际的查询个数。
 10// check for overflow, and check for # docs in index 11if (maxDocRequested < 0 || maxDocRequested > maxDoc()) maxDocRequested = maxDoc();// 最多的情况获取所有doc id
 12int supersetMaxDoc= maxDocRequested;
 13     DocList superset = null;
 14 15int flags = cmd.getFlags();
 16     Query q = cmd.getQuery();
 17if (q instanceof ExtendedQuery) {
 18       ExtendedQuery eq = (ExtendedQuery)q;
 19if (!eq.getCache()) {
 20         flags |= (NO_CHECK_QCACHE | NO_SET_QCACHE | NO_CHECK_FILTERCACHE);
 21      }
 22    }
 23 24 25// we can try and look up the complete query in the cache.
 26// we can‘t do that if filter!=null though (we don‘t want to
 27// do hashCode() and equals() for a big DocSet).
        // 先从查询结果的缓存区查找是否出现过该条件的查询，若出现过则返回缓存的结果.关于缓存的内容将会独立写一篇文章
 28if (queryResultCache != null && cmd.getFilter()==null 29         && (flags & (NO_CHECK_QCACHE|NO_SET_QCACHE)) != ((NO_CHECK_QCACHE|NO_SET_QCACHE)))
 30    {
 31// all of the current flags can be reused during warming,
 32// so set all of them on the cache key. 33         key = new QueryResultKey(q, cmd.getFilterList(), cmd.getSort(), flags);
 34if ((flags & NO_CHECK_QCACHE)==0) {
 35           superset = queryResultCache.get(key);
 36 37if (superset != null) {
 38// check that the cache entry has scores recorded if we need them 39if ((flags & GET_SCORES)==0 || superset.hasScores()) {
 40// NOTE: subset() returns null if the DocList has fewer docs than
 41// requested 42               out.docList = superset.subset(cmd.getOffset(),cmd.getLen()); //如果有缓存，就从中去除一部分子集
 43            }
 44          }
 45if (out.docList != null) {   
 46// found the docList in the cache... now check if we need the docset too.
 47// OPT: possible future optimization - if the doclist contains all the matches,
 48// use it to make the docset instead of rerunning the query.
                //获取缓存中的docSet，并传给result。
 49if (out.docSet==null && ((flags & GET_DOCSET)!=0) ) {
 50if (cmd.getFilterList()==null) {
 51                 out.docSet = getDocSet(cmd.getQuery());
 52               } else {  
 53                 List<Query> newList = new ArrayList<>(cmd.getFilterList().size()+1);
 54                newList.add(cmd.getQuery());
 55                newList.addAll(cmd.getFilterList());
 56                 out.docSet = getDocSet(newList);
 57              }
 58            }
 59return;
 60          }
 61        }
 62 63// If we are going to generate the result, bump up to the
 64// next resultWindowSize for better caching. 65       // 修改supersetMaxDoc为queryResultWindwSize的整数倍
 66if ((flags & NO_SET_QCACHE) == 0) {
 67// handle 0 special case as well as avoid idiv in the common case. 68if (maxDocRequested < queryResultWindowSize) {
 69           supersetMaxDoc=queryResultWindowSize;
 70         } else {
 71           supersetMaxDoc = ((maxDocRequested -1)/queryResultWindowSize + 1)*queryResultWindowSize;
 72if (supersetMaxDoc < 0) supersetMaxDoc=maxDocRequested;
 73        }
 74       } else {
 75         key = null;  // we won‘t be caching the result 76      }
 77    }
 78    cmd.setSupersetMaxDoc(supersetMaxDoc);
 79 80 81// OK, so now we need to generate an answer.
 82// One way to do that would be to check if we have an unordered list
 83// of results for the base query.  If so, we can apply the filters and then
 84// sort by the resulting set.  This can only be used if:
 85// - the sort doesn‘t contain score
 86// - we don‘t want score returned.
 87 88// check if we should try and use the filter cache 89boolean useFilterCache=false;
 90if ((flags & (GET_SCORES|NO_CHECK_FILTERCACHE))==0 && useFilterForSortedQuery && cmd.getSort() != null && filterCache != null) {
 91       useFilterCache=true;
 92       SortField[] sfields = cmd.getSort().getSort();
 93for (SortField sf : sfields) {
 94if (sf.getType() == SortField.Type.SCORE) {
 95           useFilterCache=false;
 96break;
 97        }
 98      }
 99    }
100101if (useFilterCache) {
102// now actually use the filter cache.
103// for large filters that match few documents, this may be
104// slower than simply re-executing the query.105if (out.docSet == null) {
106         out.docSet = getDocSet(cmd.getQuery(),cmd.getFilter());
107         DocSet bigFilt = getDocSet(cmd.getFilterList());
108if (bigFilt != null) out.docSet = out.docSet.intersection(bigFilt);
109      }
110// todo: there could be a sortDocSet that could take a list of
111// the filters instead of anding them first...
112// perhaps there should be a multi-docset-iterator113      sortDocSet(qr, cmd);  //排序查询
114     } else {
115// do it the normal way...116if ((flags & GET_DOCSET)!=0) {
117// this currently conflates returning the docset for the base query vs
118// the base query and all filters.119         DocSet qDocSet = getDocListAndSetNC(qr,cmd);
120// cache the docSet matching the query w/o filtering121if (qDocSet!=null && filterCache!=null && !qr.isPartialResults()) filterCache.put(cmd.getQuery(),qDocSet);
122       } else {
123        getDocListNC(qr,cmd); //非排序查询，这也是本文的流程。
124      }
125assertnull != out.docList : "docList is null";
126    }
127128if (null == cmd.getCursorMark()) {
129// Kludge...
130// we can‘t use DocSlice.subset, even though it should be an identity op
131// because it gets confused by situations where there are lots of matches, but
132// less docs in the slice then were requested, (due to the cursor)
133// so we have to short circuit the call.
134// None of which is really a problem since we can‘t use caching with
135// cursors anyway, but it still looks weird to have to special case this
136// behavior based on this condition - hence the long explanation.137       superset = out.docList; //根据offset和len截取查询结果
138       out.docList = superset.subset(cmd.getOffset(),cmd.getLen()); 
139     } else {
140// sanity check our cursor assumptions141assertnull == superset : "cursor: superset isn‘t null";
142assert 0 == cmd.getOffset() : "cursor: command offset mismatch";
143assert 0 == out.docList.offset() : "cursor: docList offset mismatch";
144assert cmd.getLen() >= supersetMaxDoc : "cursor: superset len mismatch: " +
145         cmd.getLen() + " vs " + supersetMaxDoc;
146    }
147148// lastly, put the superset in the cache if the size is less than or equal
149// to queryResultMaxDocsCached150if (key != null && superset.size() <= queryResultMaxDocsCached && !qr.isPartialResults()) {
151      queryResultCache.put(key, superset);    //如果结果的个数小于或者等于queryResultMaxDocsCached则将本次查询结果放入缓存
152    }
153   }

             1
            final TopDocsCollector topCollector = buildTopDocsCollector(len, cmd); //新建TopDocsCollector对象，里面会新建(offset + len(查询条          //件的len))的HitQueue，每当获取到一个符合查询条件的doc，就会将该doc id放入HitQueue,并totalhit计数加一，这个totalhit变量也就是查询结果的数量
 2       Collector collector = topCollector;
 3if (terminateEarly) {
 4         collector = new EarlyTerminatingCollector(collector, cmd.len);
 5      }
 6if( timeAllowed > 0 ) {
 7         collector = new TimeLimitingCollector(collector, TimeLimitingCollector.getGlobalCounter(), timeAllowed); 
           //TimeLimitingCollector的实现原理很简单，从第一个找到符合查询条件的doc id开始计时，在达到timeAllowed之前，会想查询得到的doc id放入HitQue           //ue,一旦timeAllowed到了，就会立即扔出错误，中断后续的查询。这对于我们优化查询是个重要的提示
 8      }
 9if (pf.postFilter != null) {
10        pf.postFilter.setLastDelegate(collector);
11         collector = pf.postFilter;
12      }
13try {
           // 进入Lucene的IndexSearch.Search()
14super.search(query, luceneFilter, collector);
15if(collector instanceof DelegatingCollector) {
16          ((DelegatingCollector)collector).finish();
17        }
18      }
19catch( TimeLimitingCollector.TimeExceededException x ) {
20         log.warn( "Query: " + query + "; " + x.getMessage() );
21         qr.setPartialResults(true);
22      }
2324       totalHits = topCollector.getTotalHits();           //返回totalhit的结果
25       TopDocs topDocs = topCollector.topDocs(0, len);    //返回优先级队列hitqueue的doc id
26      populateNextCursorMarkFromTopDocs(qr, cmd, topDocs);
2728       maxScore = totalHits>0 ? topDocs.getMaxScore() : 0.0f;
29       nDocsReturned = topDocs.scoreDocs.length;
30       ids = newint[nDocsReturned];
31       scores = (cmd.getFlags()&GET_SCORES)!=0 ? newfloat[nDocsReturned] : null;
32for (int i=0; i<nDocsReturned; i++) {
33         ScoreDoc scoreDoc = topDocs.scoreDocs[i];
34         ids[i] = scoreDoc.doc;
35if (scores != null) scores[i] = scoreDoc.score;
36       }

            /**
            
   * Calls {
            @link
             Collector#collect(int)} on the decorated {
            @link
             Collector}
   * unless the allowed time has passed, in which case it throws an exception.
   * 
   * 
            @throws
             TimeExceededException
   *           if the time allowed has exceeded.
   
            */
            
  @Override
  
            public
            void collect(finalint doc) throws IOException {
    finallong time = clock.get();
    if (timeout < time) {
      if (greedy) {
        //System.out.println(this+"  greedy: before failing, collecting doc: "+(docBase + doc)+"  "+(time-t0));        collector.collect(doc);
      }
      //System.out.println(this+"  failing on:  "+(docBase + doc)+"  "+(time-t0));thrownew TimeExceededException( timeout-t0, time-t0, docBase + doc );   
    }
    //System.out.println(this+"  collecting: "+(docBase + doc)+"  "+(time-t0));    collector.collect(doc);
  }

             1
            public BooleanWeight(IndexSearcher searcher, boolean disableCoord)
 2throws IOException {
 3this.similarity = searcher.getSimilarity();
 4this.disableCoord = disableCoord;
 5       weights = new ArrayList<>(clauses.size());
 6for (int i = 0 ; i < clauses.size(); i++) {
 7         BooleanClause c = clauses.get(i);
 8         Weight w = c.getQuery().createWeight(searcher);
 9        weights.add(w);
10if (!c.isProhibited()) {
11           maxCoord++;
12        }
13      }
14     }

             1
            /**
             2
               * Lower-level search API.

             3
               * 

             4
               * <p>

             5
               * {
            @link
             Collector#collect(int)} is called for every document. <br>

             6
               * 

             7
               * <p>

             8
               * NOTE: this method executes the searches on all given leaves exclusively.

             9
               * To search across all the searchers leaves use {
            @link
             #leafContexts}.

            10
               * 

            11
               * 
            @param
             leaves 

            12
               *          the searchers leaves to execute the searches on

            13
               * 
            @param
             weight

            14
               *          to match documents

            15
               * 
            @param
             collector

            16
               *          to receive hits

            17
               * 
            @throws
             BooleanQuery.TooManyClauses If a query would exceed 

            18
               *         {
            @link
             BooleanQuery#getMaxClauseCount()} clauses.

            19
            */
            20
            protected
            void search(List<AtomicReaderContext> leaves, Weight weight, Collector collector)
21throws IOException {
2223// TODO: should we make this
24// threaded...?  the Collector could be sync‘d?
25// always use single thread:26for (AtomicReaderContext ctx : leaves) { // search each subreader27try {
28        collector.setNextReader(ctx);
29       } catch (CollectionTerminatedException e) {
30// there is no doc of interest in this reader context
31// continue with the following leaf32continue;
33      }
34       BulkScorer scorer = weight.bulkScorer(ctx, !collector.acceptsDocsOutOfOrder(), ctx.reader().getLiveDocs());
35if (scorer != null) {
36try {
37          scorer.score(collector);
38         } catch (CollectionTerminatedException e) {
39// collection was terminated prematurely
40// continue with the following leaf41        }
42      }
43    }
44   }

             1
             @Override

             2
            public
            boolean score(Collector collector, int max) throws IOException {
 3// TODO: this may be sort of weird, when we are
 4// embedded in a BooleanScorer, because we are
 5// called for every chunk of 2048 documents.  But,
 6// then, scorer is a FakeScorer in that case, so any
 7// Collector doing something "interesting" in
 8// setScorer will be forced to use BS2 anyways: 9      collector.setScorer(scorer);
10if (max == DocIdSetIterator.NO_MORE_DOCS) {
11        scoreAll(collector, scorer);
12returnfalse;
13       } else {
14int doc = scorer.docID();
15if (doc < 0) {
16           doc = scorer.nextDoc();
17        }
18return scoreRange(collector, scorer, doc, max);
19      }
20     }

            1
            static
            void scoreAll(Collector collector, Scorer scorer) throws IOException {
2int doc;
3while ((doc = scorer.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
4        collector.collect(doc);
5      }
6     }

             1
                @Override

             2
            public
            void collect(int doc) throws IOException {
 3float score = scorer.score();
 4 5// This collector cannot handle these scores: 6assert score != Float.NEGATIVE_INFINITY;
 7assert !Float.isNaN(score);
 8 9       totalHits++;
10if (score <= pqTop.score) {
11// Since docs are returned in-order (i.e., increasing doc Id), a document
12// with equal score to pqTop.score cannot compete since HitQueue favors
13// documents with lower doc Ids. Therefore reject those docs too.14return;
15      }
16       pqTop.doc = doc + docBase;
17       pqTop.score = score;
18       pqTop = pq.updateTop();
19     }