LUCENE-3622: merge trunk (1213324:1213812)

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene3622@1213825 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -638,6 +638,10 @@ Optimizations
   boolean clauses are required and instances of TermQuery. 
   (Simon Willnauer, Robert Muir)
 
+* LUCENE-3643: FilteredQuery and IndexSearcher.search(Query, Filter,...)
+  now optimize the special case query instanceof MatchAllDocsQuery to
+  execute as ConstantScoreQuery.  (Uwe Schindler)
+
 Bug fixes
 
 * LUCENE-2803: The FieldCache can miss values if an entry for a reader
@@ -712,6 +716,10 @@ New Features
 * LUCENE-3593: Added a FieldValueFilter that accepts all documents that either
   have at least one or no value at all in a specific field. (Simon Willnauer,
   Uwe Schindler, Robert Muir)
+
+* LUCENE-3586: CheckIndex and IndexUpgrader allow you to specify the
+  specific FSDirectory implementation to use (with the new -dir-impl
+  command-line option).  (Luca Cavanna via Mike McCandless)
 
 Bug fixes
 
@@ -731,6 +739,10 @@ Bug fixes
 * LUCENE-3641: Fixed MultiReader to correctly propagate readerFinishedListeners
   to clones/reopened readers.  (Uwe Schindler)
 
+* LUCENE-3642: Fixed bugs in CharTokenizer, n-gram filters, and smart chinese 
+  where they would create invalid offsets in some situations, leading to problems
+  in highlighting. (Max Beutel via Robert Muir)
+
 Documentation
 
 * LUCENE-3597: Fixed incorrect grouping documentation. (Martijn van Groningen, Robert Muir)

lucene/src/java/org/apache/lucene/index/CheckIndex.java

@@ -28,7 +28,6 @@
 import org.apache.lucene.index.DocValues.SortedSource;
 import org.apache.lucene.index.DocValues.Source;
 import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.PerDocProducer;
 
 import java.io.File;
 import java.io.IOException;
@@ -41,10 +40,21 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.lucene.document.FieldType; // for javadocs
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.codecs.Codec;
+
 import org.apache.lucene.index.codecs.BlockTreeTermsReader;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CommandLineUtil;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.StringHelper;
 
@@ -1477,49 +1487,57 @@ public static void main(String[] args) throws IOException, InterruptedException
     boolean verbose = false;
     List<String> onlySegments = new ArrayList<String>();
     String indexPath = null;
+    String dirImpl = null;
     int i = 0;
     while(i < args.length) {
-      if (args[i].equals("-fix")) {
+      String arg = args[i];
+      if ("-fix".equals(arg)) {
         doFix = true;
-        i++;
-      } else if (args[i].equals("-codec")) {
+      } else if ("-codec".equals(arg)) {
         if (i == args.length-1) {
           System.out.println("ERROR: missing name for -codec option");
           System.exit(1);
         }
-        codec = Codec.forName(args[i+1]);
-        i+=2;
-      } else if (args[i].equals("-verbose")) {
-        verbose = true;
         i++;
-      } else if (args[i].equals("-segment")) {
+        codec = Codec.forName(args[i]);
+      } else if (arg.equals("-verbose")) {
+        verbose = true;
+      } else if (arg.equals("-segment")) {
         if (i == args.length-1) {
           System.out.println("ERROR: missing name for -segment option");
           System.exit(1);
         }
-        onlySegments.add(args[i+1]);
-        i += 2;
+        i++;
+        onlySegments.add(args[i]);
+      } else if ("-dir-impl".equals(arg)) {
+        if (i == args.length - 1) {
+          System.out.println("ERROR: missing value for -dir-impl option");
+          System.exit(1);
+        }
+        i++;
+        dirImpl = args[i];
       } else {
         if (indexPath != null) {
           System.out.println("ERROR: unexpected extra argument '" + args[i] + "'");
           System.exit(1);
         }
         indexPath = args[i];
-        i++;
       }
+      i++;
     }
 
     if (indexPath == null) {
       System.out.println("\nERROR: index path not specified");
-      System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" +
+      System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y] [-dir-impl X]\n" +
                          "\n" +
                          "  -fix: actually write a new segments_N file, removing any problematic segments\n" +
                          "  -codec X: when fixing, codec to write the new segments_N file with\n" +
                          "  -verbose: print additional details\n" +
                          "  -segment X: only check the specified segments.  This can be specified multiple\n" + 
                          "              times, to check more than one segment, eg '-segment _2 -segment _a'.\n" +
                          "              You can't use this with the -fix option\n" +
-                         "\n" + 
+                         "  -dir-impl X: use a specific " + FSDirectory.class.getSimpleName() + " implementation. " +
+                         		"If no package is specified the " + FSDirectory.class.getPackage().getName() + " package will be used.\n" +
                          "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" +
                          "documents (perhaps many) to be permanently removed from the index.  Always make\n" +
                          "a backup copy of your index before running this!  Do not run this tool on an index\n" +
@@ -1549,7 +1567,11 @@ else if (doFix) {
     System.out.println("\nOpening index @ " + indexPath + "\n");
     Directory dir = null;
     try {
-      dir = FSDirectory.open(new File(indexPath));
+      if (dirImpl == null) {
+        dir = FSDirectory.open(new File(indexPath));
+      } else {
+        dir = CommandLineUtil.newFSDirectory(dirImpl, new File(indexPath));
+      }
     } catch (Throwable t) {
       System.out.println("ERROR: could not open directory \"" + indexPath + "\"; exiting");
       t.printStackTrace(System.out);

lucene/src/java/org/apache/lucene/index/IndexReader.java

@@ -34,6 +34,7 @@
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CommandLineUtil;
 import org.apache.lucene.util.ReaderUtil;         // for javadocs
 
 /** IndexReader is an abstract class, providing an interface for accessing an
@@ -965,17 +966,28 @@ public IndexCommit getIndexCommit() throws IOException {
   public static void main(String [] args) {
     String filename = null;
     boolean extract = false;
+    String dirImpl = null;
 
-    for (int i = 0; i < args.length; ++i) {
-      if (args[i].equals("-extract")) {
+    int j = 0;
+    while(j < args.length) {
+      String arg = args[j];
+      if ("-extract".equals(arg)) {
         extract = true;
+      } else if ("-dir-impl".equals(arg)) {
+        if (j == args.length - 1) {
+          System.out.println("ERROR: missing value for -dir-impl option");
+          System.exit(1);
+        }
+        j++;
+        dirImpl = args[j];
       } else if (filename == null) {
-        filename = args[i];
+        filename = arg;
       }
+      j++;
     }
 
     if (filename == null) {
-      System.out.println("Usage: org.apache.lucene.index.IndexReader [-extract] <cfsfile>");
+      System.out.println("Usage: org.apache.lucene.index.IndexReader [-extract] [-dir-impl X] <cfsfile>");
       return;
     }
 
@@ -987,7 +999,12 @@ public static void main(String [] args) {
       File file = new File(filename);
       String dirname = file.getAbsoluteFile().getParent();
       filename = file.getName();
-      dir = FSDirectory.open(new File(dirname));
+      if (dirImpl == null) {
+        dir = FSDirectory.open(new File(dirname));
+      } else {
+        dir = CommandLineUtil.newFSDirectory(dirImpl, new File(dirname));
+      }
+
       cfr = new CompoundFileDirectory(dir, filename, IOContext.DEFAULT, false);
 
       String [] files = cfr.listAll();

lucene/src/java/org/apache/lucene/index/IndexUpgrader.java

@@ -19,6 +19,7 @@
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.CommandLineUtil;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.Version;
@@ -54,36 +55,56 @@ public final class IndexUpgrader {
   private static void printUsage() {
     System.err.println("Upgrades an index so all segments created with a previous Lucene version are rewritten.");
     System.err.println("Usage:");
-    System.err.println("  java " + IndexUpgrader.class.getName() + " [-delete-prior-commits] [-verbose] indexDir");
+    System.err.println("  java " + IndexUpgrader.class.getName() + " [-delete-prior-commits] [-verbose] [-dir-impl X] indexDir");
     System.err.println("This tool keeps only the last commit in an index; for this");
     System.err.println("reason, if the incoming index has more than one commit, the tool");
     System.err.println("refuses to run by default. Specify -delete-prior-commits to override");
     System.err.println("this, allowing the tool to delete all but the last commit.");
+    System.err.println("Specify a " + FSDirectory.class.getSimpleName() + 
+        " implementation through the -dir-impl option to force its use. If no package is specified the " 
+        + FSDirectory.class.getPackage().getName() + " package will be used.");
     System.err.println("WARNING: This tool may reorder document IDs!");
     System.exit(1);
   }
 
   @SuppressWarnings("deprecation")
   public static void main(String[] args) throws IOException {
-    String dir = null;
+    String path = null;
     boolean deletePriorCommits = false;
     PrintStream out = null;
-    for (String arg : args) {
+    String dirImpl = null;
+    int i = 0;
+    while (i<args.length) {
+      String arg = args[i];
       if ("-delete-prior-commits".equals(arg)) {
         deletePriorCommits = true;
       } else if ("-verbose".equals(arg)) {
         out = System.out;
-      } else if (dir == null) {
-        dir = arg;
-      } else {
+      } else if (path == null) {
+        path = arg;
+      } else if ("-dir-impl".equals(arg)) {
+        if (i == args.length - 1) {
+          System.out.println("ERROR: missing value for -dir-impl option");
+          System.exit(1);
+        }
+        i++;
+        dirImpl = args[i];
+      }else {
         printUsage();
       }
+      i++;
     }
-    if (dir == null) {
+    if (path == null) {
       printUsage();
     }
 
-    new IndexUpgrader(FSDirectory.open(new File(dir)), Version.LUCENE_CURRENT, out, deletePriorCommits).upgrade();
+    Directory dir = null;
+    if (dirImpl == null) {
+      dir = FSDirectory.open(new File(path));
+    } else {
+      dir = CommandLineUtil.newFSDirectory(dirImpl, new File(path));
+    }
+    new IndexUpgrader(dir, Version.LUCENE_CURRENT, out, deletePriorCommits).upgrade();
   }
 
   private final Directory dir;

lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java

@@ -38,36 +38,47 @@ public class CachingWrapperFilter extends Filter {
   Filter filter;
 
   protected final FilterCache<DocIdSet> cache;
+  private final boolean recacheDeletes;
 
-  static class FilterCache<T> {
+  private static class FilterCache<T> {
 
     /**
      * A transient Filter cache (package private because of test)
      */
-    // NOTE: not final so that we can dynamically re-init
-    // after de-serialize
-    transient Map<Object,T> cache;
+    private final Map<Object,Map<Object,T>> cache = new WeakHashMap<Object,Map<Object,T>>();
 
-    public synchronized T get(IndexReader reader, Object coreKey) throws IOException {
-      T value;
-
-      if (cache == null) {
-        cache = new WeakHashMap<Object,T>();
+    public synchronized T get(IndexReader reader, Object coreKey, Object coreSubKey) throws IOException {
+      Map<Object,T> innerCache = cache.get(coreKey);
+      if (innerCache == null) {
+        innerCache = new WeakHashMap<Object,T>();
+        cache.put(coreKey, innerCache);
       }
 
-      return cache.get(coreKey);
+      return innerCache.get(coreSubKey);
     }
 
-    public synchronized void put(Object coreKey, T value) {
-      cache.put(coreKey, value);
+    public synchronized void put(Object coreKey, Object coreSubKey, T value) {
+      cache.get(coreKey).put(coreSubKey, value);
     }
   }
 
   /** Wraps another filter's result and caches it.
    * @param filter Filter to cache results of
    */
   public CachingWrapperFilter(Filter filter) {
+    this(filter, false);
+  }
+
+  /** Wraps another filter's result and caches it.  If
+   *  recacheDeletes is true, then new deletes (for example
+   *  after {@link IndexReader#openIfChanged}) will be AND'd
+   *  and cached again.
+   *
+   *  @param filter Filter to cache results of
+   */
+  public CachingWrapperFilter(Filter filter, boolean recacheDeletes) {
     this.filter = filter;
+    this.recacheDeletes = recacheDeletes;
     cache = new FilterCache<DocIdSet>();
   }
 
@@ -106,33 +117,48 @@ public DocIdSet getDocIdSet(AtomicReaderContext context, final Bits acceptDocs)
     final IndexReader reader = context.reader;
     final Object coreKey = reader.getCoreCacheKey();
 
-    DocIdSet docIdSet = cache.get(reader, coreKey);
+    // Only cache if incoming acceptDocs is == live docs;
+    // if Lucene passes in more interesting acceptDocs in
+    // the future we don't want to over-cache:
+    final boolean doCacheSubAcceptDocs = recacheDeletes && acceptDocs == reader.getLiveDocs();
+
+    final Bits subAcceptDocs;
+    if (doCacheSubAcceptDocs) {
+      subAcceptDocs = acceptDocs;
+    } else {
+      subAcceptDocs = null;
+    }
+
+    DocIdSet docIdSet = cache.get(reader, coreKey, subAcceptDocs);
     if (docIdSet != null) {
       hitCount++;
     } else {
       missCount++;
-      // cache miss: we use no acceptDocs here
-      // (this saves time on building DocIdSet, the acceptDocs will be applied on the cached set)
-      docIdSet = docIdSetToCache(filter.getDocIdSet(context, null/**!!!*/), reader);
-      cache.put(coreKey, docIdSet);
+      docIdSet = docIdSetToCache(filter.getDocIdSet(context, subAcceptDocs), reader);
+      cache.put(coreKey, subAcceptDocs, docIdSet);
+    }
+
+    if (doCacheSubAcceptDocs) {
+      return docIdSet;
+    } else {
+      return BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs);
     }
-
-    return BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs);
   }
 
   @Override
   public String toString() {
-    return "CachingWrapperFilter("+filter+")";
+    return "CachingWrapperFilter("+filter+",recacheDeletes=" + recacheDeletes + ")";
   }
 
   @Override
   public boolean equals(Object o) {
     if (!(o instanceof CachingWrapperFilter)) return false;
-    return this.filter.equals(((CachingWrapperFilter)o).filter);
+    final CachingWrapperFilter other = (CachingWrapperFilter) o;
+    return this.filter.equals(other.filter) && this.recacheDeletes == other.recacheDeletes;
   }
 
   @Override
   public int hashCode() {
-    return filter.hashCode() ^ 0x1117BF25;  
+    return (filter.hashCode() ^ 0x1117BF25) + (recacheDeletes ? 0 : 1);
   }
 }