25f005
From f93bcfb45708895f90396552677f11881f342dc9 Mon Sep 17 00:00:00 2001
25f005
From: Mat Booth <mat.booth@redhat.com>
25f005
Date: Wed, 4 Mar 2015 15:10:27 +0000
25f005
Subject: [PATCH] Port to Lucene 4.
25f005
25f005
---
25f005
 org.eclipse.help.base/META-INF/MANIFEST.MF         |  4 +-
25f005
 .../eclipse/help/internal/search/Analyzer_en.java  | 27 +++++---
25f005
 .../help/internal/search/DefaultAnalyzer.java      | 13 ++--
25f005
 .../search/LowerCaseAndDigitsTokenizer.java        |  7 +-
25f005
 .../eclipse/help/internal/search/QueryBuilder.java | 15 +++-
25f005
 .../eclipse/help/internal/search/SearchIndex.java  | 81 ++++++++++------------
25f005
 .../help/internal/search/SmartAnalyzer.java        | 14 ++--
25f005
 .../help/internal/search/WordTokenStream.java      |  2 +
25f005
 org.eclipse.ua.tests/META-INF/MANIFEST.MF          |  7 +-
25f005
 .../help/search/PrebuiltIndexCompatibility.java    |  6 +-
25f005
 10 files changed, 94 insertions(+), 82 deletions(-)
25f005
25f005
diff --git a/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF b/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF
25f005
index ee34c8e..fdef3e6 100644
25f005
--- a/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF
25f005
+++ b/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF
25f005
@@ -43,8 +43,8 @@ Require-Bundle: org.eclipse.ant.core;bundle-version="3.2.200";resolution:=option
25f005
  org.eclipse.core.runtime;bundle-version="[3.11.0,4.0.0)",
25f005
  org.eclipse.help;bundle-version="[3.5.0,4.0.0)";visibility:=reexport,
25f005
  org.eclipse.core.expressions;bundle-version="[3.4.200,4.0.0)",
25f005
- org.apache.lucene.analysis;bundle-version="[3.5.0,4.0.0)",
25f005
- org.apache.lucene.core;bundle-version="[3.5.0,4.0.0)",
25f005
+ org.apache.lucene.analysis;bundle-version="[4.7.0,5.0.0)",
25f005
+ org.apache.lucene.core;bundle-version="[4.7.0,5.0.0)",
25f005
  org.eclipse.core.net;bundle-version="1.2.200"
25f005
 Import-Package: com.ibm.icu.text,
25f005
  org.eclipse.equinox.http.jetty;resolution:=optional
Roland Grunberg e635b2
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java
25f005
index a066aa4..6c41103 100644
Roland Grunberg e635b2
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java
Roland Grunberg e635b2
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java
25f005
@@ -11,10 +11,15 @@
Roland Grunberg e635b2
  *******************************************************************************/
Roland Grunberg e635b2
 package org.eclipse.help.internal.search;
Roland Grunberg e635b2
 import java.io.*;
Roland Grunberg e635b2
-import java.util.HashSet;
Roland Grunberg e635b2
-import java.util.Set;
Roland Grunberg e635b2
+import java.util.ArrayList;
Roland Grunberg e635b2
+import java.util.List;
Roland Grunberg e635b2
 
25f005
-import org.apache.lucene.analysis.*;
25f005
+import org.apache.lucene.analysis.Analyzer;
25f005
+import org.apache.lucene.analysis.TokenStream;
25f005
+import org.apache.lucene.analysis.Tokenizer;
25f005
+import org.apache.lucene.analysis.core.StopFilter;
25f005
+import org.apache.lucene.analysis.en.PorterStemFilter;
25f005
+import org.apache.lucene.analysis.util.CharArraySet;
25f005
 import org.apache.lucene.util.Version;
Roland Grunberg e635b2
 /**
Roland Grunberg e635b2
  * Lucene Analyzer for English. LowerCaseTokenizer->StopFilter->PorterStemFilter
25f005
@@ -27,18 +32,22 @@ public final class Analyzer_en extends Analyzer {
Roland Grunberg e635b2
 		super();
Roland Grunberg e635b2
 	}
Roland Grunberg e635b2
 	/**
Roland Grunberg e635b2
-	 * Creates a TokenStream which tokenizes all the text in the provided
Roland Grunberg e635b2
+	 * Creates a TokenStreamComponents which tokenizes all the text in the provided
Roland Grunberg e635b2
 	 * Reader.
Roland Grunberg e635b2
 	 */
Roland Grunberg e635b2
-	public final TokenStream tokenStream(String fieldName, Reader reader) {
25f005
-		return new PorterStemFilter(new StopFilter(Version.LUCENE_30, new LowerCaseAndDigitsTokenizer(reader), getStopWords(), false));
Roland Grunberg e635b2
+	@Override
Roland Grunberg e635b2
+	public final TokenStreamComponents createComponents(String fieldName, Reader reader) {
Roland Grunberg e635b2
+		CharArraySet stopWordsSet = StopFilter.makeStopSet(Version.LUCENE_47, getStopWords(), true);
Roland Grunberg e635b2
+		Tokenizer source = new LowerCaseAndDigitsTokenizer(reader);
Roland Grunberg e635b2
+		TokenStream filter = new PorterStemFilter(new StopFilter(Version.LUCENE_47, source, stopWordsSet));
Roland Grunberg e635b2
+		return new TokenStreamComponents(source, filter);
Roland Grunberg e635b2
 	}
25f005
 
Roland Grunberg e635b2
-	private Set<string> stopWords;
Roland Grunberg e635b2
+	private List<string> stopWords;
25f005
 
Roland Grunberg e635b2
-	private Set<string> getStopWords() {
Roland Grunberg e635b2
+	private List<string> getStopWords() {
Roland Grunberg e635b2
 		if ( stopWords == null ) {
Roland Grunberg e635b2
-			stopWords = new HashSet<string>();
Roland Grunberg e635b2
+			stopWords = new ArrayList<string>();
Roland Grunberg e635b2
 			for (int i = 0; i < STOP_WORDS.length; i++) {
Roland Grunberg e635b2
 			    stopWords.add(STOP_WORDS[i]);
Roland Grunberg e635b2
 			}
Roland Grunberg e635b2
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java
25f005
index 4109474..2718307 100644
Roland Grunberg e635b2
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java
Roland Grunberg e635b2
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java
25f005
@@ -17,8 +17,8 @@ import java.util.StringTokenizer;
Roland Grunberg e635b2
 import com.ibm.icu.text.BreakIterator;
Roland Grunberg e635b2
 
Roland Grunberg e635b2
 import org.apache.lucene.analysis.Analyzer;
Roland Grunberg e635b2
-import org.apache.lucene.analysis.LowerCaseFilter;
25f005
-import org.apache.lucene.analysis.TokenStream;
Roland Grunberg e635b2
+import org.apache.lucene.analysis.Tokenizer;
25f005
+import org.apache.lucene.analysis.core.LowerCaseFilter;
25f005
 import org.apache.lucene.util.Version;
Roland Grunberg e635b2
 import org.eclipse.help.internal.base.HelpBasePlugin;
Roland Grunberg e635b2
 
25f005
@@ -84,11 +84,14 @@ public final class DefaultAnalyzer extends Analyzer {
Roland Grunberg e635b2
 	}
Roland Grunberg e635b2
 
Roland Grunberg e635b2
 	/**
Roland Grunberg e635b2
-	 * Creates a TokenStream which tokenizes all the text in the provided
Roland Grunberg e635b2
+	 * Creates a TokenStreamComponents which tokenizes all the text in the provided
Roland Grunberg e635b2
 	 * Reader.
Roland Grunberg e635b2
 	 */
Roland Grunberg e635b2
-	public final TokenStream tokenStream(String fieldName, Reader reader) {
25f005
-		return new LowerCaseFilter(Version.LUCENE_30, new WordTokenStream(fieldName, reader, locale));
Roland Grunberg e635b2
+	@Override
Roland Grunberg e635b2
+	public final TokenStreamComponents createComponents(String fieldName, Reader reader) {
Roland Grunberg e635b2
+		Tokenizer source = new WordTokenStream(fieldName, reader, locale);
Roland Grunberg e635b2
+		LowerCaseFilter filter = new LowerCaseFilter(Version.LUCENE_47, source);
Roland Grunberg e635b2
+		return new TokenStreamComponents(source, filter);
Roland Grunberg e635b2
 	}
Roland Grunberg e635b2
 
Roland Grunberg e635b2
 	/**
Roland Grunberg e635b2
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java
25f005
index a475688..91e3cb4 100644
Roland Grunberg e635b2
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java
Roland Grunberg e635b2
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java
25f005
@@ -13,7 +13,7 @@ package org.eclipse.help.internal.search;
Roland Grunberg e635b2
 
Roland Grunberg e635b2
 import java.io.*;
Roland Grunberg e635b2
 
Roland Grunberg e635b2
-import org.apache.lucene.analysis.*;
25f005
+import org.apache.lucene.analysis.util.CharTokenizer;
25f005
 import org.apache.lucene.util.Version;
Roland Grunberg e635b2
 
Roland Grunberg e635b2
 /**
25f005
@@ -22,13 +22,14 @@ import org.apache.lucene.util.Version;
Roland Grunberg e635b2
 public class LowerCaseAndDigitsTokenizer extends CharTokenizer {
Roland Grunberg e635b2
 
Roland Grunberg e635b2
     public LowerCaseAndDigitsTokenizer(Reader input) {
25f005
-        super(Version.LUCENE_30, input);
Roland Grunberg e635b2
+        super(Version.LUCENE_47, input);
Roland Grunberg e635b2
     }
Roland Grunberg e635b2
     protected char normalize(char c) {
Roland Grunberg e635b2
         return Character.toLowerCase(c);
Roland Grunberg e635b2
     }
Roland Grunberg e635b2
 
Roland Grunberg e635b2
-    protected boolean isTokenChar(char c) {
Roland Grunberg e635b2
+    @Override
Roland Grunberg e635b2
+    public boolean isTokenChar(int c) {
Roland Grunberg e635b2
         return Character.isLetterOrDigit(c);
Roland Grunberg e635b2
     }
Roland Grunberg e635b2
 
Roland Grunberg e635b2
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java
25f005
index 9cc690e..6449adb 100644
Roland Grunberg e635b2
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java
Roland Grunberg e635b2
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java
25f005
@@ -243,16 +243,25 @@ public class QueryBuilder {
Roland Grunberg e635b2
 	private List<string> analyzeText(Analyzer analyzer, String fieldName, String text) {
Roland Grunberg e635b2
 		List<string> words = new ArrayList<string>(1);
Roland Grunberg e635b2
 		Reader reader = new StringReader(text);
Roland Grunberg e635b2
-		TokenStream tStream = analyzer.tokenStream(fieldName, reader);
25f005
 
1a78a8
-		CharTermAttribute termAttribute = tStream.getAttribute(CharTermAttribute.class);
Roland Grunberg 2af405
+		TokenStream tStream = null;
Roland Grunberg e635b2
 		try {
Roland Grunberg 2af405
+			tStream = analyzer.tokenStream(fieldName, reader);
Roland Grunberg 2af405
+			tStream.reset();
25f005
+			CharTermAttribute termAttribute = (CharTermAttribute) tStream
25f005
+					.getAttribute(CharTermAttribute.class);
Roland Grunberg e635b2
 			while (tStream.incrementToken()) {
Roland Grunberg e635b2
 				String term = termAttribute.toString();
Roland Grunberg e635b2
 				words.add(term);
Roland Grunberg 2af405
 			}
Roland Grunberg 2af405
-			reader.close();
Roland Grunberg 2af405
 		} catch (IOException ioe) {
Roland Grunberg 2af405
+		} finally {
Roland Grunberg 2af405
+			if (tStream != null) {
Roland Grunberg 2af405
+				try {
Roland Grunberg 2af405
+					tStream.close();
Roland Grunberg 2af405
+				} catch (IOException e) {
Roland Grunberg 2af405
+				}
Roland Grunberg 2af405
+			}
Roland Grunberg 2af405
 		}
25f005
 
Roland Grunberg 2af405
 		return words;
Roland Grunberg e635b2
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java
25f005
index 33c9476..5a46fe5 100644
Roland Grunberg e635b2
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java
Roland Grunberg e635b2
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java
25f005
@@ -33,17 +33,20 @@ import java.util.Set;
Roland Grunberg e635b2
 import java.util.zip.ZipEntry;
Roland Grunberg e635b2
 import java.util.zip.ZipInputStream;
Roland Grunberg e635b2
 
25f005
-import org.apache.lucene.analysis.LimitTokenCountAnalyzer;
Roland Grunberg e635b2
+import org.apache.lucene.analysis.miscellaneous.LimitTokenCountAnalyzer;
Roland Grunberg e635b2
 import org.apache.lucene.document.Document;
Roland Grunberg e635b2
 import org.apache.lucene.document.Field;
Roland Grunberg e635b2
+import org.apache.lucene.index.AtomicReader;
Roland Grunberg e635b2
+import org.apache.lucene.index.DirectoryReader;
Roland Grunberg e635b2
+import org.apache.lucene.index.DocsEnum;
Roland Grunberg e635b2
 import org.apache.lucene.index.IndexReader;
Roland Grunberg e635b2
 import org.apache.lucene.index.IndexWriter;
25f005
-import org.apache.lucene.index.Term;
Roland Grunberg e635b2
-import org.apache.lucene.index.TermDocs;
25f005
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
25f005
 import org.apache.lucene.index.IndexWriterConfig;
25f005
 import org.apache.lucene.index.LogByteSizeMergePolicy;
25f005
 import org.apache.lucene.index.LogMergePolicy;
25f005
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
25f005
+import org.apache.lucene.index.Term;
Roland Grunberg e635b2
 import org.apache.lucene.search.BooleanQuery;
Roland Grunberg e635b2
 import org.apache.lucene.search.IndexSearcher;
Roland Grunberg e635b2
 import org.apache.lucene.search.Query;
25f005
@@ -283,7 +286,7 @@ public class SearchIndex implements IHelpSearchIndex {
Roland Grunberg e635b2
 			indexedDocs.restore();
Roland Grunberg e635b2
 			setInconsistent(true);
25f005
 			LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), 1000000);
25f005
-			IndexWriterConfig writerConfig = new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_31, analyzer);
25f005
+			IndexWriterConfig writerConfig = new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_47, analyzer);
25f005
 			writerConfig.setOpenMode(create ? OpenMode.CREATE : OpenMode.APPEND);
25f005
 			LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
25f005
 			mergePolicy.setMergeFactor(20);
25f005
@@ -307,7 +310,7 @@ public class SearchIndex implements IHelpSearchIndex {
Roland Grunberg e635b2
 			indexedDocs = new HelpProperties(INDEXED_DOCS_FILE, indexDir);
Roland Grunberg e635b2
 			indexedDocs.restore();
Roland Grunberg e635b2
 			setInconsistent(true);
Roland Grunberg e635b2
-			ir = IndexReader.open(luceneDirectory, false);
Roland Grunberg e635b2
+			ir = DirectoryReader.open(luceneDirectory);
Roland Grunberg e635b2
 			return true;
Roland Grunberg e635b2
 		} catch (IOException e) {
Roland Grunberg e635b2
 			HelpBasePlugin.logError("Exception occurred in search indexing at beginDeleteBatch.", e); //$NON-NLS-1$
25f005
@@ -323,7 +326,7 @@ public class SearchIndex implements IHelpSearchIndex {
Roland Grunberg e635b2
 			if (ir != null) {
Roland Grunberg e635b2
 				ir.close();
Roland Grunberg e635b2
 			}
Roland Grunberg e635b2
-			ir = IndexReader.open(luceneDirectory, false);
Roland Grunberg e635b2
+			ir = DirectoryReader.open(luceneDirectory);
Roland Grunberg e635b2
 			return true;
Roland Grunberg e635b2
 		} catch (IOException e) {
Roland Grunberg e635b2
 			HelpBasePlugin.logError("Exception occurred in search indexing at beginDeleteBatch.", e); //$NON-NLS-1$
25f005
@@ -341,7 +344,7 @@ public class SearchIndex implements IHelpSearchIndex {
Roland Grunberg e635b2
 	public IStatus removeDocument(String name) {
Roland Grunberg e635b2
 		Term term = new Term(FIELD_NAME, name);
Roland Grunberg e635b2
 		try {
Roland Grunberg e635b2
-			ir.deleteDocuments(term);
Roland Grunberg e635b2
+			iw.deleteDocuments(term);
Roland Grunberg e635b2
 			indexedDocs.remove(name);
Roland Grunberg e635b2
 		} catch (IOException e) {
Roland Grunberg e635b2
 			return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR,
25f005
@@ -379,7 +382,7 @@ public class SearchIndex implements IHelpSearchIndex {
Roland Grunberg e635b2
 			 * know about this change. Close it so that it gets reloaded next search.
Roland Grunberg e635b2
 			 */
Roland Grunberg e635b2
 			if (searcher != null) {
Roland Grunberg e635b2
-				searcher.close();
Roland Grunberg e635b2
+				searcher.getIndexReader().close();
Roland Grunberg e635b2
 				searcher = null;
Roland Grunberg e635b2
 			}
Roland Grunberg e635b2
 			return true;
25f005
@@ -411,7 +414,7 @@ public class SearchIndex implements IHelpSearchIndex {
Roland Grunberg e635b2
 			 * know about this change. Close it so that it gets reloaded next search.
Roland Grunberg e635b2
 			 */
Roland Grunberg e635b2
 			if (searcher != null) {
Roland Grunberg e635b2
-				searcher.close();
Roland Grunberg e635b2
+				searcher.getIndexReader().close();
Roland Grunberg e635b2
 				searcher = null;
Roland Grunberg e635b2
 			}
Roland Grunberg e635b2
 			return true;
25f005
@@ -525,18 +528,19 @@ public class SearchIndex implements IHelpSearchIndex {
Roland Grunberg e635b2
 	}
Roland Grunberg e635b2
 
Roland Grunberg e635b2
 	public IStatus removeDuplicates(String name, String[] index_paths) {
Roland Grunberg e635b2
-		TermDocs hrefDocs = null;
Roland Grunberg e635b2
-		TermDocs indexDocs = null;
Roland Grunberg e635b2
-		Term hrefTerm = new Term(FIELD_NAME, name);
Roland Grunberg e635b2
 		try {
Roland Grunberg e635b2
+			AtomicReader ar = SlowCompositeReaderWrapper.wrap(ir);
Roland Grunberg e635b2
+			DocsEnum hrefDocs = null;
Roland Grunberg e635b2
+			DocsEnum indexDocs = null;
Roland Grunberg e635b2
+			Term hrefTerm = new Term(FIELD_NAME, name);
Roland Grunberg e635b2
 			for (int i = 0; i < index_paths.length; i++) {
Roland Grunberg e635b2
 				Term indexTerm = new Term(FIELD_INDEX_ID, index_paths[i]);
Roland Grunberg e635b2
 				if (i == 0) {
Roland Grunberg e635b2
-					hrefDocs = ir.termDocs(hrefTerm);
Roland Grunberg e635b2
-					indexDocs = ir.termDocs(indexTerm);
Roland Grunberg e635b2
+					hrefDocs = ar.termDocsEnum(hrefTerm);
Roland Grunberg e635b2
+					indexDocs = ar.termDocsEnum(indexTerm);
Roland Grunberg e635b2
 				} else {
Roland Grunberg e635b2
-					hrefDocs.seek(hrefTerm);
Roland Grunberg e635b2
-					indexDocs.seek(indexTerm);
Roland Grunberg e635b2
+					hrefDocs = ar.termDocsEnum(hrefTerm);
Roland Grunberg e635b2
+					indexDocs = ar.termDocsEnum(indexTerm);
Roland Grunberg e635b2
 				}
Roland Grunberg e635b2
 				removeDocuments(hrefDocs, indexDocs);
Roland Grunberg e635b2
 			}
25f005
@@ -545,19 +549,6 @@ public class SearchIndex implements IHelpSearchIndex {
Roland Grunberg e635b2
 					"IO exception occurred while removing duplicates of document " + name //$NON-NLS-1$
Roland Grunberg e635b2
 							+ " from index " + indexDir.getAbsolutePath() + ".", //$NON-NLS-1$ //$NON-NLS-2$
Roland Grunberg e635b2
 					ioe);
Roland Grunberg e635b2
-		} finally {
Roland Grunberg e635b2
-			if (hrefDocs != null) {
Roland Grunberg e635b2
-				try {
Roland Grunberg e635b2
-					hrefDocs.close();
Roland Grunberg e635b2
-				} catch (IOException e) {
Roland Grunberg e635b2
-				}
Roland Grunberg e635b2
-			}
Roland Grunberg e635b2
-			if (indexDocs != null) {
Roland Grunberg e635b2
-				try {
Roland Grunberg e635b2
-					indexDocs.close();
Roland Grunberg e635b2
-				} catch (IOException e) {
Roland Grunberg e635b2
-				}
Roland Grunberg e635b2
-			}
Roland Grunberg e635b2
 		}
Roland Grunberg e635b2
 		return Status.OK_STATUS;
Roland Grunberg e635b2
 	}
25f005
@@ -569,33 +560,33 @@ public class SearchIndex implements IHelpSearchIndex {
Roland Grunberg e635b2
 	 * @param docs2
Roland Grunberg e635b2
 	 * @throws IOException
Roland Grunberg e635b2
 	 */
Roland Grunberg e635b2
-	private void removeDocuments(TermDocs doc1, TermDocs docs2) throws IOException {
Roland Grunberg e635b2
-		if (!doc1.next()) {
Roland Grunberg e635b2
+	private void removeDocuments(DocsEnum doc1, DocsEnum docs2) throws IOException {
Roland Grunberg e635b2
+		if (doc1.nextDoc() == DocsEnum.NO_MORE_DOCS) {
Roland Grunberg e635b2
 			return;
Roland Grunberg e635b2
 		}
Roland Grunberg e635b2
-		if (!docs2.next()) {
Roland Grunberg e635b2
+		if (docs2.nextDoc() == DocsEnum.NO_MORE_DOCS) {
Roland Grunberg e635b2
 			return;
Roland Grunberg e635b2
 		}
Roland Grunberg e635b2
 		while (true) {
Roland Grunberg e635b2
-			if (doc1.doc() < docs2.doc()) {
Roland Grunberg e635b2
-				if (!doc1.skipTo(docs2.doc())) {
Roland Grunberg e635b2
-					if (!doc1.next()) {
Roland Grunberg e635b2
+			if (doc1.docID() < docs2.docID()) {
Roland Grunberg e635b2
+				if (doc1.advance(docs2.docID()) == DocsEnum.NO_MORE_DOCS) {
Roland Grunberg e635b2
+					if (doc1.nextDoc() == DocsEnum.NO_MORE_DOCS) {
Roland Grunberg e635b2
 						return;
Roland Grunberg e635b2
 					}
Roland Grunberg e635b2
 				}
Roland Grunberg e635b2
-			} else if (doc1.doc() > docs2.doc()) {
Roland Grunberg e635b2
-				if (!docs2.skipTo(doc1.doc())) {
Roland Grunberg e635b2
-					if (!doc1.next()) {
Roland Grunberg e635b2
+			} else if (doc1.docID() > docs2.docID()) {
Roland Grunberg e635b2
+				if (docs2.advance(doc1.docID()) == DocsEnum.NO_MORE_DOCS) {
Roland Grunberg e635b2
+					if (doc1.nextDoc() == DocsEnum.NO_MORE_DOCS) {
Roland Grunberg e635b2
 						return;
Roland Grunberg e635b2
 					}
Roland Grunberg e635b2
 				}
Roland Grunberg e635b2
 			}
Roland Grunberg e635b2
-			if (doc1.doc() == docs2.doc()) {
Roland Grunberg e635b2
-				ir.deleteDocument(doc1.doc());
Roland Grunberg e635b2
-				if (!doc1.next()) {
Roland Grunberg e635b2
+			if (doc1.docID() == docs2.docID()) {
Roland Grunberg e635b2
+				iw.tryDeleteDocument(ir, doc1.docID());
Roland Grunberg e635b2
+				if (doc1.nextDoc() == DocsEnum.NO_MORE_DOCS) {
Roland Grunberg e635b2
 					return;
Roland Grunberg e635b2
 				}
Roland Grunberg e635b2
-				if (!docs2.next()) {
Roland Grunberg e635b2
+				if (docs2.nextDoc() == DocsEnum.NO_MORE_DOCS) {
Roland Grunberg e635b2
 					return;
Roland Grunberg e635b2
 				}
Roland Grunberg e635b2
 			}
25f005
@@ -802,7 +793,7 @@ public class SearchIndex implements IHelpSearchIndex {
Roland Grunberg e635b2
 	public void openSearcher() throws IOException {
Roland Grunberg e635b2
 		synchronized (searcherCreateLock) {
Roland Grunberg e635b2
 			if (searcher == null) {
25f005
-				searcher = new IndexSearcher(IndexReader.open(luceneDirectory, false));
Roland Grunberg e635b2
+				searcher = new IndexSearcher(DirectoryReader.open(luceneDirectory));
Roland Grunberg e635b2
 			}
Roland Grunberg e635b2
 		}
Roland Grunberg e635b2
 	}
25f005
@@ -820,7 +811,7 @@ public class SearchIndex implements IHelpSearchIndex {
Mat Booth 39a3b2
 				if (searches.isEmpty()) {
Mat Booth 39a3b2
 					if (searcher != null) {
Mat Booth 39a3b2
 						try {
Mat Booth 39a3b2
-							searcher.close();
Mat Booth 39a3b2
+							searcher.getIndexReader().close();
Mat Booth 39a3b2
 						} catch (IOException ioe) {
Mat Booth 39a3b2
 						}
Mat Booth 39a3b2
 					}
25f005
@@ -903,7 +894,7 @@ public class SearchIndex implements IHelpSearchIndex {
Roland Grunberg e635b2
 		IndexWriter cleaner = null;
25f005
 		LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), 10000);
Roland Grunberg e635b2
 		try {
25f005
-			cleaner = new IndexWriter(luceneDirectory, new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_31, analyzer).setOpenMode(
25f005
+			cleaner = new IndexWriter(luceneDirectory, new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_47, analyzer).setOpenMode(
25f005
 			        OpenMode.CREATE));
Roland Grunberg e635b2
 		} catch (IOException ioe) {
Roland Grunberg e635b2
 		} finally {
Roland Grunberg e635b2
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java
25f005
index d0a7bb7..1b20d3b 100644
Roland Grunberg e635b2
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java
Roland Grunberg e635b2
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java
25f005
@@ -10,15 +10,13 @@
25f005
  *******************************************************************************/
25f005
 package org.eclipse.help.internal.search;
25f005
 
25f005
-import java.io.*;
25f005
-
25f005
 import org.apache.lucene.analysis.*;
25f005
 
25f005
 /**
Roland Grunberg e635b2
  * Smart Analyzer. Chooses underlying implementation based on the field which
Roland Grunberg e635b2
  * text is analyzed.
Roland Grunberg e635b2
  */
Roland Grunberg e635b2
-public final class SmartAnalyzer extends Analyzer {
Roland Grunberg e635b2
+public final class SmartAnalyzer extends AnalyzerWrapper {
Roland Grunberg e635b2
 	Analyzer pluggedInAnalyzer;
Roland Grunberg e635b2
 	Analyzer exactAnalyzer;
Roland Grunberg e635b2
 
25f005
@@ -31,14 +29,14 @@ public final class SmartAnalyzer extends Analyzer {
Roland Grunberg e635b2
 		this.exactAnalyzer = new DefaultAnalyzer(locale);
Roland Grunberg e635b2
 	}
Roland Grunberg e635b2
 	/**
Roland Grunberg e635b2
-	 * Creates a TokenStream which tokenizes all the text in the provided
Roland Grunberg e635b2
-	 * Reader. Delegates to DefaultAnalyzer when field used to search for exact
Roland Grunberg e635b2
+	 * Delegates to DefaultAnalyzer when field used to search for exact
Roland Grunberg e635b2
 	 * match, and to plugged-in analyzer for other fields.
Roland Grunberg e635b2
 	 */
Roland Grunberg e635b2
-	public final TokenStream tokenStream(String fieldName, Reader reader) {
Roland Grunberg e635b2
+	@Override
Roland Grunberg e635b2
+	public final Analyzer getWrappedAnalyzer(String fieldName) {
Roland Grunberg e635b2
 		if (fieldName != null && fieldName.startsWith("exact_")) { //$NON-NLS-1$
Roland Grunberg e635b2
-			return exactAnalyzer.tokenStream(fieldName, reader);
Roland Grunberg e635b2
+			return exactAnalyzer;
Roland Grunberg e635b2
 		}
Roland Grunberg e635b2
-		return pluggedInAnalyzer.tokenStream(fieldName, reader);
Roland Grunberg e635b2
+		return pluggedInAnalyzer;
Roland Grunberg e635b2
 	}
Roland Grunberg e635b2
 }
Roland Grunberg e635b2
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java
25f005
index 0b70cf7..106775a 100644
Roland Grunberg e635b2
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java
Roland Grunberg e635b2
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java
Roland Grunberg e635b2
@@ -35,6 +35,7 @@ public final class WordTokenStream extends Tokenizer {
Roland Grunberg e635b2
 	 * Constructor
Roland Grunberg e635b2
 	 */
Roland Grunberg e635b2
 	public WordTokenStream(String fieldName, Reader reader, Locale locale) {
Roland Grunberg e635b2
+		super(reader);
Roland Grunberg e635b2
 		this.reader = reader;
Roland Grunberg e635b2
 		boundary = BreakIterator.getWordInstance(locale);
Roland Grunberg e635b2
 
Roland Grunberg 2af405
@@ -105,6 +106,7 @@ public final class WordTokenStream extends Tokenizer {
Roland Grunberg 2af405
 	}
25f005
 
Roland Grunberg 2af405
 	public void close() throws IOException {
Roland Grunberg 2af405
+		super.close();
Roland Grunberg 2af405
 		/// Unlikely to be called as this is a reused
Roland Grunberg 2af405
 	    if (this.reader != null) {
Roland Grunberg 2af405
 	    	this.reader.close();
25f005
diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF b/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF
25f005
index a8177c3..cd99e8a 100644
25f005
--- a/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF
25f005
+++ b/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF
ba42d1
@@ -18,15 +18,14 @@
25f005
  org.eclipse.ui.intro.universal,
25f005
  org.eclipse.ui.forms,
25f005
  org.eclipse.ui.browser;bundle-version="3.2.300",
25f005
+ org.apache.lucene.analysis;bundle-version="[4.7.0,5.0.0)",
25f005
+ org.apache.lucene.core;bundle-version="[4.7.0,5.0.0)",
25f005
  org.eclipse.equinox.jsp.jasper;bundle-version="1.0.200",
25f005
  org.eclipse.equinox.jsp.jasper.registry;bundle-version="1.0.100"
25f005
 Bundle-ActivationPolicy: lazy
25f005
 Bundle-Vendor: Eclipse.org
03b08a
 Import-Package: javax.servlet;version="3.1.0",
03b08a
- javax.servlet.http;version="3.1.0",
25f005
- org.apache.lucene.index;core=split;version="[3.5.0,4.0.0)",
25f005
- org.apache.lucene.search;core=split;version="[3.5.0,4.0.0)",
25f005
- org.apache.lucene.store;core=split;version="[3.5.0,4.0.0)"
03b08a
+ javax.servlet.http;version="3.1.0"
25f005
 Bundle-RequiredExecutionEnvironment: JavaSE-1.6
25f005
 Export-Package: org.eclipse.ua.tests,
25f005
  org.eclipse.ua.tests.browser,
25f005
diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java b/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java
25f005
index 640d4c9..8924fa7 100644
25f005
--- a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java
25f005
+++ b/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java
25f005
@@ -24,7 +24,7 @@ import junit.framework.TestSuite;
365159
 import org.osgi.framework.Bundle;
365159
 
365159
 import org.apache.lucene.index.CorruptIndexException;
25f005
-import org.apache.lucene.index.IndexReader;
365159
+import org.apache.lucene.index.DirectoryReader;
365159
 import org.apache.lucene.search.IndexSearcher;
365159
 import org.apache.lucene.search.Query;
365159
 import org.apache.lucene.search.TopDocs;
25f005
@@ -157,7 +157,7 @@ public class PrebuiltIndexCompatibility extends TestCase {
365159
 			IndexSearcher searcher = null;
365159
 			try {
365159
 				luceneDirectory = new NIOFSDirectory(new File(filePath));
25f005
-				searcher = new IndexSearcher(IndexReader.open(luceneDirectory, true));
365159
+				searcher = new IndexSearcher(DirectoryReader.open(luceneDirectory));
365159
 				TopDocs hits = searcher.search(luceneQuery, 500);
365159
 				assertEquals(hits.totalHits, 1);
365159
 			} finally {
25f005
@@ -167,7 +167,7 @@ public class PrebuiltIndexCompatibility extends TestCase {
365159
 					} catch (IOException x) {
365159
 					}
365159
 				if (searcher != null)
365159
-					searcher.close();
365159
+					searcher.getIndexReader().close();
365159
 			}
365159
 		} else {
365159
 			fail("Cannot resolve to file protocol");
25f005
-- 
25f005
2.1.0
25f005