From e9b5adb1d0477a50111afe5cf7736c542b7e4998 Mon Sep 17 00:00:00 2001

From: Sopot Cela <scela@redhat.com>

Date: Thu, 21 May 2015 13:48:41 +0200

Subject: [PATCH] Bug 466829 - Upgrade platform.ua to Lucene 5.1.0

Change-Id: I882188205c2c1e2cc1106108680dd4e94570a975

Signed-off-by: Sopot Cela <scela@redhat.com>

---

diff --git a/eclipse.platform.ua/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs

index 34d2b77..07713d8 100644

--- a/eclipse.platform.ua/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs

+++ b/eclipse.platform.ua/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs

@@ -7,9 +7,9 @@

 org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled

 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled

 org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate

-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5

+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7

 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve

-org.eclipse.jdt.core.compiler.compliance=1.5

+org.eclipse.jdt.core.compiler.compliance=1.7

 org.eclipse.jdt.core.compiler.debug.lineNumber=generate

 org.eclipse.jdt.core.compiler.debug.localVariable=generate

 org.eclipse.jdt.core.compiler.debug.sourceFile=generate

@@ -98,7 +98,7 @@

 org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore

 org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning

 org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning

-org.eclipse.jdt.core.compiler.source=1.5

+org.eclipse.jdt.core.compiler.source=1.7

 org.eclipse.jdt.core.formatter.align_type_members_on_columns=false

 org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16

 org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16

diff --git a/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF b/org.eclipse.help.base/META-INF/MANIFEST.MF

index ee34c8e..e4bd703 100644

--- a/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF

+++ b/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF

@@ -43,9 +43,11 @@

  org.eclipse.core.runtime;bundle-version="[3.11.0,4.0.0)",

  org.eclipse.help;bundle-version="[3.5.0,4.0.0)";visibility:=reexport,

  org.eclipse.core.expressions;bundle-version="[3.4.200,4.0.0)",

- org.apache.lucene.analysis;bundle-version="[3.5.0,4.0.0)",

- org.apache.lucene.core;bundle-version="[3.5.0,4.0.0)",

- org.eclipse.core.net;bundle-version="1.2.200"

+ org.eclipse.core.net;bundle-version="1.2.200",

+ org.apache.lucene.analyzers-common;bundle-version="5.1.0",

+ org.apache.lucene.core;bundle-version="5.1.0",

+ org.apache.lucene.queryparser;bundle-version="5.1.0",

+ org.apache.lucene.analyzers-smartcn;bundle-version="5.1.0"

 Import-Package: com.ibm.icu.text,

  org.eclipse.equinox.http.jetty;resolution:=optional

 Bundle-RequiredExecutionEnvironment: J2SE-1.5

diff --git a/eclipse.platform.ua/org.eclipse.help.base/plugin.xml b/org.eclipse.help.base/plugin.xml

index 07a5a22..4daf3f0 100644

--- a/eclipse.platform.ua/org.eclipse.help.base/plugin.xml

+++ b/eclipse.platform.ua/org.eclipse.help.base/plugin.xml

@@ -83,7 +83,7 @@

       </analyzer>

             locale="zh"

-            class="org.apache.lucene.analysis.cn.ChineseAnalyzer">

+            class="org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer">

       </analyzer>

             locale="cs"

diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/AnalyzerFactory.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/AnalyzerFactory.java

index 4ea3b21..704b0e9 100644

--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/AnalyzerFactory.java

+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/AnalyzerFactory.java

@@ -1,5 +1,5 @@

 /*******************************************************************************

- * Copyright (c) 2012 IBM Corporation and others.

+ * Copyright (c) 2012, 2015 IBM Corporation and others.

  * All rights reserved. This program and the accompanying materials

  * are made available under the terms of the Eclipse Public License v1.0

  * which accompanies this distribution, and is available at

@@ -7,6 +7,7 @@

  *

  * Contributors:

  *     IBM Corporation - initial API and implementation

+ *     Sopot Cela - Bug 466829 - Migration to Lucene 5

  *******************************************************************************/

 package org.eclipse.help.internal.search;

@@ -19,8 +20,9 @@

 import org.apache.lucene.analysis.fr.FrenchAnalyzer;

 import org.apache.lucene.analysis.nl.DutchAnalyzer;

 import org.apache.lucene.analysis.ru.RussianAnalyzer;

-import org.apache.lucene.util.Version;

-import org.eclipse.core.runtime.*;

+import org.eclipse.core.runtime.CoreException;

+import org.eclipse.core.runtime.IConfigurationElement;

+import org.eclipse.core.runtime.IExecutableExtension;

 /**

  * A factory responsible for instantiating a lucene {@link Analyzer}.

@@ -30,27 +32,26 @@

 	public Analyzer create() {

 		if (locale == null)

 			return null;

-		Version version = Version.LUCENE_35;

 		if ("pt".equals(locale)) //$NON-NLS-1$

-			return new BrazilianAnalyzer(version);

+			return new BrazilianAnalyzer();

 		if ("ja".equals(locale)) //$NON-NLS-1$

-			return new CJKAnalyzer(version);

+			return new CJKAnalyzer();

 		if ("ko".equals(locale)) //$NON-NLS-1$

-			return new CJKAnalyzer(version);

+			return new CJKAnalyzer();

 		if ("pt".equals(locale)) //$NON-NLS-1$

-			return new BrazilianAnalyzer(version);

+			return new BrazilianAnalyzer();

 		if ("cs".equals(locale)) //$NON-NLS-1$

-			return new CzechAnalyzer(version);

+			return new CzechAnalyzer();

 		if ("de".equals(locale)) //$NON-NLS-1$

-			return new GermanAnalyzer(version);

+			return new GermanAnalyzer();

 		if ("el".equals(locale)) //$NON-NLS-1$

-			return new GreekAnalyzer(version);

+			return new GreekAnalyzer();

 		if ("fr".equals(locale)) //$NON-NLS-1$

-			return new FrenchAnalyzer(version);

+			return new FrenchAnalyzer();

 		if ("nl".equals(locale)) //$NON-NLS-1$

-			return new DutchAnalyzer(version);

+			return new DutchAnalyzer();

 		if ("ru".equals(locale)) //$NON-NLS-1$

-			return new RussianAnalyzer(version);

+			return new RussianAnalyzer();

 		//unknown language

 		return null;

diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java

index a066aa4..a2183f9 100644

--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java

+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java

@@ -8,18 +8,25 @@

  * Contributors:

  *     IBM Corporation - initial API and implementation

  *     Alexander Kurtakov - Bug 460787

+ *     Sopot Cela - Bug 466829 - Migration to Lucene 5

  *******************************************************************************/

 package org.eclipse.help.internal.search;

-import java.io.*;

-import java.util.HashSet;

-import java.util.Set;

+import java.util.ArrayList;

+import java.util.List;

-import org.apache.lucene.analysis.*;

-import org.apache.lucene.util.Version;

+import org.apache.lucene.analysis.Analyzer;

+import org.apache.lucene.analysis.TokenStream;

+import org.apache.lucene.analysis.Tokenizer;

+import org.apache.lucene.analysis.core.StopFilter;

+import org.apache.lucene.analysis.en.EnglishPossessiveFilter;

+import org.apache.lucene.analysis.en.PorterStemFilter;

+import org.apache.lucene.analysis.miscellaneous.KeywordRepeatFilter;

+import org.apache.lucene.analysis.util.CharArraySet;

 /**

  * Lucene Analyzer for English. LowerCaseTokenizer->StopFilter->PorterStemFilter

  */

 public final class Analyzer_en extends Analyzer {

+

 	/**

 	 * Constructor for Analyzer_en.

 	 */

@@ -27,18 +34,25 @@

 		super();

 	}

 	/**

-	 * Creates a TokenStream which tokenizes all the text in the provided

+	 * Creates a TokenStreamComponents which tokenizes all the text in the provided

 	 * Reader.

 	 */

-	public final TokenStream tokenStream(String fieldName, Reader reader) {

-		return new PorterStemFilter(new StopFilter(Version.LUCENE_30, new LowerCaseAndDigitsTokenizer(reader), getStopWords(), false));

+	@Override

+	public final TokenStreamComponents createComponents(String fieldName) {

+			final Tokenizer source;

+	      	source = new LowerCaseAndDigitsTokenizer();

+	      	TokenStream result = new EnglishPossessiveFilter(source);

+		    result = new StopFilter(result, new CharArraySet(getStopWords(), false));

+		    result = new KeywordRepeatFilter(result);

+		    result = new PorterStemFilter(result);

+		    return new TokenStreamComponents(source, result);

 	}

-	private Set<String> stopWords;

+	private List<String> stopWords;

-	private Set<String> getStopWords() {

+	private List<String> getStopWords() {

 		if ( stopWords == null ) {

-			stopWords = new HashSet<String>();

+			stopWords = new ArrayList<String>();

 			for (int i = 0; i < STOP_WORDS.length; i++) {

 			    stopWords.add(STOP_WORDS[i]);

 			}

diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java

index 4109474..e3c8722 100644

--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java

+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java

@@ -7,22 +7,20 @@

  * Contributors:

  *     IBM Corporation - initial API and implementation

  *     Alexander Kurtakov - Bug 460787

+ *     Sopot Cela - Bug 466829 - Migration to Lucene 5

  *******************************************************************************/

 package org.eclipse.help.internal.search;

-import java.io.Reader;

 import java.util.Locale;

 import java.util.StringTokenizer;

-import com.ibm.icu.text.BreakIterator;

-

 import org.apache.lucene.analysis.Analyzer;

-import org.apache.lucene.analysis.LowerCaseFilter;

-import org.apache.lucene.analysis.TokenStream;

-import org.apache.lucene.util.Version;

+import org.apache.lucene.analysis.Tokenizer;

+import org.apache.lucene.analysis.core.LowerCaseFilter;

+import org.eclipse.core.runtime.Platform;

 import org.eclipse.help.internal.base.HelpBasePlugin;

-import org.eclipse.core.runtime.Platform;

+import com.ibm.icu.text.BreakIterator;

 /**

@@ -84,11 +82,14 @@

 	}

 	/**

-	 * Creates a TokenStream which tokenizes all the text in the provided

+	 * Creates a TokenStreamComponents which tokenizes all the text in the provided

 	 * Reader.

 	 */

-	public final TokenStream tokenStream(String fieldName, Reader reader) {

-		return new LowerCaseFilter(Version.LUCENE_30, new WordTokenStream(fieldName, reader, locale));

+	@Override

+	public final TokenStreamComponents createComponents(String fieldName) {

+		Tokenizer source = new WordTokenStream(locale);

+		LowerCaseFilter filter = new LowerCaseFilter(source);

+		return new TokenStreamComponents(source, filter);

 	}

 	/**

diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java

index a475688..cbb2472 100644

--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java

+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java

@@ -8,27 +8,26 @@

  * Contributors:

  *     IBM Corporation - initial API and implementation

  *     Alexander Kurtakov - Bug 460787

+ *     Sopot Cela - Bug 466829 - Migration to Lucene 5

  *******************************************************************************/

 package org.eclipse.help.internal.search;

-import java.io.*;

-

-import org.apache.lucene.analysis.*;

-import org.apache.lucene.util.Version;

+import org.apache.lucene.analysis.util.CharTokenizer;

 /**

  * Tokenizer breaking words around letters or digits.

  */

 public class LowerCaseAndDigitsTokenizer extends CharTokenizer {

-    public LowerCaseAndDigitsTokenizer(Reader input) {

-        super(Version.LUCENE_30, input);

+    public LowerCaseAndDigitsTokenizer() {

+        super();

     }

     protected char normalize(char c) {

         return Character.toLowerCase(c);

     }

-    protected boolean isTokenChar(char c) {

+    @Override

+    public boolean isTokenChar(int c) {

         return Character.isLetterOrDigit(c);

     }

diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java

index 60a545d..804bf5e 100644

--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java

+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java

@@ -1,5 +1,5 @@

 /*******************************************************************************

- * Copyright (c) 2010, 2011 IBM Corporation and others.

+ * Copyright (c) 2010, 2015 IBM Corporation and others.

  * All rights reserved. This program and the accompanying materials

  * are made available under the terms of the Eclipse Public License v1.0

  * which accompanies this distribution, and is available at

@@ -7,6 +7,7 @@

  *

  * Contributors:

  *     IBM Corporation - initial API and implementation

+ *     Sopot Cela - Bug 466829 - Migration to Lucene 5

  *******************************************************************************/

 package org.eclipse.help.internal.search;

@@ -16,6 +17,8 @@

 import org.apache.lucene.document.Document;

 import org.apache.lucene.document.Field;

+import org.apache.lucene.document.StoredField;

+import org.apache.lucene.document.TextField;

 import org.eclipse.help.search.ISearchDocument;

 /**

@@ -31,22 +34,22 @@

 	}

 	public void setTitle(String title) {

-		doc.add(new Field("title", title, Field.Store.NO, Field.Index.ANALYZED)); //$NON-NLS-1$

-		doc.add(new Field("exact_title", title, Field.Store.NO, Field.Index.ANALYZED)); //$NON-NLS-1$

-		doc.add(new Field("raw_title", title, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$

+		doc.add(new TextField("title", title, Field.Store.NO)); //$NON-NLS-1$

+		doc.add(new TextField("exact_title", title, Field.Store.NO)); //$NON-NLS-1$

+		doc.add(new StoredField("raw_title", title)); //$NON-NLS-1$

 	}

 	public void setSummary(String summary) {

-	  	doc.add(new Field("summary", summary, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$

+	  	doc.add(new StoredField("summary", summary)); //$NON-NLS-1$

 	}

 	public void addContents(String contents) {

-		doc.add(new Field("contents", new StringReader(contents))); //$NON-NLS-1$

-		doc.add(new Field("exact_contents", new StringReader(contents))); //$NON-NLS-1$

+		doc.add(new TextField("contents", new StringReader(contents))); //$NON-NLS-1$

+		doc.add(new TextField("exact_contents", new StringReader(contents))); //$NON-NLS-1$

 	}

 	public void setHasFilters(boolean hasFilters) {

-		doc.add(new Field("filters", Boolean.toString(hasFilters), Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$

+		doc.add(new StoredField("filters", Boolean.toString(hasFilters))); //$NON-NLS-1$

 	}

 	public Document getDocument() {

@@ -54,8 +57,8 @@

 	}

 	public void addContents(Reader contents, Reader exactContents) {

-		doc.add(new Field("contents", contents)); //$NON-NLS-1$

-		doc.add(new Field("exact_contents", exactContents)); //$NON-NLS-1$

+		doc.add(new TextField("contents", contents)); //$NON-NLS-1$

+		doc.add(new TextField("exact_contents", exactContents)); //$NON-NLS-1$

 	}

 }

diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java

index ca9cd67..7952c8d 100644

--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java

+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java

@@ -1,5 +1,5 @@

 /*******************************************************************************

- * Copyright (c) 2005, 2014 IBM Corporation and others.

+ * Copyright (c) 2005, 2015 IBM Corporation and others.

  * All rights reserved. This program and the accompanying materials

  * are made available under the terms of the Eclipse Public License v1.0

  * which accompanies this distribution, and is available at

@@ -7,6 +7,7 @@

  *

  * Contributors:

  *     IBM Corporation - initial API and implementation

+ *     Sopot Cela - Bug 466829 - Migration to Lucene 5

  *******************************************************************************/

 package org.eclipse.help.internal.search;

@@ -152,6 +153,7 @@

 					.getProperty(SearchIndex.DEPENDENCIES_KEY_ANALYZER);

 			if (!targetIndex.isLuceneCompatible(lucene)

 					|| !targetIndex.isAnalyzerCompatible(analyzer)) {

+				HelpBasePlugin.logError("Error trying to consume Lucene index from bundle "+bundle.toString()+". Please use an index built with Lucene 5 or higher.", null); //$NON-NLS-1$ //$NON-NLS-2$

 				return false;

 			}

 		} catch (MalformedURLException mue) {

diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java

deleted file mode 100644

index 08cf58a..0000000

--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java

+++ /dev/null

@@ -1,459 +0,0 @@

-/*******************************************************************************

- * Copyright (c) 2000, 2012 IBM Corporation and others.

- * All rights reserved. This program and the accompanying materials

- * are made available under the terms of the Eclipse Public License v1.0

- * which accompanies this distribution, and is available at

- * http://www.eclipse.org/legal/epl-v10.html

- *

- * Contributors:

- *     IBM Corporation - initial API and implementation

- *     Chris Torrence  - patch for bug Bug 107648

- *******************************************************************************/

-package org.eclipse.help.internal.search;

-import java.io.*;

-import java.util.ArrayList;

-import java.util.Collection;

-import java.util.Iterator;

-import java.util.List;

-import java.util.Locale;

-import java.util.StringTokenizer;

-

-import org.apache.lucene.analysis.*;

-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;

-import org.apache.lucene.index.*;

-import org.apache.lucene.search.*;

-import org.eclipse.help.internal.base.*;

-/**

- * Build query acceptable by the search engine.

- */

-public class QueryBuilder {

-	// Maximum allowed number of terms

-	private static final int MAX_TERMS = 10;

-	// Maximum allowed number of ORs

-	private static final int MAX_UNIONS = 4;

-	// Maximum allowed number terms with wild cards

-	private static final int MAX_WILD_TERMS = 2;

-	// Query from user

-	private String searchWords;

-	// Descriptor of Analyzer to process the query words

-	private AnalyzerDescriptor analyzerDesc;

-	// Analyzer to process the query words

-	private Analyzer analyzer;

-	// List of QueryWordsToken

-	private List<QueryWordsToken> analyzedTokens;

-	// List of words to highlight

-	private List<String> highlightWords = new ArrayList<String>();

-	private Locale locale;

-	/**

-	 * Creates a query builder for the search word. The search word is processed

-	 * by a lexical analyzer.

-	 */

-	public QueryBuilder(String searchWords, AnalyzerDescriptor analyzerDesc) {

-		this.searchWords = searchWords;

-		String language = analyzerDesc.getLang();

-		if (language.length() >= 5) {

-			this.locale = new Locale(language.substring(0, 2), language

-					.substring(3, 5));

-		} else {

-			this.locale = new Locale(language.substring(0, 2), ""); //$NON-NLS-1$

-		}

-		this.analyzerDesc = analyzerDesc;

-		this.analyzer = analyzerDesc.getAnalyzer();

-	}

-	/**

-	 * Splits user query into tokens and returns a list of QueryWordsToken's.

-	 */

-	private List<QueryWordsToken> tokenizeUserQuery(String searchWords) {

-	    List<QueryWordsToken> tokenList = new ArrayList<QueryWordsToken>();

-		//Divide along quotation marks

-		//StringTokenizer qTokenizer = new StringTokenizer(searchWords.trim(),

-		//		"\"", true); //$NON-NLS-1$

-		boolean withinQuotation = false;

-		String quotedString = ""; //$NON-NLS-1$

-		int termCount = 0;// keep track of number of terms to disallow too many

-

-		int fromIndex = -1;

-		searchWords = searchWords.trim();

-		while((fromIndex = searchWords.indexOf("\"", fromIndex+1))!= -1){ //$NON-NLS-1$

-			withinQuotation = !withinQuotation;

-		}

-		if( withinQuotation ) {

-			searchWords = searchWords + "\""; //$NON-NLS-1$

-			withinQuotation = !withinQuotation;

-		}

-

-		StringTokenizer qTokenizer = new StringTokenizer(searchWords,"\"",true); //$NON-NLS-1$

-		int orCount = 0; // keep track of number of ORs to disallow too many

-		while (qTokenizer.hasMoreTokens()) {

-			String curToken = qTokenizer.nextToken();

-			if (curToken.equals("\"")) { //$NON-NLS-1$

-				if (withinQuotation) {

-					// check for too many terms

-					if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER

-							&& ++termCount > MAX_TERMS) {

-						throw new QueryTooComplexException();

-					}

-					tokenList.add(QueryWordsToken.exactPhrase(quotedString));

-				} else {

-					quotedString = ""; //$NON-NLS-1$

-				}

-				withinQuotation = !withinQuotation;

-				continue;

-			} else if (withinQuotation) {

-				quotedString = curToken;

-				continue;

-			} else {

-				//divide unquoted strings along white space

-				StringTokenizer parser = new StringTokenizer(curToken.trim());

-				while (parser.hasMoreTokens()) {

-					String token = parser.nextToken();

-					if (token.equalsIgnoreCase(QueryWordsToken.AND().value)) {

-						tokenList.add(QueryWordsToken.AND());

-					} else if (token

-							.equalsIgnoreCase(QueryWordsToken.OR().value)) {

-						// Check for too many OR terms

-						if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER

-								&& ++orCount > MAX_UNIONS) {

-							throw new QueryTooComplexException();

-						}

-						tokenList.add(QueryWordsToken.OR());

-					} else if (token

-							.equalsIgnoreCase(QueryWordsToken.NOT().value)) {

-						tokenList.add(QueryWordsToken.NOT());

-					} else {

-						// check for too many terms

-						if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER

-								&& ++termCount > MAX_TERMS) {

-							throw new QueryTooComplexException();

-						}

-						tokenList.add(QueryWordsToken.word(token));

-					}

-				}

-			}

-		}

-		return tokenList;

-	}

-	/**

-	 * Apply the Analyzer to the search tokens and return the list of processed

-	 * QueryWordsToken's.

-	 */

-	private List<QueryWordsToken> analyzeTokens(List<QueryWordsToken> tokens) {

-		boolean isTokenAfterNot = false;

-		List<QueryWordsToken> newTokens = new ArrayList<QueryWordsToken>();

-		int wildCardTermCount = 0;

-		for (int i = 0; i < tokens.size(); i++) {

-			QueryWordsToken token = tokens.get(i);

-			if (token.type == QueryWordsToken.WORD) {

-				int questionMIndex = token.value.indexOf('?');

-				int starIndex = token.value.indexOf('*');

-				if (starIndex >= 0 || questionMIndex >= 0) {

-					if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER

-							&& ++wildCardTermCount > MAX_WILD_TERMS) {

-						throw new QueryTooComplexException();

-					}

-					newTokens.add(QueryWordsToken.word(token.value

-							.toLowerCase(locale)));

-					// add word to the list of words to highlight

-					if (!isTokenAfterNot && !highlightWords.contains(token.value)) {

-						highlightWords.add(token.value);

-					}

-				} else {

-					List<String> wordList = analyzeText(analyzer, "contents", //$NON-NLS-1$

-							token.value);

-					if (wordList.size() > 0) {

-						if (!isTokenAfterNot && !highlightWords.contains(token.value)) {

-							// add original word to the list of words to

-							// highlight

-							highlightWords.add(token.value);

-						}

-						if (wordList.size() == 1) {

-							String word = wordList.get(0);

-							newTokens.add(QueryWordsToken.word(word));

-							// add analyzed word to the list of words to

-							// highlight

-							// this is required to highlight stemmed words

-							if (!isTokenAfterNot && !highlightWords.contains(word)) {

-								highlightWords.add(word);

-							}

-						} else {

-							QueryWordsPhrase phrase = QueryWordsToken.phrase();

-							for (Iterator<String> it = wordList.iterator(); it

-									.hasNext();) {

-								String word = it.next();

-								phrase.addWord(word);

-								// add each analyzed word to the list of words

-								// to highlight

-								// this is only required to highlight stemmed

-								// words.

-								// Adding words should not be done when

-								// DefaultAnalyzer is used,

-								// because it does not perform stemming and

-								// common words removal

-								// which would result in common characters

-								// highlighted all over (bug 30263)

-								if (!analyzerDesc.getId().startsWith(

-										HelpBasePlugin.PLUGIN_ID + "#")) { //$NON-NLS-1$

-									if (!isTokenAfterNot && !highlightWords.contains(word)) {

-										highlightWords.add(word);

-									}

-								}

-							}

-							newTokens.add(phrase);

-						}

-					}

-				}

-			} else if (// forget ANDs

-			/*

-			 * token.type == SearchQueryToken.AND ||

-			 */

-			token.type == QueryWordsToken.OR

-					|| token.type == QueryWordsToken.NOT)

-				newTokens.add(token);

-			else if (token.type == QueryWordsToken.EXACT_PHRASE) {

-				List<String> wordList = analyzeText(analyzer, "exact_contents", //$NON-NLS-1$

-						token.value);

-				if (wordList.size() > 0) {

-					if (!isTokenAfterNot && !highlightWords.contains(token.value)) {

-						// add original word to the list of words to highlight

-						highlightWords.add(token.value);

-					}

-				}

-				QueryWordsExactPhrase phrase = QueryWordsToken.exactPhrase();

-				for (Iterator<String> it = wordList.iterator(); it.hasNext();) {

-					String word = it.next();

-					phrase.addWord(word);

-					// add analyzed word to the list of words to highlight

-					// if (!highlightWords.contains(word))

-					//	highlightWords.add(word);

-				}

-				// add phrase only if not empty

-				if (phrase.getWords().size() > 0) {

-					newTokens.add(phrase);

-				}

-			}

-			isTokenAfterNot = (token.type == QueryWordsToken.NOT);

-		}

-		return newTokens;

-	}

-	/**

-	 * Get a list of tokens corresponding to a search word or phrase

-	 *

-	 * @return List of String

-	 */

-	private List<String> analyzeText(Analyzer analyzer, String fieldName, String text) {

-		List<String> words = new ArrayList<String>(1);

-		Reader reader = new StringReader(text);

-		TokenStream tStream = analyzer.tokenStream(fieldName, reader);

-

-		CharTermAttribute termAttribute = tStream.getAttribute(CharTermAttribute.class);

-		try {

-			while (tStream.incrementToken()) {

-				String term = termAttribute.toString();

-				words.add(term);

-			}

-			reader.close();

-		} catch (IOException ioe) {

-		}

-

-		return words;

-	}

-	/**

-	 * Obtains Lucene Query from tokens

-	 *

-	 * @return Query or null if no query could be created

-	 */

-	private Query createLuceneQuery(List<QueryWordsToken> searchTokens, String[] fieldNames,

-			float[] boosts) {

-		// Get queries for parts separated by OR

-		List<Query> requiredQueries = getRequiredQueries(searchTokens, fieldNames,

-				boosts);

-		if (requiredQueries.size() == 0)

-			return null;

-		else if (requiredQueries.size() <= 1)

-			return requiredQueries.get(0);

-		else

-			/* if (requiredQueries.size() > 1) */

-			// OR queries

-			return (orQueries(requiredQueries));

-	}

-	/**

-	 * Obtains Lucene queries for token sequences separated at OR.

-	 *

-	 * @return List of Query (could be empty)

-	 */

-	private List<Query> getRequiredQueries(List<QueryWordsToken> tokens, String[] fieldNames,

-			float[] boosts) {

-		List<Query> oredQueries = new ArrayList<Query>();

-		ArrayList<QueryWordsToken> requiredQueryTokens = new ArrayList<QueryWordsToken>();

-		for (int i = 0; i < tokens.size(); i++) {

-			QueryWordsToken token = tokens.get(i);

-			if (token.type != QueryWordsToken.OR) {

-				requiredQueryTokens.add(token);

-			} else {

-				Query reqQuery = getRequiredQuery(requiredQueryTokens,

-						fieldNames, boosts);

-				if (reqQuery != null)

-					oredQueries.add(reqQuery);

-				requiredQueryTokens = new ArrayList<QueryWordsToken>();

-			}

-		}

-		Query reqQuery = getRequiredQuery(requiredQueryTokens, fieldNames,

-				boosts);

-		if (reqQuery != null)

-			oredQueries.add(reqQuery);

-		return oredQueries;

-	}

-	private Query orQueries(Collection<Query> queries) {

-		BooleanQuery bq = new BooleanQuery();

-		for (Iterator<Query> it = queries.iterator(); it.hasNext();) {

-			Query q = it.next();

-			bq.add(q, BooleanClause.Occur.SHOULD);

-		}

-		return bq;

-	}

-	/**

-	 * Obtains Lucene Query for tokens containing only AND and NOT operators.

-	 *

-	 * @return BooleanQuery or null if no query could be created from the tokens

-	 */

-	private Query getRequiredQuery(List<QueryWordsToken> requiredTokens, String[] fieldNames,

-			float[] boosts) {

-		BooleanQuery retQuery = new BooleanQuery();

-		boolean requiredTermExist = false;

-		// Parse tokens left to right

-		QueryWordsToken operator = null;

-		for (int i = 0; i < requiredTokens.size(); i++) {

-			QueryWordsToken token = requiredTokens.get(i);

-			if (token.type == QueryWordsToken.AND

-					|| token.type == QueryWordsToken.NOT) {

-				operator = token;

-				continue;

-			}

-			// Creates queries for all fields

-			Query qs[] = new Query[fieldNames.length];

-			for (int f = 0; f < fieldNames.length; f++) {

-				qs[f] = token.createLuceneQuery(fieldNames[f], boosts[f]);

-			}

-			// creates the boolean query of all fields

-			Query q = qs[0];

-			if (fieldNames.length > 1) {

-				BooleanQuery allFieldsQuery = new BooleanQuery();

-				for (int f = 0; f < fieldNames.length; f++)

-					allFieldsQuery.add(qs[f], BooleanClause.Occur.SHOULD);

-				q = allFieldsQuery;

-			}

-			if (operator != null && operator.type == QueryWordsToken.NOT) {

-				retQuery.add(q, BooleanClause.Occur.MUST_NOT); // add as prohibited

-			} else {

-				retQuery.add(q, BooleanClause.Occur.MUST); // add as required

-				requiredTermExist = true;

-			}

-		}

-		if (!requiredTermExist) {

-			return null; // cannot search for prohibited only

-		}

-		return retQuery;

-	}

-	private Query getLuceneQuery(String[] fieldNames, float[] boosts) {

-		Query luceneQuery = createLuceneQuery(analyzedTokens, fieldNames,

-				boosts);

-		return luceneQuery;

-	}

-	/**

-	 * @param fieldNames -

-	 *            Collection of field names of type String (e.g. "h1"); the

-	 *            search will be performed on the given fields

-	 * @param fieldSearchOnly -

-	 *            boolean indicating if field only search should be performed;

-	 *            if set to false, default field "contents" and all other fields

-	 *            will be searched

-	 */

-	public Query getLuceneQuery(Collection<String> fieldNames, boolean fieldSearchOnly)

-			throws QueryTooComplexException {

-		// split search query into tokens

-		List<QueryWordsToken> userTokens = tokenizeUserQuery(searchWords);

-		analyzedTokens = analyzeTokens(userTokens);

-		return buildLuceneQuery(fieldNames, fieldSearchOnly);

-	}

-	/**

-	 * @param fieldNames -

-	 *            Collection of field names of type String (e.g. "h1"); the

-	 *            search will be performed on the given fields

-	 * @param fieldSearchOnly -

-	 *            boolean indicating if field only search should be performed;

-	 *            if set to false, default field "contents" and all other fields

-	 *            will be searched

-	 */

-	private Query buildLuceneQuery(Collection<String> fieldNames,

-			boolean fieldSearchOnly) {

-		String[] fields;

-		float[] boosts;

-		if (fieldSearchOnly) {

-			fields = new String[fieldNames.size()];

-			boosts = new float[fieldNames.size()];

-			Iterator<String> fieldNamesIt = fieldNames.iterator();

-			for (int i = 0; i < fieldNames.size(); i++) {

-				fields[i] = fieldNamesIt.next();

-				boosts[i] = 5.0f;

-			}

-		} else {

-			fields = new String[fieldNames.size() + 2];

-	        boosts = new float[fieldNames.size() + 2];

-			Iterator<String> fieldNamesIt = fieldNames.iterator();

-			for (int i = 0; i < fieldNames.size(); i++) {

-				fields[i] = fieldNamesIt.next();

-				boosts[i] = 5.0f;

-			}

-			fields[fieldNames.size()] = "contents"; //$NON-NLS-1$

-			boosts[fieldNames.size()] = 1.0f;

-            fields[fieldNames.size()+1] = "title"; //$NON-NLS-1$

-	        boosts[fieldNames.size()+1] = 1.0f;

-		}

-		Query query = getLuceneQuery(fields, boosts);

-		query = improveRankingForUnqotedPhrase(query, fields, boosts);

-		return query;

-	}

-	/**

-	 * If user query contained only words (no quotaions nor operators) extends

-	 * query with term phrase representing entire user query i.e for user string

-	 * a b, the query a AND b will be extended to "a b" OR a AND b

-	 */

-	private Query improveRankingForUnqotedPhrase(Query query, String[] fields,

-			float[] boosts) {

-		if (query == null)

-			return query;

-		// check if all tokens are words

-		for (int i = 0; i < analyzedTokens.size(); i++)

-			if (analyzedTokens.get(i).type != QueryWordsToken.WORD)

-				return query;

-		// Create phrase query for all tokens and OR with original query

-		BooleanQuery booleanQuery = new BooleanQuery();

-		booleanQuery.add(query, BooleanClause.Occur.SHOULD);

-		PhraseQuery[] phraseQueries = new PhraseQuery[fields.length];

-		for (int f = 0; f < fields.length; f++) {

-			phraseQueries[f] = new PhraseQuery();

-			for (int i = 0; i < analyzedTokens.size(); i++) {

-				Term t = new Term(fields[f], analyzedTokens

-						.get(i).value);

-				phraseQueries[f].add(t);

-			}

-			phraseQueries[f].setBoost(10 * boosts[f]);

-			booleanQuery.add(phraseQueries[f], BooleanClause.Occur.SHOULD);

-		}

-		return booleanQuery;

-	}

-	/**

-	 * Obtains analyzed terms from query as one string. Words are double quoted,

-	 * and separated by space. The analyzed words are needed for highlighting

-	 * word roots.

-	 */

-	public String gethighlightTerms() {

-		StringBuffer buf = new StringBuffer();

-		for (Iterator<String> it = highlightWords.iterator(); it.hasNext();) {

-			buf.append('"');

-			buf.append(it.next());

-			buf.append("\" "); //$NON-NLS-1$

-		}

-		return buf.toString();

-	}

-}

diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsExactPhrase.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsExactPhrase.java

deleted file mode 100644

index 324b8e1..0000000

--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsExactPhrase.java

+++ /dev/null

@@ -1,50 +0,0 @@

-/*******************************************************************************

- * Copyright (c) 2000, 2011 IBM Corporation and others.

- * All rights reserved. This program and the accompanying materials

- * are made available under the terms of the Eclipse Public License v1.0

- * which accompanies this distribution, and is available at

- * http://www.eclipse.org/legal/epl-v10.html

- *

- * Contributors:

- *     IBM Corporation - initial API and implementation

- *******************************************************************************/

-package org.eclipse.help.internal.search;