diff --git a/e9b5adb1.diff b/e9b5adb1.diff deleted file mode 100644 index 38cf03c..0000000 --- a/e9b5adb1.diff +++ /dev/null @@ -1,2252 +0,0 @@ -From e9b5adb1d0477a50111afe5cf7736c542b7e4998 Mon Sep 17 00:00:00 2001 -From: Sopot Cela -Date: Thu, 21 May 2015 13:48:41 +0200 -Subject: [PATCH] Bug 466829 - Upgrade platform.ua to Lucene 5.1.0 - -Change-Id: I882188205c2c1e2cc1106108680dd4e94570a975 -Signed-off-by: Sopot Cela ---- - -diff --git a/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs -index 34d2b77..07713d8 100644 ---- a/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs -+++ b/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs -@@ -7,9 +7,9 @@ - org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled - org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled - org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate --org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5 -+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 - org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve --org.eclipse.jdt.core.compiler.compliance=1.5 -+org.eclipse.jdt.core.compiler.compliance=1.7 - org.eclipse.jdt.core.compiler.debug.lineNumber=generate - org.eclipse.jdt.core.compiler.debug.localVariable=generate - org.eclipse.jdt.core.compiler.debug.sourceFile=generate -@@ -98,7 +98,7 @@ - org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore - org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning - org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning --org.eclipse.jdt.core.compiler.source=1.5 -+org.eclipse.jdt.core.compiler.source=1.7 - org.eclipse.jdt.core.formatter.align_type_members_on_columns=false - org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16 - org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16 -diff --git a/org.eclipse.help.base/META-INF/MANIFEST.MF b/org.eclipse.help.base/META-INF/MANIFEST.MF -index ee34c8e..e4bd703 100644 ---- a/org.eclipse.help.base/META-INF/MANIFEST.MF -+++ b/org.eclipse.help.base/META-INF/MANIFEST.MF -@@ -43,9 +43,11 @@ - org.eclipse.core.runtime;bundle-version="[3.11.0,4.0.0)", - org.eclipse.help;bundle-version="[3.5.0,4.0.0)";visibility:=reexport, - org.eclipse.core.expressions;bundle-version="[3.4.200,4.0.0)", -- org.apache.lucene.analysis;bundle-version="[3.5.0,4.0.0)", -- org.apache.lucene.core;bundle-version="[3.5.0,4.0.0)", -- org.eclipse.core.net;bundle-version="1.2.200" -+ org.eclipse.core.net;bundle-version="1.2.200", -+ org.apache.lucene.analyzers-common;bundle-version="5.1.0", -+ org.apache.lucene.core;bundle-version="5.1.0", -+ org.apache.lucene.queryparser;bundle-version="5.1.0", -+ org.apache.lucene.analyzers-smartcn;bundle-version="5.1.0" - Import-Package: com.ibm.icu.text, - org.eclipse.equinox.http.jetty;resolution:=optional - Bundle-RequiredExecutionEnvironment: J2SE-1.5 -diff --git a/org.eclipse.help.base/plugin.xml b/org.eclipse.help.base/plugin.xml -index 07a5a22..4daf3f0 100644 ---- a/org.eclipse.help.base/plugin.xml -+++ b/org.eclipse.help.base/plugin.xml -@@ -83,7 +83,7 @@ - - -+ class="org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer"> - - StopFilter->PorterStemFilter - */ - public final class Analyzer_en extends Analyzer { -+ - /** - * Constructor for Analyzer_en. - */ -@@ -27,18 +34,25 @@ - super(); - } - /** -- * Creates a TokenStream which tokenizes all the text in the provided -+ * Creates a TokenStreamComponents which tokenizes all the text in the provided - * Reader. - */ -- public final TokenStream tokenStream(String fieldName, Reader reader) { -- return new PorterStemFilter(new StopFilter(Version.LUCENE_30, new LowerCaseAndDigitsTokenizer(reader), getStopWords(), false)); -+ @Override -+ public final TokenStreamComponents createComponents(String fieldName) { -+ final Tokenizer source; -+ source = new LowerCaseAndDigitsTokenizer(); -+ TokenStream result = new EnglishPossessiveFilter(source); -+ result = new StopFilter(result, new CharArraySet(getStopWords(), false)); -+ result = new KeywordRepeatFilter(result); -+ result = new PorterStemFilter(result); -+ return new TokenStreamComponents(source, result); - } - -- private Set stopWords; -+ private List stopWords; - -- private Set getStopWords() { -+ private List getStopWords() { - if ( stopWords == null ) { -- stopWords = new HashSet(); -+ stopWords = new ArrayList(); - for (int i = 0; i < STOP_WORDS.length; i++) { - stopWords.add(STOP_WORDS[i]); - } -diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java -index 4109474..e3c8722 100644 ---- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java -+++ b/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java -@@ -7,22 +7,20 @@ - * Contributors: - * IBM Corporation - initial API and implementation - * Alexander Kurtakov - Bug 460787 -+ * Sopot Cela - Bug 466829 - Migration to Lucene 5 - *******************************************************************************/ - package org.eclipse.help.internal.search; - --import java.io.Reader; - import java.util.Locale; - import java.util.StringTokenizer; - --import com.ibm.icu.text.BreakIterator; -- - import org.apache.lucene.analysis.Analyzer; --import org.apache.lucene.analysis.LowerCaseFilter; --import org.apache.lucene.analysis.TokenStream; --import org.apache.lucene.util.Version; -+import org.apache.lucene.analysis.Tokenizer; -+import org.apache.lucene.analysis.core.LowerCaseFilter; -+import org.eclipse.core.runtime.Platform; - import org.eclipse.help.internal.base.HelpBasePlugin; - --import org.eclipse.core.runtime.Platform; -+import com.ibm.icu.text.BreakIterator; - - - /** -@@ -84,11 +82,14 @@ - } - - /** -- * Creates a TokenStream which tokenizes all the text in the provided -+ * Creates a TokenStreamComponents which tokenizes all the text in the provided - * Reader. - */ -- public final TokenStream tokenStream(String fieldName, Reader reader) { -- return new LowerCaseFilter(Version.LUCENE_30, new WordTokenStream(fieldName, reader, locale)); -+ @Override -+ public final TokenStreamComponents createComponents(String fieldName) { -+ Tokenizer source = new WordTokenStream(locale); -+ LowerCaseFilter filter = new LowerCaseFilter(source); -+ return new TokenStreamComponents(source, filter); - } - - /** -diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java -index a475688..cbb2472 100644 ---- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java -+++ b/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java -@@ -8,27 +8,26 @@ - * Contributors: - * IBM Corporation - initial API and implementation - * Alexander Kurtakov - Bug 460787 -+ * Sopot Cela - Bug 466829 - Migration to Lucene 5 - *******************************************************************************/ - package org.eclipse.help.internal.search; - --import java.io.*; -- --import org.apache.lucene.analysis.*; --import org.apache.lucene.util.Version; -+import org.apache.lucene.analysis.util.CharTokenizer; - - /** - * Tokenizer breaking words around letters or digits. - */ - public class LowerCaseAndDigitsTokenizer extends CharTokenizer { - -- public LowerCaseAndDigitsTokenizer(Reader input) { -- super(Version.LUCENE_30, input); -+ public LowerCaseAndDigitsTokenizer() { -+ super(); - } - protected char normalize(char c) { - return Character.toLowerCase(c); - } - -- protected boolean isTokenChar(char c) { -+ @Override -+ public boolean isTokenChar(int c) { - return Character.isLetterOrDigit(c); - } - -diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java -index 60a545d..804bf5e 100644 ---- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java -+++ b/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java -@@ -1,5 +1,5 @@ - /******************************************************************************* -- * Copyright (c) 2010, 2011 IBM Corporation and others. -+ * Copyright (c) 2010, 2015 IBM Corporation and others. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at -@@ -7,6 +7,7 @@ - * - * Contributors: - * IBM Corporation - initial API and implementation -+ * Sopot Cela - Bug 466829 - Migration to Lucene 5 - *******************************************************************************/ - - package org.eclipse.help.internal.search; -@@ -16,6 +17,8 @@ - - import org.apache.lucene.document.Document; - import org.apache.lucene.document.Field; -+import org.apache.lucene.document.StoredField; -+import org.apache.lucene.document.TextField; - import org.eclipse.help.search.ISearchDocument; - - /** -@@ -31,22 +34,22 @@ - } - - public void setTitle(String title) { -- doc.add(new Field("title", title, Field.Store.NO, Field.Index.ANALYZED)); //$NON-NLS-1$ -- doc.add(new Field("exact_title", title, Field.Store.NO, Field.Index.ANALYZED)); //$NON-NLS-1$ -- doc.add(new Field("raw_title", title, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$ -+ doc.add(new TextField("title", title, Field.Store.NO)); //$NON-NLS-1$ -+ doc.add(new TextField("exact_title", title, Field.Store.NO)); //$NON-NLS-1$ -+ doc.add(new StoredField("raw_title", title)); //$NON-NLS-1$ - } - - public void setSummary(String summary) { -- doc.add(new Field("summary", summary, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$ -+ doc.add(new StoredField("summary", summary)); //$NON-NLS-1$ - } - - public void addContents(String contents) { -- doc.add(new Field("contents", new StringReader(contents))); //$NON-NLS-1$ -- doc.add(new Field("exact_contents", new StringReader(contents))); //$NON-NLS-1$ -+ doc.add(new TextField("contents", new StringReader(contents))); //$NON-NLS-1$ -+ doc.add(new TextField("exact_contents", new StringReader(contents))); //$NON-NLS-1$ - } - - public void setHasFilters(boolean hasFilters) { -- doc.add(new Field("filters", Boolean.toString(hasFilters), Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$ -+ doc.add(new StoredField("filters", Boolean.toString(hasFilters))); //$NON-NLS-1$ - } - - public Document getDocument() { -@@ -54,8 +57,8 @@ - } - - public void addContents(Reader contents, Reader exactContents) { -- doc.add(new Field("contents", contents)); //$NON-NLS-1$ -- doc.add(new Field("exact_contents", exactContents)); //$NON-NLS-1$ -+ doc.add(new TextField("contents", contents)); //$NON-NLS-1$ -+ doc.add(new TextField("exact_contents", exactContents)); //$NON-NLS-1$ - } - - } -diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java -index ca9cd67..7952c8d 100644 ---- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java -+++ b/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java -@@ -1,5 +1,5 @@ - /******************************************************************************* -- * Copyright (c) 2005, 2014 IBM Corporation and others. -+ * Copyright (c) 2005, 2015 IBM Corporation and others. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at -@@ -7,6 +7,7 @@ - * - * Contributors: - * IBM Corporation - initial API and implementation -+ * Sopot Cela - Bug 466829 - Migration to Lucene 5 - *******************************************************************************/ - package org.eclipse.help.internal.search; - -@@ -152,6 +153,7 @@ - .getProperty(SearchIndex.DEPENDENCIES_KEY_ANALYZER); - if (!targetIndex.isLuceneCompatible(lucene) - || !targetIndex.isAnalyzerCompatible(analyzer)) { -+ HelpBasePlugin.logError("Error trying to consume Lucene index from bundle "+bundle.toString()+". Please use an index built with Lucene 5 or higher.", null); //$NON-NLS-1$ //$NON-NLS-2$ - return false; - } - } catch (MalformedURLException mue) { -diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java -deleted file mode 100644 -index 08cf58a..0000000 ---- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java -+++ /dev/null -@@ -1,459 +0,0 @@ --/******************************************************************************* -- * Copyright (c) 2000, 2012 IBM Corporation and others. -- * All rights reserved. This program and the accompanying materials -- * are made available under the terms of the Eclipse Public License v1.0 -- * which accompanies this distribution, and is available at -- * http://www.eclipse.org/legal/epl-v10.html -- * -- * Contributors: -- * IBM Corporation - initial API and implementation -- * Chris Torrence - patch for bug Bug 107648 -- *******************************************************************************/ --package org.eclipse.help.internal.search; --import java.io.*; --import java.util.ArrayList; --import java.util.Collection; --import java.util.Iterator; --import java.util.List; --import java.util.Locale; --import java.util.StringTokenizer; -- --import org.apache.lucene.analysis.*; --import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; --import org.apache.lucene.index.*; --import org.apache.lucene.search.*; --import org.eclipse.help.internal.base.*; --/** -- * Build query acceptable by the search engine. -- */ --public class QueryBuilder { -- // Maximum allowed number of terms -- private static final int MAX_TERMS = 10; -- // Maximum allowed number of ORs -- private static final int MAX_UNIONS = 4; -- // Maximum allowed number terms with wild cards -- private static final int MAX_WILD_TERMS = 2; -- // Query from user -- private String searchWords; -- // Descriptor of Analyzer to process the query words -- private AnalyzerDescriptor analyzerDesc; -- // Analyzer to process the query words -- private Analyzer analyzer; -- // List of QueryWordsToken -- private List analyzedTokens; -- // List of words to highlight -- private List highlightWords = new ArrayList(); -- private Locale locale; -- /** -- * Creates a query builder for the search word. The search word is processed -- * by a lexical analyzer. -- */ -- public QueryBuilder(String searchWords, AnalyzerDescriptor analyzerDesc) { -- this.searchWords = searchWords; -- String language = analyzerDesc.getLang(); -- if (language.length() >= 5) { -- this.locale = new Locale(language.substring(0, 2), language -- .substring(3, 5)); -- } else { -- this.locale = new Locale(language.substring(0, 2), ""); //$NON-NLS-1$ -- } -- this.analyzerDesc = analyzerDesc; -- this.analyzer = analyzerDesc.getAnalyzer(); -- } -- /** -- * Splits user query into tokens and returns a list of QueryWordsToken's. -- */ -- private List tokenizeUserQuery(String searchWords) { -- List tokenList = new ArrayList(); -- //Divide along quotation marks -- //StringTokenizer qTokenizer = new StringTokenizer(searchWords.trim(), -- // "\"", true); //$NON-NLS-1$ -- boolean withinQuotation = false; -- String quotedString = ""; //$NON-NLS-1$ -- int termCount = 0;// keep track of number of terms to disallow too many -- -- int fromIndex = -1; -- searchWords = searchWords.trim(); -- while((fromIndex = searchWords.indexOf("\"", fromIndex+1))!= -1){ //$NON-NLS-1$ -- withinQuotation = !withinQuotation; -- } -- if( withinQuotation ) { -- searchWords = searchWords + "\""; //$NON-NLS-1$ -- withinQuotation = !withinQuotation; -- } -- -- StringTokenizer qTokenizer = new StringTokenizer(searchWords,"\"",true); //$NON-NLS-1$ -- int orCount = 0; // keep track of number of ORs to disallow too many -- while (qTokenizer.hasMoreTokens()) { -- String curToken = qTokenizer.nextToken(); -- if (curToken.equals("\"")) { //$NON-NLS-1$ -- if (withinQuotation) { -- // check for too many terms -- if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER -- && ++termCount > MAX_TERMS) { -- throw new QueryTooComplexException(); -- } -- tokenList.add(QueryWordsToken.exactPhrase(quotedString)); -- } else { -- quotedString = ""; //$NON-NLS-1$ -- } -- withinQuotation = !withinQuotation; -- continue; -- } else if (withinQuotation) { -- quotedString = curToken; -- continue; -- } else { -- //divide unquoted strings along white space -- StringTokenizer parser = new StringTokenizer(curToken.trim()); -- while (parser.hasMoreTokens()) { -- String token = parser.nextToken(); -- if (token.equalsIgnoreCase(QueryWordsToken.AND().value)) { -- tokenList.add(QueryWordsToken.AND()); -- } else if (token -- .equalsIgnoreCase(QueryWordsToken.OR().value)) { -- // Check for too many OR terms -- if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER -- && ++orCount > MAX_UNIONS) { -- throw new QueryTooComplexException(); -- } -- tokenList.add(QueryWordsToken.OR()); -- } else if (token -- .equalsIgnoreCase(QueryWordsToken.NOT().value)) { -- tokenList.add(QueryWordsToken.NOT()); -- } else { -- // check for too many terms -- if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER -- && ++termCount > MAX_TERMS) { -- throw new QueryTooComplexException(); -- } -- tokenList.add(QueryWordsToken.word(token)); -- } -- } -- } -- } -- return tokenList; -- } -- /** -- * Apply the Analyzer to the search tokens and return the list of processed -- * QueryWordsToken's. -- */ -- private List analyzeTokens(List tokens) { -- boolean isTokenAfterNot = false; -- List newTokens = new ArrayList(); -- int wildCardTermCount = 0; -- for (int i = 0; i < tokens.size(); i++) { -- QueryWordsToken token = tokens.get(i); -- if (token.type == QueryWordsToken.WORD) { -- int questionMIndex = token.value.indexOf('?'); -- int starIndex = token.value.indexOf('*'); -- if (starIndex >= 0 || questionMIndex >= 0) { -- if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER -- && ++wildCardTermCount > MAX_WILD_TERMS) { -- throw new QueryTooComplexException(); -- } -- newTokens.add(QueryWordsToken.word(token.value -- .toLowerCase(locale))); -- // add word to the list of words to highlight -- if (!isTokenAfterNot && !highlightWords.contains(token.value)) { -- highlightWords.add(token.value); -- } -- } else { -- List wordList = analyzeText(analyzer, "contents", //$NON-NLS-1$ -- token.value); -- if (wordList.size() > 0) { -- if (!isTokenAfterNot && !highlightWords.contains(token.value)) { -- // add original word to the list of words to -- // highlight -- highlightWords.add(token.value); -- } -- if (wordList.size() == 1) { -- String word = wordList.get(0); -- newTokens.add(QueryWordsToken.word(word)); -- // add analyzed word to the list of words to -- // highlight -- // this is required to highlight stemmed words -- if (!isTokenAfterNot && !highlightWords.contains(word)) { -- highlightWords.add(word); -- } -- } else { -- QueryWordsPhrase phrase = QueryWordsToken.phrase(); -- for (Iterator it = wordList.iterator(); it -- .hasNext();) { -- String word = it.next(); -- phrase.addWord(word); -- // add each analyzed word to the list of words -- // to highlight -- // this is only required to highlight stemmed -- // words. -- // Adding words should not be done when -- // DefaultAnalyzer is used, -- // because it does not perform stemming and -- // common words removal -- // which would result in common characters -- // highlighted all over (bug 30263) -- if (!analyzerDesc.getId().startsWith( -- HelpBasePlugin.PLUGIN_ID + "#")) { //$NON-NLS-1$ -- if (!isTokenAfterNot && !highlightWords.contains(word)) { -- highlightWords.add(word); -- } -- } -- } -- newTokens.add(phrase); -- } -- } -- } -- } else if (// forget ANDs -- /* -- * token.type == SearchQueryToken.AND || -- */ -- token.type == QueryWordsToken.OR -- || token.type == QueryWordsToken.NOT) -- newTokens.add(token); -- else if (token.type == QueryWordsToken.EXACT_PHRASE) { -- List wordList = analyzeText(analyzer, "exact_contents", //$NON-NLS-1$ -- token.value); -- if (wordList.size() > 0) { -- if (!isTokenAfterNot && !highlightWords.contains(token.value)) { -- // add original word to the list of words to highlight -- highlightWords.add(token.value); -- } -- } -- QueryWordsExactPhrase phrase = QueryWordsToken.exactPhrase(); -- for (Iterator it = wordList.iterator(); it.hasNext();) { -- String word = it.next(); -- phrase.addWord(word); -- // add analyzed word to the list of words to highlight -- // if (!highlightWords.contains(word)) -- // highlightWords.add(word); -- } -- // add phrase only if not empty -- if (phrase.getWords().size() > 0) { -- newTokens.add(phrase); -- } -- } -- isTokenAfterNot = (token.type == QueryWordsToken.NOT); -- } -- return newTokens; -- } -- /** -- * Get a list of tokens corresponding to a search word or phrase -- * -- * @return List of String -- */ -- private List analyzeText(Analyzer analyzer, String fieldName, String text) { -- List words = new ArrayList(1); -- Reader reader = new StringReader(text); -- TokenStream tStream = analyzer.tokenStream(fieldName, reader); -- -- CharTermAttribute termAttribute = tStream.getAttribute(CharTermAttribute.class); -- try { -- while (tStream.incrementToken()) { -- String term = termAttribute.toString(); -- words.add(term); -- } -- reader.close(); -- } catch (IOException ioe) { -- } -- -- return words; -- } -- /** -- * Obtains Lucene Query from tokens -- * -- * @return Query or null if no query could be created -- */ -- private Query createLuceneQuery(List searchTokens, String[] fieldNames, -- float[] boosts) { -- // Get queries for parts separated by OR -- List requiredQueries = getRequiredQueries(searchTokens, fieldNames, -- boosts); -- if (requiredQueries.size() == 0) -- return null; -- else if (requiredQueries.size() <= 1) -- return requiredQueries.get(0); -- else -- /* if (requiredQueries.size() > 1) */ -- // OR queries -- return (orQueries(requiredQueries)); -- } -- /** -- * Obtains Lucene queries for token sequences separated at OR. -- * -- * @return List of Query (could be empty) -- */ -- private List getRequiredQueries(List tokens, String[] fieldNames, -- float[] boosts) { -- List oredQueries = new ArrayList(); -- ArrayList requiredQueryTokens = new ArrayList(); -- for (int i = 0; i < tokens.size(); i++) { -- QueryWordsToken token = tokens.get(i); -- if (token.type != QueryWordsToken.OR) { -- requiredQueryTokens.add(token); -- } else { -- Query reqQuery = getRequiredQuery(requiredQueryTokens, -- fieldNames, boosts); -- if (reqQuery != null) -- oredQueries.add(reqQuery); -- requiredQueryTokens = new ArrayList(); -- } -- } -- Query reqQuery = getRequiredQuery(requiredQueryTokens, fieldNames, -- boosts); -- if (reqQuery != null) -- oredQueries.add(reqQuery); -- return oredQueries; -- } -- private Query orQueries(Collection queries) { -- BooleanQuery bq = new BooleanQuery(); -- for (Iterator it = queries.iterator(); it.hasNext();) { -- Query q = it.next(); -- bq.add(q, BooleanClause.Occur.SHOULD); -- } -- return bq; -- } -- /** -- * Obtains Lucene Query for tokens containing only AND and NOT operators. -- * -- * @return BooleanQuery or null if no query could be created from the tokens -- */ -- private Query getRequiredQuery(List requiredTokens, String[] fieldNames, -- float[] boosts) { -- BooleanQuery retQuery = new BooleanQuery(); -- boolean requiredTermExist = false; -- // Parse tokens left to right -- QueryWordsToken operator = null; -- for (int i = 0; i < requiredTokens.size(); i++) { -- QueryWordsToken token = requiredTokens.get(i); -- if (token.type == QueryWordsToken.AND -- || token.type == QueryWordsToken.NOT) { -- operator = token; -- continue; -- } -- // Creates queries for all fields -- Query qs[] = new Query[fieldNames.length]; -- for (int f = 0; f < fieldNames.length; f++) { -- qs[f] = token.createLuceneQuery(fieldNames[f], boosts[f]); -- } -- // creates the boolean query of all fields -- Query q = qs[0]; -- if (fieldNames.length > 1) { -- BooleanQuery allFieldsQuery = new BooleanQuery(); -- for (int f = 0; f < fieldNames.length; f++) -- allFieldsQuery.add(qs[f], BooleanClause.Occur.SHOULD); -- q = allFieldsQuery; -- } -- if (operator != null && operator.type == QueryWordsToken.NOT) { -- retQuery.add(q, BooleanClause.Occur.MUST_NOT); // add as prohibited -- } else { -- retQuery.add(q, BooleanClause.Occur.MUST); // add as required -- requiredTermExist = true; -- } -- } -- if (!requiredTermExist) { -- return null; // cannot search for prohibited only -- } -- return retQuery; -- } -- private Query getLuceneQuery(String[] fieldNames, float[] boosts) { -- Query luceneQuery = createLuceneQuery(analyzedTokens, fieldNames, -- boosts); -- return luceneQuery; -- } -- /** -- * @param fieldNames - -- * Collection of field names of type String (e.g. "h1"); the -- * search will be performed on the given fields -- * @param fieldSearchOnly - -- * boolean indicating if field only search should be performed; -- * if set to false, default field "contents" and all other fields -- * will be searched -- */ -- public Query getLuceneQuery(Collection fieldNames, boolean fieldSearchOnly) -- throws QueryTooComplexException { -- // split search query into tokens -- List userTokens = tokenizeUserQuery(searchWords); -- analyzedTokens = analyzeTokens(userTokens); -- return buildLuceneQuery(fieldNames, fieldSearchOnly); -- } -- /** -- * @param fieldNames - -- * Collection of field names of type String (e.g. "h1"); the -- * search will be performed on the given fields -- * @param fieldSearchOnly - -- * boolean indicating if field only search should be performed; -- * if set to false, default field "contents" and all other fields -- * will be searched -- */ -- private Query buildLuceneQuery(Collection fieldNames, -- boolean fieldSearchOnly) { -- String[] fields; -- float[] boosts; -- if (fieldSearchOnly) { -- fields = new String[fieldNames.size()]; -- boosts = new float[fieldNames.size()]; -- Iterator fieldNamesIt = fieldNames.iterator(); -- for (int i = 0; i < fieldNames.size(); i++) { -- fields[i] = fieldNamesIt.next(); -- boosts[i] = 5.0f; -- } -- } else { -- fields = new String[fieldNames.size() + 2]; -- boosts = new float[fieldNames.size() + 2]; -- Iterator fieldNamesIt = fieldNames.iterator(); -- for (int i = 0; i < fieldNames.size(); i++) { -- fields[i] = fieldNamesIt.next(); -- boosts[i] = 5.0f; -- } -- fields[fieldNames.size()] = "contents"; //$NON-NLS-1$ -- boosts[fieldNames.size()] = 1.0f; -- fields[fieldNames.size()+1] = "title"; //$NON-NLS-1$ -- boosts[fieldNames.size()+1] = 1.0f; -- } -- Query query = getLuceneQuery(fields, boosts); -- query = improveRankingForUnqotedPhrase(query, fields, boosts); -- return query; -- } -- /** -- * If user query contained only words (no quotaions nor operators) extends -- * query with term phrase representing entire user query i.e for user string -- * a b, the query a AND b will be extended to "a b" OR a AND b -- */ -- private Query improveRankingForUnqotedPhrase(Query query, String[] fields, -- float[] boosts) { -- if (query == null) -- return query; -- // check if all tokens are words -- for (int i = 0; i < analyzedTokens.size(); i++) -- if (analyzedTokens.get(i).type != QueryWordsToken.WORD) -- return query; -- // Create phrase query for all tokens and OR with original query -- BooleanQuery booleanQuery = new BooleanQuery(); -- booleanQuery.add(query, BooleanClause.Occur.SHOULD); -- PhraseQuery[] phraseQueries = new PhraseQuery[fields.length]; -- for (int f = 0; f < fields.length; f++) { -- phraseQueries[f] = new PhraseQuery(); -- for (int i = 0; i < analyzedTokens.size(); i++) { -- Term t = new Term(fields[f], analyzedTokens -- .get(i).value); -- phraseQueries[f].add(t); -- } -- phraseQueries[f].setBoost(10 * boosts[f]); -- booleanQuery.add(phraseQueries[f], BooleanClause.Occur.SHOULD); -- } -- return booleanQuery; -- } -- /** -- * Obtains analyzed terms from query as one string. Words are double quoted, -- * and separated by space. The analyzed words are needed for highlighting -- * word roots. -- */ -- public String gethighlightTerms() { -- StringBuffer buf = new StringBuffer(); -- for (Iterator it = highlightWords.iterator(); it.hasNext();) { -- buf.append('"'); -- buf.append(it.next()); -- buf.append("\" "); //$NON-NLS-1$ -- } -- return buf.toString(); -- } --} -diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsExactPhrase.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsExactPhrase.java -deleted file mode 100644 -index 324b8e1..0000000 ---- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsExactPhrase.java -+++ /dev/null -@@ -1,50 +0,0 @@ --/******************************************************************************* -- * Copyright (c) 2000, 2011 IBM Corporation and others. -- * All rights reserved. This program and the accompanying materials -- * are made available under the terms of the Eclipse Public License v1.0 -- * which accompanies this distribution, and is available at -- * http://www.eclipse.org/legal/epl-v10.html -- * -- * Contributors: -- * IBM Corporation - initial API and implementation -- *******************************************************************************/ --package org.eclipse.help.internal.search; --import java.util.ArrayList; --import java.util.Iterator; --import java.util.List; -- --import org.apache.lucene.index.*; --import org.apache.lucene.search.*; --/** -- * Represents a quoted token in user search query words -- */ --public class QueryWordsExactPhrase extends QueryWordsToken { -- private List words; -- public QueryWordsExactPhrase() { -- super(QueryWordsToken.EXACT_PHRASE, ""); //$NON-NLS-1$ -- words = new ArrayList(); -- } -- public void addWord(String word) { -- words.add(word); -- if (words.size() <= 1) -- value = word; -- else -- value += " " + word; //$NON-NLS-1$ -- } -- public List getWords() { -- return words; -- } -- /** -- * Creates a lucene query for a field -- */ -- public Query createLuceneQuery(String field, float boost) { -- PhraseQuery q = new PhraseQuery(); -- for (Iterator it = getWords().iterator(); it.hasNext();) { -- String word = it.next(); -- Term t = new Term("exact_" + field, word); //$NON-NLS-1$ -- q.add(t); -- q.setBoost(boost); -- } -- return q; -- } --} -diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsPhrase.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsPhrase.java -deleted file mode 100644 -index 8a94e89..0000000 ---- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsPhrase.java -+++ /dev/null -@@ -1,51 +0,0 @@ --/******************************************************************************* -- * Copyright (c) 2000, 2011 IBM Corporation and others. -- * All rights reserved. This program and the accompanying materials -- * are made available under the terms of the Eclipse Public License v1.0 -- * which accompanies this distribution, and is available at -- * http://www.eclipse.org/legal/epl-v10.html -- * -- * Contributors: -- * IBM Corporation - initial API and implementation -- *******************************************************************************/ --package org.eclipse.help.internal.search; --import java.util.ArrayList; --import java.util.Iterator; --import java.util.List; -- --import org.apache.lucene.index.*; --import org.apache.lucene.search.*; --/** -- * Represents a phrase (not quoted) token in user search query words It consists -- * of several words created by an analyzer -- */ --public class QueryWordsPhrase extends QueryWordsToken { -- private List words; -- public QueryWordsPhrase() { -- super(QueryWordsToken.PHRASE, ""); //$NON-NLS-1$ -- words = new ArrayList(); -- } -- public void addWord(String word) { -- words.add(word); -- if (words.size() <= 1) -- value = word; -- else -- value += " " + word; //$NON-NLS-1$ -- } -- public List getWords() { -- return words; -- } -- /** -- * Creates a lucene query for a field -- */ -- public Query createLuceneQuery(String field, float boost) { -- PhraseQuery q = new PhraseQuery(); -- for (Iterator it = getWords().iterator(); it.hasNext();) { -- String word = it.next(); -- Term t = new Term(field, word); -- q.add(t); -- q.setBoost(boost); -- } -- return q; -- } --} -diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsToken.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsToken.java -deleted file mode 100644 -index 6ba76f2..0000000 ---- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsToken.java -+++ /dev/null -@@ -1,81 +0,0 @@ --/******************************************************************************* -- * Copyright (c) 2000, 2007 IBM Corporation and others. -- * All rights reserved. This program and the accompanying materials -- * are made available under the terms of the Eclipse Public License v1.0 -- * which accompanies this distribution, and is available at -- * http://www.eclipse.org/legal/epl-v10.html -- * -- * Contributors: -- * IBM Corporation - initial API and implementation -- *******************************************************************************/ --package org.eclipse.help.internal.search; --import org.apache.lucene.index.*; --import org.apache.lucene.search.*; --/** -- * Represents a token in user search query words -- */ --public class QueryWordsToken { -- public static final int AND = 0; -- public static final int OR = 1; -- public static final int NOT = 2; -- public static final int EXACT_PHRASE = 3; -- public static final int PHRASE = 4; -- public static final int WORD = 5; -- private static final QueryWordsToken fAND = new QueryWordsToken(AND, "AND"); //$NON-NLS-1$ -- private static final QueryWordsToken fOR = new QueryWordsToken(OR, "OR"); //$NON-NLS-1$ -- private static final QueryWordsToken fNOT = new QueryWordsToken(NOT, "NOT"); //$NON-NLS-1$ -- public int type; -- public String value; -- protected QueryWordsToken(int type, String value) { -- this.type = type; -- this.value = value; -- } -- /** -- * Creates a lucene query for a field -- */ -- public Query createLuceneQuery(String field, float boost) { -- Query q; -- int questionPos = value.indexOf('?'); -- int starPos = value.indexOf('*'); -- if (questionPos >= 0 || starPos >= 0) { -- if (questionPos == -1 && starPos == value.length() - 1) { -- Term t = new Term("exact_" + field, value.substring(0, starPos)); //$NON-NLS-1$ -- q = new PrefixQuery(t); -- ((PrefixQuery) q).setBoost(boost); -- } else { -- Term t = new Term("exact_" + field, value); //$NON-NLS-1$ -- q = new WildcardQuery(t); -- ((WildcardQuery) q).setBoost(boost); -- } -- } else { -- Term t = new Term(field, value); -- q = new TermQuery(t); -- ((TermQuery) q).setBoost(boost); -- } -- // after updating Lucene, set boost on a Query class -- return q; -- } -- public static QueryWordsToken AND() { -- return fAND; -- } -- public static QueryWordsToken OR() { -- return fOR; -- } -- public static QueryWordsToken NOT() { -- return fNOT; -- } -- public static QueryWordsToken word(String word) { -- return new QueryWordsToken(QueryWordsToken.WORD, word); -- } -- public static QueryWordsPhrase phrase() { -- return new QueryWordsPhrase(); -- } -- public static QueryWordsExactPhrase exactPhrase() { -- return new QueryWordsExactPhrase(); -- } -- public static QueryWordsExactPhrase exactPhrase(String word) { -- QueryWordsExactPhrase token = new QueryWordsExactPhrase(); -- token.addWord(word); -- return token; -- } --} -diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java -index 67963a5..131ac65 100644 ---- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java -+++ b/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java -@@ -1,14 +1,13 @@ - /******************************************************************************* -- * Copyright (c) 2000, 2014 IBM Corporation and others. -- * All rights reserved. This program and the accompanying materials -- * are made available under the terms of the Eclipse Public License v1.0 -+ * Copyright (c) 2000, 2015 IBM Corporation and others. All rights reserved. This program and the -+ * accompanying materials are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * -- * Contributors: -- * IBM Corporation - initial API and implementation -- * Holger Voormann - fix for bug 426785 (http://eclip.se/426785) -- * Alexander Kurtakov - Bug 460787 -+ * Contributors: IBM Corporation - initial API and implementation -+ * Holger Voormann - fix for bug 426785 (http://eclip.se/426785) -+ * Alexander Kurtakov - Bug 460787 -+ * Sopot Cela - Bug 466829 - Migration to Lucene 5 - *******************************************************************************/ - package org.eclipse.help.internal.search; - -@@ -17,6 +16,7 @@ - import java.io.IOException; - import java.io.InputStream; - import java.io.RandomAccessFile; -+import java.io.StringReader; - import java.net.MalformedURLException; - import java.net.URL; - import java.nio.channels.FileLock; -@@ -33,19 +33,30 @@ - import java.util.zip.ZipEntry; - import java.util.zip.ZipInputStream; - --import org.apache.lucene.analysis.LimitTokenCountAnalyzer; -+import org.apache.lucene.analysis.miscellaneous.LimitTokenCountAnalyzer; - import org.apache.lucene.document.Document; - import org.apache.lucene.document.Field; -+import org.apache.lucene.document.StoredField; -+import org.apache.lucene.document.StringField; -+import org.apache.lucene.index.DirectoryReader; - import org.apache.lucene.index.IndexReader; - import org.apache.lucene.index.IndexWriter; --import org.apache.lucene.index.Term; --import org.apache.lucene.index.TermDocs; --import org.apache.lucene.index.IndexWriterConfig.OpenMode; - import org.apache.lucene.index.IndexWriterConfig; -+import org.apache.lucene.index.IndexWriterConfig.OpenMode; -+import org.apache.lucene.index.LeafReader; - import org.apache.lucene.index.LogByteSizeMergePolicy; - import org.apache.lucene.index.LogMergePolicy; -+import org.apache.lucene.index.PostingsEnum; -+import org.apache.lucene.index.SlowCompositeReaderWrapper; -+import org.apache.lucene.index.Term; -+import org.apache.lucene.queryparser.classic.FastCharStream; -+import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; -+import org.apache.lucene.queryparser.classic.QueryParserTokenManager; -+import org.apache.lucene.queryparser.classic.Token; -+import org.apache.lucene.search.BooleanClause.Occur; - import org.apache.lucene.search.BooleanQuery; - import org.apache.lucene.search.IndexSearcher; -+import org.apache.lucene.search.MultiTermQuery; - import org.apache.lucene.search.Query; - import org.apache.lucene.search.TopDocs; - import org.apache.lucene.store.Directory; -@@ -59,7 +70,6 @@ - import org.eclipse.core.runtime.OperationCanceledException; - import org.eclipse.core.runtime.Platform; - import org.eclipse.core.runtime.Status; --import org.eclipse.help.internal.HelpPlugin; - import org.eclipse.help.internal.base.BaseHelpSystem; - import org.eclipse.help.internal.base.HelpBasePlugin; - import org.eclipse.help.internal.base.util.HelpProperties; -@@ -86,7 +96,7 @@ - - private File indexDir; - -- private Directory luceneDirectory; -+ public Directory luceneDirectory; - - private String locale; - -@@ -133,7 +143,7 @@ - private Collection searches = new ArrayList(); - - private FileLock lock; -- private RandomAccessFile raf = null; -+ private RandomAccessFile raf = null; - - /** - * Constructor. -@@ -170,7 +180,7 @@ - inconsistencyFile = new File(indexDir.getParentFile(), locale + ".inconsistent"); //$NON-NLS-1$ - htmlSearchParticipant = new HTMLSearchParticipant(indexDir.getAbsolutePath()); - try { -- luceneDirectory = new NIOFSDirectory(indexDir); -+ luceneDirectory = new NIOFSDirectory(indexDir.toPath()); - } catch (IOException e) { - } - if (!exists()) { -@@ -203,11 +213,11 @@ - public IStatus addDocument(String name, URL url) { - try { - Document doc = new Document(); -- doc.add(new Field(FIELD_NAME, name, Field.Store.YES, Field.Index.NOT_ANALYZED)); -+ doc.add(new StringField(FIELD_NAME, name, Field.Store.YES)); - addExtraFields(doc); - String pluginId = LocalSearchManager.getPluginId(name); - if (relativePath != null) { -- doc.add(new Field(FIELD_INDEX_ID, relativePath, Field.Store.YES, Field.Index.NOT_ANALYZED)); -+ doc.add(new StringField(FIELD_INDEX_ID, relativePath, Field.Store.YES)); - } - // check for the explicit search participant. - SearchParticipant participant = null; -@@ -220,20 +230,22 @@ - if (participant == null) - participant = BaseHelpSystem.getLocalSearchManager().getParticipant(pluginId, name); - if (participant != null) { -- IStatus status = participant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc)); -+ IStatus status = participant.addDocument(this, pluginId, name, url, id, -+ new LuceneSearchDocument(doc)); - if (status.getSeverity() == IStatus.OK) { - String filters = doc.get("filters"); //$NON-NLS-1$ - indexedDocs.put(name, filters != null ? filters : "0"); //$NON-NLS-1$ - if (id != null) -- doc.add(new Field("id", id, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$ -+ doc.add(new StoredField("id", id)); //$NON-NLS-1$ - if (pid != null) -- doc.add(new Field("participantId", pid, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$ -+ doc.add(new StoredField("participantId", pid)); //$NON-NLS-1$ - iw.addDocument(doc); - } - return status; - } - // default to html -- IStatus status = htmlSearchParticipant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc)); -+ IStatus status = htmlSearchParticipant.addDocument(this, pluginId, name, url, id, -+ new LuceneSearchDocument(doc)); - if (status.getSeverity() == IStatus.OK) { - String filters = doc.get("filters"); //$NON-NLS-1$ - indexedDocs.put(name, filters != null ? filters : "0"); //$NON-NLS-1$ -@@ -245,20 +257,21 @@ - "IO exception occurred while adding document " + name //$NON-NLS-1$ - + " to index " + indexDir.getAbsolutePath() + ".", //$NON-NLS-1$ //$NON-NLS-2$ - e); -- } -- catch (Exception e) { -+ } catch (Exception e) { - return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR, - "An unexpected internal error occurred while adding document " //$NON-NLS-1$ - + name + " to index " + indexDir.getAbsolutePath() //$NON-NLS-1$ -- + ".", e); //$NON-NLS-1$ -+ + ".", //$NON-NLS-1$ -+ e); - } - } - - /** -- * Add any extra fields that need to be added to this document. Subclasses -- * should override to add more fields. -+ * Add any extra fields that need to be added to this document. Subclasses should override to -+ * add more fields. - * -- * @param doc the document to add fields to -+ * @param doc -+ * the document to add fields to - */ - protected void addExtraFields(Document doc) { - } -@@ -282,8 +295,9 @@ - indexedDocs = new HelpProperties(INDEXED_DOCS_FILE, indexDir); - indexedDocs.restore(); - setInconsistent(true); -- LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), 1000000); -- IndexWriterConfig writerConfig = new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_31, analyzer); -+ LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), -+ 1000000); -+ IndexWriterConfig writerConfig = new IndexWriterConfig(analyzer); - writerConfig.setOpenMode(create ? OpenMode.CREATE : OpenMode.APPEND); - LogMergePolicy mergePolicy = new LogByteSizeMergePolicy(); - mergePolicy.setMergeFactor(20); -@@ -307,7 +321,7 @@ - indexedDocs = new HelpProperties(INDEXED_DOCS_FILE, indexDir); - indexedDocs.restore(); - setInconsistent(true); -- ir = IndexReader.open(luceneDirectory, false); -+ ir = DirectoryReader.open(luceneDirectory); - return true; - } catch (IOException e) { - HelpBasePlugin.logError("Exception occurred in search indexing at beginDeleteBatch.", e); //$NON-NLS-1$ -@@ -323,7 +337,7 @@ - if (ir != null) { - ir.close(); - } -- ir = IndexReader.open(luceneDirectory, false); -+ ir = DirectoryReader.open(luceneDirectory); - return true; - } catch (IOException e) { - HelpBasePlugin.logError("Exception occurred in search indexing at beginDeleteBatch.", e); //$NON-NLS-1$ -@@ -334,14 +348,14 @@ - /** - * Deletes a single document from the index. - * -- * @param name - -- * document name -+ * @param name -+ * - document name - * @return IStatus - */ - public IStatus removeDocument(String name) { - Term term = new Term(FIELD_NAME, name); - try { -- ir.deleteDocuments(term); -+ iw.deleteDocuments(term); - indexedDocs.remove(name); - } catch (IOException e) { - return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR, -@@ -375,11 +389,11 @@ - } - - /* -- * The searcher's index reader has it's stuff in memory so it won't -- * know about this change. Close it so that it gets reloaded next search. -+ * The searcher's index reader has it's stuff in memory so it won't know about this -+ * change. Close it so that it gets reloaded next search. - */ - if (searcher != null) { -- searcher.close(); -+ searcher.getIndexReader().close(); - searcher = null; - } - return true; -@@ -407,11 +421,11 @@ - saveDependencies(); - - /* -- * The searcher's index reader has it's stuff in memory so it won't -- * know about this change. Close it so that it gets reloaded next search. -+ * The searcher's index reader has it's stuff in memory so it won't know about this -+ * change. Close it so that it gets reloaded next search. - */ - if (searcher != null) { -- searcher.close(); -+ searcher.getIndexReader().close(); - searcher = null; - } - return true; -@@ -469,11 +483,11 @@ - String indexId = indexIds.get(i); - String indexPath = indexPaths.get(i); - try { -- dirList.add(new NIOFSDirectory(new File(indexPath))); -+ dirList.add(new NIOFSDirectory(new File(indexPath).toPath())); - } catch (IOException ioe) { -- HelpBasePlugin -- .logError( -- "Help search indexing directory could not be created for directory " + indexPath, ioe); //$NON-NLS-1$ -+ HelpBasePlugin.logError( -+ "Help search indexing directory could not be created for directory " + indexPath, //$NON-NLS-1$ -+ ioe); - continue; - } - -@@ -525,18 +539,19 @@ - } - - public IStatus removeDuplicates(String name, String[] index_paths) { -- TermDocs hrefDocs = null; -- TermDocs indexDocs = null; -- Term hrefTerm = new Term(FIELD_NAME, name); - try { -+ LeafReader ar = SlowCompositeReaderWrapper.wrap(ir); -+ PostingsEnum hrefDocs = null; -+ PostingsEnum indexDocs = null; -+ Term hrefTerm = new Term(FIELD_NAME, name); - for (int i = 0; i < index_paths.length; i++) { - Term indexTerm = new Term(FIELD_INDEX_ID, index_paths[i]); - if (i == 0) { -- hrefDocs = ir.termDocs(hrefTerm); -- indexDocs = ir.termDocs(indexTerm); -+ hrefDocs = ar.postings(hrefTerm); -+ indexDocs = ar.postings(indexTerm); - } else { -- hrefDocs.seek(hrefTerm); -- indexDocs.seek(indexTerm); -+ hrefDocs = ar.postings(hrefTerm); -+ indexDocs = ar.postings(indexTerm); - } - removeDocuments(hrefDocs, indexDocs); - } -@@ -545,19 +560,6 @@ - "IO exception occurred while removing duplicates of document " + name //$NON-NLS-1$ - + " from index " + indexDir.getAbsolutePath() + ".", //$NON-NLS-1$ //$NON-NLS-2$ - ioe); -- } finally { -- if (hrefDocs != null) { -- try { -- hrefDocs.close(); -- } catch (IOException e) { -- } -- } -- if (indexDocs != null) { -- try { -- indexDocs.close(); -- } catch (IOException e) { -- } -- } - } - return Status.OK_STATUS; - } -@@ -569,33 +571,33 @@ - * @param docs2 - * @throws IOException - */ -- private void removeDocuments(TermDocs doc1, TermDocs docs2) throws IOException { -- if (!doc1.next()) { -+ private void removeDocuments(PostingsEnum doc1, PostingsEnum docs2) throws IOException { -+ if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) { - return; - } -- if (!docs2.next()) { -+ if (docs2.nextDoc() == PostingsEnum.NO_MORE_DOCS) { - return; - } - while (true) { -- if (doc1.doc() < docs2.doc()) { -- if (!doc1.skipTo(docs2.doc())) { -- if (!doc1.next()) { -+ if (doc1.docID() < docs2.docID()) { -+ if (doc1.advance(docs2.docID()) == PostingsEnum.NO_MORE_DOCS) { -+ if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) { - return; - } - } -- } else if (doc1.doc() > docs2.doc()) { -- if (!docs2.skipTo(doc1.doc())) { -- if (!doc1.next()) { -+ } else if (doc1.docID() > docs2.docID()) { -+ if (docs2.advance(doc1.docID()) == PostingsEnum.NO_MORE_DOCS) { -+ if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) { - return; - } - } - } -- if (doc1.doc() == docs2.doc()) { -- ir.deleteDocument(doc1.doc()); -- if (!doc1.next()) { -+ if (doc1.docID() == docs2.docID()) { -+ iw.tryDeleteDocument(ir, doc1.docID()); -+ if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) { - return; - } -- if (!docs2.next()) { -+ if (docs2.nextDoc() == PostingsEnum.NO_MORE_DOCS) { - return; - } - } -@@ -623,20 +625,72 @@ - registerSearch(Thread.currentThread()); - if (closed) - return; -- QueryBuilder queryBuilder = new QueryBuilder(searchQuery.getSearchWord(), analyzerDescriptor); -- Query luceneQuery = queryBuilder.getLuceneQuery(searchQuery.getFieldNames(), searchQuery -- .isFieldSearch()); -- if (HelpPlugin.DEBUG_SEARCH) { -- System.out.println("Search Query: " + luceneQuery.toString()); //$NON-NLS-1$ -+ -+ String[] fields; -+ if (searchQuery.isFieldSearch()){ -+ //sometimes you might want to search other than the default fields -+ fields = (String[]) searchQuery.getFieldNames().toArray(); -+ }else { -+ fields = new String[]{"contents","title"}; //$NON-NLS-1$ //$NON-NLS-2$ - } -- String highlightTerms = queryBuilder.gethighlightTerms(); -- if (luceneQuery != null) { -+ -+ //prepare the parser -+ -+ MultiFieldQueryParser qb = new MultiFieldQueryParser(fields,analyzerDescriptor.getAnalyzer()); -+ qb.setAllowLeadingWildcard(true); -+ qb.setAnalyzeRangeTerms(true); -+ qb.setAutoGeneratePhraseQueries(true); -+ qb.setMultiTermRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE); -+ qb.setLowercaseExpandedTerms(true); -+ qb.setLocale(new Locale(analyzerDescriptor.getLang())); -+ -+ //parse the "pure" query (no boosting) -+ Query luceneQuery = qb.parse(searchQuery.getSearchWord()); -+ -+ //we'll merge the pure query with a some boosted queries -+ Query mergedQuery; -+ -+ if (!isWildcardQuery(searchQuery.getSearchWord())){ -+ mergedQuery = new BooleanQuery(); //merge for all fields before merging with luceneQuery -+ for (int i=0;i set = new HashSet(); -+ -+ if (mergedQuery != null) { - if (searcher == null) { - openSearcher(); - } -- TopDocs topDocs = searcher.search(luceneQuery, null, 1000); -- collector.addHits(LocalSearchManager.asList(topDocs, searcher), highlightTerms); -+ TopDocs topDocs = searcher.search(mergedQuery, 1000); -+ -+ String highlight=null; -+ QueryParserTokenManager manager = new QueryParserTokenManager(new FastCharStream(new StringReader(searchQuery.getSearchWord()))); -+ while (true){ -+ Token nextToken = manager.getNextToken(); -+ String toHighlight = null; -+ if (nextToken.kind==0) break; -+ String image = nextToken.image; -+ toHighlight=image; -+ if ((image.startsWith("\""))&&( image.endsWith("\""))){ //$NON-NLS-1$//$NON-NLS-2$ -+ toHighlight = image.substring(1,image.length()-1); -+ } -+ if (image.equals("AND") || image.equals("OR")) //$NON-NLS-1$ //$NON-NLS-2$ -+ continue; -+ set .add(toHighlight); -+ -+ } -+ highlight = buildHighlight(set); -+ collector.addHits(LocalSearchManager.asList(topDocs, searcher), highlight==null?"":highlight); //$NON-NLS-1$ - } -+ - } catch (BooleanQuery.TooManyClauses tmc) { - collector.addQTCException(new QueryTooComplexException()); - } catch (QueryTooComplexException qe) { -@@ -647,6 +701,19 @@ - } finally { - unregisterSearch(Thread.currentThread()); - } -+ } -+ -+ private boolean isWildcardQuery(String searchWord) { -+ -+ return searchWord.contains("?")|| searchWord.contains("*"); //$NON-NLS-1$//$NON-NLS-2$ -+ } -+ -+ private String buildHighlight(Set set) { -+ StringBuilder sb = new StringBuilder(); -+ for (String string : set) { -+ sb.append("\""+string+"\""); //$NON-NLS-1$//$NON-NLS-2$ -+ } -+ return sb.toString(); - } - - public String getLocale() { -@@ -660,13 +727,13 @@ - if (docPlugins == null) { - Set totalIds = new HashSet(); - IExtensionRegistry registry = Platform.getExtensionRegistry(); -- IExtensionPoint extensionPoint = registry.getExtensionPoint(TocFileProvider.EXTENSION_POINT_ID_TOC); -+ IExtensionPoint extensionPoint = registry -+ .getExtensionPoint(TocFileProvider.EXTENSION_POINT_ID_TOC); - IExtension[] extensions = extensionPoint.getExtensions(); -- for (int i=0;i= 0 ) { -+ if (luceneVersion.compareTo(indexVersion) >= 0) { - // Lucene bundle is newer than the index - return true; - } -@@ -802,7 +871,7 @@ - public void openSearcher() throws IOException { - synchronized (searcherCreateLock) { - if (searcher == null) { -- searcher = new IndexSearcher(IndexReader.open(luceneDirectory, false)); -+ searcher = new IndexSearcher(DirectoryReader.open(luceneDirectory)); - } - } - } -@@ -820,7 +889,7 @@ - if (searches.isEmpty()) { - if (searcher != null) { - try { -- searcher.close(); -+ searcher.getIndexReader().close(); - } catch (IOException ioe) { - } - } -@@ -838,7 +907,8 @@ - * Finds and unzips prebuild index specified in preferences - */ - private void unzipProductIndex() { -- String indexPluginId = Platform.getPreferencesService().getString(HelpBasePlugin.PLUGIN_ID, "productIndex", null, null); //$NON-NLS-1$ -+ String indexPluginId = Platform.getPreferencesService().getString(HelpBasePlugin.PLUGIN_ID, -+ "productIndex", null, null); //$NON-NLS-1$ - if (indexPluginId == null || indexPluginId.length() <= 0) { - return; - } -@@ -901,10 +971,11 @@ - */ - private void cleanOldIndex() { - IndexWriter cleaner = null; -- LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), 10000); -+ LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), -+ 10000); - try { -- cleaner = new IndexWriter(luceneDirectory, new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_31, analyzer).setOpenMode( -- OpenMode.CREATE)); -+ cleaner = new IndexWriter(luceneDirectory, -+ new IndexWriterConfig(analyzer).setOpenMode(OpenMode.CREATE)); - } catch (IOException ioe) { - } finally { - try { -@@ -957,7 +1028,7 @@ - * if lock already obtained - */ - public synchronized boolean tryLock() throws OverlappingFileLockException { -- if ("none".equals(System.getProperty("osgi.locking"))) { //$NON-NLS-1$//$NON-NLS-2$ -+ if ("none".equals(System.getProperty("osgi.locking"))) { //$NON-NLS-1$//$NON-NLS-2$ - return true; // Act as if lock succeeded - } - if (lock != null) { -@@ -979,7 +1050,7 @@ - lock = null; - logLockFailure(ioe); - } -- if ( raf != null ) { -+ if (raf != null) { - try { - raf.close(); - } catch (IOException e) { -@@ -1025,9 +1096,9 @@ - } - lock = null; - } -- if (raf != null ) { -+ if (raf != null) { - try { -- raf.close(); -+ raf.close(); - } catch (IOException ioe) { - } - raf = null; -@@ -1100,25 +1171,26 @@ - - public IStatus addDocument(String pluginId, String name, URL url, String id, Document doc) { - // try a registered participant for the file format -- SearchParticipant participant = BaseHelpSystem.getLocalSearchManager() -- .getParticipant(pluginId, name); -+ SearchParticipant participant = BaseHelpSystem.getLocalSearchManager().getParticipant(pluginId, name); - if (participant != null) { - try { - return participant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc)); -- } -- catch (Throwable t) { -+ } catch (Throwable t) { - return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR, - "Error while adding document to search participant (addDocument()): " //$NON-NLS-1$ -- + name + ", " + url + "for participant " + participant.getClass().getName(), t); //$NON-NLS-1$ //$NON-NLS-2$ -+ + name + ", " + url + "for participant " + participant.getClass().getName(), //$NON-NLS-1$ //$NON-NLS-2$ -+ t); - } - } - // default to html -- return htmlSearchParticipant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc)); -+ return htmlSearchParticipant.addDocument(this, pluginId, name, url, id, -+ new LuceneSearchDocument(doc)); - } - -- public IStatus addSearchableDocument(String pluginId, String name, URL url, String id, ISearchDocument doc) { -+ public IStatus addSearchableDocument(String pluginId, String name, URL url, String id, -+ ISearchDocument doc) { - // In the help system the only class that implements ISearchDocument is LuceneSearchDocument -- LuceneSearchDocument luceneDoc = (LuceneSearchDocument)doc; -+ LuceneSearchDocument luceneDoc = (LuceneSearchDocument) doc; - return addDocument(pluginId, name, url, id, luceneDoc.getDocument()); - } - } -diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java -index d0a7bb7..444b66c 100644 ---- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java -+++ b/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java -@@ -1,5 +1,5 @@ - /******************************************************************************* -- * Copyright (c) 2000, 2012 IBM Corporation and others. -+ * Copyright (c) 2000, 2015 IBM Corporation and others. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at -@@ -7,10 +7,9 @@ - * - * Contributors: - * IBM Corporation - initial API and implementation -+ * Sopot Cela - Bug 466829 - Migration to Lucene 5 - *******************************************************************************/ - package org.eclipse.help.internal.search; -- --import java.io.*; - - import org.apache.lucene.analysis.*; - -@@ -18,7 +17,7 @@ - * Smart Analyzer. Chooses underlying implementation based on the field which - * text is analyzed. - */ --public final class SmartAnalyzer extends Analyzer { -+public final class SmartAnalyzer extends AnalyzerWrapper { - Analyzer pluggedInAnalyzer; - Analyzer exactAnalyzer; - -@@ -26,19 +25,19 @@ - * Constructor for SmartAnalyzer. - */ - public SmartAnalyzer(String locale, Analyzer pluggedInAnalyzer) { -- super(); -+ super(pluggedInAnalyzer.getReuseStrategy()); - this.pluggedInAnalyzer = pluggedInAnalyzer; - this.exactAnalyzer = new DefaultAnalyzer(locale); - } - /** -- * Creates a TokenStream which tokenizes all the text in the provided -- * Reader. Delegates to DefaultAnalyzer when field used to search for exact -+ * Delegates to DefaultAnalyzer when field used to search for exact - * match, and to plugged-in analyzer for other fields. - */ -- public final TokenStream tokenStream(String fieldName, Reader reader) { -+ @Override -+ public final Analyzer getWrappedAnalyzer(String fieldName) { - if (fieldName != null && fieldName.startsWith("exact_")) { //$NON-NLS-1$ -- return exactAnalyzer.tokenStream(fieldName, reader); -+ return exactAnalyzer; - } -- return pluggedInAnalyzer.tokenStream(fieldName, reader); -+ return pluggedInAnalyzer; - } - } -diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java -index 0b70cf7..79d5592 100644 ---- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java -+++ b/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java -@@ -1,5 +1,5 @@ - /******************************************************************************* -- * Copyright (c) 2000, 2012 IBM Corporation and others. -+ * Copyright (c) 2000, 2015 IBM Corporation and others. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at -@@ -7,16 +7,18 @@ - * - * Contributors: - * IBM Corporation - initial API and implementation -+ * Sopot Cela - Bug 466829 - Migration to Lucene 5 - *******************************************************************************/ - package org.eclipse.help.internal.search; - --import com.ibm.icu.text.BreakIterator; - import java.io.IOException; --import java.io.Reader; - import java.util.Locale; -+ - import org.apache.lucene.analysis.TokenStream; - import org.apache.lucene.analysis.Tokenizer; - import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -+ -+import com.ibm.icu.text.BreakIterator; - - /** - * WordTokenStream obtains tokens containing words appropriate for use with -@@ -24,7 +26,6 @@ - */ - public final class WordTokenStream extends Tokenizer { - private static final int BUF_LEN = 4096; -- private final Reader reader; - private final BreakIterator boundary; - private StringBuffer strbuf; - -@@ -34,8 +35,8 @@ - /** - * Constructor - */ -- public WordTokenStream(String fieldName, Reader reader, Locale locale) { -- this.reader = reader; -+ public WordTokenStream(Locale locale) { -+ super(); - boundary = BreakIterator.getWordInstance(locale); - - } -@@ -52,9 +53,9 @@ - if(strbuf == null) { - int available; - char[] cbuf = new char[BUF_LEN]; -- while ((available = reader.read(cbuf)) <= 0) { -+ while ((available = input.read(cbuf)) <= 0) { - if (available < 0) { -- reader.close(); -+ input.close(); - return false; - } - } -@@ -62,7 +63,7 @@ - strbuf.append(cbuf, 0, available); - // read more until white space (or EOF) - int c; -- while (0 <= (c = reader.read())) { -+ while (0 <= (c = input.read())) { - strbuf.append((char) c); - if (c == ' ' || c == '\r' || c == '\n' || c == '\t') { - break; -@@ -70,7 +71,7 @@ - } - - if (c < 0) { -- reader.close(); -+ input.close(); - } - - boundary.setText(strbuf.toString()); -@@ -105,9 +106,10 @@ - } - - public void close() throws IOException { -+ super.close(); - /// Unlikely to be called as this is a reused -- if (this.reader != null) { -- this.reader.close(); -+ if (this.input != null) { -+ this.input.close(); - } - } - } -diff --git a/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml b/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml -index 8438100..174e92a 100644 ---- a/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml -+++ b/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml -@@ -74,20 +74,6 @@ - unpack="false"/> - - -- -- -- -- - -+ -+ -+ -+ -+ -+ -+ -+ - -diff --git a/org.eclipse.ua.tests/META-INF/MANIFEST.MF b/org.eclipse.ua.tests/META-INF/MANIFEST.MF -index 6bcf9bc..23f8910 100644 ---- a/org.eclipse.ua.tests/META-INF/MANIFEST.MF -+++ b/org.eclipse.ua.tests/META-INF/MANIFEST.MF -@@ -19,14 +19,13 @@ - org.eclipse.ui.forms, - org.eclipse.ui.browser;bundle-version="3.2.300", - org.eclipse.equinox.jsp.jasper;bundle-version="1.0.200", -- org.eclipse.equinox.jsp.jasper.registry;bundle-version="1.0.100" -+ org.eclipse.equinox.jsp.jasper.registry;bundle-version="1.0.100", -+ org.apache.lucene.analyzers-common;bundle-version="5.1.0", -+ org.apache.lucene.core;bundle-version="5.1.0" - Bundle-ActivationPolicy: lazy - Bundle-Vendor: Eclipse.org - Import-Package: javax.servlet;version="3.1.0", -- javax.servlet.http;version="3.1.0", -- org.apache.lucene.index;core=split;version="[3.5.0,4.0.0)", -- org.apache.lucene.search;core=split;version="[3.5.0,4.0.0)", -- org.apache.lucene.store;core=split;version="[3.5.0,4.0.0)" -+ javax.servlet.http;version="3.1.0" - Bundle-RequiredExecutionEnvironment: JavaSE-1.6 - Export-Package: org.eclipse.ua.tests, - org.eclipse.ua.tests.browser, -diff --git a/org.eclipse.ua.tests/data/help/searchindex/index291/write.lock b/org.eclipse.ua.tests/data/help/searchindex/index291/write.lock -new file mode 100644 -index 0000000..e69de29 ---- /dev/null -+++ b/org.eclipse.ua.tests/data/help/searchindex/index291/write.lock -diff --git a/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfe b/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfe -new file mode 100644 -index 0000000..09ec2c9 ---- /dev/null -+++ b/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfe -Binary files differ -diff --git a/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfs b/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfs -new file mode 100644 -index 0000000..3aa288a ---- /dev/null -+++ b/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfs -Binary files differ -diff --git a/org.eclipse.ua.tests/data/help/searchindex/index510/_0.si b/org.eclipse.ua.tests/data/help/searchindex/index510/_0.si -new file mode 100644 -index 0000000..d897bfc ---- /dev/null -+++ b/org.eclipse.ua.tests/data/help/searchindex/index510/_0.si -Binary files differ -diff --git a/org.eclipse.ua.tests/data/help/searchindex/index510/segments_1 b/org.eclipse.ua.tests/data/help/searchindex/index510/segments_1 -new file mode 100644 -index 0000000..4878901 ---- /dev/null -+++ b/org.eclipse.ua.tests/data/help/searchindex/index510/segments_1 -Binary files differ -diff --git a/org.eclipse.ua.tests/data/help/searchindex/index510/write.lock b/org.eclipse.ua.tests/data/help/searchindex/index510/write.lock -new file mode 100644 -index 0000000..e69de29 ---- /dev/null -+++ b/org.eclipse.ua.tests/data/help/searchindex/index510/write.lock -diff --git a/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/EncodedCharacterSearch.java b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/EncodedCharacterSearch.java -index 1dd4876..02b2e5b 100644 ---- a/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/EncodedCharacterSearch.java -+++ b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/EncodedCharacterSearch.java -@@ -7,6 +7,7 @@ - * - * Contributors: - * IBM Corporation - initial API and implementation -+ * Sopot Cela - Bug 466829 - Migration to Lucene 5 - *******************************************************************************/ - package org.eclipse.ua.tests.help.search; - -@@ -43,7 +44,7 @@ - - public void testUtf8Hebrew() { - SearchTestUtils.searchOneLocale("\u05D0\u05B7\u05E1\u05B0\u05D8\u05B0\u05E8\u05D5\u05B9\u05E0\u05D5\u05B9\u05DE" -- + "\u05B0\u05D9\u05B8\u05D4) \u05DC\u05B4\u05E7\u05BC\u05D5\u05BC\u05D9 (\u05D9\u05E8\u05D7 \u05D0\u05D5 \u05E9\u05DE\u05E9", new String[] {"/org.eclipse.ua.tests/data/help/search/testnlUTF8.htm" }, "en"); -+ + "\u05B0\u05D9\u05B8\u05D4\\) \u05DC\u05B4\u05E7\u05BC\u05D5\u05BC\u05D9 \\(\u05D9\u05E8\u05D7 \u05D0\u05D5 \u05E9\u05DE\u05E9", new String[] {"/org.eclipse.ua.tests/data/help/search/testnlUTF8.htm" }, "en"); - } - - -diff --git a/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java -index 640d4c9..33ae5ba 100644 ---- a/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java -+++ b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java -@@ -8,6 +8,7 @@ - * Contributors: - * IBM Corporation - initial API and implementation - * Alexander Kurtakov - Bug 460787 -+ * Sopot Cela - Bug 466829 - Migration to Lucene 5 - *******************************************************************************/ - - package org.eclipse.ua.tests.help.search; -@@ -15,43 +16,38 @@ - import java.io.File; - import java.io.IOException; - import java.net.URL; --import java.util.ArrayList; - --import junit.framework.Test; --import junit.framework.TestCase; --import junit.framework.TestSuite; -- --import org.osgi.framework.Bundle; -- -+import org.apache.lucene.analysis.standard.StandardAnalyzer; - import org.apache.lucene.index.CorruptIndexException; --import org.apache.lucene.index.IndexReader; -+import org.apache.lucene.index.DirectoryReader; - import org.apache.lucene.search.IndexSearcher; - import org.apache.lucene.search.Query; - import org.apache.lucene.search.TopDocs; - import org.apache.lucene.store.Directory; - import org.apache.lucene.store.NIOFSDirectory; -- --import org.eclipse.help.internal.base.BaseHelpSystem; --import org.eclipse.help.internal.search.AnalyzerDescriptor; --import org.eclipse.help.internal.search.PluginIndex; --import org.eclipse.help.internal.search.QueryBuilder; --import org.eclipse.help.internal.search.SearchIndexWithIndexingProgress; --import org.eclipse.ua.tests.plugin.UserAssistanceTestPlugin; -- -+import org.apache.lucene.util.QueryBuilder; - import org.eclipse.core.runtime.FileLocator; - import org.eclipse.core.runtime.NullProgressMonitor; - import org.eclipse.core.runtime.Path; -+import org.eclipse.help.internal.base.BaseHelpSystem; -+import org.eclipse.help.internal.search.PluginIndex; -+import org.eclipse.help.internal.search.SearchIndexWithIndexingProgress; -+import org.eclipse.ua.tests.plugin.UserAssistanceTestPlugin; -+import org.osgi.framework.Bundle; -+ -+import junit.framework.Test; -+import junit.framework.TestCase; -+import junit.framework.TestSuite; - - /** -- * Verify that older versions of the index can be read by this -- * version of Eclipse. -+ * Verify that older versions of the index can be read by this version of -+ * Eclipse. - * -- * How to maintain this test - if when upgrading to a new version -- * of Lucene one of the IndexReadable tests fails you need to -- * make the following changes: -- * 1. Change the corresponding Compatible() test to expect a result of false -- * 2. Comment out the failing test -- * 3. Change the help system to recognize that version of Lucene as being incompatible -+ * How to maintain this test - if when upgrading to a new version of Lucene one -+ * of the IndexReadable tests fails you need to make the following changes: 1. -+ * Change the corresponding Compatible() test to expect a result of false 2. -+ * Comment out the failing test 3. Change the help system to recognize that -+ * version of Lucene as being incompatible - */ - - public class PrebuiltIndexCompatibility extends TestCase { -@@ -64,48 +60,33 @@ - } - - /** -- * Test index built with Lucene 1.9.1 -- */ -- public void test1_9_1_IndexReadable() throws Exception { -- checkReadable("data/help/searchindex/index191"); -- } -- -- /** -- * Test index built with Lucene 2.9.1 -- */ -- public void test2_9_1_IndexReadable() throws Exception { -- checkReadable("data/help/searchindex/index291"); -- } -- -- /** - ** Test compatibility of Lucene 1.9.1 index with current Lucene - */ -- public void test1_9_1Compatible() -- { -- checkCompatible("data/help/searchindex/index191", true); -+ public void test1_9_1Compatible() { -+ checkCompatible("data/help/searchindex/index191", false); - } -- -+ - /** - ** Test compatibility of Lucene 2.9.1 index with current Lucene - */ -- public void test2_9_1Compatible() -- { -- checkCompatible("data/help/searchindex/index291", true); -+ public void test2_9_1Compatible() { -+ checkCompatible("data/help/searchindex/index291", false); - } - -- public void test1_9_1LuceneCompatible() -- { -- checkLuceneCompatible("1.9.1", true); -+ public void test1_9_1LuceneCompatible() { -+ checkLuceneCompatible("1.9.1", false); - } - -- public void test1_4_103NotLuceneCompatible() -- { -+ public void test1_4_103NotLuceneCompatible() { - checkLuceneCompatible("1.4.103", false); - } - -- public void test2_9_1LuceneCompatible() -- { -- checkLuceneCompatible("2.9.1", true); -+ public void test2_9_1LuceneCompatible() { -+ checkLuceneCompatible("2.9.1", false); -+ } -+ -+ public void test5_1_0LuceneCompatible() { -+ checkLuceneCompatible("5.1.0", true); - } - - public void testPluginIndexEqualToItself() { -@@ -114,52 +95,59 @@ - } - - /** -- * Verify that if the paths and plugins are the same two PluginIndex objects are equal -+ * Verify that if the paths and plugins are the same two PluginIndex objects -+ * are equal - */ - public void testPluginIndexEquality() { -- PluginIndex index1a = createPluginIndex("data/help/searchindex/index191"); -- PluginIndex index1b = createPluginIndex("data/help/searchindex/index191"); -+ PluginIndex index1a = createPluginIndex("data/help/searchindex/index510"); -+ PluginIndex index1b = createPluginIndex("data/help/searchindex/index510"); - assertTrue(index1a.equals(index1b)); - } -- -+ - /** -- * Verify that if the paths and plugins are the same two PluginIndex objects are equal -+ * Verify that if the paths and plugins are the same two PluginIndex objects -+ * are equal - */ - public void testPluginIndexHash() { - PluginIndex index1a = createPluginIndex("data/help/searchindex/index191"); - PluginIndex index1b = createPluginIndex("data/help/searchindex/index191"); - assertEquals(index1a.hashCode(), index1b.hashCode()); - } -- -+ - /** -- * Verify that if the paths are different two PluginIndex objects are not equal -+ * Verify that if the paths are different two PluginIndex objects are not -+ * equal - */ - public void testPluginIndexInequality() { - PluginIndex index1 = createPluginIndex("data/help/searchindex/index191"); - PluginIndex index2 = createPluginIndex("data/help/searchindex/index291"); - assertFalse(index1.equals(index2)); - } -+ -+ public void test5_1_0IndexReadable() throws CorruptIndexException, IOException { -+ checkReadable("data/help/searchindex/index510"); -+ } - -- /* -- * Verifies that a prebuilt index can be searched -- */ -- private void checkReadable(String indexPath) throws IOException, -- CorruptIndexException { -+ /* -+ * Verifies that a prebuilt index can be searched -+ */ -+ private void checkReadable(String indexPath) throws IOException, CorruptIndexException { - Path path = new Path(indexPath); -- Bundle bundle = UserAssistanceTestPlugin.getDefault().getBundle(); -+ Bundle bundle = UserAssistanceTestPlugin.getDefault().getBundle(); - URL url = FileLocator.find(bundle, path, null); - URL resolved = FileLocator.resolve(url); - if ("file".equals(resolved.getProtocol())) { //$NON-NLS-1$ - String filePath = resolved.getFile(); -- QueryBuilder queryBuilder = new QueryBuilder("eclipse", new AnalyzerDescriptor("en-us")); -- Query luceneQuery = queryBuilder.getLuceneQuery(new ArrayList() , false); -+ StandardAnalyzer standardAnalyzer = new StandardAnalyzer(); -+ QueryBuilder builder = new QueryBuilder(standardAnalyzer); -+ Query luceneQuery = builder.createBooleanQuery("contents", "eclipse"); - Directory luceneDirectory = null; - IndexSearcher searcher = null; - try { -- luceneDirectory = new NIOFSDirectory(new File(filePath)); -- searcher = new IndexSearcher(IndexReader.open(luceneDirectory, true)); -+ luceneDirectory = new NIOFSDirectory(new File(filePath).toPath()); -+ searcher = new IndexSearcher(DirectoryReader.open(luceneDirectory)); - TopDocs hits = searcher.search(luceneQuery, 500); -- assertEquals(hits.totalHits, 1); -+ assertEquals(1, hits.totalHits ); - } finally { - if (luceneDirectory != null) - try { -@@ -167,13 +155,13 @@ - } catch (IOException x) { - } - if (searcher != null) -- searcher.close(); -+ searcher.getIndexReader().close(); - } - } else { - fail("Cannot resolve to file protocol"); - } - } -- -+ - /* - * Tests the isCompatible method in PluginIndex - */ -@@ -186,22 +174,20 @@ - public PluginIndex createPluginIndex(String versionDirectory) { - PluginIndex pluginIndex; - SearchIndexWithIndexingProgress index = BaseHelpSystem.getLocalSearchManager().getIndex("en_us".toString()); -- BaseHelpSystem.getLocalSearchManager().ensureIndexUpdated( -- new NullProgressMonitor(), -- index); -+ BaseHelpSystem.getLocalSearchManager().ensureIndexUpdated(new NullProgressMonitor(), index); - pluginIndex = new PluginIndex("org.eclipse.ua.tests", "data/help/searchindex/" + versionDirectory, index); - return pluginIndex; - } -- -+ - /* -- * Tests the isLuceneCompatible method in SearchIndex -+ * Tests the isLuceneCompatible method in SearchIndex - */ - private void checkLuceneCompatible(String version, boolean expected) { - SearchIndexWithIndexingProgress index = BaseHelpSystem.getLocalSearchManager().getIndex("en_us".toString()); -- BaseHelpSystem.getLocalSearchManager().ensureIndexUpdated( -- new NullProgressMonitor(), -- index); -+ BaseHelpSystem.getLocalSearchManager().ensureIndexUpdated(new NullProgressMonitor(), index); - assertEquals(expected, index.isLuceneCompatible(version)); - } - -+ -+ - } -diff --git a/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/SearchParticipantTest.java b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/SearchParticipantTest.java -index 223e42a..2e782c3 100644 ---- a/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/SearchParticipantTest.java -+++ b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/SearchParticipantTest.java -@@ -1,5 +1,5 @@ - /******************************************************************************* -- * Copyright (c) 2010 IBM Corporation and others. -+ * Copyright (c) 2010, 2015 IBM Corporation and others. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at -@@ -7,6 +7,7 @@ - * - * Contributors: - * IBM Corporation - initial API and implementation -+ * Sopot Cela - Bug 466829 - Migration to Lucene 5 - *******************************************************************************/ - package org.eclipse.ua.tests.help.search; - -@@ -40,7 +41,7 @@ - } - - public void testSearchUsingAndInSeparateDocs() { -- SearchTestUtils.searchAllLocales("jduehdye and olhoykk", new String[0]); -+ SearchTestUtils.searchAllLocales("jduehdye AND olhoykk", new String[0]); - } - - public void testSearchExactMatch() { diff --git a/eclipse-bug-466499.patch b/eclipse-bug-466499.patch deleted file mode 100644 index 196f971..0000000 --- a/eclipse-bug-466499.patch +++ /dev/null @@ -1,90 +0,0 @@ -From a41f152c868984b41b73945f374966ad621c3e13 Mon Sep 17 00:00:00 2001 -From: Snjezana Peco -Date: Thu, 21 May 2015 10:41:39 +0200 -Subject: Bug 466499 - [GTK3] Path text not drawn in column of commit dialog - window - -Change-Id: I508432a354314ca11839e03fbac7b5adc44fff3c -Signed-off-by: Snjezana Peco ---- - .../Eclipse SWT/gtk/org/eclipse/swt/widgets/Table.java | 3 ++- - .../Eclipse SWT/gtk/org/eclipse/swt/widgets/Tree.java | 17 ++++------------- - 2 files changed, 6 insertions(+), 14 deletions(-) - -diff --git a/eclipse.platform.swt/bundles/org.eclipse.swt/Eclipse SWT/gtk/org/eclipse/swt/widgets/Table.java b/eclipse.platform.swt/bundles/org.eclipse.swt/Eclipse SWT/gtk/org/eclipse/swt/widgets/Table.java -index e49865b..9b904ad 100644 ---- a/eclipse.platform.swt/bundles/org.eclipse.swt/Eclipse SWT/gtk/org/eclipse/swt/widgets/Table.java -+++ b/eclipse.platform.swt/bundles/org.eclipse.swt/Eclipse SWT/gtk/org/eclipse/swt/widgets/Table.java -@@ -2662,7 +2662,8 @@ void rendererRender (long /*int*/ cell, long /*int*/ cr, long /*int*/ window, lo - } - } - if (item != null) { -- if (OS.GTK_IS_CELL_RENDERER_TOGGLE (cell) || (OS.GTK_IS_CELL_RENDERER_PIXBUF (cell) && (columnIndex != 0 || (style & SWT.CHECK) == 0))) { -+ if (OS.GTK_IS_CELL_RENDERER_TOGGLE (cell) || -+ ( (OS.GTK_IS_CELL_RENDERER_PIXBUF (cell) || OS.GTK_VERSION > OS.VERSION(3, 13, 0)) && (columnIndex != 0 || (style & SWT.CHECK) == 0))) { - drawFlags = (int)/*64*/flags; - drawState = SWT.FOREGROUND; - long /*int*/ [] ptr = new long /*int*/ [1]; -diff --git a/eclipse.platform.swt/bundles/org.eclipse.swt/Eclipse SWT/gtk/org/eclipse/swt/widgets/Tree.java b/eclipse.platform.swt/bundles/org.eclipse.swt/Eclipse SWT/gtk/org/eclipse/swt/widgets/Tree.java -index 4d5448a..bc64152 100644 ---- a/eclipse.platform.swt/bundles/org.eclipse.swt/Eclipse SWT/gtk/org/eclipse/swt/widgets/Tree.java -+++ b/eclipse.platform.swt/bundles/org.eclipse.swt/Eclipse SWT/gtk/org/eclipse/swt/widgets/Tree.java -@@ -2675,7 +2675,8 @@ void rendererRender (long /*int*/ cell, long /*int*/ cr, long /*int*/ window, lo - } - } - if (item != null) { -- if (OS.GTK_IS_CELL_RENDERER_TOGGLE (cell) || (OS.GTK_IS_CELL_RENDERER_PIXBUF (cell) && (columnIndex != 0 || (style & SWT.CHECK) == 0))) { -+ if (OS.GTK_IS_CELL_RENDERER_TOGGLE (cell) || -+ ( (OS.GTK_IS_CELL_RENDERER_PIXBUF (cell) || OS.GTK_VERSION > OS.VERSION(3, 13, 0)) && (columnIndex != 0 || (style & SWT.CHECK) == 0))) { - drawFlags = (int)/*64*/flags; - drawState = SWT.FOREGROUND; - long /*int*/ [] ptr = new long /*int*/ [1]; -@@ -2775,9 +2776,6 @@ void rendererRender (long /*int*/ cell, long /*int*/ cr, long /*int*/ window, lo - sendEvent (SWT.EraseItem, event); - drawForeground = null; - drawState = event.doit ? event.detail : 0; -- if (OS.GTK3 && OS.GTK_VERSION <= OS.VERSION(3, 14, 8)) { -- drawState |= SWT.FOREGROUND; -- } - drawFlags &= ~(OS.GTK_CELL_RENDERER_FOCUSED | OS.GTK_CELL_RENDERER_SELECTED); - if ((drawState & SWT.SELECTED) != 0) drawFlags |= OS.GTK_CELL_RENDERER_SELECTED; - if ((drawState & SWT.FOCUSED) != 0) drawFlags |= OS.GTK_CELL_RENDERER_FOCUSED; -@@ -2821,10 +2819,8 @@ void rendererRender (long /*int*/ cell, long /*int*/ cr, long /*int*/ window, lo - if (OS.GTK_IS_CELL_RENDERER_TEXT (cell)) { - if (hooks (SWT.PaintItem)) { - GdkRectangle rect = new GdkRectangle (); -- GdkRectangle clipRect = new GdkRectangle (); - long /*int*/ path = OS.gtk_tree_model_get_path (modelHandle, iter); - OS.gtk_tree_view_get_cell_area (handle, path, columnHandle, rect); -- OS.gtk_tree_view_get_background_area (handle, path, columnHandle, clipRect); - OS.gtk_tree_path_free (path); - // A workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=459117 - if (cr != 0 && OS.GTK_VERSION > OS.VERSION(3, 9, 0) && OS.GTK_VERSION <= OS.VERSION(3, 14, 8)) { -@@ -2877,12 +2873,9 @@ void rendererRender (long /*int*/ cell, long /*int*/ cr, long /*int*/ window, lo - gc.setFont (item.getFont (columnIndex)); - if ((style & SWT.MIRRORED) != 0) { - rect.x = getClientWidth () - rect.width - rect.x; -- clipRect.x = getClientWidth () - clipRect.width - clipRect.x; - } - -- if (!OS.GTK3){ -- gc.setClipping (clipRect.x, clipRect.y, clipRect.width, clipRect.height); -- } -+ gc.setClipping (rect.x, rect.y, rect.width, rect.height); - - Event event = new Event (); - event.item = item; -@@ -2893,9 +2886,7 @@ void rendererRender (long /*int*/ cell, long /*int*/ cr, long /*int*/ window, lo - event.width = contentWidth [0]; - event.height = rect.height; - event.detail = drawState; -- if (!OS.GTK3 || OS.GTK_VERSION > OS.VERSION(3, 14, 8)) { -- sendEvent (SWT.PaintItem, event); -- } -+ sendEvent(SWT.PaintItem, event); - gc.dispose(); - } - } --- -cgit v0.11.2-4-g4a35 - diff --git a/eclipse-change-build-packagings.patch b/eclipse-change-build-packagings.patch index 7b7beb3..62f62eb 100644 --- a/eclipse-change-build-packagings.patch +++ b/eclipse-change-build-packagings.patch @@ -1,23 +1,23 @@ --- eclipse.platform.releng/features/org.eclipse.sdk/feature.xml.orig 2014-04-08 15:47:19.927087084 +0100 +++ eclipse.platform.releng/features/org.eclipse.sdk/feature.xml 2014-04-08 15:48:00.045367311 +0100 -@@ -19,7 +19,7 @@ - %license - +@@ -59,7 +59,7 @@ + id="org.eclipse.help.source" + version="0.0.0"/> - + - - + + + -+ ++ + + + @@ -385,7 +385,7 @@ index 6f6526b..c05d703 100644 + + + -+ ++ + + + diff --git a/eclipse-fix-tests.patch b/eclipse-fix-tests.patch index 6f45b07..4b24a01 100644 --- a/eclipse-fix-tests.patch +++ b/eclipse-fix-tests.patch @@ -31,3 +31,85 @@ index f1f38f0..f9111e6 100644 } }); if (children != null && children.length == 1) +diff --git eclipse.jdt.core/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/LambdaExpressionsTest.java eclipse.jdt.core/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/LambdaExpressionsTest.java +index 0245d89..e95ff8f 100644 +--- eclipse.jdt.core/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/LambdaExpressionsTest.java ++++ eclipse.jdt.core/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/LambdaExpressionsTest.java +@@ -4416,6 +4416,10 @@ public void test434297() { + public void test436542() throws Exception { + String jreDirectory = Util.getJREDirectory(); + String jfxJar = Util.toNativePath(jreDirectory + "/lib/ext/jfxrt.jar"); ++ File file = new File(jfxJar); ++ if (!file.exists()) { ++ return; ++ } + this.runConformTest( + new String[] { + "Utility.java", +diff --git eclipse.jdt.core/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/NullTypeAnnotationTest.java eclipse.jdt.core/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/NullTypeAnnotationTest.java +index 1a0cbcc..ddad62f 100644 +--- eclipse.jdt.core/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/NullTypeAnnotationTest.java ++++ eclipse.jdt.core/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/NullTypeAnnotationTest.java +@@ -6781,6 +6781,10 @@ public void testBug434582a() { + public void test443467() throws Exception { + String jreDirectory = Util.getJREDirectory(); + String jfxJar = Util.toNativePath(jreDirectory + "/lib/ext/jfxrt.jar"); ++ File file = new File(jfxJar); ++ if (!file.exists()) { ++ return; ++ } + this.runNegativeTestWithExtraLibs( + new String[] { + "BuildIdeMain.java", +diff --git eclipse.jdt.core/org.eclipse.jdt.apt.pluggable.tests/build.properties eclipse.jdt.core/org.eclipse.jdt.apt.pluggable.tests/build.properties +index 376daa6..1742a8d 100644 +--- eclipse.jdt.core/org.eclipse.jdt.apt.pluggable.tests/build.properties ++++ eclipse.jdt.core/org.eclipse.jdt.apt.pluggable.tests/build.properties +@@ -14,6 +14,7 @@ bin.includes = .,\ + resources/,\ + lib/ + src.includes = about.html +-jars.compile.order = . ++jars.compile.order = .,\ ++ lib/annotations.jar + source.. = src/ +-output.. = bin/ ++source.lib/annotations.jar = src/ +diff --git eclipse.jdt.core/org.eclipse.jdt.apt.tests/build.properties eclipse.jdt.core/org.eclipse.jdt.apt.tests/build.properties +index f34ae40..0f9877d 100644 +--- eclipse.jdt.core/org.eclipse.jdt.apt.tests/build.properties ++++ eclipse.jdt.core/org.eclipse.jdt.apt.tests/build.properties +@@ -21,4 +21,6 @@ bin.includes = plugin.xml,\ + plugin.properties + src.includes = about.html + src.excludes = src-resources/ +-jars.compile.order = aptext.jar,apt.jar,. ++jars.compile.order = aptext.jar,apt.jar,.,\ ++ resources/question.jar ++source.resources/question.jar = src-resources/ +diff --git eclipse.jdt.core/org.eclipse.jdt.compiler.apt.tests/build.properties eclipse.jdt.core/org.eclipse.jdt.compiler.apt.tests/build.properties +index bc8963d..fbe4da3 100644 +--- eclipse.jdt.core/org.eclipse.jdt.compiler.apt.tests/build.properties ++++ eclipse.jdt.core/org.eclipse.jdt.compiler.apt.tests/build.properties +@@ -10,11 +10,6 @@ + # Red Hat Inc. (mistria) - Avoid nested jars + ############################################################################### + source.. = src/ +-output.. = bin/ +-# Don't rebuild apttestprocessors everytime to avoid dirty working tree +-# lib/apttestprocessors.jar considered as a resource set a dev-timeY +-#source.lib/apttestprocessors.jar = processors/ +-#jars.compile.order = lib/apttestprocessors.jar,. + bin.includes = about.html,\ + plugin.properties,\ + test.xml,\ +@@ -24,3 +19,9 @@ bin.includes = about.html,\ + . + src.includes = about.html + compilerArg=-proc:none ++source.lib/apttestprocessors.jar = processors/ ++source.lib/apttestprocessors8.jar = processors/,\ ++ processors8/ ++jars.compile.order = lib/apttestprocessors.jar,\ ++ lib/apttestprocessors8.jar,\ ++ . diff --git a/eclipse-lucene-4.patch b/eclipse-lucene-4.patch new file mode 100644 index 0000000..21a24fa --- /dev/null +++ b/eclipse-lucene-4.patch @@ -0,0 +1,524 @@ +diff --git a/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF b/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF +index ee34c8e..fdef3e6 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF ++++ b/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF +@@ -43,8 +43,8 @@ Require-Bundle: org.eclipse.ant.core;bundle-version="3.2.200";resolution:=option + org.eclipse.core.runtime;bundle-version="[3.11.0,4.0.0)", + org.eclipse.help;bundle-version="[3.5.0,4.0.0)";visibility:=reexport, + org.eclipse.core.expressions;bundle-version="[3.4.200,4.0.0)", +- org.apache.lucene.analysis;bundle-version="[3.5.0,4.0.0)", +- org.apache.lucene.core;bundle-version="[3.5.0,4.0.0)", ++ org.apache.lucene.analysis;bundle-version="[4.7.0,5.0.0)", ++ org.apache.lucene.core;bundle-version="[4.7.0,5.0.0)", + org.eclipse.core.net;bundle-version="1.2.200" + Import-Package: com.ibm.icu.text, + org.eclipse.equinox.http.jetty;resolution:=optional +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java +index a066aa4..6c41103 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java +@@ -11,10 +11,15 @@ + *******************************************************************************/ + package org.eclipse.help.internal.search; + import java.io.*; +-import java.util.HashSet; +-import java.util.Set; ++import java.util.ArrayList; ++import java.util.List; + +-import org.apache.lucene.analysis.*; ++import org.apache.lucene.analysis.Analyzer; ++import org.apache.lucene.analysis.TokenStream; ++import org.apache.lucene.analysis.Tokenizer; ++import org.apache.lucene.analysis.core.StopFilter; ++import org.apache.lucene.analysis.en.PorterStemFilter; ++import org.apache.lucene.analysis.util.CharArraySet; + import org.apache.lucene.util.Version; + /** + * Lucene Analyzer for English. LowerCaseTokenizer->StopFilter->PorterStemFilter +@@ -27,18 +32,22 @@ public final class Analyzer_en extends Analyzer { + super(); + } + /** +- * Creates a TokenStream which tokenizes all the text in the provided ++ * Creates a TokenStreamComponents which tokenizes all the text in the provided + * Reader. + */ +- public final TokenStream tokenStream(String fieldName, Reader reader) { +- return new PorterStemFilter(new StopFilter(Version.LUCENE_30, new LowerCaseAndDigitsTokenizer(reader), getStopWords(), false)); ++ @Override ++ public final TokenStreamComponents createComponents(String fieldName, Reader reader) { ++ CharArraySet stopWordsSet = StopFilter.makeStopSet(Version.LUCENE_47, getStopWords(), true); ++ Tokenizer source = new LowerCaseAndDigitsTokenizer(reader); ++ TokenStream filter = new PorterStemFilter(new StopFilter(Version.LUCENE_47, source, stopWordsSet)); ++ return new TokenStreamComponents(source, filter); + } + +- private Set stopWords; ++ private List stopWords; + +- private Set getStopWords() { ++ private List getStopWords() { + if ( stopWords == null ) { +- stopWords = new HashSet(); ++ stopWords = new ArrayList(); + for (int i = 0; i < STOP_WORDS.length; i++) { + stopWords.add(STOP_WORDS[i]); + } +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java +index 4109474..2718307 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java +@@ -17,8 +17,8 @@ import java.util.StringTokenizer; + import com.ibm.icu.text.BreakIterator; + + import org.apache.lucene.analysis.Analyzer; +-import org.apache.lucene.analysis.LowerCaseFilter; +-import org.apache.lucene.analysis.TokenStream; ++import org.apache.lucene.analysis.Tokenizer; ++import org.apache.lucene.analysis.core.LowerCaseFilter; + import org.apache.lucene.util.Version; + import org.eclipse.help.internal.base.HelpBasePlugin; + +@@ -84,11 +84,14 @@ public final class DefaultAnalyzer extends Analyzer { + } + + /** +- * Creates a TokenStream which tokenizes all the text in the provided ++ * Creates a TokenStreamComponents which tokenizes all the text in the provided + * Reader. + */ +- public final TokenStream tokenStream(String fieldName, Reader reader) { +- return new LowerCaseFilter(Version.LUCENE_30, new WordTokenStream(fieldName, reader, locale)); ++ @Override ++ public final TokenStreamComponents createComponents(String fieldName, Reader reader) { ++ Tokenizer source = new WordTokenStream(fieldName, reader, locale); ++ LowerCaseFilter filter = new LowerCaseFilter(Version.LUCENE_47, source); ++ return new TokenStreamComponents(source, filter); + } + + /** +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java +index a475688..91e3cb4 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java +@@ -13,7 +13,7 @@ package org.eclipse.help.internal.search; + + import java.io.*; + +-import org.apache.lucene.analysis.*; ++import org.apache.lucene.analysis.util.CharTokenizer; + import org.apache.lucene.util.Version; + + /** +@@ -22,13 +22,14 @@ import org.apache.lucene.util.Version; + public class LowerCaseAndDigitsTokenizer extends CharTokenizer { + + public LowerCaseAndDigitsTokenizer(Reader input) { +- super(Version.LUCENE_30, input); ++ super(Version.LUCENE_47, input); + } + protected char normalize(char c) { + return Character.toLowerCase(c); + } + +- protected boolean isTokenChar(char c) { ++ @Override ++ public boolean isTokenChar(int c) { + return Character.isLetterOrDigit(c); + } + +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java +index 9cc690e..6449adb 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java +@@ -243,16 +243,25 @@ public class QueryBuilder { + private List analyzeText(Analyzer analyzer, String fieldName, String text) { + List words = new ArrayList(1); + Reader reader = new StringReader(text); +- TokenStream tStream = analyzer.tokenStream(fieldName, reader); + +- CharTermAttribute termAttribute = tStream.getAttribute(CharTermAttribute.class); ++ TokenStream tStream = null; + try { ++ tStream = analyzer.tokenStream(fieldName, reader); ++ tStream.reset(); ++ CharTermAttribute termAttribute = (CharTermAttribute) tStream ++ .getAttribute(CharTermAttribute.class); + while (tStream.incrementToken()) { + String term = termAttribute.toString(); + words.add(term); + } +- reader.close(); + } catch (IOException ioe) { ++ } finally { ++ if (tStream != null) { ++ try { ++ tStream.close(); ++ } catch (IOException e) { ++ } ++ } + } + + return words; +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java +index 33c9476..5a46fe5 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java +@@ -33,17 +33,20 @@ import java.util.Set; + import java.util.zip.ZipEntry; + import java.util.zip.ZipInputStream; + +-import org.apache.lucene.analysis.LimitTokenCountAnalyzer; ++import org.apache.lucene.analysis.miscellaneous.LimitTokenCountAnalyzer; + import org.apache.lucene.document.Document; + import org.apache.lucene.document.Field; ++import org.apache.lucene.index.AtomicReader; ++import org.apache.lucene.index.DirectoryReader; ++import org.apache.lucene.index.DocsEnum; + import org.apache.lucene.index.IndexReader; + import org.apache.lucene.index.IndexWriter; +-import org.apache.lucene.index.Term; +-import org.apache.lucene.index.TermDocs; + import org.apache.lucene.index.IndexWriterConfig.OpenMode; + import org.apache.lucene.index.IndexWriterConfig; + import org.apache.lucene.index.LogByteSizeMergePolicy; + import org.apache.lucene.index.LogMergePolicy; ++import org.apache.lucene.index.SlowCompositeReaderWrapper; ++import org.apache.lucene.index.Term; + import org.apache.lucene.search.BooleanQuery; + import org.apache.lucene.search.IndexSearcher; + import org.apache.lucene.search.Query; +@@ -283,7 +286,7 @@ public class SearchIndex implements IHelpSearchIndex { + indexedDocs.restore(); + setInconsistent(true); + LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), 1000000); +- IndexWriterConfig writerConfig = new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_31, analyzer); ++ IndexWriterConfig writerConfig = new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_47, analyzer); + writerConfig.setOpenMode(create ? OpenMode.CREATE : OpenMode.APPEND); + LogMergePolicy mergePolicy = new LogByteSizeMergePolicy(); + mergePolicy.setMergeFactor(20); +@@ -307,7 +310,7 @@ public class SearchIndex implements IHelpSearchIndex { + indexedDocs = new HelpProperties(INDEXED_DOCS_FILE, indexDir); + indexedDocs.restore(); + setInconsistent(true); +- ir = IndexReader.open(luceneDirectory, false); ++ ir = DirectoryReader.open(luceneDirectory); + return true; + } catch (IOException e) { + HelpBasePlugin.logError("Exception occurred in search indexing at beginDeleteBatch.", e); //$NON-NLS-1$ +@@ -323,7 +326,7 @@ public class SearchIndex implements IHelpSearchIndex { + if (ir != null) { + ir.close(); + } +- ir = IndexReader.open(luceneDirectory, false); ++ ir = DirectoryReader.open(luceneDirectory); + return true; + } catch (IOException e) { + HelpBasePlugin.logError("Exception occurred in search indexing at beginDeleteBatch.", e); //$NON-NLS-1$ +@@ -341,7 +344,7 @@ public class SearchIndex implements IHelpSearchIndex { + public IStatus removeDocument(String name) { + Term term = new Term(FIELD_NAME, name); + try { +- ir.deleteDocuments(term); ++ iw.deleteDocuments(term); + indexedDocs.remove(name); + } catch (IOException e) { + return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR, +@@ -379,7 +382,7 @@ public class SearchIndex implements IHelpSearchIndex { + * know about this change. Close it so that it gets reloaded next search. + */ + if (searcher != null) { +- searcher.close(); ++ searcher.getIndexReader().close(); + searcher = null; + } + return true; +@@ -411,7 +414,7 @@ public class SearchIndex implements IHelpSearchIndex { + * know about this change. Close it so that it gets reloaded next search. + */ + if (searcher != null) { +- searcher.close(); ++ searcher.getIndexReader().close(); + searcher = null; + } + return true; +@@ -525,18 +528,19 @@ public class SearchIndex implements IHelpSearchIndex { + } + + public IStatus removeDuplicates(String name, String[] index_paths) { +- TermDocs hrefDocs = null; +- TermDocs indexDocs = null; +- Term hrefTerm = new Term(FIELD_NAME, name); + try { ++ AtomicReader ar = SlowCompositeReaderWrapper.wrap(ir); ++ DocsEnum hrefDocs = null; ++ DocsEnum indexDocs = null; ++ Term hrefTerm = new Term(FIELD_NAME, name); + for (int i = 0; i < index_paths.length; i++) { + Term indexTerm = new Term(FIELD_INDEX_ID, index_paths[i]); + if (i == 0) { +- hrefDocs = ir.termDocs(hrefTerm); +- indexDocs = ir.termDocs(indexTerm); ++ hrefDocs = ar.termDocsEnum(hrefTerm); ++ indexDocs = ar.termDocsEnum(indexTerm); + } else { +- hrefDocs.seek(hrefTerm); +- indexDocs.seek(indexTerm); ++ hrefDocs = ar.termDocsEnum(hrefTerm); ++ indexDocs = ar.termDocsEnum(indexTerm); + } + removeDocuments(hrefDocs, indexDocs); + } +@@ -545,19 +549,6 @@ public class SearchIndex implements IHelpSearchIndex { + "IO exception occurred while removing duplicates of document " + name //$NON-NLS-1$ + + " from index " + indexDir.getAbsolutePath() + ".", //$NON-NLS-1$ //$NON-NLS-2$ + ioe); +- } finally { +- if (hrefDocs != null) { +- try { +- hrefDocs.close(); +- } catch (IOException e) { +- } +- } +- if (indexDocs != null) { +- try { +- indexDocs.close(); +- } catch (IOException e) { +- } +- } + } + return Status.OK_STATUS; + } +@@ -569,33 +560,33 @@ public class SearchIndex implements IHelpSearchIndex { + * @param docs2 + * @throws IOException + */ +- private void removeDocuments(TermDocs doc1, TermDocs docs2) throws IOException { +- if (!doc1.next()) { ++ private void removeDocuments(DocsEnum doc1, DocsEnum docs2) throws IOException { ++ if (doc1.nextDoc() == DocsEnum.NO_MORE_DOCS) { + return; + } +- if (!docs2.next()) { ++ if (docs2.nextDoc() == DocsEnum.NO_MORE_DOCS) { + return; + } + while (true) { +- if (doc1.doc() < docs2.doc()) { +- if (!doc1.skipTo(docs2.doc())) { +- if (!doc1.next()) { ++ if (doc1.docID() < docs2.docID()) { ++ if (doc1.advance(docs2.docID()) == DocsEnum.NO_MORE_DOCS) { ++ if (doc1.nextDoc() == DocsEnum.NO_MORE_DOCS) { + return; + } + } +- } else if (doc1.doc() > docs2.doc()) { +- if (!docs2.skipTo(doc1.doc())) { +- if (!doc1.next()) { ++ } else if (doc1.docID() > docs2.docID()) { ++ if (docs2.advance(doc1.docID()) == DocsEnum.NO_MORE_DOCS) { ++ if (doc1.nextDoc() == DocsEnum.NO_MORE_DOCS) { + return; + } + } + } +- if (doc1.doc() == docs2.doc()) { +- ir.deleteDocument(doc1.doc()); +- if (!doc1.next()) { ++ if (doc1.docID() == docs2.docID()) { ++ iw.tryDeleteDocument(ir, doc1.docID()); ++ if (doc1.nextDoc() == DocsEnum.NO_MORE_DOCS) { + return; + } +- if (!docs2.next()) { ++ if (docs2.nextDoc() == DocsEnum.NO_MORE_DOCS) { + return; + } + } +@@ -802,7 +793,7 @@ public class SearchIndex implements IHelpSearchIndex { + public void openSearcher() throws IOException { + synchronized (searcherCreateLock) { + if (searcher == null) { +- searcher = new IndexSearcher(IndexReader.open(luceneDirectory, false)); ++ searcher = new IndexSearcher(DirectoryReader.open(luceneDirectory)); + } + } + } +@@ -820,7 +811,7 @@ public class SearchIndex implements IHelpSearchIndex { + if (searches.isEmpty()) { + if (searcher != null) { + try { +- searcher.close(); ++ searcher.getIndexReader().close(); + } catch (IOException ioe) { + } + } +@@ -903,7 +894,7 @@ public class SearchIndex implements IHelpSearchIndex { + IndexWriter cleaner = null; + LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), 10000); + try { +- cleaner = new IndexWriter(luceneDirectory, new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_31, analyzer).setOpenMode( ++ cleaner = new IndexWriter(luceneDirectory, new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_47, analyzer).setOpenMode( + OpenMode.CREATE)); + } catch (IOException ioe) { + } finally { +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java +index d0a7bb7..1b20d3b 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java +@@ -10,15 +10,13 @@ + *******************************************************************************/ + package org.eclipse.help.internal.search; + +-import java.io.*; +- + import org.apache.lucene.analysis.*; + + /** + * Smart Analyzer. Chooses underlying implementation based on the field which + * text is analyzed. + */ +-public final class SmartAnalyzer extends Analyzer { ++public final class SmartAnalyzer extends AnalyzerWrapper { + Analyzer pluggedInAnalyzer; + Analyzer exactAnalyzer; + +@@ -31,14 +29,14 @@ public final class SmartAnalyzer extends Analyzer { + this.exactAnalyzer = new DefaultAnalyzer(locale); + } + /** +- * Creates a TokenStream which tokenizes all the text in the provided +- * Reader. Delegates to DefaultAnalyzer when field used to search for exact ++ * Delegates to DefaultAnalyzer when field used to search for exact + * match, and to plugged-in analyzer for other fields. + */ +- public final TokenStream tokenStream(String fieldName, Reader reader) { ++ @Override ++ public final Analyzer getWrappedAnalyzer(String fieldName) { + if (fieldName != null && fieldName.startsWith("exact_")) { //$NON-NLS-1$ +- return exactAnalyzer.tokenStream(fieldName, reader); ++ return exactAnalyzer; + } +- return pluggedInAnalyzer.tokenStream(fieldName, reader); ++ return pluggedInAnalyzer; + } + } +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java +index 0b70cf7..106775a 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java +@@ -35,6 +35,7 @@ public final class WordTokenStream extends Tokenizer { + * Constructor + */ + public WordTokenStream(String fieldName, Reader reader, Locale locale) { ++ super(reader); + this.reader = reader; + boundary = BreakIterator.getWordInstance(locale); + +@@ -105,6 +106,7 @@ public final class WordTokenStream extends Tokenizer { + } + + public void close() throws IOException { ++ super.close(); + /// Unlikely to be called as this is a reused + if (this.reader != null) { + this.reader.close(); +diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF b/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF +index a8177c3..cd99e8a 100644 +--- a/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF ++++ b/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF +@@ -18,15 +18,14 @@ + org.eclipse.ui.intro.universal, + org.eclipse.ui.forms, + org.eclipse.ui.browser;bundle-version="3.2.300", ++ org.apache.lucene.analysis;bundle-version="[4.7.0,5.0.0)", ++ org.apache.lucene.core;bundle-version="[4.7.0,5.0.0)", + org.eclipse.equinox.jsp.jasper;bundle-version="1.0.200", + org.eclipse.equinox.jsp.jasper.registry;bundle-version="1.0.100" + Bundle-ActivationPolicy: lazy + Bundle-Vendor: Eclipse.org + Import-Package: javax.servlet;version="3.1.0", +- javax.servlet.http;version="3.1.0", +- org.apache.lucene.index;core=split;version="[3.5.0,4.0.0)", +- org.apache.lucene.search;core=split;version="[3.5.0,4.0.0)", +- org.apache.lucene.store;core=split;version="[3.5.0,4.0.0)" ++ javax.servlet.http;version="3.1.0" + Bundle-RequiredExecutionEnvironment: JavaSE-1.6 + Export-Package: org.eclipse.ua.tests, + org.eclipse.ua.tests.browser, +diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java b/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java +index 640d4c9..8924fa7 100644 +--- a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java ++++ b/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java +@@ -24,7 +24,7 @@ import junit.framework.TestSuite; + import org.osgi.framework.Bundle; + + import org.apache.lucene.index.CorruptIndexException; +-import org.apache.lucene.index.IndexReader; ++import org.apache.lucene.index.DirectoryReader; + import org.apache.lucene.search.IndexSearcher; + import org.apache.lucene.search.Query; + import org.apache.lucene.search.TopDocs; +@@ -157,7 +157,7 @@ public class PrebuiltIndexCompatibility extends TestCase { + IndexSearcher searcher = null; + try { + luceneDirectory = new NIOFSDirectory(new File(filePath)); +- searcher = new IndexSearcher(IndexReader.open(luceneDirectory, true)); ++ searcher = new IndexSearcher(DirectoryReader.open(luceneDirectory)); + TopDocs hits = searcher.search(luceneQuery, 500); + assertEquals(hits.totalHits, 1); + } finally { +@@ -167,7 +167,7 @@ public class PrebuiltIndexCompatibility extends TestCase { + } catch (IOException x) { + } + if (searcher != null) +- searcher.close(); ++ searcher.getIndexReader().close(); + } + } else { + fail("Cannot resolve to file protocol"); +diff --git a/dependencies/replace_platform_plugins_with_symlinks.sh b/dependencies/replace_platform_plugins_with_symlinks.sh +index b818924..f1a67b3 100755 +--- a/dependencies/replace_platform_plugins_with_symlinks.sh ++++ b/dependencies/replace_platform_plugins_with_symlinks.sh +@@ -45,15 +45,7 @@ pushd $1 + _symlink org.apache.felix.gogo.shell_ felix-gogo-shell/org.apache.felix.gogo.shell.jar + _symlink javax.annotation-api_ glassfish-annotation-api.jar + _symlink org.apache.lucene.core_ lucene/lucene-core.jar +- _symlink org.apache.lucene.analyzers-common_ lucene/lucene-analyzers-common.jar +- _symlink org.apache.lucene.analyzers-smartcn_ lucene/lucene-analyzers-smartcn.jar +- _symlink org.apache.lucene.queries lucene/lucene-queries.jar +- _symlink org.apache.lucene.queryparser lucene/lucene-queryparser.jar +- _symlink org.apache.lucene.sandbox lucene/lucene-sandbox.jar +- _symlink org.apache.xalan xalan-j2.jar +- _symlink org.apache.xerces xerces-j2.jar +- _symlink org.apache.xml.resolver xml-commons-resolver.jar +- _symlink org.apache.xml.serializer xalan-j2-serializer.jar ++ _symlink org.apache.lucene.analysis_ lucene/lucene-analyzers-common.jar + _symlink org.eclipse.jetty.util_ jetty/jetty-util.jar + _symlink org.eclipse.jetty.server_ jetty/jetty-server.jar + _symlink org.eclipse.jetty.http_ jetty/jetty-http.jar +--- a/eclipse.platform.releng/features/org.eclipse.help-feature/feature.xml ++++ b/eclipse.platform.releng/features/org.eclipse.help-feature/feature.xml +@@ -95,7 +95,7 @@ + + + +- ++ + + + +--- a/eclipse.platform.releng.tychoeclipsebuilder/eclipse.platform.repository/category.xml ++++ b/eclipse.platform.releng.tychoeclipsebuilder/eclipse.platform.repository/category.xml +@@ -50,7 +50,7 @@ + + + +- ++ + + + diff --git a/eclipse-lucene-5.patch b/eclipse-lucene-5.patch new file mode 100644 index 0000000..1a5d281 --- /dev/null +++ b/eclipse-lucene-5.patch @@ -0,0 +1,2263 @@ +From e9b5adb1d0477a50111afe5cf7736c542b7e4998 Mon Sep 17 00:00:00 2001 +From: Sopot Cela +Date: Thu, 21 May 2015 13:48:41 +0200 +Subject: [PATCH] Bug 466829 - Upgrade platform.ua to Lucene 5.1.0 + +Change-Id: I882188205c2c1e2cc1106108680dd4e94570a975 +Signed-off-by: Sopot Cela +--- + +diff --git a/eclipse.platform.ua/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs +index 34d2b77..07713d8 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs ++++ b/eclipse.platform.ua/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs +@@ -7,9 +7,9 @@ + org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled + org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled + org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate +-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5 ++org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 + org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +-org.eclipse.jdt.core.compiler.compliance=1.5 ++org.eclipse.jdt.core.compiler.compliance=1.7 + org.eclipse.jdt.core.compiler.debug.lineNumber=generate + org.eclipse.jdt.core.compiler.debug.localVariable=generate + org.eclipse.jdt.core.compiler.debug.sourceFile=generate +@@ -98,7 +98,7 @@ + org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore + org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning + org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +-org.eclipse.jdt.core.compiler.source=1.5 ++org.eclipse.jdt.core.compiler.source=1.7 + org.eclipse.jdt.core.formatter.align_type_members_on_columns=false + org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16 + org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16 +diff --git a/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF b/org.eclipse.help.base/META-INF/MANIFEST.MF +index ee34c8e..e4bd703 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF ++++ b/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF +@@ -43,9 +43,11 @@ + org.eclipse.core.runtime;bundle-version="[3.11.0,4.0.0)", + org.eclipse.help;bundle-version="[3.5.0,4.0.0)";visibility:=reexport, + org.eclipse.core.expressions;bundle-version="[3.4.200,4.0.0)", +- org.apache.lucene.analysis;bundle-version="[3.5.0,4.0.0)", +- org.apache.lucene.core;bundle-version="[3.5.0,4.0.0)", +- org.eclipse.core.net;bundle-version="1.2.200" ++ org.eclipse.core.net;bundle-version="1.2.200", ++ org.apache.lucene.analyzers-common;bundle-version="5.1.0", ++ org.apache.lucene.core;bundle-version="5.1.0", ++ org.apache.lucene.queryparser;bundle-version="5.1.0", ++ org.apache.lucene.analyzers-smartcn;bundle-version="5.1.0" + Import-Package: com.ibm.icu.text, + org.eclipse.equinox.http.jetty;resolution:=optional + Bundle-RequiredExecutionEnvironment: J2SE-1.5 +diff --git a/eclipse.platform.ua/org.eclipse.help.base/plugin.xml b/org.eclipse.help.base/plugin.xml +index 07a5a22..4daf3f0 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/plugin.xml ++++ b/eclipse.platform.ua/org.eclipse.help.base/plugin.xml +@@ -83,7 +83,7 @@ + + ++ class="org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer"> + + StopFilter->PorterStemFilter + */ + public final class Analyzer_en extends Analyzer { ++ + /** + * Constructor for Analyzer_en. + */ +@@ -27,18 +34,25 @@ + super(); + } + /** +- * Creates a TokenStream which tokenizes all the text in the provided ++ * Creates a TokenStreamComponents which tokenizes all the text in the provided + * Reader. + */ +- public final TokenStream tokenStream(String fieldName, Reader reader) { +- return new PorterStemFilter(new StopFilter(Version.LUCENE_30, new LowerCaseAndDigitsTokenizer(reader), getStopWords(), false)); ++ @Override ++ public final TokenStreamComponents createComponents(String fieldName) { ++ final Tokenizer source; ++ source = new LowerCaseAndDigitsTokenizer(); ++ TokenStream result = new EnglishPossessiveFilter(source); ++ result = new StopFilter(result, new CharArraySet(getStopWords(), false)); ++ result = new KeywordRepeatFilter(result); ++ result = new PorterStemFilter(result); ++ return new TokenStreamComponents(source, result); + } + +- private Set stopWords; ++ private List stopWords; + +- private Set getStopWords() { ++ private List getStopWords() { + if ( stopWords == null ) { +- stopWords = new HashSet(); ++ stopWords = new ArrayList(); + for (int i = 0; i < STOP_WORDS.length; i++) { + stopWords.add(STOP_WORDS[i]); + } +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java +index 4109474..e3c8722 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java +@@ -7,22 +7,20 @@ + * Contributors: + * IBM Corporation - initial API and implementation + * Alexander Kurtakov - Bug 460787 ++ * Sopot Cela - Bug 466829 - Migration to Lucene 5 + *******************************************************************************/ + package org.eclipse.help.internal.search; + +-import java.io.Reader; + import java.util.Locale; + import java.util.StringTokenizer; + +-import com.ibm.icu.text.BreakIterator; +- + import org.apache.lucene.analysis.Analyzer; +-import org.apache.lucene.analysis.LowerCaseFilter; +-import org.apache.lucene.analysis.TokenStream; +-import org.apache.lucene.util.Version; ++import org.apache.lucene.analysis.Tokenizer; ++import org.apache.lucene.analysis.core.LowerCaseFilter; ++import org.eclipse.core.runtime.Platform; + import org.eclipse.help.internal.base.HelpBasePlugin; + +-import org.eclipse.core.runtime.Platform; ++import com.ibm.icu.text.BreakIterator; + + + /** +@@ -84,11 +82,14 @@ + } + + /** +- * Creates a TokenStream which tokenizes all the text in the provided ++ * Creates a TokenStreamComponents which tokenizes all the text in the provided + * Reader. + */ +- public final TokenStream tokenStream(String fieldName, Reader reader) { +- return new LowerCaseFilter(Version.LUCENE_30, new WordTokenStream(fieldName, reader, locale)); ++ @Override ++ public final TokenStreamComponents createComponents(String fieldName) { ++ Tokenizer source = new WordTokenStream(locale); ++ LowerCaseFilter filter = new LowerCaseFilter(source); ++ return new TokenStreamComponents(source, filter); + } + + /** +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java +index a475688..cbb2472 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java +@@ -8,27 +8,26 @@ + * Contributors: + * IBM Corporation - initial API and implementation + * Alexander Kurtakov - Bug 460787 ++ * Sopot Cela - Bug 466829 - Migration to Lucene 5 + *******************************************************************************/ + package org.eclipse.help.internal.search; + +-import java.io.*; +- +-import org.apache.lucene.analysis.*; +-import org.apache.lucene.util.Version; ++import org.apache.lucene.analysis.util.CharTokenizer; + + /** + * Tokenizer breaking words around letters or digits. + */ + public class LowerCaseAndDigitsTokenizer extends CharTokenizer { + +- public LowerCaseAndDigitsTokenizer(Reader input) { +- super(Version.LUCENE_30, input); ++ public LowerCaseAndDigitsTokenizer() { ++ super(); + } + protected char normalize(char c) { + return Character.toLowerCase(c); + } + +- protected boolean isTokenChar(char c) { ++ @Override ++ public boolean isTokenChar(int c) { + return Character.isLetterOrDigit(c); + } + +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java +index 60a545d..804bf5e 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java +@@ -1,5 +1,5 @@ + /******************************************************************************* +- * Copyright (c) 2010, 2011 IBM Corporation and others. ++ * Copyright (c) 2010, 2015 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at +@@ -7,6 +7,7 @@ + * + * Contributors: + * IBM Corporation - initial API and implementation ++ * Sopot Cela - Bug 466829 - Migration to Lucene 5 + *******************************************************************************/ + + package org.eclipse.help.internal.search; +@@ -16,6 +17,8 @@ + + import org.apache.lucene.document.Document; + import org.apache.lucene.document.Field; ++import org.apache.lucene.document.StoredField; ++import org.apache.lucene.document.TextField; + import org.eclipse.help.search.ISearchDocument; + + /** +@@ -31,22 +34,22 @@ + } + + public void setTitle(String title) { +- doc.add(new Field("title", title, Field.Store.NO, Field.Index.ANALYZED)); //$NON-NLS-1$ +- doc.add(new Field("exact_title", title, Field.Store.NO, Field.Index.ANALYZED)); //$NON-NLS-1$ +- doc.add(new Field("raw_title", title, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$ ++ doc.add(new TextField("title", title, Field.Store.NO)); //$NON-NLS-1$ ++ doc.add(new TextField("exact_title", title, Field.Store.NO)); //$NON-NLS-1$ ++ doc.add(new StoredField("raw_title", title)); //$NON-NLS-1$ + } + + public void setSummary(String summary) { +- doc.add(new Field("summary", summary, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$ ++ doc.add(new StoredField("summary", summary)); //$NON-NLS-1$ + } + + public void addContents(String contents) { +- doc.add(new Field("contents", new StringReader(contents))); //$NON-NLS-1$ +- doc.add(new Field("exact_contents", new StringReader(contents))); //$NON-NLS-1$ ++ doc.add(new TextField("contents", new StringReader(contents))); //$NON-NLS-1$ ++ doc.add(new TextField("exact_contents", new StringReader(contents))); //$NON-NLS-1$ + } + + public void setHasFilters(boolean hasFilters) { +- doc.add(new Field("filters", Boolean.toString(hasFilters), Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$ ++ doc.add(new StoredField("filters", Boolean.toString(hasFilters))); //$NON-NLS-1$ + } + + public Document getDocument() { +@@ -54,8 +57,8 @@ + } + + public void addContents(Reader contents, Reader exactContents) { +- doc.add(new Field("contents", contents)); //$NON-NLS-1$ +- doc.add(new Field("exact_contents", exactContents)); //$NON-NLS-1$ ++ doc.add(new TextField("contents", contents)); //$NON-NLS-1$ ++ doc.add(new TextField("exact_contents", exactContents)); //$NON-NLS-1$ + } + + } +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java +index ca9cd67..7952c8d 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java +@@ -1,5 +1,5 @@ + /******************************************************************************* +- * Copyright (c) 2005, 2014 IBM Corporation and others. ++ * Copyright (c) 2005, 2015 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at +@@ -7,6 +7,7 @@ + * + * Contributors: + * IBM Corporation - initial API and implementation ++ * Sopot Cela - Bug 466829 - Migration to Lucene 5 + *******************************************************************************/ + package org.eclipse.help.internal.search; + +@@ -152,6 +153,7 @@ + .getProperty(SearchIndex.DEPENDENCIES_KEY_ANALYZER); + if (!targetIndex.isLuceneCompatible(lucene) + || !targetIndex.isAnalyzerCompatible(analyzer)) { ++ HelpBasePlugin.logError("Error trying to consume Lucene index from bundle "+bundle.toString()+". Please use an index built with Lucene 5 or higher.", null); //$NON-NLS-1$ //$NON-NLS-2$ + return false; + } + } catch (MalformedURLException mue) { +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java +deleted file mode 100644 +index 08cf58a..0000000 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java ++++ /dev/null +@@ -1,459 +0,0 @@ +-/******************************************************************************* +- * Copyright (c) 2000, 2012 IBM Corporation and others. +- * All rights reserved. This program and the accompanying materials +- * are made available under the terms of the Eclipse Public License v1.0 +- * which accompanies this distribution, and is available at +- * http://www.eclipse.org/legal/epl-v10.html +- * +- * Contributors: +- * IBM Corporation - initial API and implementation +- * Chris Torrence - patch for bug Bug 107648 +- *******************************************************************************/ +-package org.eclipse.help.internal.search; +-import java.io.*; +-import java.util.ArrayList; +-import java.util.Collection; +-import java.util.Iterator; +-import java.util.List; +-import java.util.Locale; +-import java.util.StringTokenizer; +- +-import org.apache.lucene.analysis.*; +-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +-import org.apache.lucene.index.*; +-import org.apache.lucene.search.*; +-import org.eclipse.help.internal.base.*; +-/** +- * Build query acceptable by the search engine. +- */ +-public class QueryBuilder { +- // Maximum allowed number of terms +- private static final int MAX_TERMS = 10; +- // Maximum allowed number of ORs +- private static final int MAX_UNIONS = 4; +- // Maximum allowed number terms with wild cards +- private static final int MAX_WILD_TERMS = 2; +- // Query from user +- private String searchWords; +- // Descriptor of Analyzer to process the query words +- private AnalyzerDescriptor analyzerDesc; +- // Analyzer to process the query words +- private Analyzer analyzer; +- // List of QueryWordsToken +- private List analyzedTokens; +- // List of words to highlight +- private List highlightWords = new ArrayList(); +- private Locale locale; +- /** +- * Creates a query builder for the search word. The search word is processed +- * by a lexical analyzer. +- */ +- public QueryBuilder(String searchWords, AnalyzerDescriptor analyzerDesc) { +- this.searchWords = searchWords; +- String language = analyzerDesc.getLang(); +- if (language.length() >= 5) { +- this.locale = new Locale(language.substring(0, 2), language +- .substring(3, 5)); +- } else { +- this.locale = new Locale(language.substring(0, 2), ""); //$NON-NLS-1$ +- } +- this.analyzerDesc = analyzerDesc; +- this.analyzer = analyzerDesc.getAnalyzer(); +- } +- /** +- * Splits user query into tokens and returns a list of QueryWordsToken's. +- */ +- private List tokenizeUserQuery(String searchWords) { +- List tokenList = new ArrayList(); +- //Divide along quotation marks +- //StringTokenizer qTokenizer = new StringTokenizer(searchWords.trim(), +- // "\"", true); //$NON-NLS-1$ +- boolean withinQuotation = false; +- String quotedString = ""; //$NON-NLS-1$ +- int termCount = 0;// keep track of number of terms to disallow too many +- +- int fromIndex = -1; +- searchWords = searchWords.trim(); +- while((fromIndex = searchWords.indexOf("\"", fromIndex+1))!= -1){ //$NON-NLS-1$ +- withinQuotation = !withinQuotation; +- } +- if( withinQuotation ) { +- searchWords = searchWords + "\""; //$NON-NLS-1$ +- withinQuotation = !withinQuotation; +- } +- +- StringTokenizer qTokenizer = new StringTokenizer(searchWords,"\"",true); //$NON-NLS-1$ +- int orCount = 0; // keep track of number of ORs to disallow too many +- while (qTokenizer.hasMoreTokens()) { +- String curToken = qTokenizer.nextToken(); +- if (curToken.equals("\"")) { //$NON-NLS-1$ +- if (withinQuotation) { +- // check for too many terms +- if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER +- && ++termCount > MAX_TERMS) { +- throw new QueryTooComplexException(); +- } +- tokenList.add(QueryWordsToken.exactPhrase(quotedString)); +- } else { +- quotedString = ""; //$NON-NLS-1$ +- } +- withinQuotation = !withinQuotation; +- continue; +- } else if (withinQuotation) { +- quotedString = curToken; +- continue; +- } else { +- //divide unquoted strings along white space +- StringTokenizer parser = new StringTokenizer(curToken.trim()); +- while (parser.hasMoreTokens()) { +- String token = parser.nextToken(); +- if (token.equalsIgnoreCase(QueryWordsToken.AND().value)) { +- tokenList.add(QueryWordsToken.AND()); +- } else if (token +- .equalsIgnoreCase(QueryWordsToken.OR().value)) { +- // Check for too many OR terms +- if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER +- && ++orCount > MAX_UNIONS) { +- throw new QueryTooComplexException(); +- } +- tokenList.add(QueryWordsToken.OR()); +- } else if (token +- .equalsIgnoreCase(QueryWordsToken.NOT().value)) { +- tokenList.add(QueryWordsToken.NOT()); +- } else { +- // check for too many terms +- if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER +- && ++termCount > MAX_TERMS) { +- throw new QueryTooComplexException(); +- } +- tokenList.add(QueryWordsToken.word(token)); +- } +- } +- } +- } +- return tokenList; +- } +- /** +- * Apply the Analyzer to the search tokens and return the list of processed +- * QueryWordsToken's. +- */ +- private List analyzeTokens(List tokens) { +- boolean isTokenAfterNot = false; +- List newTokens = new ArrayList(); +- int wildCardTermCount = 0; +- for (int i = 0; i < tokens.size(); i++) { +- QueryWordsToken token = tokens.get(i); +- if (token.type == QueryWordsToken.WORD) { +- int questionMIndex = token.value.indexOf('?'); +- int starIndex = token.value.indexOf('*'); +- if (starIndex >= 0 || questionMIndex >= 0) { +- if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER +- && ++wildCardTermCount > MAX_WILD_TERMS) { +- throw new QueryTooComplexException(); +- } +- newTokens.add(QueryWordsToken.word(token.value +- .toLowerCase(locale))); +- // add word to the list of words to highlight +- if (!isTokenAfterNot && !highlightWords.contains(token.value)) { +- highlightWords.add(token.value); +- } +- } else { +- List wordList = analyzeText(analyzer, "contents", //$NON-NLS-1$ +- token.value); +- if (wordList.size() > 0) { +- if (!isTokenAfterNot && !highlightWords.contains(token.value)) { +- // add original word to the list of words to +- // highlight +- highlightWords.add(token.value); +- } +- if (wordList.size() == 1) { +- String word = wordList.get(0); +- newTokens.add(QueryWordsToken.word(word)); +- // add analyzed word to the list of words to +- // highlight +- // this is required to highlight stemmed words +- if (!isTokenAfterNot && !highlightWords.contains(word)) { +- highlightWords.add(word); +- } +- } else { +- QueryWordsPhrase phrase = QueryWordsToken.phrase(); +- for (Iterator it = wordList.iterator(); it +- .hasNext();) { +- String word = it.next(); +- phrase.addWord(word); +- // add each analyzed word to the list of words +- // to highlight +- // this is only required to highlight stemmed +- // words. +- // Adding words should not be done when +- // DefaultAnalyzer is used, +- // because it does not perform stemming and +- // common words removal +- // which would result in common characters +- // highlighted all over (bug 30263) +- if (!analyzerDesc.getId().startsWith( +- HelpBasePlugin.PLUGIN_ID + "#")) { //$NON-NLS-1$ +- if (!isTokenAfterNot && !highlightWords.contains(word)) { +- highlightWords.add(word); +- } +- } +- } +- newTokens.add(phrase); +- } +- } +- } +- } else if (// forget ANDs +- /* +- * token.type == SearchQueryToken.AND || +- */ +- token.type == QueryWordsToken.OR +- || token.type == QueryWordsToken.NOT) +- newTokens.add(token); +- else if (token.type == QueryWordsToken.EXACT_PHRASE) { +- List wordList = analyzeText(analyzer, "exact_contents", //$NON-NLS-1$ +- token.value); +- if (wordList.size() > 0) { +- if (!isTokenAfterNot && !highlightWords.contains(token.value)) { +- // add original word to the list of words to highlight +- highlightWords.add(token.value); +- } +- } +- QueryWordsExactPhrase phrase = QueryWordsToken.exactPhrase(); +- for (Iterator it = wordList.iterator(); it.hasNext();) { +- String word = it.next(); +- phrase.addWord(word); +- // add analyzed word to the list of words to highlight +- // if (!highlightWords.contains(word)) +- // highlightWords.add(word); +- } +- // add phrase only if not empty +- if (phrase.getWords().size() > 0) { +- newTokens.add(phrase); +- } +- } +- isTokenAfterNot = (token.type == QueryWordsToken.NOT); +- } +- return newTokens; +- } +- /** +- * Get a list of tokens corresponding to a search word or phrase +- * +- * @return List of String +- */ +- private List analyzeText(Analyzer analyzer, String fieldName, String text) { +- List words = new ArrayList(1); +- Reader reader = new StringReader(text); +- TokenStream tStream = analyzer.tokenStream(fieldName, reader); +- +- CharTermAttribute termAttribute = tStream.getAttribute(CharTermAttribute.class); +- try { +- while (tStream.incrementToken()) { +- String term = termAttribute.toString(); +- words.add(term); +- } +- reader.close(); +- } catch (IOException ioe) { +- } +- +- return words; +- } +- /** +- * Obtains Lucene Query from tokens +- * +- * @return Query or null if no query could be created +- */ +- private Query createLuceneQuery(List searchTokens, String[] fieldNames, +- float[] boosts) { +- // Get queries for parts separated by OR +- List requiredQueries = getRequiredQueries(searchTokens, fieldNames, +- boosts); +- if (requiredQueries.size() == 0) +- return null; +- else if (requiredQueries.size() <= 1) +- return requiredQueries.get(0); +- else +- /* if (requiredQueries.size() > 1) */ +- // OR queries +- return (orQueries(requiredQueries)); +- } +- /** +- * Obtains Lucene queries for token sequences separated at OR. +- * +- * @return List of Query (could be empty) +- */ +- private List getRequiredQueries(List tokens, String[] fieldNames, +- float[] boosts) { +- List oredQueries = new ArrayList(); +- ArrayList requiredQueryTokens = new ArrayList(); +- for (int i = 0; i < tokens.size(); i++) { +- QueryWordsToken token = tokens.get(i); +- if (token.type != QueryWordsToken.OR) { +- requiredQueryTokens.add(token); +- } else { +- Query reqQuery = getRequiredQuery(requiredQueryTokens, +- fieldNames, boosts); +- if (reqQuery != null) +- oredQueries.add(reqQuery); +- requiredQueryTokens = new ArrayList(); +- } +- } +- Query reqQuery = getRequiredQuery(requiredQueryTokens, fieldNames, +- boosts); +- if (reqQuery != null) +- oredQueries.add(reqQuery); +- return oredQueries; +- } +- private Query orQueries(Collection queries) { +- BooleanQuery bq = new BooleanQuery(); +- for (Iterator it = queries.iterator(); it.hasNext();) { +- Query q = it.next(); +- bq.add(q, BooleanClause.Occur.SHOULD); +- } +- return bq; +- } +- /** +- * Obtains Lucene Query for tokens containing only AND and NOT operators. +- * +- * @return BooleanQuery or null if no query could be created from the tokens +- */ +- private Query getRequiredQuery(List requiredTokens, String[] fieldNames, +- float[] boosts) { +- BooleanQuery retQuery = new BooleanQuery(); +- boolean requiredTermExist = false; +- // Parse tokens left to right +- QueryWordsToken operator = null; +- for (int i = 0; i < requiredTokens.size(); i++) { +- QueryWordsToken token = requiredTokens.get(i); +- if (token.type == QueryWordsToken.AND +- || token.type == QueryWordsToken.NOT) { +- operator = token; +- continue; +- } +- // Creates queries for all fields +- Query qs[] = new Query[fieldNames.length]; +- for (int f = 0; f < fieldNames.length; f++) { +- qs[f] = token.createLuceneQuery(fieldNames[f], boosts[f]); +- } +- // creates the boolean query of all fields +- Query q = qs[0]; +- if (fieldNames.length > 1) { +- BooleanQuery allFieldsQuery = new BooleanQuery(); +- for (int f = 0; f < fieldNames.length; f++) +- allFieldsQuery.add(qs[f], BooleanClause.Occur.SHOULD); +- q = allFieldsQuery; +- } +- if (operator != null && operator.type == QueryWordsToken.NOT) { +- retQuery.add(q, BooleanClause.Occur.MUST_NOT); // add as prohibited +- } else { +- retQuery.add(q, BooleanClause.Occur.MUST); // add as required +- requiredTermExist = true; +- } +- } +- if (!requiredTermExist) { +- return null; // cannot search for prohibited only +- } +- return retQuery; +- } +- private Query getLuceneQuery(String[] fieldNames, float[] boosts) { +- Query luceneQuery = createLuceneQuery(analyzedTokens, fieldNames, +- boosts); +- return luceneQuery; +- } +- /** +- * @param fieldNames - +- * Collection of field names of type String (e.g. "h1"); the +- * search will be performed on the given fields +- * @param fieldSearchOnly - +- * boolean indicating if field only search should be performed; +- * if set to false, default field "contents" and all other fields +- * will be searched +- */ +- public Query getLuceneQuery(Collection fieldNames, boolean fieldSearchOnly) +- throws QueryTooComplexException { +- // split search query into tokens +- List userTokens = tokenizeUserQuery(searchWords); +- analyzedTokens = analyzeTokens(userTokens); +- return buildLuceneQuery(fieldNames, fieldSearchOnly); +- } +- /** +- * @param fieldNames - +- * Collection of field names of type String (e.g. "h1"); the +- * search will be performed on the given fields +- * @param fieldSearchOnly - +- * boolean indicating if field only search should be performed; +- * if set to false, default field "contents" and all other fields +- * will be searched +- */ +- private Query buildLuceneQuery(Collection fieldNames, +- boolean fieldSearchOnly) { +- String[] fields; +- float[] boosts; +- if (fieldSearchOnly) { +- fields = new String[fieldNames.size()]; +- boosts = new float[fieldNames.size()]; +- Iterator fieldNamesIt = fieldNames.iterator(); +- for (int i = 0; i < fieldNames.size(); i++) { +- fields[i] = fieldNamesIt.next(); +- boosts[i] = 5.0f; +- } +- } else { +- fields = new String[fieldNames.size() + 2]; +- boosts = new float[fieldNames.size() + 2]; +- Iterator fieldNamesIt = fieldNames.iterator(); +- for (int i = 0; i < fieldNames.size(); i++) { +- fields[i] = fieldNamesIt.next(); +- boosts[i] = 5.0f; +- } +- fields[fieldNames.size()] = "contents"; //$NON-NLS-1$ +- boosts[fieldNames.size()] = 1.0f; +- fields[fieldNames.size()+1] = "title"; //$NON-NLS-1$ +- boosts[fieldNames.size()+1] = 1.0f; +- } +- Query query = getLuceneQuery(fields, boosts); +- query = improveRankingForUnqotedPhrase(query, fields, boosts); +- return query; +- } +- /** +- * If user query contained only words (no quotaions nor operators) extends +- * query with term phrase representing entire user query i.e for user string +- * a b, the query a AND b will be extended to "a b" OR a AND b +- */ +- private Query improveRankingForUnqotedPhrase(Query query, String[] fields, +- float[] boosts) { +- if (query == null) +- return query; +- // check if all tokens are words +- for (int i = 0; i < analyzedTokens.size(); i++) +- if (analyzedTokens.get(i).type != QueryWordsToken.WORD) +- return query; +- // Create phrase query for all tokens and OR with original query +- BooleanQuery booleanQuery = new BooleanQuery(); +- booleanQuery.add(query, BooleanClause.Occur.SHOULD); +- PhraseQuery[] phraseQueries = new PhraseQuery[fields.length]; +- for (int f = 0; f < fields.length; f++) { +- phraseQueries[f] = new PhraseQuery(); +- for (int i = 0; i < analyzedTokens.size(); i++) { +- Term t = new Term(fields[f], analyzedTokens +- .get(i).value); +- phraseQueries[f].add(t); +- } +- phraseQueries[f].setBoost(10 * boosts[f]); +- booleanQuery.add(phraseQueries[f], BooleanClause.Occur.SHOULD); +- } +- return booleanQuery; +- } +- /** +- * Obtains analyzed terms from query as one string. Words are double quoted, +- * and separated by space. The analyzed words are needed for highlighting +- * word roots. +- */ +- public String gethighlightTerms() { +- StringBuffer buf = new StringBuffer(); +- for (Iterator it = highlightWords.iterator(); it.hasNext();) { +- buf.append('"'); +- buf.append(it.next()); +- buf.append("\" "); //$NON-NLS-1$ +- } +- return buf.toString(); +- } +-} +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsExactPhrase.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsExactPhrase.java +deleted file mode 100644 +index 324b8e1..0000000 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsExactPhrase.java ++++ /dev/null +@@ -1,50 +0,0 @@ +-/******************************************************************************* +- * Copyright (c) 2000, 2011 IBM Corporation and others. +- * All rights reserved. This program and the accompanying materials +- * are made available under the terms of the Eclipse Public License v1.0 +- * which accompanies this distribution, and is available at +- * http://www.eclipse.org/legal/epl-v10.html +- * +- * Contributors: +- * IBM Corporation - initial API and implementation +- *******************************************************************************/ +-package org.eclipse.help.internal.search; +-import java.util.ArrayList; +-import java.util.Iterator; +-import java.util.List; +- +-import org.apache.lucene.index.*; +-import org.apache.lucene.search.*; +-/** +- * Represents a quoted token in user search query words +- */ +-public class QueryWordsExactPhrase extends QueryWordsToken { +- private List words; +- public QueryWordsExactPhrase() { +- super(QueryWordsToken.EXACT_PHRASE, ""); //$NON-NLS-1$ +- words = new ArrayList(); +- } +- public void addWord(String word) { +- words.add(word); +- if (words.size() <= 1) +- value = word; +- else +- value += " " + word; //$NON-NLS-1$ +- } +- public List getWords() { +- return words; +- } +- /** +- * Creates a lucene query for a field +- */ +- public Query createLuceneQuery(String field, float boost) { +- PhraseQuery q = new PhraseQuery(); +- for (Iterator it = getWords().iterator(); it.hasNext();) { +- String word = it.next(); +- Term t = new Term("exact_" + field, word); //$NON-NLS-1$ +- q.add(t); +- q.setBoost(boost); +- } +- return q; +- } +-} +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsPhrase.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsPhrase.java +deleted file mode 100644 +index 8a94e89..0000000 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsPhrase.java ++++ /dev/null +@@ -1,51 +0,0 @@ +-/******************************************************************************* +- * Copyright (c) 2000, 2011 IBM Corporation and others. +- * All rights reserved. This program and the accompanying materials +- * are made available under the terms of the Eclipse Public License v1.0 +- * which accompanies this distribution, and is available at +- * http://www.eclipse.org/legal/epl-v10.html +- * +- * Contributors: +- * IBM Corporation - initial API and implementation +- *******************************************************************************/ +-package org.eclipse.help.internal.search; +-import java.util.ArrayList; +-import java.util.Iterator; +-import java.util.List; +- +-import org.apache.lucene.index.*; +-import org.apache.lucene.search.*; +-/** +- * Represents a phrase (not quoted) token in user search query words It consists +- * of several words created by an analyzer +- */ +-public class QueryWordsPhrase extends QueryWordsToken { +- private List words; +- public QueryWordsPhrase() { +- super(QueryWordsToken.PHRASE, ""); //$NON-NLS-1$ +- words = new ArrayList(); +- } +- public void addWord(String word) { +- words.add(word); +- if (words.size() <= 1) +- value = word; +- else +- value += " " + word; //$NON-NLS-1$ +- } +- public List getWords() { +- return words; +- } +- /** +- * Creates a lucene query for a field +- */ +- public Query createLuceneQuery(String field, float boost) { +- PhraseQuery q = new PhraseQuery(); +- for (Iterator it = getWords().iterator(); it.hasNext();) { +- String word = it.next(); +- Term t = new Term(field, word); +- q.add(t); +- q.setBoost(boost); +- } +- return q; +- } +-} +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsToken.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsToken.java +deleted file mode 100644 +index 6ba76f2..0000000 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsToken.java ++++ /dev/null +@@ -1,81 +0,0 @@ +-/******************************************************************************* +- * Copyright (c) 2000, 2007 IBM Corporation and others. +- * All rights reserved. This program and the accompanying materials +- * are made available under the terms of the Eclipse Public License v1.0 +- * which accompanies this distribution, and is available at +- * http://www.eclipse.org/legal/epl-v10.html +- * +- * Contributors: +- * IBM Corporation - initial API and implementation +- *******************************************************************************/ +-package org.eclipse.help.internal.search; +-import org.apache.lucene.index.*; +-import org.apache.lucene.search.*; +-/** +- * Represents a token in user search query words +- */ +-public class QueryWordsToken { +- public static final int AND = 0; +- public static final int OR = 1; +- public static final int NOT = 2; +- public static final int EXACT_PHRASE = 3; +- public static final int PHRASE = 4; +- public static final int WORD = 5; +- private static final QueryWordsToken fAND = new QueryWordsToken(AND, "AND"); //$NON-NLS-1$ +- private static final QueryWordsToken fOR = new QueryWordsToken(OR, "OR"); //$NON-NLS-1$ +- private static final QueryWordsToken fNOT = new QueryWordsToken(NOT, "NOT"); //$NON-NLS-1$ +- public int type; +- public String value; +- protected QueryWordsToken(int type, String value) { +- this.type = type; +- this.value = value; +- } +- /** +- * Creates a lucene query for a field +- */ +- public Query createLuceneQuery(String field, float boost) { +- Query q; +- int questionPos = value.indexOf('?'); +- int starPos = value.indexOf('*'); +- if (questionPos >= 0 || starPos >= 0) { +- if (questionPos == -1 && starPos == value.length() - 1) { +- Term t = new Term("exact_" + field, value.substring(0, starPos)); //$NON-NLS-1$ +- q = new PrefixQuery(t); +- ((PrefixQuery) q).setBoost(boost); +- } else { +- Term t = new Term("exact_" + field, value); //$NON-NLS-1$ +- q = new WildcardQuery(t); +- ((WildcardQuery) q).setBoost(boost); +- } +- } else { +- Term t = new Term(field, value); +- q = new TermQuery(t); +- ((TermQuery) q).setBoost(boost); +- } +- // after updating Lucene, set boost on a Query class +- return q; +- } +- public static QueryWordsToken AND() { +- return fAND; +- } +- public static QueryWordsToken OR() { +- return fOR; +- } +- public static QueryWordsToken NOT() { +- return fNOT; +- } +- public static QueryWordsToken word(String word) { +- return new QueryWordsToken(QueryWordsToken.WORD, word); +- } +- public static QueryWordsPhrase phrase() { +- return new QueryWordsPhrase(); +- } +- public static QueryWordsExactPhrase exactPhrase() { +- return new QueryWordsExactPhrase(); +- } +- public static QueryWordsExactPhrase exactPhrase(String word) { +- QueryWordsExactPhrase token = new QueryWordsExactPhrase(); +- token.addWord(word); +- return token; +- } +-} +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java +index 67963a5..131ac65 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java +@@ -1,14 +1,13 @@ + /******************************************************************************* +- * Copyright (c) 2000, 2014 IBM Corporation and others. +- * All rights reserved. This program and the accompanying materials +- * are made available under the terms of the Eclipse Public License v1.0 ++ * Copyright (c) 2000, 2015 IBM Corporation and others. All rights reserved. This program and the ++ * accompanying materials are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * +- * Contributors: +- * IBM Corporation - initial API and implementation +- * Holger Voormann - fix for bug 426785 (http://eclip.se/426785) +- * Alexander Kurtakov - Bug 460787 ++ * Contributors: IBM Corporation - initial API and implementation ++ * Holger Voormann - fix for bug 426785 (http://eclip.se/426785) ++ * Alexander Kurtakov - Bug 460787 ++ * Sopot Cela - Bug 466829 - Migration to Lucene 5 + *******************************************************************************/ + package org.eclipse.help.internal.search; + +@@ -17,6 +16,7 @@ + import java.io.IOException; + import java.io.InputStream; + import java.io.RandomAccessFile; ++import java.io.StringReader; + import java.net.MalformedURLException; + import java.net.URL; + import java.nio.channels.FileLock; +@@ -33,19 +33,30 @@ + import java.util.zip.ZipEntry; + import java.util.zip.ZipInputStream; + +-import org.apache.lucene.analysis.LimitTokenCountAnalyzer; ++import org.apache.lucene.analysis.miscellaneous.LimitTokenCountAnalyzer; + import org.apache.lucene.document.Document; + import org.apache.lucene.document.Field; ++import org.apache.lucene.document.StoredField; ++import org.apache.lucene.document.StringField; ++import org.apache.lucene.index.DirectoryReader; + import org.apache.lucene.index.IndexReader; + import org.apache.lucene.index.IndexWriter; +-import org.apache.lucene.index.Term; +-import org.apache.lucene.index.TermDocs; +-import org.apache.lucene.index.IndexWriterConfig.OpenMode; + import org.apache.lucene.index.IndexWriterConfig; ++import org.apache.lucene.index.IndexWriterConfig.OpenMode; ++import org.apache.lucene.index.LeafReader; + import org.apache.lucene.index.LogByteSizeMergePolicy; + import org.apache.lucene.index.LogMergePolicy; ++import org.apache.lucene.index.PostingsEnum; ++import org.apache.lucene.index.SlowCompositeReaderWrapper; ++import org.apache.lucene.index.Term; ++import org.apache.lucene.queryparser.classic.FastCharStream; ++import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; ++import org.apache.lucene.queryparser.classic.QueryParserTokenManager; ++import org.apache.lucene.queryparser.classic.Token; ++import org.apache.lucene.search.BooleanClause.Occur; + import org.apache.lucene.search.BooleanQuery; + import org.apache.lucene.search.IndexSearcher; ++import org.apache.lucene.search.MultiTermQuery; + import org.apache.lucene.search.Query; + import org.apache.lucene.search.TopDocs; + import org.apache.lucene.store.Directory; +@@ -59,7 +70,6 @@ + import org.eclipse.core.runtime.OperationCanceledException; + import org.eclipse.core.runtime.Platform; + import org.eclipse.core.runtime.Status; +-import org.eclipse.help.internal.HelpPlugin; + import org.eclipse.help.internal.base.BaseHelpSystem; + import org.eclipse.help.internal.base.HelpBasePlugin; + import org.eclipse.help.internal.base.util.HelpProperties; +@@ -86,7 +96,7 @@ + + private File indexDir; + +- private Directory luceneDirectory; ++ public Directory luceneDirectory; + + private String locale; + +@@ -133,7 +143,7 @@ + private Collection searches = new ArrayList(); + + private FileLock lock; +- private RandomAccessFile raf = null; ++ private RandomAccessFile raf = null; + + /** + * Constructor. +@@ -170,7 +180,7 @@ + inconsistencyFile = new File(indexDir.getParentFile(), locale + ".inconsistent"); //$NON-NLS-1$ + htmlSearchParticipant = new HTMLSearchParticipant(indexDir.getAbsolutePath()); + try { +- luceneDirectory = new NIOFSDirectory(indexDir); ++ luceneDirectory = new NIOFSDirectory(indexDir.toPath()); + } catch (IOException e) { + } + if (!exists()) { +@@ -203,11 +213,11 @@ + public IStatus addDocument(String name, URL url) { + try { + Document doc = new Document(); +- doc.add(new Field(FIELD_NAME, name, Field.Store.YES, Field.Index.NOT_ANALYZED)); ++ doc.add(new StringField(FIELD_NAME, name, Field.Store.YES)); + addExtraFields(doc); + String pluginId = LocalSearchManager.getPluginId(name); + if (relativePath != null) { +- doc.add(new Field(FIELD_INDEX_ID, relativePath, Field.Store.YES, Field.Index.NOT_ANALYZED)); ++ doc.add(new StringField(FIELD_INDEX_ID, relativePath, Field.Store.YES)); + } + // check for the explicit search participant. + SearchParticipant participant = null; +@@ -220,20 +230,22 @@ + if (participant == null) + participant = BaseHelpSystem.getLocalSearchManager().getParticipant(pluginId, name); + if (participant != null) { +- IStatus status = participant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc)); ++ IStatus status = participant.addDocument(this, pluginId, name, url, id, ++ new LuceneSearchDocument(doc)); + if (status.getSeverity() == IStatus.OK) { + String filters = doc.get("filters"); //$NON-NLS-1$ + indexedDocs.put(name, filters != null ? filters : "0"); //$NON-NLS-1$ + if (id != null) +- doc.add(new Field("id", id, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$ ++ doc.add(new StoredField("id", id)); //$NON-NLS-1$ + if (pid != null) +- doc.add(new Field("participantId", pid, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$ ++ doc.add(new StoredField("participantId", pid)); //$NON-NLS-1$ + iw.addDocument(doc); + } + return status; + } + // default to html +- IStatus status = htmlSearchParticipant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc)); ++ IStatus status = htmlSearchParticipant.addDocument(this, pluginId, name, url, id, ++ new LuceneSearchDocument(doc)); + if (status.getSeverity() == IStatus.OK) { + String filters = doc.get("filters"); //$NON-NLS-1$ + indexedDocs.put(name, filters != null ? filters : "0"); //$NON-NLS-1$ +@@ -245,20 +257,21 @@ + "IO exception occurred while adding document " + name //$NON-NLS-1$ + + " to index " + indexDir.getAbsolutePath() + ".", //$NON-NLS-1$ //$NON-NLS-2$ + e); +- } +- catch (Exception e) { ++ } catch (Exception e) { + return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR, + "An unexpected internal error occurred while adding document " //$NON-NLS-1$ + + name + " to index " + indexDir.getAbsolutePath() //$NON-NLS-1$ +- + ".", e); //$NON-NLS-1$ ++ + ".", //$NON-NLS-1$ ++ e); + } + } + + /** +- * Add any extra fields that need to be added to this document. Subclasses +- * should override to add more fields. ++ * Add any extra fields that need to be added to this document. Subclasses should override to ++ * add more fields. + * +- * @param doc the document to add fields to ++ * @param doc ++ * the document to add fields to + */ + protected void addExtraFields(Document doc) { + } +@@ -282,8 +295,9 @@ + indexedDocs = new HelpProperties(INDEXED_DOCS_FILE, indexDir); + indexedDocs.restore(); + setInconsistent(true); +- LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), 1000000); +- IndexWriterConfig writerConfig = new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_31, analyzer); ++ LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), ++ 1000000); ++ IndexWriterConfig writerConfig = new IndexWriterConfig(analyzer); + writerConfig.setOpenMode(create ? OpenMode.CREATE : OpenMode.APPEND); + LogMergePolicy mergePolicy = new LogByteSizeMergePolicy(); + mergePolicy.setMergeFactor(20); +@@ -307,7 +321,7 @@ + indexedDocs = new HelpProperties(INDEXED_DOCS_FILE, indexDir); + indexedDocs.restore(); + setInconsistent(true); +- ir = IndexReader.open(luceneDirectory, false); ++ ir = DirectoryReader.open(luceneDirectory); + return true; + } catch (IOException e) { + HelpBasePlugin.logError("Exception occurred in search indexing at beginDeleteBatch.", e); //$NON-NLS-1$ +@@ -323,7 +337,7 @@ + if (ir != null) { + ir.close(); + } +- ir = IndexReader.open(luceneDirectory, false); ++ ir = DirectoryReader.open(luceneDirectory); + return true; + } catch (IOException e) { + HelpBasePlugin.logError("Exception occurred in search indexing at beginDeleteBatch.", e); //$NON-NLS-1$ +@@ -334,14 +348,14 @@ + /** + * Deletes a single document from the index. + * +- * @param name - +- * document name ++ * @param name ++ * - document name + * @return IStatus + */ + public IStatus removeDocument(String name) { + Term term = new Term(FIELD_NAME, name); + try { +- ir.deleteDocuments(term); ++ iw.deleteDocuments(term); + indexedDocs.remove(name); + } catch (IOException e) { + return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR, +@@ -375,11 +389,11 @@ + } + + /* +- * The searcher's index reader has it's stuff in memory so it won't +- * know about this change. Close it so that it gets reloaded next search. ++ * The searcher's index reader has it's stuff in memory so it won't know about this ++ * change. Close it so that it gets reloaded next search. + */ + if (searcher != null) { +- searcher.close(); ++ searcher.getIndexReader().close(); + searcher = null; + } + return true; +@@ -407,11 +421,11 @@ + saveDependencies(); + + /* +- * The searcher's index reader has it's stuff in memory so it won't +- * know about this change. Close it so that it gets reloaded next search. ++ * The searcher's index reader has it's stuff in memory so it won't know about this ++ * change. Close it so that it gets reloaded next search. + */ + if (searcher != null) { +- searcher.close(); ++ searcher.getIndexReader().close(); + searcher = null; + } + return true; +@@ -469,11 +483,11 @@ + String indexId = indexIds.get(i); + String indexPath = indexPaths.get(i); + try { +- dirList.add(new NIOFSDirectory(new File(indexPath))); ++ dirList.add(new NIOFSDirectory(new File(indexPath).toPath())); + } catch (IOException ioe) { +- HelpBasePlugin +- .logError( +- "Help search indexing directory could not be created for directory " + indexPath, ioe); //$NON-NLS-1$ ++ HelpBasePlugin.logError( ++ "Help search indexing directory could not be created for directory " + indexPath, //$NON-NLS-1$ ++ ioe); + continue; + } + +@@ -525,18 +539,19 @@ + } + + public IStatus removeDuplicates(String name, String[] index_paths) { +- TermDocs hrefDocs = null; +- TermDocs indexDocs = null; +- Term hrefTerm = new Term(FIELD_NAME, name); + try { ++ LeafReader ar = SlowCompositeReaderWrapper.wrap(ir); ++ PostingsEnum hrefDocs = null; ++ PostingsEnum indexDocs = null; ++ Term hrefTerm = new Term(FIELD_NAME, name); + for (int i = 0; i < index_paths.length; i++) { + Term indexTerm = new Term(FIELD_INDEX_ID, index_paths[i]); + if (i == 0) { +- hrefDocs = ir.termDocs(hrefTerm); +- indexDocs = ir.termDocs(indexTerm); ++ hrefDocs = ar.postings(hrefTerm); ++ indexDocs = ar.postings(indexTerm); + } else { +- hrefDocs.seek(hrefTerm); +- indexDocs.seek(indexTerm); ++ hrefDocs = ar.postings(hrefTerm); ++ indexDocs = ar.postings(indexTerm); + } + removeDocuments(hrefDocs, indexDocs); + } +@@ -545,19 +560,6 @@ + "IO exception occurred while removing duplicates of document " + name //$NON-NLS-1$ + + " from index " + indexDir.getAbsolutePath() + ".", //$NON-NLS-1$ //$NON-NLS-2$ + ioe); +- } finally { +- if (hrefDocs != null) { +- try { +- hrefDocs.close(); +- } catch (IOException e) { +- } +- } +- if (indexDocs != null) { +- try { +- indexDocs.close(); +- } catch (IOException e) { +- } +- } + } + return Status.OK_STATUS; + } +@@ -569,33 +571,33 @@ + * @param docs2 + * @throws IOException + */ +- private void removeDocuments(TermDocs doc1, TermDocs docs2) throws IOException { +- if (!doc1.next()) { ++ private void removeDocuments(PostingsEnum doc1, PostingsEnum docs2) throws IOException { ++ if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) { + return; + } +- if (!docs2.next()) { ++ if (docs2.nextDoc() == PostingsEnum.NO_MORE_DOCS) { + return; + } + while (true) { +- if (doc1.doc() < docs2.doc()) { +- if (!doc1.skipTo(docs2.doc())) { +- if (!doc1.next()) { ++ if (doc1.docID() < docs2.docID()) { ++ if (doc1.advance(docs2.docID()) == PostingsEnum.NO_MORE_DOCS) { ++ if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) { + return; + } + } +- } else if (doc1.doc() > docs2.doc()) { +- if (!docs2.skipTo(doc1.doc())) { +- if (!doc1.next()) { ++ } else if (doc1.docID() > docs2.docID()) { ++ if (docs2.advance(doc1.docID()) == PostingsEnum.NO_MORE_DOCS) { ++ if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) { + return; + } + } + } +- if (doc1.doc() == docs2.doc()) { +- ir.deleteDocument(doc1.doc()); +- if (!doc1.next()) { ++ if (doc1.docID() == docs2.docID()) { ++ iw.tryDeleteDocument(ir, doc1.docID()); ++ if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) { + return; + } +- if (!docs2.next()) { ++ if (docs2.nextDoc() == PostingsEnum.NO_MORE_DOCS) { + return; + } + } +@@ -623,20 +625,72 @@ + registerSearch(Thread.currentThread()); + if (closed) + return; +- QueryBuilder queryBuilder = new QueryBuilder(searchQuery.getSearchWord(), analyzerDescriptor); +- Query luceneQuery = queryBuilder.getLuceneQuery(searchQuery.getFieldNames(), searchQuery +- .isFieldSearch()); +- if (HelpPlugin.DEBUG_SEARCH) { +- System.out.println("Search Query: " + luceneQuery.toString()); //$NON-NLS-1$ ++ ++ String[] fields; ++ if (searchQuery.isFieldSearch()){ ++ //sometimes you might want to search other than the default fields ++ fields = (String[]) searchQuery.getFieldNames().toArray(); ++ }else { ++ fields = new String[]{"contents","title"}; //$NON-NLS-1$ //$NON-NLS-2$ + } +- String highlightTerms = queryBuilder.gethighlightTerms(); +- if (luceneQuery != null) { ++ ++ //prepare the parser ++ ++ MultiFieldQueryParser qb = new MultiFieldQueryParser(fields,analyzerDescriptor.getAnalyzer()); ++ qb.setAllowLeadingWildcard(true); ++ qb.setAnalyzeRangeTerms(true); ++ qb.setAutoGeneratePhraseQueries(true); ++ qb.setMultiTermRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE); ++ qb.setLowercaseExpandedTerms(true); ++ qb.setLocale(new Locale(analyzerDescriptor.getLang())); ++ ++ //parse the "pure" query (no boosting) ++ Query luceneQuery = qb.parse(searchQuery.getSearchWord()); ++ ++ //we'll merge the pure query with a some boosted queries ++ Query mergedQuery; ++ ++ if (!isWildcardQuery(searchQuery.getSearchWord())){ ++ mergedQuery = new BooleanQuery(); //merge for all fields before merging with luceneQuery ++ for (int i=0;i set = new HashSet(); ++ ++ if (mergedQuery != null) { + if (searcher == null) { + openSearcher(); + } +- TopDocs topDocs = searcher.search(luceneQuery, null, 1000); +- collector.addHits(LocalSearchManager.asList(topDocs, searcher), highlightTerms); ++ TopDocs topDocs = searcher.search(mergedQuery, 1000); ++ ++ String highlight=null; ++ QueryParserTokenManager manager = new QueryParserTokenManager(new FastCharStream(new StringReader(searchQuery.getSearchWord()))); ++ while (true){ ++ Token nextToken = manager.getNextToken(); ++ String toHighlight = null; ++ if (nextToken.kind==0) break; ++ String image = nextToken.image; ++ toHighlight=image; ++ if ((image.startsWith("\""))&&( image.endsWith("\""))){ //$NON-NLS-1$//$NON-NLS-2$ ++ toHighlight = image.substring(1,image.length()-1); ++ } ++ if (image.equals("AND") || image.equals("OR")) //$NON-NLS-1$ //$NON-NLS-2$ ++ continue; ++ set .add(toHighlight); ++ ++ } ++ highlight = buildHighlight(set); ++ collector.addHits(LocalSearchManager.asList(topDocs, searcher), highlight==null?"":highlight); //$NON-NLS-1$ + } ++ + } catch (BooleanQuery.TooManyClauses tmc) { + collector.addQTCException(new QueryTooComplexException()); + } catch (QueryTooComplexException qe) { +@@ -647,6 +701,19 @@ + } finally { + unregisterSearch(Thread.currentThread()); + } ++ } ++ ++ private boolean isWildcardQuery(String searchWord) { ++ ++ return searchWord.contains("?")|| searchWord.contains("*"); //$NON-NLS-1$//$NON-NLS-2$ ++ } ++ ++ private String buildHighlight(Set set) { ++ StringBuilder sb = new StringBuilder(); ++ for (String string : set) { ++ sb.append("\""+string+"\""); //$NON-NLS-1$//$NON-NLS-2$ ++ } ++ return sb.toString(); + } + + public String getLocale() { +@@ -660,13 +727,13 @@ + if (docPlugins == null) { + Set totalIds = new HashSet(); + IExtensionRegistry registry = Platform.getExtensionRegistry(); +- IExtensionPoint extensionPoint = registry.getExtensionPoint(TocFileProvider.EXTENSION_POINT_ID_TOC); ++ IExtensionPoint extensionPoint = registry ++ .getExtensionPoint(TocFileProvider.EXTENSION_POINT_ID_TOC); + IExtension[] extensions = extensionPoint.getExtensions(); +- for (int i=0;i= 0 ) { ++ if (luceneVersion.compareTo(indexVersion) >= 0) { + // Lucene bundle is newer than the index + return true; + } +@@ -802,7 +871,7 @@ + public void openSearcher() throws IOException { + synchronized (searcherCreateLock) { + if (searcher == null) { +- searcher = new IndexSearcher(IndexReader.open(luceneDirectory, false)); ++ searcher = new IndexSearcher(DirectoryReader.open(luceneDirectory)); + } + } + } +@@ -820,7 +889,7 @@ + if (searches.isEmpty()) { + if (searcher != null) { + try { +- searcher.close(); ++ searcher.getIndexReader().close(); + } catch (IOException ioe) { + } + } +@@ -838,7 +907,8 @@ + * Finds and unzips prebuild index specified in preferences + */ + private void unzipProductIndex() { +- String indexPluginId = Platform.getPreferencesService().getString(HelpBasePlugin.PLUGIN_ID, "productIndex", null, null); //$NON-NLS-1$ ++ String indexPluginId = Platform.getPreferencesService().getString(HelpBasePlugin.PLUGIN_ID, ++ "productIndex", null, null); //$NON-NLS-1$ + if (indexPluginId == null || indexPluginId.length() <= 0) { + return; + } +@@ -901,10 +971,11 @@ + */ + private void cleanOldIndex() { + IndexWriter cleaner = null; +- LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), 10000); ++ LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), ++ 10000); + try { +- cleaner = new IndexWriter(luceneDirectory, new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_31, analyzer).setOpenMode( +- OpenMode.CREATE)); ++ cleaner = new IndexWriter(luceneDirectory, ++ new IndexWriterConfig(analyzer).setOpenMode(OpenMode.CREATE)); + } catch (IOException ioe) { + } finally { + try { +@@ -957,7 +1028,7 @@ + * if lock already obtained + */ + public synchronized boolean tryLock() throws OverlappingFileLockException { +- if ("none".equals(System.getProperty("osgi.locking"))) { //$NON-NLS-1$//$NON-NLS-2$ ++ if ("none".equals(System.getProperty("osgi.locking"))) { //$NON-NLS-1$//$NON-NLS-2$ + return true; // Act as if lock succeeded + } + if (lock != null) { +@@ -979,7 +1050,7 @@ + lock = null; + logLockFailure(ioe); + } +- if ( raf != null ) { ++ if (raf != null) { + try { + raf.close(); + } catch (IOException e) { +@@ -1025,9 +1096,9 @@ + } + lock = null; + } +- if (raf != null ) { ++ if (raf != null) { + try { +- raf.close(); ++ raf.close(); + } catch (IOException ioe) { + } + raf = null; +@@ -1100,25 +1171,26 @@ + + public IStatus addDocument(String pluginId, String name, URL url, String id, Document doc) { + // try a registered participant for the file format +- SearchParticipant participant = BaseHelpSystem.getLocalSearchManager() +- .getParticipant(pluginId, name); ++ SearchParticipant participant = BaseHelpSystem.getLocalSearchManager().getParticipant(pluginId, name); + if (participant != null) { + try { + return participant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc)); +- } +- catch (Throwable t) { ++ } catch (Throwable t) { + return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR, + "Error while adding document to search participant (addDocument()): " //$NON-NLS-1$ +- + name + ", " + url + "for participant " + participant.getClass().getName(), t); //$NON-NLS-1$ //$NON-NLS-2$ ++ + name + ", " + url + "for participant " + participant.getClass().getName(), //$NON-NLS-1$ //$NON-NLS-2$ ++ t); + } + } + // default to html +- return htmlSearchParticipant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc)); ++ return htmlSearchParticipant.addDocument(this, pluginId, name, url, id, ++ new LuceneSearchDocument(doc)); + } + +- public IStatus addSearchableDocument(String pluginId, String name, URL url, String id, ISearchDocument doc) { ++ public IStatus addSearchableDocument(String pluginId, String name, URL url, String id, ++ ISearchDocument doc) { + // In the help system the only class that implements ISearchDocument is LuceneSearchDocument +- LuceneSearchDocument luceneDoc = (LuceneSearchDocument)doc; ++ LuceneSearchDocument luceneDoc = (LuceneSearchDocument) doc; + return addDocument(pluginId, name, url, id, luceneDoc.getDocument()); + } + } +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java +index d0a7bb7..444b66c 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java +@@ -1,5 +1,5 @@ + /******************************************************************************* +- * Copyright (c) 2000, 2012 IBM Corporation and others. ++ * Copyright (c) 2000, 2015 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at +@@ -7,10 +7,9 @@ + * + * Contributors: + * IBM Corporation - initial API and implementation ++ * Sopot Cela - Bug 466829 - Migration to Lucene 5 + *******************************************************************************/ + package org.eclipse.help.internal.search; +- +-import java.io.*; + + import org.apache.lucene.analysis.*; + +@@ -18,7 +17,7 @@ + * Smart Analyzer. Chooses underlying implementation based on the field which + * text is analyzed. + */ +-public final class SmartAnalyzer extends Analyzer { ++public final class SmartAnalyzer extends AnalyzerWrapper { + Analyzer pluggedInAnalyzer; + Analyzer exactAnalyzer; + +@@ -26,19 +25,19 @@ + * Constructor for SmartAnalyzer. + */ + public SmartAnalyzer(String locale, Analyzer pluggedInAnalyzer) { +- super(); ++ super(pluggedInAnalyzer.getReuseStrategy()); + this.pluggedInAnalyzer = pluggedInAnalyzer; + this.exactAnalyzer = new DefaultAnalyzer(locale); + } + /** +- * Creates a TokenStream which tokenizes all the text in the provided +- * Reader. Delegates to DefaultAnalyzer when field used to search for exact ++ * Delegates to DefaultAnalyzer when field used to search for exact + * match, and to plugged-in analyzer for other fields. + */ +- public final TokenStream tokenStream(String fieldName, Reader reader) { ++ @Override ++ public final Analyzer getWrappedAnalyzer(String fieldName) { + if (fieldName != null && fieldName.startsWith("exact_")) { //$NON-NLS-1$ +- return exactAnalyzer.tokenStream(fieldName, reader); ++ return exactAnalyzer; + } +- return pluggedInAnalyzer.tokenStream(fieldName, reader); ++ return pluggedInAnalyzer; + } + } +diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java +index 0b70cf7..79d5592 100644 +--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java ++++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java +@@ -1,5 +1,5 @@ + /******************************************************************************* +- * Copyright (c) 2000, 2012 IBM Corporation and others. ++ * Copyright (c) 2000, 2015 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at +@@ -7,16 +7,18 @@ + * + * Contributors: + * IBM Corporation - initial API and implementation ++ * Sopot Cela - Bug 466829 - Migration to Lucene 5 + *******************************************************************************/ + package org.eclipse.help.internal.search; + +-import com.ibm.icu.text.BreakIterator; + import java.io.IOException; +-import java.io.Reader; + import java.util.Locale; ++ + import org.apache.lucene.analysis.TokenStream; + import org.apache.lucene.analysis.Tokenizer; + import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; ++ ++import com.ibm.icu.text.BreakIterator; + + /** + * WordTokenStream obtains tokens containing words appropriate for use with +@@ -24,7 +26,6 @@ + */ + public final class WordTokenStream extends Tokenizer { + private static final int BUF_LEN = 4096; +- private final Reader reader; + private final BreakIterator boundary; + private StringBuffer strbuf; + +@@ -34,8 +35,8 @@ + /** + * Constructor + */ +- public WordTokenStream(String fieldName, Reader reader, Locale locale) { +- this.reader = reader; ++ public WordTokenStream(Locale locale) { ++ super(); + boundary = BreakIterator.getWordInstance(locale); + + } +@@ -52,9 +53,9 @@ + if(strbuf == null) { + int available; + char[] cbuf = new char[BUF_LEN]; +- while ((available = reader.read(cbuf)) <= 0) { ++ while ((available = input.read(cbuf)) <= 0) { + if (available < 0) { +- reader.close(); ++ input.close(); + return false; + } + } +@@ -62,7 +63,7 @@ + strbuf.append(cbuf, 0, available); + // read more until white space (or EOF) + int c; +- while (0 <= (c = reader.read())) { ++ while (0 <= (c = input.read())) { + strbuf.append((char) c); + if (c == ' ' || c == '\r' || c == '\n' || c == '\t') { + break; +@@ -70,7 +71,7 @@ + } + + if (c < 0) { +- reader.close(); ++ input.close(); + } + + boundary.setText(strbuf.toString()); +@@ -105,9 +106,10 @@ + } + + public void close() throws IOException { ++ super.close(); + /// Unlikely to be called as this is a reused +- if (this.reader != null) { +- this.reader.close(); ++ if (this.input != null) { ++ this.input.close(); + } + } + } +diff --git a/eclipse.platform.ua/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml b/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml +index 8438100..174e92a 100644 +--- a/eclipse.platform.ua/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml ++++ b/eclipse.platform.ua/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml +@@ -74,20 +74,6 @@ + unpack="false"/> + + +- +- +- +- + ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF b/org.eclipse.ua.tests/META-INF/MANIFEST.MF +index 6bcf9bc..23f8910 100644 +--- a/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF ++++ b/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF +@@ -19,14 +19,13 @@ + org.eclipse.ui.forms, + org.eclipse.ui.browser;bundle-version="3.2.300", + org.eclipse.equinox.jsp.jasper;bundle-version="1.0.200", +- org.eclipse.equinox.jsp.jasper.registry;bundle-version="1.0.100" ++ org.eclipse.equinox.jsp.jasper.registry;bundle-version="1.0.100", ++ org.apache.lucene.analyzers-common;bundle-version="5.1.0", ++ org.apache.lucene.core;bundle-version="5.1.0" + Bundle-ActivationPolicy: lazy + Bundle-Vendor: Eclipse.org + Import-Package: javax.servlet;version="3.1.0", +- javax.servlet.http;version="3.1.0", +- org.apache.lucene.index;core=split;version="[3.5.0,4.0.0)", +- org.apache.lucene.search;core=split;version="[3.5.0,4.0.0)", +- org.apache.lucene.store;core=split;version="[3.5.0,4.0.0)" ++ javax.servlet.http;version="3.1.0" + Bundle-RequiredExecutionEnvironment: JavaSE-1.6 + Export-Package: org.eclipse.ua.tests, + org.eclipse.ua.tests.browser, +diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index291/write.lock b/org.eclipse.ua.tests/data/help/searchindex/index291/write.lock +new file mode 100644 +index 0000000..e69de29 +--- /dev/null ++++ b/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index291/write.lock +diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfe b/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfe +new file mode 100644 +index 0000000..09ec2c9 +--- /dev/null ++++ b/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfe +Binary files differ +diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfs b/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfs +new file mode 100644 +index 0000000..3aa288a +--- /dev/null ++++ b/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfs +Binary files differ +diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/_0.si b/org.eclipse.ua.tests/data/help/searchindex/index510/_0.si +new file mode 100644 +index 0000000..d897bfc +--- /dev/null ++++ b/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/_0.si +Binary files differ +diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/segments_1 b/org.eclipse.ua.tests/data/help/searchindex/index510/segments_1 +new file mode 100644 +index 0000000..4878901 +--- /dev/null ++++ b/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/segments_1 +Binary files differ +diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/write.lock b/org.eclipse.ua.tests/data/help/searchindex/index510/write.lock +new file mode 100644 +index 0000000..e69de29 +--- /dev/null ++++ b/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/write.lock +diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/EncodedCharacterSearch.java b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/EncodedCharacterSearch.java +index 1dd4876..02b2e5b 100644 +--- a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/EncodedCharacterSearch.java ++++ b/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/EncodedCharacterSearch.java +@@ -7,6 +7,7 @@ + * + * Contributors: + * IBM Corporation - initial API and implementation ++ * Sopot Cela - Bug 466829 - Migration to Lucene 5 + *******************************************************************************/ + package org.eclipse.ua.tests.help.search; + +@@ -43,7 +44,7 @@ + + public void testUtf8Hebrew() { + SearchTestUtils.searchOneLocale("\u05D0\u05B7\u05E1\u05B0\u05D8\u05B0\u05E8\u05D5\u05B9\u05E0\u05D5\u05B9\u05DE" +- + "\u05B0\u05D9\u05B8\u05D4) \u05DC\u05B4\u05E7\u05BC\u05D5\u05BC\u05D9 (\u05D9\u05E8\u05D7 \u05D0\u05D5 \u05E9\u05DE\u05E9", new String[] {"/org.eclipse.ua.tests/data/help/search/testnlUTF8.htm" }, "en"); ++ + "\u05B0\u05D9\u05B8\u05D4\\) \u05DC\u05B4\u05E7\u05BC\u05D5\u05BC\u05D9 \\(\u05D9\u05E8\u05D7 \u05D0\u05D5 \u05E9\u05DE\u05E9", new String[] {"/org.eclipse.ua.tests/data/help/search/testnlUTF8.htm" }, "en"); + } + + +diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java +index 640d4c9..33ae5ba 100644 +--- a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java ++++ b/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java +@@ -8,6 +8,7 @@ + * Contributors: + * IBM Corporation - initial API and implementation + * Alexander Kurtakov - Bug 460787 ++ * Sopot Cela - Bug 466829 - Migration to Lucene 5 + *******************************************************************************/ + + package org.eclipse.ua.tests.help.search; +@@ -15,43 +16,38 @@ + import java.io.File; + import java.io.IOException; + import java.net.URL; +-import java.util.ArrayList; + +-import junit.framework.Test; +-import junit.framework.TestCase; +-import junit.framework.TestSuite; +- +-import org.osgi.framework.Bundle; +- ++import org.apache.lucene.analysis.standard.StandardAnalyzer; + import org.apache.lucene.index.CorruptIndexException; +-import org.apache.lucene.index.IndexReader; ++import org.apache.lucene.index.DirectoryReader; + import org.apache.lucene.search.IndexSearcher; + import org.apache.lucene.search.Query; + import org.apache.lucene.search.TopDocs; + import org.apache.lucene.store.Directory; + import org.apache.lucene.store.NIOFSDirectory; +- +-import org.eclipse.help.internal.base.BaseHelpSystem; +-import org.eclipse.help.internal.search.AnalyzerDescriptor; +-import org.eclipse.help.internal.search.PluginIndex; +-import org.eclipse.help.internal.search.QueryBuilder; +-import org.eclipse.help.internal.search.SearchIndexWithIndexingProgress; +-import org.eclipse.ua.tests.plugin.UserAssistanceTestPlugin; +- ++import org.apache.lucene.util.QueryBuilder; + import org.eclipse.core.runtime.FileLocator; + import org.eclipse.core.runtime.NullProgressMonitor; + import org.eclipse.core.runtime.Path; ++import org.eclipse.help.internal.base.BaseHelpSystem; ++import org.eclipse.help.internal.search.PluginIndex; ++import org.eclipse.help.internal.search.SearchIndexWithIndexingProgress; ++import org.eclipse.ua.tests.plugin.UserAssistanceTestPlugin; ++import org.osgi.framework.Bundle; ++ ++import junit.framework.Test; ++import junit.framework.TestCase; ++import junit.framework.TestSuite; + + /** +- * Verify that older versions of the index can be read by this +- * version of Eclipse. ++ * Verify that older versions of the index can be read by this version of ++ * Eclipse. + * +- * How to maintain this test - if when upgrading to a new version +- * of Lucene one of the IndexReadable tests fails you need to +- * make the following changes: +- * 1. Change the corresponding Compatible() test to expect a result of false +- * 2. Comment out the failing test +- * 3. Change the help system to recognize that version of Lucene as being incompatible ++ * How to maintain this test - if when upgrading to a new version of Lucene one ++ * of the IndexReadable tests fails you need to make the following changes: 1. ++ * Change the corresponding Compatible() test to expect a result of false 2. ++ * Comment out the failing test 3. Change the help system to recognize that ++ * version of Lucene as being incompatible + */ + + public class PrebuiltIndexCompatibility extends TestCase { +@@ -64,48 +60,33 @@ + } + + /** +- * Test index built with Lucene 1.9.1 +- */ +- public void test1_9_1_IndexReadable() throws Exception { +- checkReadable("data/help/searchindex/index191"); +- } +- +- /** +- * Test index built with Lucene 2.9.1 +- */ +- public void test2_9_1_IndexReadable() throws Exception { +- checkReadable("data/help/searchindex/index291"); +- } +- +- /** + ** Test compatibility of Lucene 1.9.1 index with current Lucene + */ +- public void test1_9_1Compatible() +- { +- checkCompatible("data/help/searchindex/index191", true); ++ public void test1_9_1Compatible() { ++ checkCompatible("data/help/searchindex/index191", false); + } +- ++ + /** + ** Test compatibility of Lucene 2.9.1 index with current Lucene + */ +- public void test2_9_1Compatible() +- { +- checkCompatible("data/help/searchindex/index291", true); ++ public void test2_9_1Compatible() { ++ checkCompatible("data/help/searchindex/index291", false); + } + +- public void test1_9_1LuceneCompatible() +- { +- checkLuceneCompatible("1.9.1", true); ++ public void test1_9_1LuceneCompatible() { ++ checkLuceneCompatible("1.9.1", false); + } + +- public void test1_4_103NotLuceneCompatible() +- { ++ public void test1_4_103NotLuceneCompatible() { + checkLuceneCompatible("1.4.103", false); + } + +- public void test2_9_1LuceneCompatible() +- { +- checkLuceneCompatible("2.9.1", true); ++ public void test2_9_1LuceneCompatible() { ++ checkLuceneCompatible("2.9.1", false); ++ } ++ ++ public void test5_1_0LuceneCompatible() { ++ checkLuceneCompatible("5.1.0", true); + } + + public void testPluginIndexEqualToItself() { +@@ -114,52 +95,59 @@ + } + + /** +- * Verify that if the paths and plugins are the same two PluginIndex objects are equal ++ * Verify that if the paths and plugins are the same two PluginIndex objects ++ * are equal + */ + public void testPluginIndexEquality() { +- PluginIndex index1a = createPluginIndex("data/help/searchindex/index191"); +- PluginIndex index1b = createPluginIndex("data/help/searchindex/index191"); ++ PluginIndex index1a = createPluginIndex("data/help/searchindex/index510"); ++ PluginIndex index1b = createPluginIndex("data/help/searchindex/index510"); + assertTrue(index1a.equals(index1b)); + } +- ++ + /** +- * Verify that if the paths and plugins are the same two PluginIndex objects are equal ++ * Verify that if the paths and plugins are the same two PluginIndex objects ++ * are equal + */ + public void testPluginIndexHash() { + PluginIndex index1a = createPluginIndex("data/help/searchindex/index191"); + PluginIndex index1b = createPluginIndex("data/help/searchindex/index191"); + assertEquals(index1a.hashCode(), index1b.hashCode()); + } +- ++ + /** +- * Verify that if the paths are different two PluginIndex objects are not equal ++ * Verify that if the paths are different two PluginIndex objects are not ++ * equal + */ + public void testPluginIndexInequality() { + PluginIndex index1 = createPluginIndex("data/help/searchindex/index191"); + PluginIndex index2 = createPluginIndex("data/help/searchindex/index291"); + assertFalse(index1.equals(index2)); + } ++ ++ public void test5_1_0IndexReadable() throws CorruptIndexException, IOException { ++ checkReadable("data/help/searchindex/index510"); ++ } + +- /* +- * Verifies that a prebuilt index can be searched +- */ +- private void checkReadable(String indexPath) throws IOException, +- CorruptIndexException { ++ /* ++ * Verifies that a prebuilt index can be searched ++ */ ++ private void checkReadable(String indexPath) throws IOException, CorruptIndexException { + Path path = new Path(indexPath); +- Bundle bundle = UserAssistanceTestPlugin.getDefault().getBundle(); ++ Bundle bundle = UserAssistanceTestPlugin.getDefault().getBundle(); + URL url = FileLocator.find(bundle, path, null); + URL resolved = FileLocator.resolve(url); + if ("file".equals(resolved.getProtocol())) { //$NON-NLS-1$ + String filePath = resolved.getFile(); +- QueryBuilder queryBuilder = new QueryBuilder("eclipse", new AnalyzerDescriptor("en-us")); +- Query luceneQuery = queryBuilder.getLuceneQuery(new ArrayList() , false); ++ StandardAnalyzer standardAnalyzer = new StandardAnalyzer(); ++ QueryBuilder builder = new QueryBuilder(standardAnalyzer); ++ Query luceneQuery = builder.createBooleanQuery("contents", "eclipse"); + Directory luceneDirectory = null; + IndexSearcher searcher = null; + try { +- luceneDirectory = new NIOFSDirectory(new File(filePath)); +- searcher = new IndexSearcher(IndexReader.open(luceneDirectory, true)); ++ luceneDirectory = new NIOFSDirectory(new File(filePath).toPath()); ++ searcher = new IndexSearcher(DirectoryReader.open(luceneDirectory)); + TopDocs hits = searcher.search(luceneQuery, 500); +- assertEquals(hits.totalHits, 1); ++ assertEquals(1, hits.totalHits ); + } finally { + if (luceneDirectory != null) + try { +@@ -167,13 +155,13 @@ + } catch (IOException x) { + } + if (searcher != null) +- searcher.close(); ++ searcher.getIndexReader().close(); + } + } else { + fail("Cannot resolve to file protocol"); + } + } +- ++ + /* + * Tests the isCompatible method in PluginIndex + */ +@@ -186,22 +174,20 @@ + public PluginIndex createPluginIndex(String versionDirectory) { + PluginIndex pluginIndex; + SearchIndexWithIndexingProgress index = BaseHelpSystem.getLocalSearchManager().getIndex("en_us".toString()); +- BaseHelpSystem.getLocalSearchManager().ensureIndexUpdated( +- new NullProgressMonitor(), +- index); ++ BaseHelpSystem.getLocalSearchManager().ensureIndexUpdated(new NullProgressMonitor(), index); + pluginIndex = new PluginIndex("org.eclipse.ua.tests", "data/help/searchindex/" + versionDirectory, index); + return pluginIndex; + } +- ++ + /* +- * Tests the isLuceneCompatible method in SearchIndex ++ * Tests the isLuceneCompatible method in SearchIndex + */ + private void checkLuceneCompatible(String version, boolean expected) { + SearchIndexWithIndexingProgress index = BaseHelpSystem.getLocalSearchManager().getIndex("en_us".toString()); +- BaseHelpSystem.getLocalSearchManager().ensureIndexUpdated( +- new NullProgressMonitor(), +- index); ++ BaseHelpSystem.getLocalSearchManager().ensureIndexUpdated(new NullProgressMonitor(), index); + assertEquals(expected, index.isLuceneCompatible(version)); + } + ++ ++ + } +diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/SearchParticipantTest.java b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/SearchParticipantTest.java +index 223e42a..2e782c3 100644 +--- a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/SearchParticipantTest.java ++++ b/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/SearchParticipantTest.java +@@ -1,5 +1,5 @@ + /******************************************************************************* +- * Copyright (c) 2010 IBM Corporation and others. ++ * Copyright (c) 2010, 2015 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at +@@ -7,6 +7,7 @@ + * + * Contributors: + * IBM Corporation - initial API and implementation ++ * Sopot Cela - Bug 466829 - Migration to Lucene 5 + *******************************************************************************/ + package org.eclipse.ua.tests.help.search; + +@@ -40,7 +41,7 @@ + } + + public void testSearchUsingAndInSeparateDocs() { +- SearchTestUtils.searchAllLocales("jduehdye and olhoykk", new String[0]); ++ SearchTestUtils.searchAllLocales("jduehdye AND olhoykk", new String[0]); + } + + public void testSearchExactMatch() { +--- a/eclipse.platform.common/bundles/org.eclipse.platform.doc.isv/pom.xml ++++ b/eclipse.platform.common/bundles/org.eclipse.platform.doc.isv/pom.xml +@@ -101,7 +101,7 @@ + + + eclipse-plugin +- org.apache.lucene.analysis ++ org.apache.lucene.analyzers-common + 0.0.0 + + diff --git a/eclipse-no-target-platform.patch b/eclipse-no-target-platform.patch deleted file mode 100644 index ae8873b..0000000 --- a/eclipse-no-target-platform.patch +++ /dev/null @@ -1,16 +0,0 @@ ---- eclipse-platform-parent/pom.xml.orig 2015-01-30 11:09:37.156904060 +0000 -+++ eclipse-platform-parent/pom.xml 2015-01-30 11:11:05.154445821 +0000 -@@ -204,13 +204,6 @@ - -- -- -- org.eclipse -- eclipse-sdk-prereqs -- 4.5.0-SNAPSHOT -- -- - - - linux diff --git a/eclipse-secondary-arches.patch b/eclipse-secondary-arches.patch index c25e8f5..7871f28 100644 --- a/eclipse-secondary-arches.patch +++ b/eclipse-secondary-arches.patch @@ -2,7 +2,7 @@ diff --git a/eclipse.platform.swt.binaries/pom.xml b/eclipse.platform.swt.binari index 7dd0536..66d79ee 100644 --- a/eclipse.platform.swt.binaries/pom.xml +++ b/eclipse.platform.swt.binaries/pom.xml -@@ -53,6 +53,8 @@ +@@ -63,6 +63,8 @@ bundles/org.eclipse.swt.gtk.linux.ppc64le bundles/org.eclipse.swt.gtk.linux.s390 bundles/org.eclipse.swt.gtk.linux.s390x @@ -15,7 +15,7 @@ diff --git a/rt.equinox.framework/pom.xml b/rt.equinox.framework/pom.xml index b46cdc6..0fb36a5 100644 --- a/rt.equinox.framework/pom.xml +++ b/rt.equinox.framework/pom.xml -@@ -54,6 +54,8 @@ +@@ -64,6 +64,8 @@ bundles/org.eclipse.equinox.launcher.gtk.aix.ppc bundles/org.eclipse.equinox.launcher.gtk.aix.ppc64 bundles/org.eclipse.equinox.launcher.gtk.hpux.ia64 diff --git a/eclipse-test-support.patch b/eclipse-test-support.patch index 46ee189..f6b4c1b 100644 --- a/eclipse-test-support.patch +++ b/eclipse-test-support.patch @@ -122,7 +122,7 @@ -@@ -1430,6 +1370,15 @@ +@@ -1431,6 +1371,15 @@ value="end longRunningTests" /> @@ -138,7 +138,7 @@