From e9b5adb1d0477a50111afe5cf7736c542b7e4998 Mon Sep 17 00:00:00 2001
From: Sopot Cela <scela@redhat.com>
Date: Thu, 21 May 2015 13:48:41 +0200
Subject: [PATCH] Bug 466829 - Upgrade platform.ua to Lucene 5.1.0
Change-Id: I882188205c2c1e2cc1106108680dd4e94570a975
Signed-off-by: Sopot Cela <scela@redhat.com>
---
diff --git a/eclipse.platform.ua/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs
index 34d2b77..07713d8 100644
--- a/eclipse.platform.ua/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs
+++ b/eclipse.platform.ua/org.eclipse.help.base/.settings/org.eclipse.jdt.core.prefs
@@ -7,9 +7,9 @@
org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.5
+org.eclipse.jdt.core.compiler.compliance=1.7
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
@@ -98,7 +98,7 @@
org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore
org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
-org.eclipse.jdt.core.compiler.source=1.5
+org.eclipse.jdt.core.compiler.source=1.7
org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16
diff --git a/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF b/org.eclipse.help.base/META-INF/MANIFEST.MF
index ee34c8e..e4bd703 100644
--- a/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF
+++ b/eclipse.platform.ua/org.eclipse.help.base/META-INF/MANIFEST.MF
@@ -43,9 +43,11 @@
org.eclipse.core.runtime;bundle-version="[3.11.0,4.0.0)",
org.eclipse.help;bundle-version="[3.5.0,4.0.0)";visibility:=reexport,
org.eclipse.core.expressions;bundle-version="[3.4.200,4.0.0)",
- org.apache.lucene.analysis;bundle-version="[3.5.0,4.0.0)",
- org.apache.lucene.core;bundle-version="[3.5.0,4.0.0)",
- org.eclipse.core.net;bundle-version="1.2.200"
+ org.eclipse.core.net;bundle-version="1.2.200",
+ org.apache.lucene.analyzers-common;bundle-version="5.1.0",
+ org.apache.lucene.core;bundle-version="5.1.0",
+ org.apache.lucene.queryparser;bundle-version="5.1.0",
+ org.apache.lucene.analyzers-smartcn;bundle-version="5.1.0"
Import-Package: com.ibm.icu.text,
org.eclipse.equinox.http.jetty;resolution:=optional
Bundle-RequiredExecutionEnvironment: J2SE-1.5
diff --git a/eclipse.platform.ua/org.eclipse.help.base/plugin.xml b/org.eclipse.help.base/plugin.xml
index 07a5a22..4daf3f0 100644
--- a/eclipse.platform.ua/org.eclipse.help.base/plugin.xml
+++ b/eclipse.platform.ua/org.eclipse.help.base/plugin.xml
@@ -83,7 +83,7 @@
</analyzer>
<analyzer
locale="zh"
- class="org.apache.lucene.analysis.cn.ChineseAnalyzer">
+ class="org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer">
</analyzer>
<analyzer
locale="cs"
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/AnalyzerFactory.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/AnalyzerFactory.java
index 4ea3b21..704b0e9 100644
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/AnalyzerFactory.java
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/AnalyzerFactory.java
@@ -1,5 +1,5 @@
/*******************************************************************************
- * Copyright (c) 2012 IBM Corporation and others.
+ * Copyright (c) 2012, 2015 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -7,6 +7,7 @@
*
* Contributors:
* IBM Corporation - initial API and implementation
+ * Sopot Cela - Bug 466829 - Migration to Lucene 5
*******************************************************************************/
package org.eclipse.help.internal.search;
@@ -19,8 +20,9 @@
import org.apache.lucene.analysis.fr.FrenchAnalyzer;
import org.apache.lucene.analysis.nl.DutchAnalyzer;
import org.apache.lucene.analysis.ru.RussianAnalyzer;
-import org.apache.lucene.util.Version;
-import org.eclipse.core.runtime.*;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IConfigurationElement;
+import org.eclipse.core.runtime.IExecutableExtension;
/**
* A factory responsible for instantiating a lucene {@link Analyzer}.
@@ -30,27 +32,26 @@
public Analyzer create() {
if (locale == null)
return null;
- Version version = Version.LUCENE_35;
if ("pt".equals(locale)) //$NON-NLS-1$
- return new BrazilianAnalyzer(version);
+ return new BrazilianAnalyzer();
if ("ja".equals(locale)) //$NON-NLS-1$
- return new CJKAnalyzer(version);
+ return new CJKAnalyzer();
if ("ko".equals(locale)) //$NON-NLS-1$
- return new CJKAnalyzer(version);
+ return new CJKAnalyzer();
if ("pt".equals(locale)) //$NON-NLS-1$
- return new BrazilianAnalyzer(version);
+ return new BrazilianAnalyzer();
if ("cs".equals(locale)) //$NON-NLS-1$
- return new CzechAnalyzer(version);
+ return new CzechAnalyzer();
if ("de".equals(locale)) //$NON-NLS-1$
- return new GermanAnalyzer(version);
+ return new GermanAnalyzer();
if ("el".equals(locale)) //$NON-NLS-1$
- return new GreekAnalyzer(version);
+ return new GreekAnalyzer();
if ("fr".equals(locale)) //$NON-NLS-1$
- return new FrenchAnalyzer(version);
+ return new FrenchAnalyzer();
if ("nl".equals(locale)) //$NON-NLS-1$
- return new DutchAnalyzer(version);
+ return new DutchAnalyzer();
if ("ru".equals(locale)) //$NON-NLS-1$
- return new RussianAnalyzer(version);
+ return new RussianAnalyzer();
//unknown language
return null;
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java
index a066aa4..a2183f9 100644
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java
@@ -8,18 +8,25 @@
* Contributors:
* IBM Corporation - initial API and implementation
* Alexander Kurtakov - Bug 460787
+ * Sopot Cela - Bug 466829 - Migration to Lucene 5
*******************************************************************************/
package org.eclipse.help.internal.search;
-import java.io.*;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.ArrayList;
+import java.util.List;
-import org.apache.lucene.analysis.*;
-import org.apache.lucene.util.Version;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.analysis.en.EnglishPossessiveFilter;
+import org.apache.lucene.analysis.en.PorterStemFilter;
+import org.apache.lucene.analysis.miscellaneous.KeywordRepeatFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
/**
* Lucene Analyzer for English. LowerCaseTokenizer->StopFilter->PorterStemFilter
*/
public final class Analyzer_en extends Analyzer {
+
/**
* Constructor for Analyzer_en.
*/
@@ -27,18 +34,25 @@
super();
}
/**
- * Creates a TokenStream which tokenizes all the text in the provided
+ * Creates a TokenStreamComponents which tokenizes all the text in the provided
* Reader.
*/
- public final TokenStream tokenStream(String fieldName, Reader reader) {
- return new PorterStemFilter(new StopFilter(Version.LUCENE_30, new LowerCaseAndDigitsTokenizer(reader), getStopWords(), false));
+ @Override
+ public final TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source;
+ source = new LowerCaseAndDigitsTokenizer();
+ TokenStream result = new EnglishPossessiveFilter(source);
+ result = new StopFilter(result, new CharArraySet(getStopWords(), false));
+ result = new KeywordRepeatFilter(result);
+ result = new PorterStemFilter(result);
+ return new TokenStreamComponents(source, result);
}
- private Set<String> stopWords;
+ private List<String> stopWords;
- private Set<String> getStopWords() {
+ private List<String> getStopWords() {
if ( stopWords == null ) {
- stopWords = new HashSet<String>();
+ stopWords = new ArrayList<String>();
for (int i = 0; i < STOP_WORDS.length; i++) {
stopWords.add(STOP_WORDS[i]);
}
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java
index 4109474..e3c8722 100644
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/DefaultAnalyzer.java
@@ -7,22 +7,20 @@
* Contributors:
* IBM Corporation - initial API and implementation
* Alexander Kurtakov - Bug 460787
+ * Sopot Cela - Bug 466829 - Migration to Lucene 5
*******************************************************************************/
package org.eclipse.help.internal.search;
-import java.io.Reader;
import java.util.Locale;
import java.util.StringTokenizer;
-import com.ibm.icu.text.BreakIterator;
-
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.LowerCaseFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.util.Version;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.eclipse.core.runtime.Platform;
import org.eclipse.help.internal.base.HelpBasePlugin;
-import org.eclipse.core.runtime.Platform;
+import com.ibm.icu.text.BreakIterator;
/**
@@ -84,11 +82,14 @@
}
/**
- * Creates a TokenStream which tokenizes all the text in the provided
+ * Creates a TokenStreamComponents which tokenizes all the text in the provided
* Reader.
*/
- public final TokenStream tokenStream(String fieldName, Reader reader) {
- return new LowerCaseFilter(Version.LUCENE_30, new WordTokenStream(fieldName, reader, locale));
+ @Override
+ public final TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new WordTokenStream(locale);
+ LowerCaseFilter filter = new LowerCaseFilter(source);
+ return new TokenStreamComponents(source, filter);
}
/**
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java
index a475688..cbb2472 100644
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LowerCaseAndDigitsTokenizer.java
@@ -8,27 +8,26 @@
* Contributors:
* IBM Corporation - initial API and implementation
* Alexander Kurtakov - Bug 460787
+ * Sopot Cela - Bug 466829 - Migration to Lucene 5
*******************************************************************************/
package org.eclipse.help.internal.search;
-import java.io.*;
-
-import org.apache.lucene.analysis.*;
-import org.apache.lucene.util.Version;
+import org.apache.lucene.analysis.util.CharTokenizer;
/**
* Tokenizer breaking words around letters or digits.
*/
public class LowerCaseAndDigitsTokenizer extends CharTokenizer {
- public LowerCaseAndDigitsTokenizer(Reader input) {
- super(Version.LUCENE_30, input);
+ public LowerCaseAndDigitsTokenizer() {
+ super();
}
protected char normalize(char c) {
return Character.toLowerCase(c);
}
- protected boolean isTokenChar(char c) {
+ @Override
+ public boolean isTokenChar(int c) {
return Character.isLetterOrDigit(c);
}
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java
index 60a545d..804bf5e 100644
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/LuceneSearchDocument.java
@@ -1,5 +1,5 @@
/*******************************************************************************
- * Copyright (c) 2010, 2011 IBM Corporation and others.
+ * Copyright (c) 2010, 2015 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -7,6 +7,7 @@
*
* Contributors:
* IBM Corporation - initial API and implementation
+ * Sopot Cela - Bug 466829 - Migration to Lucene 5
*******************************************************************************/
package org.eclipse.help.internal.search;
@@ -16,6 +17,8 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.TextField;
import org.eclipse.help.search.ISearchDocument;
/**
@@ -31,22 +34,22 @@
}
public void setTitle(String title) {
- doc.add(new Field("title", title, Field.Store.NO, Field.Index.ANALYZED)); //$NON-NLS-1$
- doc.add(new Field("exact_title", title, Field.Store.NO, Field.Index.ANALYZED)); //$NON-NLS-1$
- doc.add(new Field("raw_title", title, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$
+ doc.add(new TextField("title", title, Field.Store.NO)); //$NON-NLS-1$
+ doc.add(new TextField("exact_title", title, Field.Store.NO)); //$NON-NLS-1$
+ doc.add(new StoredField("raw_title", title)); //$NON-NLS-1$
}
public void setSummary(String summary) {
- doc.add(new Field("summary", summary, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$
+ doc.add(new StoredField("summary", summary)); //$NON-NLS-1$
}
public void addContents(String contents) {
- doc.add(new Field("contents", new StringReader(contents))); //$NON-NLS-1$
- doc.add(new Field("exact_contents", new StringReader(contents))); //$NON-NLS-1$
+ doc.add(new TextField("contents", new StringReader(contents))); //$NON-NLS-1$
+ doc.add(new TextField("exact_contents", new StringReader(contents))); //$NON-NLS-1$
}
public void setHasFilters(boolean hasFilters) {
- doc.add(new Field("filters", Boolean.toString(hasFilters), Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$
+ doc.add(new StoredField("filters", Boolean.toString(hasFilters))); //$NON-NLS-1$
}
public Document getDocument() {
@@ -54,8 +57,8 @@
}
public void addContents(Reader contents, Reader exactContents) {
- doc.add(new Field("contents", contents)); //$NON-NLS-1$
- doc.add(new Field("exact_contents", exactContents)); //$NON-NLS-1$
+ doc.add(new TextField("contents", contents)); //$NON-NLS-1$
+ doc.add(new TextField("exact_contents", exactContents)); //$NON-NLS-1$
}
}
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java
index ca9cd67..7952c8d 100644
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java
@@ -1,5 +1,5 @@
/*******************************************************************************
- * Copyright (c) 2005, 2014 IBM Corporation and others.
+ * Copyright (c) 2005, 2015 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -7,6 +7,7 @@
*
* Contributors:
* IBM Corporation - initial API and implementation
+ * Sopot Cela - Bug 466829 - Migration to Lucene 5
*******************************************************************************/
package org.eclipse.help.internal.search;
@@ -152,6 +153,7 @@
.getProperty(SearchIndex.DEPENDENCIES_KEY_ANALYZER);
if (!targetIndex.isLuceneCompatible(lucene)
|| !targetIndex.isAnalyzerCompatible(analyzer)) {
+ HelpBasePlugin.logError("Error trying to consume Lucene index from bundle "+bundle.toString()+". Please use an index built with Lucene 5 or higher.", null); //$NON-NLS-1$ //$NON-NLS-2$
return false;
}
} catch (MalformedURLException mue) {
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java
deleted file mode 100644
index 08cf58a..0000000
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryBuilder.java
+++ /dev/null
@@ -1,459 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2000, 2012 IBM Corporation and others.
- * All rights reserved. This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License v1.0
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/epl-v10.html
- *
- * Contributors:
- * IBM Corporation - initial API and implementation
- * Chris Torrence - patch for bug Bug 107648
- *******************************************************************************/
-package org.eclipse.help.internal.search;
-import java.io.*;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.StringTokenizer;
-
-import org.apache.lucene.analysis.*;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.index.*;
-import org.apache.lucene.search.*;
-import org.eclipse.help.internal.base.*;
-/**
- * Build query acceptable by the search engine.
- */
-public class QueryBuilder {
- // Maximum allowed number of terms
- private static final int MAX_TERMS = 10;
- // Maximum allowed number of ORs
- private static final int MAX_UNIONS = 4;
- // Maximum allowed number terms with wild cards
- private static final int MAX_WILD_TERMS = 2;
- // Query from user
- private String searchWords;
- // Descriptor of Analyzer to process the query words
- private AnalyzerDescriptor analyzerDesc;
- // Analyzer to process the query words
- private Analyzer analyzer;
- // List of QueryWordsToken
- private List<QueryWordsToken> analyzedTokens;
- // List of words to highlight
- private List<String> highlightWords = new ArrayList<String>();
- private Locale locale;
- /**
- * Creates a query builder for the search word. The search word is processed
- * by a lexical analyzer.
- */
- public QueryBuilder(String searchWords, AnalyzerDescriptor analyzerDesc) {
- this.searchWords = searchWords;
- String language = analyzerDesc.getLang();
- if (language.length() >= 5) {
- this.locale = new Locale(language.substring(0, 2), language
- .substring(3, 5));
- } else {
- this.locale = new Locale(language.substring(0, 2), ""); //$NON-NLS-1$
- }
- this.analyzerDesc = analyzerDesc;
- this.analyzer = analyzerDesc.getAnalyzer();
- }
- /**
- * Splits user query into tokens and returns a list of QueryWordsToken's.
- */
- private List<QueryWordsToken> tokenizeUserQuery(String searchWords) {
- List<QueryWordsToken> tokenList = new ArrayList<QueryWordsToken>();
- //Divide along quotation marks
- //StringTokenizer qTokenizer = new StringTokenizer(searchWords.trim(),
- // "\"", true); //$NON-NLS-1$
- boolean withinQuotation = false;
- String quotedString = ""; //$NON-NLS-1$
- int termCount = 0;// keep track of number of terms to disallow too many
-
- int fromIndex = -1;
- searchWords = searchWords.trim();
- while((fromIndex = searchWords.indexOf("\"", fromIndex+1))!= -1){ //$NON-NLS-1$
- withinQuotation = !withinQuotation;
- }
- if( withinQuotation ) {
- searchWords = searchWords + "\""; //$NON-NLS-1$
- withinQuotation = !withinQuotation;
- }
-
- StringTokenizer qTokenizer = new StringTokenizer(searchWords,"\"",true); //$NON-NLS-1$
- int orCount = 0; // keep track of number of ORs to disallow too many
- while (qTokenizer.hasMoreTokens()) {
- String curToken = qTokenizer.nextToken();
- if (curToken.equals("\"")) { //$NON-NLS-1$
- if (withinQuotation) {
- // check for too many terms
- if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER
- && ++termCount > MAX_TERMS) {
- throw new QueryTooComplexException();
- }
- tokenList.add(QueryWordsToken.exactPhrase(quotedString));
- } else {
- quotedString = ""; //$NON-NLS-1$
- }
- withinQuotation = !withinQuotation;
- continue;
- } else if (withinQuotation) {
- quotedString = curToken;
- continue;
- } else {
- //divide unquoted strings along white space
- StringTokenizer parser = new StringTokenizer(curToken.trim());
- while (parser.hasMoreTokens()) {
- String token = parser.nextToken();
- if (token.equalsIgnoreCase(QueryWordsToken.AND().value)) {
- tokenList.add(QueryWordsToken.AND());
- } else if (token
- .equalsIgnoreCase(QueryWordsToken.OR().value)) {
- // Check for too many OR terms
- if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER
- && ++orCount > MAX_UNIONS) {
- throw new QueryTooComplexException();
- }
- tokenList.add(QueryWordsToken.OR());
- } else if (token
- .equalsIgnoreCase(QueryWordsToken.NOT().value)) {
- tokenList.add(QueryWordsToken.NOT());
- } else {
- // check for too many terms
- if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER
- && ++termCount > MAX_TERMS) {
- throw new QueryTooComplexException();
- }
- tokenList.add(QueryWordsToken.word(token));
- }
- }
- }
- }
- return tokenList;
- }
- /**
- * Apply the Analyzer to the search tokens and return the list of processed
- * QueryWordsToken's.
- */
- private List<QueryWordsToken> analyzeTokens(List<QueryWordsToken> tokens) {
- boolean isTokenAfterNot = false;
- List<QueryWordsToken> newTokens = new ArrayList<QueryWordsToken>();
- int wildCardTermCount = 0;
- for (int i = 0; i < tokens.size(); i++) {
- QueryWordsToken token = tokens.get(i);
- if (token.type == QueryWordsToken.WORD) {
- int questionMIndex = token.value.indexOf('?');
- int starIndex = token.value.indexOf('*');
- if (starIndex >= 0 || questionMIndex >= 0) {
- if (BaseHelpSystem.getMode() == BaseHelpSystem.MODE_INFOCENTER
- && ++wildCardTermCount > MAX_WILD_TERMS) {
- throw new QueryTooComplexException();
- }
- newTokens.add(QueryWordsToken.word(token.value
- .toLowerCase(locale)));
- // add word to the list of words to highlight
- if (!isTokenAfterNot && !highlightWords.contains(token.value)) {
- highlightWords.add(token.value);
- }
- } else {
- List<String> wordList = analyzeText(analyzer, "contents", //$NON-NLS-1$
- token.value);
- if (wordList.size() > 0) {
- if (!isTokenAfterNot && !highlightWords.contains(token.value)) {
- // add original word to the list of words to
- // highlight
- highlightWords.add(token.value);
- }
- if (wordList.size() == 1) {
- String word = wordList.get(0);
- newTokens.add(QueryWordsToken.word(word));
- // add analyzed word to the list of words to
- // highlight
- // this is required to highlight stemmed words
- if (!isTokenAfterNot && !highlightWords.contains(word)) {
- highlightWords.add(word);
- }
- } else {
- QueryWordsPhrase phrase = QueryWordsToken.phrase();
- for (Iterator<String> it = wordList.iterator(); it
- .hasNext();) {
- String word = it.next();
- phrase.addWord(word);
- // add each analyzed word to the list of words
- // to highlight
- // this is only required to highlight stemmed
- // words.
- // Adding words should not be done when
- // DefaultAnalyzer is used,
- // because it does not perform stemming and
- // common words removal
- // which would result in common characters
- // highlighted all over (bug 30263)
- if (!analyzerDesc.getId().startsWith(
- HelpBasePlugin.PLUGIN_ID + "#")) { //$NON-NLS-1$
- if (!isTokenAfterNot && !highlightWords.contains(word)) {
- highlightWords.add(word);
- }
- }
- }
- newTokens.add(phrase);
- }
- }
- }
- } else if (// forget ANDs
- /*
- * token.type == SearchQueryToken.AND ||
- */
- token.type == QueryWordsToken.OR
- || token.type == QueryWordsToken.NOT)
- newTokens.add(token);
- else if (token.type == QueryWordsToken.EXACT_PHRASE) {
- List<String> wordList = analyzeText(analyzer, "exact_contents", //$NON-NLS-1$
- token.value);
- if (wordList.size() > 0) {
- if (!isTokenAfterNot && !highlightWords.contains(token.value)) {
- // add original word to the list of words to highlight
- highlightWords.add(token.value);
- }
- }
- QueryWordsExactPhrase phrase = QueryWordsToken.exactPhrase();
- for (Iterator<String> it = wordList.iterator(); it.hasNext();) {
- String word = it.next();
- phrase.addWord(word);
- // add analyzed word to the list of words to highlight
- // if (!highlightWords.contains(word))
- // highlightWords.add(word);
- }
- // add phrase only if not empty
- if (phrase.getWords().size() > 0) {
- newTokens.add(phrase);
- }
- }
- isTokenAfterNot = (token.type == QueryWordsToken.NOT);
- }
- return newTokens;
- }
- /**
- * Get a list of tokens corresponding to a search word or phrase
- *
- * @return List of String
- */
- private List<String> analyzeText(Analyzer analyzer, String fieldName, String text) {
- List<String> words = new ArrayList<String>(1);
- Reader reader = new StringReader(text);
- TokenStream tStream = analyzer.tokenStream(fieldName, reader);
-
- CharTermAttribute termAttribute = tStream.getAttribute(CharTermAttribute.class);
- try {
- while (tStream.incrementToken()) {
- String term = termAttribute.toString();
- words.add(term);
- }
- reader.close();
- } catch (IOException ioe) {
- }
-
- return words;
- }
- /**
- * Obtains Lucene Query from tokens
- *
- * @return Query or null if no query could be created
- */
- private Query createLuceneQuery(List<QueryWordsToken> searchTokens, String[] fieldNames,
- float[] boosts) {
- // Get queries for parts separated by OR
- List<Query> requiredQueries = getRequiredQueries(searchTokens, fieldNames,
- boosts);
- if (requiredQueries.size() == 0)
- return null;
- else if (requiredQueries.size() <= 1)
- return requiredQueries.get(0);
- else
- /* if (requiredQueries.size() > 1) */
- // OR queries
- return (orQueries(requiredQueries));
- }
- /**
- * Obtains Lucene queries for token sequences separated at OR.
- *
- * @return List of Query (could be empty)
- */
- private List<Query> getRequiredQueries(List<QueryWordsToken> tokens, String[] fieldNames,
- float[] boosts) {
- List<Query> oredQueries = new ArrayList<Query>();
- ArrayList<QueryWordsToken> requiredQueryTokens = new ArrayList<QueryWordsToken>();
- for (int i = 0; i < tokens.size(); i++) {
- QueryWordsToken token = tokens.get(i);
- if (token.type != QueryWordsToken.OR) {
- requiredQueryTokens.add(token);
- } else {
- Query reqQuery = getRequiredQuery(requiredQueryTokens,
- fieldNames, boosts);
- if (reqQuery != null)
- oredQueries.add(reqQuery);
- requiredQueryTokens = new ArrayList<QueryWordsToken>();
- }
- }
- Query reqQuery = getRequiredQuery(requiredQueryTokens, fieldNames,
- boosts);
- if (reqQuery != null)
- oredQueries.add(reqQuery);
- return oredQueries;
- }
- private Query orQueries(Collection<Query> queries) {
- BooleanQuery bq = new BooleanQuery();
- for (Iterator<Query> it = queries.iterator(); it.hasNext();) {
- Query q = it.next();
- bq.add(q, BooleanClause.Occur.SHOULD);
- }
- return bq;
- }
- /**
- * Obtains Lucene Query for tokens containing only AND and NOT operators.
- *
- * @return BooleanQuery or null if no query could be created from the tokens
- */
- private Query getRequiredQuery(List<QueryWordsToken> requiredTokens, String[] fieldNames,
- float[] boosts) {
- BooleanQuery retQuery = new BooleanQuery();
- boolean requiredTermExist = false;
- // Parse tokens left to right
- QueryWordsToken operator = null;
- for (int i = 0; i < requiredTokens.size(); i++) {
- QueryWordsToken token = requiredTokens.get(i);
- if (token.type == QueryWordsToken.AND
- || token.type == QueryWordsToken.NOT) {
- operator = token;
- continue;
- }
- // Creates queries for all fields
- Query qs[] = new Query[fieldNames.length];
- for (int f = 0; f < fieldNames.length; f++) {
- qs[f] = token.createLuceneQuery(fieldNames[f], boosts[f]);
- }
- // creates the boolean query of all fields
- Query q = qs[0];
- if (fieldNames.length > 1) {
- BooleanQuery allFieldsQuery = new BooleanQuery();
- for (int f = 0; f < fieldNames.length; f++)
- allFieldsQuery.add(qs[f], BooleanClause.Occur.SHOULD);
- q = allFieldsQuery;
- }
- if (operator != null && operator.type == QueryWordsToken.NOT) {
- retQuery.add(q, BooleanClause.Occur.MUST_NOT); // add as prohibited
- } else {
- retQuery.add(q, BooleanClause.Occur.MUST); // add as required
- requiredTermExist = true;
- }
- }
- if (!requiredTermExist) {
- return null; // cannot search for prohibited only
- }
- return retQuery;
- }
- private Query getLuceneQuery(String[] fieldNames, float[] boosts) {
- Query luceneQuery = createLuceneQuery(analyzedTokens, fieldNames,
- boosts);
- return luceneQuery;
- }
- /**
- * @param fieldNames -
- * Collection of field names of type String (e.g. "h1"); the
- * search will be performed on the given fields
- * @param fieldSearchOnly -
- * boolean indicating if field only search should be performed;
- * if set to false, default field "contents" and all other fields
- * will be searched
- */
- public Query getLuceneQuery(Collection<String> fieldNames, boolean fieldSearchOnly)
- throws QueryTooComplexException {
- // split search query into tokens
- List<QueryWordsToken> userTokens = tokenizeUserQuery(searchWords);
- analyzedTokens = analyzeTokens(userTokens);
- return buildLuceneQuery(fieldNames, fieldSearchOnly);
- }
- /**
- * @param fieldNames -
- * Collection of field names of type String (e.g. "h1"); the
- * search will be performed on the given fields
- * @param fieldSearchOnly -
- * boolean indicating if field only search should be performed;
- * if set to false, default field "contents" and all other fields
- * will be searched
- */
- private Query buildLuceneQuery(Collection<String> fieldNames,
- boolean fieldSearchOnly) {
- String[] fields;
- float[] boosts;
- if (fieldSearchOnly) {
- fields = new String[fieldNames.size()];
- boosts = new float[fieldNames.size()];
- Iterator<String> fieldNamesIt = fieldNames.iterator();
- for (int i = 0; i < fieldNames.size(); i++) {
- fields[i] = fieldNamesIt.next();
- boosts[i] = 5.0f;
- }
- } else {
- fields = new String[fieldNames.size() + 2];
- boosts = new float[fieldNames.size() + 2];
- Iterator<String> fieldNamesIt = fieldNames.iterator();
- for (int i = 0; i < fieldNames.size(); i++) {
- fields[i] = fieldNamesIt.next();
- boosts[i] = 5.0f;
- }
- fields[fieldNames.size()] = "contents"; //$NON-NLS-1$
- boosts[fieldNames.size()] = 1.0f;
- fields[fieldNames.size()+1] = "title"; //$NON-NLS-1$
- boosts[fieldNames.size()+1] = 1.0f;
- }
- Query query = getLuceneQuery(fields, boosts);
- query = improveRankingForUnqotedPhrase(query, fields, boosts);
- return query;
- }
- /**
- * If user query contained only words (no quotaions nor operators) extends
- * query with term phrase representing entire user query i.e for user string
- * a b, the query a AND b will be extended to "a b" OR a AND b
- */
- private Query improveRankingForUnqotedPhrase(Query query, String[] fields,
- float[] boosts) {
- if (query == null)
- return query;
- // check if all tokens are words
- for (int i = 0; i < analyzedTokens.size(); i++)
- if (analyzedTokens.get(i).type != QueryWordsToken.WORD)
- return query;
- // Create phrase query for all tokens and OR with original query
- BooleanQuery booleanQuery = new BooleanQuery();
- booleanQuery.add(query, BooleanClause.Occur.SHOULD);
- PhraseQuery[] phraseQueries = new PhraseQuery[fields.length];
- for (int f = 0; f < fields.length; f++) {
- phraseQueries[f] = new PhraseQuery();
- for (int i = 0; i < analyzedTokens.size(); i++) {
- Term t = new Term(fields[f], analyzedTokens
- .get(i).value);
- phraseQueries[f].add(t);
- }
- phraseQueries[f].setBoost(10 * boosts[f]);
- booleanQuery.add(phraseQueries[f], BooleanClause.Occur.SHOULD);
- }
- return booleanQuery;
- }
- /**
- * Obtains analyzed terms from query as one string. Words are double quoted,
- * and separated by space. The analyzed words are needed for highlighting
- * word roots.
- */
- public String gethighlightTerms() {
- StringBuffer buf = new StringBuffer();
- for (Iterator<String> it = highlightWords.iterator(); it.hasNext();) {
- buf.append('"');
- buf.append(it.next());
- buf.append("\" "); //$NON-NLS-1$
- }
- return buf.toString();
- }
-}
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsExactPhrase.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsExactPhrase.java
deleted file mode 100644
index 324b8e1..0000000
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsExactPhrase.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2000, 2011 IBM Corporation and others.
- * All rights reserved. This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License v1.0
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/epl-v10.html
- *
- * Contributors:
- * IBM Corporation - initial API and implementation
- *******************************************************************************/
-package org.eclipse.help.internal.search;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.lucene.index.*;
-import org.apache.lucene.search.*;
-/**
- * Represents a quoted token in user search query words
- */
-public class QueryWordsExactPhrase extends QueryWordsToken {
- private List<String> words;
- public QueryWordsExactPhrase() {
- super(QueryWordsToken.EXACT_PHRASE, ""); //$NON-NLS-1$
- words = new ArrayList<String>();
- }
- public void addWord(String word) {
- words.add(word);
- if (words.size() <= 1)
- value = word;
- else
- value += " " + word; //$NON-NLS-1$
- }
- public List<String> getWords() {
- return words;
- }
- /**
- * Creates a lucene query for a field
- */
- public Query createLuceneQuery(String field, float boost) {
- PhraseQuery q = new PhraseQuery();
- for (Iterator<String> it = getWords().iterator(); it.hasNext();) {
- String word = it.next();
- Term t = new Term("exact_" + field, word); //$NON-NLS-1$
- q.add(t);
- q.setBoost(boost);
- }
- return q;
- }
-}
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsPhrase.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsPhrase.java
deleted file mode 100644
index 8a94e89..0000000
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsPhrase.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2000, 2011 IBM Corporation and others.
- * All rights reserved. This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License v1.0
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/epl-v10.html
- *
- * Contributors:
- * IBM Corporation - initial API and implementation
- *******************************************************************************/
-package org.eclipse.help.internal.search;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.lucene.index.*;
-import org.apache.lucene.search.*;
-/**
- * Represents a phrase (not quoted) token in user search query words It consists
- * of several words created by an analyzer
- */
-public class QueryWordsPhrase extends QueryWordsToken {
- private List<String> words;
- public QueryWordsPhrase() {
- super(QueryWordsToken.PHRASE, ""); //$NON-NLS-1$
- words = new ArrayList<String>();
- }
- public void addWord(String word) {
- words.add(word);
- if (words.size() <= 1)
- value = word;
- else
- value += " " + word; //$NON-NLS-1$
- }
- public List<String> getWords() {
- return words;
- }
- /**
- * Creates a lucene query for a field
- */
- public Query createLuceneQuery(String field, float boost) {
- PhraseQuery q = new PhraseQuery();
- for (Iterator<String> it = getWords().iterator(); it.hasNext();) {
- String word = it.next();
- Term t = new Term(field, word);
- q.add(t);
- q.setBoost(boost);
- }
- return q;
- }
-}
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsToken.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsToken.java
deleted file mode 100644
index 6ba76f2..0000000
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/QueryWordsToken.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2000, 2007 IBM Corporation and others.
- * All rights reserved. This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License v1.0
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/epl-v10.html
- *
- * Contributors:
- * IBM Corporation - initial API and implementation
- *******************************************************************************/
-package org.eclipse.help.internal.search;
-import org.apache.lucene.index.*;
-import org.apache.lucene.search.*;
-/**
- * Represents a token in user search query words
- */
-public class QueryWordsToken {
- public static final int AND = 0;
- public static final int OR = 1;
- public static final int NOT = 2;
- public static final int EXACT_PHRASE = 3;
- public static final int PHRASE = 4;
- public static final int WORD = 5;
- private static final QueryWordsToken fAND = new QueryWordsToken(AND, "AND"); //$NON-NLS-1$
- private static final QueryWordsToken fOR = new QueryWordsToken(OR, "OR"); //$NON-NLS-1$
- private static final QueryWordsToken fNOT = new QueryWordsToken(NOT, "NOT"); //$NON-NLS-1$
- public int type;
- public String value;
- protected QueryWordsToken(int type, String value) {
- this.type = type;
- this.value = value;
- }
- /**
- * Creates a lucene query for a field
- */
- public Query createLuceneQuery(String field, float boost) {
- Query q;
- int questionPos = value.indexOf('?');
- int starPos = value.indexOf('*');
- if (questionPos >= 0 || starPos >= 0) {
- if (questionPos == -1 && starPos == value.length() - 1) {
- Term t = new Term("exact_" + field, value.substring(0, starPos)); //$NON-NLS-1$
- q = new PrefixQuery(t);
- ((PrefixQuery) q).setBoost(boost);
- } else {
- Term t = new Term("exact_" + field, value); //$NON-NLS-1$
- q = new WildcardQuery(t);
- ((WildcardQuery) q).setBoost(boost);
- }
- } else {
- Term t = new Term(field, value);
- q = new TermQuery(t);
- ((TermQuery) q).setBoost(boost);
- }
- // after updating Lucene, set boost on a Query class
- return q;
- }
- public static QueryWordsToken AND() {
- return fAND;
- }
- public static QueryWordsToken OR() {
- return fOR;
- }
- public static QueryWordsToken NOT() {
- return fNOT;
- }
- public static QueryWordsToken word(String word) {
- return new QueryWordsToken(QueryWordsToken.WORD, word);
- }
- public static QueryWordsPhrase phrase() {
- return new QueryWordsPhrase();
- }
- public static QueryWordsExactPhrase exactPhrase() {
- return new QueryWordsExactPhrase();
- }
- public static QueryWordsExactPhrase exactPhrase(String word) {
- QueryWordsExactPhrase token = new QueryWordsExactPhrase();
- token.addWord(word);
- return token;
- }
-}
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java
index 67963a5..131ac65 100644
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java
@@ -1,14 +1,13 @@
/*******************************************************************************
- * Copyright (c) 2000, 2014 IBM Corporation and others.
- * All rights reserved. This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License v1.0
+ * Copyright (c) 2000, 2015 IBM Corporation and others. All rights reserved. This program and the
+ * accompanying materials are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
- * Contributors:
- * IBM Corporation - initial API and implementation
- * Holger Voormann - fix for bug 426785 (http://eclip.se/426785)
- * Alexander Kurtakov - Bug 460787
+ * Contributors: IBM Corporation - initial API and implementation
+ * Holger Voormann - fix for bug 426785 (http://eclip.se/426785)
+ * Alexander Kurtakov - Bug 460787
+ * Sopot Cela - Bug 466829 - Migration to Lucene 5
*******************************************************************************/
package org.eclipse.help.internal.search;
@@ -17,6 +16,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
+import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.channels.FileLock;
@@ -33,19 +33,30 @@
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
-import org.apache.lucene.analysis.LimitTokenCountAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.LimitTokenCountAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LogByteSizeMergePolicy;
import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queryparser.classic.FastCharStream;
+import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
+import org.apache.lucene.queryparser.classic.QueryParserTokenManager;
+import org.apache.lucene.queryparser.classic.Token;
+import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
@@ -59,7 +70,6 @@
import org.eclipse.core.runtime.OperationCanceledException;
import org.eclipse.core.runtime.Platform;
import org.eclipse.core.runtime.Status;
-import org.eclipse.help.internal.HelpPlugin;
import org.eclipse.help.internal.base.BaseHelpSystem;
import org.eclipse.help.internal.base.HelpBasePlugin;
import org.eclipse.help.internal.base.util.HelpProperties;
@@ -86,7 +96,7 @@
private File indexDir;
- private Directory luceneDirectory;
+ public Directory luceneDirectory;
private String locale;
@@ -133,7 +143,7 @@
private Collection<Thread> searches = new ArrayList<Thread>();
private FileLock lock;
- private RandomAccessFile raf = null;
+ private RandomAccessFile raf = null;
/**
* Constructor.
@@ -170,7 +180,7 @@
inconsistencyFile = new File(indexDir.getParentFile(), locale + ".inconsistent"); //$NON-NLS-1$
htmlSearchParticipant = new HTMLSearchParticipant(indexDir.getAbsolutePath());
try {
- luceneDirectory = new NIOFSDirectory(indexDir);
+ luceneDirectory = new NIOFSDirectory(indexDir.toPath());
} catch (IOException e) {
}
if (!exists()) {
@@ -203,11 +213,11 @@
public IStatus addDocument(String name, URL url) {
try {
Document doc = new Document();
- doc.add(new Field(FIELD_NAME, name, Field.Store.YES, Field.Index.NOT_ANALYZED));
+ doc.add(new StringField(FIELD_NAME, name, Field.Store.YES));
addExtraFields(doc);
String pluginId = LocalSearchManager.getPluginId(name);
if (relativePath != null) {
- doc.add(new Field(FIELD_INDEX_ID, relativePath, Field.Store.YES, Field.Index.NOT_ANALYZED));
+ doc.add(new StringField(FIELD_INDEX_ID, relativePath, Field.Store.YES));
}
// check for the explicit search participant.
SearchParticipant participant = null;
@@ -220,20 +230,22 @@
if (participant == null)
participant = BaseHelpSystem.getLocalSearchManager().getParticipant(pluginId, name);
if (participant != null) {
- IStatus status = participant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc));
+ IStatus status = participant.addDocument(this, pluginId, name, url, id,
+ new LuceneSearchDocument(doc));
if (status.getSeverity() == IStatus.OK) {
String filters = doc.get("filters"); //$NON-NLS-1$
indexedDocs.put(name, filters != null ? filters : "0"); //$NON-NLS-1$
if (id != null)
- doc.add(new Field("id", id, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$
+ doc.add(new StoredField("id", id)); //$NON-NLS-1$
if (pid != null)
- doc.add(new Field("participantId", pid, Field.Store.YES, Field.Index.NO)); //$NON-NLS-1$
+ doc.add(new StoredField("participantId", pid)); //$NON-NLS-1$
iw.addDocument(doc);
}
return status;
}
// default to html
- IStatus status = htmlSearchParticipant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc));
+ IStatus status = htmlSearchParticipant.addDocument(this, pluginId, name, url, id,
+ new LuceneSearchDocument(doc));
if (status.getSeverity() == IStatus.OK) {
String filters = doc.get("filters"); //$NON-NLS-1$
indexedDocs.put(name, filters != null ? filters : "0"); //$NON-NLS-1$
@@ -245,20 +257,21 @@
"IO exception occurred while adding document " + name //$NON-NLS-1$
+ " to index " + indexDir.getAbsolutePath() + ".", //$NON-NLS-1$ //$NON-NLS-2$
e);
- }
- catch (Exception e) {
+ } catch (Exception e) {
return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR,
"An unexpected internal error occurred while adding document " //$NON-NLS-1$
+ name + " to index " + indexDir.getAbsolutePath() //$NON-NLS-1$
- + ".", e); //$NON-NLS-1$
+ + ".", //$NON-NLS-1$
+ e);
}
}
/**
- * Add any extra fields that need to be added to this document. Subclasses
- * should override to add more fields.
+ * Add any extra fields that need to be added to this document. Subclasses should override to
+ * add more fields.
*
- * @param doc the document to add fields to
+ * @param doc
+ * the document to add fields to
*/
protected void addExtraFields(Document doc) {
}
@@ -282,8 +295,9 @@
indexedDocs = new HelpProperties(INDEXED_DOCS_FILE, indexDir);
indexedDocs.restore();
setInconsistent(true);
- LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), 1000000);
- IndexWriterConfig writerConfig = new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_31, analyzer);
+ LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(),
+ 1000000);
+ IndexWriterConfig writerConfig = new IndexWriterConfig(analyzer);
writerConfig.setOpenMode(create ? OpenMode.CREATE : OpenMode.APPEND);
LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
mergePolicy.setMergeFactor(20);
@@ -307,7 +321,7 @@
indexedDocs = new HelpProperties(INDEXED_DOCS_FILE, indexDir);
indexedDocs.restore();
setInconsistent(true);
- ir = IndexReader.open(luceneDirectory, false);
+ ir = DirectoryReader.open(luceneDirectory);
return true;
} catch (IOException e) {
HelpBasePlugin.logError("Exception occurred in search indexing at beginDeleteBatch.", e); //$NON-NLS-1$
@@ -323,7 +337,7 @@
if (ir != null) {
ir.close();
}
- ir = IndexReader.open(luceneDirectory, false);
+ ir = DirectoryReader.open(luceneDirectory);
return true;
} catch (IOException e) {
HelpBasePlugin.logError("Exception occurred in search indexing at beginDeleteBatch.", e); //$NON-NLS-1$
@@ -334,14 +348,14 @@
/**
* Deletes a single document from the index.
*
- * @param name -
- * document name
+ * @param name
+ * - document name
* @return IStatus
*/
public IStatus removeDocument(String name) {
Term term = new Term(FIELD_NAME, name);
try {
- ir.deleteDocuments(term);
+ iw.deleteDocuments(term);
indexedDocs.remove(name);
} catch (IOException e) {
return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR,
@@ -375,11 +389,11 @@
}
/*
- * The searcher's index reader has it's stuff in memory so it won't
- * know about this change. Close it so that it gets reloaded next search.
+ * The searcher's index reader has it's stuff in memory so it won't know about this
+ * change. Close it so that it gets reloaded next search.
*/
if (searcher != null) {
- searcher.close();
+ searcher.getIndexReader().close();
searcher = null;
}
return true;
@@ -407,11 +421,11 @@
saveDependencies();
/*
- * The searcher's index reader has it's stuff in memory so it won't
- * know about this change. Close it so that it gets reloaded next search.
+ * The searcher's index reader has it's stuff in memory so it won't know about this
+ * change. Close it so that it gets reloaded next search.
*/
if (searcher != null) {
- searcher.close();
+ searcher.getIndexReader().close();
searcher = null;
}
return true;
@@ -469,11 +483,11 @@
String indexId = indexIds.get(i);
String indexPath = indexPaths.get(i);
try {
- dirList.add(new NIOFSDirectory(new File(indexPath)));
+ dirList.add(new NIOFSDirectory(new File(indexPath).toPath()));
} catch (IOException ioe) {
- HelpBasePlugin
- .logError(
- "Help search indexing directory could not be created for directory " + indexPath, ioe); //$NON-NLS-1$
+ HelpBasePlugin.logError(
+ "Help search indexing directory could not be created for directory " + indexPath, //$NON-NLS-1$
+ ioe);
continue;
}
@@ -525,18 +539,19 @@
}
public IStatus removeDuplicates(String name, String[] index_paths) {
- TermDocs hrefDocs = null;
- TermDocs indexDocs = null;
- Term hrefTerm = new Term(FIELD_NAME, name);
try {
+ LeafReader ar = SlowCompositeReaderWrapper.wrap(ir);
+ PostingsEnum hrefDocs = null;
+ PostingsEnum indexDocs = null;
+ Term hrefTerm = new Term(FIELD_NAME, name);
for (int i = 0; i < index_paths.length; i++) {
Term indexTerm = new Term(FIELD_INDEX_ID, index_paths[i]);
if (i == 0) {
- hrefDocs = ir.termDocs(hrefTerm);
- indexDocs = ir.termDocs(indexTerm);
+ hrefDocs = ar.postings(hrefTerm);
+ indexDocs = ar.postings(indexTerm);
} else {
- hrefDocs.seek(hrefTerm);
- indexDocs.seek(indexTerm);
+ hrefDocs = ar.postings(hrefTerm);
+ indexDocs = ar.postings(indexTerm);
}
removeDocuments(hrefDocs, indexDocs);
}
@@ -545,19 +560,6 @@
"IO exception occurred while removing duplicates of document " + name //$NON-NLS-1$
+ " from index " + indexDir.getAbsolutePath() + ".", //$NON-NLS-1$ //$NON-NLS-2$
ioe);
- } finally {
- if (hrefDocs != null) {
- try {
- hrefDocs.close();
- } catch (IOException e) {
- }
- }
- if (indexDocs != null) {
- try {
- indexDocs.close();
- } catch (IOException e) {
- }
- }
}
return Status.OK_STATUS;
}
@@ -569,33 +571,33 @@
* @param docs2
* @throws IOException
*/
- private void removeDocuments(TermDocs doc1, TermDocs docs2) throws IOException {
- if (!doc1.next()) {
+ private void removeDocuments(PostingsEnum doc1, PostingsEnum docs2) throws IOException {
+ if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) {
return;
}
- if (!docs2.next()) {
+ if (docs2.nextDoc() == PostingsEnum.NO_MORE_DOCS) {
return;
}
while (true) {
- if (doc1.doc() < docs2.doc()) {
- if (!doc1.skipTo(docs2.doc())) {
- if (!doc1.next()) {
+ if (doc1.docID() < docs2.docID()) {
+ if (doc1.advance(docs2.docID()) == PostingsEnum.NO_MORE_DOCS) {
+ if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) {
return;
}
}
- } else if (doc1.doc() > docs2.doc()) {
- if (!docs2.skipTo(doc1.doc())) {
- if (!doc1.next()) {
+ } else if (doc1.docID() > docs2.docID()) {
+ if (docs2.advance(doc1.docID()) == PostingsEnum.NO_MORE_DOCS) {
+ if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) {
return;
}
}
}
- if (doc1.doc() == docs2.doc()) {
- ir.deleteDocument(doc1.doc());
- if (!doc1.next()) {
+ if (doc1.docID() == docs2.docID()) {
+ iw.tryDeleteDocument(ir, doc1.docID());
+ if (doc1.nextDoc() == PostingsEnum.NO_MORE_DOCS) {
return;
}
- if (!docs2.next()) {
+ if (docs2.nextDoc() == PostingsEnum.NO_MORE_DOCS) {
return;
}
}
@@ -623,20 +625,72 @@
registerSearch(Thread.currentThread());
if (closed)
return;
- QueryBuilder queryBuilder = new QueryBuilder(searchQuery.getSearchWord(), analyzerDescriptor);
- Query luceneQuery = queryBuilder.getLuceneQuery(searchQuery.getFieldNames(), searchQuery
- .isFieldSearch());
- if (HelpPlugin.DEBUG_SEARCH) {
- System.out.println("Search Query: " + luceneQuery.toString()); //$NON-NLS-1$
+
+ String[] fields;
+ if (searchQuery.isFieldSearch()){
+ //sometimes you might want to search other than the default fields
+ fields = (String[]) searchQuery.getFieldNames().toArray();
+ }else {
+ fields = new String[]{"contents","title"}; //$NON-NLS-1$ //$NON-NLS-2$
}
- String highlightTerms = queryBuilder.gethighlightTerms();
- if (luceneQuery != null) {
+
+ //prepare the parser
+
+ MultiFieldQueryParser qb = new MultiFieldQueryParser(fields,analyzerDescriptor.getAnalyzer());
+ qb.setAllowLeadingWildcard(true);
+ qb.setAnalyzeRangeTerms(true);
+ qb.setAutoGeneratePhraseQueries(true);
+ qb.setMultiTermRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE);
+ qb.setLowercaseExpandedTerms(true);
+ qb.setLocale(new Locale(analyzerDescriptor.getLang()));
+
+ //parse the "pure" query (no boosting)
+ Query luceneQuery = qb.parse(searchQuery.getSearchWord());
+
+ //we'll merge the pure query with a some boosted queries
+ Query mergedQuery;
+
+ if (!isWildcardQuery(searchQuery.getSearchWord())){
+ mergedQuery = new BooleanQuery(); //merge for all fields before merging with luceneQuery
+ for (int i=0;i<fields.length;i++){
+ Query exactBoostQuery= qb.createPhraseQuery(fields[i], searchQuery.getSearchWord());
+ exactBoostQuery.setBoost(10.0f);
+ ((BooleanQuery) mergedQuery).add(exactBoostQuery,Occur.SHOULD);
+ }
+ ((BooleanQuery) mergedQuery).add(luceneQuery,Occur.SHOULD);
+
+ }else {
+ mergedQuery = luceneQuery;
+ }
+
+ Set<String> set = new HashSet<String>();
+
+ if (mergedQuery != null) {
if (searcher == null) {
openSearcher();
}
- TopDocs topDocs = searcher.search(luceneQuery, null, 1000);
- collector.addHits(LocalSearchManager.asList(topDocs, searcher), highlightTerms);
+ TopDocs topDocs = searcher.search(mergedQuery, 1000);
+
+ String highlight=null;
+ QueryParserTokenManager manager = new QueryParserTokenManager(new FastCharStream(new StringReader(searchQuery.getSearchWord())));
+ while (true){
+ Token nextToken = manager.getNextToken();
+ String toHighlight = null;
+ if (nextToken.kind==0) break;
+ String image = nextToken.image;
+ toHighlight=image;
+ if ((image.startsWith("\""))&&( image.endsWith("\""))){ //$NON-NLS-1$//$NON-NLS-2$
+ toHighlight = image.substring(1,image.length()-1);
+ }
+ if (image.equals("AND") || image.equals("OR")) //$NON-NLS-1$ //$NON-NLS-2$
+ continue;
+ set .add(toHighlight);
+
+ }
+ highlight = buildHighlight(set);
+ collector.addHits(LocalSearchManager.asList(topDocs, searcher), highlight==null?"":highlight); //$NON-NLS-1$
}
+
} catch (BooleanQuery.TooManyClauses tmc) {
collector.addQTCException(new QueryTooComplexException());
} catch (QueryTooComplexException qe) {
@@ -647,6 +701,19 @@
} finally {
unregisterSearch(Thread.currentThread());
}
+ }
+
+ private boolean isWildcardQuery(String searchWord) {
+
+ return searchWord.contains("?")|| searchWord.contains("*"); //$NON-NLS-1$//$NON-NLS-2$
+ }
+
+ private String buildHighlight(Set<String> set) {
+ StringBuilder sb = new StringBuilder();
+ for (String string : set) {
+ sb.append("\""+string+"\""); //$NON-NLS-1$//$NON-NLS-2$
+ }
+ return sb.toString();
}
public String getLocale() {
@@ -660,13 +727,13 @@
if (docPlugins == null) {
Set<String> totalIds = new HashSet<String>();
IExtensionRegistry registry = Platform.getExtensionRegistry();
- IExtensionPoint extensionPoint = registry.getExtensionPoint(TocFileProvider.EXTENSION_POINT_ID_TOC);
+ IExtensionPoint extensionPoint = registry
+ .getExtensionPoint(TocFileProvider.EXTENSION_POINT_ID_TOC);
IExtension[] extensions = extensionPoint.getExtensions();
- for (int i=0;i<extensions.length;++i) {
+ for (int i = 0; i < extensions.length; ++i) {
try {
totalIds.add(extensions[i].getNamespaceIdentifier());
- }
- catch (InvalidRegistryObjectException e) {
+ } catch (InvalidRegistryObjectException e) {
// ignore this extension and move on
}
}
@@ -717,25 +784,27 @@
/**
* Determines whether an index can be read by the Lucene bundle
- * @param indexVersionString The version of an Index directory
+ *
+ * @param indexVersionString
+ * The version of an Index directory
* @return
*/
public boolean isLuceneCompatible(String indexVersionString) {
- if (indexVersionString==null) return false;
+ if (indexVersionString == null)
+ return false;
String luceneVersionString = ""; //$NON-NLS-1$
Bundle luceneBundle = Platform.getBundle(LUCENE_BUNDLE_ID);
if (luceneBundle != null) {
- luceneVersionString += luceneBundle.getHeaders()
- .get(Constants.BUNDLE_VERSION);
+ luceneVersionString += luceneBundle.getHeaders().get(Constants.BUNDLE_VERSION);
}
Version luceneVersion = new Version(luceneVersionString);
Version indexVersion = new Version(indexVersionString);
- Version v191 = new Version(1, 9, 1);
- if (indexVersion.compareTo(v191) < 0) {
- // index is older than Lucene 1.9.1
+ Version v500 = new Version(5, 0, 0);
+ if (indexVersion.compareTo(v500) < 0) {
+ // index is older than Lucene 5.0.0
return false;
}
- if ( luceneVersion.compareTo(indexVersion) >= 0 ) {
+ if (luceneVersion.compareTo(indexVersion) >= 0) {
// Lucene bundle is newer than the index
return true;
}
@@ -802,7 +871,7 @@
public void openSearcher() throws IOException {
synchronized (searcherCreateLock) {
if (searcher == null) {
- searcher = new IndexSearcher(IndexReader.open(luceneDirectory, false));
+ searcher = new IndexSearcher(DirectoryReader.open(luceneDirectory));
}
}
}
@@ -820,7 +889,7 @@
if (searches.isEmpty()) {
if (searcher != null) {
try {
- searcher.close();
+ searcher.getIndexReader().close();
} catch (IOException ioe) {
}
}
@@ -838,7 +907,8 @@
* Finds and unzips prebuild index specified in preferences
*/
private void unzipProductIndex() {
- String indexPluginId = Platform.getPreferencesService().getString(HelpBasePlugin.PLUGIN_ID, "productIndex", null, null); //$NON-NLS-1$
+ String indexPluginId = Platform.getPreferencesService().getString(HelpBasePlugin.PLUGIN_ID,
+ "productIndex", null, null); //$NON-NLS-1$
if (indexPluginId == null || indexPluginId.length() <= 0) {
return;
}
@@ -901,10 +971,11 @@
*/
private void cleanOldIndex() {
IndexWriter cleaner = null;
- LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(), 10000);
+ LimitTokenCountAnalyzer analyzer = new LimitTokenCountAnalyzer(analyzerDescriptor.getAnalyzer(),
+ 10000);
try {
- cleaner = new IndexWriter(luceneDirectory, new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_31, analyzer).setOpenMode(
- OpenMode.CREATE));
+ cleaner = new IndexWriter(luceneDirectory,
+ new IndexWriterConfig(analyzer).setOpenMode(OpenMode.CREATE));
} catch (IOException ioe) {
} finally {
try {
@@ -957,7 +1028,7 @@
* if lock already obtained
*/
public synchronized boolean tryLock() throws OverlappingFileLockException {
- if ("none".equals(System.getProperty("osgi.locking"))) { //$NON-NLS-1$//$NON-NLS-2$
+ if ("none".equals(System.getProperty("osgi.locking"))) { //$NON-NLS-1$//$NON-NLS-2$
return true; // Act as if lock succeeded
}
if (lock != null) {
@@ -979,7 +1050,7 @@
lock = null;
logLockFailure(ioe);
}
- if ( raf != null ) {
+ if (raf != null) {
try {
raf.close();
} catch (IOException e) {
@@ -1025,9 +1096,9 @@
}
lock = null;
}
- if (raf != null ) {
+ if (raf != null) {
try {
- raf.close();
+ raf.close();
} catch (IOException ioe) {
}
raf = null;
@@ -1100,25 +1171,26 @@
public IStatus addDocument(String pluginId, String name, URL url, String id, Document doc) {
// try a registered participant for the file format
- SearchParticipant participant = BaseHelpSystem.getLocalSearchManager()
- .getParticipant(pluginId, name);
+ SearchParticipant participant = BaseHelpSystem.getLocalSearchManager().getParticipant(pluginId, name);
if (participant != null) {
try {
return participant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc));
- }
- catch (Throwable t) {
+ } catch (Throwable t) {
return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR,
"Error while adding document to search participant (addDocument()): " //$NON-NLS-1$
- + name + ", " + url + "for participant " + participant.getClass().getName(), t); //$NON-NLS-1$ //$NON-NLS-2$
+ + name + ", " + url + "for participant " + participant.getClass().getName(), //$NON-NLS-1$ //$NON-NLS-2$
+ t);
}
}
// default to html
- return htmlSearchParticipant.addDocument(this, pluginId, name, url, id, new LuceneSearchDocument(doc));
+ return htmlSearchParticipant.addDocument(this, pluginId, name, url, id,
+ new LuceneSearchDocument(doc));
}
- public IStatus addSearchableDocument(String pluginId, String name, URL url, String id, ISearchDocument doc) {
+ public IStatus addSearchableDocument(String pluginId, String name, URL url, String id,
+ ISearchDocument doc) {
// In the help system the only class that implements ISearchDocument is LuceneSearchDocument
- LuceneSearchDocument luceneDoc = (LuceneSearchDocument)doc;
+ LuceneSearchDocument luceneDoc = (LuceneSearchDocument) doc;
return addDocument(pluginId, name, url, id, luceneDoc.getDocument());
}
}
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java
index d0a7bb7..444b66c 100644
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/SmartAnalyzer.java
@@ -1,5 +1,5 @@
/*******************************************************************************
- * Copyright (c) 2000, 2012 IBM Corporation and others.
+ * Copyright (c) 2000, 2015 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -7,10 +7,9 @@
*
* Contributors:
* IBM Corporation - initial API and implementation
+ * Sopot Cela - Bug 466829 - Migration to Lucene 5
*******************************************************************************/
package org.eclipse.help.internal.search;
-
-import java.io.*;
import org.apache.lucene.analysis.*;
@@ -18,7 +17,7 @@
* Smart Analyzer. Chooses underlying implementation based on the field which
* text is analyzed.
*/
-public final class SmartAnalyzer extends Analyzer {
+public final class SmartAnalyzer extends AnalyzerWrapper {
Analyzer pluggedInAnalyzer;
Analyzer exactAnalyzer;
@@ -26,19 +25,19 @@
* Constructor for SmartAnalyzer.
*/
public SmartAnalyzer(String locale, Analyzer pluggedInAnalyzer) {
- super();
+ super(pluggedInAnalyzer.getReuseStrategy());
this.pluggedInAnalyzer = pluggedInAnalyzer;
this.exactAnalyzer = new DefaultAnalyzer(locale);
}
/**
- * Creates a TokenStream which tokenizes all the text in the provided
- * Reader. Delegates to DefaultAnalyzer when field used to search for exact
+ * Delegates to DefaultAnalyzer when field used to search for exact
* match, and to plugged-in analyzer for other fields.
*/
- public final TokenStream tokenStream(String fieldName, Reader reader) {
+ @Override
+ public final Analyzer getWrappedAnalyzer(String fieldName) {
if (fieldName != null && fieldName.startsWith("exact_")) { //$NON-NLS-1$
- return exactAnalyzer.tokenStream(fieldName, reader);
+ return exactAnalyzer;
}
- return pluggedInAnalyzer.tokenStream(fieldName, reader);
+ return pluggedInAnalyzer;
}
}
diff --git a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java
index 0b70cf7..79d5592 100644
--- a/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java
+++ b/eclipse.platform.ua/org.eclipse.help.base/src/org/eclipse/help/internal/search/WordTokenStream.java
@@ -1,5 +1,5 @@
/*******************************************************************************
- * Copyright (c) 2000, 2012 IBM Corporation and others.
+ * Copyright (c) 2000, 2015 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -7,16 +7,18 @@
*
* Contributors:
* IBM Corporation - initial API and implementation
+ * Sopot Cela - Bug 466829 - Migration to Lucene 5
*******************************************************************************/
package org.eclipse.help.internal.search;
-import com.ibm.icu.text.BreakIterator;
import java.io.IOException;
-import java.io.Reader;
import java.util.Locale;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+import com.ibm.icu.text.BreakIterator;
/**
* WordTokenStream obtains tokens containing words appropriate for use with
@@ -24,7 +26,6 @@
*/
public final class WordTokenStream extends Tokenizer {
private static final int BUF_LEN = 4096;
- private final Reader reader;
private final BreakIterator boundary;
private StringBuffer strbuf;
@@ -34,8 +35,8 @@
/**
* Constructor
*/
- public WordTokenStream(String fieldName, Reader reader, Locale locale) {
- this.reader = reader;
+ public WordTokenStream(Locale locale) {
+ super();
boundary = BreakIterator.getWordInstance(locale);
}
@@ -52,9 +53,9 @@
if(strbuf == null) {
int available;
char[] cbuf = new char[BUF_LEN];
- while ((available = reader.read(cbuf)) <= 0) {
+ while ((available = input.read(cbuf)) <= 0) {
if (available < 0) {
- reader.close();
+ input.close();
return false;
}
}
@@ -62,7 +63,7 @@
strbuf.append(cbuf, 0, available);
// read more until white space (or EOF)
int c;
- while (0 <= (c = reader.read())) {
+ while (0 <= (c = input.read())) {
strbuf.append((char) c);
if (c == ' ' || c == '\r' || c == '\n' || c == '\t') {
break;
@@ -70,7 +71,7 @@
}
if (c < 0) {
- reader.close();
+ input.close();
}
boundary.setText(strbuf.toString());
@@ -105,9 +106,10 @@
}
public void close() throws IOException {
+ super.close();
/// Unlikely to be called as this is a reused
- if (this.reader != null) {
- this.reader.close();
+ if (this.input != null) {
+ this.input.close();
}
}
}
diff --git a/eclipse.platform.ua/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml b/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml
index 8438100..174e92a 100644
--- a/eclipse.platform.ua/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml
+++ b/eclipse.platform.ua/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml
@@ -74,20 +74,6 @@
unpack="false"/>
<plugin
- id="org.apache.lucene.analysis"
- download-size="0"
- install-size="0"
- version="0.0.0"
- unpack="false"/>
-
- <plugin
- id="org.apache.lucene.core"
- download-size="0"
- install-size="0"
- version="0.0.0"
- unpack="false"/>
-
- <plugin
id="org.eclipse.core.contenttype"
download-size="0"
install-size="0"
@@ -227,4 +213,32 @@
version="0.0.0"
unpack="false"/>
+ <plugin
+ id="org.apache.lucene.analyzers-common"
+ download-size="0"
+ install-size="0"
+ version="0.0.0"
+ unpack="false"/>
+
+ <plugin
+ id="org.apache.lucene.analyzers-smartcn"
+ download-size="0"
+ install-size="0"
+ version="0.0.0"
+ unpack="false"/>
+
+ <plugin
+ id="org.apache.lucene.core"
+ download-size="0"
+ install-size="0"
+ version="0.0.0"
+ unpack="false"/>
+
+ <plugin
+ id="org.apache.lucene.queryparser"
+ download-size="0"
+ install-size="0"
+ version="0.0.0"
+ unpack="false"/>
+
</feature>
diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF b/org.eclipse.ua.tests/META-INF/MANIFEST.MF
index 6bcf9bc..23f8910 100644
--- a/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF
+++ b/eclipse.platform.ua/org.eclipse.ua.tests/META-INF/MANIFEST.MF
@@ -19,14 +19,13 @@
org.eclipse.ui.forms,
org.eclipse.ui.browser;bundle-version="3.2.300",
org.eclipse.equinox.jsp.jasper;bundle-version="1.0.200",
- org.eclipse.equinox.jsp.jasper.registry;bundle-version="1.0.100"
+ org.eclipse.equinox.jsp.jasper.registry;bundle-version="1.0.100",
+ org.apache.lucene.analyzers-common;bundle-version="5.1.0",
+ org.apache.lucene.core;bundle-version="5.1.0"
Bundle-ActivationPolicy: lazy
Bundle-Vendor: Eclipse.org
Import-Package: javax.servlet;version="3.1.0",
- javax.servlet.http;version="3.1.0",
- org.apache.lucene.index;core=split;version="[3.5.0,4.0.0)",
- org.apache.lucene.search;core=split;version="[3.5.0,4.0.0)",
- org.apache.lucene.store;core=split;version="[3.5.0,4.0.0)"
+ javax.servlet.http;version="3.1.0"
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Export-Package: org.eclipse.ua.tests,
org.eclipse.ua.tests.browser,
diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index291/write.lock b/org.eclipse.ua.tests/data/help/searchindex/index291/write.lock
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index291/write.lock
diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfe b/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfe
new file mode 100644
index 0000000..09ec2c9
--- /dev/null
+++ b/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfe
Binary files differ
diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfs b/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfs
new file mode 100644
index 0000000..3aa288a
--- /dev/null
+++ b/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/_0.cfs
Binary files differ
diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/_0.si b/org.eclipse.ua.tests/data/help/searchindex/index510/_0.si
new file mode 100644
index 0000000..d897bfc
--- /dev/null
+++ b/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/_0.si
Binary files differ
diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/segments_1 b/org.eclipse.ua.tests/data/help/searchindex/index510/segments_1
new file mode 100644
index 0000000..4878901
--- /dev/null
+++ b/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/segments_1
Binary files differ
diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/write.lock b/org.eclipse.ua.tests/data/help/searchindex/index510/write.lock
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/eclipse.platform.ua/org.eclipse.ua.tests/data/help/searchindex/index510/write.lock
diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/EncodedCharacterSearch.java b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/EncodedCharacterSearch.java
index 1dd4876..02b2e5b 100644
--- a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/EncodedCharacterSearch.java
+++ b/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/EncodedCharacterSearch.java
@@ -7,6 +7,7 @@
*
* Contributors:
* IBM Corporation - initial API and implementation
+ * Sopot Cela - Bug 466829 - Migration to Lucene 5
*******************************************************************************/
package org.eclipse.ua.tests.help.search;
@@ -43,7 +44,7 @@
public void testUtf8Hebrew() {
SearchTestUtils.searchOneLocale("\u05D0\u05B7\u05E1\u05B0\u05D8\u05B0\u05E8\u05D5\u05B9\u05E0\u05D5\u05B9\u05DE"
- + "\u05B0\u05D9\u05B8\u05D4) \u05DC\u05B4\u05E7\u05BC\u05D5\u05BC\u05D9 (\u05D9\u05E8\u05D7 \u05D0\u05D5 \u05E9\u05DE\u05E9", new String[] {"/org.eclipse.ua.tests/data/help/search/testnlUTF8.htm" }, "en");
+ + "\u05B0\u05D9\u05B8\u05D4\\) \u05DC\u05B4\u05E7\u05BC\u05D5\u05BC\u05D9 \\(\u05D9\u05E8\u05D7 \u05D0\u05D5 \u05E9\u05DE\u05E9", new String[] {"/org.eclipse.ua.tests/data/help/search/testnlUTF8.htm" }, "en");
}
diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java
index 640d4c9..33ae5ba 100644
--- a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java
+++ b/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java
@@ -8,6 +8,7 @@
* Contributors:
* IBM Corporation - initial API and implementation
* Alexander Kurtakov - Bug 460787
+ * Sopot Cela - Bug 466829 - Migration to Lucene 5
*******************************************************************************/
package org.eclipse.ua.tests.help.search;
@@ -15,43 +16,38 @@
import java.io.File;
import java.io.IOException;
import java.net.URL;
-import java.util.ArrayList;
-import junit.framework.Test;
-import junit.framework.TestCase;
-import junit.framework.TestSuite;
-
-import org.osgi.framework.Bundle;
-
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NIOFSDirectory;
-
-import org.eclipse.help.internal.base.BaseHelpSystem;
-import org.eclipse.help.internal.search.AnalyzerDescriptor;
-import org.eclipse.help.internal.search.PluginIndex;
-import org.eclipse.help.internal.search.QueryBuilder;
-import org.eclipse.help.internal.search.SearchIndexWithIndexingProgress;
-import org.eclipse.ua.tests.plugin.UserAssistanceTestPlugin;
-
+import org.apache.lucene.util.QueryBuilder;
import org.eclipse.core.runtime.FileLocator;
import org.eclipse.core.runtime.NullProgressMonitor;
import org.eclipse.core.runtime.Path;
+import org.eclipse.help.internal.base.BaseHelpSystem;
+import org.eclipse.help.internal.search.PluginIndex;
+import org.eclipse.help.internal.search.SearchIndexWithIndexingProgress;
+import org.eclipse.ua.tests.plugin.UserAssistanceTestPlugin;
+import org.osgi.framework.Bundle;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
/**
- * Verify that older versions of the index can be read by this
- * version of Eclipse.
+ * Verify that older versions of the index can be read by this version of
+ * Eclipse.
*
- * How to maintain this test - if when upgrading to a new version
- * of Lucene one of the IndexReadable tests fails you need to
- * make the following changes:
- * 1. Change the corresponding Compatible() test to expect a result of false
- * 2. Comment out the failing test
- * 3. Change the help system to recognize that version of Lucene as being incompatible
+ * How to maintain this test - if when upgrading to a new version of Lucene one
+ * of the IndexReadable tests fails you need to make the following changes: 1.
+ * Change the corresponding Compatible() test to expect a result of false 2.
+ * Comment out the failing test 3. Change the help system to recognize that
+ * version of Lucene as being incompatible
*/
public class PrebuiltIndexCompatibility extends TestCase {
@@ -64,48 +60,33 @@
}
/**
- * Test index built with Lucene 1.9.1
- */
- public void test1_9_1_IndexReadable() throws Exception {
- checkReadable("data/help/searchindex/index191");
- }
-
- /**
- * Test index built with Lucene 2.9.1
- */
- public void test2_9_1_IndexReadable() throws Exception {
- checkReadable("data/help/searchindex/index291");
- }
-
- /**
** Test compatibility of Lucene 1.9.1 index with current Lucene
*/
- public void test1_9_1Compatible()
- {
- checkCompatible("data/help/searchindex/index191", true);
+ public void test1_9_1Compatible() {
+ checkCompatible("data/help/searchindex/index191", false);
}
-
+
/**
** Test compatibility of Lucene 2.9.1 index with current Lucene
*/
- public void test2_9_1Compatible()
- {
- checkCompatible("data/help/searchindex/index291", true);
+ public void test2_9_1Compatible() {
+ checkCompatible("data/help/searchindex/index291", false);
}
- public void test1_9_1LuceneCompatible()
- {
- checkLuceneCompatible("1.9.1", true);
+ public void test1_9_1LuceneCompatible() {
+ checkLuceneCompatible("1.9.1", false);
}
- public void test1_4_103NotLuceneCompatible()
- {
+ public void test1_4_103NotLuceneCompatible() {
checkLuceneCompatible("1.4.103", false);
}
- public void test2_9_1LuceneCompatible()
- {
- checkLuceneCompatible("2.9.1", true);
+ public void test2_9_1LuceneCompatible() {
+ checkLuceneCompatible("2.9.1", false);
+ }
+
+ public void test5_1_0LuceneCompatible() {
+ checkLuceneCompatible("5.1.0", true);
}
public void testPluginIndexEqualToItself() {
@@ -114,52 +95,59 @@
}
/**
- * Verify that if the paths and plugins are the same two PluginIndex objects are equal
+ * Verify that if the paths and plugins are the same two PluginIndex objects
+ * are equal
*/
public void testPluginIndexEquality() {
- PluginIndex index1a = createPluginIndex("data/help/searchindex/index191");
- PluginIndex index1b = createPluginIndex("data/help/searchindex/index191");
+ PluginIndex index1a = createPluginIndex("data/help/searchindex/index510");
+ PluginIndex index1b = createPluginIndex("data/help/searchindex/index510");
assertTrue(index1a.equals(index1b));
}
-
+
/**
- * Verify that if the paths and plugins are the same two PluginIndex objects are equal
+ * Verify that if the paths and plugins are the same two PluginIndex objects
+ * are equal
*/
public void testPluginIndexHash() {
PluginIndex index1a = createPluginIndex("data/help/searchindex/index191");
PluginIndex index1b = createPluginIndex("data/help/searchindex/index191");
assertEquals(index1a.hashCode(), index1b.hashCode());
}
-
+
/**
- * Verify that if the paths are different two PluginIndex objects are not equal
+ * Verify that if the paths are different two PluginIndex objects are not
+ * equal
*/
public void testPluginIndexInequality() {
PluginIndex index1 = createPluginIndex("data/help/searchindex/index191");
PluginIndex index2 = createPluginIndex("data/help/searchindex/index291");
assertFalse(index1.equals(index2));
}
+
+ public void test5_1_0IndexReadable() throws CorruptIndexException, IOException {
+ checkReadable("data/help/searchindex/index510");
+ }
- /*
- * Verifies that a prebuilt index can be searched
- */
- private void checkReadable(String indexPath) throws IOException,
- CorruptIndexException {
+ /*
+ * Verifies that a prebuilt index can be searched
+ */
+ private void checkReadable(String indexPath) throws IOException, CorruptIndexException {
Path path = new Path(indexPath);
- Bundle bundle = UserAssistanceTestPlugin.getDefault().getBundle();
+ Bundle bundle = UserAssistanceTestPlugin.getDefault().getBundle();
URL url = FileLocator.find(bundle, path, null);
URL resolved = FileLocator.resolve(url);
if ("file".equals(resolved.getProtocol())) { //$NON-NLS-1$
String filePath = resolved.getFile();
- QueryBuilder queryBuilder = new QueryBuilder("eclipse", new AnalyzerDescriptor("en-us"));
- Query luceneQuery = queryBuilder.getLuceneQuery(new ArrayList<String>() , false);
+ StandardAnalyzer standardAnalyzer = new StandardAnalyzer();
+ QueryBuilder builder = new QueryBuilder(standardAnalyzer);
+ Query luceneQuery = builder.createBooleanQuery("contents", "eclipse");
Directory luceneDirectory = null;
IndexSearcher searcher = null;
try {
- luceneDirectory = new NIOFSDirectory(new File(filePath));
- searcher = new IndexSearcher(IndexReader.open(luceneDirectory, true));
+ luceneDirectory = new NIOFSDirectory(new File(filePath).toPath());
+ searcher = new IndexSearcher(DirectoryReader.open(luceneDirectory));
TopDocs hits = searcher.search(luceneQuery, 500);
- assertEquals(hits.totalHits, 1);
+ assertEquals(1, hits.totalHits );
} finally {
if (luceneDirectory != null)
try {
@@ -167,13 +155,13 @@
} catch (IOException x) {
}
if (searcher != null)
- searcher.close();
+ searcher.getIndexReader().close();
}
} else {
fail("Cannot resolve to file protocol");
}
}
-
+
/*
* Tests the isCompatible method in PluginIndex
*/
@@ -186,22 +174,20 @@
public PluginIndex createPluginIndex(String versionDirectory) {
PluginIndex pluginIndex;
SearchIndexWithIndexingProgress index = BaseHelpSystem.getLocalSearchManager().getIndex("en_us".toString());
- BaseHelpSystem.getLocalSearchManager().ensureIndexUpdated(
- new NullProgressMonitor(),
- index);
+ BaseHelpSystem.getLocalSearchManager().ensureIndexUpdated(new NullProgressMonitor(), index);
pluginIndex = new PluginIndex("org.eclipse.ua.tests", "data/help/searchindex/" + versionDirectory, index);
return pluginIndex;
}
-
+
/*
- * Tests the isLuceneCompatible method in SearchIndex
+ * Tests the isLuceneCompatible method in SearchIndex
*/
private void checkLuceneCompatible(String version, boolean expected) {
SearchIndexWithIndexingProgress index = BaseHelpSystem.getLocalSearchManager().getIndex("en_us".toString());
- BaseHelpSystem.getLocalSearchManager().ensureIndexUpdated(
- new NullProgressMonitor(),
- index);
+ BaseHelpSystem.getLocalSearchManager().ensureIndexUpdated(new NullProgressMonitor(), index);
assertEquals(expected, index.isLuceneCompatible(version));
}
+
+
}
diff --git a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/SearchParticipantTest.java b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/SearchParticipantTest.java
index 223e42a..2e782c3 100644
--- a/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/SearchParticipantTest.java
+++ b/eclipse.platform.ua/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/SearchParticipantTest.java
@@ -1,5 +1,5 @@
/*******************************************************************************
- * Copyright (c) 2010 IBM Corporation and others.
+ * Copyright (c) 2010, 2015 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -7,6 +7,7 @@
*
* Contributors:
* IBM Corporation - initial API and implementation
+ * Sopot Cela - Bug 466829 - Migration to Lucene 5
*******************************************************************************/
package org.eclipse.ua.tests.help.search;
@@ -40,7 +41,7 @@
}
public void testSearchUsingAndInSeparateDocs() {
- SearchTestUtils.searchAllLocales("jduehdye and olhoykk", new String[0]);
+ SearchTestUtils.searchAllLocales("jduehdye AND olhoykk", new String[0]);
}
public void testSearchExactMatch() {
--- a/eclipse.platform.common/bundles/org.eclipse.platform.doc.isv/pom.xml
+++ b/eclipse.platform.common/bundles/org.eclipse.platform.doc.isv/pom.xml
@@ -101,7 +101,7 @@
</requirement>
<requirement>
<type>eclipse-plugin</type>
- <id>org.apache.lucene.analysis</id>
+ <id>org.apache.lucene.analyzers-common</id>
<versionRange>0.0.0</versionRange>
</requirement>
<requirement>