1 |
msevigny |
1.1 |
/* |
2 |
|
|
SDX: Documentary System in XML. |
3 |
|
|
Copyright (C) 2000, 2001, 2002 Ministere de la culture et de la communication (France), AJLSM |
4 |
|
|
|
5 |
|
|
Ministere de la culture et de la communication, |
6 |
|
|
Mission de la recherche et de la technologie |
7 |
|
|
3 rue de Valois, 75042 Paris Cedex 01 (France) |
8 |
|
|
mrt@culture.fr, michel.bottin@culture.fr |
9 |
|
|
|
10 |
|
|
AJLSM, 17, rue Vital Carles, 33000 Bordeaux (France) |
11 |
|
|
sevigny@ajlsm.com |
12 |
|
|
|
13 |
|
|
This program is free software; you can redistribute it and/or |
14 |
|
|
modify it under the terms of the GNU General Public License |
15 |
|
|
as published by the Free Software Foundation; either version 2 |
16 |
|
|
of the License, or (at your option) any later version. |
17 |
|
|
|
18 |
|
|
This program is distributed in the hope that it will be useful, |
19 |
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of |
20 |
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
21 |
|
|
See the GNU General Public License for more details. |
22 |
|
|
|
23 |
|
|
You should have received a copy of the GNU General Public License |
24 |
|
|
along with this program; if not, write to the |
25 |
|
|
Free Software Foundation, Inc. |
26 |
|
|
59 Temple Place - Suite 330, Boston, MA 02111-1307, USA |
27 |
|
|
or connect to: |
28 |
|
|
http://www.fsf.org/copyleft/gpl.html |
29 |
|
|
*/ |
30 |
|
|
package fr.gouv.culture.sdx.search.lucene.analysis; |
31 |
|
|
|
32 |
rpandey |
1.3 |
import fr.gouv.culture.sdx.exception.SDXException; |
33 |
msevigny |
1.1 |
import org.apache.avalon.framework.configuration.Configuration; |
34 |
|
|
import org.apache.avalon.framework.configuration.ConfigurationException; |
35 |
rpandey |
1.3 |
import org.apache.lucene.analysis.LowerCaseTokenizer; |
36 |
|
|
import org.apache.lucene.analysis.StopFilter; |
37 |
|
|
import org.apache.lucene.analysis.TokenStream; |
38 |
msevigny |
1.1 |
|
39 |
rpandey |
1.3 |
import java.io.Reader; |
40 |
msevigny |
1.1 |
import java.util.Hashtable; |
41 |
|
|
|
42 |
|
|
/** |
43 |
|
|
* A default Lucene analyzer used by SDX. |
44 |
|
|
*/ |
45 |
rpandey |
1.4 |
public class DefaultAnalyzer extends AbstractAnalyzer { |
46 |
msevigny |
1.1 |
|
47 |
|
|
/** The element enclosing the list of stop words. */ |
48 |
|
|
private static final String STOP_WORDS_ELEMENT = "stopWords"; |
49 |
|
|
|
50 |
|
|
/** The element containing a stop word. */ |
51 |
|
|
private static final String STOP_WORD_ELEMENT = "stopWord"; |
52 |
|
|
|
53 |
rpandey |
1.10 |
/** The attribute indicating the use of stop words or not. */ |
54 |
rpandey |
1.13 |
protected static final String ATTRIBUTE_USE_STOP_WORDS = "useStopWords"; |
55 |
|
|
|
56 |
|
|
/** The attribute indicating the use of exclusion stem words or not. */ |
57 |
|
|
protected static final String ATTRIBUTE_EXCLUDE_STEMS = "excludeStems"; |
58 |
|
|
|
59 |
rpandey |
1.10 |
|
60 |
msevigny |
1.1 |
/** The list of stop words used. */ |
61 |
rpandey |
1.13 |
protected Hashtable stopTable = null; |
62 |
|
|
|
63 |
|
|
/**The table for stemming exclusions*/ |
64 |
|
|
protected Hashtable excludeTable = null; |
65 |
|
|
/**String representation of the element name in the analyzer config file*/ |
66 |
|
|
protected final String EXCLUDE_STEMS_ELEMENT = "excludeStems"; |
67 |
|
|
/**String representation of the element name in the analyzer config file*/ |
68 |
|
|
protected final String EXCLUDE_STEM_ELEMENT = "excludeStem"; |
69 |
msevigny |
1.1 |
|
70 |
rpandey |
1.10 |
/** An array containing some common English words that are not usually useful for searching. */ |
71 |
rpandey |
1.13 |
public static final String[] DEFAULT_STOP_WORDS = { |
72 |
msevigny |
1.1 |
"a", "and", "are", "as", "at", "be", "but", "by", |
73 |
|
|
"for", "if", "in", "into", "is", "it", |
74 |
|
|
"no", "not", "of", "on", "or", "s", "such", |
75 |
|
|
"t", "that", "the", "their", "then", "there", "these", |
76 |
|
|
"they", "this", "to", "was", "will", "with" |
77 |
|
|
}; |
78 |
|
|
|
79 |
|
|
/** |
80 |
|
|
* Builds a default analyzer. |
81 |
|
|
* |
82 |
|
|
* <p> |
83 |
|
|
* This analyzer will use Lucene's StopAnalyzer. |
84 |
|
|
*/ |
85 |
|
|
public DefaultAnalyzer() { |
86 |
|
|
} |
87 |
|
|
|
88 |
|
|
/** |
89 |
|
|
* Configures this analyzer. |
90 |
|
|
* |
91 |
|
|
* <p> |
92 |
|
|
* The class will search for <stopWord> elements and |
93 |
|
|
* use them as a stop word list. If none is found or the configuration |
94 |
|
|
* object is null, the default list wi be used. |
95 |
|
|
* <p> |
96 |
|
|
* If the top-level element <cconfiguration> has a false |
97 |
|
|
* value for its useStopWords attribute, no stop words will |
98 |
|
|
* be used. |
99 |
|
|
*/ |
100 |
|
|
public void configure(Configuration configuration) throws ConfigurationException { |
101 |
rpandey |
1.9 |
if (configuration != null) { |
102 |
rpandey |
1.13 |
boolean useStopWords = configuration.getAttributeAsBoolean(ATTRIBUTE_USE_STOP_WORDS, true); |
103 |
rpandey |
1.9 |
if (useStopWords) { |
104 |
|
|
try { |
105 |
|
|
stopTable = buildStopTable(configuration); |
106 |
|
|
} catch (SDXException e) { |
107 |
|
|
throw new ConfigurationException(e.getMessage(), e.fillInStackTrace()); |
108 |
|
|
} |
109 |
|
|
} else |
110 |
|
|
stopTable = null; |
111 |
rpandey |
1.13 |
|
112 |
|
|
boolean excludeStems = configuration.getAttributeAsBoolean(ATTRIBUTE_EXCLUDE_STEMS, true); |
113 |
|
|
|
114 |
|
|
if (excludeStems) { |
115 |
|
|
//configuration a table of exclusions |
116 |
|
|
excludeTable = buildExcludeTable(configuration); |
117 |
|
|
} else |
118 |
|
|
excludeTable = null; |
119 |
|
|
|
120 |
rpandey |
1.9 |
} else { |
121 |
|
|
// No configuration, so we will use default stop words |
122 |
larcaini |
1.17 |
stopTable = StopFilter.makeStopTable(getDefaultStopWords()); |
123 |
rpandey |
1.13 |
//TODO same thing for the excludeTable |
124 |
|
|
//excludeTable = StopFilter.makeStopTable(getDefaultExludedStems()); |
125 |
|
|
|
126 |
msevigny |
1.1 |
} |
127 |
|
|
} |
128 |
|
|
|
129 |
|
|
/** Filters LowerCaseTokenizer with StopFilter. */ |
130 |
|
|
public TokenStream tokenStream(String fieldName, Reader reader) { |
131 |
|
|
if (stopTable != null) |
132 |
|
|
return new StopFilter(new LowerCaseTokenizer(reader), stopTable); |
133 |
|
|
else |
134 |
|
|
return new LowerCaseTokenizer(reader); |
135 |
|
|
} |
136 |
|
|
|
137 |
|
|
/** |
138 |
rpandey |
1.11 |
* Builds a stop word table from a configuration. |
139 |
msevigny |
1.1 |
* |
140 |
|
|
* @param conf The configuration to use. |
141 |
|
|
*/ |
142 |
rpandey |
1.8 |
protected Hashtable buildStopTable(Configuration conf) throws SDXException, ConfigurationException { |
143 |
msevigny |
1.1 |
Hashtable stops = null; |
144 |
|
|
if (conf != null) { |
145 |
rpandey |
1.9 |
Configuration cStop = conf.getChild(STOP_WORDS_ELEMENT, false); |
146 |
msevigny |
1.1 |
if (cStop != null) { |
147 |
|
|
Configuration[] cStops = cStop.getChildren(STOP_WORD_ELEMENT); |
148 |
|
|
if (cStops != null) { |
149 |
|
|
String[] words = new String[cStops.length]; |
150 |
rpandey |
1.9 |
for (int i = 0; i < cStops.length; i++) { |
151 |
|
|
words[i] = cStops[i].getValue(); |
152 |
|
|
} |
153 |
larcaini |
1.17 |
stops = StopFilter.makeStopTable(words); |
154 |
msevigny |
1.1 |
} |
155 |
|
|
} |
156 |
|
|
} |
157 |
rpandey |
1.11 |
if (stops == null) |
158 |
larcaini |
1.17 |
stops = StopFilter.makeStopTable(getDefaultStopWords()); |
159 |
rpandey |
1.11 |
|
160 |
msevigny |
1.1 |
return stops; |
161 |
|
|
} |
162 |
|
|
|
163 |
|
|
/** |
164 |
|
|
* Returns a default list of stop words. |
165 |
|
|
*/ |
166 |
|
|
protected String[] getDefaultStopWords() { |
167 |
|
|
return DEFAULT_STOP_WORDS; |
168 |
rpandey |
1.13 |
} |
169 |
|
|
|
170 |
|
|
/** |
171 |
|
|
* Builds a stop word table from a configuration. |
172 |
|
|
* |
173 |
|
|
* @param conf The configuration to use. |
174 |
|
|
*/ |
175 |
|
|
protected Hashtable buildExcludeTable(Configuration conf) throws ConfigurationException { |
176 |
|
|
Hashtable excludes = null; |
177 |
|
|
if (conf != null) { |
178 |
|
|
Configuration cExclude = conf.getChild(EXCLUDE_STEMS_ELEMENT, false); |
179 |
|
|
if (cExclude != null) { |
180 |
|
|
Configuration[] cExcludes = cExclude.getChildren(EXCLUDE_STEM_ELEMENT); |
181 |
|
|
if (cExcludes != null) { |
182 |
|
|
String[] words = new String[cExcludes.length]; |
183 |
|
|
for (int i = 0; i < cExcludes.length; i++) { |
184 |
|
|
words[i] = cExcludes[i].getValue(); |
185 |
|
|
} |
186 |
larcaini |
1.17 |
excludes = StopFilter.makeStopTable(words); |
187 |
rpandey |
1.13 |
} |
188 |
|
|
} |
189 |
|
|
} |
190 |
|
|
if (excludes == null) |
191 |
larcaini |
1.17 |
excludes = StopFilter.makeStopTable(getDefaultStopWords()); |
192 |
rpandey |
1.13 |
|
193 |
|
|
return excludes; |
194 |
msevigny |
1.1 |
} |
195 |
rpandey |
1.14 |
|
196 |
msevigny |
1.1 |
} |