/[sdx]/sdx_v2/src/java/fr/gouv/culture/sdx/search/lucene/analysis/DefaultAnalyzer.java
ViewVC logotype

Annotation of /sdx_v2/src/java/fr/gouv/culture/sdx/search/lucene/analysis/DefaultAnalyzer.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.18 - (hide annotations) (download)
Thu Jun 17 11:18:40 2004 UTC (19 years, 10 months ago) by larcaini
Branch: MAIN
Changes since 1.17: +0 -3 lines
Some import cleaning...

1 msevigny 1.1 /*
2     SDX: Documentary System in XML.
3     Copyright (C) 2000, 2001, 2002 Ministere de la culture et de la communication (France), AJLSM
4    
5     Ministere de la culture et de la communication,
6     Mission de la recherche et de la technologie
7     3 rue de Valois, 75042 Paris Cedex 01 (France)
8     mrt@culture.fr, michel.bottin@culture.fr
9    
10     AJLSM, 17, rue Vital Carles, 33000 Bordeaux (France)
11     sevigny@ajlsm.com
12    
13     This program is free software; you can redistribute it and/or
14     modify it under the terms of the GNU General Public License
15     as published by the Free Software Foundation; either version 2
16     of the License, or (at your option) any later version.
17    
18     This program is distributed in the hope that it will be useful,
19     but WITHOUT ANY WARRANTY; without even the implied warranty of
20     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
21     See the GNU General Public License for more details.
22    
23     You should have received a copy of the GNU General Public License
24     along with this program; if not, write to the
25     Free Software Foundation, Inc.
26     59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
27     or connect to:
28     http://www.fsf.org/copyleft/gpl.html
29     */
30     package fr.gouv.culture.sdx.search.lucene.analysis;
31    
32 rpandey 1.3 import fr.gouv.culture.sdx.exception.SDXException;
33 msevigny 1.1 import org.apache.avalon.framework.configuration.Configuration;
34     import org.apache.avalon.framework.configuration.ConfigurationException;
35 rpandey 1.3 import org.apache.lucene.analysis.LowerCaseTokenizer;
36     import org.apache.lucene.analysis.StopFilter;
37     import org.apache.lucene.analysis.TokenStream;
38 msevigny 1.1
39 rpandey 1.3 import java.io.Reader;
40 msevigny 1.1 import java.util.Hashtable;
41    
42     /**
43     * A default Lucene analyzer used by SDX.
44     */
45 rpandey 1.4 public class DefaultAnalyzer extends AbstractAnalyzer {
46 msevigny 1.1
47     /** The element enclosing the list of stop words. */
48     private static final String STOP_WORDS_ELEMENT = "stopWords";
49    
50     /** The element containing a stop word. */
51     private static final String STOP_WORD_ELEMENT = "stopWord";
52    
53 rpandey 1.10 /** The attribute indicating the use of stop words or not. */
54 rpandey 1.13 protected static final String ATTRIBUTE_USE_STOP_WORDS = "useStopWords";
55    
56     /** The attribute indicating the use of exclusion stem words or not. */
57     protected static final String ATTRIBUTE_EXCLUDE_STEMS = "excludeStems";
58    
59 rpandey 1.10
60 msevigny 1.1 /** The list of stop words used. */
61 rpandey 1.13 protected Hashtable stopTable = null;
62    
63     /**The table for stemming exclusions*/
64     protected Hashtable excludeTable = null;
65     /**String representation of the element name in the analyzer config file*/
66     protected final String EXCLUDE_STEMS_ELEMENT = "excludeStems";
67     /**String representation of the element name in the analyzer config file*/
68     protected final String EXCLUDE_STEM_ELEMENT = "excludeStem";
69 msevigny 1.1
70 rpandey 1.10 /** An array containing some common English words that are not usually useful for searching. */
71 rpandey 1.13 public static final String[] DEFAULT_STOP_WORDS = {
72 msevigny 1.1 "a", "and", "are", "as", "at", "be", "but", "by",
73     "for", "if", "in", "into", "is", "it",
74     "no", "not", "of", "on", "or", "s", "such",
75     "t", "that", "the", "their", "then", "there", "these",
76     "they", "this", "to", "was", "will", "with"
77     };
78    
79     /**
80     * Builds a default analyzer.
81     *
82     * <p>
83     * This analyzer will use Lucene's StopAnalyzer.
84     */
85     public DefaultAnalyzer() {
86     }
87    
88     /**
89     * Configures this analyzer.
90     *
91     * <p>
92     * The class will search for &lt;stopWord&gt; elements and
93     * use them as a stop word list. If none is found or the configuration
94     * object is null, the default list wi be used.
95     * <p>
96     * If the top-level element &lt;cconfiguration&gt; has a false
97     * value for its useStopWords attribute, no stop words will
98     * be used.
99     */
100     public void configure(Configuration configuration) throws ConfigurationException {
101 rpandey 1.9 if (configuration != null) {
102 rpandey 1.13 boolean useStopWords = configuration.getAttributeAsBoolean(ATTRIBUTE_USE_STOP_WORDS, true);
103 rpandey 1.9 if (useStopWords) {
104     try {
105     stopTable = buildStopTable(configuration);
106     } catch (SDXException e) {
107     throw new ConfigurationException(e.getMessage(), e.fillInStackTrace());
108     }
109     } else
110     stopTable = null;
111 rpandey 1.13
112     boolean excludeStems = configuration.getAttributeAsBoolean(ATTRIBUTE_EXCLUDE_STEMS, true);
113    
114     if (excludeStems) {
115     //configuration a table of exclusions
116     excludeTable = buildExcludeTable(configuration);
117     } else
118     excludeTable = null;
119    
120 rpandey 1.9 } else {
121     // No configuration, so we will use default stop words
122 larcaini 1.17 stopTable = StopFilter.makeStopTable(getDefaultStopWords());
123 rpandey 1.13 //TODO same thing for the excludeTable
124     //excludeTable = StopFilter.makeStopTable(getDefaultExludedStems());
125    
126 msevigny 1.1 }
127     }
128    
129     /** Filters LowerCaseTokenizer with StopFilter. */
130     public TokenStream tokenStream(String fieldName, Reader reader) {
131     if (stopTable != null)
132     return new StopFilter(new LowerCaseTokenizer(reader), stopTable);
133     else
134     return new LowerCaseTokenizer(reader);
135     }
136    
137     /**
138 rpandey 1.11 * Builds a stop word table from a configuration.
139 msevigny 1.1 *
140     * @param conf The configuration to use.
141     */
142 rpandey 1.8 protected Hashtable buildStopTable(Configuration conf) throws SDXException, ConfigurationException {
143 msevigny 1.1 Hashtable stops = null;
144     if (conf != null) {
145 rpandey 1.9 Configuration cStop = conf.getChild(STOP_WORDS_ELEMENT, false);
146 msevigny 1.1 if (cStop != null) {
147     Configuration[] cStops = cStop.getChildren(STOP_WORD_ELEMENT);
148     if (cStops != null) {
149     String[] words = new String[cStops.length];
150 rpandey 1.9 for (int i = 0; i < cStops.length; i++) {
151     words[i] = cStops[i].getValue();
152     }
153 larcaini 1.17 stops = StopFilter.makeStopTable(words);
154 msevigny 1.1 }
155     }
156     }
157 rpandey 1.11 if (stops == null)
158 larcaini 1.17 stops = StopFilter.makeStopTable(getDefaultStopWords());
159 rpandey 1.11
160 msevigny 1.1 return stops;
161     }
162    
163     /**
164     * Returns a default list of stop words.
165     */
166     protected String[] getDefaultStopWords() {
167     return DEFAULT_STOP_WORDS;
168 rpandey 1.13 }
169    
170     /**
171     * Builds a stop word table from a configuration.
172     *
173     * @param conf The configuration to use.
174     */
175     protected Hashtable buildExcludeTable(Configuration conf) throws ConfigurationException {
176     Hashtable excludes = null;
177     if (conf != null) {
178     Configuration cExclude = conf.getChild(EXCLUDE_STEMS_ELEMENT, false);
179     if (cExclude != null) {
180     Configuration[] cExcludes = cExclude.getChildren(EXCLUDE_STEM_ELEMENT);
181     if (cExcludes != null) {
182     String[] words = new String[cExcludes.length];
183     for (int i = 0; i < cExcludes.length; i++) {
184     words[i] = cExcludes[i].getValue();
185     }
186 larcaini 1.17 excludes = StopFilter.makeStopTable(words);
187 rpandey 1.13 }
188     }
189     }
190     if (excludes == null)
191 larcaini 1.17 excludes = StopFilter.makeStopTable(getDefaultStopWords());
192 rpandey 1.13
193     return excludes;
194 msevigny 1.1 }
195 rpandey 1.14
196 msevigny 1.1 }

savannah-hackers-public@gnu.org
ViewVC Help
Powered by ViewVC 1.1.26