1 |
/* |
2 |
SDX: Documentary System in XML. |
3 |
Copyright (C) 2000, 2001, 2002 Ministere de la culture et de la communication (France), AJLSM |
4 |
|
5 |
Ministere de la culture et de la communication, |
6 |
Mission de la recherche et de la technologie |
7 |
3 rue de Valois, 75042 Paris Cedex 01 (France) |
8 |
mrt@culture.fr, michel.bottin@culture.fr |
9 |
|
10 |
AJLSM, 17, rue Vital Carles, 33000 Bordeaux (France) |
11 |
sevigny@ajlsm.com |
12 |
|
13 |
This program is free software; you can redistribute it and/or |
14 |
modify it under the terms of the GNU General Public License |
15 |
as published by the Free Software Foundation; either version 2 |
16 |
of the License, or (at your option) any later version. |
17 |
|
18 |
This program is distributed in the hope that it will be useful, |
19 |
but WITHOUT ANY WARRANTY; without even the implied warranty of |
20 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
21 |
See the GNU General Public License for more details. |
22 |
|
23 |
You should have received a copy of the GNU General Public License |
24 |
along with this program; if not, write to the |
25 |
Free Software Foundation, Inc. |
26 |
59 Temple Place - Suite 330, Boston, MA 02111-1307, USA |
27 |
or connect to: |
28 |
http://www.fsf.org/copyleft/gpl.html |
29 |
*/ |
30 |
package fr.gouv.culture.sdx.search.lucene.query; |
31 |
|
32 |
import fr.gouv.culture.sdx.documentbase.LuceneDocumentBase; |
33 |
import fr.gouv.culture.sdx.exception.SDXException; |
34 |
import fr.gouv.culture.sdx.exception.SDXExceptionCode; |
35 |
import fr.gouv.culture.sdx.framework.Framework; |
36 |
import fr.gouv.culture.sdx.search.lucene.DateField; |
37 |
import fr.gouv.culture.sdx.search.lucene.Field; |
38 |
import fr.gouv.culture.sdx.search.lucene.filter.Criteria; |
39 |
import fr.gouv.culture.sdx.search.lucene.filter.Filter; |
40 |
import fr.gouv.culture.sdx.utils.Bits; |
41 |
import fr.gouv.culture.sdx.utils.Date; |
42 |
import fr.gouv.culture.sdx.utils.Utilities; |
43 |
import fr.gouv.culture.sdx.utils.constants.Node; |
44 |
import fr.gouv.culture.sdx.utils.logging.LoggingUtils; |
45 |
import org.apache.lucene.document.Document; |
46 |
import org.apache.lucene.index.IndexReader; |
47 |
import org.apache.lucene.index.Term; |
48 |
import org.apache.lucene.index.TermDocs; |
49 |
import org.apache.lucene.index.TermEnum; |
50 |
import org.apache.lucene.search.Hits; |
51 |
import org.apache.regexp.RE; |
52 |
import org.apache.regexp.RESyntaxException; |
53 |
import org.xml.sax.ContentHandler; |
54 |
import org.xml.sax.SAXException; |
55 |
import org.xml.sax.helpers.AttributesImpl; |
56 |
|
57 |
import java.io.IOException; |
58 |
import java.text.Collator; |
59 |
import java.util.*; |
60 |
|
61 |
/* |
62 |
* Une liste de termes de la base de documents. |
63 |
* |
64 |
* Les termes sont des valeurs uniques de champs. On peut construire une |
65 |
* liste de tous les termes, une liste de termes qui respectent une |
66 |
* expression reguliere (dans le but de gerer la troncature), une |
67 |
* liste de termes pour un champ, ainsi qu'une liste de termes pour |
68 |
* un champ en fonction d'une requete de recherche. |
69 |
*/ |
70 |
|
71 |
/** A list of terms for a document base. |
72 |
* |
73 |
* The terms are single values of fields. One can build one |
74 |
* list of all the terms, a list of terms which respect one |
75 |
* regular expression (with an aim of managing truncation), one |
76 |
* list of terms for a field, as well as a list of terms for |
77 |
* a field according to a search acceptRequest. |
78 |
* |
79 |
*/ |
80 |
public class Terms extends AbstractResponse implements fr.gouv.culture.sdx.search.Terms { |
81 |
|
82 |
/** The ordered list of terms. */ |
83 |
private TreeMap termList; |
84 |
|
85 |
/** The fields of filters. */ |
86 |
private String[] fieldFilters; |
87 |
|
88 |
/** The values of filters. */ |
89 |
private String[] valueFilters; |
90 |
|
91 |
/** THe filter used for the document base. */ |
92 |
private Filter filter; |
93 |
|
94 |
/** The object of comparison for sorting */ |
95 |
private static Collator sortCollator; |
96 |
|
97 |
/** |
98 |
* Create a list of all the terms of the documents base |
99 |
* |
100 |
* @param db Information about the documents base |
101 |
*/ |
102 |
/* |
103 |
public Terms(LuceneIndex index) throws IOException |
104 |
{ |
105 |
|
106 |
this.index = index; |
107 |
|
108 |
IndexReader reader = index.getReader(); |
109 |
TermEnum terms = reader.terms(); |
110 |
reader.close(); |
111 |
|
112 |
// Il faut les trier |
113 |
initCollator(index); |
114 |
termList = new TreeMap(sortCollator); |
115 |
while ( terms.next() ) |
116 |
termList.put(terms.term().text(), new TermInfo(reader, terms.term())); |
117 |
nbPages = countPages(); |
118 |
} |
119 |
*/ |
120 |
|
121 |
/** |
122 |
* Builds a term list. |
123 |
* |
124 |
* @param sLocs The SearchLocations object (indices to be searched). |
125 |
* @param field The field name. |
126 |
*/ |
127 |
public void setUp(SearchLocations sLocs, String field) throws SDXException { |
128 |
if (sLocs == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SEARCHLOCATIONS_NULL, null, null); |
129 |
|
130 |
if (!Utilities.checkString(field)) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_FIELD_NAME_NULL, null, null); |
131 |
setFieldFilter(field); |
132 |
//interning the field name for better performance with lucene |
133 |
String internedfieldName = field.intern(); |
134 |
|
135 |
super.setSearchLocations(sLocs); |
136 |
|
137 |
initCollator(super._searchLocations.getField(field)); |
138 |
termList = new TreeMap(sortCollator); |
139 |
IndexReader reader = null; |
140 |
TermEnum ts = null; |
141 |
try { |
142 |
reader = super._searchLocations.getReader(); |
143 |
if (reader != null) { |
144 |
ts = reader.terms(new Term(internedfieldName, "")); |
145 |
// Term t; |
146 |
// String text; |
147 |
boolean collecting = false; |
148 |
if (ts != null) { |
149 |
do { |
150 |
if (ts.term() != null && ts.term().field() == internedfieldName) { |
151 |
collecting = true; |
152 |
TermInfo ti = (TermInfo) termList.get(ts.term().text()); |
153 |
if (ti != null) |
154 |
ti.update(reader, ts.term()); |
155 |
else { |
156 |
ti = new TermInfo(); |
157 |
ti.enableLogging(super.getLog()); |
158 |
ti.setUp(reader, ts.term()); |
159 |
termList.put(ts.term().text(), ti); |
160 |
} |
161 |
/* |
162 |
TermInfo ti = new TermInfo(); |
163 |
ti.enableLogging(super.getLog()); |
164 |
ti.setUp(reader, ts.term()); |
165 |
termList.put(ts.term().text(), ti); |
166 |
*/ |
167 |
|
168 |
} else if (collecting) break; |
169 |
} while (ts.next()); |
170 |
} |
171 |
} |
172 |
} catch (IOException e) { |
173 |
String[] args = new String[1]; |
174 |
args[0] = e.getMessage(); |
175 |
throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SETUP_TERMS, args, e); |
176 |
} finally { |
177 |
//freeing resources |
178 |
if (ts != null) { |
179 |
try { |
180 |
ts.close(); |
181 |
} catch (IOException e) { |
182 |
String[] args = new String[1]; |
183 |
args[0] = e.getMessage(); |
184 |
throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_LUCENE_TERMENUM_CLOSE, args, e); |
185 |
} |
186 |
} |
187 |
if (reader != null) { |
188 |
try { |
189 |
reader.close(); |
190 |
} catch (IOException e) { |
191 |
String[] args = new String[1]; |
192 |
args[0] = e.getMessage(); |
193 |
throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_LUCENE_READER_CLOSE, args, e); |
194 |
} |
195 |
} |
196 |
} |
197 |
|
198 |
super.setNbPages(countPages()); |
199 |
} |
200 |
|
201 |
/** |
202 |
* Builds a term list. |
203 |
* |
204 |
* @param sLocs The SearchLocations object (indices to be searched). |
205 |
* @param field The field name. |
206 |
* @param str The term |
207 |
*/ |
208 |
public void setUp(SearchLocations sLocs, String field, String str) throws SDXException { |
209 |
//TODO?:if str == null we call the more simple constructor, but should we really do this or throw and exception here?-rbp |
210 |
if (sLocs == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SEARCHLOCATIONS_NULL, null, null); |
211 |
|
212 |
if (!Utilities.checkString(field)) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_FIELD_NAME_NULL, null, null); |
213 |
setFieldFilter(field); |
214 |
//interning the field name for better performance with lucene |
215 |
String internedfieldName = field.intern(); |
216 |
|
217 |
if (str == null) |
218 |
setUp(sLocs, field); |
219 |
else { |
220 |
super.setSearchLocations(sLocs); |
221 |
|
222 |
initCollator(super._searchLocations.getField(field)); |
223 |
termList = new TreeMap(sortCollator); |
224 |
IndexReader reader = null; |
225 |
TermEnum ts = null; |
226 |
RE re = null; |
227 |
try { |
228 |
re = getRE(str); |
229 |
} catch (RESyntaxException e) { |
230 |
String[] args = new String[1]; |
231 |
args[0] = e.getMessage(); |
232 |
throw new SDXException(null, SDXExceptionCode.ERROR_SETUP_TERMS, args, e); |
233 |
} |
234 |
reader = super._searchLocations.getReader(); |
235 |
if (reader != null) { |
236 |
try { |
237 |
ts = reader.terms(new Term(internedfieldName, "")); |
238 |
Term t = null; |
239 |
String text; |
240 |
boolean collecting = false; |
241 |
if (ts != null) { |
242 |
do { |
243 |
if (ts.term() != null) |
244 |
t = ts.term(); |
245 |
if (t != null && t.field() == internedfieldName) { |
246 |
collecting = true; |
247 |
text = t.text(); |
248 |
if (re.match(text)) { |
249 |
TermInfo ti = (TermInfo) termList.get(text); |
250 |
if (ti != null) |
251 |
ti.update(reader, ts.term()); |
252 |
else { |
253 |
ti = new TermInfo(); |
254 |
ti.enableLogging(super.getLog()); |
255 |
ti.setUp(reader, ts.term()); |
256 |
termList.put(text, ti); |
257 |
} |
258 |
/* |
259 |
TermInfo ti = new TermInfo(); |
260 |
ti.enableLogging(super.getLog()); |
261 |
ti.setUp(reader, t); |
262 |
termList.put(text, ti); |
263 |
*/ |
264 |
} |
265 |
} else if (collecting) break; |
266 |
} while (ts.next()); |
267 |
} |
268 |
} catch (IOException e) { |
269 |
String[] args = new String[1]; |
270 |
args[0] = e.getMessage(); |
271 |
throw new SDXException(null, SDXExceptionCode.ERROR_SETUP_TERMS, args, e); |
272 |
} finally { |
273 |
//freeing resources |
274 |
if (ts != null) { |
275 |
try { |
276 |
ts.close(); |
277 |
} catch (IOException e) { |
278 |
String[] args = new String[1]; |
279 |
args[0] = e.getMessage(); |
280 |
throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_TERMENUM_CLOSE, args, e); |
281 |
} |
282 |
} |
283 |
if (reader != null) { |
284 |
try { |
285 |
reader.close(); |
286 |
} catch (IOException e) { |
287 |
String[] args = new String[1]; |
288 |
args[0] = e.getMessage(); |
289 |
throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_READER_CLOSE, args, e); |
290 |
} |
291 |
} |
292 |
} |
293 |
} |
294 |
super.setNbPages(countPages()); |
295 |
} |
296 |
} |
297 |
|
298 |
protected void setFieldFilter(String field) { |
299 |
String[] fields = {field}; |
300 |
fieldFilters = fields; |
301 |
} |
302 |
|
303 |
/** |
304 |
* Builds a term list. |
305 |
* |
306 |
* @param results The Results object (indices to be searched). |
307 |
* @param field The field name. |
308 |
*/ |
309 |
public void setUp(Results results, String field, String value) throws SDXException { |
310 |
setFieldFilter(field); |
311 |
String[] fields = new String[1]; |
312 |
fields[0] = field; |
313 |
String[] values = new String[1]; |
314 |
values[0] = value; |
315 |
setUp(results, fields, values); |
316 |
} |
317 |
|
318 |
|
319 |
public void setUp(Results results, String[] fields, String[] values) throws SDXException { |
320 |
if (results != null) { |
321 |
super.setSearchLocations(results.getSearchLocations()); |
322 |
if (super._searchLocations == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SEARCHLOCATIONS_NULL, null, null); |
323 |
if (fields == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_FIELD_NAME_NULL, null, null); |
324 |
|
325 |
// If there is more than one Reader in the searchlocation, then log a warning |
326 |
// TODO Warn: specific message... |
327 |
if (super._searchLocations.size() > 1) LoggingUtils.logWarn(super.getLog(), "Only the first search location will be used...", null); |
328 |
// We store the fields and values |
329 |
fieldFilters = fields; |
330 |
termList = getTerms(results.getHits(), fields, values); |
331 |
super.setNbPages(countPages()); |
332 |
} |
333 |
} |
334 |
|
335 |
|
336 |
/** |
337 |
* Builds a lits of terms from multiple criterias. |
338 |
* |
339 |
* <p>IMPORTANT : only one search location is actually used. |
340 |
* TODO : fix this... |
341 |
* |
342 |
* <p>Criterias are field=value pairs. This method lets a |
343 |
* developer creates hierarchical lists. |
344 |
* |
345 |
* <p>There should be one value less than fields; the last |
346 |
* field will then be the field to return values from. |
347 |
* |
348 |
* <p>For instance, if the first field is 'region', the |
349 |
* second is 'department' and the third is 'city', and |
350 |
* the first value is 'Aquitaine' and the second value is |
351 |
* 'Gironde', you will get a list of cities that appear in |
352 |
* documents that also have region=Aquitaine and |
353 |
* department=Gironde. |
354 |
* |
355 |
* @param sLocs Where to find the terms (may not bu null). |
356 |
* @param fields The list of fields (may not be null). |
357 |
* @param values The list of values (there should be one less value than fields). |
358 |
*/ |
359 |
public void setUp(SearchLocations sLocs, String[] fields, String[] values) throws SDXException { |
360 |
if (sLocs == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SEARCHLOCATIONS_NULL, null, null); |
361 |
if (fields == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_FIELD_NAME_NULL, null, null); |
362 |
|
363 |
super.setSearchLocations(sLocs); |
364 |
|
365 |
// If there is more than one Reader in the searchlocation, then log a warning |
366 |
// TODO : specific message... |
367 |
if (sLocs.size() > 1) LoggingUtils.logWarn(super.getLog(), "Only the first search location will be used...", null); |
368 |
|
369 |
// We store the fields and values |
370 |
fieldFilters = fields; |
371 |
valueFilters = values; |
372 |
|
373 |
// Consider a null values as an array of length 0, or no filter... |
374 |
if (values == null) values = new String[0]; |
375 |
|
376 |
// We only build the terms if there is one more field than values |
377 |
if (fields.length == (values.length + 1)) { |
378 |
|
379 |
// We'll build a simple filter, by ANDing all field=value |
380 |
// pairs except the last field. |
381 |
|
382 |
BitSet documentSet = null; |
383 |
IndexReader r = sLocs.getReader(); |
384 |
if (fields.length > 1) { |
385 |
filter = new Filter(Filter.BOOLEAN_OPERATOR_AND); |
386 |
|
387 |
for (int i = 0; i < values.length; i++) { |
388 |
Criteria c = new Criteria(); |
389 |
c.enableLogging(super.getLog()); |
390 |
c.setUp(sLocs.getField(fields[i]), values[i]); |
391 |
filter.add(c); |
392 |
} |
393 |
documentSet = filter.bits(r); |
394 |
} |
395 |
|
396 |
|
397 |
// Then we build the list of values of the last field |
398 |
// with at least one document in the bitset. |
399 |
try { |
400 |
termList = getTerms(r, sLocs.getField(fields[fields.length - 1]), documentSet); |
401 |
} finally { |
402 |
if (r != null) { |
403 |
try { |
404 |
r.close(); |
405 |
} catch (IOException e) { |
406 |
String[] args = new String[1]; |
407 |
args[0] = e.getMessage(); |
408 |
throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_READER_CLOSE, args, e); |
409 |
} |
410 |
} |
411 |
} |
412 |
super.setNbPages(countPages()); |
413 |
|
414 |
} else { |
415 |
// Here, we don't mett the criteria that there must be one more field than value |
416 |
// TODO : log a warning... |
417 |
} |
418 |
} |
419 |
|
420 |
/** |
421 |
* Cree une liste filtree de termes. |
422 |
* |
423 |
* Le filtre specifie les documents dont les termes |
424 |
* doivent etre extraits. |
425 |
* |
426 |
* @param db L'informations sur la base de documents. |
427 |
* @param field Le champ. |
428 |
* @param filter Le filtre. |
429 |
*/ |
430 |
/* |
431 |
public Terms(SearchLocations sLocs, String field, Filter filter) throws SDXException, IOException { |
432 |
this.searchLocations = sLocs; |
433 |
|
434 |
//TODO? if ( field == null || db == null || filter == null ) throw new SDXException("fr", "Impossible de constuire une liste de termes."); |
435 |
{ |
436 |
this.filter = filter; |
437 |
IndexReader r = index.getReader(); |
438 |
BitSet documentSet = filter.bits(r); |
439 |
|
440 |
// Ensuite, on doit trouver les valeurs du champ qui ont |
441 |
// au moins un document dans le filtre, et trouver le nombre |
442 |
// de documents. |
443 |
|
444 |
termList = getTerms(r, field, documentSet); |
445 |
r.close(); |
446 |
nbPages = countPages(); |
447 |
} |
448 |
} |
449 |
*/ |
450 |
|
451 |
/**Creates a list of terms according to a chain with truncation |
452 |
* |
453 |
* <p> |
454 |
* A super.getLog() must be set and then the Terms must be setUp. |
455 |
* |
456 |
* @see #enableLogging |
457 |
* @see #setUp |
458 |
*/ |
459 |
public Terms() { |
460 |
} |
461 |
|
462 |
|
463 |
/** |
464 |
* Genere une sortie XML de la liste de termes. |
465 |
* |
466 |
* @param factory Le document hote pour cette sortie XML. |
467 |
*/ |
468 |
/* |
469 |
public Element toDOM(Document factory) throws IOException |
470 |
{ |
471 |
return toDOM(factory, -1); |
472 |
} |
473 |
*/ |
474 |
|
475 |
/** |
476 |
* Returns an XML representation of this term list. |
477 |
* |
478 |
* @param hdl The ContentHandler to feed with SAX events. |
479 |
*/ |
480 |
public void toSAX(ContentHandler hdl) throws SAXException { |
481 |
toSAX(hdl, -1); |
482 |
} |
483 |
|
484 |
/* |
485 |
//Genere une sortie XML de la liste de termes. |
486 |
// @param factory Le document hote pour cette sortie XML. |
487 |
//@param page Le numero de la page a sortir. |
488 |
public Element toDOM(Document factory, int page) throws IOException |
489 |
{ |
490 |
String nsURI = Utilities.getSDXNamespaceURI(); |
491 |
String nsPrefix = Utilities.getSDXNamespacePrefix(); |
492 |
|
493 |
if ( page < 1 ) page = 1; |
494 |
if ( page > nbPages ) page = nbPages; |
495 |
int sIdx = ( (page - 1) * hitsPerPage ); |
496 |
if ( sIdx < 0 ) sIdx = 0; |
497 |
|
498 |
int eIdx = ( page * hitsPerPage ) - 1; |
499 |
if ( eIdx >= termList.size() ) eIdx = termList.size() - 1; |
500 |
|
501 |
Element top = factory.createElementNS(nsURI, nsPrefix + ":terms"); |
502 |
top.setAttribute("xmlns:" + nsPrefix, nsURI); |
503 |
//forward compatibility : use of old attribute name (take care !) |
504 |
if ( fieldFilters != null && fieldFilters.length > 0 ) top.setAttribute("code", fieldFilters[fieldFilters.length - 1]); |
505 |
if ( id != null ) top.setAttribute("id", id); |
506 |
|
507 |
// Les informations sur le nombre de pages |
508 |
top.setAttribute("start", String.valueOf(sIdx+1)); |
509 |
top.setAttribute("end", String.valueOf(eIdx+1)); |
510 |
top.setAttribute("nb", String.valueOf(termList.size())); |
511 |
top.setAttribute("nbPages", String.valueOf(nbPages)); |
512 |
top.setAttribute("currentPage", String.valueOf(page)); |
513 |
|
514 |
// On va ajouter des informations sur les filtres utilises. |
515 |
if ( filter != null ) top.appendChild(filter.toDOM(factory)); |
516 |
|
517 |
if ( hitsPerPage != 0 ) |
518 |
{ |
519 |
Element termEl; |
520 |
if ( termList != null ) |
521 |
{ |
522 |
Iterator it = termList.keySet().iterator(); |
523 |
int i=0; |
524 |
while ( it.hasNext() ) |
525 |
{ |
526 |
i++; |
527 |
|
528 |
if ( i > sIdx && i <= eIdx+1 ) |
529 |
{ |
530 |
fr.gouv.culture.sdx.query.lucene.TermInfo term = (fr.gouv.culture.sdx.query.lucene.TermInfo)termList.get((String)it.next()); |
531 |
termEl = factory.createElementNS(nsURI, nsPrefix + ":term"); |
532 |
termEl.setAttribute("docFreq", String.valueOf(term.getDocFrequency())); |
533 |
String fieldname = term.getField(); |
534 |
String content; |
535 |
if ( db != null && db.getFieldType(fieldname) == DBField.DATE ) content = Utilities.formatDate(DateField.stringToDate(term.getContent())); |
536 |
else |
537 |
content = term.getContent(); |
538 |
termEl.setAttribute("escapedValue", java.net.URLEncoder.encodeURL(content)); |
539 |
termEl.setAttribute("field", fieldname); |
540 |
termEl.setAttribute("value", content); |
541 |
termEl.setAttribute("no", String.valueOf(i)); |
542 |
|
543 |
// S'il y a un seul document, alors on retourne aussi son identificateur. |
544 |
if ( term.getDocFrequency() == 1 ) termEl.setAttribute("docId", term.getDocId()); |
545 |
|
546 |
top.appendChild(termEl); |
547 |
} |
548 |
else if ( i > eIdx+1 ) break; else it.next(); |
549 |
} |
550 |
} |
551 |
} |
552 |
return top; |
553 |
} |
554 |
*/ |
555 |
|
556 |
/** |
557 |
* Returns an XML representation of this term list. |
558 |
* |
559 |
* @param hdl The ContentHandler to feed with SAX events. |
560 |
* @param page The number of the page to show. |
561 |
*/ |
562 |
public void toSAX(ContentHandler hdl, int page) throws SAXException { |
563 |
String sdxNsUri = Framework.SDXNamespaceURI; |
564 |
String sdxNsPrefix = Framework.SDXNamespacePrefix; |
565 |
|
566 |
if (page < 1) page = 1; |
567 |
if (page > super.getNbPages()) page = super.getNbPages(); |
568 |
int sIdx = ((page - 1) * super.getHitsPerPage()); |
569 |
if (sIdx < 0) sIdx = 0; |
570 |
|
571 |
int eIdx = (page * super.getHitsPerPage()) - 1; |
572 |
if (termList != null && eIdx >= termList.size()) eIdx = termList.size() - 1; |
573 |
|
574 |
//Creation of local variables which are later passed into startElement() and endElement() methods-rbp13/03/02 |
575 |
String localName = Node.Name.TERMS; |
576 |
String qName = sdxNsPrefix + ":" + localName; |
577 |
AttributesImpl atts = new AttributesImpl(); |
578 |
|
579 |
|
580 |
if (fieldFilters != null && fieldFilters.length > 0) { |
581 |
atts.addAttribute("", Node.Name.NAME, Node.Name.NAME, Node.Type.CDATA, fieldFilters[fieldFilters.length - 1]); |
582 |
atts.addAttribute("", Node.Name.FIELD, Node.Name.FIELD, Node.Type.CDATA, fieldFilters[fieldFilters.length - 1]); |
583 |
} |
584 |
|
585 |
// Les informations sur le nombre de pages |
586 |
//TODO?:should we also add the field name here or other information?-rbp |
587 |
atts.addAttribute("", Node.Name.QID, Node.Name.QID, Node.Type.CDATA, String.valueOf(super.getId())); |
588 |
atts.addAttribute("", Node.Name.PAGE, Node.Name.PAGE, Node.Type.CDATA, String.valueOf(page)); |
589 |
atts.addAttribute("", Node.Name.HPP, Node.Name.HPP, Node.Type.CDATA, String.valueOf(super.getHitsPerPage())); |
590 |
atts.addAttribute("", Node.Name.PAGES, Node.Name.PAGES, Node.Type.CDATA, String.valueOf(super.getNbPages())); |
591 |
if (termList != null) |
592 |
atts.addAttribute("", Node.Name.NB, Node.Name.NB, Node.Type.CDATA, String.valueOf(termList.size())); |
593 |
atts.addAttribute("", Node.Name.START, Node.Name.START, Node.Type.CDATA, String.valueOf(sIdx + 1)); |
594 |
atts.addAttribute("", Node.Name.END, Node.Name.END, Node.Type.CDATA, String.valueOf(eIdx + 1)); |
595 |
|
596 |
// for backward compatibility but not documented |
597 |
atts.addAttribute("", Node.Name.ID, Node.Name.ID, Node.Type.CDATA, String.valueOf(super.getId())); |
598 |
atts.addAttribute("", Node.Name.CURRENT_PAGE, Node.Name.CURRENT_PAGE, Node.Type.CDATA, String.valueOf(page)); |
599 |
atts.addAttribute("", Node.Name.NB_PAGES, Node.Name.NB_PAGES, Node.Type.CDATA, String.valueOf(super.getNbPages())); |
600 |
|
601 |
//startElement() method is called for "terms" and local variables are passed-rbp12/03/02 |
602 |
hdl.startElement(sdxNsUri, localName, qName, atts); |
603 |
|
604 |
// On va ajouter des informations sur les filtres utilises. |
605 |
if (filter != null) filter.toSAX(hdl); |
606 |
|
607 |
if (super.getHitsPerPage() != 0) { |
608 |
if (termList != null) { |
609 |
Iterator it = termList.keySet().iterator(); |
610 |
int i = 0; |
611 |
while (it.hasNext()) { |
612 |
i++; |
613 |
|
614 |
if (i > sIdx && i <= eIdx + 1) { |
615 |
TermInfo term = (TermInfo) termList.get(it.next()); |
616 |
//Creation of local variables which are later passed into startElement() and endElement() methods-rbp12/03/02 |
617 |
String subLocalName = Node.Name.TERM; |
618 |
String subQName = sdxNsPrefix + ":" + subLocalName; |
619 |
AttributesImpl subAtts = new AttributesImpl(); |
620 |
subAtts.addAttribute("", Node.Name.NO, Node.Name.NO, Node.Type.CDATA, String.valueOf(i)); |
621 |
|
622 |
String fieldname = term.getField(); |
623 |
String content; |
624 |
|
625 |
int fieldType = -1; |
626 |
try { |
627 |
fieldType = super._searchLocations.getFieldType(fieldname); |
628 |
} catch (SDXException sdxE) { |
629 |
throw new SAXException(sdxE.getMessage(), sdxE); |
630 |
} |
631 |
if (super._searchLocations != null && fieldType == Field.DATE) |
632 |
content = Date.formatDate(DateField.stringToDate(term.getContent())); |
633 |
else |
634 |
content = term.getContent(); |
635 |
|
636 |
String esc = null; |
637 |
esc = Utilities.encodeURL(content, super.getEncoding()); |
638 |
// try { esc=java.net.URLEncoder.encodeURL(content, Framework.URL_ENCODING); } catch (java.io.UnsupportedEncodingException e) {} // 1.4 |
639 |
subAtts.addAttribute("", Node.Name.VALUE, Node.Name.VALUE, Node.Type.CDATA, content); |
640 |
subAtts.addAttribute("", Node.Name.ESCAPED_VALUE, Node.Name.ESCAPED_VALUE, Node.Type.CDATA, esc); |
641 |
subAtts.addAttribute("", Node.Name.FIELD, Node.Name.FIELD, Node.Type.CDATA, fieldname); |
642 |
|
643 |
|
644 |
subAtts.addAttribute("", Node.Name.DOCS, Node.Name.DOCS, Node.Type.CDATA, String.valueOf(term.getDocFrequency())); |
645 |
// S'il y a un seul document, alors on retourne aussi son identificateur. |
646 |
if (term.getDocFrequency() == 1) { |
647 |
subAtts.addAttribute("", Node.Name.ID, Node.Name.ID, Node.Type.CDATA, term.getDocId()); |
648 |
subAtts.addAttribute("", Node.Name.DB_ID, Node.Name.DB_ID, Node.Type.CDATA, term.getDbId()); |
649 |
subAtts.addAttribute("", Node.Name.APPID, Node.Name.APPID, Node.Type.CDATA, term.getAppId()); |
650 |
subAtts.addAttribute("", Node.Name.DOC_ID, Node.Name.DOC_ID, Node.Type.CDATA, term.getDocId()); |
651 |
} |
652 |
// not documented but preserved for backward compatibility |
653 |
subAtts.addAttribute("", Node.Name.DOC_FREQ, Node.Name.DOC_FREQ, Node.Type.CDATA, String.valueOf(term.getDocFrequency())); |
654 |
//startElement() method is called for "term" and local variables are passed-rbp12/03/02 |
655 |
hdl.startElement(sdxNsUri, subLocalName, subQName, subAtts); |
656 |
|
657 |
//endElement() method is called for "term" and local variables are passed-rbp12/03/02 |
658 |
hdl.endElement(sdxNsUri, subLocalName, subQName); |
659 |
|
660 |
} else if (i > eIdx + 1) break; else it.next(); |
661 |
} |
662 |
} |
663 |
} |
664 |
//endElement() method is called for "terms" and local variables are passed-rbp12/03/02 |
665 |
hdl.endElement(sdxNsUri, localName, qName); |
666 |
} |
667 |
|
668 |
/** |
669 |
* Returns a list of the terms. |
670 |
*/ |
671 |
public TreeMap getList() { |
672 |
return termList; |
673 |
} |
674 |
|
675 |
/** |
676 |
* Constructs a regular expression from a term of the acceptRequest. |
677 |
* |
678 |
* @param str The term. |
679 |
*/ |
680 |
private static RE getRE(String str) throws RESyntaxException { |
681 |
if (str == null) return null; |
682 |
StringBuffer st = new StringBuffer("^"); |
683 |
for (int i = 0; i < str.length(); i++) { |
684 |
char c = str.charAt(i); |
685 |
switch (c) { |
686 |
case '*': |
687 |
st.append("(.*)"); |
688 |
break; |
689 |
|
690 |
case '?': |
691 |
st.append("(.?)"); |
692 |
break; |
693 |
|
694 |
case '(': |
695 |
case ')': |
696 |
case '.': |
697 |
case '\\': |
698 |
case '[': |
699 |
case ']': |
700 |
case '^': |
701 |
case '$': |
702 |
case '+': |
703 |
case '{': |
704 |
case '}': |
705 |
case '|': |
706 |
st.append('\\'); |
707 |
st.append(c); |
708 |
break; |
709 |
|
710 |
default: |
711 |
st.append(c); |
712 |
} |
713 |
} |
714 |
st.append("$"); |
715 |
return new RE(st.toString()); |
716 |
} |
717 |
|
718 |
/* |
719 |
* Retourne une liste de termes en fonction d'un champ et d'une valeur. |
720 |
* |
721 |
* La valeur peut contenir des masques * et ?. Tous les termes de ce champ |
722 |
* qui respectent la valeur masquee seront retournes. Si aucun terme ne |
723 |
* satisfait le critere, on retourne une liste vide. |
724 |
* |
725 |
* @param r Un objet pour lire l'index. |
726 |
* @param f Le nom du champ |
727 |
* @param pattern La valeur masquee. |
728 |
*/ |
729 |
/** |
730 |
* Returns a list of terms according to a field and a value. |
731 |
* |
732 |
* The value can contain masks and?. All terms of this field |
733 |
* which respects the masked value will be turned over. If no term |
734 |
* satisfied the criterion, an empty list is returned. |
735 |
* |
736 |
* @param r The index reader |
737 |
* @param f The field |
738 |
* @param pattern The pattern |
739 |
* @throws SDXException |
740 |
*/ |
741 |
public static TreeMap getTerms(IndexReader r, Field f, String pattern) throws SDXException { |
742 |
TreeMap ret = null; |
743 |
|
744 |
if (r == null || f == null || pattern == null) return ret; |
745 |
String internedfieldName = ""; |
746 |
if (Utilities.checkString(f.getCode())) |
747 |
//interning the field name for better performance with lucene |
748 |
internedfieldName = f.getCode().intern(); |
749 |
|
750 |
if (f.getLocale() != null) |
751 |
ret = new TreeMap(Collator.getInstance(f.getLocale())); |
752 |
else |
753 |
//TODO?:is this acceptable default behavior?-rbp |
754 |
ret = new TreeMap(Collator.getInstance(new Locale("fr", "FR"))); |
755 |
|
756 |
TermEnum ts = null; |
757 |
try { |
758 |
RE re = getRE(pattern); |
759 |
ts = r.terms(new Term(internedfieldName, "")); |
760 |
boolean collecting = false; |
761 |
if (ts != null) { |
762 |
do { |
763 |
if (ts.term() != null && ts.term().field() == internedfieldName) { |
764 |
collecting = true; |
765 |
if (ts.term() != null && re != null && re.match(ts.term().text())) { |
766 |
// First check if the term is already in the list |
767 |
TermInfo ti = (TermInfo) ret.get(ts.term().text()); |
768 |
if (ti != null) |
769 |
ti.update(r, ts.term()); |
770 |
else { |
771 |
ti = new TermInfo(); |
772 |
//ti.enableLogging(_logger); |
773 |
ti.setUp(r, ts.term()); |
774 |
ret.put(ts.term().text(), ti); |
775 |
} |
776 |
} |
777 |
} else if (collecting) break; |
778 |
} while (ts.next()); |
779 |
} |
780 |
} catch (RESyntaxException e) { |
781 |
String[] args = new String[1]; |
782 |
args[0] = e.getMessage(); |
783 |
throw new SDXException(null, SDXExceptionCode.ERROR_GET_TERMS, args, e); |
784 |
} catch (IOException e) { |
785 |
String[] args = new String[1]; |
786 |
args[0] = e.getMessage(); |
787 |
throw new SDXException(null, SDXExceptionCode.ERROR_GET_TERMS, args, e); |
788 |
} finally { |
789 |
try { |
790 |
//freeing resources |
791 |
if (ts != null) ts.close(); |
792 |
} catch (IOException e) { |
793 |
String[] args = new String[2]; |
794 |
args[0] = f.getCode(); |
795 |
args[1] = e.getMessage(); |
796 |
throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_TERMENUM_CLOSE, args, e); |
797 |
} |
798 |
} |
799 |
|
800 |
return ret; |
801 |
} |
802 |
|
803 |
/** |
804 |
* Retourne une liste de valeurs pour un champ. |
805 |
* |
806 |
* @param r L'objet pour lire l'index. |
807 |
* @param field Le nom du champ. |
808 |
*/ |
809 |
/* Not used anymore ? MS |
810 |
public static TreeMap getTerms(IndexReader r, String field) throws SDXException, IOException { |
811 |
TreeMap ret = new TreeMap(Collator.getInstance(new Locale("fr", "FR"))); |
812 |
TermEnum ts = r.terms(); |
813 |
boolean collecting = false; |
814 |
while (ts.next()) { |
815 |
if (ts.term().field().equals(field)) { |
816 |
collecting = true; |
817 |
TermInfo ti = (TermInfo)ret.get(ts.term().text()); |
818 |
if ( ti != null ) ti.update(r, ts.term()); |
819 |
else { |
820 |
ti = new TermInfo(); |
821 |
ti.enableLogging(super.getLog()); |
822 |
ti.setUp(r, ts.term()); |
823 |
ret.put(ts.term().text(), ti); |
824 |
} |
825 |
} else if (collecting) break; |
826 |
} |
827 |
return ret; |
828 |
} |
829 |
*/ |
830 |
|
831 |
/** |
832 |
* Builds a list of values for a filtered f. |
833 |
* |
834 |
* @param hits The results. |
835 |
* @param fields The fields in question. |
836 |
* @param values The values for the fields in question. |
837 |
*/ |
838 |
private TreeMap getTerms(Hits hits, String[] fields, String[] values) throws SDXException { |
839 |
|
840 |
TreeMap ret = new TreeMap(); |
841 |
if (hits == null || hits.length() == 0) return ret; // TODO : is the the better solution? Harmonize with other lists of terms... |
842 |
Hashtable fieldMaps = new Hashtable(); |
843 |
for (int i = 0; i < hits.length(); i++) { |
844 |
try { |
845 |
Document hitDoc = hits.doc(i); |
846 |
if (hitDoc != null) { |
847 |
for (int j = 0; j < fields.length; j++) { |
848 |
Field f = super._searchLocations.getField(fields[j]); |
849 |
String lfname = f.getCode(); |
850 |
if (f != null) { |
851 |
TreeMap map = null; |
852 |
if (fieldMaps.size() > 0 && fieldMaps.containsKey(lfname)) |
853 |
map = (TreeMap) fieldMaps.get(lfname); |
854 |
else { |
855 |
if (f.getCollator() == null) |
856 |
map = new TreeMap(Collator.getInstance(new Locale("fr", "FR"))); // TODO : is it OK? |
857 |
else |
858 |
map = new TreeMap(f.getCollator()); |
859 |
} |
860 |
String[] fieldsValues = hitDoc.getValues(lfname); |
861 |
if (fieldsValues != null) { |
862 |
HashSet termsCollected = new HashSet(); |
863 |
for (int k = 0; k < fieldsValues.length; k++) { |
864 |
String lftext = fieldsValues[k]; |
865 |
if (Utilities.checkString(lftext)) { |
866 |
if (values == null || values.length == 0 || (j < values.length && (!Utilities.checkString(values[j]) || lftext.equals(values[j])))) { |
867 |
if (!termsCollected.contains(lftext)) { |
868 |
TermInfo ti = (TermInfo) map.get(lftext); |
869 |
if (ti != null) { |
870 |
ti.update(lfname, lftext); |
871 |
} else { |
872 |
ti = new TermInfo(); |
873 |
ti.enableLogging(super.getLog()); |
874 |
ti.setUp(lfname, lftext); |
875 |
ti.update(lfname, lftext); |
876 |
ti.setDocId(hitDoc.getField(LuceneIndex.ID_FIELD).stringValue()); |
877 |
ti.setDbId(hitDoc.getField(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXDBID).stringValue()); |
878 |
ti.setAppId(hitDoc.getField(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXAPPID).stringValue()); |
879 |
map.put(lftext, ti); |
880 |
} |
881 |
termsCollected.add(lftext); |
882 |
} |
883 |
} |
884 |
} |
885 |
} |
886 |
} |
887 |
if (map != null) fieldMaps.put(lfname, map); |
888 |
} |
889 |
} |
890 |
} |
891 |
} catch (IOException e) { |
892 |
String[] args = new String[1]; |
893 |
args[0] = e.getMessage(); |
894 |
new SDXException(null, SDXExceptionCode.ERROR_GET_TERMS, args, e); |
895 |
} |
896 |
} |
897 |
|
898 |
|
899 |
Enumeration maps = fieldMaps.elements(); |
900 |
if (maps != null) { |
901 |
while (maps.hasMoreElements()) { |
902 |
TreeMap outmap = (TreeMap) maps.nextElement(); |
903 |
ret.putAll(outmap); |
904 |
} |
905 |
maps = null; |
906 |
} |
907 |
return ret; |
908 |
} |
909 |
|
910 |
|
911 |
/** |
912 |
* Builds a list of values for a filtered f. |
913 |
* |
914 |
* @param r The reader. |
915 |
* @param f The f. |
916 |
* @param docs A list of documents as a filter. |
917 |
*/ |
918 |
public TreeMap getTerms(IndexReader r, Field f, BitSet docs) throws SDXException { |
919 |
|
920 |
if (f == null) return new TreeMap(); // TODO : is the the better solution? Harmonize with other lists of terms... |
921 |
|
922 |
//interning the f name for better performance with lucene |
923 |
String internedfieldName = ""; |
924 |
if (Utilities.checkString(f.getCode())) internedfieldName = f.getCode().intern(); |
925 |
|
926 |
TreeMap ret; |
927 |
if (f.getCollator() == null) |
928 |
ret = new TreeMap(Collator.getInstance(new Locale("fr", "FR"))); // TODO : is it OK? |
929 |
else |
930 |
ret = new TreeMap(f.getCollator()); |
931 |
|
932 |
TermEnum ts = null; |
933 |
try { |
934 |
ts = r.terms(new Term(internedfieldName, "")); |
935 |
boolean collecting = false; |
936 |
BitSet termDocumentSet; |
937 |
int nbDocs; |
938 |
do { |
939 |
if (ts.term() != null && ts.term().field() == internedfieldName) { |
940 |
collecting = true; |
941 |
|
942 |
// We build the bitset for this term and compare |
943 |
// it with the one we have, and then count what's |
944 |
// left. |
945 |
|
946 |
termDocumentSet = getDocumentSet(r, ts.term()); |
947 |
if (docs != null) termDocumentSet.and(docs); |
948 |
nbDocs = Bits.countBits(termDocumentSet); |
949 |
if (nbDocs > 0) { |
950 |
TermInfo ti = (TermInfo) ret.get(ts.term().text()); |
951 |
if (ti != null) |
952 |
ti.update(r, ts.term()); |
953 |
else { |
954 |
ti = new TermInfo(); |
955 |
ti.enableLogging(super.getLog()); |
956 |
ti.setUp(r, ts.term()); |
957 |
ret.put(ts.term().text(), ti); |
958 |
} |
959 |
} |
960 |
} else if (collecting) break; |
961 |
} while (ts.next()); |
962 |
return ret; |
963 |
} catch (IOException e) { |
964 |
String[] args = new String[1]; |
965 |
args[0] = e.getMessage(); |
966 |
throw new SDXException(null, SDXExceptionCode.ERROR_GET_TERMS, args, e); |
967 |
} finally { |
968 |
try { |
969 |
//freeing resources |
970 |
if (ts != null) ts.close(); |
971 |
} catch (IOException e) { |
972 |
String[] args = new String[2]; |
973 |
args[0] = f.getCode(); |
974 |
args[1] = e.getMessage(); |
975 |
throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_TERMENUM_CLOSE, args, e); |
976 |
} |
977 |
} |
978 |
} |
979 |
|
980 |
|
981 |
/** |
982 |
* Returns a list of the documents associated with a term within the index. |
983 |
* |
984 |
* @param r The index reader. |
985 |
* @param term The term for searching. |
986 |
*/ |
987 |
public static BitSet getDocumentSet(IndexReader r, Term term) throws IOException { |
988 |
BitSet bits = new BitSet(r.numDocs()); |
989 |
TermDocs docs = r.termDocs(term); |
990 |
while (docs.next()) bits.set(docs.doc()); |
991 |
return bits; |
992 |
} |
993 |
|
994 |
/* |
995 |
// Ecrit la liste de termes en XML dans un writer. |
996 |
// @param writer Le writer ou ecrire. |
997 |
// @param page La page a envoyer. |
998 |
public void toXML(Writer writer, int page) throws SDXException |
999 |
{ |
1000 |
XMLSerializer ser = new XMLSerializer(writer, new OutputFormat("xml", "UTF-8", true)); |
1001 |
try |
1002 |
{ |
1003 |
ser.serialize(toDOM(new DocumentImpl(), page)); |
1004 |
} |
1005 |
catch ( IOException e ) |
1006 |
{ |
1007 |
throw new SDXException(e, "fr", "Impossible de serialiser le document XML"); |
1008 |
} |
1009 |
} |
1010 |
*/ |
1011 |
|
1012 |
/* No need for this anymore as it is being handled in the servlet class, GetTerms.java.-rbp21/03/02 |
1013 |
// Ecrit la liste de termes en XML dans un flux de sortie. |
1014 |
// @param os Le flux ou ecrire. |
1015 |
// @param page La page a envoyer. |
1016 |
public void toXML(OutputStream os, int page) throws SDXException |
1017 |
{ |
1018 |
XMLSerializer ser = new XMLSerializer(os, new OutputFormat("xml", "UTF-8", true)); |
1019 |
try |
1020 |
{ |
1021 |
ser.serialize(toDOM(new DocumentImpl(), page)); |
1022 |
} |
1023 |
catch ( IOException e ) |
1024 |
{ |
1025 |
throw new SDXException(e, "fr", "Impossible de serialiser le document XML"); |
1026 |
} |
1027 |
} |
1028 |
*/ |
1029 |
|
1030 |
/** |
1031 |
* Counts the number of pages for these terms. |
1032 |
*/ |
1033 |
public int countPages() { |
1034 |
if (termList == null) |
1035 |
return 0; |
1036 |
else if (termList.size() == 0) |
1037 |
return 0; |
1038 |
else { |
1039 |
if (super.getHitsPerPage() == 0) return 1; |
1040 |
float temp = ((float) termList.size() / (float) super.getHitsPerPage()); |
1041 |
if (temp != Math.round(temp)) |
1042 |
return (int) (Math.round(Math.floor(temp) + 1)); |
1043 |
else |
1044 |
return Math.round(temp); |
1045 |
} |
1046 |
} |
1047 |
|
1048 |
/** |
1049 |
* Set the number of hits per page. |
1050 |
* |
1051 |
* @param nb The number of hits. |
1052 |
*/ |
1053 |
public void setHitsPerPage(int nb) { |
1054 |
if (nb < 0) |
1055 |
setAllHits(); |
1056 |
else { |
1057 |
super.setHitsPerPage(nb); |
1058 |
super.setNbPages(countPages()); |
1059 |
} |
1060 |
} |
1061 |
|
1062 |
/** |
1063 |
* Initializes the collator used for sorting . |
1064 |
* |
1065 |
* @param field The field from the document base. |
1066 |
*/ |
1067 |
private void initCollator(Field field) { |
1068 |
if (field != null) sortCollator = field.getCollator(); |
1069 |
} |
1070 |
|
1071 |
/** |
1072 |
* Indicate to return all the results on only one page. |
1073 |
*/ |
1074 |
public void setAllHits() { |
1075 |
if (termList != null) |
1076 |
super.setHitsPerPage(termList.size()); |
1077 |
super.setNbPages(1); |
1078 |
} |
1079 |
|
1080 |
protected String getClassNameSuffix() { |
1081 |
return Terms.CLASS_NAME_SUFFIX; |
1082 |
} |
1083 |
|
1084 |
|
1085 |
} |