/[sdx]/sdx_v2/src/java/fr/gouv/culture/sdx/search/lucene/query/Terms.java
ViewVC logotype

Contents of /sdx_v2/src/java/fr/gouv/culture/sdx/search/lucene/query/Terms.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.75 - (show annotations) (download)
Mon Nov 27 10:57:57 2006 UTC (17 years, 5 months ago) by malo_pichot
Branch: MAIN
CVS Tags: V_23_FINAL, HEAD
Changes since 1.74: +27 -40 lines
Comments.

1 /*
2 SDX: Documentary System in XML.
3 Copyright (C) 2000, 2001, 2002 Ministere de la culture et de la communication (France), AJLSM
4
5 Ministere de la culture et de la communication,
6 Mission de la recherche et de la technologie
7 3 rue de Valois, 75042 Paris Cedex 01 (France)
8 mrt@culture.fr, michel.bottin@culture.fr
9
10 AJLSM, 17, rue Vital Carles, 33000 Bordeaux (France)
11 sevigny@ajlsm.com
12
13 This program is free software; you can redistribute it and/or
14 modify it under the terms of the GNU General Public License
15 as published by the Free Software Foundation; either version 2
16 of the License, or (at your option) any later version.
17
18 This program is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
21 See the GNU General Public License for more details.
22
23 You should have received a copy of the GNU General Public License
24 along with this program; if not, write to the
25 Free Software Foundation, Inc.
26 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
27 or connect to:
28 http://www.fsf.org/copyleft/gpl.html
29 */
30 package fr.gouv.culture.sdx.search.lucene.query;
31
32 import fr.gouv.culture.sdx.documentbase.LuceneDocumentBase;
33 import fr.gouv.culture.sdx.exception.SDXException;
34 import fr.gouv.culture.sdx.exception.SDXExceptionCode;
35 import fr.gouv.culture.sdx.framework.Framework;
36 import fr.gouv.culture.sdx.search.lucene.DateField;
37 import fr.gouv.culture.sdx.search.lucene.Field;
38 import fr.gouv.culture.sdx.search.lucene.filter.Criteria;
39 import fr.gouv.culture.sdx.search.lucene.filter.Filter;
40 import fr.gouv.culture.sdx.utils.Bits;
41 import fr.gouv.culture.sdx.utils.Date;
42 import fr.gouv.culture.sdx.utils.Utilities;
43 import fr.gouv.culture.sdx.utils.constants.Node;
44 import fr.gouv.culture.sdx.utils.logging.LoggingUtils;
45 import org.apache.lucene.document.Document;
46 import org.apache.lucene.index.IndexReader;
47 import org.apache.lucene.index.Term;
48 import org.apache.lucene.index.TermDocs;
49 import org.apache.lucene.index.TermEnum;
50 import org.apache.lucene.search.Hits;
51 import org.apache.regexp.RE;
52 import org.apache.regexp.RESyntaxException;
53 import org.xml.sax.ContentHandler;
54 import org.xml.sax.SAXException;
55 import org.xml.sax.helpers.AttributesImpl;
56
57 import java.io.IOException;
58 import java.text.Collator;
59 import java.util.*;
60
61 /*
62 * Une liste de termes de la base de documents.
63 *
64 * Les termes sont des valeurs uniques de champs. On peut construire une
65 * liste de tous les termes, une liste de termes qui respectent une
66 * expression reguliere (dans le but de gerer la troncature), une
67 * liste de termes pour un champ, ainsi qu'une liste de termes pour
68 * un champ en fonction d'une requete de recherche.
69 */
70
71 /** A list of terms for a document base.
72 *
73 * The terms are single values of fields. One can build one
74 * list of all the terms, a list of terms which respect one
75 * regular expression (with an aim of managing truncation), one
76 * list of terms for a field, as well as a list of terms for
77 * a field according to a search acceptRequest.
78 *
79 */
80 public class Terms extends AbstractResponse implements fr.gouv.culture.sdx.search.Terms {
81
82 /** The ordered list of terms. */
83 private TreeMap termList;
84
85 /** The fields of filters. */
86 private String[] fieldFilters;
87
88 /** The values of filters. */
89 private String[] valueFilters;
90
91 /** THe filter used for the document base. */
92 private Filter filter;
93
94 /** The object of comparison for sorting */
95 private static Collator sortCollator;
96
97 /**
98 * Create a list of all the terms of the documents base
99 *
100 * @param db Information about the documents base
101 */
102 /*
103 public Terms(LuceneIndex index) throws IOException
104 {
105
106 this.index = index;
107
108 IndexReader reader = index.getReader();
109 TermEnum terms = reader.terms();
110 reader.close();
111
112 // Il faut les trier
113 initCollator(index);
114 termList = new TreeMap(sortCollator);
115 while ( terms.next() )
116 termList.put(terms.term().text(), new TermInfo(reader, terms.term()));
117 nbPages = countPages();
118 }
119 */
120
121 /**
122 * Builds a term list.
123 *
124 * @param sLocs The SearchLocations object (indices to be searched).
125 * @param field The field name.
126 */
127 public void setUp(SearchLocations sLocs, String field) throws SDXException {
128 if (sLocs == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SEARCHLOCATIONS_NULL, null, null);
129
130 if (!Utilities.checkString(field)) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_FIELD_NAME_NULL, null, null);
131 setFieldFilter(field);
132 //interning the field name for better performance with lucene
133 String internedfieldName = field.intern();
134
135 super.setSearchLocations(sLocs);
136
137 initCollator(super._searchLocations.getField(field));
138 termList = new TreeMap(sortCollator);
139 IndexReader reader = null;
140 TermEnum ts = null;
141 try {
142 reader = super._searchLocations.getReader();
143 if (reader != null) {
144 ts = reader.terms(new Term(internedfieldName, ""));
145 // Term t;
146 // String text;
147 boolean collecting = false;
148 if (ts != null) {
149 do {
150 if (ts.term() != null && ts.term().field() == internedfieldName) {
151 collecting = true;
152 TermInfo ti = (TermInfo) termList.get(ts.term().text());
153 if (ti != null)
154 ti.update(reader, ts.term());
155 else {
156 ti = new TermInfo();
157 ti.enableLogging(super.getLog());
158 ti.setUp(reader, ts.term());
159 termList.put(ts.term().text(), ti);
160 }
161 /*
162 TermInfo ti = new TermInfo();
163 ti.enableLogging(super.getLog());
164 ti.setUp(reader, ts.term());
165 termList.put(ts.term().text(), ti);
166 */
167
168 } else if (collecting) break;
169 } while (ts.next());
170 }
171 }
172 } catch (IOException e) {
173 String[] args = new String[1];
174 args[0] = e.getMessage();
175 throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SETUP_TERMS, args, e);
176 } finally {
177 //freeing resources
178 if (ts != null) {
179 try {
180 ts.close();
181 } catch (IOException e) {
182 String[] args = new String[1];
183 args[0] = e.getMessage();
184 throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_LUCENE_TERMENUM_CLOSE, args, e);
185 }
186 }
187 if (reader != null) {
188 try {
189 reader.close();
190 } catch (IOException e) {
191 String[] args = new String[1];
192 args[0] = e.getMessage();
193 throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_LUCENE_READER_CLOSE, args, e);
194 }
195 }
196 }
197
198 super.setNbPages(countPages());
199 }
200
201 /**
202 * Builds a term list.
203 *
204 * @param sLocs The SearchLocations object (indices to be searched).
205 * @param field The field name.
206 * @param str The term
207 */
208 public void setUp(SearchLocations sLocs, String field, String str) throws SDXException {
209 //TODO?:if str == null we call the more simple constructor, but should we really do this or throw and exception here?-rbp
210 if (sLocs == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SEARCHLOCATIONS_NULL, null, null);
211
212 if (!Utilities.checkString(field)) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_FIELD_NAME_NULL, null, null);
213 setFieldFilter(field);
214 //interning the field name for better performance with lucene
215 String internedfieldName = field.intern();
216
217 if (str == null)
218 setUp(sLocs, field);
219 else {
220 super.setSearchLocations(sLocs);
221
222 initCollator(super._searchLocations.getField(field));
223 termList = new TreeMap(sortCollator);
224 IndexReader reader = null;
225 TermEnum ts = null;
226 RE re = null;
227 try {
228 re = getRE(str);
229 } catch (RESyntaxException e) {
230 String[] args = new String[1];
231 args[0] = e.getMessage();
232 throw new SDXException(null, SDXExceptionCode.ERROR_SETUP_TERMS, args, e);
233 }
234 reader = super._searchLocations.getReader();
235 if (reader != null) {
236 try {
237 ts = reader.terms(new Term(internedfieldName, ""));
238 Term t = null;
239 String text;
240 boolean collecting = false;
241 if (ts != null) {
242 do {
243 if (ts.term() != null)
244 t = ts.term();
245 if (t != null && t.field() == internedfieldName) {
246 collecting = true;
247 text = t.text();
248 if (re.match(text)) {
249 TermInfo ti = (TermInfo) termList.get(text);
250 if (ti != null)
251 ti.update(reader, ts.term());
252 else {
253 ti = new TermInfo();
254 ti.enableLogging(super.getLog());
255 ti.setUp(reader, ts.term());
256 termList.put(text, ti);
257 }
258 /*
259 TermInfo ti = new TermInfo();
260 ti.enableLogging(super.getLog());
261 ti.setUp(reader, t);
262 termList.put(text, ti);
263 */
264 }
265 } else if (collecting) break;
266 } while (ts.next());
267 }
268 } catch (IOException e) {
269 String[] args = new String[1];
270 args[0] = e.getMessage();
271 throw new SDXException(null, SDXExceptionCode.ERROR_SETUP_TERMS, args, e);
272 } finally {
273 //freeing resources
274 if (ts != null) {
275 try {
276 ts.close();
277 } catch (IOException e) {
278 String[] args = new String[1];
279 args[0] = e.getMessage();
280 throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_TERMENUM_CLOSE, args, e);
281 }
282 }
283 if (reader != null) {
284 try {
285 reader.close();
286 } catch (IOException e) {
287 String[] args = new String[1];
288 args[0] = e.getMessage();
289 throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_READER_CLOSE, args, e);
290 }
291 }
292 }
293 }
294 super.setNbPages(countPages());
295 }
296 }
297
298 protected void setFieldFilter(String field) {
299 String[] fields = {field};
300 fieldFilters = fields;
301 }
302
303 /**
304 * Builds a term list.
305 *
306 * @param results The Results object (indices to be searched).
307 * @param field The field name.
308 */
309 public void setUp(Results results, String field, String value) throws SDXException {
310 setFieldFilter(field);
311 String[] fields = new String[1];
312 fields[0] = field;
313 String[] values = new String[1];
314 values[0] = value;
315 setUp(results, fields, values);
316 }
317
318
319 public void setUp(Results results, String[] fields, String[] values) throws SDXException {
320 if (results != null) {
321 super.setSearchLocations(results.getSearchLocations());
322 if (super._searchLocations == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SEARCHLOCATIONS_NULL, null, null);
323 if (fields == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_FIELD_NAME_NULL, null, null);
324
325 // If there is more than one Reader in the searchlocation, then log a warning
326 // TODO Warn: specific message...
327 if (super._searchLocations.size() > 1) LoggingUtils.logWarn(super.getLog(), "Only the first search location will be used...", null);
328 // We store the fields and values
329 fieldFilters = fields;
330 termList = getTerms(results.getHits(), fields, values);
331 super.setNbPages(countPages());
332 }
333 }
334
335
336 /**
337 * Builds a lits of terms from multiple criterias.
338 *
339 * <p>IMPORTANT : only one search location is actually used.
340 * TODO : fix this...
341 *
342 * <p>Criterias are field=value pairs. This method lets a
343 * developer creates hierarchical lists.
344 *
345 * <p>There should be one value less than fields; the last
346 * field will then be the field to return values from.
347 *
348 * <p>For instance, if the first field is 'region', the
349 * second is 'department' and the third is 'city', and
350 * the first value is 'Aquitaine' and the second value is
351 * 'Gironde', you will get a list of cities that appear in
352 * documents that also have region=Aquitaine and
353 * department=Gironde.
354 *
355 * @param sLocs Where to find the terms (may not bu null).
356 * @param fields The list of fields (may not be null).
357 * @param values The list of values (there should be one less value than fields).
358 */
359 public void setUp(SearchLocations sLocs, String[] fields, String[] values) throws SDXException {
360 if (sLocs == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SEARCHLOCATIONS_NULL, null, null);
361 if (fields == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_FIELD_NAME_NULL, null, null);
362
363 super.setSearchLocations(sLocs);
364
365 // If there is more than one Reader in the searchlocation, then log a warning
366 // TODO : specific message...
367 if (sLocs.size() > 1) LoggingUtils.logWarn(super.getLog(), "Only the first search location will be used...", null);
368
369 // We store the fields and values
370 fieldFilters = fields;
371 valueFilters = values;
372
373 // Consider a null values as an array of length 0, or no filter...
374 if (values == null) values = new String[0];
375
376 // We only build the terms if there is one more field than values
377 if (fields.length == (values.length + 1)) {
378
379 // We'll build a simple filter, by ANDing all field=value
380 // pairs except the last field.
381
382 BitSet documentSet = null;
383 IndexReader r = sLocs.getReader();
384 if (fields.length > 1) {
385 filter = new Filter(Filter.BOOLEAN_OPERATOR_AND);
386
387 for (int i = 0; i < values.length; i++) {
388 Criteria c = new Criteria();
389 c.enableLogging(super.getLog());
390 c.setUp(sLocs.getField(fields[i]), values[i]);
391 filter.add(c);
392 }
393 documentSet = filter.bits(r);
394 }
395
396
397 // Then we build the list of values of the last field
398 // with at least one document in the bitset.
399 try {
400 termList = getTerms(r, sLocs.getField(fields[fields.length - 1]), documentSet);
401 } finally {
402 if (r != null) {
403 try {
404 r.close();
405 } catch (IOException e) {
406 String[] args = new String[1];
407 args[0] = e.getMessage();
408 throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_READER_CLOSE, args, e);
409 }
410 }
411 }
412 super.setNbPages(countPages());
413
414 } else {
415 // Here, we don't mett the criteria that there must be one more field than value
416 // TODO : log a warning...
417 }
418 }
419
420 /**
421 * Cree une liste filtree de termes.
422 *
423 * Le filtre specifie les documents dont les termes
424 * doivent etre extraits.
425 *
426 * @param db L'informations sur la base de documents.
427 * @param field Le champ.
428 * @param filter Le filtre.
429 */
430 /*
431 public Terms(SearchLocations sLocs, String field, Filter filter) throws SDXException, IOException {
432 this.searchLocations = sLocs;
433
434 //TODO? if ( field == null || db == null || filter == null ) throw new SDXException("fr", "Impossible de constuire une liste de termes.");
435 {
436 this.filter = filter;
437 IndexReader r = index.getReader();
438 BitSet documentSet = filter.bits(r);
439
440 // Ensuite, on doit trouver les valeurs du champ qui ont
441 // au moins un document dans le filtre, et trouver le nombre
442 // de documents.
443
444 termList = getTerms(r, field, documentSet);
445 r.close();
446 nbPages = countPages();
447 }
448 }
449 */
450
451 /**Creates a list of terms according to a chain with truncation
452 *
453 * <p>
454 * A super.getLog() must be set and then the Terms must be setUp.
455 *
456 * @see #enableLogging
457 * @see #setUp
458 */
459 public Terms() {
460 }
461
462
463 /**
464 * Genere une sortie XML de la liste de termes.
465 *
466 * @param factory Le document hote pour cette sortie XML.
467 */
468 /*
469 public Element toDOM(Document factory) throws IOException
470 {
471 return toDOM(factory, -1);
472 }
473 */
474
475 /**
476 * Returns an XML representation of this term list.
477 *
478 * @param hdl The ContentHandler to feed with SAX events.
479 */
480 public void toSAX(ContentHandler hdl) throws SAXException {
481 toSAX(hdl, -1);
482 }
483
484 /*
485 //Genere une sortie XML de la liste de termes.
486 // @param factory Le document hote pour cette sortie XML.
487 //@param page Le numero de la page a sortir.
488 public Element toDOM(Document factory, int page) throws IOException
489 {
490 String nsURI = Utilities.getSDXNamespaceURI();
491 String nsPrefix = Utilities.getSDXNamespacePrefix();
492
493 if ( page < 1 ) page = 1;
494 if ( page > nbPages ) page = nbPages;
495 int sIdx = ( (page - 1) * hitsPerPage );
496 if ( sIdx < 0 ) sIdx = 0;
497
498 int eIdx = ( page * hitsPerPage ) - 1;
499 if ( eIdx >= termList.size() ) eIdx = termList.size() - 1;
500
501 Element top = factory.createElementNS(nsURI, nsPrefix + ":terms");
502 top.setAttribute("xmlns:" + nsPrefix, nsURI);
503 //forward compatibility : use of old attribute name (take care !)
504 if ( fieldFilters != null && fieldFilters.length > 0 ) top.setAttribute("code", fieldFilters[fieldFilters.length - 1]);
505 if ( id != null ) top.setAttribute("id", id);
506
507 // Les informations sur le nombre de pages
508 top.setAttribute("start", String.valueOf(sIdx+1));
509 top.setAttribute("end", String.valueOf(eIdx+1));
510 top.setAttribute("nb", String.valueOf(termList.size()));
511 top.setAttribute("nbPages", String.valueOf(nbPages));
512 top.setAttribute("currentPage", String.valueOf(page));
513
514 // On va ajouter des informations sur les filtres utilises.
515 if ( filter != null ) top.appendChild(filter.toDOM(factory));
516
517 if ( hitsPerPage != 0 )
518 {
519 Element termEl;
520 if ( termList != null )
521 {
522 Iterator it = termList.keySet().iterator();
523 int i=0;
524 while ( it.hasNext() )
525 {
526 i++;
527
528 if ( i > sIdx && i <= eIdx+1 )
529 {
530 fr.gouv.culture.sdx.query.lucene.TermInfo term = (fr.gouv.culture.sdx.query.lucene.TermInfo)termList.get((String)it.next());
531 termEl = factory.createElementNS(nsURI, nsPrefix + ":term");
532 termEl.setAttribute("docFreq", String.valueOf(term.getDocFrequency()));
533 String fieldname = term.getField();
534 String content;
535 if ( db != null && db.getFieldType(fieldname) == DBField.DATE ) content = Utilities.formatDate(DateField.stringToDate(term.getContent()));
536 else
537 content = term.getContent();
538 termEl.setAttribute("escapedValue", java.net.URLEncoder.encodeURL(content));
539 termEl.setAttribute("field", fieldname);
540 termEl.setAttribute("value", content);
541 termEl.setAttribute("no", String.valueOf(i));
542
543 // S'il y a un seul document, alors on retourne aussi son identificateur.
544 if ( term.getDocFrequency() == 1 ) termEl.setAttribute("docId", term.getDocId());
545
546 top.appendChild(termEl);
547 }
548 else if ( i > eIdx+1 ) break; else it.next();
549 }
550 }
551 }
552 return top;
553 }
554 */
555
556 /**
557 * Returns an XML representation of this term list.
558 *
559 * @param hdl The ContentHandler to feed with SAX events.
560 * @param page The number of the page to show.
561 */
562 public void toSAX(ContentHandler hdl, int page) throws SAXException {
563 String sdxNsUri = Framework.SDXNamespaceURI;
564 String sdxNsPrefix = Framework.SDXNamespacePrefix;
565
566 if (page < 1) page = 1;
567 if (page > super.getNbPages()) page = super.getNbPages();
568 int sIdx = ((page - 1) * super.getHitsPerPage());
569 if (sIdx < 0) sIdx = 0;
570
571 int eIdx = (page * super.getHitsPerPage()) - 1;
572 if (termList != null && eIdx >= termList.size()) eIdx = termList.size() - 1;
573
574 //Creation of local variables which are later passed into startElement() and endElement() methods-rbp13/03/02
575 String localName = Node.Name.TERMS;
576 String qName = sdxNsPrefix + ":" + localName;
577 AttributesImpl atts = new AttributesImpl();
578
579
580 if (fieldFilters != null && fieldFilters.length > 0) {
581 atts.addAttribute("", Node.Name.NAME, Node.Name.NAME, Node.Type.CDATA, fieldFilters[fieldFilters.length - 1]);
582 atts.addAttribute("", Node.Name.FIELD, Node.Name.FIELD, Node.Type.CDATA, fieldFilters[fieldFilters.length - 1]);
583 }
584
585 // Les informations sur le nombre de pages
586 //TODO?:should we also add the field name here or other information?-rbp
587 atts.addAttribute("", Node.Name.QID, Node.Name.QID, Node.Type.CDATA, String.valueOf(super.getId()));
588 atts.addAttribute("", Node.Name.PAGE, Node.Name.PAGE, Node.Type.CDATA, String.valueOf(page));
589 atts.addAttribute("", Node.Name.HPP, Node.Name.HPP, Node.Type.CDATA, String.valueOf(super.getHitsPerPage()));
590 atts.addAttribute("", Node.Name.PAGES, Node.Name.PAGES, Node.Type.CDATA, String.valueOf(super.getNbPages()));
591 if (termList != null)
592 atts.addAttribute("", Node.Name.NB, Node.Name.NB, Node.Type.CDATA, String.valueOf(termList.size()));
593 atts.addAttribute("", Node.Name.START, Node.Name.START, Node.Type.CDATA, String.valueOf(sIdx + 1));
594 atts.addAttribute("", Node.Name.END, Node.Name.END, Node.Type.CDATA, String.valueOf(eIdx + 1));
595
596 // for backward compatibility but not documented
597 atts.addAttribute("", Node.Name.ID, Node.Name.ID, Node.Type.CDATA, String.valueOf(super.getId()));
598 atts.addAttribute("", Node.Name.CURRENT_PAGE, Node.Name.CURRENT_PAGE, Node.Type.CDATA, String.valueOf(page));
599 atts.addAttribute("", Node.Name.NB_PAGES, Node.Name.NB_PAGES, Node.Type.CDATA, String.valueOf(super.getNbPages()));
600
601 //startElement() method is called for "terms" and local variables are passed-rbp12/03/02
602 hdl.startElement(sdxNsUri, localName, qName, atts);
603
604 // On va ajouter des informations sur les filtres utilises.
605 if (filter != null) filter.toSAX(hdl);
606
607 if (super.getHitsPerPage() != 0) {
608 if (termList != null) {
609 Iterator it = termList.keySet().iterator();
610 int i = 0;
611 while (it.hasNext()) {
612 i++;
613
614 if (i > sIdx && i <= eIdx + 1) {
615 TermInfo term = (TermInfo) termList.get(it.next());
616 //Creation of local variables which are later passed into startElement() and endElement() methods-rbp12/03/02
617 String subLocalName = Node.Name.TERM;
618 String subQName = sdxNsPrefix + ":" + subLocalName;
619 AttributesImpl subAtts = new AttributesImpl();
620 subAtts.addAttribute("", Node.Name.NO, Node.Name.NO, Node.Type.CDATA, String.valueOf(i));
621
622 String fieldname = term.getField();
623 String content;
624
625 int fieldType = -1;
626 try {
627 fieldType = super._searchLocations.getFieldType(fieldname);
628 } catch (SDXException sdxE) {
629 throw new SAXException(sdxE.getMessage(), sdxE);
630 }
631 if (super._searchLocations != null && fieldType == Field.DATE)
632 content = Date.formatDate(DateField.stringToDate(term.getContent()));
633 else
634 content = term.getContent();
635
636 String esc = null;
637 esc = Utilities.encodeURL(content, super.getEncoding());
638 // try { esc=java.net.URLEncoder.encodeURL(content, Framework.URL_ENCODING); } catch (java.io.UnsupportedEncodingException e) {} // 1.4
639 subAtts.addAttribute("", Node.Name.VALUE, Node.Name.VALUE, Node.Type.CDATA, content);
640 subAtts.addAttribute("", Node.Name.ESCAPED_VALUE, Node.Name.ESCAPED_VALUE, Node.Type.CDATA, esc);
641 subAtts.addAttribute("", Node.Name.FIELD, Node.Name.FIELD, Node.Type.CDATA, fieldname);
642
643
644 subAtts.addAttribute("", Node.Name.DOCS, Node.Name.DOCS, Node.Type.CDATA, String.valueOf(term.getDocFrequency()));
645 // S'il y a un seul document, alors on retourne aussi son identificateur.
646 if (term.getDocFrequency() == 1) {
647 subAtts.addAttribute("", Node.Name.ID, Node.Name.ID, Node.Type.CDATA, term.getDocId());
648 subAtts.addAttribute("", Node.Name.DB_ID, Node.Name.DB_ID, Node.Type.CDATA, term.getDbId());
649 subAtts.addAttribute("", Node.Name.APPID, Node.Name.APPID, Node.Type.CDATA, term.getAppId());
650 subAtts.addAttribute("", Node.Name.DOC_ID, Node.Name.DOC_ID, Node.Type.CDATA, term.getDocId());
651 }
652 // not documented but preserved for backward compatibility
653 subAtts.addAttribute("", Node.Name.DOC_FREQ, Node.Name.DOC_FREQ, Node.Type.CDATA, String.valueOf(term.getDocFrequency()));
654 //startElement() method is called for "term" and local variables are passed-rbp12/03/02
655 hdl.startElement(sdxNsUri, subLocalName, subQName, subAtts);
656
657 //endElement() method is called for "term" and local variables are passed-rbp12/03/02
658 hdl.endElement(sdxNsUri, subLocalName, subQName);
659
660 } else if (i > eIdx + 1) break; else it.next();
661 }
662 }
663 }
664 //endElement() method is called for "terms" and local variables are passed-rbp12/03/02
665 hdl.endElement(sdxNsUri, localName, qName);
666 }
667
668 /**
669 * Returns a list of the terms.
670 */
671 public TreeMap getList() {
672 return termList;
673 }
674
675 /**
676 * Constructs a regular expression from a term of the acceptRequest.
677 *
678 * @param str The term.
679 */
680 private static RE getRE(String str) throws RESyntaxException {
681 if (str == null) return null;
682 StringBuffer st = new StringBuffer("^");
683 for (int i = 0; i < str.length(); i++) {
684 char c = str.charAt(i);
685 switch (c) {
686 case '*':
687 st.append("(.*)");
688 break;
689
690 case '?':
691 st.append("(.?)");
692 break;
693
694 case '(':
695 case ')':
696 case '.':
697 case '\\':
698 case '[':
699 case ']':
700 case '^':
701 case '$':
702 case '+':
703 case '{':
704 case '}':
705 case '|':
706 st.append('\\');
707 st.append(c);
708 break;
709
710 default:
711 st.append(c);
712 }
713 }
714 st.append("$");
715 return new RE(st.toString());
716 }
717
718 /*
719 * Retourne une liste de termes en fonction d'un champ et d'une valeur.
720 *
721 * La valeur peut contenir des masques * et ?. Tous les termes de ce champ
722 * qui respectent la valeur masquee seront retournes. Si aucun terme ne
723 * satisfait le critere, on retourne une liste vide.
724 *
725 * @param r Un objet pour lire l'index.
726 * @param f Le nom du champ
727 * @param pattern La valeur masquee.
728 */
729 /**
730 * Returns a list of terms according to a field and a value.
731 *
732 * The value can contain masks and?. All terms of this field
733 * which respects the masked value will be turned over. If no term
734 * satisfied the criterion, an empty list is returned.
735 *
736 * @param r The index reader
737 * @param f The field
738 * @param pattern The pattern
739 * @throws SDXException
740 */
741 public static TreeMap getTerms(IndexReader r, Field f, String pattern) throws SDXException {
742 TreeMap ret = null;
743
744 if (r == null || f == null || pattern == null) return ret;
745 String internedfieldName = "";
746 if (Utilities.checkString(f.getCode()))
747 //interning the field name for better performance with lucene
748 internedfieldName = f.getCode().intern();
749
750 if (f.getLocale() != null)
751 ret = new TreeMap(Collator.getInstance(f.getLocale()));
752 else
753 //TODO?:is this acceptable default behavior?-rbp
754 ret = new TreeMap(Collator.getInstance(new Locale("fr", "FR")));
755
756 TermEnum ts = null;
757 try {
758 RE re = getRE(pattern);
759 ts = r.terms(new Term(internedfieldName, ""));
760 boolean collecting = false;
761 if (ts != null) {
762 do {
763 if (ts.term() != null && ts.term().field() == internedfieldName) {
764 collecting = true;
765 if (ts.term() != null && re != null && re.match(ts.term().text())) {
766 // First check if the term is already in the list
767 TermInfo ti = (TermInfo) ret.get(ts.term().text());
768 if (ti != null)
769 ti.update(r, ts.term());
770 else {
771 ti = new TermInfo();
772 //ti.enableLogging(_logger);
773 ti.setUp(r, ts.term());
774 ret.put(ts.term().text(), ti);
775 }
776 }
777 } else if (collecting) break;
778 } while (ts.next());
779 }
780 } catch (RESyntaxException e) {
781 String[] args = new String[1];
782 args[0] = e.getMessage();
783 throw new SDXException(null, SDXExceptionCode.ERROR_GET_TERMS, args, e);
784 } catch (IOException e) {
785 String[] args = new String[1];
786 args[0] = e.getMessage();
787 throw new SDXException(null, SDXExceptionCode.ERROR_GET_TERMS, args, e);
788 } finally {
789 try {
790 //freeing resources
791 if (ts != null) ts.close();
792 } catch (IOException e) {
793 String[] args = new String[2];
794 args[0] = f.getCode();
795 args[1] = e.getMessage();
796 throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_TERMENUM_CLOSE, args, e);
797 }
798 }
799
800 return ret;
801 }
802
803 /**
804 * Retourne une liste de valeurs pour un champ.
805 *
806 * @param r L'objet pour lire l'index.
807 * @param field Le nom du champ.
808 */
809 /* Not used anymore ? MS
810 public static TreeMap getTerms(IndexReader r, String field) throws SDXException, IOException {
811 TreeMap ret = new TreeMap(Collator.getInstance(new Locale("fr", "FR")));
812 TermEnum ts = r.terms();
813 boolean collecting = false;
814 while (ts.next()) {
815 if (ts.term().field().equals(field)) {
816 collecting = true;
817 TermInfo ti = (TermInfo)ret.get(ts.term().text());
818 if ( ti != null ) ti.update(r, ts.term());
819 else {
820 ti = new TermInfo();
821 ti.enableLogging(super.getLog());
822 ti.setUp(r, ts.term());
823 ret.put(ts.term().text(), ti);
824 }
825 } else if (collecting) break;
826 }
827 return ret;
828 }
829 */
830
831 /**
832 * Builds a list of values for a filtered f.
833 *
834 * @param hits The results.
835 * @param fields The fields in question.
836 * @param values The values for the fields in question.
837 */
838 private TreeMap getTerms(Hits hits, String[] fields, String[] values) throws SDXException {
839
840 TreeMap ret = new TreeMap();
841 if (hits == null || hits.length() == 0) return ret; // TODO : is the the better solution? Harmonize with other lists of terms...
842 Hashtable fieldMaps = new Hashtable();
843 for (int i = 0; i < hits.length(); i++) {
844 try {
845 Document hitDoc = hits.doc(i);
846 if (hitDoc != null) {
847 for (int j = 0; j < fields.length; j++) {
848 Field f = super._searchLocations.getField(fields[j]);
849 String lfname = f.getCode();
850 if (f != null) {
851 TreeMap map = null;
852 if (fieldMaps.size() > 0 && fieldMaps.containsKey(lfname))
853 map = (TreeMap) fieldMaps.get(lfname);
854 else {
855 if (f.getCollator() == null)
856 map = new TreeMap(Collator.getInstance(new Locale("fr", "FR"))); // TODO : is it OK?
857 else
858 map = new TreeMap(f.getCollator());
859 }
860 String[] fieldsValues = hitDoc.getValues(lfname);
861 if (fieldsValues != null) {
862 HashSet termsCollected = new HashSet();
863 for (int k = 0; k < fieldsValues.length; k++) {
864 String lftext = fieldsValues[k];
865 if (Utilities.checkString(lftext)) {
866 if (values == null || values.length == 0 || (j < values.length && (!Utilities.checkString(values[j]) || lftext.equals(values[j])))) {
867 if (!termsCollected.contains(lftext)) {
868 TermInfo ti = (TermInfo) map.get(lftext);
869 if (ti != null) {
870 ti.update(lfname, lftext);
871 } else {
872 ti = new TermInfo();
873 ti.enableLogging(super.getLog());
874 ti.setUp(lfname, lftext);
875 ti.update(lfname, lftext);
876 ti.setDocId(hitDoc.getField(LuceneIndex.ID_FIELD).stringValue());
877 ti.setDbId(hitDoc.getField(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXDBID).stringValue());
878 ti.setAppId(hitDoc.getField(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXAPPID).stringValue());
879 map.put(lftext, ti);
880 }
881 termsCollected.add(lftext);
882 }
883 }
884 }
885 }
886 }
887 if (map != null) fieldMaps.put(lfname, map);
888 }
889 }
890 }
891 } catch (IOException e) {
892 String[] args = new String[1];
893 args[0] = e.getMessage();
894 new SDXException(null, SDXExceptionCode.ERROR_GET_TERMS, args, e);
895 }
896 }
897
898
899 Enumeration maps = fieldMaps.elements();
900 if (maps != null) {
901 while (maps.hasMoreElements()) {
902 TreeMap outmap = (TreeMap) maps.nextElement();
903 ret.putAll(outmap);
904 }
905 maps = null;
906 }
907 return ret;
908 }
909
910
911 /**
912 * Builds a list of values for a filtered f.
913 *
914 * @param r The reader.
915 * @param f The f.
916 * @param docs A list of documents as a filter.
917 */
918 public TreeMap getTerms(IndexReader r, Field f, BitSet docs) throws SDXException {
919
920 if (f == null) return new TreeMap(); // TODO : is the the better solution? Harmonize with other lists of terms...
921
922 //interning the f name for better performance with lucene
923 String internedfieldName = "";
924 if (Utilities.checkString(f.getCode())) internedfieldName = f.getCode().intern();
925
926 TreeMap ret;
927 if (f.getCollator() == null)
928 ret = new TreeMap(Collator.getInstance(new Locale("fr", "FR"))); // TODO : is it OK?
929 else
930 ret = new TreeMap(f.getCollator());
931
932 TermEnum ts = null;
933 try {
934 ts = r.terms(new Term(internedfieldName, ""));
935 boolean collecting = false;
936 BitSet termDocumentSet;
937 int nbDocs;
938 do {
939 if (ts.term() != null && ts.term().field() == internedfieldName) {
940 collecting = true;
941
942 // We build the bitset for this term and compare
943 // it with the one we have, and then count what's
944 // left.
945
946 termDocumentSet = getDocumentSet(r, ts.term());
947 if (docs != null) termDocumentSet.and(docs);
948 nbDocs = Bits.countBits(termDocumentSet);
949 if (nbDocs > 0) {
950 TermInfo ti = (TermInfo) ret.get(ts.term().text());
951 if (ti != null)
952 ti.update(r, ts.term());
953 else {
954 ti = new TermInfo();
955 ti.enableLogging(super.getLog());
956 ti.setUp(r, ts.term());
957 ret.put(ts.term().text(), ti);
958 }
959 }
960 } else if (collecting) break;
961 } while (ts.next());
962 return ret;
963 } catch (IOException e) {
964 String[] args = new String[1];
965 args[0] = e.getMessage();
966 throw new SDXException(null, SDXExceptionCode.ERROR_GET_TERMS, args, e);
967 } finally {
968 try {
969 //freeing resources
970 if (ts != null) ts.close();
971 } catch (IOException e) {
972 String[] args = new String[2];
973 args[0] = f.getCode();
974 args[1] = e.getMessage();
975 throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_TERMENUM_CLOSE, args, e);
976 }
977 }
978 }
979
980
981 /**
982 * Returns a list of the documents associated with a term within the index.
983 *
984 * @param r The index reader.
985 * @param term The term for searching.
986 */
987 public static BitSet getDocumentSet(IndexReader r, Term term) throws IOException {
988 BitSet bits = new BitSet(r.numDocs());
989 TermDocs docs = r.termDocs(term);
990 while (docs.next()) bits.set(docs.doc());
991 return bits;
992 }
993
994 /*
995 // Ecrit la liste de termes en XML dans un writer.
996 // @param writer Le writer ou ecrire.
997 // @param page La page a envoyer.
998 public void toXML(Writer writer, int page) throws SDXException
999 {
1000 XMLSerializer ser = new XMLSerializer(writer, new OutputFormat("xml", "UTF-8", true));
1001 try
1002 {
1003 ser.serialize(toDOM(new DocumentImpl(), page));
1004 }
1005 catch ( IOException e )
1006 {
1007 throw new SDXException(e, "fr", "Impossible de serialiser le document XML");
1008 }
1009 }
1010 */
1011
1012 /* No need for this anymore as it is being handled in the servlet class, GetTerms.java.-rbp21/03/02
1013 // Ecrit la liste de termes en XML dans un flux de sortie.
1014 // @param os Le flux ou ecrire.
1015 // @param page La page a envoyer.
1016 public void toXML(OutputStream os, int page) throws SDXException
1017 {
1018 XMLSerializer ser = new XMLSerializer(os, new OutputFormat("xml", "UTF-8", true));
1019 try
1020 {
1021 ser.serialize(toDOM(new DocumentImpl(), page));
1022 }
1023 catch ( IOException e )
1024 {
1025 throw new SDXException(e, "fr", "Impossible de serialiser le document XML");
1026 }
1027 }
1028 */
1029
1030 /**
1031 * Counts the number of pages for these terms.
1032 */
1033 public int countPages() {
1034 if (termList == null)
1035 return 0;
1036 else if (termList.size() == 0)
1037 return 0;
1038 else {
1039 if (super.getHitsPerPage() == 0) return 1;
1040 float temp = ((float) termList.size() / (float) super.getHitsPerPage());
1041 if (temp != Math.round(temp))
1042 return (int) (Math.round(Math.floor(temp) + 1));
1043 else
1044 return Math.round(temp);
1045 }
1046 }
1047
1048 /**
1049 * Set the number of hits per page.
1050 *
1051 * @param nb The number of hits.
1052 */
1053 public void setHitsPerPage(int nb) {
1054 if (nb < 0)
1055 setAllHits();
1056 else {
1057 super.setHitsPerPage(nb);
1058 super.setNbPages(countPages());
1059 }
1060 }
1061
1062 /**
1063 * Initializes the collator used for sorting .
1064 *
1065 * @param field The field from the document base.
1066 */
1067 private void initCollator(Field field) {
1068 if (field != null) sortCollator = field.getCollator();
1069 }
1070
1071 /**
1072 * Indicate to return all the results on only one page.
1073 */
1074 public void setAllHits() {
1075 if (termList != null)
1076 super.setHitsPerPage(termList.size());
1077 super.setNbPages(1);
1078 }
1079
1080 protected String getClassNameSuffix() {
1081 return Terms.CLASS_NAME_SUFFIX;
1082 }
1083
1084
1085 }

savannah-hackers-public@gnu.org
ViewVC Help
Powered by ViewVC 1.1.26