/[sdx]/sdx_v2/src/java/fr/gouv/culture/sdx/oai/LuceneDocumentBaseOAIRepository.java
ViewVC logotype

Contents of /sdx_v2/src/java/fr/gouv/culture/sdx/oai/LuceneDocumentBaseOAIRepository.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.54 - (show annotations) (download)
Tue Apr 19 10:30:09 2005 UTC (19 years, 1 month ago) by malo_pichot
Branch: MAIN
CVS Tags: V_23_BETA1
Changes since 1.53: +2 -2 lines
DEBUG: use binaries operators instead of logics operators

1 /*
2 SDX: Documentary System in XML.
3 Copyright (C) 2000, 2001, 2002 Ministere de la culture et de la communication (France), AJLSM
4
5 Ministere de la culture et de la communication,
6 Mission de la recherche et de la technologie
7 3 rue de Valois, 75042 Paris Cedex 01 (France)
8 mrt@culture.fr, michel.bottin@culture.fr
9
10 AJLSM, 17, rue Vital Carles, 33000 Bordeaux (France)
11 sevigny@ajlsm.com
12
13 This program is free software; you can redistribute it and/or
14 modify it under the terms of the GNU General Public License
15 as published by the Free Software Foundation; either version 2
16 of the License, or (at your option) any later version.
17
18 This program is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of f
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
21 See the GNU General Public License for more details.
22
23 You should have received a copy of the GNU General Public License
24 along with this program; if not, write to the
25 Free Software Foundation, Inc.
26 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
27 or connect to:
28 http://www.fsf.org/copyleft/gpl.html
29 */
30 package fr.gouv.culture.sdx.oai;
31
32 import fr.gouv.culture.oai.OAIError;
33 import fr.gouv.culture.oai.OAIObject;
34 import fr.gouv.culture.oai.OAIRequest;
35 import fr.gouv.culture.oai.util.OAIUtilities;
36 import fr.gouv.culture.sdx.document.XMLDocument;
37 import fr.gouv.culture.sdx.documentbase.LuceneDocumentBase;
38 import fr.gouv.culture.sdx.exception.SDXException;
39 import fr.gouv.culture.sdx.exception.SDXExceptionCode;
40 import fr.gouv.culture.sdx.framework.Framework;
41 import fr.gouv.culture.sdx.pipeline.Pipeline;
42 import fr.gouv.culture.sdx.search.lucene.DateField;
43 import fr.gouv.culture.sdx.search.lucene.FieldList;
44 import fr.gouv.culture.sdx.search.lucene.analysis.MetaAnalyzer;
45 import fr.gouv.culture.sdx.search.lucene.query.*;
46 import fr.gouv.culture.sdx.utils.Utilities;
47 import fr.gouv.culture.sdx.utils.configuration.ConfigurationUtils;
48 import fr.gouv.culture.sdx.utils.constants.ContextKeys;
49 import fr.gouv.culture.sdx.utils.database.DatabaseEntity;
50 import fr.gouv.culture.sdx.utils.logging.LoggingUtils;
51 import org.apache.avalon.framework.service.ServiceException;
52 import org.apache.avalon.framework.configuration.Configuration;
53 import org.apache.avalon.framework.configuration.ConfigurationException;
54 import org.apache.avalon.framework.context.ContextException;
55 import org.apache.avalon.framework.parameters.Parameters;
56 import org.apache.cocoon.xml.IncludeXMLConsumer;
57 import org.apache.cocoon.xml.XMLPipe;
58 import org.apache.lucene.document.Document;
59 import org.apache.lucene.document.Field;
60 import org.apache.lucene.index.Term;
61 import org.apache.lucene.search.BooleanQuery;
62 import org.apache.lucene.search.Hits;
63 import org.apache.lucene.search.TermQuery;
64 import org.apache.lucene.search.WildcardQuery;
65 import org.xml.sax.SAXException;
66 import org.xml.sax.helpers.AttributesImpl;
67
68 import java.io.IOException;
69 import java.util.*;
70
71 /**
72 * Created by IntelliJ IDEA.
73 * User: rpandey
74 * Date: May 12, 2003
75 * Time: 12:31:36 PM
76 * To change this template use Options | File Templates.
77 */
78 //TODO: better abstraction here
79
80 public class LuceneDocumentBaseOAIRepository extends AbstractDocumentBaseOAIRepository {
81
82
83 /**Search index of the underlying LuceneDocumentBase*/
84 protected LuceneIndex luceneSearchIndex = null;
85
86 /**query string for document selection*/
87 protected SimpleQuery includeQuery = null;
88 /**query string for document selection*/
89 protected SimpleQuery excludeQuery = null;
90
91 /**Set mappings from configuration file*/
92 protected Hashtable setMappings = null;
93
94
95 protected final String ATTRIBUTE_NAME_SDXFIELD = "sdxField";
96
97
98 protected final String ELEMENT_NAME_OAI_FORMAT = "oai-format";
99
100 /*Lucene Specific*/
101 protected final String ELEMENT_NAME_OAI_SUBSET = "oai-subset";
102 protected final String ELEMENT_NAME_INCLUDE = "include";
103 protected final String ELEMENT_NAME_EXCLUDE = "exclude";
104 protected static final String ATTRIBUTE_NAME_INCLUDE_QUERY = "includeQuery";
105 protected static final String ATTRIBUTE_NAME_EXCLUDE_QUERY = "excludeQuery";
106 public static final String PARAMETER_NAME_INCLUDE_QUERY = ATTRIBUTE_NAME_INCLUDE_QUERY;
107 public static final String PARAMETER_NAME_EXCLUDE_QUERY = ATTRIBUTE_NAME_EXCLUDE_QUERY;
108
109
110 public LuceneDocumentBaseOAIRepository(LuceneDocumentBase base) {
111 if (base != null) {
112 this.documentBase = base;
113 this.documentBaseId = base.getId();
114 this.luceneSearchIndex = (LuceneIndex) base.getIndex();
115 }
116 //indicating the protocol version supported
117 super.protocolVersion = "2.0";
118 //yyyy-MM-dd'T'HH:mm:ssz this is the java date format string we use in lucene document base
119 //A repository that supports YYYY-MM-DDThh:mm:ssZ should indicate so in the Identify response
120 super.granularity = "YYYY-MM-DDThh:mm:ssZ";
121 //indicating level of deleted record support according to OAI-PMH guidelines
122 super.deletedRecord = "transient";
123 }
124
125
126 public void configure(Configuration configuration) throws ConfigurationException {
127 //should configure protocolVersion, granularity, deleteRecord, repositoryName, and admin email
128 super.configure(configuration);
129 configureSets(configuration);
130 configureFormats(configuration);
131 configureSubset(configuration);
132 }
133
134 protected void configureSets(Configuration configuration) throws ConfigurationException {
135 Configuration setsConf = configuration.getChild(DocumentBaseOAIRepository.ConfigurationNode.OAI_SETS, false);
136 if (setsConf != null) {
137 Configuration[] sets = setsConf.getChildren(DocumentBaseOAIRepository.ConfigurationNode.OAI_SET);
138 for (int i = 0; i < sets.length; i++) {
139 Configuration set = sets[i];
140 if (set != null) {
141 String sdxFieldName = set.getAttribute(ATTRIBUTE_NAME_SDXFIELD, "");
142 String setName = set.getAttribute(DocumentBaseOAIRepository.ConfigurationNode.NAME, null);
143 String setSpec = set.getAttribute(DocumentBaseOAIRepository.ConfigurationNode.SPEC, sdxFieldName);//using sdxFieldName as default setSpec
144 String includeQuery = set.getAttribute(ATTRIBUTE_NAME_INCLUDE_QUERY, "");
145 String excludeQuery = set.getAttribute(ATTRIBUTE_NAME_EXCLUDE_QUERY, "");
146 ConfigurationUtils.checkConfAttributeValue(ATTRIBUTE_NAME_SDXFIELD, sdxFieldName, set.getLocation());
147 ConfigurationUtils.checkConfAttributeValue(DocumentBaseOAIRepository.ConfigurationNode.NAME, setName, set.getLocation());
148 if (this.luceneSearchIndex != null) {
149 MetaAnalyzer ma = this.luceneSearchIndex.getMetaAnalyzer();
150 if (ma != null) {
151 FieldList fieldsDef = ma.getFieldList();
152 if (fieldsDef != null) {
153 fr.gouv.culture.sdx.search.lucene.Field sdxField = fieldsDef.getField(sdxFieldName);
154 if (sdxField != null && !sdxField.isInBrief()) {
155 String[] args = {sdxFieldName};
156 SDXException sdxE = new SDXException(logger, SDXExceptionCode.ERROR_SDX_OAI_FIELD_NOT_BRIEF, args, null);
157 throw new ConfigurationException(sdxE.getMessage(), sdxE);
158 }
159 }
160 }
161 }
162 Parameters setParams = new Parameters();
163 if (Utilities.checkString(setName))
164 setParams.setParameter(PARAMETER_NAME_SET_NAME, setName);
165 if (Utilities.checkString(sdxFieldName))
166 setParams.setParameter(PARAMETER_NAME_SDX_FIELD, sdxFieldName);
167 if (Utilities.checkString(setSpec))
168 setParams.setParameter(PARAMETER_NAME_SET_SPEC, sdxFieldName);
169 try {
170 if (Utilities.checkString(includeQuery)) {
171 getLuceneSimpleQuery(includeQuery);//parsing the query to reveal any config problems
172 setParams.setParameter(PARAMETER_NAME_INCLUDE_QUERY, includeQuery);
173 }
174
175 if (Utilities.checkString(excludeQuery)) {
176 getLuceneSimpleQuery(excludeQuery);//parsing the query to reveal any config problems
177 setParams.setParameter(PARAMETER_NAME_EXCLUDE_QUERY, excludeQuery);
178 }
179 if (this.setMappings == null) this.setMappings = new Hashtable();
180 Utilities.isObjectUnique(this.setMappings, setSpec, setParams);
181 this.setMappings.put(setSpec, setParams);
182 } catch (SDXException e) {
183 throw new ConfigurationException(e.getMessage(), e);
184 }
185 }
186 }
187
188 }
189 }
190
191
192 protected void configureFormats(Configuration configuration) throws ConfigurationException {
193 //should configure metadata formats here from config file
194 Configuration[] formats = configuration.getChildren(ELEMENT_NAME_OAI_FORMAT);
195 if (formats.length > 0) super.metadataFormats = new Hashtable();
196 for (int i = 0; i < formats.length; i++) {
197 BasicOAIMetadataFormat format = null;
198 String formatPrefix = formats[i].getAttribute(OAIObject.Node.Name.METADATA_PREFIX);
199 if (formatPrefix.equalsIgnoreCase(Framework.SDXNamespacePrefix)) {
200 format = new SDXOAIMetadataFormat();
201 } else
202 format = new BasicOAIMetadataFormat();
203 try {
204 format.enableLogging(this.logger);
205 format.contextualize(Utilities.createNewReadOnlyContext(getContext()));
206 format.service(this.manager);
207 format.configure(formats[i]);
208 formatPrefix = format.getPrefix();
209 if (Utilities.checkString(formatPrefix)) {
210 Utilities.isObjectUnique(super.metadataFormats, formatPrefix, format);
211 super.metadataFormats.put(formatPrefix, format);
212 }
213 } catch (ServiceException e) {
214 throw new ConfigurationException(e.getMessage(), e);
215 } catch (SDXException e) {
216 throw new ConfigurationException(e.getMessage(), e);
217 } catch (ContextException e) {
218 throw new ConfigurationException(e.getMessage(), e);
219 }
220
221 }
222
223 if (!super.metadataFormats.containsKey(OAIObject.Node.Prefix.OAI_DC)) {
224 String[] args = new String[1];
225 args[0] = this.documentBaseId;
226 SDXException sdxE = new SDXException(logger, SDXExceptionCode.ERROR_NO_OAI_DC_METADATA_FORMAT, args, null);
227 throw new ConfigurationException(sdxE.getMessage(), sdxE);
228 }
229 }
230
231
232 protected void configureSubset(Configuration configuration) throws ConfigurationException {
233 //the subset of documents which should be include and excluded via lucene query syntax
234 Configuration oaiSubset = configuration.getChild(ELEMENT_NAME_OAI_SUBSET, false);
235 if (oaiSubset != null) {
236
237 SearchLocations sLocs = this.getSearchLocation();
238
239 Configuration include = oaiSubset.getChild(ELEMENT_NAME_INCLUDE, false);
240 Configuration exclude = oaiSubset.getChild(ELEMENT_NAME_EXCLUDE, false);
241
242 /*include and no exclude*/
243 if (include != null && exclude == null) {
244 String includeQueryString = include.getAttribute(DocumentBaseOAIRepository.ConfigurationNode.QUERY);
245 if (Utilities.checkString(includeQueryString)) {
246
247 this.includeQuery = new SimpleQuery();
248 this.includeQuery.enableLogging(this.logger);
249 try {
250 this.includeQuery.setUp(sLocs, null, includeQueryString);
251 } catch (SDXException e) {
252 throw new ConfigurationException(e.getMessage(), e);
253 }
254 }
255
256 }
257
258 /*exclude and no include*/
259 if (exclude != null && include == null) {
260 String excludeQueryString = exclude.getAttribute(DocumentBaseOAIRepository.ConfigurationNode.QUERY);
261 if (Utilities.checkString(excludeQueryString)) {
262 this.excludeQuery = new SimpleQuery();
263 this.excludeQuery.enableLogging(this.logger);
264 try {
265 this.excludeQuery.setUp(sLocs, null, excludeQueryString);
266 } catch (SDXException e) {
267 throw new ConfigurationException(e.getMessage(), e);
268 }
269 }
270
271 }
272
273 /*both include and exclude*/
274 if (exclude != null && include != null) {
275 String excludeQueryString = exclude.getAttribute(DocumentBaseOAIRepository.ConfigurationNode.QUERY);
276 if (Utilities.checkString(excludeQueryString)) {
277 this.excludeQuery = new SimpleQuery();
278 this.excludeQuery.enableLogging(this.logger);
279 try {
280 this.excludeQuery.setUp(sLocs, null, excludeQueryString);
281 } catch (SDXException e) {
282 throw new ConfigurationException(e.getMessage(), e);
283 }
284 }
285 String includeQueryString = include.getAttribute(DocumentBaseOAIRepository.ConfigurationNode.QUERY);
286 if (Utilities.checkString(includeQueryString)) {
287
288 this.includeQuery = new SimpleQuery();
289 this.includeQuery.enableLogging(this.logger);
290 try {
291 this.includeQuery.setUp(sLocs, null, includeQueryString);
292 } catch (SDXException e) {
293 throw new ConfigurationException(e.getMessage(), e);
294 }
295 }
296 }
297
298 }
299
300 }
301
302
303 public void listSets(OAIRequest request) throws SAXException {
304 /*
305 Summary and Usage Notes
306 This verb is used to retrieve the set structure of a repository, useful for selective harvesting.
307
308 Arguments
309 resumptionToken an exclusive argument with a value that is the flow control token returned by a previous ListSets request that issued an incomplete list
310 Error and Exception Conditions
311 badArgument - The request includes illegal arguments or is missing required arguments.
312 badResumptionToken - The value of the resumptionToken argument is invalid or expired.
313 noSetHierarchy - The repository does not support sets.
314 */
315 if (this.setMappings == null || this.setMappings.size() == 0)
316 super.sendNoSetHierarchyError();
317 else if (Utilities.checkString(request.getResumptionToken())) {
318 new OAIError(OAIError.ERROR_BAD_RESUMPTION_TOKEN, "This repository does not support resumptionTokens for ListSets requests").toSAX(this);
319 } else {
320 super.startVerbEvent(request);
321 Enumeration sets = this.setMappings.keys();
322 if (sets != null) {
323 while (sets.hasMoreElements()) {
324 String setSpec = (String) sets.nextElement();
325 if (Utilities.checkString(setSpec)) {
326
327 Parameters setParams = (Parameters) this.setMappings.get(setSpec);
328 String setName = "";
329 String sdxFieldName = "";
330 if (setParams != null) {
331 setName = setParams.getParameter(PARAMETER_NAME_SET_NAME, "");
332 sdxFieldName = setParams.getParameter(PARAMETER_NAME_SDX_FIELD, "");
333 }
334 Terms terms = new Terms();
335 if (Utilities.checkString(sdxFieldName)) {
336 terms.enableLogging(this.logger);
337 try {
338 terms.setUp(getSearchLocation(), sdxFieldName, "*");
339 } catch (SDXException e) {
340 LoggingUtils.logException(logger, e);
341 }
342 }
343
344 TreeMap termsMap = terms.getList();
345 if (termsMap != null && termsMap.size() > 0) {
346 //using each field value for the defined set
347 Iterator termsItr = termsMap.keySet().iterator();
348 while (termsItr.hasNext()) {
349 String key = (String) termsItr.next();
350 setSpec = key;
351
352 super.startElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.SET, OAIObject.Node.Name.SET, null);
353
354 if (Utilities.checkString(setName))
355 super.sendElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.SET_NAME, OAIObject.Node.Name.SET_NAME, null, setName);
356 if (Utilities.checkString(setSpec))
357 super.sendElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.SET_SPEC, OAIObject.Node.Name.SET_SPEC, null, setSpec);
358
359 super.endElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.SET, OAIObject.Node.Name.SET);
360
361
362 }
363 } else {
364 //using the application.xconf definitions
365 super.startElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.SET, OAIObject.Node.Name.SET, null);
366
367 if (Utilities.checkString(setName))
368 super.sendElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.SET_NAME, OAIObject.Node.Name.SET_NAME, null, setName);
369 if (Utilities.checkString(setSpec))
370 super.sendElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.SET_SPEC, OAIObject.Node.Name.SET_SPEC, null, setSpec);
371
372 super.endElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.SET, OAIObject.Node.Name.SET);
373 }
374 }
375 }
376 }
377
378 super.endVerbEvent(request);
379 }
380
381 }
382
383
384 public void listIdentifiers(OAIRequest request) throws SAXException {
385 /*
386 Summary and Usage Notes
387 This verb is an abbreviated form of ListRecords, retrieving only headers rather than records. Optional arguments permit selective harvesting of headers based on set membership and/or _datestamp. Depending on the repository's support for deletions, a returned header may have a status attribute of "deleted" if a record matching the arguments specified in the request has been deleted.
388
389 Arguments
390 from an optional argument with a UTCdatetime value , which specifies a lower bound for _datestamp-based selective harvesting.
391 until an optional argument with a UTCdatetime value , which specifies a upper bound for _datestamp-based selective harvesting.
392 metadataPrefix a required argument, which specifies that headers should be returned only if the metadata format matching the supplied metadataPrefix is available or, depending on the repository's support for deletions, has been deleted. The metadata formats supported by a repository and for a particular item can be retrieved using the ListMetadataFormats request.
393 set an optional argument with a setSpec value , which specifies set criteria for selective harvesting.
394 resumptionToken an exclusive argument with a value that is the flow control token returned by a previous ListIdentifiers request that issued an incomplete list.
395 Error and Exception Conditions
396 badArgument - The request includes illegal arguments or is missing required arguments.
397 badResumptionToken - The value of the resumptionToken argument is invalid or expired.
398 cannotDisseminateFormat - The value of the metadataPrefix argument is not supported by the repository.
399 noRecordsMatch- The combination of the values of the from, until , and set arguments results in an empty list.
400 noSetHierarchy - The repository does not support sets.
401 */
402 listIdentifiersOrRecords(request);
403 }
404
405 public void getRecord(OAIRequest request) throws SAXException {
406 /*
407 Arguments
408 identifier a required argument that specifies the unique identifier of the item in the repository from which the record must be disseminated.
409 metadataPrefix a required argument that specifies the metadataPrefix of the format that should be included in the metadata part of the returned record .
410 - A record should only be returned if the format specified by the metadataPrefix can be disseminated from the item identified by the value of the identifier argument.
411 -The metadata formats supported by a repository and for a particular record can be retrieved using the ListMetadataFormats request.
412 Error and Exception Conditions
413 badArgument - The request includes illegal arguments or is missing required arguments.
414 cannotDisseminateFormat - The value of the metadataPrefix argument is not supported by the item identified by the value of the identifier argument
415 idDoesNotExist - The value of the identifier argument is unknown or illegal in this repository.
416 */
417 String mdPrefix = request.getMetadataPrefix();
418 if (!Utilities.checkString(mdPrefix) || !metadataFormats.containsKey(mdPrefix)) return;
419 String id = request.getIdentifier();
420 Hits hits = executeIdQuery(request);
421 if (hits == null || hits.length() == 0) {
422 new OAIError(OAIError.ERROR_ID_DOES_NOT_EXIST, "The value of the identifier argument, " + id + ", is unknown or illegal in this repository.").toSAX(this);
423 return;//idDoesNotExist
424 } else if (hits.length() == 1) {
425 try {
426 ArrayList availableFormats = verifyMetadataFormatForRecord(hits.doc(0), mdPrefix);
427 if (availableFormats != null && !availableFormats.contains(this.metadataFormats.get(mdPrefix)))
428 new OAIError(OAIError.ERROR_CANNOT_DISSEMINATE_FORMAT, "The value of the metadataPrefix argument, " + mdPrefix + ", is not supported by the item identified by the value of the identifier argument, " + id + " .").toSAX(this);
429 else {
430 super.startVerbEvent(request);
431 sendRecord(request, hits.doc(0), mdPrefix);
432 super.endVerbEvent(request);
433 }
434 } catch (IOException e) {
435 throw new SAXException(e.getMessage(), e);
436 }
437 }
438 }
439
440
441 public void listRecords(OAIRequest request) throws SAXException {
442 /*
443 Summary and Usage Notes
444 This verb is used to harvest records from a repository. Optional arguments permit selective harvesting of records based on set membership and/or _datestamp. Depending on the repository's support for deletions, a returned header may have a status attribute of "deleted" if a record matching the arguments specified in the request has been deleted. No metadata will be present for records with deleted status.
445
446 Arguments
447 from an optional argument with a UTCdatetime value , which specifies a lower bound for _datestamp-based selective harvesting.
448 until an optional argument with a UTCdatetime value , which specifies a upper bound for _datestamp-based selective harvesting.
449 set an optional argument with a setSpec value , which specifies set criteria for selective harvesting.
450 resumptionToken an exclusive argument with a value that is the flow control token returned by a previous ListRecords request that issued an incomplete list.
451 metadataPrefix a required argument (unless the exclusive argument resumptionToken is used) that specifies the metadataPrefix of the format that should be included in the metadata part of the returned records. Records should be included only for items from which the metadata format
452 matching the metadataPrefix can be disseminated. The metadata formats supported by a repository and for a particular item can be retrieved using the ListMetadataFormats request.
453 Error and Exception Conditions
454 badArgument - The request includes illegal arguments or is missing required arguments.
455 badResumptionToken - The value of the resumptionToken argument is invalid or expired.
456 cannotDisseminateFormat - The value of the metadataPrefix argument is not supported by the repository.
457 noRecordsMatch - The combination of the values of the from, until, set, and metadataPrefix arguments results in an empty list.
458 noSetHierarchy - The repository does not support sets.
459 */
460 listIdentifiersOrRecords(request);
461 }
462
463 /**Protected method which search documents in the LuceneDocumentBase and constructs OAI records.
464 * The method to search documents is the same for the verbs ListIdentifiers and ListRecords.
465 *
466 * @params request The harvester request
467 * @author Malo Pichot*/
468 protected void listIdentifiersOrRecords(OAIRequest request) throws SAXException {
469 int verb = request.getVerb();
470 String mdPrefix = request.getMetadataPrefix();
471 String resumptionToken = request.getResumptionToken();
472 String newResumptionToken = "";
473 int currentBatch = 0;
474 int cursor = -1;
475 Hits hits = null;
476
477 if (Utilities.checkString(resumptionToken)){
478 //we have a resumptionToken
479
480 if (!_database.entityExists(resumptionToken)){//miss the DBE, we fail
481 new OAIError(OAIError.ERROR_BAD_RESUMPTION_TOKEN, "The value of the resumptionToken argument is invalid or expired.").toSAX(this);
482 return;
483 }
484
485 else {
486 //we have the databaseEntity, we can go on
487 mdPrefix = getResumptionTokenProperty(resumptionToken, OAIRequest.URL_PARAM_NAME_METADATA_PREFIX);
488
489 cursor = Integer.parseInt(getResumptionTokenCursor(resumptionToken));
490 currentBatch = (cursor/this.numRecordsPerResponse)+1;
491
492 try {//Get the results in context
493 hits = (Hits) _context.get(/*resultsId*/getResumptionTokenProperty(resumptionToken, "resultsId"));
494 } catch (ContextException e) {
495 LoggingUtils.logException(logger, e);
496 new OAIError(OAIError.ERROR_BAD_RESUMPTION_TOKEN, "The value of the resumptionToken argument is invalid or expired.").toSAX(this);
497 resetInternalObjects(resumptionToken, newResumptionToken);
498 }
499
500 }
501
502 } else{//we do not have resumptionToken
503 //Check for errors first
504 if (!Utilities.checkString(mdPrefix)) {
505 new OAIError(OAIError.ERROR_NO_METADATA_FORMATS, "The request does not contain any metadataPrefix").toSAX(this);
506 resetInternalObjects(resumptionToken, newResumptionToken);
507 return;
508 } else if(!metadataFormats.containsKey(mdPrefix)){
509 new OAIError(OAIError.ERROR_NO_METADATA_FORMATS, "The metadataPrefix is not available: \""+mdPrefix+"\"").toSAX(this);
510 resetInternalObjects(resumptionToken, newResumptionToken);
511 return;
512 }
513
514 try {
515 hits = executeQueryForRequestParams(request);//execute the query
516 } catch (SDXException e) {
517 LoggingUtils.logException(logger, e);
518 resetInternalObjects(resumptionToken, newResumptionToken);
519 }
520 }
521
522 //Work on results
523 if (hits == null || hits.length() == 0) {//noRecordsMatch
524 new OAIError(OAIError.ERROR_NO_RECORDS_MATCH, "The combination of the values of the from \""+request.getFrom()+"\", until \""+request.getUntil()+"\", set \""+request.getSetIdentifier()+"\", and metadataPrefix \""+mdPrefix+"\" arguments results in an empty list.").toSAX(this);
525 if ( Utilities.checkString(resumptionToken) | Utilities.checkString(newResumptionToken) )//if we were working with resumtionToken, we have to delete info about in database
526 resetInternalObjects(resumptionToken, newResumptionToken);
527 return;
528 } else{//we have results
529 boolean startVerbEventSent = false;
530 int resultsLength = hits.length();
531 int end = resultsLength - 1;
532 int start = currentBatch * this.numRecordsPerResponse;
533 int endBatch = start + (this.numRecordsPerResponse - 1);
534 for (int i = start; i < resultsLength && i <= endBatch; i++) {
535 try {
536 Document hitDoc = hits.doc(i);
537 if (i == endBatch && i != end) {
538 //last document of the current batch, create a new resumptionToken
539 if (newResumptionToken == null || newResumptionToken.equals("")){
540 try {
541 newResumptionToken = createResumptionToken(request);
542 } catch (SDXException e) {
543 LoggingUtils.logException(logger, e);
544 resetInternalObjects(resumptionToken, newResumptionToken);
545 }
546 }
547 _context.put(newResumptionToken+"_results", hits);//put the results in context ; so, later, we do not need to execute the query again
548 }
549 if (i == end && Utilities.checkString(resumptionToken)) {//reset all things at the last document
550 resetInternalObjects(resumptionToken, newResumptionToken);
551 }
552 ArrayList avaliableFormats = verifyMetadataFormatForRecord(hitDoc, mdPrefix);
553 if (avaliableFormats != null && avaliableFormats.contains(this.metadataFormats.get(mdPrefix))) {
554 if (!startVerbEventSent) {
555 super.startVerbEvent(request);
556 startVerbEventSent = true;
557 }
558
559 switch(verb){
560 case OAIRequest.VERB_LIST_IDENTIFIERS:
561 sendIdentifier(request, hitDoc);
562 break;
563 case OAIRequest.VERB_LIST_RECORDS:
564 sendRecord(request, hitDoc, mdPrefix);
565 break;
566 default:
567 case OAIRequest.VERB_UNKNOWN:
568 String verbString = request.getVerbString();
569 String errorString = "The verb";
570 if (OAIUtilities.checkString(verbString)) errorString += ", " + verbString + ",";
571 errorString += " is unknown or a value was not provided.";
572 new OAIError(OAIError.ERROR_BAD_VERB, errorString).toSAX(this);
573 break;
574 }
575 }
576 } catch (IOException e) {
577 LoggingUtils.logException(logger, e);
578 }
579 }
580
581 if (startVerbEventSent) {
582 //Do we have to send resumption token informations
583 if (OAIUtilities.checkString(newResumptionToken)){//we have to send info about a resumptionToken
584 super.sendResumptionToken(newResumptionToken, getResumptionTokenCursor(newResumptionToken), new String(""+resultsLength));
585 } else if (OAIUtilities.checkString(resumptionToken)){//harvest is ending, only send info about cursor and completeListSize
586 super.sendResumptionToken(null, new String(""+resultsLength), new String(""+resultsLength));
587 }
588 super.endVerbEvent(request);
589 } else {
590 new OAIError(OAIError.ERROR_NO_RECORDS_MATCH, "The combination of the values of the from \""+request.getFrom()+"\", until \""+request.getUntil()+"\", set \""+request.getSetIdentifier()+"\", and metadataPrefix \""+mdPrefix+"\" arguments results in an empty list.").toSAX(this);
591 resetInternalObjects(resumptionToken, newResumptionToken);
592 return;
593 }
594 }
595 }
596
597 //Delete DBE and reset newResumptionToken
598 protected void resetInternalObjects (String resumptionToken, String newResumptionToken){
599 String currentRt = resumptionToken;
600 if (currentRt==null || currentRt.equals(""))
601 currentRt = newResumptionToken;
602 if ( currentRt!=null && !currentRt.equals("")) {//we have to be sure that we have to delete something
603 try {//reset the DBE
604 String resultsId = getResumptionTokenProperty(currentRt, "resultsId");
605 if (resultsId != null) _context.put(resultsId,null);//reset resultsId in context
606 super._database.delete(new DatabaseEntity(currentRt));//delete the DBE
607 super._database.optimize();
608 newResumptionToken = null;//no resumptionToken needed
609 } catch (SDXException e) {
610 LoggingUtils.logException(logger, e);
611 }
612 }
613 }
614
615 public void listMetadataFormats(OAIRequest request) throws SAXException {
616 /*
617 Summary and Usage Notes
618 This verb is used to retrieve the metadata formats available from a repository. An optional argument restricts the request to the formats available for a specific item.
619
620 Arguments
621 identifier an optional argument that specifies the unique identifier of the item for which available metadata formats are being requested. If this argument is omitted, then the response includes all metadata formats supported by this repository. Note that the fact that a metadata format is supported by a repository does not mean that it can be disseminated from all items in the repository.
622 Error and Exception Conditions
623 badArgument - The request includes illegal arguments or is missing required arguments.
624 idDoesNotExist - The value of the identifier argument is unknown or illegal in this repository.
625 noMetadataFormats - There are no metadata formats available for the specified item.
626 */
627 /*output should look like this for each format
628 <metadataFormat>
629 <metadataPrefix>oai_dc</metadataPrefix>
630 <schema>http://www.openarchives.org/OAI/2.0/oai_dc.xsd
631 </schema>
632 <metadataNamespace>http://www.openarchives.org/OAI/2.0/oai_dc/
633 </metadataNamespace>
634 </metadataFormat>
635 */
636
637 /*NOTE: we don't use the request object here to get an identifier because
638 *based upon our architecture we can provide all formats via a pipeline transformation
639 * but what if a document doesn't contain any of the mapped fields or a field
640 */
641
642 //if we don't have an id we show all mdformates
643 if (request != null) {
644 String id = request.getIdentifier();
645 if (!Utilities.checkString(id)) {
646 Enumeration mdformats = this.metadataFormats.elements();
647 if (mdformats != null) {
648 super.startVerbEvent(request);
649 while (mdformats.hasMoreElements()) {
650 BasicOAIMetadataFormat mdformat = (BasicOAIMetadataFormat) mdformats.nextElement();
651 if (mdformat != null)
652 mdformat.toSAX(this);
653 }
654 super.endVerbEvent(request);
655 }
656 } else {
657 Hits hits = executeIdQuery(request);
658 if (hits == null || hits.length() == 0) {
659 new OAIError(OAIError.ERROR_ID_DOES_NOT_EXIST, "The value of the identifier argument, " + id + ", is unknown or illegal in this repository.").toSAX(this);
660 return;//idDoesNotExist
661 } else if (hits.length() == 1) {
662 try {
663 ArrayList availableFormats = verifyMetadataFormatForRecord(hits.doc(0));
664 if (availableFormats == null || availableFormats.size() <= 0)
665 new OAIError(OAIError.ERROR_CANNOT_DISSEMINATE_FORMAT, "There are no metadata formats available for the specified item, " + id + " .").toSAX(this);
666 else {
667 super.startVerbEvent(request);
668 for (int i = 0; i < availableFormats.size(); i++) {
669 BasicOAIMetadataFormat mdFormat = (BasicOAIMetadataFormat) availableFormats.get(i);
670 if (mdFormat != null)
671 mdFormat.toSAX(this);
672 }
673 super.endVerbEvent(request);
674 }
675 } catch (IOException e) {
676 throw new SAXException(e.getMessage(), e);
677 }
678 }
679 }
680
681 }
682
683
684 }
685
686 /**Verifies that a document/record is available in the oai_dc format
687 *
688 * @param document The document verify
689 * @return an ArrayList of the available formats
690 */
691 protected ArrayList verifyMetadataFormatForRecord(Document document) {
692 return verifyMetadataFormatForRecord(document, null);
693 }
694
695 /**Verifies that a document/record is available in the specified format
696 *
697 * @param document The document verify
698 * @param mdPrefix The format to verify
699 * @return an ArrayList of the available formats
700 */
701 protected ArrayList verifyMetadataFormatForRecord(Document document, String mdPrefix) {
702 ArrayList availableFormats = null;
703
704 if (document != null) {
705 Enumeration formatsToCheck = null;
706 if (Utilities.checkString(mdPrefix)) {
707 BasicOAIMetadataFormat metadataFormat = (BasicOAIMetadataFormat) this.metadataFormats.get(mdPrefix);
708 Vector format = new Vector();
709 format.add(metadataFormat);
710 formatsToCheck = format.elements();
711 } else
712 formatsToCheck = metadataFormats.elements();
713
714
715 if (formatsToCheck != null) {
716 availableFormats = new ArrayList();
717 while (formatsToCheck.hasMoreElements()) {
718 BasicOAIMetadataFormat formatToCheck = (BasicOAIMetadataFormat) formatsToCheck.nextElement();
719 if (formatToCheck != null) {
720 if (Framework.SDXNamespacePrefix.equalsIgnoreCase(formatToCheck.getPrefix()))
721 availableFormats.add(formatToCheck);
722 else {
723 Pipeline pipe = formatToCheck.getPipeline();
724 Parameters mappings = formatToCheck.getMappings();
725 boolean metadataFormatHasMappings = true;
726 if (mappings == null || mappings.getNames() == null && mappings.getNames().length == 0)
727 metadataFormatHasMappings = false;//thie metadataFormat has no mappings so we will send the sdx fields to its pipeline
728 if (pipe != null) {
729 availableFormats.add(formatToCheck);
730 } else if (metadataFormatHasMappings) {
731 String[] sdxFieldNames = mappings.getNames();
732 if (sdxFieldNames != null) {
733 for (int i = 0; i < sdxFieldNames.length; i++) {
734 String sdxFieldName = sdxFieldNames[i];
735 if (Utilities.checkString(sdxFieldName) && !sdxFieldName.equals(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXDOCID)) {
736 if (document.getField(sdxFieldName) != null || document.getField(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDX_OAI_DELETED_RECORD) != null) {
737 availableFormats.add(formatToCheck);
738 }
739 }
740 }
741 }
742 }
743 }
744 }
745 }
746 }
747 }
748
749 return availableFormats;
750
751 }
752
753 /**Builds a search locations object
754 * using the underlying LuceneIndex
755 *
756 * @return
757 */
758 protected SearchLocations getSearchLocation() {
759 SearchLocations sLocs = new SearchLocations();
760 sLocs.enableLogging(this.logger);
761 sLocs.setUp(this.documentBase); // using new method
762 return sLocs;
763 }
764
765 /**Builds a basic boolean query
766 * adding the global inclusion
767 * and exclusion queries.
768 *
769 * @return
770 */
771 protected BooleanQuery getBaseQuery() {
772 BooleanQuery bq = Utilities.newBooleanQuery();
773
774 if (this.includeQuery != null)
775 bq.add(this.includeQuery.getLuceneQuery(), true, false);
776 if (this.excludeQuery != null)
777 bq.add(this.excludeQuery.getLuceneQuery(), false, true);
778 bq.add(new WildcardQuery(new Term(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDX_OAI_DELETED_RECORD, "*")), false, false);
779
780 return bq;
781 }
782
783 protected BooleanQuery getBaseQueryForResults() {
784 BooleanQuery bq = Utilities.newBooleanQuery();;
785 /*request for the oai deleted records*/
786 WildcardQuery oaiDeleted = new WildcardQuery(new Term(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDX_OAI_DELETED_RECORD, "*"));
787 /*request for all docs*/
788 org.apache.lucene.search.Query query = new TermQuery(new Term(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXALL, "1"));
789 /*if no include and no exclude, we only return deleted*/
790 if (this.includeQuery == null && this.excludeQuery == null){
791 bq.add (query, false, false);
792 bq.add (oaiDeleted, false, false);
793 }
794 else {
795 if (this.includeQuery != null && this.excludeQuery == null){
796 /*query between include and exclude but here no exclude*/
797 BooleanQuery bq2 = Utilities.newBooleanQuery();;
798 bq2.add(this.includeQuery.getLuceneQuery(), true, false);
799 /*query with include and exclude*/
800 bq.add(bq2,false,false);
801 /*query with deleted*/
802 bq.add(oaiDeleted, false, false);
803 }
804 else if (this.excludeQuery != null && this.includeQuery == null){
805 /*query between include and exclude but here no include*/
806 BooleanQuery bq2 = Utilities.newBooleanQuery();;
807 bq2.add(query, true, false);
808 bq2.add(this.excludeQuery.getLuceneQuery(), true, true);
809
810 bq.add(bq2,false,false);
811 /*query with deleted*/
812 bq.add(oaiDeleted, false, false);
813 }
814 else if (this.excludeQuery != null && this.includeQuery != null){
815 /*We want only include and except exclude*/
816 BooleanQuery bq2 = Utilities.newBooleanQuery();;
817 bq2.add(this.includeQuery.getLuceneQuery(), true, false);
818 bq2.add(this.excludeQuery.getLuceneQuery(), true, true);
819 bq.add(bq2,false,false);
820 /*query with deleted*/
821 bq.add(oaiDeleted, false, false);
822 }
823 }
824
825 return bq;
826 }
827
828
829 protected TermQuery getIdQuery(String id) {
830 TermQuery tq = null;
831 if (Utilities.checkString(id)) {
832 tq = new TermQuery(new Term(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXDOCID, id));
833 }
834 return tq;
835 }
836
837 /**Executes a query from an OAI request for the identifier param*/
838 protected Hits executeIdQuery(OAIRequest request) throws SAXException {
839 String id = request.getIdentifier();
840 String internalId = deriveInternalSdxId(request, id);
841 Hits hits = null;
842 if (Utilities.checkString(internalId)) {
843 try {
844 BooleanQuery bq = getBaseQueryForResults();
845 TermQuery idQuery = getIdQuery(internalId);
846 bq.add(idQuery, true, false);
847
848 if (this.luceneSearchIndex != null)
849 hits = luceneSearchIndex.search(bq);
850 } catch (SDXException e) {
851 throw new SAXException(e.getMessage(), e);
852 }
853
854 }
855 return hits;
856 }
857
858
859 //TODO: use DocumentBaseOAIRecord
860 protected void sendRecord(OAIRequest request, org.apache.lucene.document.Document doc, String mdPrefix) throws SAXException {
861 /*
862 <record>
863 <header>
864 <identifier>oai:arXiv.org:cs/0112017</identifier>
865 <datestamp>2001-12-14</datestamp>
866 <setSpec>cs</setSpec>
867 <setSpec>math</setSpec>
868 </header>
869 <metadata>
870 <oai_dc:dc
871 xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
872 xmlns:dc="http://purl.org/dc/elements/1.1/"
873 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
874 xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/
875 http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
876 <dc:title>Using Structural Metadata to Localize Experience of
877 Digital Content</dc:title>
878 <dc:creator>Dushay, Naomi</dc:creator>
879 <dc:subject>Digital Libraries</dc:subject>
880 <dc:description>With the increasing technical sophistication of
881 both information consumers and providers, there is
882 increasing demand for more meaningful experiences of digital
883 information. We present a framework that separates digital
884 object experience, or rendering, from digital object storage
885 and manipulation, so the rendering can be tailored to
886 particular communities of users.
887 </dc:description>
888 <dc:description>Comment: 23 pages including 2 appendices,
889 8 figures</dc:description>
890 <dc:date>2001-12-14</dc:date>
891 </oai_dc:dc>
892 </metadata>
893 </record>
894 */
895
896 BasicOAIMetadataFormat metadataFormat = (BasicOAIMetadataFormat) this.metadataFormats.get(mdPrefix);
897 String formatPrefix = metadataFormat.getPrefix();
898 boolean isSDXFormat = formatPrefix.equalsIgnoreCase(Framework.SDXNamespacePrefix);
899 if (doc != null) {
900 if (doc.getField(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXDOCID) == null || doc.getField(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDX_OAI_DELETED_RECORD) != null){
901 super.startElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.RECORD, OAIObject.Node.Name.RECORD, null);
902 sendHeaderEvents(request, doc);
903 super.endElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.RECORD, OAIObject.Node.Name.RECORD);
904 }
905 else if (metadataFormat != null) {
906 //start element 'record'
907 super.startElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.RECORD, OAIObject.Node.Name.RECORD, null);
908
909 sendHeaderEvents(request, doc);
910
911 Enumeration fields = doc.fields();
912 XMLPipe pipe = null;
913 boolean metadataFormatHasMappings = true;
914 if (fields != null) {
915 boolean startEventsSent = false;
916 while (fields.hasMoreElements()) {
917 if (metadataFormat.getMappings() == null || metadataFormat.getMappings().getNames() == null
918 && metadataFormat.getMappings().getNames().length == 0)
919 metadataFormatHasMappings = false;//thie metadataFormat has no mappings so we will send the sdx fields to its pipeline
920 pipe = metadataFormat.getPipeline();
921 if (pipe != null) {
922 Pipeline addRootAttPipe = metadataFormat.getAddRootAttributePipe();
923 if (addRootAttPipe != null)
924 addRootAttPipe.setConsumer(new IncludeXMLConsumer(this));
925 pipe.setConsumer(addRootAttPipe);
926 }
927 if (!startEventsSent) {
928 super.startElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.METADATA, OAIObject.Node.Name.METADATA, null);
929 metadataFormat.setConsumer(this);
930 if (metadataFormatHasMappings || isSDXFormat)//we don't have a pipe so each mapped element needs to be contained within a rootElement
931 metadataFormat.startMetadataFormatRootElement();
932 startEventsSent = true;
933 }
934 Field field = (Field) fields.nextElement();
935 if (field != null) {
936 String origName = field.name();
937 String val = field.stringValue();
938 if (Utilities.checkString(origName)) {
939 try {
940 String newVal = val;
941 //if there is a pipeline and no field mappings we get the document and send it to the pipe
942 if (origName.equals(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXDOCID)) {
943 if (pipe != null && !metadataFormatHasMappings) {
944 XMLDocument xdoc = new XMLDocument();
945 xdoc.enableLogging(this.logger);
946 xdoc.setId(val);
947 this.documentBase.getDocument(xdoc, pipe, false);
948 } else if (isSDXFormat)
949 metadataFormat.sendElement(origName, newVal);
950 else {//we have mappings and so we build and external representation TODO: is this a good idea in all cases?
951 newVal = buildUrlLocator(request, val);
952 metadataFormat.sendElement(origName, newVal);
953 }
954 } else {
955 //if anything was mapped we send the elements from the mappings
956 metadataFormat.sendElement(origName, newVal);
957 }
958 } catch (SDXException e) {
959 LoggingUtils.logException(logger, e);
960 }
961
962 }
963 }
964 }
965
966 if (startEventsSent) {
967 if (metadataFormatHasMappings || isSDXFormat)//we don't have a pipe so each mapped element needs to be contained within a rootElement
968 metadataFormat.endMetadataFormatRootElement();
969 super.endElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.METADATA, OAIObject.Node.Name.METADATA);
970 // if (pipe != null) pipe.setConsumer(null);//recycling the pipe
971 // metadataFormat.setConsumer(null);//recycling the metatdata format
972 }
973
974 }
975
976
977
978
979 //end element 'record'
980 super.endElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.RECORD, OAIObject.Node.Name.RECORD);
981 }
982 }
983
984
985 }
986
987
988 protected void sendIdentifier(OAIRequest request, org.apache.lucene.document.Document doc) throws SAXException {
989 /*
990 <record>
991 <header>
992 <identifier>oai:arXiv.org:cs/0112017</identifier>
993 <datestamp>2001-12-14</datestamp>
994 <setSpec>cs</setSpec>
995 <setSpec>math</setSpec>
996 </header>
997 <metadata>
998 <oai_dc:dc
999 xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
1000 xmlns:dc="http://purl.org/dc/elements/1.1/"
1001 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
1002 xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/
1003 http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
1004 <dc:title>Using Structural Metadata to Localize Experience of
1005 Digital Content</dc:title>
1006 <dc:creator>Dushay, Naomi</dc:creator>
1007 <dc:subject>Digital Libraries</dc:subject>
1008 <dc:description>With the increasing technical sophistication of
1009 both information consumers and providers, there is
1010 increasing demand for more meaningful experiences of digital
1011 information. We present a framework that separates digital
1012 object experience, or rendering, from digital object storage
1013 and manipulation, so the rendering can be tailored to
1014 particular communities of users.
1015 </dc:description>
1016 <dc:description>Comment: 23 pages including 2 appendices,
1017 8 figures</dc:description>
1018 <dc:date>2001-12-14</dc:date>
1019 </oai_dc:dc>
1020 </metadata>
1021 </record>
1022 */
1023
1024
1025
1026 if (doc != null)
1027 sendHeaderEvents(request, doc);
1028
1029 }
1030
1031
1032 /**Sends the OAI-PMH header events for the provided lucene document
1033 *
1034 * @param request The request
1035 * @param doc The lucene document
1036 * @throws SAXException
1037 */
1038 //TODO: use DocumentBaseOAIRecord
1039 protected void sendHeaderEvents(OAIRequest request, Document doc) throws SAXException {
1040
1041 String id = doc.get(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXDOCID);
1042 String externalId = "";
1043 String status = null;
1044 if (!Utilities.checkString(id)) {
1045 String deletedId = doc.get(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDX_OAI_DELETED_RECORD);
1046 if (Utilities.checkString(deletedId)) {
1047 status = OAIObject.Node.Value.DELETED;
1048 id = deletedId;
1049 }
1050 }
1051 externalId = buildExternalOaiId(request, id);
1052 String datestamp = doc.get(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXMODDATE);
1053
1054
1055 AttributesImpl atts = new AttributesImpl();
1056 if (Utilities.checkString(status))
1057 atts.addAttribute("", OAIObject.Node.Name.STATUS, OAIObject.Node.Name.STATUS, OAIObject.Node.Type.CDATA, status);
1058
1059 super.startElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.HEADER, OAIObject.Node.Name.HEADER, atts);
1060
1061 datestamp = fr.gouv.culture.sdx.utils.Date.formatUtcISO8601Date(DateField.stringToDate(datestamp));
1062
1063 super.sendElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.IDENTIFIER, OAIObject.Node.Name.IDENTIFIER, null, externalId);
1064 super.sendElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.DATESTAMP, OAIObject.Node.Name.DATESTAMP, null, datestamp);
1065 if (this.setMappings != null) {
1066 Enumeration sets = this.setMappings.keys();
1067 if (sets != null) {
1068 while (sets.hasMoreElements()) {
1069 Parameters setParams = getDocumentSetParameters(doc);
1070 if (setParams != null) {
1071 String setSpec = setParams.getParameter(PARAMETER_NAME_SET_SPEC, "");
1072 if (Utilities.checkString(setSpec)) {
1073 String encoding = Utilities.getStringFromContext(ContextKeys.SDX.ENCODING, getContext());
1074 setSpec = Utilities.encodeURL(setSpec, encoding);
1075 super.sendElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.SET_SPEC, OAIObject.Node.Name.SET_SPEC, null, setSpec);
1076 }
1077 }
1078 }
1079 }
1080 }
1081
1082 super.endElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.HEADER, OAIObject.Node.Name.HEADER);
1083
1084 }
1085
1086
1087 /**Adds a deleted record lucene document
1088 * TO the search index for the specific id
1089 *
1090 * @param id The id of the deleted record
1091 * @throws SDXException
1092 */
1093 public void addDeletedRecord(String id) throws SDXException {
1094 if (!Utilities.checkString(id)) return;
1095 String docId = id;
1096 org.apache.lucene.document.Document deletedRecord = new org.apache.lucene.document.Document();
1097 Hits hits = null;
1098 BooleanQuery bq = getBaseQuery();
1099 bq.add(getIdQuery(docId), true, false);
1100 hits = this.luceneSearchIndex.search(bq);
1101
1102 try {
1103 if (hits != null && hits.length() == 1 && docId.equals(hits.doc(0).get(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXDOCID))) {
1104 deletedRecord.add(Field.Keyword(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDX_OAI_DELETED_RECORD, id));
1105 Date date = fr.gouv.culture.sdx.utils.Date.getUtcIso8601Date();
1106 deletedRecord.add(Field.Keyword(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXMODDATE, DateField.dateToString(date)));
1107 luceneSearchIndex.writeDocument(deletedRecord, false);//passing false here as a calling method should be responsible for any index optimization.
1108 }
1109 } catch (IOException e) {
1110 LoggingUtils.logException(logger, e);//TODOException, can we do anything better here
1111 }
1112
1113 }
1114
1115 /**Removes a deleted record entry from lucene document
1116 * FROM the search index for the specific id
1117 *
1118 * @param id The id of the deleted record
1119 * @throws SDXException
1120 */
1121 public void removeDeletedRecord(String id) throws SDXException {
1122 if (!Utilities.checkString(id)) return;
1123 Term term = new Term(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDX_OAI_DELETED_RECORD, id);
1124 luceneSearchIndex.delete(term);
1125 }
1126
1127 /**Returns a boolean query for a specific set for a specific set
1128 *
1129 * @param setSpec The setSpec to use
1130 */
1131 protected BooleanQuery addSetQuery(String setSpec) {
1132 BooleanQuery bq = Utilities.newBooleanQuery();
1133 if (this.setMappings != null && Utilities.checkString(setSpec)) {
1134 Parameters setParams = (Parameters) this.setMappings.get(setSpec);
1135 if (setParams != null) {
1136 String sdxFieldName = setParams.getParameter(PARAMETER_NAME_SDX_FIELD, "");
1137 String includeQuery = setParams.getParameter(PARAMETER_NAME_INCLUDE_QUERY, "");
1138 String excludeQuery = setParams.getParameter(PARAMETER_NAME_EXCLUDE_QUERY, "");
1139 if (Utilities.checkString(includeQuery) || Utilities.checkString(excludeQuery)) {
1140 try {
1141 org.apache.lucene.search.Query inQuery = getLuceneSimpleQuery(includeQuery);
1142 if (inQuery != null)
1143 bq.add(inQuery, true, false);
1144 } catch (SDXException e) {
1145 //this exception should never be thrown, as configuration will fail before if the query is unparsable
1146 LoggingUtils.logException(logger, e);
1147 }
1148
1149 try {
1150 org.apache.lucene.search.Query exQuery = getLuceneSimpleQuery(excludeQuery);
1151 if (exQuery != null)
1152 bq.add(exQuery, false, true);
1153 } catch (SDXException e) {
1154 //this exception should never be thrown, as configuration will fail before if the query is unparsable
1155 LoggingUtils.logException(logger, e);
1156 }
1157
1158 } else if (Utilities.checkString(sdxFieldName))
1159 bq.add(new WildcardQuery(new Term(sdxFieldName, setSpec)), true, false);
1160 }
1161 }
1162 return bq;
1163 }
1164
1165 /**Builds an sdx simple query for the provided query string
1166 * and returns the underlying lucene query object
1167 *
1168 * @param query The query string
1169 * @return
1170 * @throws SDXException
1171 */
1172 protected org.apache.lucene.search.Query getLuceneSimpleQuery(String query) throws SDXException {
1173 org.apache.lucene.search.Query ret = null;
1174 if (Utilities.checkString(query)) {
1175 SearchLocations sLocs = this.getSearchLocation();
1176 SimpleQuery sQuery = new SimpleQuery();
1177 sQuery.enableLogging(this.logger);
1178 sQuery.setUp(sLocs, null, query);
1179 ret = sQuery.getLuceneQuery();
1180 }
1181 return ret;
1182 }
1183
1184
1185 /**Returns a parameters object containing
1186 * set information for a lucene document
1187 *
1188 * @param doc The lucene document
1189 * @return
1190 */
1191 protected Parameters getDocumentSetParameters(Document doc) {
1192 Parameters ret = null;
1193 String docId = doc.get(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXDOCID);
1194 Enumeration setSpecs = this.setMappings.keys();
1195 if (setSpecs != null) {
1196 while (setSpecs.hasMoreElements()) {
1197 String setSpec = (String) setSpecs.nextElement();
1198 Parameters setParams = (Parameters) this.setMappings.get(setSpec);
1199 String sdxFieldName = setParams.getParameter(PARAMETER_NAME_SDX_FIELD, "");
1200 if (Utilities.checkString(sdxFieldName)) {
1201 Field field = doc.getField(sdxFieldName);
1202 if (field != null) {
1203 String fieldValue = field.stringValue();
1204 //checking if the setSpec indicates a field name or its value
1205 if (Utilities.checkString(setSpec) && setSpec.equals(sdxFieldName) || setSpec.equals(fieldValue)) {
1206 ret = setParams;
1207 break;
1208 }
1209 }
1210 }
1211 // if the above doesn't work then we will evaluate the inlcude/exclude queries
1212 BooleanQuery bq = getBaseQueryForResults();
1213 String includeQuery = setParams.getParameter(PARAMETER_NAME_INCLUDE_QUERY, "");
1214 String excludeQuery = setParams.getParameter(PARAMETER_NAME_EXCLUDE_QUERY, "");
1215 if (Utilities.checkString(includeQuery) || Utilities.checkString(excludeQuery)) {
1216 try {
1217 org.apache.lucene.search.Query inQuery = getLuceneSimpleQuery(includeQuery);
1218 if (inQuery != null)
1219 bq.add(inQuery, true, false);
1220 } catch (SDXException e) {
1221 //this exception should never be thrown, as configuration will fail before if the query is unparsable
1222 LoggingUtils.logException(logger, e);
1223 }
1224 try {
1225 org.apache.lucene.search.Query exQuery = getLuceneSimpleQuery(excludeQuery);
1226 if (exQuery != null)
1227 bq.add(exQuery, false, true);
1228 } catch (SDXException e) {
1229 //this exception should never be thrown, as configuration will fail before if the query is unparsable
1230 LoggingUtils.logException(logger, e);
1231 }
1232 //now execute the query and then compare the docids
1233 if (this.luceneSearchIndex != null) {
1234 try {
1235 Hits hits = luceneSearchIndex.search(bq);
1236 if (hits != null && hits.length() > 0) {
1237 for (int i = 0; i < hits.length(); i++) {
1238 Document hitDoc = hits.doc(i);
1239 String id = hitDoc.get(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXDOCID);
1240 if (Utilities.checkString(id) && Utilities.checkString(docId) && id.equals(docId)) {
1241 ret = setParams;
1242 break;
1243 }
1244 }
1245 }
1246 } catch (SDXException e) {
1247 LoggingUtils.logException(logger, e);
1248 } catch (IOException e) {
1249 LoggingUtils.logException(logger, e);
1250 }
1251 }
1252
1253
1254 }
1255
1256
1257 }
1258
1259
1260 }
1261
1262
1263 return ret;
1264 }
1265
1266
1267 /** Executes a query for a request and sorts them in
1268 * ascending date order
1269 *
1270 * @param request The request from which to build the query to execute
1271 */
1272 protected Hits executeQueryForRequestParams(OAIRequest request) throws SDXException, SAXException {
1273 String set = "";
1274 String resumptionToken = "";
1275 String from = "";
1276 String until = "";
1277 if (request != null) {
1278 set = request.getSetIdentifier();
1279 resumptionToken = request.getResumptionToken();
1280 from = request.getFrom();
1281 until = request.getUntil();
1282 }
1283
1284 BooleanQuery bq = getBaseQueryForResults();
1285
1286 org.apache.lucene.search.Filter rangeFilter = null;
1287 java.util.Date fromDate = null;
1288 java.util.Date untilDate = null;
1289
1290 //Validation of set hierarchy
1291 if (Utilities.checkString(set) && (this.setMappings == null || this.setMappings.size() == 0)) {
1292 super.sendNoSetHierarchyError();
1293 return null;
1294 }
1295
1296 /*Start Date validation*/
1297 //from param
1298 if (fromDate == null)//we check for null, as we may be using a date from the last record in a resumptionToken set
1299 fromDate = fr.gouv.culture.sdx.utils.Date.parseUtcISO8601Date(from);
1300 if (fromDate == null && Utilities.checkString(from)) {
1301 //Bad date
1302 new OAIError(OAIError.ERROR_BAD_DATE, "The value of the \"from\" argument, " + from + ", is invalid.").toSAX(this);
1303 return null;
1304 }
1305
1306 untilDate = fr.gouv.culture.sdx.utils.Date.parseUtcISO8601Date(until);
1307 if (untilDate == null && Utilities.checkString(until)) {
1308 new OAIError(OAIError.ERROR_BAD_DATE, "The value of the \"until\" argument, " + until + ", is invalid.").toSAX(this);
1309 return null;//Bad date
1310 }
1311 /*End Date validation*/
1312 if (fromDate != null || untilDate != null) {
1313
1314 if (fromDate != null && untilDate != null) {
1315 /*see OAI 2.0 spec 2.7.1 Selective Harvesting and Datestamps :
1316 * Therefore, the from arugment must be less than or equal to the until argument.
1317 *Otherwise, a repository must issue a badArgument error .
1318 */
1319 if (fromDate.getTime() > untilDate.getTime()) {
1320 new OAIError(OAIError.ERROR_BAD_ARGUMENT, "The value of the \"from\" argument, " + from + ", must be less than or equal to the value of the \"until\" argument," + until + " .").toSAX(this);
1321 return null;//Bad dates
1322 }
1323 }
1324
1325 DateIntervalQuery dateQuery = new DateIntervalQuery();
1326 dateQuery.enableLogging(this.logger);
1327 dateQuery.setUpOai(getSearchLocation(), LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXMODDATE, fromDate, untilDate);
1328 rangeFilter = dateQuery.prepare();
1329 }
1330
1331
1332 bq.add(addSetQuery(set), false, false);//adding any set information to the query
1333
1334 Hits hits = null;
1335 if (this.luceneSearchIndex != null)
1336 hits = luceneSearchIndex.search(bq, rangeFilter);
1337
1338 return hits;
1339
1340 }
1341
1342
1343 /**Returns the earliest _datestamp for this repository
1344 *
1345 * @return
1346 */
1347 public String getEarliestDatestamp() {
1348 if (!Utilities.checkString(super.earliestDatestamp)) {
1349 Date utcDate = fr.gouv.culture.sdx.utils.Date.getUtcIso8601Date(this.documentBase.creationDate());
1350 super.earliestDatestamp = fr.gouv.culture.sdx.utils.Date.formatUtcISO8601Date(utcDate);
1351 }
1352 return super.earliestDatestamp;
1353 }
1354
1355 /**Destroys all deleted record entries
1356 *
1357 */
1358 public void purgeDeletedRecords() {
1359 try {
1360 LuceneIndex luceneSearchIndex = this.luceneSearchIndex;
1361 luceneSearchIndex.delete(new Term(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDX_OAI_DELETED_RECORD, "*"));
1362 luceneSearchIndex.optimize();
1363 } catch (SDXException e) {
1364 LoggingUtils.logException(logger, e);
1365 }
1366
1367 }
1368
1369
1370 }
1371
1372

savannah-hackers-public@gnu.org
ViewVC Help
Powered by ViewVC 1.1.26