/[sdx]/sdx_v2/src/java/fr/gouv/culture/sdx/oai/AbstractDocumentBaseOAIHarvester.java
ViewVC logotype

Annotation of /sdx_v2/src/java/fr/gouv/culture/sdx/oai/AbstractDocumentBaseOAIHarvester.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.24.2.16 - (hide annotations) (download)
Wed Apr 20 08:28:14 2005 UTC (19 years, 1 month ago) by malo_pichot
Branch: V_22
CVS Tags: V_222_FINAL
Changes since 1.24.2.15: +12 -22 lines
DEBUG: we did not consider sdxoaidate as an date field and so can not build date queries on

1 rpandey 1.1 /*
2     SDX: Documentary System in XML.
3     Copyright (C) 2000, 2001, 2002 Ministere de la culture et de la communication (France), AJLSM
4    
5     Ministere de la culture et de la communication,
6     Mission de la recherche et de la technologie
7     3 rue de Valois, 75042 Paris Cedex 01 (France)
8     mrt@culture.fr, michel.bottin@culture.fr
9    
10     AJLSM, 17, rue Vital Carles, 33000 Bordeaux (France)
11     sevigny@ajlsm.com
12    
13     This program is free software; you can redistribute it and/or
14     modify it under the terms of the GNU General Public License
15     as published by the Free Software Foundation; either version 2
16     of the License, or (at your option) any later version.
17    
18     This program is distributed in the hope that it will be useful,
19     but WITHOUT ANY WARRANTY; without even the implied warranty of
20     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
21     See the GNU General Public License for more details.
22    
23     You should have received a copy of the GNU General Public License
24     along with this program; if not, write to the
25     Free Software Foundation, Inc.
26     59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
27     or connect to:
28     http://www.fsf.org/copyleft/gpl.html
29     */
30     package fr.gouv.culture.sdx.oai;
31    
32 malo_pichot 1.24.2.3 import java.io.File;
33     import java.io.FileOutputStream;
34     import java.io.IOException;
35     import java.net.MalformedURLException;
36     import java.net.URL;
37     import java.util.ArrayList;
38     import java.util.Date;
39     import java.util.Enumeration;
40     import java.util.Hashtable;
41 malo_pichot 1.24.2.6 import java.util.Properties;
42 malo_pichot 1.24.2.3
43 malo_pichot 1.24.2.4 import org.apache.avalon.excalibur.io.FileUtil;
44 malo_pichot 1.24.2.3 import org.apache.avalon.excalibur.xml.Parser;
45     import org.apache.avalon.framework.component.ComponentException;
46     import org.apache.avalon.framework.configuration.Configuration;
47     import org.apache.avalon.framework.configuration.ConfigurationException;
48     import org.apache.cocoon.Constants;
49     import org.apache.cocoon.ProcessingException;
50     import org.apache.cocoon.serialization.XMLSerializer;
51     import org.apache.cocoon.xml.IncludeXMLConsumer;
52     import org.apache.cocoon.xml.XMLConsumer;
53     import org.apache.cocoon.xml.XMLMulticaster;
54     import org.apache.cocoon.xml.XMLPipe;
55     import org.xml.sax.Attributes;
56     import org.xml.sax.SAXException;
57     import org.xml.sax.helpers.AttributesImpl;
58    
59 rpandey 1.3 import fr.gouv.culture.oai.AbstractOAIHarvester;
60     import fr.gouv.culture.oai.OAIObject;
61     import fr.gouv.culture.oai.OAIRequest;
62     import fr.gouv.culture.oai.OAIRequestImpl;
63 malo_pichot 1.24.2.6 import fr.gouv.culture.oai.util.OAIUtilities;
64 rpandey 1.1 import fr.gouv.culture.sdx.application.Application;
65     import fr.gouv.culture.sdx.document.IndexableDocument;
66 malo_pichot 1.24.2.3 import fr.gouv.culture.sdx.document.OAIDocument;
67 rpandey 1.1 import fr.gouv.culture.sdx.document.XMLDocument;
68 malo_pichot 1.24.2.3 import fr.gouv.culture.sdx.documentbase.AbstractDocumentBase;
69     import fr.gouv.culture.sdx.documentbase.DocumentBase;
70     import fr.gouv.culture.sdx.documentbase.IDGenerator;
71     import fr.gouv.culture.sdx.documentbase.IndexParameters;
72 rpandey 1.1 import fr.gouv.culture.sdx.exception.SDXException;
73     import fr.gouv.culture.sdx.exception.SDXExceptionCode;
74     import fr.gouv.culture.sdx.framework.Framework;
75 rpandey 1.10 import fr.gouv.culture.sdx.framework.FrameworkImpl;
76 rpandey 1.1 import fr.gouv.culture.sdx.pipeline.GenericPipeline;
77     import fr.gouv.culture.sdx.pipeline.Pipeline;
78     import fr.gouv.culture.sdx.repository.Repository;
79 malo_pichot 1.24.2.3 import fr.gouv.culture.sdx.search.lucene.query.ComplexQuery;
80 malo_pichot 1.24.2.15 import fr.gouv.culture.sdx.search.lucene.query.DateIntervalQuery;
81 malo_pichot 1.24.2.3 import fr.gouv.culture.sdx.search.lucene.query.FieldQuery;
82     import fr.gouv.culture.sdx.search.lucene.query.SearchLocations;
83 rpandey 1.21 import fr.gouv.culture.sdx.utils.ConfigurationUtilities;
84 rpandey 1.1 import fr.gouv.culture.sdx.utils.Utilities;
85 rpandey 1.19 import fr.gouv.culture.sdx.utils.database.Database;
86     import fr.gouv.culture.sdx.utils.database.DatabaseBacked;
87 rpandey 1.22 import fr.gouv.culture.sdx.utils.database.DatabaseEntity;
88     import fr.gouv.culture.sdx.utils.database.Property;
89 malo_pichot 1.24.2.3 import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.SimpleTimeScheduler;
90     import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.Target;
91     import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeScheduler;
92     import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeTrigger;
93     import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeTriggerFactory;
94 rpandey 1.10 import fr.gouv.culture.util.apache.cocoon.xml.SynchronizedXMLConsumerWrapper;
95 rpandey 1.1
96     /**
97     * Created by IntelliJ IDEA.
98     * User: rpandey
99     * Date: May 12, 2003
100     * Time: 12:32:16 PM
101     */
102     public abstract class AbstractDocumentBaseOAIHarvester extends AbstractOAIHarvester implements Target {
103    
104     protected String ELEMENT_NAME_PIPELINE = "pipeline";
105    
106 rpandey 1.23 /**The underlying document base*/
107     protected DocumentBase docbase = null;
108     /**Id of the underlying document base*/
109     protected String docbaseId = "";
110     /**Properties from the document base*/
111     protected Hashtable docBaseProps = null;
112    
113     /**Pre-indexation pipeline*/
114 rpandey 1.1 protected Pipeline pipe = null;
115 rpandey 1.23 /**Underlying database to store any info*/
116 rpandey 1.19 protected Database database = null;
117 rpandey 1.23 /**Requests in application.xconf*/
118 rpandey 1.1 protected Hashtable storedRequests = null;
119 rpandey 1.23 /**References to the underlying documentbase's/application's repositories*/
120 rpandey 1.1 protected Hashtable storeRepositoriesRefs = null;
121 rpandey 1.23 /**Time scheduler for stored requests*/
122 rpandey 1.1 protected TimeScheduler scheduler = null;
123 rpandey 1.23 /**IDGenerator for this object*/
124     protected IDGenerator harvesterIdGen = null;
125    
126     //variables for sax stream handling
127     protected String TEMPFILE_SUFFIX = ".sdx";
128 rpandey 1.10 protected File tempDir = null;
129     protected File harvestDoc = null;
130     protected FileOutputStream fileOs = null;
131 rpandey 1.6 protected XMLDocument urlResource = null;
132 rpandey 1.1 protected ArrayList deletedDocs = null;
133 rpandey 1.19 protected ArrayList harvestedDocs = null;
134 malo_pichot 1.24.2.10 protected int noDocsHarvested = 0;
135     protected int noDocsDeleted = 0;
136 rpandey 1.19
137 rpandey 1.23 protected boolean keepDeletedRecords = false;//Defaulted, we will delete "deletedRecords" keeping our harvester in sync with source repository
138     protected int noRecordsPerBatch = OAIObject.NUMBER_RECORDS_PER_RESPONSE;
139    
140     //configuration node names
141     /**Configuration node name*/
142     protected static final String ELEMENT_NAME_OAI_DATA_PROVIDERS = "oai-data-providers";
143     /**Configuration node name*/
144     protected static final String ELEMENT_NAME_OAI_VERB = "oai-verb";
145     /**Configuration node name*/
146     protected static final String ELEMENT_NAME_OAI_IDENTIFIER = "oai-identifier";
147     /**Configuration node name*/
148     protected static final String ATTRIBUTE_NAME_NAME = "name";
149     /**Configuration node name*/
150     protected static final String ATTRIBUTE_NAME_ADMIN_EMAIL = OAIObject.Node.Name.ADMIN_EMAIL;
151     /**Configuration node name*/
152     protected static final String ATTRIBUTE_NAME_USER_AGENT = "userAgent";
153     /**Configuration node name*/
154     protected static final String ATTRIBUTE_NAME_URL = "url";
155     /**Configuration node name*/
156     protected static final String ATTRIBUTE_NAME_UPDATE = "update";
157     /**Configuration node name*/
158     protected static final String ATTRIBUTE_NAME_METADATA_PREFIX = OAIObject.Node.Name.METADATA_PREFIX;
159     /**Configuration node name*/
160     protected static final String ATTRIBUTE_NAME_SDX_REPOSITORY = "sdxRepository";
161     /**Configuration node name*/
162     protected static final String ATTRIBUTE_NAME_FROM = "from";
163     /**Configuration node name*/
164 malo_pichot 1.24.2.10 protected static final String ATTRIBUTE_USE_LAST_HARVEST_DATE = "useLastHarvestDate";
165     /**Configuration node name*/
166 rpandey 1.23 protected static final String ATTRIBUTE_NAME_UNTIL = "until";
167     /**Configuration node name*/
168     protected static final String ATTRIBUTE_NAME_SET = "set";
169     protected static final String ATTRIBUTE_NAME_KEEP_DELETED_RECORD = "keepDeletedRecords";
170     /**Configuration node name*/
171     protected static final String ATTRIBUTE_NO_RECORDS_PER_BATCH = "noRecordsPerBatch";
172    
173    
174     //strings for internal database properties and sax output
175 rpandey 1.22 protected static final String OAI_HARVEST_ID = "oaiHarvestId";
176 rpandey 1.24 //internal database property for failed harvests caused by internal errors( errors in this implementation and not OAI repository errors)
177     protected static final String OAI_FAILED_HARVEST = "oaiFailedHarvest";
178 rpandey 1.22 protected static final String OAI_HARVESTER_LAST_UPDATED = "oaiHarvesterLastUpdated";
179     protected static final String OAI_HARVESTER_RESUMPTION_TOKEN = "oaiHarvesterResumptionToken";
180     /*Field names for common request parameters*/
181     protected static final String OAI_VERB = "oaiVerb";
182     protected static final String OAI_IDENTIFIER = "oaiIdentifier";
183     protected static final String OAI_METADATA_PREFIX = "oaiMetadataPrefix";
184     protected static final String OAI_FROM = "oaiFrom";
185     protected static final String OAI_UNTIL = "oaiUntil";
186     protected static final String OAI_SET = "oaiSet";
187     protected static final String NO_DOCS_DELETED = "noDocDeleted";
188     protected static final String NO_DOCS_HARVESTED = "noDocHarvested";
189 rpandey 1.1
190 malo_pichot 1.24.2.6 /**List OAI files with OAI properties*/
191     protected Hashtable filesProperties = null;
192    
193 rpandey 1.1
194 rpandey 1.23 /**Basic constructor*/
195 rpandey 1.1 public AbstractDocumentBaseOAIHarvester(DocumentBase base) {
196 rpandey 1.23 this.docbase = base;
197     if (this.docbase != null)
198     this.docbaseId = this.docbase.getId();
199 rpandey 1.1 }
200    
201 rpandey 1.23 /**Set's the properties for this object*/
202 rpandey 1.1 public void setProperties(Hashtable props) {
203 uid67066 1.18 this.docBaseProps = props;
204 rpandey 1.1 }
205    
206    
207 rpandey 1.23 /**Configures this object*/
208 rpandey 1.1 public void configure(Configuration configuration) throws ConfigurationException {
209    
210 uid67066 1.18 super.userAgent = configuration.getAttribute(ATTRIBUTE_NAME_USER_AGENT, "SDX OAI Harvester");
211 rpandey 1.20 this.keepDeletedRecords = configuration.getAttributeAsBoolean(ATTRIBUTE_NAME_KEEP_DELETED_RECORD, this.keepDeletedRecords);
212 rpandey 1.23 this.noRecordsPerBatch = configuration.getAttributeAsInteger(ATTRIBUTE_NO_RECORDS_PER_BATCH, this.noRecordsPerBatch);
213 uid67066 1.18 configureAdminEmails(configuration);
214 rpandey 1.1 this.configureDataProviders(configuration);
215     this.configurePipeline(configuration);
216 rpandey 1.19 configureDatabase(configuration);
217 rpandey 1.21 configureHarvestIDGenerator(configuration);
218 rpandey 1.19 }
219    
220 rpandey 1.21
221 rpandey 1.23 /**Configures the internal database*/
222 rpandey 1.19 protected void configureDatabase(Configuration configuration) throws ConfigurationException {
223     DatabaseBacked internalDb = new DatabaseBacked();
224     try {
225     internalDb.setId(getHarvesterId());
226 rpandey 1.24.2.1 internalDb.compose(this.manager);
227 rpandey 1.19 internalDb.setProperties(this.docBaseProps);
228     internalDb.configure(configuration);
229     internalDb.init();
230     this.database = internalDb.getDatabase();
231     } catch (SDXException e) {
232 rpandey 1.24.2.1 throw new ConfigurationException(e.getMessage(), e);
233     } catch (ComponentException e) {
234 rpandey 1.19 throw new ConfigurationException(e.getMessage(), e);
235     }
236    
237     }
238    
239 rpandey 1.23 /**Configures the id generator for harvests*/
240 rpandey 1.21 protected void configureHarvestIDGenerator(Configuration configuration) throws ConfigurationException {
241     this.harvesterIdGen = ConfigurationUtilities.configureIDGenerator(this.logger, configuration);
242     this.harvesterIdGen.setDatabase(this.database);
243     }
244    
245    
246 rpandey 1.23 /**Returns an id for this harvester based upon the underlying document base id*/
247 rpandey 1.19 protected String getHarvesterId() {
248 rpandey 1.23 String hid = "";
249     hid += Framework.SDXNamespacePrefix + "_";
250     hid += OAIObject.Node.Prefix.OAI + "_";
251     hid += "harvester" + "_";
252     hid += this.docbaseId;
253     return hid;
254 uid67066 1.18 }
255    
256 rpandey 1.23 /**Configures a list of admin emails
257     * can be sub-elements, a single attribute,
258     * or both
259     *
260     * @param configuration
261     * @throws ConfigurationException
262     */
263 uid67066 1.18 protected void configureAdminEmails(Configuration configuration) throws ConfigurationException {
264     //configure the admin email
265     ArrayList locAdminEmailsList = new ArrayList();
266     String firstAdminEmail = configuration.getAttribute(ATTRIBUTE_NAME_ADMIN_EMAIL, null);
267     Configuration[] locAdminEmails = configuration.getChildren(ATTRIBUTE_NAME_ADMIN_EMAIL);
268     if (Utilities.checkString(firstAdminEmail))
269     locAdminEmailsList.add(firstAdminEmail);
270     for (int i = 0; i < locAdminEmails.length; i++) {
271     Configuration locAdminEmail = locAdminEmails[i];
272     if (locAdminEmail != null) {
273     String value = locAdminEmail.getValue();
274     if (Utilities.checkString(value))
275     locAdminEmailsList.add(value);
276     }
277     }
278     if (locAdminEmailsList.size() <= 0)//no admin email throw an error TODO:make this exception better
279     Utilities.checkConfAttributeValue(ATTRIBUTE_NAME_ADMIN_EMAIL, null, configuration.getLocation());
280    
281     super.adminEmails = (String[]) locAdminEmailsList.toArray(new String[0]);
282     //releasing resources
283     locAdminEmailsList.clear();
284     locAdminEmailsList = null;
285    
286 rpandey 1.1 }
287    
288 rpandey 1.23 /**Configures data providers info that can be reused
289     * and from which requests can be automatically executed
290     *
291     * @param configuration
292     * @throws ConfigurationException
293     * @see #storedRequests
294     */
295 rpandey 1.1 protected void configureDataProviders(Configuration configuration) throws ConfigurationException {
296     if (configuration != null) {
297 rpandey 1.3 Configuration dataProvidersConf = configuration.getChild(ELEMENT_NAME_OAI_DATA_PROVIDERS, false);
298 rpandey 1.1 if (dataProvidersConf != null) {
299 rpandey 1.3 Configuration[] repoRequestConfs = dataProvidersConf.getChildren(AbstractDocumentBase.ELEMENT_NAME_OAI_REPOSITORY);
300 rpandey 1.1 if (repoRequestConfs != null) {
301 rpandey 1.21 for (int x = 0; x < repoRequestConfs.length; x++) {
302     Configuration repoRequestConf = repoRequestConfs[x];
303 rpandey 1.1 if (repoRequestConf != null) {
304     if (storedRequests == null) storedRequests = new Hashtable();
305 rpandey 1.3 String repoUrl = repoRequestConf.getAttribute(ATTRIBUTE_NAME_URL);
306 rpandey 1.1 configureStoreRepositories(repoUrl, repoRequestConf);
307 rpandey 1.3 Configuration updateConf = repoRequestConf.getChild(ATTRIBUTE_NAME_UPDATE, false);
308     Configuration[] verbConfs = repoRequestConf.getChildren(ELEMENT_NAME_OAI_VERB);
309 rpandey 1.1 if (verbConfs != null) {
310 rpandey 1.21 for (int y = 0; y < verbConfs.length; y++) {
311 rpandey 1.1 OAIRequest request = null;
312 rpandey 1.21 Configuration verbConf = verbConfs[y];
313 rpandey 1.1 if (verbConf != null) {
314 rpandey 1.3 String verb = verbConf.getAttribute(ATTRIBUTE_NAME_NAME);
315     String mdPrefix = verbConf.getAttribute(ATTRIBUTE_NAME_METADATA_PREFIX);
316     Utilities.checkConfAttributeValue(ATTRIBUTE_NAME_METADATA_PREFIX, mdPrefix, verbConf.getLocation());
317 malo_pichot 1.24.2.10 String verbId = verbConf.getAttribute("id");
318     Utilities.checkConfAttributeValue("id", verbId, verbConf.getLocation());
319 rpandey 1.1 if (verb.equalsIgnoreCase(OAIRequest.VERB_STRING_GET_RECORD)) {
320     verb = OAIRequest.VERB_STRING_GET_RECORD;
321 rpandey 1.3 Configuration[] idsConf = verbConf.getChildren(ELEMENT_NAME_OAI_IDENTIFIER);
322 rpandey 1.1 if (idsConf != null) {
323 rpandey 1.21 for (int z = 0; z < idsConf.length; z++) {
324     Configuration idConf = idsConf[z];
325 rpandey 1.1 if (idConf != null) {
326     String id = idConf.getValue();
327     request = new OAIRequestImpl();
328     request.enableLogging(this.logger);
329     request.setRepositoryURL(repoUrl);
330     request.setVerbString(verb);
331     request.setMetadataPrefix(mdPrefix);
332 malo_pichot 1.24.2.10 request.setVerbId(verbId);
333 rpandey 1.1 request.setIdentifier(id);
334     storedRequests.put(request.getRequestURL(), request);
335     configureUpdateTriggers(request.getRequestURL(), updateConf);
336     }
337    
338     }
339     }
340    
341     } else if (verb.equalsIgnoreCase(OAIRequest.VERB_STRING_LIST_RECORDS)) {
342     verb = OAIRequest.VERB_STRING_LIST_RECORDS;
343 rpandey 1.3 String from = verbConf.getAttribute(ATTRIBUTE_NAME_FROM, null);
344 malo_pichot 1.24.2.16 boolean useLastHarvestDate = verbConf.getAttributeAsBoolean(ATTRIBUTE_USE_LAST_HARVEST_DATE, /*true by default*/true);
345 rpandey 1.3 String until = verbConf.getAttribute(ATTRIBUTE_NAME_UNTIL, null);
346     String set = verbConf.getAttribute(ATTRIBUTE_NAME_SET, null);
347 rpandey 1.1 request = new OAIRequestImpl();
348     request.enableLogging(this.logger);
349     request.setRepositoryURL(repoUrl);
350     request.setVerbString(verb);
351     request.setMetadataPrefix(mdPrefix);
352 malo_pichot 1.24.2.10 request.setVerbId(verbId);
353 rpandey 1.1 request.setFrom(from);
354 malo_pichot 1.24.2.10 request.setUseLastHarvestDate(useLastHarvestDate);
355 rpandey 1.1 request.setUntil(until);
356     request.setSetIdentifier(set);
357     this.storedRequests.put(request.getRequestURL(), request);
358     configureUpdateTriggers(request.getRequestURL(), updateConf);
359     } else//TODOException:
360     throw new ConfigurationException("this verb action is not supported for harvesting : " + verb);
361     }
362    
363    
364     }
365     }
366    
367 malo_pichot 1.24.2.3 // now we can configure indexation pipeline for these repository
368     this.configurePipeline(repoRequestConf);
369 rpandey 1.1
370 malo_pichot 1.24.2.3 }
371 rpandey 1.1
372     }
373     if (this.scheduler != null) this.scheduler.start();
374     }
375    
376     }
377    
378     }
379     }
380    
381 rpandey 1.23 /**Configures time triggers for
382     * stored requests
383     *
384     * @param requestUrl The request url
385     * @param updateConf The configuration for updates
386     * @throws ConfigurationException
387     * @see #scheduler, #storedRequests
388     */
389 rpandey 1.1 protected void configureUpdateTriggers(String requestUrl, Configuration updateConf) throws ConfigurationException {
390     if (Utilities.checkString(requestUrl) && updateConf != null) {
391     TimeTrigger trigger = new TimeTriggerFactory().createTimeTrigger(updateConf);
392     if (trigger != null) {
393     if (this.scheduler == null) this.scheduler = new SimpleTimeScheduler();
394     this.scheduler.addTrigger(requestUrl, trigger, this);
395     }
396     }
397     }
398    
399 rpandey 1.23
400     /**Configures the repositories
401     * to which data will be stored
402     * based upon their repository url
403     *
404     * @param repoUrl The repository/data provider url
405     * @param oaiRepoConf The configuration
406     * @throws ConfigurationException
407     */
408 rpandey 1.1 protected void configureStoreRepositories(String repoUrl, Configuration oaiRepoConf) throws ConfigurationException {
409     if (Utilities.checkString(repoUrl)) {
410 rpandey 1.3 String ref = oaiRepoConf.getAttribute(ATTRIBUTE_NAME_SDX_REPOSITORY, null);
411 malo_pichot 1.24.2.3 /*check for the sdxrepository attribute, if it exists, get the repository object and add it to the local hashtable*/
412     if (Utilities.checkString(ref)) {
413 rpandey 1.1 Repository repo = null;
414 uid67066 1.18 Hashtable appRepos = (Hashtable) this.docBaseProps.get(Application.APPLICATION_REPOSITORIES);
415 rpandey 1.1 if (appRepos != null)
416     repo = (Repository) appRepos.get(ref);
417     if (repo == null)
418     try {
419 rpandey 1.23 repo = this.docbase.getRepository(ref);
420 rpandey 1.1 } catch (SDXException e) {
421     String[] args = new String[1];
422     args[0] = ref;
423     SDXException sdxE = new SDXException(logger, SDXExceptionCode.ERROR_LOAD_REFERENCED_REPO, args, null);
424     throw new ConfigurationException(sdxE.getMessage(), sdxE);
425     }
426     if (this.storeRepositoriesRefs == null) this.storeRepositoriesRefs = new Hashtable();
427     //populating the hashtable
428     storeRepositoriesRefs.put(repoUrl, repo);
429     }
430     }
431    
432    
433     }
434    
435    
436 rpandey 1.23 /**Configures the preIndexation pipeline
437     *
438     * @param configuration
439     * @throws ConfigurationException
440     * @see #pipe
441     */
442 rpandey 1.1 protected void configurePipeline(Configuration configuration) throws ConfigurationException {
443     //at this point, we should have a <sdx:pipeline> element
444     Configuration pipeConf = configuration.getChild(ELEMENT_NAME_PIPELINE, false);
445     //testing if we have something
446     if (pipeConf != null) {
447     //creating the pipeline and assigning the class field
448     this.pipe = new GenericPipeline();
449     //setting the logger for the pipeline object
450     this.pipe.enableLogging(this.logger);
451     //giving the object the cocoon component manager
452     try {
453     this.pipe.compose(this.manager);
454     } catch (ComponentException e) {
455     throw new ConfigurationException(e.getMessage(), e);
456     }
457    
458 malo_pichot 1.24.2.3 //pass OAI base properties to the pipe
459     this.pipe.setProperties(docBaseProps);
460    
461 rpandey 1.1 //configuring the pipeline object from 'application.xconf'
462     this.pipe.configure(pipeConf);
463     }
464     }
465    
466 rpandey 1.23 /**Creates a new temporary directory for
467     * writing harvested records before the will
468     * be indexed
469     * @return
470     * @throws SDXException
471     * @throws IOException
472     */
473     protected File getNewTempDir() throws SDXException, IOException {
474     File ret = (File) this.docBaseProps.get(Constants.CONTEXT_UPLOAD_DIR);
475 malo_pichot 1.24.2.3 String childDir = Utilities.getStringFromHashtable(FrameworkImpl.APP_PATH_NAME, this.docBaseProps) + "_oaiHarvests" + File.separator + this.docbaseId + File.separator + "harvest-" + Utilities.encodeURL(Long.toString(new Date().getTime()), null);
476 rpandey 1.23 if (Utilities.checkString(super.resumptionToken))
477     childDir += "-" + OAIObject.Node.Name.RESUMPTION_TOKEN + "-" + super.resumptionToken;
478     if (ret.canWrite())
479     ret = new File(ret, childDir);
480     else
481     ret = new File(Utilities.getSystemTempDir(), childDir);
482     Utilities.checkDirectory(ret.getCanonicalPath(), logger);
483     return ret;
484     }
485    
486     /**Deletes the directory
487     * represented by the tempDir
488     * class field
489     *
490     */
491     protected void deleteTempDir() {
492     if (tempDir != null) {
493     try {
494     FileUtil.deleteDirectory(this.tempDir);//deleting the old tempdir if possible
495     } catch (IOException e) {
496     Utilities.logWarn(logger, e.getMessage(), e);
497     }
498     }
499     }
500    
501     /**Establishes the tempDir class field
502     *
503     * @throws SDXException
504     * @throws IOException
505     */
506     protected void initTempDir() throws SDXException, IOException {
507     this.tempDir = getNewTempDir();
508     }
509    
510     /**Get's the current date in iso8601 format
511     *
512     * @return
513     */
514     protected String getIsoDate() {
515     return fr.gouv.culture.sdx.utils.Date.formatDate(fr.gouv.culture.sdx.utils.Date.getUtcIso8601Date());
516     }
517    
518 rpandey 1.1
519 rpandey 1.23 /**Sets up resources to capture an oai record
520     *
521     *
522     * @throws SAXException
523     */
524 rpandey 1.12 protected void prepareRecordCapture() throws SAXException {
525 rpandey 1.22 try {
526     if (this.tempDir == null || !this.tempDir.exists()) initTempDir();
527 rpandey 1.23 this.harvestDoc = File.createTempFile("oaiHarvestedRecord", TEMPFILE_SUFFIX, tempDir);
528 rpandey 1.22 this.fileOs = new FileOutputStream(this.harvestDoc);
529 malo_pichot 1.24.2.6
530     //Initialise OAI properties for the record
531     if (filesProperties == null)
532     filesProperties = new Hashtable();
533     Properties props = new Properties();
534     filesProperties.put(this.harvestDoc.getName(), props);//we can work with file name, automatically generated and unique
535 malo_pichot 1.24.2.7 this.currentDatestamp = this.currentMetadtaUrlIdentifier = this.currentOaiIdentifier = this.currentOaiStatus = null;
536 malo_pichot 1.24.2.15 this.deleteRecord=false;
537 rpandey 1.22 XMLSerializer cBytes = new XMLSerializer();
538     cBytes.setOutputStream(this.fileOs);
539 malo_pichot 1.24.2.3 // We must strip the OAI envelop
540     XMLPipe oaiStripper = new OAIEnvelopStripper();
541     oaiStripper.setConsumer(cBytes);
542 rpandey 1.22 XMLConsumer newConsumer = null;
543     if (firstXmlConsumer != null)
544 malo_pichot 1.24.2.3 newConsumer = new XMLMulticaster(oaiStripper, firstXmlConsumer);
545 rpandey 1.22 else
546 malo_pichot 1.24.2.3 newConsumer = oaiStripper;
547 rpandey 1.23
548     super.setSynchronizedConsumer(new SynchronizedXMLConsumerWrapper(newConsumer));
549 malo_pichot 1.24.2.3
550 rpandey 1.22 } catch (IOException e) {
551     throw new SAXException(e.getMessage(), e);
552     } catch (SDXException e) {
553     throw new SAXException(e.getMessage(), e);
554 rpandey 1.10 }
555 rpandey 1.22
556 rpandey 1.10 }
557 rpandey 1.16
558 malo_pichot 1.24.2.6 /**Ends the capture of an oai record.
559     * Store properties (dateStamp and OAI identifier) needed for indexation.
560 rpandey 1.23 *
561     * @throws Exception
562     */
563 rpandey 1.12 protected void captureRecord() throws Exception {
564 rpandey 1.22 if (this.fileOs != null && this.harvestDoc != null) {
565 malo_pichot 1.24.2.6
566 rpandey 1.10 this.fileOs.flush();
567     this.fileOs.close();
568 malo_pichot 1.24.2.14 if (OAIObject.Node.Value.DELETED.equalsIgnoreCase(currentOaiStatus))
569     resetRecordCaptureFields(true);
570     else{
571     /*Setting OAI properties (dateStamp, OAI identifier, OAI status) for the current record.
572     Those informations will be used for indexation*/
573     Properties oaiProps = (Properties)filesProperties.get(this.harvestDoc.getName());//TODO: do we need to prevent the case where properties are null ? -mp
574     if (this.currentDatestamp != null)
575     oaiProps.setProperty("datestamp", this.currentDatestamp);
576     else oaiProps.setProperty("datestamp", "");
577     if (this.currentOaiIdentifier != null)
578     oaiProps.setProperty("oaiid", this.currentOaiIdentifier);
579     else oaiProps.setProperty("oaiid", "");
580     if (this.currentOaiStatus != null)
581     oaiProps.setProperty("oaistatus", this.currentOaiStatus);
582     else oaiProps.setProperty("oaistatus", "");
583     resetRecordCaptureFields(false);
584     }
585 rpandey 1.22 }
586     }
587    
588 rpandey 1.23 /**Resets the class fields for record capture
589     * possibility deleting the current <code>harvetDoc</code>
590     * object underlying file
591     *
592     * @param deleteDoc flag for deletion of actual file
593     */
594 rpandey 1.22 protected void resetRecordCaptureFields(boolean deleteDoc) {
595     if (this.fileOs != null) {
596     try {
597     this.fileOs.flush();
598     this.fileOs.close();
599     this.fileOs = null;
600     } catch (IOException e) {
601     Utilities.logException(logger, e);
602     }
603     }
604    
605     if (this.harvestDoc != null) {
606     if (deleteDoc)
607     this.harvestDoc.delete();
608 rpandey 1.13 this.harvestDoc = null;
609 rpandey 1.1 }
610 rpandey 1.22
611     super.setConsumer(super.firstXmlConsumer);//resetting the consumer to the first external consumer
612 rpandey 1.1 }
613    
614 rpandey 1.23
615     /**Sets up resources to delete an oai record
616 malo_pichot 1.24.2.3 *Add the record to the list of the records to removed
617 rpandey 1.23 */
618     protected void prepareRecordForDeletion() {
619     if (this.deletedDocs == null) this.deletedDocs = new ArrayList();
620     if (Utilities.checkString(this.currentOaiIdentifier)) {
621     XMLDocument deleteDoc = null;
622     try {
623 malo_pichot 1.24.2.3 deleteDoc = new XMLDocument(this.currentOaiIdentifier);//constructs an XMLDocument. We should pass the sdxdocid, not the sdxoaiid
624 rpandey 1.23 } catch (SDXException e) {
625     //do nothing here
626     }
627     if (deleteDoc != null) this.deletedDocs.add(deleteDoc);
628     }
629     }
630    
631    
632     /**Reads the documents from <code>tempDir</code>
633     * and indexes them in the corresponding document
634     * base, any marked deletions will be carried out
635     * as well
636     *
637     * @return
638 malo_pichot 1.24.2.3 * @throws ProcessingException
639 rpandey 1.23 * @throws SDXException
640     * @throws SAXException
641 malo_pichot 1.24.2.3 * @throws IOException
642 malo_pichot 1.24.2.6 * @see fr.gouv.culture.oai.AbstractOAIHarvester#storeHarvestedData()
643 rpandey 1.23 */
644 malo_pichot 1.24.2.3 protected boolean storeHarvestedData() throws ProcessingException, IOException, SDXException, SAXException {
645 rpandey 1.1 boolean dataHarvested = false;
646 rpandey 1.23 if (docbase != null) {
647 malo_pichot 1.24.2.15
648 rpandey 1.11 //deleting any docs which have been removed from the repo
649 malo_pichot 1.24.2.15 if (this.deletedDocs != null && this.deletedDocs.size() > 0 )
650 malo_pichot 1.24.2.3 deleteOAIDocuments();
651 malo_pichot 1.24.2.15
652 rpandey 1.1 IndexableDocument[] indexDocs = null;
653 rpandey 1.11 //creating our docs from the disk
654 rpandey 1.10 if (this.tempDir != null) {
655     String[] files = this.tempDir.list();
656 rpandey 1.19 this.harvestedDocs = new ArrayList();
657 rpandey 1.21 ArrayList docs = new ArrayList();
658     IndexParameters indexParams = getIndexParameters();
659 rpandey 1.10 for (int i = 0; i < files.length; i++) {
660 malo_pichot 1.24.2.15
661 rpandey 1.10 String fileName = files[i];
662 malo_pichot 1.24.2.15
663 malo_pichot 1.24.2.6 //Recovers OAI properties (dateStamp, OAI identifier) for the current OAI record
664     Properties oaiProps = (Properties)filesProperties.get(fileName);
665     this.currentDatestamp = oaiProps.getProperty("datestamp");
666     this.currentOaiIdentifier = oaiProps.getProperty("oaiid");
667     this.currentOaiStatus = oaiProps.getProperty("oaistatus");
668 malo_pichot 1.24.2.15
669 malo_pichot 1.24.2.6 /*We don't want to indexe document that must be deleted*/
670     if (this.currentOaiStatus != null && !OAIObject.Node.Value.DELETED.equalsIgnoreCase(this.currentOaiStatus)) {
671 malo_pichot 1.24.2.15
672 malo_pichot 1.24.2.6 //Constructs OAIDocument wich will be indexed
673     OAIDocument metadataDoc = new OAIDocument();
674     metadataDoc.setDateString(this.currentDatestamp);
675     metadataDoc.setIdentifier(this.currentOaiIdentifier);
676 malo_pichot 1.24.2.15
677 malo_pichot 1.24.2.6 try {
678     metadataDoc.setContent(new File(tempDir, fileName).toURL());
679     if (docs == null) docs = new ArrayList();
680     docs.add(metadataDoc);
681     this.harvestedDocs.add(metadataDoc);//keeping track of all additions
682     } catch (MalformedURLException e) {
683     Utilities.logException(logger, e);
684     }
685 malo_pichot 1.24.2.15 }
686    
687     //at every 1000th document we will index into the base and/or at the end of the group
688     int modulusTopLimit = indexParams.getBatchMax() - 1;
689     if ( i == (files.length - 1) || ( i > 0 && (i % modulusTopLimit) == 0 ) ) {
690     indexDocs = (IndexableDocument[]) docs.toArray(new IndexableDocument[0]);
691     if (indexDocs != null) {
692     Repository repo = null;
693     if (Utilities.checkString(this.repoUrl) && storeRepositoriesRefs != null)
694     repo = (Repository) this.storeRepositoriesRefs.get(this.repoUrl);
695     this.docbase.index(indexDocs, repo, indexParams, /*this.contentHandler*/this);
696     dataHarvested = true;
697     docs = null;
698     indexDocs = null;
699 rpandey 1.11 }
700     }
701 rpandey 1.10 }
702 malo_pichot 1.24.2.15
703 malo_pichot 1.24.2.10 this.noDocsHarvested = this.noDocsHarvested + this.harvestedDocs.size();
704 rpandey 1.10 }
705 malo_pichot 1.24.2.15
706    
707 rpandey 1.1 }
708     return dataHarvested;
709     }
710    
711 malo_pichot 1.24.2.3 /**Delete OAI documents from the current document base.
712     *
713     * @throws IOException
714     * @throws ProcessingException
715     * @throws SDXException
716     * @throws SAXException
717     */
718     protected void deleteOAIDocuments() throws IOException, ProcessingException, SDXException, SAXException {
719    
720     if (this.deletedDocs != null && this.deletedDocs.size() > 0){
721    
722     //Construct the SearchLocations for the query
723     SearchLocations slocs = new SearchLocations();
724     slocs.enableLogging(this.logger);
725     slocs.addIndex(this.docbase.getIndex());
726    
727     ComplexQuery cq = new ComplexQuery();
728     cq.enableLogging(this.logger);
729     cq.setUp(slocs, fr.gouv.culture.sdx.search.lucene.query.Query.OPERATOR_OR);
730    
731 malo_pichot 1.24.2.11 //boucle sur les documents a supprimer
732 malo_pichot 1.24.2.3 for( int i = 0; i < this.deletedDocs.size(); i++ ){
733     fr.gouv.culture.sdx.document.XMLDocument doc = (XMLDocument) this.deletedDocs.get(i);
734     String sdxoaiid = doc.getId();
735     FieldQuery fq = new FieldQuery();
736     fq.setUp(slocs, /*sdxoaiid*/sdxoaiid, /*fieldname: sdxoaiid*/OAIDocument.INTERNAL_FIELD_NAME_SDXOAIID);
737     cq.addComponent(fq);
738     }
739    
740     cq.prepare();//prepare the query
741    
742     fr.gouv.culture.sdx.search.lucene.query.Results res = cq.execute();//execute the query
743    
744     //we have results
745 malo_pichot 1.24.2.15 if ( res != null && res.count() > 0 ) {
746 malo_pichot 1.24.2.3 String[] ids = res.getDocIds();
747     XMLDocument[] docs = new XMLDocument[ids.length];
748     for ( int j = 0; j < docs.length ; j++ ){
749     docs[j] = new XMLDocument(ids[j]);
750     }
751     //delete the documents
752     this.docbase.delete(docs, null);
753 malo_pichot 1.24.2.10 noDocsDeleted = noDocsDeleted + docs.length;
754 malo_pichot 1.24.2.3 }
755     }
756    
757     }
758    
759 rpandey 1.23 /**Handles the resumption token by issuing another request
760     * based upon the request from which the resumption token was received.
761     */
762 rpandey 1.1 protected void handleResumptionToken() {
763     if (Utilities.checkString(super.resumptionToken) && Utilities.checkString(super.repoUrl)) {
764 malo_pichot 1.24.2.12 String verb = "";
765     if (this.requestParams!=null)
766     verb = this.requestParams.getParameter(OAIObject.Node.Name.VERB, "");
767     else if (this.storedRequests.size() > 0){
768     verb = ( (OAIRequest)this.storedRequests.get(super.requestUrl) ).getVerbString();
769     }
770    
771     try{
772     if (Utilities.checkString(verb)) {
773     newRequestUrl = this.repoUrl + OAIRequest.URL_CHARACTER_QUESTION_MARK + OAIObject.Node.Name.VERB + OAIRequest.URL_CHARACTER_EQUALS + verb + OAIRequest.URL_CHARACTER_AMPERSAND + OAIObject.Node.Name.RESUMPTION_TOKEN + OAIRequest.URL_CHARACTER_EQUALS + super.resumptionToken;
774     this.resetAllFields();
775     try {
776     this.initTempDir();//trying to build a tempDir with the resumptionToken in the dir name
777     } catch (SDXException e) {
778     Utilities.logException(logger, e);//if this fails we log it as there will be another opportunity during the harvest
779     } catch (IOException e) {
780     Utilities.logException(logger, e);
781     }
782     }
783     } catch (java.lang.NullPointerException npe){
784     Utilities.logError(logger, "Request parameters have been reset or expired..", npe);
785     } finally{
786     super.resetResumptionToken();//resetting the resumption token here so we don't loop
787     }
788 rpandey 1.1 }
789     }
790    
791 rpandey 1.23 //TODO-TEST: this should be preparing for a merge with the metadata, done
792     /**Prepares to read a url value from an oai record and
793     * retrieve the XML behind.
794     *@see #identifierName, #currentMetadtaUrlIdentifier
795     */
796     protected void prepareResourceFromUrlIdentifierCapture() {
797 rpandey 1.1 if (Utilities.checkString(this.currentMetadtaUrlIdentifier)) {
798 rpandey 1.23 try {
799     URL resourceUrl = new URL(this.currentMetadtaUrlIdentifier);
800     XMLDocument resource = new XMLDocument();
801     resource.setId(this.currentMetadtaUrlIdentifier);
802     resource.setContent(resourceUrl);
803     this.urlResource = resource;
804     } catch (MalformedURLException e) {
805     Utilities.logException(logger, e);
806     } catch (SDXException e) {
807     Utilities.logException(logger, e);
808     }
809 rpandey 1.6 /*
810 rpandey 1.1 if (docs == null) docs = new ArrayList();
811     docs.add(resource);
812 rpandey 1.6 */
813    
814     /*InputStream contents = resource.openStream();
815     byte[] chars = new byte[contents.available()];
816     contents.read(chars, 0, chars.length);
817     contents.close();
818     docBytes.
819     docBytes.write(chars);*/
820    
821     }
822     }
823    
824 rpandey 1.23 /**Captures the xml from a url taken from an oai record and adds
825     * it to the oai-record as a sibling of the <metadata/> element
826     *
827     */
828     protected void captureResourceFromUrlIdentifier() {
829 rpandey 1.6 if (this.urlResource != null) {
830     try {
831 rpandey 1.23 IncludeXMLConsumer include = new IncludeXMLConsumer(super.synchronizedXmlConsumer);
832     urlResource.setConsumer(include);
833     include.startElement(Framework.SDXNamespaceURI, "urlResource", "sdx:urlResource", new AttributesImpl());
834     Parser parser = null;
835     try {
836     parser = (Parser) this.manager.lookup(Parser.ROLE);
837     urlResource.parse(parser);
838     } finally {
839     if (parser != null)
840     this.manager.release(parser);
841     this.urlResource = null;
842     include.endElement(Framework.SDXNamespaceURI, "urlResource", "sdx:urlResource");
843     }
844     } catch (SAXException e) {
845     Utilities.logException(logger, e);
846     } catch (ComponentException e) {
847     Utilities.logException(logger, e);
848     } catch (SDXException e) {
849     Utilities.logException(logger, e);
850 rpandey 1.6 }
851 rpandey 1.1 }
852 rpandey 1.6
853 rpandey 1.1 }
854    
855    
856 rpandey 1.23 /**Resets necessary class fields
857     *
858     */
859 rpandey 1.1 protected void resetAllFields() {
860 malo_pichot 1.24.2.13 noDocsHarvested = noDocsDeleted = 0;
861     deletedDocs = harvestedDocs = null;
862 rpandey 1.6 this.urlResource = null;
863 rpandey 1.22 resetRecordCaptureFields(false);
864 rpandey 1.10 deleteTempDir();
865 rpandey 1.14 this.tempDir = null;
866 malo_pichot 1.24.2.12 super.resetAllFields();
867 rpandey 1.1 }
868    
869 rpandey 1.23
870     /**Builds simple index parameters for indexation of
871     * oai records into the undelryi
872     * @return
873     */
874 rpandey 1.1 protected IndexParameters getIndexParameters() {
875     IndexParameters params = new IndexParameters();
876     params.setSendIndexationEvents(IndexParameters.SEND_ALL_EVENTS);
877 rpandey 1.23 params.setBatchMax(this.noRecordsPerBatch);
878 malo_pichot 1.24.2.3 //if (this.pipe != null && this.docbase != null) {
879     // this.pipe.setConsumer(this.docbase.getIndexationPipeline());
880     // params.setPipeline(this.pipe);
881     //}
882     if ( this.pipe != null ) params.setPipeline(this.pipe);
883     else if ( this.docbase != null ) params.setPipeline(this.docbase.getIndexationPipeline());
884 rpandey 1.17 params.setPipelineParams(super.getHarvestParameters());
885 rpandey 1.1 return params;
886     }
887    
888 rpandey 1.23 /**Sends the details of stored harvesting requests
889     * to the current consumer
890     *
891     * @throws SAXException
892     */
893 rpandey 1.6 public void sendStoredHarvestingRequests() throws SAXException {
894 rpandey 1.4 try {
895 rpandey 1.6 if (this.storedRequests != null && this.storedRequests.size() > 0) {
896 rpandey 1.4 this.acquire();
897     super.acquireSynchronizedXMLConsumer();
898     super.startElement(Framework.SDXNamespaceURI,
899     fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS,
900     Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS,
901     null);
902     Enumeration requests = storedRequests.elements();
903     if (requests != null) {
904     while (requests.hasMoreElements()) {
905     OAIRequest request = (OAIRequest) requests.nextElement();
906     if (request != null)
907     request.toSAX(this);
908     }
909 rpandey 1.1 }
910 rpandey 1.4
911     super.endElement(Framework.SDXNamespaceURI,
912     fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS, Framework.SDXNamespacePrefix + ":" +
913     fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS);
914 rpandey 1.1 }
915 rpandey 1.4 } catch (InterruptedException e) {
916     throw new SAXException(e.getMessage(), e);
917     } finally {
918     super.releaseSynchronizedXMLConsumer();
919     this.release();
920 rpandey 1.1 }
921    
922     }
923    
924    
925 rpandey 1.23 /**Triggers a oai request to a repository based
926     * upon a trigger name (also a request url)
927     *
928     * @param triggerName
929     */
930 rpandey 1.1 public synchronized void targetTriggered(String triggerName) {
931 rpandey 1.3 //DEBUG System.out.println("i was triggered :" + triggerName);
932 malo_pichot 1.24.2.11
933 rpandey 1.23 //verify that we have this trigger defined
934 rpandey 1.1 OAIRequest request = (OAIRequest) this.storedRequests.get(triggerName);
935 malo_pichot 1.24.2.11
936 rpandey 1.1 if (request != null) {
937 malo_pichot 1.24.2.11
938     // Test if we have to force the use of the from parameter (in application.xconf) instead of the last haverst date
939 malo_pichot 1.24.2.10 boolean useLastHarvestDate = request.getUseLastHarvestDate();
940     if ( useLastHarvestDate ){
941     try {
942     DatabaseEntity dbe = this.database.getEntity(request.getVerbId());
943     if ( dbe != null ){
944     String lastHarvestDate = dbe.getProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED);
945     request.setFrom(lastHarvestDate);
946     }
947     } catch (SDXException e) {
948     Utilities.logException(logger, e);
949     }
950     }
951 malo_pichot 1.24.2.11
952 rpandey 1.1 String requestUrl = request.getRequestURL();
953 malo_pichot 1.24.2.11
954 malo_pichot 1.24.2.10 this.receiveSynchronizedRequest(requestUrl, triggerName);
955 malo_pichot 1.24.2.11
956 rpandey 1.3 //DEBUG System.out.println("i was requested :" + requestUrl);
957 rpandey 1.1 }
958     }
959    
960 rpandey 1.3
961     public void startElement(String s, String s1, String s2, Attributes attributes) throws SAXException {
962     if (Utilities.checkString(this.identifierName)) {
963     /*looking for an element name or the VALUE of an attribute with the name "name" matching super.indentifierName
964     *so one could create a metadata identifier element like
965     *<super.identifierName>myIdentifierValue</super.identifierName>
966     *or
967     *<anyElementName name="super.identifierName">myIdentifierValue</anyElementName>
968     */
969 rpandey 1.4 if (attributes != null && this.identifierName.equals(attributes.getValue(fr.gouv.culture.sdx.utils.constants.Node.Name.NAME)))
970 rpandey 1.3 this.captureElemContent = true;
971     }
972     super.startElement(s, s1, s2, attributes);
973     }
974    
975     public void endElement(String s, String s1, String s2) throws SAXException {
976     if (super.sBuff != null && super.sBuff.length() > 0) {
977     String content = super.sBuff.toString();
978    
979     if (!OAIObject.Node.Xmlns.OAI_2_0.equals(s)) {
980     if (fr.gouv.culture.sdx.utils.constants.Node.Name.FIELD.equals(s1)) {
981     super.currentMetadtaUrlIdentifier = content;
982     try {
983 rpandey 1.12 prepareResourceFromUrlIdentifierCapture();
984 rpandey 1.3 } catch (Exception e) {
985     //if we can't build the document we don't just fail completely we will continue
986 rpandey 1.9 Utilities.logException(logger, e);
987 rpandey 1.3 }
988     }
989     }
990    
991     }
992     super.endElement(s, s1, s2);
993 rpandey 1.22 }
994    
995 rpandey 1.23 /**Querys the underlying data structures
996     * based upon current sax flow
997     * position/set class fields and
998     * determines whether an oai record should be
999     * harvested
1000     *
1001     * @return boolean indicates whether the record should be handled
1002     */
1003 rpandey 1.22 protected boolean shouldHarvestDocument() {
1004     boolean ret = true;
1005     try {
1006 malo_pichot 1.24.2.14
1007 malo_pichot 1.24.2.16 //First control: do we have a previous harvest for this provider/verb ? If so, look for the datestamp of the current oai doc vs. the time of the last harvest
1008 malo_pichot 1.24.2.14 OAIRequest request = (OAIRequest) this.storedRequests.get(super.requestUrl);
1009 malo_pichot 1.24.2.16 if (request != null && request.getVerbId()!=null && !request.getVerbId().equals("") && this.database.entityExists(request.getVerbId())){
1010     DatabaseEntity dbe = this.database.getEntity(request.getVerbId());
1011     String formattedDate = dbe.getProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED);
1012     java.util.Date lastUpdatedDate = fr.gouv.culture.sdx.utils.Date.parseDate(formattedDate);
1013     java.util.Date currentDocumentDate = fr.gouv.culture.sdx.utils.Date.parseDate(super.currentDatestamp);
1014     if (currentDocumentDate.getTime() > lastUpdatedDate.getTime())
1015     return ret;//the timestamp of the current doc is more recent than the last harvest, we should harvest it
1016 malo_pichot 1.24.2.14 }
1017    
1018 malo_pichot 1.24.2.16 //Second control: search for an sdxdocument which correspond to the current oai id
1019     else if (Utilities.checkString(super.currentOaiIdentifier)) {
1020 malo_pichot 1.24.2.14
1021     //The complexQuery
1022     ComplexQuery cq = new ComplexQuery();
1023     SearchLocations slocs = new SearchLocations();
1024     slocs.enableLogging(this.logger);
1025 malo_pichot 1.24.2.15 slocs.addIndex(this.docbase.getIndex());
1026 malo_pichot 1.24.2.14 cq.enableLogging(this.logger);
1027     cq.setUp(slocs,fr.gouv.culture.sdx.search.lucene.query.Query.OPERATOR_AND);
1028     //The fieldQuery
1029     FieldQuery fq = new FieldQuery();
1030     fq.setUp(slocs, /*sdxoaiid*/super.currentOaiIdentifier, /*fieldname: sdxoaiid*/OAIDocument.INTERNAL_FIELD_NAME_SDXOAIID);
1031     cq.addComponent(fq);
1032     //The dateQuery
1033     DateIntervalQuery dq = new DateIntervalQuery();
1034     java.util.Date currentOaiDocumentDate= fr.gouv.culture.sdx.utils.Date.parseDate(super.currentDatestamp);
1035 malo_pichot 1.24.2.16 dq.setUp(slocs, /*sdxoaidate*/OAIDocument.INTERNAL_FIELD_NAME_SDXOAIDATE, /*from*/currentOaiDocumentDate, /*to*/null, /*bounds inclusive*/true);
1036 malo_pichot 1.24.2.14 cq.addComponent(dq);
1037    
1038     cq.prepare();
1039     fr.gouv.culture.sdx.search.lucene.query.Results res = cq.execute();
1040    
1041     //we have result(s)
1042     if (res != null && res.count() == 1 ){//1 result cool !
1043     //String docid = res.getDocIds()[0];//we need to compare the timestamp of this sdxdoc and the current oaidoc
1044     ret = false;
1045     }
1046     else if(res != null && res.count() > 1) {//More than 1 result: error
1047     OAIUtilities.logError(logger, "A problem occured during harvesting: "+res.count()+" documents correspond to the OAI identifier: "+super.currentOaiIdentifier, null);
1048     }
1049     else return ret;//no document with this id exists, so we should harvest it
1050     }
1051     } catch (SDXException e) {
1052 malo_pichot 1.24.2.15 Utilities.logException(logger, e);
1053 malo_pichot 1.24.2.14 }
1054    
1055     return ret;
1056 rpandey 1.22 }
1057    
1058 rpandey 1.23 /**Saves critical data about a harvest
1059     *
1060     * @param dataHarvested
1061     * @throws SAXException
1062     */
1063 rpandey 1.22 protected void saveCriticalFields(boolean dataHarvested) throws SAXException {
1064 malo_pichot 1.24.2.11
1065 malo_pichot 1.24.2.10 try {
1066 malo_pichot 1.24.2.11
1067 malo_pichot 1.24.2.10 OAIRequest request = (OAIRequest) this.storedRequests.get(super.requestUrl);
1068 malo_pichot 1.24.2.11
1069     if (request != null && request.getVerbId() != null && !request.getVerbId().equals("")) {
1070    
1071 malo_pichot 1.24.2.10 DatabaseEntity dbe = this.database.getEntity(request.getVerbId());
1072 malo_pichot 1.24.2.11 if (dbe == null) dbe = new DatabaseEntity(request.getVerbId());
1073    
1074 malo_pichot 1.24.2.10 //Now we have to clear the database
1075     Property[] props = dbe.getProperties();
1076     if (props.length > 0){
1077     for(int i=0 ; i < props.length; i++){
1078     Property prop = props[i];
1079     dbe.deleteProperty(props[i].getName());
1080     }
1081     }
1082 malo_pichot 1.24.2.11
1083 malo_pichot 1.24.2.10 if (Utilities.checkString(super.repoUrl))
1084     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_REQUEST_URL, super.requestUrl);
1085 malo_pichot 1.24.2.11
1086 malo_pichot 1.24.2.10 if (Utilities.checkString(super.repoUrl))
1087     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL, super.repoUrl);
1088 malo_pichot 1.24.2.11
1089 malo_pichot 1.24.2.10 if (requestParams != null) {
1090 malo_pichot 1.24.2.11
1091 malo_pichot 1.24.2.10 String verb = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, null);
1092     if (Utilities.checkString(verb)) {
1093 malo_pichot 1.24.2.11
1094 malo_pichot 1.24.2.10 String mdPrefix = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_METADATA_PREFIX, null);
1095 malo_pichot 1.24.2.11
1096 malo_pichot 1.24.2.10 if (verb.equals(OAIRequest.VERB_STRING_GET_RECORD)) {
1097 malo_pichot 1.24.2.11
1098 malo_pichot 1.24.2.10 dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb);
1099 malo_pichot 1.24.2.11
1100 malo_pichot 1.24.2.10 if (Utilities.checkString(mdPrefix))
1101     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_METADATA_PREFIX, mdPrefix);
1102 malo_pichot 1.24.2.11
1103 malo_pichot 1.24.2.10 String id = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_IDENTIFIER, null);
1104     if (Utilities.checkString(id))
1105     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_IDENTIFIER, id);
1106     }
1107 malo_pichot 1.24.2.11
1108 malo_pichot 1.24.2.10 if (verb.equals(OAIRequest.VERB_STRING_LIST_RECORDS)) {
1109 malo_pichot 1.24.2.11
1110 malo_pichot 1.24.2.10 dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb);
1111 malo_pichot 1.24.2.11
1112 malo_pichot 1.24.2.10 if (Utilities.checkString(mdPrefix))
1113     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_METADATA_PREFIX, mdPrefix);
1114 malo_pichot 1.24.2.11
1115 malo_pichot 1.24.2.10 String from = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_FROM, null);
1116     if (Utilities.checkString(from))
1117     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_FROM, from);
1118 malo_pichot 1.24.2.11
1119 malo_pichot 1.24.2.10 String until = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_UNTIL, null);
1120     if (Utilities.checkString(until))
1121     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_UNTIL, until);
1122 malo_pichot 1.24.2.11
1123 malo_pichot 1.24.2.10 String set = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_SET, null);
1124     if (Utilities.checkString(set))
1125     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_SET, set);
1126 malo_pichot 1.24.2.11
1127 malo_pichot 1.24.2.10 }
1128 malo_pichot 1.24.2.11
1129 malo_pichot 1.24.2.10 }
1130     }
1131 malo_pichot 1.24.2.11
1132    
1133     dbe.setId(request.getVerbId());
1134 malo_pichot 1.24.2.10 //deleting old info.
1135     //adding latest info.
1136     if (Utilities.checkString(super.responseDate))
1137     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED, super.responseDate);
1138     if (Utilities.checkString(super.resumptionToken))
1139     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_RESUMPTION_TOKEN, super.resumptionToken);
1140     dbe.addProperty(LuceneDocumentBaseOAIHarvester.NO_DOCS_DELETED, Integer.toString(noDocsDeleted));
1141     dbe.addProperty(LuceneDocumentBaseOAIHarvester.NO_DOCS_HARVESTED, Integer.toString(noDocsHarvested));
1142 malo_pichot 1.24.2.11
1143 malo_pichot 1.24.2.10 this.database.update(dbe);
1144     this.database.optimize();
1145 malo_pichot 1.24.2.11
1146 malo_pichot 1.24.2.10 }
1147 malo_pichot 1.24.2.11
1148 malo_pichot 1.24.2.10 } catch (Exception e) {
1149     throw new SAXException(e.getMessage(), e);
1150     }
1151     }
1152 rpandey 1.22
1153 rpandey 1.23 /**Generates an id to associate
1154     * with a harvest
1155     *
1156     * @return
1157     */
1158 rpandey 1.22 protected String generateNewHarvestId() {
1159     return this.harvesterIdGen.generate();
1160 malo_pichot 1.24.2.10 //Here we must generate a known ID to work with later
1161 rpandey 1.22 }
1162    
1163 rpandey 1.23 /**Sends sax events to the current consumer
1164     * with summary details of the all the past harvests
1165     *
1166     * @throws SAXException
1167     */
1168 rpandey 1.22 public void sendPastHarvestsSummary() throws SAXException {
1169     //build a query get the doc
1170     try {
1171 rpandey 1.24 DatabaseEntity[] dbes = this.database.getEntities();
1172 rpandey 1.22 if (dbes != null && dbes.length > 0) {
1173     this.acquire();
1174     acquireSynchronizedXMLConsumer();
1175     super.startElement(Framework.SDXNamespaceURI, fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS,
1176     Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS, null);
1177     for (int i = 0; i < dbes.length; i++) {
1178 rpandey 1.24 DatabaseEntity dbe = dbes[i];
1179 rpandey 1.22 AttributesImpl atts = new AttributesImpl();
1180     String repoUrl = null;
1181     if (dbe != null) {
1182     Property[] props = dbe.getProperties();
1183     if (props != null) {
1184     for (int j = 0; j < props.length; j++) {
1185     Property prop = props[j];
1186     if (prop != null) {
1187     String propName = prop.getName();
1188     if (Utilities.checkString(propName) && !LuceneDocumentBaseOAIHarvester.OAI_HARVEST_ID.equals(propName)) {
1189     String propVal = prop.getValue();
1190     if (propName.equals(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL)) {
1191     repoUrl = propVal;
1192     } else {
1193     if (Utilities.checkString(propVal))
1194     atts.addAttribute("", propName, propName, OAIObject.Node.Type.CDATA, propVal);
1195     }
1196     }
1197     }
1198    
1199     }
1200     }
1201    
1202     }
1203     sendElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.REQUEST, OAIObject.Node.Name.REQUEST,
1204     atts, repoUrl);
1205     }
1206     super.endElement(Framework.SDXNamespaceURI, fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS,
1207     Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS);
1208     }
1209    
1210     } catch (/*IO*/Exception e) {
1211     throw new SAXException(e.getMessage(), e);
1212     } /* Utilities.logException(this.logger, e);TODO: what is the best way to handle this?
1213     } catch (SDXException e) {
1214     try {
1215     e.toSAX(this);
1216     } catch (ProcessingException e1) {
1217     Utilities.logException(this.logger, e1);
1218     }
1219     } */ finally {
1220     releaseSynchronizedXMLConsumer();
1221     this.release();
1222     }
1223    
1224    
1225     }
1226    
1227 rpandey 1.23 /**Retrieves the time when the harvester was last updated
1228     *
1229     * @return
1230     */
1231 rpandey 1.22 public Date lastUpdated() {
1232 rpandey 1.23 return this.docbase.lastModificationDate();
1233 rpandey 1.22 }
1234    
1235 rpandey 1.23 /**Destroys all summary data pertaining to past harvests
1236     * but not the actual oai records harvested
1237     *
1238     */
1239 rpandey 1.22 public void purgePastHarvestsData() {
1240     try {
1241 rpandey 1.24 DatabaseEntity[] dbes = this.database.getEntities();
1242 rpandey 1.22 for (int i = 0; i < dbes.length; i++) {
1243 rpandey 1.24 DatabaseEntity dbe = dbes[i];
1244 rpandey 1.22 this.database.delete(dbe);
1245     }
1246     this.database.optimize();
1247     } catch (SDXException e) {
1248     Utilities.logException(logger, e);
1249 rpandey 1.24 }
1250     }
1251    
1252     /**Stores data about harvesting failures caused
1253     * by problems other than oai errors sent from
1254     * a queried repository
1255     *
1256     * @param e
1257     */
1258     protected void storeFailedHarvestData(Exception e) {
1259     try {
1260     //TODO: call this method when harvesting fails/major exception thrown
1261     //create a special property "failedHarvest"
1262     DatabaseEntity dbe = new DatabaseEntity(generateNewHarvestId());
1263     String message = "noMessage";
1264     if (e != null) message = e.getMessage();
1265     dbe.addProperty(OAI_FAILED_HARVEST, message);
1266     //create a database entity and store all relevant class fields
1267     dbe.addProperty(OAI_REQUEST_URL, this.requestUrl);
1268     //save it to the database
1269     this.database.update(dbe);
1270     //TODO:later perhaps we can try to auto-reexecute these failed request, keying on the OAI_FAILED_HARVEST property ???
1271     } catch (SDXException e1) {
1272     Utilities.logException(logger, e1);
1273 rpandey 1.22 }
1274 rpandey 1.1 }
1275    
1276    
1277 malo_pichot 1.24.2.6 }

savannah-hackers-public@gnu.org
ViewVC Help
Powered by ViewVC 1.1.26