/[sdx]/sdx_v2/src/java/fr/gouv/culture/sdx/oai/AbstractDocumentBaseOAIHarvester.java
ViewVC logotype

Annotation of /sdx_v2/src/java/fr/gouv/culture/sdx/oai/AbstractDocumentBaseOAIHarvester.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.24.2.2 - (hide annotations) (download)
Thu Oct 14 10:05:21 2004 UTC (19 years, 7 months ago) by mdelperier
Branch: V_22
CVS Tags: V_221_FINAL
Changes since 1.24.2.1: +3 -15 lines
pb aoi harvester in storeHarvestedData corrected

1 rpandey 1.1 /*
2     SDX: Documentary System in XML.
3     Copyright (C) 2000, 2001, 2002 Ministere de la culture et de la communication (France), AJLSM
4    
5     Ministere de la culture et de la communication,
6     Mission de la recherche et de la technologie
7     3 rue de Valois, 75042 Paris Cedex 01 (France)
8     mrt@culture.fr, michel.bottin@culture.fr
9    
10     AJLSM, 17, rue Vital Carles, 33000 Bordeaux (France)
11     sevigny@ajlsm.com
12    
13     This program is free software; you can redistribute it and/or
14     modify it under the terms of the GNU General Public License
15     as published by the Free Software Foundation; either version 2
16     of the License, or (at your option) any later version.
17    
18     This program is distributed in the hope that it will be useful,
19     but WITHOUT ANY WARRANTY; without even the implied warranty of
20     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
21     See the GNU General Public License for more details.
22    
23     You should have received a copy of the GNU General Public License
24     along with this program; if not, write to the
25     Free Software Foundation, Inc.
26     59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
27     or connect to:
28     http://www.fsf.org/copyleft/gpl.html
29     */
30     package fr.gouv.culture.sdx.oai;
31    
32 rpandey 1.3 import fr.gouv.culture.oai.AbstractOAIHarvester;
33     import fr.gouv.culture.oai.OAIObject;
34     import fr.gouv.culture.oai.OAIRequest;
35     import fr.gouv.culture.oai.OAIRequestImpl;
36 rpandey 1.1 import fr.gouv.culture.sdx.application.Application;
37     import fr.gouv.culture.sdx.document.Document;
38     import fr.gouv.culture.sdx.document.IndexableDocument;
39     import fr.gouv.culture.sdx.document.XMLDocument;
40 rpandey 1.22 import fr.gouv.culture.sdx.documentbase.*;
41 rpandey 1.1 import fr.gouv.culture.sdx.exception.SDXException;
42     import fr.gouv.culture.sdx.exception.SDXExceptionCode;
43     import fr.gouv.culture.sdx.framework.Framework;
44 rpandey 1.10 import fr.gouv.culture.sdx.framework.FrameworkImpl;
45 rpandey 1.1 import fr.gouv.culture.sdx.pipeline.GenericPipeline;
46     import fr.gouv.culture.sdx.pipeline.Pipeline;
47     import fr.gouv.culture.sdx.repository.Repository;
48 rpandey 1.21 import fr.gouv.culture.sdx.utils.ConfigurationUtilities;
49 rpandey 1.1 import fr.gouv.culture.sdx.utils.Utilities;
50 rpandey 1.19 import fr.gouv.culture.sdx.utils.database.Database;
51     import fr.gouv.culture.sdx.utils.database.DatabaseBacked;
52 rpandey 1.22 import fr.gouv.culture.sdx.utils.database.DatabaseEntity;
53     import fr.gouv.culture.sdx.utils.database.Property;
54 rpandey 1.5 import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.*;
55 rpandey 1.10 import fr.gouv.culture.util.apache.cocoon.xml.SynchronizedXMLConsumerWrapper;
56 rpandey 1.12 import org.apache.avalon.excalibur.io.FileUtil;
57 rpandey 1.7 import org.apache.avalon.excalibur.xml.Parser;
58 rpandey 1.1 import org.apache.avalon.framework.component.ComponentException;
59     import org.apache.avalon.framework.configuration.Configuration;
60     import org.apache.avalon.framework.configuration.ConfigurationException;
61 rpandey 1.22 import org.apache.avalon.framework.parameters.Parameters;
62 rpandey 1.12 import org.apache.cocoon.Constants;
63 rpandey 1.1 import org.apache.cocoon.ProcessingException;
64 rpandey 1.10 import org.apache.cocoon.serialization.XMLSerializer;
65 rpandey 1.6 import org.apache.cocoon.xml.IncludeXMLConsumer;
66 rpandey 1.10 import org.apache.cocoon.xml.XMLConsumer;
67     import org.apache.cocoon.xml.XMLMulticaster;
68 rpandey 1.3 import org.xml.sax.Attributes;
69 rpandey 1.1 import org.xml.sax.SAXException;
70 rpandey 1.6 import org.xml.sax.helpers.AttributesImpl;
71 rpandey 1.1
72 rpandey 1.12 import java.io.File;
73     import java.io.FileOutputStream;
74     import java.io.IOException;
75     import java.net.MalformedURLException;
76 rpandey 1.1 import java.net.URL;
77     import java.util.ArrayList;
78 rpandey 1.22 import java.util.Date;
79 rpandey 1.1 import java.util.Enumeration;
80     import java.util.Hashtable;
81    
82     /**
83     * Created by IntelliJ IDEA.
84     * User: rpandey
85     * Date: May 12, 2003
86     * Time: 12:32:16 PM
87     * To change this template use Options | File Templates.
88     */
89     public abstract class AbstractDocumentBaseOAIHarvester extends AbstractOAIHarvester implements Target {
90    
91     protected String ELEMENT_NAME_PIPELINE = "pipeline";
92    
93 rpandey 1.23 /**The underlying document base*/
94     protected DocumentBase docbase = null;
95     /**Id of the underlying document base*/
96     protected String docbaseId = "";
97     /**Properties from the document base*/
98     protected Hashtable docBaseProps = null;
99    
100     /**Pre-indexation pipeline*/
101 rpandey 1.1 protected Pipeline pipe = null;
102 rpandey 1.23 /**Underlying database to store any info*/
103 rpandey 1.19 protected Database database = null;
104 rpandey 1.23 /**Requests in application.xconf*/
105 rpandey 1.1 protected Hashtable storedRequests = null;
106 rpandey 1.23 /**References to the underlying documentbase's/application's repositories*/
107 rpandey 1.1 protected Hashtable storeRepositoriesRefs = null;
108 rpandey 1.23 /**Time scheduler for stored requests*/
109 rpandey 1.1 protected TimeScheduler scheduler = null;
110 rpandey 1.23 /**IDGenerator for this object*/
111     protected IDGenerator harvesterIdGen = null;
112    
113     //variables for sax stream handling
114     protected String TEMPFILE_SUFFIX = ".sdx";
115 rpandey 1.10 protected File tempDir = null;
116     protected File harvestDoc = null;
117     protected FileOutputStream fileOs = null;
118 rpandey 1.6 protected XMLDocument urlResource = null;
119 rpandey 1.1 protected ArrayList deletedDocs = null;
120 rpandey 1.19 protected ArrayList harvestedDocs = null;
121    
122 rpandey 1.23 protected boolean keepDeletedRecords = false;//Defaulted, we will delete "deletedRecords" keeping our harvester in sync with source repository
123     protected int noRecordsPerBatch = OAIObject.NUMBER_RECORDS_PER_RESPONSE;
124    
125     //configuration node names
126     /**Configuration node name*/
127     protected static final String ELEMENT_NAME_OAI_DATA_PROVIDERS = "oai-data-providers";
128     /**Configuration node name*/
129     protected static final String ELEMENT_NAME_OAI_VERB = "oai-verb";
130     /**Configuration node name*/
131     protected static final String ELEMENT_NAME_OAI_IDENTIFIER = "oai-identifier";
132     /**Configuration node name*/
133     protected static final String ATTRIBUTE_NAME_NAME = "name";
134     /**Configuration node name*/
135     protected static final String ATTRIBUTE_NAME_ADMIN_EMAIL = OAIObject.Node.Name.ADMIN_EMAIL;
136     /**Configuration node name*/
137     protected static final String ATTRIBUTE_NAME_USER_AGENT = "userAgent";
138     /**Configuration node name*/
139     protected static final String ATTRIBUTE_NAME_URL = "url";
140     /**Configuration node name*/
141     protected static final String ATTRIBUTE_NAME_UPDATE = "update";
142     /**Configuration node name*/
143     protected static final String ATTRIBUTE_NAME_METADATA_PREFIX = OAIObject.Node.Name.METADATA_PREFIX;
144     /**Configuration node name*/
145     protected static final String ATTRIBUTE_NAME_SDX_REPOSITORY = "sdxRepository";
146     /**Configuration node name*/
147     protected static final String ATTRIBUTE_NAME_FROM = "from";
148     /**Configuration node name*/
149     protected static final String ATTRIBUTE_NAME_UNTIL = "until";
150     /**Configuration node name*/
151     protected static final String ATTRIBUTE_NAME_SET = "set";
152     protected static final String ATTRIBUTE_NAME_KEEP_DELETED_RECORD = "keepDeletedRecords";
153     /**Configuration node name*/
154     protected static final String ATTRIBUTE_NO_RECORDS_PER_BATCH = "noRecordsPerBatch";
155    
156    
157     //strings for internal database properties and sax output
158 rpandey 1.22 protected static final String OAI_HARVEST_ID = "oaiHarvestId";
159 rpandey 1.24 //internal database property for failed harvests caused by internal errors( errors in this implementation and not OAI repository errors)
160     protected static final String OAI_FAILED_HARVEST = "oaiFailedHarvest";
161 rpandey 1.22 protected static final String OAI_HARVESTER_LAST_UPDATED = "oaiHarvesterLastUpdated";
162     protected static final String OAI_HARVESTER_RESUMPTION_TOKEN = "oaiHarvesterResumptionToken";
163     /*Field names for common request parameters*/
164     protected static final String OAI_VERB = "oaiVerb";
165     protected static final String OAI_IDENTIFIER = "oaiIdentifier";
166     protected static final String OAI_METADATA_PREFIX = "oaiMetadataPrefix";
167     protected static final String OAI_FROM = "oaiFrom";
168     protected static final String OAI_UNTIL = "oaiUntil";
169     protected static final String OAI_SET = "oaiSet";
170     protected static final String NO_DOCS_DELETED = "noDocDeleted";
171     protected static final String NO_DOCS_HARVESTED = "noDocHarvested";
172 rpandey 1.1
173    
174 rpandey 1.23 /**Basic constructor*/
175 rpandey 1.1 public AbstractDocumentBaseOAIHarvester(DocumentBase base) {
176 rpandey 1.23 this.docbase = base;
177     if (this.docbase != null)
178     this.docbaseId = this.docbase.getId();
179 rpandey 1.1 }
180    
181 rpandey 1.23 /**Set's the properties for this object*/
182 rpandey 1.1 public void setProperties(Hashtable props) {
183 uid67066 1.18 this.docBaseProps = props;
184 rpandey 1.1 }
185    
186    
187 rpandey 1.23 /**Configures this object*/
188 rpandey 1.1 public void configure(Configuration configuration) throws ConfigurationException {
189    
190 uid67066 1.18 super.userAgent = configuration.getAttribute(ATTRIBUTE_NAME_USER_AGENT, "SDX OAI Harvester");
191 rpandey 1.20 this.keepDeletedRecords = configuration.getAttributeAsBoolean(ATTRIBUTE_NAME_KEEP_DELETED_RECORD, this.keepDeletedRecords);
192 rpandey 1.23 this.noRecordsPerBatch = configuration.getAttributeAsInteger(ATTRIBUTE_NO_RECORDS_PER_BATCH, this.noRecordsPerBatch);
193 uid67066 1.18 configureAdminEmails(configuration);
194 rpandey 1.1 this.configureDataProviders(configuration);
195     this.configurePipeline(configuration);
196 rpandey 1.19 configureDatabase(configuration);
197 rpandey 1.21 configureHarvestIDGenerator(configuration);
198 rpandey 1.19 }
199    
200 rpandey 1.21
201 rpandey 1.23 /**Configures the internal database*/
202 rpandey 1.19 protected void configureDatabase(Configuration configuration) throws ConfigurationException {
203     DatabaseBacked internalDb = new DatabaseBacked();
204     try {
205     internalDb.setId(getHarvesterId());
206 rpandey 1.24.2.1 internalDb.compose(this.manager);
207 rpandey 1.19 internalDb.setProperties(this.docBaseProps);
208     internalDb.configure(configuration);
209     internalDb.init();
210     this.database = internalDb.getDatabase();
211     } catch (SDXException e) {
212 rpandey 1.24.2.1 throw new ConfigurationException(e.getMessage(), e);
213     } catch (ComponentException e) {
214 rpandey 1.19 throw new ConfigurationException(e.getMessage(), e);
215     }
216    
217     }
218    
219 rpandey 1.23 /**Configures the id generator for harvests*/
220 rpandey 1.21 protected void configureHarvestIDGenerator(Configuration configuration) throws ConfigurationException {
221     this.harvesterIdGen = ConfigurationUtilities.configureIDGenerator(this.logger, configuration);
222     this.harvesterIdGen.setDatabase(this.database);
223     }
224    
225    
226 rpandey 1.23 /**Returns an id for this harvester based upon the underlying document base id*/
227 rpandey 1.19 protected String getHarvesterId() {
228 rpandey 1.23 String hid = "";
229     hid += Framework.SDXNamespacePrefix + "_";
230     hid += OAIObject.Node.Prefix.OAI + "_";
231     hid += "harvester" + "_";
232     hid += this.docbaseId;
233     return hid;
234 uid67066 1.18 }
235    
236 rpandey 1.23 /**Configures a list of admin emails
237     * can be sub-elements, a single attribute,
238     * or both
239     *
240     * @param configuration
241     * @throws ConfigurationException
242     */
243 uid67066 1.18 protected void configureAdminEmails(Configuration configuration) throws ConfigurationException {
244     //configure the admin email
245     ArrayList locAdminEmailsList = new ArrayList();
246     String firstAdminEmail = configuration.getAttribute(ATTRIBUTE_NAME_ADMIN_EMAIL, null);
247     Configuration[] locAdminEmails = configuration.getChildren(ATTRIBUTE_NAME_ADMIN_EMAIL);
248     if (Utilities.checkString(firstAdminEmail))
249     locAdminEmailsList.add(firstAdminEmail);
250     for (int i = 0; i < locAdminEmails.length; i++) {
251     Configuration locAdminEmail = locAdminEmails[i];
252     if (locAdminEmail != null) {
253     String value = locAdminEmail.getValue();
254     if (Utilities.checkString(value))
255     locAdminEmailsList.add(value);
256     }
257     }
258     if (locAdminEmailsList.size() <= 0)//no admin email throw an error TODO:make this exception better
259     Utilities.checkConfAttributeValue(ATTRIBUTE_NAME_ADMIN_EMAIL, null, configuration.getLocation());
260    
261     super.adminEmails = (String[]) locAdminEmailsList.toArray(new String[0]);
262     //releasing resources
263     locAdminEmailsList.clear();
264     locAdminEmailsList = null;
265    
266 rpandey 1.1 }
267    
268 rpandey 1.23 /**Configures data providers info that can be reused
269     * and from which requests can be automatically executed
270     *
271     * @param configuration
272     * @throws ConfigurationException
273     * @see #storedRequests
274     */
275 rpandey 1.1 protected void configureDataProviders(Configuration configuration) throws ConfigurationException {
276     if (configuration != null) {
277 rpandey 1.3 Configuration dataProvidersConf = configuration.getChild(ELEMENT_NAME_OAI_DATA_PROVIDERS, false);
278 rpandey 1.1 if (dataProvidersConf != null) {
279 rpandey 1.3 Configuration[] repoRequestConfs = dataProvidersConf.getChildren(AbstractDocumentBase.ELEMENT_NAME_OAI_REPOSITORY);
280 rpandey 1.1 if (repoRequestConfs != null) {
281 rpandey 1.21 for (int x = 0; x < repoRequestConfs.length; x++) {
282     Configuration repoRequestConf = repoRequestConfs[x];
283 rpandey 1.1 if (repoRequestConf != null) {
284     if (storedRequests == null) storedRequests = new Hashtable();
285 rpandey 1.3 String repoUrl = repoRequestConf.getAttribute(ATTRIBUTE_NAME_URL);
286 rpandey 1.1 configureStoreRepositories(repoUrl, repoRequestConf);
287 rpandey 1.3 Configuration updateConf = repoRequestConf.getChild(ATTRIBUTE_NAME_UPDATE, false);
288     Configuration[] verbConfs = repoRequestConf.getChildren(ELEMENT_NAME_OAI_VERB);
289 rpandey 1.1 if (verbConfs != null) {
290 rpandey 1.21 for (int y = 0; y < verbConfs.length; y++) {
291 rpandey 1.1 OAIRequest request = null;
292 rpandey 1.21 Configuration verbConf = verbConfs[y];
293 rpandey 1.1 if (verbConf != null) {
294 rpandey 1.3 String verb = verbConf.getAttribute(ATTRIBUTE_NAME_NAME);
295     String mdPrefix = verbConf.getAttribute(ATTRIBUTE_NAME_METADATA_PREFIX);
296     Utilities.checkConfAttributeValue(ATTRIBUTE_NAME_METADATA_PREFIX, mdPrefix, verbConf.getLocation());
297 rpandey 1.1 if (verb.equalsIgnoreCase(OAIRequest.VERB_STRING_GET_RECORD)) {
298     verb = OAIRequest.VERB_STRING_GET_RECORD;
299 rpandey 1.3 Configuration[] idsConf = verbConf.getChildren(ELEMENT_NAME_OAI_IDENTIFIER);
300 rpandey 1.1 if (idsConf != null) {
301 rpandey 1.21 for (int z = 0; z < idsConf.length; z++) {
302     Configuration idConf = idsConf[z];
303 rpandey 1.1 if (idConf != null) {
304     String id = idConf.getValue();
305     request = new OAIRequestImpl();
306     request.enableLogging(this.logger);
307     request.setRepositoryURL(repoUrl);
308     request.setVerbString(verb);
309     request.setMetadataPrefix(mdPrefix);
310     request.setIdentifier(id);
311     storedRequests.put(request.getRequestURL(), request);
312     configureUpdateTriggers(request.getRequestURL(), updateConf);
313     }
314    
315     }
316     }
317    
318     } else if (verb.equalsIgnoreCase(OAIRequest.VERB_STRING_LIST_RECORDS)) {
319     verb = OAIRequest.VERB_STRING_LIST_RECORDS;
320 rpandey 1.3 String from = verbConf.getAttribute(ATTRIBUTE_NAME_FROM, null);
321     String until = verbConf.getAttribute(ATTRIBUTE_NAME_UNTIL, null);
322     String set = verbConf.getAttribute(ATTRIBUTE_NAME_SET, null);
323 rpandey 1.1 request = new OAIRequestImpl();
324     request.enableLogging(this.logger);
325     request.setRepositoryURL(repoUrl);
326     request.setVerbString(verb);
327     request.setMetadataPrefix(mdPrefix);
328     request.setFrom(from);
329     request.setUntil(until);
330     request.setSetIdentifier(set);
331     this.storedRequests.put(request.getRequestURL(), request);
332     configureUpdateTriggers(request.getRequestURL(), updateConf);
333     } else//TODOException:
334     throw new ConfigurationException("this verb action is not supported for harvesting : " + verb);
335     }
336    
337    
338     }
339     }
340    
341     }
342    
343    
344     }
345     if (this.scheduler != null) this.scheduler.start();
346     }
347    
348     }
349    
350     }
351     }
352    
353 rpandey 1.23 /**Configures time triggers for
354     * stored requests
355     *
356     * @param requestUrl The request url
357     * @param updateConf The configuration for updates
358     * @throws ConfigurationException
359     * @see #scheduler, #storedRequests
360     */
361 rpandey 1.1 protected void configureUpdateTriggers(String requestUrl, Configuration updateConf) throws ConfigurationException {
362     if (Utilities.checkString(requestUrl) && updateConf != null) {
363     TimeTrigger trigger = new TimeTriggerFactory().createTimeTrigger(updateConf);
364     if (trigger != null) {
365     if (this.scheduler == null) this.scheduler = new SimpleTimeScheduler();
366     this.scheduler.addTrigger(requestUrl, trigger, this);
367     }
368     }
369     }
370    
371 rpandey 1.23
372     /**Configures the repositories
373     * to which data will be stored
374     * based upon their repository url
375     *
376     * @param repoUrl The repository/data provider url
377     * @param oaiRepoConf The configuration
378     * @throws ConfigurationException
379     */
380 rpandey 1.1 protected void configureStoreRepositories(String repoUrl, Configuration oaiRepoConf) throws ConfigurationException {
381     if (Utilities.checkString(repoUrl)) {
382 rpandey 1.3 String ref = oaiRepoConf.getAttribute(ATTRIBUTE_NAME_SDX_REPOSITORY, null);
383 uid67066 1.18 /*check for the sdxrepository attribute, if it exists, get the repository object and add it to the local hashtable*/ if (Utilities.checkString(ref)) {
384 rpandey 1.1 Repository repo = null;
385 uid67066 1.18 Hashtable appRepos = (Hashtable) this.docBaseProps.get(Application.APPLICATION_REPOSITORIES);
386 rpandey 1.1 if (appRepos != null)
387     repo = (Repository) appRepos.get(ref);
388     if (repo == null)
389     try {
390 rpandey 1.23 repo = this.docbase.getRepository(ref);
391 rpandey 1.1 } catch (SDXException e) {
392     String[] args = new String[1];
393     args[0] = ref;
394     SDXException sdxE = new SDXException(logger, SDXExceptionCode.ERROR_LOAD_REFERENCED_REPO, args, null);
395     throw new ConfigurationException(sdxE.getMessage(), sdxE);
396     }
397     if (this.storeRepositoriesRefs == null) this.storeRepositoriesRefs = new Hashtable();
398     //populating the hashtable
399     storeRepositoriesRefs.put(repoUrl, repo);
400     }
401     }
402    
403    
404     }
405    
406    
407 rpandey 1.23 /**Configures the preIndexation pipeline
408     *
409     * @param configuration
410     * @throws ConfigurationException
411     * @see #pipe
412     */
413 rpandey 1.1 protected void configurePipeline(Configuration configuration) throws ConfigurationException {
414     //at this point, we should have a <sdx:pipeline> element
415     Configuration pipeConf = configuration.getChild(ELEMENT_NAME_PIPELINE, false);
416     //testing if we have something
417     if (pipeConf != null) {
418     //creating the pipeline and assigning the class field
419     this.pipe = new GenericPipeline();
420     //setting the logger for the pipeline object
421     this.pipe.enableLogging(this.logger);
422     //giving the object the cocoon component manager
423     try {
424     this.pipe.compose(this.manager);
425     } catch (ComponentException e) {
426     throw new ConfigurationException(e.getMessage(), e);
427     }
428    
429     //configuring the pipeline object from 'application.xconf'
430     this.pipe.configure(pipeConf);
431     }
432     }
433    
434 rpandey 1.23 /**Creates a new temporary directory for
435     * writing harvested records before the will
436     * be indexed
437     * @return
438     * @throws SDXException
439     * @throws IOException
440     */
441     protected File getNewTempDir() throws SDXException, IOException {
442     File ret = (File) this.docBaseProps.get(Constants.CONTEXT_UPLOAD_DIR);
443     String childDir = Utilities.getStringFromHashtable(FrameworkImpl.APP_PATH_NAME, this.docBaseProps) + "_oaiHarvests" + File.separator + this.docbaseId + File.separator + "harvest-" + Utilities.encodeURL(getIsoDate(), null);
444     if (Utilities.checkString(super.resumptionToken))
445     childDir += "-" + OAIObject.Node.Name.RESUMPTION_TOKEN + "-" + super.resumptionToken;
446     if (ret.canWrite())
447     ret = new File(ret, childDir);
448     else
449     ret = new File(Utilities.getSystemTempDir(), childDir);
450     Utilities.checkDirectory(ret.getCanonicalPath(), logger);
451     return ret;
452     }
453    
454     /**Deletes the directory
455     * represented by the tempDir
456     * class field
457     *
458     */
459     protected void deleteTempDir() {
460     if (tempDir != null) {
461     try {
462     FileUtil.deleteDirectory(this.tempDir);//deleting the old tempdir if possible
463     } catch (IOException e) {
464     Utilities.logWarn(logger, e.getMessage(), e);
465     }
466     }
467     }
468    
469     /**Establishes the tempDir class field
470     *
471     * @throws SDXException
472     * @throws IOException
473     */
474     protected void initTempDir() throws SDXException, IOException {
475     this.tempDir = getNewTempDir();
476     }
477    
478     /**Get's the current date in iso8601 format
479     *
480     * @return
481     */
482     protected String getIsoDate() {
483     return fr.gouv.culture.sdx.utils.Date.formatDate(fr.gouv.culture.sdx.utils.Date.getUtcIso8601Date());
484     }
485    
486 rpandey 1.1
487 rpandey 1.23 /**Sets up resources to capture an oai record
488     *
489     *
490     * @throws SAXException
491     */
492 rpandey 1.12 protected void prepareRecordCapture() throws SAXException {
493 rpandey 1.22 try {
494     if (this.tempDir == null || !this.tempDir.exists()) initTempDir();
495 rpandey 1.23 this.harvestDoc = File.createTempFile("oaiHarvestedRecord", TEMPFILE_SUFFIX, tempDir);
496 rpandey 1.22 this.fileOs = new FileOutputStream(this.harvestDoc);
497     XMLSerializer cBytes = new XMLSerializer();
498     cBytes.setOutputStream(this.fileOs);
499     XMLConsumer newConsumer = null;
500     if (firstXmlConsumer != null)
501     newConsumer = new XMLMulticaster(cBytes, firstXmlConsumer);
502     else
503     newConsumer = cBytes;
504 rpandey 1.23
505     super.setSynchronizedConsumer(new SynchronizedXMLConsumerWrapper(newConsumer));
506 rpandey 1.22 } catch (IOException e) {
507     throw new SAXException(e.getMessage(), e);
508     } catch (SDXException e) {
509     throw new SAXException(e.getMessage(), e);
510 rpandey 1.10 }
511 rpandey 1.22
512 rpandey 1.10 }
513 rpandey 1.16
514 rpandey 1.23 /**Ends the capture of an oai record
515     * and renames the file written in cocoon's
516     * work directory to correspond to it's id
517     * TODO: is this a bad idea, id's could be long?
518     *
519     * @throws Exception
520     */
521 rpandey 1.12 protected void captureRecord() throws Exception {
522 rpandey 1.22 if (this.fileOs != null && this.harvestDoc != null) {
523 rpandey 1.10 this.fileOs.flush();
524     this.fileOs.close();
525 uid67066 1.18 String encoding = Utilities.getStringFromHashtable(FrameworkImpl.ATTRIBUTE_NAME_ENCODING, this.docBaseProps);
526     File newHarvestDoc = new File(tempDir, Utilities.encodeURL(super.currentOaiIdentifier, encoding) + TEMPFILE_SUFFIX);
527 rpandey 1.19 this.harvestDoc.renameTo(newHarvestDoc);//TODO:Exception if this fails
528 rpandey 1.22 resetRecordCaptureFields(false);
529     }
530     }
531    
532 rpandey 1.23 /**Resets the class fields for record capture
533     * possibility deleting the current <code>harvetDoc</code>
534     * object underlying file
535     *
536     * @param deleteDoc flag for deletion of actual file
537     */
538 rpandey 1.22 protected void resetRecordCaptureFields(boolean deleteDoc) {
539     if (this.fileOs != null) {
540     try {
541     this.fileOs.flush();
542     this.fileOs.close();
543     this.fileOs = null;
544     } catch (IOException e) {
545     Utilities.logException(logger, e);
546     }
547     }
548    
549     if (this.harvestDoc != null) {
550     if (deleteDoc)
551     this.harvestDoc.delete();
552 rpandey 1.13 this.harvestDoc = null;
553 rpandey 1.1 }
554 rpandey 1.22
555     super.setConsumer(super.firstXmlConsumer);//resetting the consumer to the first external consumer
556 rpandey 1.1 }
557    
558 rpandey 1.23
559     /**Sets up resources to delete an oai record
560     *
561     */
562     protected void prepareRecordForDeletion() {
563     if (this.deletedDocs == null) this.deletedDocs = new ArrayList();
564     if (Utilities.checkString(this.currentOaiIdentifier)) {
565     XMLDocument deleteDoc = null;
566     try {
567     deleteDoc = new XMLDocument(this.currentOaiIdentifier);
568     } catch (SDXException e) {
569     //do nothing here
570     }
571     if (deleteDoc != null) this.deletedDocs.add(deleteDoc);
572     }
573     }
574    
575    
576     /**Reads the documents from <code>tempDir</code>
577     * and indexes them in the corresponding document
578     * base, any marked deletions will be carried out
579     * as well
580     *
581     * @return
582     * @throws SDXException
583     * @throws SAXException
584     * @throws ProcessingException
585     */
586 rpandey 1.1 protected boolean storeHarvestedData() throws SDXException, SAXException, ProcessingException {
587     boolean dataHarvested = false;
588 rpandey 1.23 if (docbase != null) {
589 rpandey 1.11 //deleting any docs which have been removed from the repo
590     Document[] deleteDocs = null;
591     if (this.deletedDocs != null && this.deletedDocs.size() > 0)
592     deleteDocs = (Document[]) this.deletedDocs.toArray(new Document[0]);
593     if (deleteDocs != null) {
594 rpandey 1.23 this.docbase.delete(deleteDocs, /*this.contentHandler*/this);
595 rpandey 1.11 dataHarvested = true;
596     }
597    
598 rpandey 1.1 IndexableDocument[] indexDocs = null;
599 rpandey 1.11 //creating our docs from the disk
600 rpandey 1.10 if (this.tempDir != null) {
601     String[] files = this.tempDir.list();
602 rpandey 1.19 this.harvestedDocs = new ArrayList();
603 rpandey 1.21 ArrayList docs = new ArrayList();
604     IndexParameters indexParams = getIndexParameters();
605 rpandey 1.10 for (int i = 0; i < files.length; i++) {
606     String fileName = files[i];
607 mdelperier 1.24.2.2
608 rpandey 1.10 XMLDocument metadataDoc = new XMLDocument();
609     String id = fileName.substring(0, fileName.lastIndexOf(TEMPFILE_SUFFIX));
610 mdelperier 1.24.2.2
611 uid67066 1.18 if (Utilities.checkString(id)) {
612     String encoding = Utilities.getStringFromHashtable(FrameworkImpl.ATTRIBUTE_NAME_ENCODING, this.docBaseProps);
613     metadataDoc.setId(Utilities.decodeURL(id, encoding));
614     }
615 rpandey 1.10 try {
616 mdelperier 1.24.2.2 metadataDoc.setContent(new File(tempDir, fileName).toURL());
617 rpandey 1.10 if (docs == null) docs = new ArrayList();
618     docs.add(metadataDoc);
619 rpandey 1.19 this.harvestedDocs.add(metadataDoc);//keeping track of all additions
620 rpandey 1.10 } catch (MalformedURLException e) {
621     Utilities.logException(logger, e);
622     }
623 rpandey 1.11 //at every 1000th document we will index into the base and/or at the end of the group
624 rpandey 1.21 int modulusTopLimit = indexParams.getBatchMax() - 1;
625     if (((i % (modulusTopLimit) == 0 || i == (files.length - 1)) && docs.size() > 0)) {
626 rpandey 1.15 indexDocs = (IndexableDocument[]) docs.toArray(new IndexableDocument[0]);
627 rpandey 1.11 if (indexDocs != null) {
628     Repository repo = null;
629     if (Utilities.checkString(this.repoUrl) && storeRepositoriesRefs != null)
630     repo = (Repository) this.storeRepositoriesRefs.get(this.repoUrl);
631 rpandey 1.23 this.docbase.index(indexDocs, repo, indexParams, /*this.contentHandler*/this);
632 rpandey 1.11 dataHarvested = true;
633 rpandey 1.15 docs = null;
634 rpandey 1.11 indexDocs = null;
635     }
636     }
637 rpandey 1.10 }
638     }
639 rpandey 1.1
640 rpandey 1.11
641 rpandey 1.1 }
642     return dataHarvested;
643     }
644    
645 rpandey 1.23 /**Handles the resumption token by issuing another request
646     * based upon the request from which the resumption token was received.
647     *
648     */
649 rpandey 1.1 protected void handleResumptionToken() {
650     if (Utilities.checkString(super.resumptionToken) && Utilities.checkString(super.repoUrl)) {
651     String verb = this.requestParams.getParameter(OAIObject.Node.Name.VERB, "");
652     if (Utilities.checkString(verb)) {
653     String requestUrl = this.repoUrl + OAIRequest.URL_CHARACTER_QUESTION_MARK + OAIObject.Node.Name.VERB + OAIRequest.URL_CHARACTER_EQUALS + verb + OAIRequest.URL_CHARACTER_AMPERSAND + OAIObject.Node.Name.RESUMPTION_TOKEN + OAIRequest.URL_CHARACTER_EQUALS + super.resumptionToken;
654 rpandey 1.14 this.resetAllFields();
655 rpandey 1.15 try {
656     this.initTempDir();//trying to build a tempDir with the resumptionToken in the dir name
657     } catch (SDXException e) {
658     Utilities.logException(logger, e);//if this fails we log it as there will be another opportunity during the harvest
659     } catch (IOException e) {
660     Utilities.logException(logger, e);
661     }
662 rpandey 1.1 super.resetResumptionToken();//resetting the resumption token here so we don't loop
663 uid67066 1.18 this.receiveRequest(requestUrl);
664 rpandey 1.1 }
665     }
666     }
667    
668 rpandey 1.23 //TODO-TEST: this should be preparing for a merge with the metadata, done
669     /**Prepares to read a url value from an oai record and
670     * retrieve the XML behind.
671     *@see #identifierName, #currentMetadtaUrlIdentifier
672     */
673     protected void prepareResourceFromUrlIdentifierCapture() {
674 rpandey 1.1 if (Utilities.checkString(this.currentMetadtaUrlIdentifier)) {
675 rpandey 1.23 try {
676     URL resourceUrl = new URL(this.currentMetadtaUrlIdentifier);
677     XMLDocument resource = new XMLDocument();
678     resource.setId(this.currentMetadtaUrlIdentifier);
679     resource.setContent(resourceUrl);
680     this.urlResource = resource;
681     } catch (MalformedURLException e) {
682     Utilities.logException(logger, e);
683     } catch (SDXException e) {
684     Utilities.logException(logger, e);
685     }
686 rpandey 1.6 /*
687 rpandey 1.1 if (docs == null) docs = new ArrayList();
688     docs.add(resource);
689 rpandey 1.6 */
690    
691     /*InputStream contents = resource.openStream();
692     byte[] chars = new byte[contents.available()];
693     contents.read(chars, 0, chars.length);
694     contents.close();
695     docBytes.
696     docBytes.write(chars);*/
697    
698     }
699     }
700    
701 rpandey 1.23 /**Captures the xml from a url taken from an oai record and adds
702     * it to the oai-record as a sibling of the <metadata/> element
703     *
704     */
705     protected void captureResourceFromUrlIdentifier() {
706 rpandey 1.6 if (this.urlResource != null) {
707     try {
708 rpandey 1.23 IncludeXMLConsumer include = new IncludeXMLConsumer(super.synchronizedXmlConsumer);
709     urlResource.setConsumer(include);
710     include.startElement(Framework.SDXNamespaceURI, "urlResource", "sdx:urlResource", new AttributesImpl());
711     Parser parser = null;
712     try {
713     parser = (Parser) this.manager.lookup(Parser.ROLE);
714     urlResource.parse(parser);
715     } finally {
716     if (parser != null)
717     this.manager.release(parser);
718     this.urlResource = null;
719     include.endElement(Framework.SDXNamespaceURI, "urlResource", "sdx:urlResource");
720     }
721     } catch (SAXException e) {
722     Utilities.logException(logger, e);
723     } catch (ComponentException e) {
724     Utilities.logException(logger, e);
725     } catch (SDXException e) {
726     Utilities.logException(logger, e);
727 rpandey 1.6 }
728 rpandey 1.1 }
729 rpandey 1.6
730 rpandey 1.1 }
731    
732    
733 rpandey 1.23 /**Resets necessary class fields
734     *
735     */
736 rpandey 1.1 protected void resetAllFields() {
737     this.deletedDocs = null;
738 rpandey 1.19 this.harvestedDocs = null;
739 rpandey 1.6 this.urlResource = null;
740 rpandey 1.22 resetRecordCaptureFields(false);
741 rpandey 1.10 deleteTempDir();
742 rpandey 1.14 this.tempDir = null;
743 rpandey 1.1 }
744    
745 rpandey 1.23
746     /**Builds simple index parameters for indexation of
747     * oai records into the undelryi
748     * @return
749     */
750 rpandey 1.1 protected IndexParameters getIndexParameters() {
751     IndexParameters params = new IndexParameters();
752     params.setSendIndexationEvents(IndexParameters.SEND_ALL_EVENTS);
753 rpandey 1.23 params.setBatchMax(this.noRecordsPerBatch);
754     if (this.pipe != null && this.docbase != null) {
755     this.pipe.setConsumer(this.docbase.getIndexationPipeline());
756 rpandey 1.1 params.setPipeline(this.pipe);
757     }
758 rpandey 1.17 params.setPipelineParams(super.getHarvestParameters());
759 rpandey 1.1 return params;
760     }
761    
762 rpandey 1.23 /**Sends the details of stored harvesting requests
763     * to the current consumer
764     *
765     * @throws SAXException
766     */
767 rpandey 1.6 public void sendStoredHarvestingRequests() throws SAXException {
768 rpandey 1.4 try {
769 rpandey 1.6 if (this.storedRequests != null && this.storedRequests.size() > 0) {
770 rpandey 1.4 this.acquire();
771     super.acquireSynchronizedXMLConsumer();
772     super.startElement(Framework.SDXNamespaceURI,
773     fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS,
774     Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS,
775     null);
776     Enumeration requests = storedRequests.elements();
777     if (requests != null) {
778     while (requests.hasMoreElements()) {
779     OAIRequest request = (OAIRequest) requests.nextElement();
780     if (request != null)
781     request.toSAX(this);
782     }
783 rpandey 1.1 }
784 rpandey 1.4
785     super.endElement(Framework.SDXNamespaceURI,
786     fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS, Framework.SDXNamespacePrefix + ":" +
787     fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS);
788 rpandey 1.1 }
789 rpandey 1.4 } catch (InterruptedException e) {
790     throw new SAXException(e.getMessage(), e);
791     } finally {
792     super.releaseSynchronizedXMLConsumer();
793     this.release();
794 rpandey 1.1 }
795    
796     }
797    
798    
799 rpandey 1.23 /**Triggers a oai request to a repository based
800     * upon a trigger name (also a request url)
801     *
802     * @param triggerName
803     */
804 rpandey 1.1 public synchronized void targetTriggered(String triggerName) {
805 rpandey 1.3 //DEBUG System.out.println("i was triggered :" + triggerName);
806 rpandey 1.23 //verify that we have this trigger defined
807 rpandey 1.1 OAIRequest request = (OAIRequest) this.storedRequests.get(triggerName);
808     if (request != null) {
809     String requestUrl = request.getRequestURL();
810 rpandey 1.17 /*TODO: query datastructs and get laste update time for this url so
811     *that we harvest only what may be changed and add this time to the request
812     *object
813     *TODO:? but should allow force complete re-harvest?
814     */
815 uid67066 1.18 this.receiveSynchronizedRequest(requestUrl);
816 rpandey 1.3 //DEBUG System.out.println("i was requested :" + requestUrl);
817 rpandey 1.1 }
818    
819    
820     }
821    
822 rpandey 1.3
823     public void startElement(String s, String s1, String s2, Attributes attributes) throws SAXException {
824     if (Utilities.checkString(this.identifierName)) {
825     /*looking for an element name or the VALUE of an attribute with the name "name" matching super.indentifierName
826     *so one could create a metadata identifier element like
827     *<super.identifierName>myIdentifierValue</super.identifierName>
828     *or
829     *<anyElementName name="super.identifierName">myIdentifierValue</anyElementName>
830     */
831 rpandey 1.4 if (attributes != null && this.identifierName.equals(attributes.getValue(fr.gouv.culture.sdx.utils.constants.Node.Name.NAME)))
832 rpandey 1.3 this.captureElemContent = true;
833     }
834     super.startElement(s, s1, s2, attributes);
835     }
836    
837     public void endElement(String s, String s1, String s2) throws SAXException {
838     if (super.sBuff != null && super.sBuff.length() > 0) {
839     String content = super.sBuff.toString();
840    
841     if (!OAIObject.Node.Xmlns.OAI_2_0.equals(s)) {
842     if (fr.gouv.culture.sdx.utils.constants.Node.Name.FIELD.equals(s1)) {
843     super.currentMetadtaUrlIdentifier = content;
844     try {
845 rpandey 1.12 prepareResourceFromUrlIdentifierCapture();
846 rpandey 1.3 } catch (Exception e) {
847     //if we can't build the document we don't just fail completely we will continue
848 rpandey 1.9 Utilities.logException(logger, e);
849 rpandey 1.3 }
850     }
851     }
852    
853     }
854     super.endElement(s, s1, s2);
855 rpandey 1.22 }
856    
857 rpandey 1.23 /**Querys the underlying data structures
858     * based upon current sax flow
859     * position/set class fields and
860     * determines whether an oai record should be
861     * harvested
862     *
863     * @return boolean indicates whether the record should be handled
864     */
865 rpandey 1.22 protected boolean shouldHarvestDocument() {
866     boolean ret = true;
867     try {
868     String verb = super.requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, null);
869     Parameters params = new Parameters();
870     params.setParameter(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL, super.repoUrl);
871     params.setParameter(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb);
872     String[] dbes = this.database.search(params);
873     // bq.add(new TermQuery(new Term(OAI_VERB, super.requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, OAIRequest.VERB_STRING_LIST_RECORDS))), false, false);
874     // bq.add(new TermQuery(new Term(OAI_VERB, super.requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, OAIRequest.VERB_STRING_GET_RECORD))), false, false);
875     //this harvest is a result of a resumptionToken request
876     // if (Utilities.checkString(resTok))
877     // bq.add(new TermQuery(new Term(OAI_HARVESTER_RESUMPTION_TOKEN, resTok)), true, false);
878     // Hits hits = getSearchIndex().search(bq);
879     if (dbes == null || dbes.length == 0)
880     return ret;//no harvest executed as yet
881     else {//we'll do comparisons on harvested data
882     for (int i = 0; i < dbes.length; i++) {
883     String dbeId = dbes[i];
884     DatabaseEntity dbe = this.database.getEntity(dbeId);
885     /*TODO: determine if this is really necessary as the following comparisons, in theory, are sufficient
886     //first comparison is based on a previously received resumption token
887     String resTok = super.requestParams.getParameter(OAIRequest.URL_PARAM_NAME_RESUMPTION_TOKEN, "");
888     String dbeResTok = dbe.getProperty(OAI_HARVESTER_RESUMPTION_TOKEN);
889     if (Utilities.checkString(dbeResTok) && Utilities.checkString(resTok)) {
890     if (resTok.equals(dbeResTok))
891     return ret;
892     }*/
893    
894     //second comparison: we make a query to see if a document with this identifier exists
895     if (Utilities.checkString(super.currentOaiIdentifier)) {
896 rpandey 1.23 DatabaseEntity docDbe = ((SDXDocumentBase) this.docbase).getDatabase().getEntity(super.currentOaiIdentifier);
897 rpandey 1.22 if (docDbe == null)
898     return ret;//no document with this id exists, so we should harvest it
899     }
900    
901     //third comparison will be based upon the timestamp of the current document vs. the time of the last harvest
902     //for ListRecords or GetRecords
903     String formattedDate = dbe.getProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED);
904     java.util.Date lastUpdatedDate = fr.gouv.culture.sdx.utils.Date.parseDate(formattedDate);
905     java.util.Date currentDocumentDate = fr.gouv.culture.sdx.utils.Date.parseDate(super.currentDatestamp);
906     if (currentDocumentDate.getTime() > lastUpdatedDate.getTime())
907     return ret;
908    
909    
910    
911     /*document is being delivered via a previously received resumpToken
912     *or
913     *document has not changed since last harvest withing the respective repository
914     *or
915     *the document exists and is upToDate
916     */
917     ret = false;
918    
919     }
920    
921    
922     }
923     } catch (SDXException e) {
924     Utilities.logException(logger, e);
925     } finally {
926     return ret;
927     }
928    
929    
930     }
931    
932 rpandey 1.23 /**Saves critical data about a harvest
933     *
934     * @param dataHarvested
935     * @throws SAXException
936     */
937 rpandey 1.22 protected void saveCriticalFields(boolean dataHarvested) throws SAXException {
938     if (dataHarvested) {//if a harvest took place we store the details
939     try {
940     String harvestId = generateNewHarvestId();
941    
942     DatabaseEntity dbe = new DatabaseEntity(harvestId);//this will be changed later
943    
944     if (Utilities.checkString(super.repoUrl))
945     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_REQUEST_URL, super.requestUrl);
946    
947     if (Utilities.checkString(super.repoUrl))
948     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL, super.repoUrl);
949    
950     if (requestParams != null) {
951    
952     String verb = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, null);
953     if (Utilities.checkString(verb)) {
954    
955     String mdPrefix = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_METADATA_PREFIX, null);
956    
957     if (verb.equals(OAIRequest.VERB_STRING_GET_RECORD)) {
958    
959     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb);
960    
961     if (Utilities.checkString(mdPrefix))
962     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_METADATA_PREFIX, mdPrefix);
963    
964     String id = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_IDENTIFIER, null);
965     if (Utilities.checkString(id))
966     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_IDENTIFIER, id);
967     }
968    
969     if (verb.equals(OAIRequest.VERB_STRING_LIST_RECORDS)) {
970    
971     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb);
972    
973     if (Utilities.checkString(mdPrefix))
974     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_METADATA_PREFIX, mdPrefix);
975    
976     String from = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_FROM, null);
977     if (Utilities.checkString(from))
978     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_FROM, from);
979    
980     String until = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_UNTIL, null);
981     if (Utilities.checkString(until))
982     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_UNTIL, until);
983    
984     String set = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_SET, null);
985     if (Utilities.checkString(set))
986     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_SET, set);
987    
988     }
989    
990     }
991     }
992    
993    
994     dbe.setId(harvestId);
995     //deleting old info.
996     //adding latest info.
997 rpandey 1.23 if (Utilities.checkString(super.responseDate))
998     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED, super.responseDate);
999 rpandey 1.22 if (Utilities.checkString(super.resumptionToken))
1000     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_RESUMPTION_TOKEN, super.resumptionToken);
1001     if (deletedDocs != null)
1002     dbe.addProperty(LuceneDocumentBaseOAIHarvester.NO_DOCS_DELETED, Integer.toString(this.deletedDocs.size()));
1003     if (harvestedDocs != null)
1004     dbe.addProperty(LuceneDocumentBaseOAIHarvester.NO_DOCS_HARVESTED, Integer.toString(this.harvestedDocs.size()));
1005    
1006     this.database.update(dbe);
1007     this.database.optimize();
1008    
1009     } catch (Exception e) {
1010     throw new SAXException(e.getMessage(), e);
1011     }
1012     }
1013     }
1014    
1015 rpandey 1.23 /**Generates an id to associate
1016     * with a harvest
1017     *
1018     * @return
1019     */
1020 rpandey 1.22 protected String generateNewHarvestId() {
1021     return this.harvesterIdGen.generate();
1022     }
1023    
1024 rpandey 1.23 /**Sends sax events to the current consumer
1025     * with summary details of the all the past harvests
1026     *
1027     * @throws SAXException
1028     */
1029 rpandey 1.22 public void sendPastHarvestsSummary() throws SAXException {
1030     //build a query get the doc
1031     try {
1032 rpandey 1.24 DatabaseEntity[] dbes = this.database.getEntities();
1033 rpandey 1.22 if (dbes != null && dbes.length > 0) {
1034     this.acquire();
1035     acquireSynchronizedXMLConsumer();
1036     super.startElement(Framework.SDXNamespaceURI, fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS,
1037     Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS, null);
1038     for (int i = 0; i < dbes.length; i++) {
1039 rpandey 1.24 DatabaseEntity dbe = dbes[i];
1040 rpandey 1.22 AttributesImpl atts = new AttributesImpl();
1041     String repoUrl = null;
1042     if (dbe != null) {
1043     Property[] props = dbe.getProperties();
1044     if (props != null) {
1045     for (int j = 0; j < props.length; j++) {
1046     Property prop = props[j];
1047     if (prop != null) {
1048     String propName = prop.getName();
1049     if (Utilities.checkString(propName) && !LuceneDocumentBaseOAIHarvester.OAI_HARVEST_ID.equals(propName)) {
1050     String propVal = prop.getValue();
1051     if (propName.equals(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL)) {
1052     repoUrl = propVal;
1053     } else {
1054     if (Utilities.checkString(propVal))
1055     atts.addAttribute("", propName, propName, OAIObject.Node.Type.CDATA, propVal);
1056     }
1057     }
1058     }
1059    
1060     }
1061     }
1062    
1063     }
1064     sendElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.REQUEST, OAIObject.Node.Name.REQUEST,
1065     atts, repoUrl);
1066     }
1067     super.endElement(Framework.SDXNamespaceURI, fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS,
1068     Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS);
1069     }
1070    
1071     } catch (/*IO*/Exception e) {
1072     throw new SAXException(e.getMessage(), e);
1073     } /* Utilities.logException(this.logger, e);TODO: what is the best way to handle this?
1074     } catch (SDXException e) {
1075     try {
1076     e.toSAX(this);
1077     } catch (ProcessingException e1) {
1078     Utilities.logException(this.logger, e1);
1079     }
1080     } */ finally {
1081     releaseSynchronizedXMLConsumer();
1082     this.release();
1083     }
1084    
1085    
1086     }
1087    
1088 rpandey 1.23 /**Retrieves the time when the harvester was last updated
1089     *
1090     * @return
1091     */
1092 rpandey 1.22 public Date lastUpdated() {
1093 rpandey 1.23 return this.docbase.lastModificationDate();
1094 rpandey 1.22 }
1095    
1096 rpandey 1.23 /**Destroys all summary data pertaining to past harvests
1097     * but not the actual oai records harvested
1098     *
1099     */
1100 rpandey 1.22 public void purgePastHarvestsData() {
1101     try {
1102 rpandey 1.24 DatabaseEntity[] dbes = this.database.getEntities();
1103 rpandey 1.22 for (int i = 0; i < dbes.length; i++) {
1104 rpandey 1.24 DatabaseEntity dbe = dbes[i];
1105 rpandey 1.22 this.database.delete(dbe);
1106     }
1107     this.database.optimize();
1108     } catch (SDXException e) {
1109     Utilities.logException(logger, e);
1110 rpandey 1.24 }
1111     }
1112    
1113     /**Stores data about harvesting failures caused
1114     * by problems other than oai errors sent from
1115     * a queried repository
1116     *
1117     * @param e
1118     */
1119     protected void storeFailedHarvestData(Exception e) {
1120     try {
1121     //TODO: call this method when harvesting fails/major exception thrown
1122     //create a special property "failedHarvest"
1123     DatabaseEntity dbe = new DatabaseEntity(generateNewHarvestId());
1124     String message = "noMessage";
1125     if (e != null) message = e.getMessage();
1126     dbe.addProperty(OAI_FAILED_HARVEST, message);
1127     //create a database entity and store all relevant class fields
1128     dbe.addProperty(OAI_REQUEST_URL, this.requestUrl);
1129     //save it to the database
1130     this.database.update(dbe);
1131     //TODO:later perhaps we can try to auto-reexecute these failed request, keying on the OAI_FAILED_HARVEST property ???
1132     } catch (SDXException e1) {
1133     Utilities.logException(logger, e1);
1134 rpandey 1.22 }
1135 rpandey 1.1 }
1136    
1137    
1138     }
1139    
1140    

savannah-hackers-public@gnu.org
ViewVC Help
Powered by ViewVC 1.1.26