/[sdx]/sdx_v2/src/java/fr/gouv/culture/sdx/oai/AbstractDocumentBaseOAIHarvester.java
ViewVC logotype

Annotation of /sdx_v2/src/java/fr/gouv/culture/sdx/oai/AbstractDocumentBaseOAIHarvester.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.43 - (hide annotations) (download)
Tue Aug 3 13:09:33 2004 UTC (19 years, 9 months ago) by jlecren
Branch: MAIN
Changes since 1.42: +2 -2 lines
change improper _identifier param to identifier param

1 rpandey 1.1 /*
2     SDX: Documentary System in XML.
3     Copyright (C) 2000, 2001, 2002 Ministere de la culture et de la communication (France), AJLSM
4    
5     Ministere de la culture et de la communication,
6     Mission de la recherche et de la technologie
7     3 rue de Valois, 75042 Paris Cedex 01 (France)
8     mrt@culture.fr, michel.bottin@culture.fr
9    
10     AJLSM, 17, rue Vital Carles, 33000 Bordeaux (France)
11     sevigny@ajlsm.com
12    
13     This program is free software; you can redistribute it and/or
14     modify it under the terms of the GNU General Public License
15     as published by the Free Software Foundation; either version 2
16     of the License, or (at your option) any later version.
17    
18     This program is distributed in the hope that it will be useful,
19     but WITHOUT ANY WARRANTY; without even the implied warranty of
20     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
21     See the GNU General Public License for more details.
22    
23     You should have received a copy of the GNU General Public License
24     along with this program; if not, write to the
25     Free Software Foundation, Inc.
26     59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
27     or connect to:
28     http://www.fsf.org/copyleft/gpl.html
29     */
30     package fr.gouv.culture.sdx.oai;
31    
32 rpandey 1.3 import fr.gouv.culture.oai.AbstractOAIHarvester;
33     import fr.gouv.culture.oai.OAIObject;
34     import fr.gouv.culture.oai.OAIRequest;
35     import fr.gouv.culture.oai.OAIRequestImpl;
36 rpandey 1.1 import fr.gouv.culture.sdx.document.Document;
37     import fr.gouv.culture.sdx.document.IndexableDocument;
38     import fr.gouv.culture.sdx.document.XMLDocument;
39 rpandey 1.36 import fr.gouv.culture.sdx.documentbase.DocumentBase;
40     import fr.gouv.culture.sdx.documentbase.IDGenerator;
41     import fr.gouv.culture.sdx.documentbase.IndexParameters;
42     import fr.gouv.culture.sdx.documentbase.SDXDocumentBase;
43 rpandey 1.1 import fr.gouv.culture.sdx.exception.SDXException;
44     import fr.gouv.culture.sdx.exception.SDXExceptionCode;
45     import fr.gouv.culture.sdx.framework.Framework;
46     import fr.gouv.culture.sdx.pipeline.GenericPipeline;
47     import fr.gouv.culture.sdx.pipeline.Pipeline;
48     import fr.gouv.culture.sdx.repository.Repository;
49     import fr.gouv.culture.sdx.utils.Utilities;
50 rpandey 1.30 import fr.gouv.culture.sdx.utils.configuration.ConfigurationUtils;
51 rpandey 1.34 import fr.gouv.culture.sdx.utils.constants.ContextKeys;
52 rpandey 1.19 import fr.gouv.culture.sdx.utils.database.Database;
53     import fr.gouv.culture.sdx.utils.database.DatabaseBacked;
54 rpandey 1.22 import fr.gouv.culture.sdx.utils.database.DatabaseEntity;
55     import fr.gouv.culture.sdx.utils.database.Property;
56 rpandey 1.26 import fr.gouv.culture.sdx.utils.logging.LoggingUtils;
57 rpandey 1.36 import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.SimpleTimeScheduler;
58     import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeScheduler;
59     import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeTrigger;
60     import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeTriggerFactory;
61 rpandey 1.10 import fr.gouv.culture.util.apache.cocoon.xml.SynchronizedXMLConsumerWrapper;
62 rpandey 1.12 import org.apache.avalon.excalibur.io.FileUtil;
63 larcaini 1.38 import org.apache.excalibur.xml.sax.SAXParser;
64 jlecren 1.39 import org.apache.avalon.framework.service.ServiceException;
65 rpandey 1.1 import org.apache.avalon.framework.configuration.Configuration;
66     import org.apache.avalon.framework.configuration.ConfigurationException;
67 rpandey 1.41 import org.apache.avalon.framework.configuration.DefaultConfiguration;
68 rpandey 1.35 import org.apache.avalon.framework.context.Context;
69 rpandey 1.30 import org.apache.avalon.framework.context.ContextException;
70 rpandey 1.22 import org.apache.avalon.framework.parameters.Parameters;
71 rpandey 1.12 import org.apache.cocoon.Constants;
72 rpandey 1.1 import org.apache.cocoon.ProcessingException;
73 rpandey 1.10 import org.apache.cocoon.serialization.XMLSerializer;
74 rpandey 1.6 import org.apache.cocoon.xml.IncludeXMLConsumer;
75 rpandey 1.10 import org.apache.cocoon.xml.XMLConsumer;
76     import org.apache.cocoon.xml.XMLMulticaster;
77 rpandey 1.3 import org.xml.sax.Attributes;
78 rpandey 1.1 import org.xml.sax.SAXException;
79 rpandey 1.6 import org.xml.sax.helpers.AttributesImpl;
80 rpandey 1.1
81 rpandey 1.12 import java.io.File;
82     import java.io.FileOutputStream;
83     import java.io.IOException;
84     import java.net.MalformedURLException;
85 rpandey 1.1 import java.net.URL;
86     import java.util.ArrayList;
87 rpandey 1.22 import java.util.Date;
88 rpandey 1.1 import java.util.Enumeration;
89     import java.util.Hashtable;
90    
91     /**
92     * Created by IntelliJ IDEA.
93     * User: rpandey
94     * Date: May 12, 2003
95     * Time: 12:32:16 PM
96     * To change this template use Options | File Templates.
97     */
98 rpandey 1.36 public abstract class AbstractDocumentBaseOAIHarvester extends AbstractOAIHarvester implements DocumentBaseOAIHarvester {
99 rpandey 1.1
100 rpandey 1.23 /**The underlying document base*/
101     protected DocumentBase docbase = null;
102     /**Id of the underlying document base*/
103     protected String docbaseId = "";
104     /**Pre-indexation pipeline*/
105 rpandey 1.1 protected Pipeline pipe = null;
106 rpandey 1.23 /**Underlying database to store any info*/
107 rpandey 1.33 protected Database _database = null;
108 rpandey 1.23 /**Requests in application.xconf*/
109 rpandey 1.1 protected Hashtable storedRequests = null;
110 rpandey 1.23 /**References to the underlying documentbase's/application's repositories*/
111 rpandey 1.1 protected Hashtable storeRepositoriesRefs = null;
112 rpandey 1.23 /**Time scheduler for stored requests*/
113 rpandey 1.1 protected TimeScheduler scheduler = null;
114 rpandey 1.23 /**IDGenerator for this object*/
115     protected IDGenerator harvesterIdGen = null;
116    
117     //variables for sax stream handling
118     protected String TEMPFILE_SUFFIX = ".sdx";
119 rpandey 1.10 protected File tempDir = null;
120     protected File harvestDoc = null;
121     protected FileOutputStream fileOs = null;
122 rpandey 1.6 protected XMLDocument urlResource = null;
123 rpandey 1.1 protected ArrayList deletedDocs = null;
124 rpandey 1.26 protected int noHarvestedDocs = 0;
125 rpandey 1.19
126 rpandey 1.23 protected boolean keepDeletedRecords = false;//Defaulted, we will delete "deletedRecords" keeping our harvester in sync with source repository
127     protected int noRecordsPerBatch = OAIObject.NUMBER_RECORDS_PER_RESPONSE;
128    
129     //strings for internal database properties and sax output
130 rpandey 1.22 protected static final String OAI_HARVEST_ID = "oaiHarvestId";
131 rpandey 1.24 //internal database property for failed harvests caused by internal errors( errors in this implementation and not OAI repository errors)
132     protected static final String OAI_FAILED_HARVEST = "oaiFailedHarvest";
133 rpandey 1.22 protected static final String OAI_HARVESTER_LAST_UPDATED = "oaiHarvesterLastUpdated";
134     protected static final String OAI_HARVESTER_RESUMPTION_TOKEN = "oaiHarvesterResumptionToken";
135     /*Field names for common request parameters*/
136     protected static final String OAI_VERB = "oaiVerb";
137     protected static final String OAI_IDENTIFIER = "oaiIdentifier";
138     protected static final String OAI_METADATA_PREFIX = "oaiMetadataPrefix";
139     protected static final String OAI_FROM = "oaiFrom";
140     protected static final String OAI_UNTIL = "oaiUntil";
141     protected static final String OAI_SET = "oaiSet";
142     protected static final String NO_DOCS_DELETED = "noDocDeleted";
143     protected static final String NO_DOCS_HARVESTED = "noDocHarvested";
144 rpandey 1.1
145    
146 rpandey 1.23 /**Basic constructor*/
147 rpandey 1.1 public AbstractDocumentBaseOAIHarvester(DocumentBase base) {
148 rpandey 1.23 this.docbase = base;
149     if (this.docbase != null)
150     this.docbaseId = this.docbase.getId();
151 rpandey 1.1 }
152    
153    
154 rpandey 1.23 /**Configures this object*/
155 rpandey 1.1 public void configure(Configuration configuration) throws ConfigurationException {
156    
157 rpandey 1.36 super.userAgent = configuration.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.USER_AGENT, "SDX OAI Harvester");
158     this.keepDeletedRecords = configuration.getAttributeAsBoolean(DocumentBaseOAIHarvester.ConfigurationNode.KEEP_DELETED_RECORDS, this.keepDeletedRecords);
159     this.noRecordsPerBatch = configuration.getAttributeAsInteger(DocumentBaseOAIHarvester.ConfigurationNode.NO_RECORDS_PER_BATCH, this.noRecordsPerBatch);
160 uid67066 1.18 configureAdminEmails(configuration);
161 rpandey 1.1 this.configureDataProviders(configuration);
162     this.configurePipeline(configuration);
163 rpandey 1.19 configureDatabase(configuration);
164 rpandey 1.21 configureHarvestIDGenerator(configuration);
165 rpandey 1.19 }
166    
167 rpandey 1.21
168 rpandey 1.23 /**Configures the internal database*/
169 rpandey 1.19 protected void configureDatabase(Configuration configuration) throws ConfigurationException {
170     DatabaseBacked internalDb = new DatabaseBacked();
171     try {
172 rpandey 1.27 internalDb.enableLogging(this.logger);
173 rpandey 1.33 internalDb.contextualize(Utilities.createNewReadOnlyContext(getContext()));
174 jlecren 1.39 internalDb.service(this.manager);
175 rpandey 1.19 internalDb.setId(getHarvesterId());
176     internalDb.configure(configuration);
177     internalDb.init();
178 rpandey 1.33 this._database = internalDb.getDatabase();
179 rpandey 1.19 } catch (SDXException e) {
180 rpandey 1.25 throw new ConfigurationException(e.getMessage(), e);
181 jlecren 1.39 } catch (ServiceException e) {
182 rpandey 1.29 throw new ConfigurationException(e.getMessage(), e);
183     } catch (ContextException e) {
184 rpandey 1.19 throw new ConfigurationException(e.getMessage(), e);
185     }
186    
187     }
188    
189 rpandey 1.23 /**Configures the id generator for harvests*/
190 rpandey 1.21 protected void configureHarvestIDGenerator(Configuration configuration) throws ConfigurationException {
191 rpandey 1.26 this.harvesterIdGen = ConfigurationUtils.configureIDGenerator(this.logger, configuration);
192 rpandey 1.33 this.harvesterIdGen.setDatabase(this._database);
193 rpandey 1.21 }
194    
195    
196 rpandey 1.23 /**Returns an id for this harvester based upon the underlying document base id*/
197 rpandey 1.19 protected String getHarvesterId() {
198 rpandey 1.23 String hid = "";
199     hid += Framework.SDXNamespacePrefix + "_";
200     hid += OAIObject.Node.Prefix.OAI + "_";
201     hid += "harvester" + "_";
202     hid += this.docbaseId;
203     return hid;
204 uid67066 1.18 }
205    
206 rpandey 1.23 /**Configures a list of admin emails
207     * can be sub-elements, a single attribute,
208     * or both
209     *
210     * @param configuration
211     * @throws ConfigurationException
212     */
213 uid67066 1.18 protected void configureAdminEmails(Configuration configuration) throws ConfigurationException {
214     //configure the admin email
215     ArrayList locAdminEmailsList = new ArrayList();
216 rpandey 1.36 String firstAdminEmail = configuration.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.ADMIN_EMAIL, null);
217     Configuration[] locAdminEmails = configuration.getChildren(DocumentBaseOAIHarvester.ConfigurationNode.ADMIN_EMAIL);
218 uid67066 1.18 if (Utilities.checkString(firstAdminEmail))
219     locAdminEmailsList.add(firstAdminEmail);
220     for (int i = 0; i < locAdminEmails.length; i++) {
221     Configuration locAdminEmail = locAdminEmails[i];
222     if (locAdminEmail != null) {
223     String value = locAdminEmail.getValue();
224     if (Utilities.checkString(value))
225     locAdminEmailsList.add(value);
226     }
227     }
228     if (locAdminEmailsList.size() <= 0)//no admin email throw an error TODO:make this exception better
229 rpandey 1.36 ConfigurationUtils.checkConfAttributeValue(DocumentBaseOAIHarvester.ConfigurationNode.ADMIN_EMAIL, null, configuration.getLocation());
230 uid67066 1.18
231     super.adminEmails = (String[]) locAdminEmailsList.toArray(new String[0]);
232     //releasing resources
233     locAdminEmailsList.clear();
234     locAdminEmailsList = null;
235    
236 rpandey 1.1 }
237    
238 rpandey 1.23 /**Configures data providers info that can be reused
239     * and from which requests can be automatically executed
240     *
241     * @param configuration
242     * @throws ConfigurationException
243     * @see #storedRequests
244     */
245 rpandey 1.1 protected void configureDataProviders(Configuration configuration) throws ConfigurationException {
246     if (configuration != null) {
247 rpandey 1.36 Configuration dataProvidersConf = configuration.getChild(DocumentBaseOAIHarvester.ConfigurationNode.OAI_DATA_PROVIDERS, false);
248 rpandey 1.1 if (dataProvidersConf != null) {
249 rpandey 1.36 Configuration[] repoRequestConfs = dataProvidersConf.getChildren(DocumentBase.ConfigurationNode.OAI_REPOSITORY);
250 rpandey 1.1 if (repoRequestConfs != null) {
251 rpandey 1.21 for (int x = 0; x < repoRequestConfs.length; x++) {
252     Configuration repoRequestConf = repoRequestConfs[x];
253 rpandey 1.1 if (repoRequestConf != null) {
254     if (storedRequests == null) storedRequests = new Hashtable();
255 rpandey 1.36 String repoUrl = repoRequestConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.URL);
256 rpandey 1.1 configureStoreRepositories(repoUrl, repoRequestConf);
257 rpandey 1.36 Configuration updateConf = repoRequestConf.getChild(DocumentBaseOAIHarvester.ConfigurationNode.UPDATE, false);
258     Configuration[] verbConfs = repoRequestConf.getChildren(DocumentBaseOAIHarvester.ConfigurationNode.OAI_VERB);
259 rpandey 1.1 if (verbConfs != null) {
260 rpandey 1.21 for (int y = 0; y < verbConfs.length; y++) {
261 rpandey 1.28 try {
262     OAIRequest request = null;
263     Configuration verbConf = verbConfs[y];
264     if (verbConf != null) {
265 rpandey 1.36 String verb = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.NAME);
266     String mdPrefix = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.METADATA_PREFIX);
267     ConfigurationUtils.checkConfAttributeValue(DocumentBaseOAIHarvester.ConfigurationNode.METADATA_PREFIX, mdPrefix, verbConf.getLocation());
268 rpandey 1.28 if (verb.equalsIgnoreCase(OAIRequest.VERB_STRING_GET_RECORD)) {
269     verb = OAIRequest.VERB_STRING_GET_RECORD;
270 rpandey 1.36 Configuration[] idsConf = verbConf.getChildren(DocumentBaseOAIHarvester.ConfigurationNode.OAI_IDENTIFIER);
271 rpandey 1.28 if (idsConf != null) {
272     for (int z = 0; z < idsConf.length; z++) {
273     Configuration idConf = idsConf[z];
274     if (idConf != null) {
275     String id = idConf.getValue();
276     request = new OAIRequestImpl();
277     request.enableLogging(this.logger);
278     request.setRepositoryURL(repoUrl);
279     request.setVerbString(verb);
280     request.setMetadataPrefix(mdPrefix);
281     request.setIdentifier(id);
282     Utilities.isObjectUnique(storedRequests, request.getRequestURL(), request);
283     storedRequests.put(request.getRequestURL(), request);
284     configureUpdateTriggers(request.getRequestURL(), updateConf);
285     }
286    
287 rpandey 1.1 }
288     }
289    
290 rpandey 1.28 } else if (verb.equalsIgnoreCase(OAIRequest.VERB_STRING_LIST_RECORDS)) {
291     verb = OAIRequest.VERB_STRING_LIST_RECORDS;
292 rpandey 1.36 String from = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.FROM, null);
293     String until = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.UNTIL, null);
294     String set = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.SET, null);
295 rpandey 1.28 request = new OAIRequestImpl();
296     request.enableLogging(this.logger);
297     request.setRepositoryURL(repoUrl);
298     request.setVerbString(verb);
299     request.setMetadataPrefix(mdPrefix);
300     request.setFrom(from);
301     request.setUntil(until);
302     request.setSetIdentifier(set);
303     Utilities.isObjectUnique(storedRequests, request.getRequestURL(), request);
304     this.storedRequests.put(request.getRequestURL(), request);
305     configureUpdateTriggers(request.getRequestURL(), updateConf);
306     } else//TODOException:
307     throw new ConfigurationException("this verb action is not supported for harvesting : " + verb);
308     }
309     } catch (ConfigurationException e) {
310     LoggingUtils.logWarn(this.logger, e.getMessage(), e);//logging this so that all configs don't fail
311     } catch (SDXException e) {
312     LoggingUtils.logWarn(this.logger, e.getMessage(), e);//logging this so that all configs don't fail
313 rpandey 1.1 }
314    
315    
316     }
317     }
318    
319     }
320    
321    
322     }
323     if (this.scheduler != null) this.scheduler.start();
324     }
325    
326     }
327    
328     }
329     }
330    
331 rpandey 1.23 /**Configures time triggers for
332     * stored requests
333     *
334     * @param requestUrl The request url
335     * @param updateConf The configuration for updates
336     * @throws ConfigurationException
337     * @see #scheduler, #storedRequests
338     */
339 rpandey 1.1 protected void configureUpdateTriggers(String requestUrl, Configuration updateConf) throws ConfigurationException {
340     if (Utilities.checkString(requestUrl) && updateConf != null) {
341     TimeTrigger trigger = new TimeTriggerFactory().createTimeTrigger(updateConf);
342     if (trigger != null) {
343     if (this.scheduler == null) this.scheduler = new SimpleTimeScheduler();
344     this.scheduler.addTrigger(requestUrl, trigger, this);
345     }
346     }
347     }
348    
349 rpandey 1.23
350     /**Configures the repositories
351     * to which data will be stored
352     * based upon their repository url
353     *
354     * @param repoUrl The repository/data provider url
355     * @param oaiRepoConf The configuration
356     * @throws ConfigurationException
357     */
358 rpandey 1.1 protected void configureStoreRepositories(String repoUrl, Configuration oaiRepoConf) throws ConfigurationException {
359     if (Utilities.checkString(repoUrl)) {
360 rpandey 1.36 String ref = oaiRepoConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.SDX_REPOSITORY, null);
361 rpandey 1.28 /*check for the sdxrepository attribute, if it exists, get the repository object and add it to the local hashtable*/
362     if (Utilities.checkString(ref)) {
363     try {
364     Repository repo = null;
365 rpandey 1.35 Context appRepos = (Context) getContext().get(ContextKeys.SDX.Application.REPOSITORIES);
366 rpandey 1.28 if (appRepos != null)
367     repo = (Repository) appRepos.get(ref);
368     if (repo == null)
369 rpandey 1.23 repo = this.docbase.getRepository(ref);
370 rpandey 1.28 if (this.storeRepositoriesRefs == null) this.storeRepositoriesRefs = new Hashtable();
371     //populating the hashtable
372     Utilities.isObjectUnique(this.storeRepositoriesRefs, repoUrl, repo);
373     storeRepositoriesRefs.put(repoUrl, repo);
374     } catch (SDXException e) {
375     String[] args = new String[1];
376     args[0] = ref;
377     SDXException sdxE = new SDXException(logger, SDXExceptionCode.ERROR_LOAD_REFERENCED_REPO, args, null);
378     throw new ConfigurationException(sdxE.getMessage(), sdxE);
379 rpandey 1.33 } catch (ContextException e) {
380     throw new ConfigurationException(e.getMessage(), e);
381 rpandey 1.28 }
382 rpandey 1.1 }
383     }
384    
385    
386     }
387    
388    
389 rpandey 1.23 /**Configures the preIndexation pipeline
390     *
391     * @param configuration
392     * @throws ConfigurationException
393     * @see #pipe
394     */
395 rpandey 1.1 protected void configurePipeline(Configuration configuration) throws ConfigurationException {
396     //at this point, we should have a <sdx:pipeline> element
397 rpandey 1.30 Configuration pipeConf = configuration.getChild(Utilities.getElementName(Pipeline.CLASS_NAME_SUFFIX), false);
398 rpandey 1.1 //testing if we have something
399     if (pipeConf != null) {
400     //creating the pipeline and assigning the class field
401     this.pipe = new GenericPipeline();
402 rpandey 1.27 //setting the super.getLog() for the pipeline object
403 rpandey 1.1 this.pipe.enableLogging(this.logger);
404 jlecren 1.39 //giving the object the cocoon service manager
405 rpandey 1.1 try {
406 jlecren 1.40 this.pipe.service(this.manager);
407 jlecren 1.39 } catch (ServiceException e) {
408 rpandey 1.1 throw new ConfigurationException(e.getMessage(), e);
409     }
410    
411     //configuring the pipeline object from 'application.xconf'
412     this.pipe.configure(pipeConf);
413     }
414     }
415    
416 rpandey 1.23 /**Creates a new temporary directory for
417     * writing harvested records before the will
418     * be indexed
419     * @return
420     * @throws SDXException
421     * @throws IOException
422     */
423     protected File getNewTempDir() throws SDXException, IOException {
424 rpandey 1.33 File ret = null;
425     try {
426     ret = (File) getContext().get(Constants.CONTEXT_UPLOAD_DIR);
427     } catch (ContextException e) {
428     throw new SDXException(logger, SDXExceptionCode.ERROR_GENERIC, null, e);//TODOException: better message here
429     }
430 rpandey 1.34 String childDir = Utilities.getStringFromContext(ContextKeys.SDX.Application.DIRECTORY_NAME, getContext()) + "_oaiHarvests" + File.separator + this.docbaseId + File.separator + "harvest-" + Utilities.encodeURL(getIsoDate(), null);
431 rpandey 1.23 if (Utilities.checkString(super.resumptionToken))
432     childDir += "-" + OAIObject.Node.Name.RESUMPTION_TOKEN + "-" + super.resumptionToken;
433     if (ret.canWrite())
434     ret = new File(ret, childDir);
435     else
436     ret = new File(Utilities.getSystemTempDir(), childDir);
437     Utilities.checkDirectory(ret.getCanonicalPath(), logger);
438     return ret;
439     }
440    
441     /**Deletes the directory
442     * represented by the tempDir
443     * class field
444     *
445     */
446     protected void deleteTempDir() {
447     if (tempDir != null) {
448     try {
449     FileUtil.deleteDirectory(this.tempDir);//deleting the old tempdir if possible
450     } catch (IOException e) {
451 rpandey 1.26 LoggingUtils.logWarn(logger, e.getMessage(), e);
452 rpandey 1.23 }
453     }
454     }
455    
456     /**Establishes the tempDir class field
457     *
458     * @throws SDXException
459     * @throws IOException
460     */
461     protected void initTempDir() throws SDXException, IOException {
462     this.tempDir = getNewTempDir();
463     }
464    
465     /**Get's the current date in iso8601 format
466     *
467     * @return
468     */
469     protected String getIsoDate() {
470     return fr.gouv.culture.sdx.utils.Date.formatDate(fr.gouv.culture.sdx.utils.Date.getUtcIso8601Date());
471     }
472    
473 rpandey 1.1
474 rpandey 1.23 /**Sets up resources to capture an oai record
475     *
476     *
477     * @throws SAXException
478     */
479 rpandey 1.12 protected void prepareRecordCapture() throws SAXException {
480 rpandey 1.22 try {
481     if (this.tempDir == null || !this.tempDir.exists()) initTempDir();
482 rpandey 1.23 this.harvestDoc = File.createTempFile("oaiHarvestedRecord", TEMPFILE_SUFFIX, tempDir);
483 rpandey 1.22 this.fileOs = new FileOutputStream(this.harvestDoc);
484     XMLSerializer cBytes = new XMLSerializer();
485 rpandey 1.42 cBytes.enableLogging(logger);
486 rpandey 1.41 cBytes.configure(new DefaultConfiguration("hack", ""));
487 rpandey 1.22 cBytes.setOutputStream(this.fileOs);
488     XMLConsumer newConsumer = null;
489     if (firstXmlConsumer != null)
490     newConsumer = new XMLMulticaster(cBytes, firstXmlConsumer);
491     else
492     newConsumer = cBytes;
493 rpandey 1.23
494     super.setSynchronizedConsumer(new SynchronizedXMLConsumerWrapper(newConsumer));
495 rpandey 1.22 } catch (IOException e) {
496     throw new SAXException(e.getMessage(), e);
497     } catch (SDXException e) {
498     throw new SAXException(e.getMessage(), e);
499 rpandey 1.41 } catch (ConfigurationException e) {
500     throw new SAXException(e.getMessage(), e);
501 rpandey 1.10 }
502 rpandey 1.22
503 rpandey 1.10 }
504 rpandey 1.16
505 rpandey 1.23 /**Ends the capture of an oai record
506     * and renames the file written in cocoon's
507     * work directory to correspond to it's id
508     * TODO: is this a bad idea, id's could be long?
509     *
510     * @throws Exception
511     */
512 rpandey 1.12 protected void captureRecord() throws Exception {
513 rpandey 1.22 if (this.fileOs != null && this.harvestDoc != null) {
514 rpandey 1.10 this.fileOs.flush();
515     this.fileOs.close();
516 rpandey 1.36 String encoding = Utilities.getStringFromContext(ContextKeys.SDX.ENCODING, getContext());
517 uid67066 1.18 File newHarvestDoc = new File(tempDir, Utilities.encodeURL(super.currentOaiIdentifier, encoding) + TEMPFILE_SUFFIX);
518 rpandey 1.19 this.harvestDoc.renameTo(newHarvestDoc);//TODO:Exception if this fails
519 rpandey 1.22 resetRecordCaptureFields(false);
520     }
521     }
522    
523 rpandey 1.23 /**Resets the class fields for record capture
524     * possibility deleting the current <code>harvetDoc</code>
525     * object underlying file
526     *
527     * @param deleteDoc flag for deletion of actual file
528     */
529 rpandey 1.22 protected void resetRecordCaptureFields(boolean deleteDoc) {
530     if (this.fileOs != null) {
531     try {
532     this.fileOs.flush();
533     this.fileOs.close();
534     this.fileOs = null;
535     } catch (IOException e) {
536 rpandey 1.26 LoggingUtils.logException(logger, e);
537 rpandey 1.22 }
538     }
539    
540     if (this.harvestDoc != null) {
541     if (deleteDoc)
542     this.harvestDoc.delete();
543 rpandey 1.13 this.harvestDoc = null;
544 rpandey 1.1 }
545 rpandey 1.22
546     super.setConsumer(super.firstXmlConsumer);//resetting the consumer to the first external consumer
547 rpandey 1.1 }
548    
549 rpandey 1.23
550     /**Sets up resources to delete an oai record
551     *
552     */
553     protected void prepareRecordForDeletion() {
554     if (this.deletedDocs == null) this.deletedDocs = new ArrayList();
555     if (Utilities.checkString(this.currentOaiIdentifier)) {
556     XMLDocument deleteDoc = null;
557     try {
558     deleteDoc = new XMLDocument(this.currentOaiIdentifier);
559     } catch (SDXException e) {
560     //do nothing here
561     }
562     if (deleteDoc != null) this.deletedDocs.add(deleteDoc);
563     }
564     }
565    
566    
567     /**Reads the documents from <code>tempDir</code>
568     * and indexes them in the corresponding document
569     * base, any marked deletions will be carried out
570     * as well
571     *
572     * @return
573     * @throws SDXException
574     * @throws SAXException
575     * @throws ProcessingException
576     */
577 rpandey 1.1 protected boolean storeHarvestedData() throws SDXException, SAXException, ProcessingException {
578     boolean dataHarvested = false;
579 rpandey 1.23 if (docbase != null) {
580 rpandey 1.11 //deleting any docs which have been removed from the repo
581     Document[] deleteDocs = null;
582     if (this.deletedDocs != null && this.deletedDocs.size() > 0)
583     deleteDocs = (Document[]) this.deletedDocs.toArray(new Document[0]);
584     if (deleteDocs != null) {
585 rpandey 1.23 this.docbase.delete(deleteDocs, /*this.contentHandler*/this);
586 rpandey 1.11 dataHarvested = true;
587     }
588    
589 rpandey 1.1 IndexableDocument[] indexDocs = null;
590 rpandey 1.11 //creating our docs from the disk
591 rpandey 1.10 if (this.tempDir != null) {
592     String[] files = this.tempDir.list();
593 rpandey 1.26 this.noHarvestedDocs = 0;
594 rpandey 1.21 ArrayList docs = new ArrayList();
595     IndexParameters indexParams = getIndexParameters();
596 rpandey 1.10 for (int i = 0; i < files.length; i++) {
597     String fileName = files[i];
598     //TODO refactor this
599     String absolute = new File(tempDir, fileName).getAbsolutePath();
600     StringBuffer url = new StringBuffer("file://");
601 rpandey 1.11 for (int j = 0; j < absolute.length(); j++) {
602     char c = absolute.charAt(j);
603     if (c == File.separatorChar)
604     url.append('/');
605 uid67066 1.18 else {
606 rpandey 1.36 String encoding = Utilities.getStringFromContext(ContextKeys.SDX.ENCODING, getContext());
607 uid67066 1.18 url.append(Utilities.encodeURL(String.valueOf(c), encoding));
608     }
609 rpandey 1.11 }
610 rpandey 1.10 XMLDocument metadataDoc = new XMLDocument();
611     String id = fileName.substring(0, fileName.lastIndexOf(TEMPFILE_SUFFIX));
612     fileName = null;
613     //DEBUG System.out.println("a document id : " + id);
614 uid67066 1.18 if (Utilities.checkString(id)) {
615 rpandey 1.36 String encoding = Utilities.getStringFromContext(ContextKeys.SDX.ENCODING, getContext());
616 uid67066 1.18 metadataDoc.setId(Utilities.decodeURL(id, encoding));
617     }
618 rpandey 1.10 try {
619     metadataDoc.setContent(new URL(url.toString()));
620     if (docs == null) docs = new ArrayList();
621     docs.add(metadataDoc);
622 rpandey 1.26 this.noHarvestedDocs++;//keeping track of all additions
623 rpandey 1.10 } catch (MalformedURLException e) {
624 rpandey 1.26 LoggingUtils.logException(logger, e);
625 rpandey 1.10 }
626 rpandey 1.11 //at every 1000th document we will index into the base and/or at the end of the group
627 rpandey 1.21 int modulusTopLimit = indexParams.getBatchMax() - 1;
628     if (((i % (modulusTopLimit) == 0 || i == (files.length - 1)) && docs.size() > 0)) {
629 rpandey 1.15 indexDocs = (IndexableDocument[]) docs.toArray(new IndexableDocument[0]);
630 rpandey 1.11 if (indexDocs != null) {
631     Repository repo = null;
632     if (Utilities.checkString(this.repoUrl) && storeRepositoriesRefs != null)
633     repo = (Repository) this.storeRepositoriesRefs.get(this.repoUrl);
634 rpandey 1.23 this.docbase.index(indexDocs, repo, indexParams, /*this.contentHandler*/this);
635 rpandey 1.11 dataHarvested = true;
636 rpandey 1.15 docs = null;
637 rpandey 1.11 indexDocs = null;
638     }
639     }
640 rpandey 1.10 }
641     }
642 rpandey 1.1
643 rpandey 1.11
644 rpandey 1.1 }
645     return dataHarvested;
646     }
647    
648 rpandey 1.23 /**Handles the resumption token by issuing another request
649     * based upon the request from which the resumption token was received.
650     *
651     */
652 rpandey 1.1 protected void handleResumptionToken() {
653     if (Utilities.checkString(super.resumptionToken) && Utilities.checkString(super.repoUrl)) {
654     String verb = this.requestParams.getParameter(OAIObject.Node.Name.VERB, "");
655     if (Utilities.checkString(verb)) {
656     String requestUrl = this.repoUrl + OAIRequest.URL_CHARACTER_QUESTION_MARK + OAIObject.Node.Name.VERB + OAIRequest.URL_CHARACTER_EQUALS + verb + OAIRequest.URL_CHARACTER_AMPERSAND + OAIObject.Node.Name.RESUMPTION_TOKEN + OAIRequest.URL_CHARACTER_EQUALS + super.resumptionToken;
657 rpandey 1.14 this.resetAllFields();
658 rpandey 1.15 try {
659     this.initTempDir();//trying to build a tempDir with the resumptionToken in the dir name
660     } catch (SDXException e) {
661 rpandey 1.26 LoggingUtils.logException(logger, e);//if this fails we log it as there will be another opportunity during the harvest
662 rpandey 1.15 } catch (IOException e) {
663 rpandey 1.26 LoggingUtils.logException(logger, e);
664 rpandey 1.15 }
665 rpandey 1.1 super.resetResumptionToken();//resetting the resumption token here so we don't loop
666 uid67066 1.18 this.receiveRequest(requestUrl);
667 rpandey 1.1 }
668     }
669     }
670    
671 rpandey 1.23 //TODO-TEST: this should be preparing for a merge with the metadata, done
672     /**Prepares to read a url value from an oai record and
673     * retrieve the XML behind.
674     *@see #identifierName, #currentMetadtaUrlIdentifier
675     */
676     protected void prepareResourceFromUrlIdentifierCapture() {
677 rpandey 1.1 if (Utilities.checkString(this.currentMetadtaUrlIdentifier)) {
678 rpandey 1.23 try {
679     URL resourceUrl = new URL(this.currentMetadtaUrlIdentifier);
680     XMLDocument resource = new XMLDocument();
681     resource.setId(this.currentMetadtaUrlIdentifier);
682     resource.setContent(resourceUrl);
683     this.urlResource = resource;
684     } catch (MalformedURLException e) {
685 rpandey 1.26 LoggingUtils.logException(logger, e);
686 rpandey 1.23 } catch (SDXException e) {
687 rpandey 1.26 LoggingUtils.logException(logger, e);
688 rpandey 1.23 }
689 rpandey 1.6 /*
690 rpandey 1.1 if (docs == null) docs = new ArrayList();
691     docs.add(resource);
692 rpandey 1.6 */
693    
694     /*InputStream contents = resource.openStream();
695     byte[] chars = new byte[contents.available()];
696     contents.read(chars, 0, chars.length);
697     contents.close();
698     docBytes.
699     docBytes.write(chars);*/
700    
701     }
702     }
703    
704 rpandey 1.23 /**Captures the xml from a url taken from an oai record and adds
705     * it to the oai-record as a sibling of the <metadata/> element
706     *
707     */
708     protected void captureResourceFromUrlIdentifier() {
709 rpandey 1.6 if (this.urlResource != null) {
710     try {
711 rpandey 1.23 IncludeXMLConsumer include = new IncludeXMLConsumer(super.synchronizedXmlConsumer);
712     urlResource.setConsumer(include);
713     include.startElement(Framework.SDXNamespaceURI, "urlResource", "sdx:urlResource", new AttributesImpl());
714 larcaini 1.38 SAXParser parser = null;
715 rpandey 1.23 try {
716 larcaini 1.38 parser = (SAXParser) this.manager.lookup(SAXParser.ROLE);
717 rpandey 1.23 urlResource.parse(parser);
718     } finally {
719     if (parser != null)
720     this.manager.release(parser);
721     this.urlResource = null;
722     include.endElement(Framework.SDXNamespaceURI, "urlResource", "sdx:urlResource");
723     }
724     } catch (SAXException e) {
725 rpandey 1.26 LoggingUtils.logException(logger, e);
726 jlecren 1.39 } catch (ServiceException e) {
727 rpandey 1.26 LoggingUtils.logException(logger, e);
728 rpandey 1.23 } catch (SDXException e) {
729 rpandey 1.26 LoggingUtils.logException(logger, e);
730 rpandey 1.6 }
731 rpandey 1.1 }
732 rpandey 1.6
733 rpandey 1.1 }
734    
735    
736 rpandey 1.23 /**Resets necessary class fields
737     *
738     */
739 rpandey 1.1 protected void resetAllFields() {
740     this.deletedDocs = null;
741 rpandey 1.26 this.noHarvestedDocs = 0;
742 rpandey 1.6 this.urlResource = null;
743 rpandey 1.22 resetRecordCaptureFields(false);
744 rpandey 1.10 deleteTempDir();
745 rpandey 1.14 this.tempDir = null;
746 rpandey 1.1 }
747    
748 rpandey 1.23
749     /**Builds simple index parameters for indexation of
750     * oai records into the undelryi
751     * @return
752     */
753 rpandey 1.1 protected IndexParameters getIndexParameters() {
754     IndexParameters params = new IndexParameters();
755     params.setSendIndexationEvents(IndexParameters.SEND_ALL_EVENTS);
756 rpandey 1.23 params.setBatchMax(this.noRecordsPerBatch);
757     if (this.pipe != null && this.docbase != null) {
758     this.pipe.setConsumer(this.docbase.getIndexationPipeline());
759 rpandey 1.1 params.setPipeline(this.pipe);
760     }
761 rpandey 1.17 params.setPipelineParams(super.getHarvestParameters());
762 rpandey 1.1 return params;
763     }
764    
765 rpandey 1.23 /**Sends the details of stored harvesting requests
766     * to the current consumer
767     *
768     * @throws SAXException
769     */
770 rpandey 1.6 public void sendStoredHarvestingRequests() throws SAXException {
771 rpandey 1.4 try {
772 rpandey 1.6 if (this.storedRequests != null && this.storedRequests.size() > 0) {
773 rpandey 1.4 this.acquire();
774     super.acquireSynchronizedXMLConsumer();
775     super.startElement(Framework.SDXNamespaceURI,
776     fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS,
777     Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS,
778     null);
779     Enumeration requests = storedRequests.elements();
780     if (requests != null) {
781     while (requests.hasMoreElements()) {
782     OAIRequest request = (OAIRequest) requests.nextElement();
783     if (request != null)
784     request.toSAX(this);
785     }
786 rpandey 1.1 }
787 rpandey 1.4
788     super.endElement(Framework.SDXNamespaceURI,
789     fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS, Framework.SDXNamespacePrefix + ":" +
790     fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS);
791 rpandey 1.1 }
792 rpandey 1.4 } catch (InterruptedException e) {
793     throw new SAXException(e.getMessage(), e);
794     } finally {
795     super.releaseSynchronizedXMLConsumer();
796     this.release();
797 rpandey 1.1 }
798    
799     }
800    
801    
802 rpandey 1.23 /**Triggers a oai request to a repository based
803     * upon a trigger name (also a request url)
804     *
805     * @param triggerName
806     */
807 rpandey 1.1 public synchronized void targetTriggered(String triggerName) {
808 rpandey 1.3 //DEBUG System.out.println("i was triggered :" + triggerName);
809 rpandey 1.23 //verify that we have this trigger defined
810 rpandey 1.1 OAIRequest request = (OAIRequest) this.storedRequests.get(triggerName);
811     if (request != null) {
812     String requestUrl = request.getRequestURL();
813 rpandey 1.17 /*TODO: query datastructs and get laste update time for this url so
814     *that we harvest only what may be changed and add this time to the request
815     *object
816     *TODO:? but should allow force complete re-harvest?
817     */
818 uid67066 1.18 this.receiveSynchronizedRequest(requestUrl);
819 rpandey 1.3 //DEBUG System.out.println("i was requested :" + requestUrl);
820 rpandey 1.1 }
821    
822    
823     }
824    
825 rpandey 1.3
826     public void startElement(String s, String s1, String s2, Attributes attributes) throws SAXException {
827     if (Utilities.checkString(this.identifierName)) {
828     /*looking for an element name or the VALUE of an attribute with the name "name" matching super.indentifierName
829 jlecren 1.43 *so one could create a metadata identifier element like
830 rpandey 1.3 *<super.identifierName>myIdentifierValue</super.identifierName>
831     *or
832     *<anyElementName name="super.identifierName">myIdentifierValue</anyElementName>
833     */
834 rpandey 1.4 if (attributes != null && this.identifierName.equals(attributes.getValue(fr.gouv.culture.sdx.utils.constants.Node.Name.NAME)))
835 rpandey 1.3 this.captureElemContent = true;
836     }
837     super.startElement(s, s1, s2, attributes);
838     }
839    
840     public void endElement(String s, String s1, String s2) throws SAXException {
841     if (super.sBuff != null && super.sBuff.length() > 0) {
842     String content = super.sBuff.toString();
843    
844     if (!OAIObject.Node.Xmlns.OAI_2_0.equals(s)) {
845     if (fr.gouv.culture.sdx.utils.constants.Node.Name.FIELD.equals(s1)) {
846     super.currentMetadtaUrlIdentifier = content;
847     try {
848 rpandey 1.12 prepareResourceFromUrlIdentifierCapture();
849 rpandey 1.3 } catch (Exception e) {
850     //if we can't build the document we don't just fail completely we will continue
851 rpandey 1.26 LoggingUtils.logException(logger, e);
852 rpandey 1.3 }
853     }
854     }
855    
856     }
857     super.endElement(s, s1, s2);
858 rpandey 1.22 }
859    
860 rpandey 1.23 /**Querys the underlying data structures
861     * based upon current sax flow
862     * position/set class fields and
863     * determines whether an oai record should be
864     * harvested
865     *
866     * @return boolean indicates whether the record should be handled
867     */
868 rpandey 1.22 protected boolean shouldHarvestDocument() {
869     boolean ret = true;
870     try {
871     String verb = super.requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, null);
872     Parameters params = new Parameters();
873     params.setParameter(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL, super.repoUrl);
874     params.setParameter(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb);
875 rpandey 1.33 String[] dbes = this._database.search(params);
876 rpandey 1.22 // bq.add(new TermQuery(new Term(OAI_VERB, super.requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, OAIRequest.VERB_STRING_LIST_RECORDS))), false, false);
877     // bq.add(new TermQuery(new Term(OAI_VERB, super.requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, OAIRequest.VERB_STRING_GET_RECORD))), false, false);
878     //this harvest is a result of a resumptionToken request
879     // if (Utilities.checkString(resTok))
880     // bq.add(new TermQuery(new Term(OAI_HARVESTER_RESUMPTION_TOKEN, resTok)), true, false);
881     // Hits hits = getSearchIndex().search(bq);
882     if (dbes == null || dbes.length == 0)
883     return ret;//no harvest executed as yet
884     else {//we'll do comparisons on harvested data
885     for (int i = 0; i < dbes.length; i++) {
886     String dbeId = dbes[i];
887 rpandey 1.33 DatabaseEntity dbe = this._database.getEntity(dbeId);
888 rpandey 1.22 /*TODO: determine if this is really necessary as the following comparisons, in theory, are sufficient
889     //first comparison is based on a previously received resumption token
890     String resTok = super.requestParams.getParameter(OAIRequest.URL_PARAM_NAME_RESUMPTION_TOKEN, "");
891     String dbeResTok = dbe.getProperty(OAI_HARVESTER_RESUMPTION_TOKEN);
892     if (Utilities.checkString(dbeResTok) && Utilities.checkString(resTok)) {
893     if (resTok.equals(dbeResTok))
894     return ret;
895     }*/
896    
897 jlecren 1.43 //second comparison: we make a query to see if a document with this identifier exists
898 rpandey 1.22 if (Utilities.checkString(super.currentOaiIdentifier)) {
899 rpandey 1.23 DatabaseEntity docDbe = ((SDXDocumentBase) this.docbase).getDatabase().getEntity(super.currentOaiIdentifier);
900 rpandey 1.22 if (docDbe == null)
901     return ret;//no document with this id exists, so we should harvest it
902     }
903    
904     //third comparison will be based upon the timestamp of the current document vs. the time of the last harvest
905     //for ListRecords or GetRecords
906     String formattedDate = dbe.getProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED);
907     java.util.Date lastUpdatedDate = fr.gouv.culture.sdx.utils.Date.parseDate(formattedDate);
908     java.util.Date currentDocumentDate = fr.gouv.culture.sdx.utils.Date.parseDate(super.currentDatestamp);
909     if (currentDocumentDate.getTime() > lastUpdatedDate.getTime())
910     return ret;
911    
912     /*document is being delivered via a previously received resumpToken
913     *or
914     *document has not changed since last harvest withing the respective repository
915     *or
916     *the document exists and is upToDate
917     */
918 rpandey 1.33
919 rpandey 1.22 ret = false;
920     }
921     }
922     } catch (SDXException e) {
923 rpandey 1.26 LoggingUtils.logException(logger, e);
924 rpandey 1.22 }
925 rpandey 1.33
926     return ret;
927 rpandey 1.22 }
928    
929 rpandey 1.23 /**Saves critical data about a harvest
930     *
931     * @param dataHarvested
932     * @throws SAXException
933     */
934 rpandey 1.22 protected void saveCriticalFields(boolean dataHarvested) throws SAXException {
935     if (dataHarvested) {//if a harvest took place we store the details
936     try {
937     String harvestId = generateNewHarvestId();
938    
939     DatabaseEntity dbe = new DatabaseEntity(harvestId);//this will be changed later
940    
941     if (Utilities.checkString(super.repoUrl))
942     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_REQUEST_URL, super.requestUrl);
943    
944     if (Utilities.checkString(super.repoUrl))
945     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL, super.repoUrl);
946    
947     if (requestParams != null) {
948    
949     String verb = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, null);
950     if (Utilities.checkString(verb)) {
951    
952     String mdPrefix = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_METADATA_PREFIX, null);
953    
954     if (verb.equals(OAIRequest.VERB_STRING_GET_RECORD)) {
955    
956     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb);
957    
958     if (Utilities.checkString(mdPrefix))
959     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_METADATA_PREFIX, mdPrefix);
960    
961     String id = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_IDENTIFIER, null);
962     if (Utilities.checkString(id))
963     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_IDENTIFIER, id);
964     }
965    
966     if (verb.equals(OAIRequest.VERB_STRING_LIST_RECORDS)) {
967    
968     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb);
969    
970     if (Utilities.checkString(mdPrefix))
971     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_METADATA_PREFIX, mdPrefix);
972    
973     String from = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_FROM, null);
974     if (Utilities.checkString(from))
975     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_FROM, from);
976    
977     String until = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_UNTIL, null);
978     if (Utilities.checkString(until))
979     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_UNTIL, until);
980    
981     String set = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_SET, null);
982     if (Utilities.checkString(set))
983     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_SET, set);
984    
985     }
986    
987     }
988     }
989    
990    
991     dbe.setId(harvestId);
992     //deleting old info.
993     //adding latest info.
994 rpandey 1.23 if (Utilities.checkString(super.responseDate))
995     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED, super.responseDate);
996 rpandey 1.22 if (Utilities.checkString(super.resumptionToken))
997     dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_RESUMPTION_TOKEN, super.resumptionToken);
998     if (deletedDocs != null)
999     dbe.addProperty(LuceneDocumentBaseOAIHarvester.NO_DOCS_DELETED, Integer.toString(this.deletedDocs.size()));
1000 rpandey 1.26 dbe.addProperty(LuceneDocumentBaseOAIHarvester.NO_DOCS_HARVESTED, Integer.toString(this.noHarvestedDocs));
1001 rpandey 1.22
1002 rpandey 1.33 this._database.update(dbe);
1003     this._database.optimize();
1004 rpandey 1.22
1005     } catch (Exception e) {
1006     throw new SAXException(e.getMessage(), e);
1007     }
1008     }
1009     }
1010    
1011 rpandey 1.23 /**Generates an id to associate
1012     * with a harvest
1013     *
1014     * @return
1015     */
1016 rpandey 1.22 protected String generateNewHarvestId() {
1017     return this.harvesterIdGen.generate();
1018     }
1019    
1020 rpandey 1.23 /**Sends sax events to the current consumer
1021     * with summary details of the all the past harvests
1022     *
1023     * @throws SAXException
1024     */
1025 rpandey 1.22 public void sendPastHarvestsSummary() throws SAXException {
1026     //build a query get the doc
1027     try {
1028 rpandey 1.33 DatabaseEntity[] dbes = this._database.getEntities();
1029 rpandey 1.22 if (dbes != null && dbes.length > 0) {
1030     this.acquire();
1031     acquireSynchronizedXMLConsumer();
1032     super.startElement(Framework.SDXNamespaceURI, fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS,
1033     Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS, null);
1034     for (int i = 0; i < dbes.length; i++) {
1035 rpandey 1.24 DatabaseEntity dbe = dbes[i];
1036 rpandey 1.22 AttributesImpl atts = new AttributesImpl();
1037     String repoUrl = null;
1038     if (dbe != null) {
1039     Property[] props = dbe.getProperties();
1040     if (props != null) {
1041     for (int j = 0; j < props.length; j++) {
1042     Property prop = props[j];
1043     if (prop != null) {
1044     String propName = prop.getName();
1045     if (Utilities.checkString(propName) && !LuceneDocumentBaseOAIHarvester.OAI_HARVEST_ID.equals(propName)) {
1046     String propVal = prop.getValue();
1047     if (propName.equals(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL)) {
1048     repoUrl = propVal;
1049     } else {
1050     if (Utilities.checkString(propVal))
1051     atts.addAttribute("", propName, propName, OAIObject.Node.Type.CDATA, propVal);
1052     }
1053     }
1054     }
1055    
1056     }
1057     }
1058    
1059     }
1060     sendElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.REQUEST, OAIObject.Node.Name.REQUEST,
1061     atts, repoUrl);
1062     }
1063     super.endElement(Framework.SDXNamespaceURI, fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS,
1064     Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS);
1065     }
1066    
1067     } catch (/*IO*/Exception e) {
1068     throw new SAXException(e.getMessage(), e);
1069 rpandey 1.27 } /* Utilities.logException(super.getLog(), e);TODO: what is the best way to handle this?
1070 rpandey 1.22 } catch (SDXException e) {
1071     try {
1072     e.toSAX(this);
1073     } catch (ProcessingException e1) {
1074 rpandey 1.27 Utilities.logException(super.getLog(), e1);
1075 rpandey 1.22 }
1076     } */ finally {
1077     releaseSynchronizedXMLConsumer();
1078     this.release();
1079     }
1080    
1081    
1082     }
1083    
1084 rpandey 1.23 /**Retrieves the time when the harvester was last updated
1085     *
1086     * @return
1087     */
1088 rpandey 1.22 public Date lastUpdated() {
1089 rpandey 1.23 return this.docbase.lastModificationDate();
1090 rpandey 1.22 }
1091    
1092 rpandey 1.23 /**Destroys all summary data pertaining to past harvests
1093     * but not the actual oai records harvested
1094     *
1095     */
1096 rpandey 1.22 public void purgePastHarvestsData() {
1097     try {
1098 rpandey 1.33 DatabaseEntity[] dbes = this._database.getEntities();
1099 rpandey 1.22 for (int i = 0; i < dbes.length; i++) {
1100 rpandey 1.24 DatabaseEntity dbe = dbes[i];
1101 rpandey 1.33 this._database.delete(dbe);
1102 rpandey 1.22 }
1103 rpandey 1.33 this._database.optimize();
1104 rpandey 1.22 } catch (SDXException e) {
1105 rpandey 1.26 LoggingUtils.logException(logger, e);
1106 rpandey 1.24 }
1107     }
1108    
1109     /**Stores data about harvesting failures caused
1110     * by problems other than oai errors sent from
1111     * a queried repository
1112     *
1113     * @param e
1114     */
1115     protected void storeFailedHarvestData(Exception e) {
1116     try {
1117     //TODO: call this method when harvesting fails/major exception thrown
1118     //create a special property "failedHarvest"
1119     DatabaseEntity dbe = new DatabaseEntity(generateNewHarvestId());
1120     String message = "noMessage";
1121     if (e != null) message = e.getMessage();
1122     dbe.addProperty(OAI_FAILED_HARVEST, message);
1123 rpandey 1.33 //create a _database entity and store all relevant class fields
1124 rpandey 1.24 dbe.addProperty(OAI_REQUEST_URL, this.requestUrl);
1125     //save it to the database
1126 rpandey 1.33 this._database.update(dbe);
1127 rpandey 1.37 this._database.optimize();
1128 rpandey 1.24 //TODO:later perhaps we can try to auto-reexecute these failed request, keying on the OAI_FAILED_HARVEST property ???
1129     } catch (SDXException e1) {
1130 rpandey 1.26 LoggingUtils.logException(logger, e1);
1131 rpandey 1.22 }
1132 rpandey 1.1 }
1133    
1134    
1135     }
1136    
1137    

savannah-hackers-public@gnu.org
ViewVC Help
Powered by ViewVC 1.1.26