1 |
rpandey |
1.1 |
/* |
2 |
|
|
SDX: Documentary System in XML. |
3 |
|
|
Copyright (C) 2000, 2001, 2002 Ministere de la culture et de la communication (France), AJLSM |
4 |
|
|
|
5 |
|
|
Ministere de la culture et de la communication, |
6 |
|
|
Mission de la recherche et de la technologie |
7 |
|
|
3 rue de Valois, 75042 Paris Cedex 01 (France) |
8 |
|
|
mrt@culture.fr, michel.bottin@culture.fr |
9 |
|
|
|
10 |
|
|
AJLSM, 17, rue Vital Carles, 33000 Bordeaux (France) |
11 |
|
|
sevigny@ajlsm.com |
12 |
|
|
|
13 |
|
|
This program is free software; you can redistribute it and/or |
14 |
|
|
modify it under the terms of the GNU General Public License |
15 |
|
|
as published by the Free Software Foundation; either version 2 |
16 |
|
|
of the License, or (at your option) any later version. |
17 |
|
|
|
18 |
|
|
This program is distributed in the hope that it will be useful, |
19 |
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of |
20 |
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
21 |
|
|
See the GNU General Public License for more details. |
22 |
|
|
|
23 |
|
|
You should have received a copy of the GNU General Public License |
24 |
|
|
along with this program; if not, write to the |
25 |
|
|
Free Software Foundation, Inc. |
26 |
|
|
59 Temple Place - Suite 330, Boston, MA 02111-1307, USA |
27 |
|
|
or connect to: |
28 |
|
|
http://www.fsf.org/copyleft/gpl.html |
29 |
|
|
*/ |
30 |
|
|
package fr.gouv.culture.sdx.oai; |
31 |
|
|
|
32 |
rpandey |
1.3 |
import fr.gouv.culture.oai.AbstractOAIHarvester; |
33 |
|
|
import fr.gouv.culture.oai.OAIObject; |
34 |
|
|
import fr.gouv.culture.oai.OAIRequest; |
35 |
|
|
import fr.gouv.culture.oai.OAIRequestImpl; |
36 |
rpandey |
1.1 |
import fr.gouv.culture.sdx.document.Document; |
37 |
|
|
import fr.gouv.culture.sdx.document.IndexableDocument; |
38 |
|
|
import fr.gouv.culture.sdx.document.XMLDocument; |
39 |
rpandey |
1.36 |
import fr.gouv.culture.sdx.documentbase.DocumentBase; |
40 |
|
|
import fr.gouv.culture.sdx.documentbase.IDGenerator; |
41 |
|
|
import fr.gouv.culture.sdx.documentbase.IndexParameters; |
42 |
|
|
import fr.gouv.culture.sdx.documentbase.SDXDocumentBase; |
43 |
rpandey |
1.1 |
import fr.gouv.culture.sdx.exception.SDXException; |
44 |
|
|
import fr.gouv.culture.sdx.exception.SDXExceptionCode; |
45 |
|
|
import fr.gouv.culture.sdx.framework.Framework; |
46 |
|
|
import fr.gouv.culture.sdx.pipeline.GenericPipeline; |
47 |
|
|
import fr.gouv.culture.sdx.pipeline.Pipeline; |
48 |
|
|
import fr.gouv.culture.sdx.repository.Repository; |
49 |
|
|
import fr.gouv.culture.sdx.utils.Utilities; |
50 |
rpandey |
1.30 |
import fr.gouv.culture.sdx.utils.configuration.ConfigurationUtils; |
51 |
rpandey |
1.34 |
import fr.gouv.culture.sdx.utils.constants.ContextKeys; |
52 |
rpandey |
1.19 |
import fr.gouv.culture.sdx.utils.database.Database; |
53 |
|
|
import fr.gouv.culture.sdx.utils.database.DatabaseBacked; |
54 |
rpandey |
1.22 |
import fr.gouv.culture.sdx.utils.database.DatabaseEntity; |
55 |
|
|
import fr.gouv.culture.sdx.utils.database.Property; |
56 |
rpandey |
1.26 |
import fr.gouv.culture.sdx.utils.logging.LoggingUtils; |
57 |
rpandey |
1.36 |
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.SimpleTimeScheduler; |
58 |
|
|
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeScheduler; |
59 |
|
|
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeTrigger; |
60 |
|
|
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeTriggerFactory; |
61 |
rpandey |
1.10 |
import fr.gouv.culture.util.apache.cocoon.xml.SynchronizedXMLConsumerWrapper; |
62 |
rpandey |
1.12 |
import org.apache.avalon.excalibur.io.FileUtil; |
63 |
larcaini |
1.38 |
import org.apache.excalibur.xml.sax.SAXParser; |
64 |
jlecren |
1.39 |
import org.apache.avalon.framework.service.ServiceException; |
65 |
rpandey |
1.1 |
import org.apache.avalon.framework.configuration.Configuration; |
66 |
|
|
import org.apache.avalon.framework.configuration.ConfigurationException; |
67 |
rpandey |
1.41 |
import org.apache.avalon.framework.configuration.DefaultConfiguration; |
68 |
rpandey |
1.35 |
import org.apache.avalon.framework.context.Context; |
69 |
rpandey |
1.30 |
import org.apache.avalon.framework.context.ContextException; |
70 |
rpandey |
1.22 |
import org.apache.avalon.framework.parameters.Parameters; |
71 |
rpandey |
1.12 |
import org.apache.cocoon.Constants; |
72 |
rpandey |
1.1 |
import org.apache.cocoon.ProcessingException; |
73 |
rpandey |
1.10 |
import org.apache.cocoon.serialization.XMLSerializer; |
74 |
rpandey |
1.6 |
import org.apache.cocoon.xml.IncludeXMLConsumer; |
75 |
rpandey |
1.10 |
import org.apache.cocoon.xml.XMLConsumer; |
76 |
|
|
import org.apache.cocoon.xml.XMLMulticaster; |
77 |
rpandey |
1.3 |
import org.xml.sax.Attributes; |
78 |
rpandey |
1.1 |
import org.xml.sax.SAXException; |
79 |
rpandey |
1.6 |
import org.xml.sax.helpers.AttributesImpl; |
80 |
rpandey |
1.1 |
|
81 |
rpandey |
1.12 |
import java.io.File; |
82 |
|
|
import java.io.FileOutputStream; |
83 |
|
|
import java.io.IOException; |
84 |
|
|
import java.net.MalformedURLException; |
85 |
rpandey |
1.1 |
import java.net.URL; |
86 |
|
|
import java.util.ArrayList; |
87 |
rpandey |
1.22 |
import java.util.Date; |
88 |
rpandey |
1.1 |
import java.util.Enumeration; |
89 |
|
|
import java.util.Hashtable; |
90 |
|
|
|
91 |
|
|
/** |
92 |
|
|
* Created by IntelliJ IDEA. |
93 |
|
|
* User: rpandey |
94 |
|
|
* Date: May 12, 2003 |
95 |
|
|
* Time: 12:32:16 PM |
96 |
|
|
* To change this template use Options | File Templates. |
97 |
|
|
*/ |
98 |
rpandey |
1.36 |
public abstract class AbstractDocumentBaseOAIHarvester extends AbstractOAIHarvester implements DocumentBaseOAIHarvester { |
99 |
rpandey |
1.1 |
|
100 |
rpandey |
1.23 |
/**The underlying document base*/ |
101 |
|
|
protected DocumentBase docbase = null; |
102 |
|
|
/**Id of the underlying document base*/ |
103 |
|
|
protected String docbaseId = ""; |
104 |
|
|
/**Pre-indexation pipeline*/ |
105 |
rpandey |
1.1 |
protected Pipeline pipe = null; |
106 |
rpandey |
1.23 |
/**Underlying database to store any info*/ |
107 |
rpandey |
1.33 |
protected Database _database = null; |
108 |
rpandey |
1.23 |
/**Requests in application.xconf*/ |
109 |
rpandey |
1.1 |
protected Hashtable storedRequests = null; |
110 |
rpandey |
1.23 |
/**References to the underlying documentbase's/application's repositories*/ |
111 |
rpandey |
1.1 |
protected Hashtable storeRepositoriesRefs = null; |
112 |
rpandey |
1.23 |
/**Time scheduler for stored requests*/ |
113 |
rpandey |
1.1 |
protected TimeScheduler scheduler = null; |
114 |
rpandey |
1.23 |
/**IDGenerator for this object*/ |
115 |
|
|
protected IDGenerator harvesterIdGen = null; |
116 |
|
|
|
117 |
|
|
//variables for sax stream handling |
118 |
|
|
protected String TEMPFILE_SUFFIX = ".sdx"; |
119 |
rpandey |
1.10 |
protected File tempDir = null; |
120 |
|
|
protected File harvestDoc = null; |
121 |
|
|
protected FileOutputStream fileOs = null; |
122 |
rpandey |
1.6 |
protected XMLDocument urlResource = null; |
123 |
rpandey |
1.1 |
protected ArrayList deletedDocs = null; |
124 |
rpandey |
1.26 |
protected int noHarvestedDocs = 0; |
125 |
rpandey |
1.19 |
|
126 |
rpandey |
1.23 |
protected boolean keepDeletedRecords = false;//Defaulted, we will delete "deletedRecords" keeping our harvester in sync with source repository |
127 |
|
|
protected int noRecordsPerBatch = OAIObject.NUMBER_RECORDS_PER_RESPONSE; |
128 |
|
|
|
129 |
|
|
//strings for internal database properties and sax output |
130 |
rpandey |
1.22 |
protected static final String OAI_HARVEST_ID = "oaiHarvestId"; |
131 |
rpandey |
1.24 |
//internal database property for failed harvests caused by internal errors( errors in this implementation and not OAI repository errors) |
132 |
|
|
protected static final String OAI_FAILED_HARVEST = "oaiFailedHarvest"; |
133 |
rpandey |
1.22 |
protected static final String OAI_HARVESTER_LAST_UPDATED = "oaiHarvesterLastUpdated"; |
134 |
|
|
protected static final String OAI_HARVESTER_RESUMPTION_TOKEN = "oaiHarvesterResumptionToken"; |
135 |
|
|
/*Field names for common request parameters*/ |
136 |
|
|
protected static final String OAI_VERB = "oaiVerb"; |
137 |
|
|
protected static final String OAI_IDENTIFIER = "oaiIdentifier"; |
138 |
|
|
protected static final String OAI_METADATA_PREFIX = "oaiMetadataPrefix"; |
139 |
|
|
protected static final String OAI_FROM = "oaiFrom"; |
140 |
|
|
protected static final String OAI_UNTIL = "oaiUntil"; |
141 |
|
|
protected static final String OAI_SET = "oaiSet"; |
142 |
|
|
protected static final String NO_DOCS_DELETED = "noDocDeleted"; |
143 |
|
|
protected static final String NO_DOCS_HARVESTED = "noDocHarvested"; |
144 |
rpandey |
1.1 |
|
145 |
|
|
|
146 |
rpandey |
1.23 |
/**Basic constructor*/ |
147 |
rpandey |
1.1 |
public AbstractDocumentBaseOAIHarvester(DocumentBase base) { |
148 |
rpandey |
1.23 |
this.docbase = base; |
149 |
|
|
if (this.docbase != null) |
150 |
|
|
this.docbaseId = this.docbase.getId(); |
151 |
rpandey |
1.1 |
} |
152 |
|
|
|
153 |
|
|
|
154 |
rpandey |
1.23 |
/**Configures this object*/ |
155 |
rpandey |
1.1 |
public void configure(Configuration configuration) throws ConfigurationException { |
156 |
|
|
|
157 |
rpandey |
1.36 |
super.userAgent = configuration.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.USER_AGENT, "SDX OAI Harvester"); |
158 |
|
|
this.keepDeletedRecords = configuration.getAttributeAsBoolean(DocumentBaseOAIHarvester.ConfigurationNode.KEEP_DELETED_RECORDS, this.keepDeletedRecords); |
159 |
|
|
this.noRecordsPerBatch = configuration.getAttributeAsInteger(DocumentBaseOAIHarvester.ConfigurationNode.NO_RECORDS_PER_BATCH, this.noRecordsPerBatch); |
160 |
uid67066 |
1.18 |
configureAdminEmails(configuration); |
161 |
rpandey |
1.1 |
this.configureDataProviders(configuration); |
162 |
|
|
this.configurePipeline(configuration); |
163 |
rpandey |
1.19 |
configureDatabase(configuration); |
164 |
rpandey |
1.21 |
configureHarvestIDGenerator(configuration); |
165 |
rpandey |
1.19 |
} |
166 |
|
|
|
167 |
rpandey |
1.21 |
|
168 |
rpandey |
1.23 |
/**Configures the internal database*/ |
169 |
rpandey |
1.19 |
protected void configureDatabase(Configuration configuration) throws ConfigurationException { |
170 |
|
|
DatabaseBacked internalDb = new DatabaseBacked(); |
171 |
|
|
try { |
172 |
rpandey |
1.27 |
internalDb.enableLogging(this.logger); |
173 |
rpandey |
1.33 |
internalDb.contextualize(Utilities.createNewReadOnlyContext(getContext())); |
174 |
jlecren |
1.39 |
internalDb.service(this.manager); |
175 |
rpandey |
1.19 |
internalDb.setId(getHarvesterId()); |
176 |
|
|
internalDb.configure(configuration); |
177 |
|
|
internalDb.init(); |
178 |
rpandey |
1.33 |
this._database = internalDb.getDatabase(); |
179 |
rpandey |
1.19 |
} catch (SDXException e) { |
180 |
rpandey |
1.25 |
throw new ConfigurationException(e.getMessage(), e); |
181 |
jlecren |
1.39 |
} catch (ServiceException e) { |
182 |
rpandey |
1.29 |
throw new ConfigurationException(e.getMessage(), e); |
183 |
|
|
} catch (ContextException e) { |
184 |
rpandey |
1.19 |
throw new ConfigurationException(e.getMessage(), e); |
185 |
|
|
} |
186 |
|
|
|
187 |
|
|
} |
188 |
|
|
|
189 |
rpandey |
1.23 |
/**Configures the id generator for harvests*/ |
190 |
rpandey |
1.21 |
protected void configureHarvestIDGenerator(Configuration configuration) throws ConfigurationException { |
191 |
rpandey |
1.26 |
this.harvesterIdGen = ConfigurationUtils.configureIDGenerator(this.logger, configuration); |
192 |
rpandey |
1.33 |
this.harvesterIdGen.setDatabase(this._database); |
193 |
rpandey |
1.21 |
} |
194 |
|
|
|
195 |
|
|
|
196 |
rpandey |
1.23 |
/**Returns an id for this harvester based upon the underlying document base id*/ |
197 |
rpandey |
1.19 |
protected String getHarvesterId() { |
198 |
rpandey |
1.23 |
String hid = ""; |
199 |
|
|
hid += Framework.SDXNamespacePrefix + "_"; |
200 |
|
|
hid += OAIObject.Node.Prefix.OAI + "_"; |
201 |
|
|
hid += "harvester" + "_"; |
202 |
|
|
hid += this.docbaseId; |
203 |
|
|
return hid; |
204 |
uid67066 |
1.18 |
} |
205 |
|
|
|
206 |
rpandey |
1.23 |
/**Configures a list of admin emails |
207 |
|
|
* can be sub-elements, a single attribute, |
208 |
|
|
* or both |
209 |
|
|
* |
210 |
|
|
* @param configuration |
211 |
|
|
* @throws ConfigurationException |
212 |
|
|
*/ |
213 |
uid67066 |
1.18 |
protected void configureAdminEmails(Configuration configuration) throws ConfigurationException { |
214 |
|
|
//configure the admin email |
215 |
|
|
ArrayList locAdminEmailsList = new ArrayList(); |
216 |
rpandey |
1.36 |
String firstAdminEmail = configuration.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.ADMIN_EMAIL, null); |
217 |
|
|
Configuration[] locAdminEmails = configuration.getChildren(DocumentBaseOAIHarvester.ConfigurationNode.ADMIN_EMAIL); |
218 |
uid67066 |
1.18 |
if (Utilities.checkString(firstAdminEmail)) |
219 |
|
|
locAdminEmailsList.add(firstAdminEmail); |
220 |
|
|
for (int i = 0; i < locAdminEmails.length; i++) { |
221 |
|
|
Configuration locAdminEmail = locAdminEmails[i]; |
222 |
|
|
if (locAdminEmail != null) { |
223 |
|
|
String value = locAdminEmail.getValue(); |
224 |
|
|
if (Utilities.checkString(value)) |
225 |
|
|
locAdminEmailsList.add(value); |
226 |
|
|
} |
227 |
|
|
} |
228 |
|
|
if (locAdminEmailsList.size() <= 0)//no admin email throw an error TODO:make this exception better |
229 |
rpandey |
1.36 |
ConfigurationUtils.checkConfAttributeValue(DocumentBaseOAIHarvester.ConfigurationNode.ADMIN_EMAIL, null, configuration.getLocation()); |
230 |
uid67066 |
1.18 |
|
231 |
|
|
super.adminEmails = (String[]) locAdminEmailsList.toArray(new String[0]); |
232 |
|
|
//releasing resources |
233 |
|
|
locAdminEmailsList.clear(); |
234 |
|
|
locAdminEmailsList = null; |
235 |
|
|
|
236 |
rpandey |
1.1 |
} |
237 |
|
|
|
238 |
rpandey |
1.23 |
/**Configures data providers info that can be reused |
239 |
|
|
* and from which requests can be automatically executed |
240 |
|
|
* |
241 |
|
|
* @param configuration |
242 |
|
|
* @throws ConfigurationException |
243 |
|
|
* @see #storedRequests |
244 |
|
|
*/ |
245 |
rpandey |
1.1 |
protected void configureDataProviders(Configuration configuration) throws ConfigurationException { |
246 |
|
|
if (configuration != null) { |
247 |
rpandey |
1.36 |
Configuration dataProvidersConf = configuration.getChild(DocumentBaseOAIHarvester.ConfigurationNode.OAI_DATA_PROVIDERS, false); |
248 |
rpandey |
1.1 |
if (dataProvidersConf != null) { |
249 |
rpandey |
1.36 |
Configuration[] repoRequestConfs = dataProvidersConf.getChildren(DocumentBase.ConfigurationNode.OAI_REPOSITORY); |
250 |
rpandey |
1.1 |
if (repoRequestConfs != null) { |
251 |
rpandey |
1.21 |
for (int x = 0; x < repoRequestConfs.length; x++) { |
252 |
|
|
Configuration repoRequestConf = repoRequestConfs[x]; |
253 |
rpandey |
1.1 |
if (repoRequestConf != null) { |
254 |
|
|
if (storedRequests == null) storedRequests = new Hashtable(); |
255 |
rpandey |
1.36 |
String repoUrl = repoRequestConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.URL); |
256 |
rpandey |
1.1 |
configureStoreRepositories(repoUrl, repoRequestConf); |
257 |
rpandey |
1.36 |
Configuration updateConf = repoRequestConf.getChild(DocumentBaseOAIHarvester.ConfigurationNode.UPDATE, false); |
258 |
|
|
Configuration[] verbConfs = repoRequestConf.getChildren(DocumentBaseOAIHarvester.ConfigurationNode.OAI_VERB); |
259 |
rpandey |
1.1 |
if (verbConfs != null) { |
260 |
rpandey |
1.21 |
for (int y = 0; y < verbConfs.length; y++) { |
261 |
rpandey |
1.28 |
try { |
262 |
|
|
OAIRequest request = null; |
263 |
|
|
Configuration verbConf = verbConfs[y]; |
264 |
|
|
if (verbConf != null) { |
265 |
rpandey |
1.36 |
String verb = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.NAME); |
266 |
|
|
String mdPrefix = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.METADATA_PREFIX); |
267 |
|
|
ConfigurationUtils.checkConfAttributeValue(DocumentBaseOAIHarvester.ConfigurationNode.METADATA_PREFIX, mdPrefix, verbConf.getLocation()); |
268 |
rpandey |
1.28 |
if (verb.equalsIgnoreCase(OAIRequest.VERB_STRING_GET_RECORD)) { |
269 |
|
|
verb = OAIRequest.VERB_STRING_GET_RECORD; |
270 |
rpandey |
1.36 |
Configuration[] idsConf = verbConf.getChildren(DocumentBaseOAIHarvester.ConfigurationNode.OAI_IDENTIFIER); |
271 |
rpandey |
1.28 |
if (idsConf != null) { |
272 |
|
|
for (int z = 0; z < idsConf.length; z++) { |
273 |
|
|
Configuration idConf = idsConf[z]; |
274 |
|
|
if (idConf != null) { |
275 |
|
|
String id = idConf.getValue(); |
276 |
|
|
request = new OAIRequestImpl(); |
277 |
|
|
request.enableLogging(this.logger); |
278 |
|
|
request.setRepositoryURL(repoUrl); |
279 |
|
|
request.setVerbString(verb); |
280 |
|
|
request.setMetadataPrefix(mdPrefix); |
281 |
|
|
request.setIdentifier(id); |
282 |
|
|
Utilities.isObjectUnique(storedRequests, request.getRequestURL(), request); |
283 |
|
|
storedRequests.put(request.getRequestURL(), request); |
284 |
|
|
configureUpdateTriggers(request.getRequestURL(), updateConf); |
285 |
|
|
} |
286 |
|
|
|
287 |
rpandey |
1.1 |
} |
288 |
|
|
} |
289 |
|
|
|
290 |
rpandey |
1.28 |
} else if (verb.equalsIgnoreCase(OAIRequest.VERB_STRING_LIST_RECORDS)) { |
291 |
|
|
verb = OAIRequest.VERB_STRING_LIST_RECORDS; |
292 |
rpandey |
1.36 |
String from = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.FROM, null); |
293 |
|
|
String until = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.UNTIL, null); |
294 |
|
|
String set = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.SET, null); |
295 |
rpandey |
1.28 |
request = new OAIRequestImpl(); |
296 |
|
|
request.enableLogging(this.logger); |
297 |
|
|
request.setRepositoryURL(repoUrl); |
298 |
|
|
request.setVerbString(verb); |
299 |
|
|
request.setMetadataPrefix(mdPrefix); |
300 |
|
|
request.setFrom(from); |
301 |
|
|
request.setUntil(until); |
302 |
|
|
request.setSetIdentifier(set); |
303 |
|
|
Utilities.isObjectUnique(storedRequests, request.getRequestURL(), request); |
304 |
|
|
this.storedRequests.put(request.getRequestURL(), request); |
305 |
|
|
configureUpdateTriggers(request.getRequestURL(), updateConf); |
306 |
|
|
} else//TODOException: |
307 |
|
|
throw new ConfigurationException("this verb action is not supported for harvesting : " + verb); |
308 |
|
|
} |
309 |
|
|
} catch (ConfigurationException e) { |
310 |
|
|
LoggingUtils.logWarn(this.logger, e.getMessage(), e);//logging this so that all configs don't fail |
311 |
|
|
} catch (SDXException e) { |
312 |
|
|
LoggingUtils.logWarn(this.logger, e.getMessage(), e);//logging this so that all configs don't fail |
313 |
rpandey |
1.1 |
} |
314 |
|
|
|
315 |
|
|
|
316 |
|
|
} |
317 |
|
|
} |
318 |
|
|
|
319 |
|
|
} |
320 |
|
|
|
321 |
|
|
|
322 |
|
|
} |
323 |
|
|
if (this.scheduler != null) this.scheduler.start(); |
324 |
|
|
} |
325 |
|
|
|
326 |
|
|
} |
327 |
|
|
|
328 |
|
|
} |
329 |
|
|
} |
330 |
|
|
|
331 |
rpandey |
1.23 |
/**Configures time triggers for |
332 |
|
|
* stored requests |
333 |
|
|
* |
334 |
|
|
* @param requestUrl The request url |
335 |
|
|
* @param updateConf The configuration for updates |
336 |
|
|
* @throws ConfigurationException |
337 |
|
|
* @see #scheduler, #storedRequests |
338 |
|
|
*/ |
339 |
rpandey |
1.1 |
protected void configureUpdateTriggers(String requestUrl, Configuration updateConf) throws ConfigurationException { |
340 |
|
|
if (Utilities.checkString(requestUrl) && updateConf != null) { |
341 |
|
|
TimeTrigger trigger = new TimeTriggerFactory().createTimeTrigger(updateConf); |
342 |
|
|
if (trigger != null) { |
343 |
|
|
if (this.scheduler == null) this.scheduler = new SimpleTimeScheduler(); |
344 |
|
|
this.scheduler.addTrigger(requestUrl, trigger, this); |
345 |
|
|
} |
346 |
|
|
} |
347 |
|
|
} |
348 |
|
|
|
349 |
rpandey |
1.23 |
|
350 |
|
|
/**Configures the repositories |
351 |
|
|
* to which data will be stored |
352 |
|
|
* based upon their repository url |
353 |
|
|
* |
354 |
|
|
* @param repoUrl The repository/data provider url |
355 |
|
|
* @param oaiRepoConf The configuration |
356 |
|
|
* @throws ConfigurationException |
357 |
|
|
*/ |
358 |
rpandey |
1.1 |
protected void configureStoreRepositories(String repoUrl, Configuration oaiRepoConf) throws ConfigurationException { |
359 |
|
|
if (Utilities.checkString(repoUrl)) { |
360 |
rpandey |
1.36 |
String ref = oaiRepoConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.SDX_REPOSITORY, null); |
361 |
rpandey |
1.28 |
/*check for the sdxrepository attribute, if it exists, get the repository object and add it to the local hashtable*/ |
362 |
|
|
if (Utilities.checkString(ref)) { |
363 |
|
|
try { |
364 |
|
|
Repository repo = null; |
365 |
rpandey |
1.35 |
Context appRepos = (Context) getContext().get(ContextKeys.SDX.Application.REPOSITORIES); |
366 |
rpandey |
1.28 |
if (appRepos != null) |
367 |
|
|
repo = (Repository) appRepos.get(ref); |
368 |
|
|
if (repo == null) |
369 |
rpandey |
1.23 |
repo = this.docbase.getRepository(ref); |
370 |
rpandey |
1.28 |
if (this.storeRepositoriesRefs == null) this.storeRepositoriesRefs = new Hashtable(); |
371 |
|
|
//populating the hashtable |
372 |
|
|
Utilities.isObjectUnique(this.storeRepositoriesRefs, repoUrl, repo); |
373 |
|
|
storeRepositoriesRefs.put(repoUrl, repo); |
374 |
|
|
} catch (SDXException e) { |
375 |
|
|
String[] args = new String[1]; |
376 |
|
|
args[0] = ref; |
377 |
|
|
SDXException sdxE = new SDXException(logger, SDXExceptionCode.ERROR_LOAD_REFERENCED_REPO, args, null); |
378 |
|
|
throw new ConfigurationException(sdxE.getMessage(), sdxE); |
379 |
rpandey |
1.33 |
} catch (ContextException e) { |
380 |
|
|
throw new ConfigurationException(e.getMessage(), e); |
381 |
rpandey |
1.28 |
} |
382 |
rpandey |
1.1 |
} |
383 |
|
|
} |
384 |
|
|
|
385 |
|
|
|
386 |
|
|
} |
387 |
|
|
|
388 |
|
|
|
389 |
rpandey |
1.23 |
/**Configures the preIndexation pipeline |
390 |
|
|
* |
391 |
|
|
* @param configuration |
392 |
|
|
* @throws ConfigurationException |
393 |
|
|
* @see #pipe |
394 |
|
|
*/ |
395 |
rpandey |
1.1 |
protected void configurePipeline(Configuration configuration) throws ConfigurationException { |
396 |
|
|
//at this point, we should have a <sdx:pipeline> element |
397 |
rpandey |
1.30 |
Configuration pipeConf = configuration.getChild(Utilities.getElementName(Pipeline.CLASS_NAME_SUFFIX), false); |
398 |
rpandey |
1.1 |
//testing if we have something |
399 |
|
|
if (pipeConf != null) { |
400 |
|
|
//creating the pipeline and assigning the class field |
401 |
|
|
this.pipe = new GenericPipeline(); |
402 |
rpandey |
1.27 |
//setting the super.getLog() for the pipeline object |
403 |
rpandey |
1.1 |
this.pipe.enableLogging(this.logger); |
404 |
jlecren |
1.39 |
//giving the object the cocoon service manager |
405 |
rpandey |
1.1 |
try { |
406 |
jlecren |
1.40 |
this.pipe.service(this.manager); |
407 |
jlecren |
1.39 |
} catch (ServiceException e) { |
408 |
rpandey |
1.1 |
throw new ConfigurationException(e.getMessage(), e); |
409 |
|
|
} |
410 |
|
|
|
411 |
|
|
//configuring the pipeline object from 'application.xconf' |
412 |
|
|
this.pipe.configure(pipeConf); |
413 |
|
|
} |
414 |
|
|
} |
415 |
|
|
|
416 |
rpandey |
1.23 |
/**Creates a new temporary directory for |
417 |
|
|
* writing harvested records before the will |
418 |
|
|
* be indexed |
419 |
|
|
* @return |
420 |
|
|
* @throws SDXException |
421 |
|
|
* @throws IOException |
422 |
|
|
*/ |
423 |
|
|
protected File getNewTempDir() throws SDXException, IOException { |
424 |
rpandey |
1.33 |
File ret = null; |
425 |
|
|
try { |
426 |
|
|
ret = (File) getContext().get(Constants.CONTEXT_UPLOAD_DIR); |
427 |
|
|
} catch (ContextException e) { |
428 |
|
|
throw new SDXException(logger, SDXExceptionCode.ERROR_GENERIC, null, e);//TODOException: better message here |
429 |
|
|
} |
430 |
rpandey |
1.34 |
String childDir = Utilities.getStringFromContext(ContextKeys.SDX.Application.DIRECTORY_NAME, getContext()) + "_oaiHarvests" + File.separator + this.docbaseId + File.separator + "harvest-" + Utilities.encodeURL(getIsoDate(), null); |
431 |
rpandey |
1.23 |
if (Utilities.checkString(super.resumptionToken)) |
432 |
|
|
childDir += "-" + OAIObject.Node.Name.RESUMPTION_TOKEN + "-" + super.resumptionToken; |
433 |
|
|
if (ret.canWrite()) |
434 |
|
|
ret = new File(ret, childDir); |
435 |
|
|
else |
436 |
|
|
ret = new File(Utilities.getSystemTempDir(), childDir); |
437 |
|
|
Utilities.checkDirectory(ret.getCanonicalPath(), logger); |
438 |
|
|
return ret; |
439 |
|
|
} |
440 |
|
|
|
441 |
|
|
/**Deletes the directory |
442 |
|
|
* represented by the tempDir |
443 |
|
|
* class field |
444 |
|
|
* |
445 |
|
|
*/ |
446 |
|
|
protected void deleteTempDir() { |
447 |
|
|
if (tempDir != null) { |
448 |
|
|
try { |
449 |
|
|
FileUtil.deleteDirectory(this.tempDir);//deleting the old tempdir if possible |
450 |
|
|
} catch (IOException e) { |
451 |
rpandey |
1.26 |
LoggingUtils.logWarn(logger, e.getMessage(), e); |
452 |
rpandey |
1.23 |
} |
453 |
|
|
} |
454 |
|
|
} |
455 |
|
|
|
456 |
|
|
/**Establishes the tempDir class field |
457 |
|
|
* |
458 |
|
|
* @throws SDXException |
459 |
|
|
* @throws IOException |
460 |
|
|
*/ |
461 |
|
|
protected void initTempDir() throws SDXException, IOException { |
462 |
|
|
this.tempDir = getNewTempDir(); |
463 |
|
|
} |
464 |
|
|
|
465 |
|
|
/**Get's the current date in iso8601 format |
466 |
|
|
* |
467 |
|
|
* @return |
468 |
|
|
*/ |
469 |
|
|
protected String getIsoDate() { |
470 |
|
|
return fr.gouv.culture.sdx.utils.Date.formatDate(fr.gouv.culture.sdx.utils.Date.getUtcIso8601Date()); |
471 |
|
|
} |
472 |
|
|
|
473 |
rpandey |
1.1 |
|
474 |
rpandey |
1.23 |
/**Sets up resources to capture an oai record |
475 |
|
|
* |
476 |
|
|
* |
477 |
|
|
* @throws SAXException |
478 |
|
|
*/ |
479 |
rpandey |
1.12 |
protected void prepareRecordCapture() throws SAXException { |
480 |
rpandey |
1.22 |
try { |
481 |
|
|
if (this.tempDir == null || !this.tempDir.exists()) initTempDir(); |
482 |
rpandey |
1.23 |
this.harvestDoc = File.createTempFile("oaiHarvestedRecord", TEMPFILE_SUFFIX, tempDir); |
483 |
rpandey |
1.22 |
this.fileOs = new FileOutputStream(this.harvestDoc); |
484 |
|
|
XMLSerializer cBytes = new XMLSerializer(); |
485 |
rpandey |
1.42 |
cBytes.enableLogging(logger); |
486 |
rpandey |
1.41 |
cBytes.configure(new DefaultConfiguration("hack", "")); |
487 |
rpandey |
1.22 |
cBytes.setOutputStream(this.fileOs); |
488 |
|
|
XMLConsumer newConsumer = null; |
489 |
|
|
if (firstXmlConsumer != null) |
490 |
|
|
newConsumer = new XMLMulticaster(cBytes, firstXmlConsumer); |
491 |
|
|
else |
492 |
|
|
newConsumer = cBytes; |
493 |
rpandey |
1.23 |
|
494 |
|
|
super.setSynchronizedConsumer(new SynchronizedXMLConsumerWrapper(newConsumer)); |
495 |
rpandey |
1.22 |
} catch (IOException e) { |
496 |
|
|
throw new SAXException(e.getMessage(), e); |
497 |
|
|
} catch (SDXException e) { |
498 |
|
|
throw new SAXException(e.getMessage(), e); |
499 |
rpandey |
1.41 |
} catch (ConfigurationException e) { |
500 |
|
|
throw new SAXException(e.getMessage(), e); |
501 |
rpandey |
1.10 |
} |
502 |
rpandey |
1.22 |
|
503 |
rpandey |
1.10 |
} |
504 |
rpandey |
1.16 |
|
505 |
rpandey |
1.23 |
/**Ends the capture of an oai record |
506 |
|
|
* and renames the file written in cocoon's |
507 |
|
|
* work directory to correspond to it's id |
508 |
|
|
* TODO: is this a bad idea, id's could be long? |
509 |
|
|
* |
510 |
|
|
* @throws Exception |
511 |
|
|
*/ |
512 |
rpandey |
1.12 |
protected void captureRecord() throws Exception { |
513 |
rpandey |
1.22 |
if (this.fileOs != null && this.harvestDoc != null) { |
514 |
rpandey |
1.10 |
this.fileOs.flush(); |
515 |
|
|
this.fileOs.close(); |
516 |
rpandey |
1.36 |
String encoding = Utilities.getStringFromContext(ContextKeys.SDX.ENCODING, getContext()); |
517 |
uid67066 |
1.18 |
File newHarvestDoc = new File(tempDir, Utilities.encodeURL(super.currentOaiIdentifier, encoding) + TEMPFILE_SUFFIX); |
518 |
rpandey |
1.19 |
this.harvestDoc.renameTo(newHarvestDoc);//TODO:Exception if this fails |
519 |
rpandey |
1.22 |
resetRecordCaptureFields(false); |
520 |
|
|
} |
521 |
|
|
} |
522 |
|
|
|
523 |
rpandey |
1.23 |
/**Resets the class fields for record capture |
524 |
|
|
* possibility deleting the current <code>harvetDoc</code> |
525 |
|
|
* object underlying file |
526 |
|
|
* |
527 |
|
|
* @param deleteDoc flag for deletion of actual file |
528 |
|
|
*/ |
529 |
rpandey |
1.22 |
protected void resetRecordCaptureFields(boolean deleteDoc) { |
530 |
|
|
if (this.fileOs != null) { |
531 |
|
|
try { |
532 |
|
|
this.fileOs.flush(); |
533 |
|
|
this.fileOs.close(); |
534 |
|
|
this.fileOs = null; |
535 |
|
|
} catch (IOException e) { |
536 |
rpandey |
1.26 |
LoggingUtils.logException(logger, e); |
537 |
rpandey |
1.22 |
} |
538 |
|
|
} |
539 |
|
|
|
540 |
|
|
if (this.harvestDoc != null) { |
541 |
|
|
if (deleteDoc) |
542 |
|
|
this.harvestDoc.delete(); |
543 |
rpandey |
1.13 |
this.harvestDoc = null; |
544 |
rpandey |
1.1 |
} |
545 |
rpandey |
1.22 |
|
546 |
|
|
super.setConsumer(super.firstXmlConsumer);//resetting the consumer to the first external consumer |
547 |
rpandey |
1.1 |
} |
548 |
|
|
|
549 |
rpandey |
1.23 |
|
550 |
|
|
/**Sets up resources to delete an oai record |
551 |
|
|
* |
552 |
|
|
*/ |
553 |
|
|
protected void prepareRecordForDeletion() { |
554 |
|
|
if (this.deletedDocs == null) this.deletedDocs = new ArrayList(); |
555 |
|
|
if (Utilities.checkString(this.currentOaiIdentifier)) { |
556 |
|
|
XMLDocument deleteDoc = null; |
557 |
|
|
try { |
558 |
|
|
deleteDoc = new XMLDocument(this.currentOaiIdentifier); |
559 |
|
|
} catch (SDXException e) { |
560 |
|
|
//do nothing here |
561 |
|
|
} |
562 |
|
|
if (deleteDoc != null) this.deletedDocs.add(deleteDoc); |
563 |
|
|
} |
564 |
|
|
} |
565 |
|
|
|
566 |
|
|
|
567 |
|
|
/**Reads the documents from <code>tempDir</code> |
568 |
|
|
* and indexes them in the corresponding document |
569 |
|
|
* base, any marked deletions will be carried out |
570 |
|
|
* as well |
571 |
|
|
* |
572 |
|
|
* @return |
573 |
|
|
* @throws SDXException |
574 |
|
|
* @throws SAXException |
575 |
|
|
* @throws ProcessingException |
576 |
|
|
*/ |
577 |
rpandey |
1.1 |
protected boolean storeHarvestedData() throws SDXException, SAXException, ProcessingException { |
578 |
|
|
boolean dataHarvested = false; |
579 |
rpandey |
1.23 |
if (docbase != null) { |
580 |
rpandey |
1.11 |
//deleting any docs which have been removed from the repo |
581 |
|
|
Document[] deleteDocs = null; |
582 |
|
|
if (this.deletedDocs != null && this.deletedDocs.size() > 0) |
583 |
|
|
deleteDocs = (Document[]) this.deletedDocs.toArray(new Document[0]); |
584 |
|
|
if (deleteDocs != null) { |
585 |
rpandey |
1.23 |
this.docbase.delete(deleteDocs, /*this.contentHandler*/this); |
586 |
rpandey |
1.11 |
dataHarvested = true; |
587 |
|
|
} |
588 |
|
|
|
589 |
rpandey |
1.1 |
IndexableDocument[] indexDocs = null; |
590 |
rpandey |
1.11 |
//creating our docs from the disk |
591 |
rpandey |
1.10 |
if (this.tempDir != null) { |
592 |
|
|
String[] files = this.tempDir.list(); |
593 |
rpandey |
1.26 |
this.noHarvestedDocs = 0; |
594 |
rpandey |
1.21 |
ArrayList docs = new ArrayList(); |
595 |
|
|
IndexParameters indexParams = getIndexParameters(); |
596 |
rpandey |
1.10 |
for (int i = 0; i < files.length; i++) { |
597 |
|
|
String fileName = files[i]; |
598 |
|
|
//TODO refactor this |
599 |
|
|
String absolute = new File(tempDir, fileName).getAbsolutePath(); |
600 |
|
|
StringBuffer url = new StringBuffer("file://"); |
601 |
rpandey |
1.11 |
for (int j = 0; j < absolute.length(); j++) { |
602 |
|
|
char c = absolute.charAt(j); |
603 |
|
|
if (c == File.separatorChar) |
604 |
|
|
url.append('/'); |
605 |
uid67066 |
1.18 |
else { |
606 |
rpandey |
1.36 |
String encoding = Utilities.getStringFromContext(ContextKeys.SDX.ENCODING, getContext()); |
607 |
uid67066 |
1.18 |
url.append(Utilities.encodeURL(String.valueOf(c), encoding)); |
608 |
|
|
} |
609 |
rpandey |
1.11 |
} |
610 |
rpandey |
1.10 |
XMLDocument metadataDoc = new XMLDocument(); |
611 |
|
|
String id = fileName.substring(0, fileName.lastIndexOf(TEMPFILE_SUFFIX)); |
612 |
|
|
fileName = null; |
613 |
|
|
//DEBUG System.out.println("a document id : " + id); |
614 |
uid67066 |
1.18 |
if (Utilities.checkString(id)) { |
615 |
rpandey |
1.36 |
String encoding = Utilities.getStringFromContext(ContextKeys.SDX.ENCODING, getContext()); |
616 |
uid67066 |
1.18 |
metadataDoc.setId(Utilities.decodeURL(id, encoding)); |
617 |
|
|
} |
618 |
rpandey |
1.10 |
try { |
619 |
|
|
metadataDoc.setContent(new URL(url.toString())); |
620 |
|
|
if (docs == null) docs = new ArrayList(); |
621 |
|
|
docs.add(metadataDoc); |
622 |
rpandey |
1.26 |
this.noHarvestedDocs++;//keeping track of all additions |
623 |
rpandey |
1.10 |
} catch (MalformedURLException e) { |
624 |
rpandey |
1.26 |
LoggingUtils.logException(logger, e); |
625 |
rpandey |
1.10 |
} |
626 |
rpandey |
1.11 |
//at every 1000th document we will index into the base and/or at the end of the group |
627 |
rpandey |
1.21 |
int modulusTopLimit = indexParams.getBatchMax() - 1; |
628 |
|
|
if (((i % (modulusTopLimit) == 0 || i == (files.length - 1)) && docs.size() > 0)) { |
629 |
rpandey |
1.15 |
indexDocs = (IndexableDocument[]) docs.toArray(new IndexableDocument[0]); |
630 |
rpandey |
1.11 |
if (indexDocs != null) { |
631 |
|
|
Repository repo = null; |
632 |
|
|
if (Utilities.checkString(this.repoUrl) && storeRepositoriesRefs != null) |
633 |
|
|
repo = (Repository) this.storeRepositoriesRefs.get(this.repoUrl); |
634 |
rpandey |
1.23 |
this.docbase.index(indexDocs, repo, indexParams, /*this.contentHandler*/this); |
635 |
rpandey |
1.11 |
dataHarvested = true; |
636 |
rpandey |
1.15 |
docs = null; |
637 |
rpandey |
1.11 |
indexDocs = null; |
638 |
|
|
} |
639 |
|
|
} |
640 |
rpandey |
1.10 |
} |
641 |
|
|
} |
642 |
rpandey |
1.1 |
|
643 |
rpandey |
1.11 |
|
644 |
rpandey |
1.1 |
} |
645 |
|
|
return dataHarvested; |
646 |
|
|
} |
647 |
|
|
|
648 |
rpandey |
1.23 |
/**Handles the resumption token by issuing another request |
649 |
|
|
* based upon the request from which the resumption token was received. |
650 |
|
|
* |
651 |
|
|
*/ |
652 |
rpandey |
1.1 |
protected void handleResumptionToken() { |
653 |
|
|
if (Utilities.checkString(super.resumptionToken) && Utilities.checkString(super.repoUrl)) { |
654 |
|
|
String verb = this.requestParams.getParameter(OAIObject.Node.Name.VERB, ""); |
655 |
|
|
if (Utilities.checkString(verb)) { |
656 |
|
|
String requestUrl = this.repoUrl + OAIRequest.URL_CHARACTER_QUESTION_MARK + OAIObject.Node.Name.VERB + OAIRequest.URL_CHARACTER_EQUALS + verb + OAIRequest.URL_CHARACTER_AMPERSAND + OAIObject.Node.Name.RESUMPTION_TOKEN + OAIRequest.URL_CHARACTER_EQUALS + super.resumptionToken; |
657 |
rpandey |
1.14 |
this.resetAllFields(); |
658 |
rpandey |
1.15 |
try { |
659 |
|
|
this.initTempDir();//trying to build a tempDir with the resumptionToken in the dir name |
660 |
|
|
} catch (SDXException e) { |
661 |
rpandey |
1.26 |
LoggingUtils.logException(logger, e);//if this fails we log it as there will be another opportunity during the harvest |
662 |
rpandey |
1.15 |
} catch (IOException e) { |
663 |
rpandey |
1.26 |
LoggingUtils.logException(logger, e); |
664 |
rpandey |
1.15 |
} |
665 |
rpandey |
1.1 |
super.resetResumptionToken();//resetting the resumption token here so we don't loop |
666 |
uid67066 |
1.18 |
this.receiveRequest(requestUrl); |
667 |
rpandey |
1.1 |
} |
668 |
|
|
} |
669 |
|
|
} |
670 |
|
|
|
671 |
rpandey |
1.23 |
//TODO-TEST: this should be preparing for a merge with the metadata, done |
672 |
|
|
/**Prepares to read a url value from an oai record and |
673 |
|
|
* retrieve the XML behind. |
674 |
|
|
*@see #identifierName, #currentMetadtaUrlIdentifier |
675 |
|
|
*/ |
676 |
|
|
protected void prepareResourceFromUrlIdentifierCapture() { |
677 |
rpandey |
1.1 |
if (Utilities.checkString(this.currentMetadtaUrlIdentifier)) { |
678 |
rpandey |
1.23 |
try { |
679 |
|
|
URL resourceUrl = new URL(this.currentMetadtaUrlIdentifier); |
680 |
|
|
XMLDocument resource = new XMLDocument(); |
681 |
|
|
resource.setId(this.currentMetadtaUrlIdentifier); |
682 |
|
|
resource.setContent(resourceUrl); |
683 |
|
|
this.urlResource = resource; |
684 |
|
|
} catch (MalformedURLException e) { |
685 |
rpandey |
1.26 |
LoggingUtils.logException(logger, e); |
686 |
rpandey |
1.23 |
} catch (SDXException e) { |
687 |
rpandey |
1.26 |
LoggingUtils.logException(logger, e); |
688 |
rpandey |
1.23 |
} |
689 |
rpandey |
1.6 |
/* |
690 |
rpandey |
1.1 |
if (docs == null) docs = new ArrayList(); |
691 |
|
|
docs.add(resource); |
692 |
rpandey |
1.6 |
*/ |
693 |
|
|
|
694 |
|
|
/*InputStream contents = resource.openStream(); |
695 |
|
|
byte[] chars = new byte[contents.available()]; |
696 |
|
|
contents.read(chars, 0, chars.length); |
697 |
|
|
contents.close(); |
698 |
|
|
docBytes. |
699 |
|
|
docBytes.write(chars);*/ |
700 |
|
|
|
701 |
|
|
} |
702 |
|
|
} |
703 |
|
|
|
704 |
rpandey |
1.23 |
/**Captures the xml from a url taken from an oai record and adds |
705 |
|
|
* it to the oai-record as a sibling of the <metadata/> element |
706 |
|
|
* |
707 |
|
|
*/ |
708 |
|
|
protected void captureResourceFromUrlIdentifier() { |
709 |
rpandey |
1.6 |
if (this.urlResource != null) { |
710 |
|
|
try { |
711 |
rpandey |
1.23 |
IncludeXMLConsumer include = new IncludeXMLConsumer(super.synchronizedXmlConsumer); |
712 |
|
|
urlResource.setConsumer(include); |
713 |
|
|
include.startElement(Framework.SDXNamespaceURI, "urlResource", "sdx:urlResource", new AttributesImpl()); |
714 |
larcaini |
1.38 |
SAXParser parser = null; |
715 |
rpandey |
1.23 |
try { |
716 |
larcaini |
1.38 |
parser = (SAXParser) this.manager.lookup(SAXParser.ROLE); |
717 |
rpandey |
1.23 |
urlResource.parse(parser); |
718 |
|
|
} finally { |
719 |
|
|
if (parser != null) |
720 |
|
|
this.manager.release(parser); |
721 |
|
|
this.urlResource = null; |
722 |
|
|
include.endElement(Framework.SDXNamespaceURI, "urlResource", "sdx:urlResource"); |
723 |
|
|
} |
724 |
|
|
} catch (SAXException e) { |
725 |
rpandey |
1.26 |
LoggingUtils.logException(logger, e); |
726 |
jlecren |
1.39 |
} catch (ServiceException e) { |
727 |
rpandey |
1.26 |
LoggingUtils.logException(logger, e); |
728 |
rpandey |
1.23 |
} catch (SDXException e) { |
729 |
rpandey |
1.26 |
LoggingUtils.logException(logger, e); |
730 |
rpandey |
1.6 |
} |
731 |
rpandey |
1.1 |
} |
732 |
rpandey |
1.6 |
|
733 |
rpandey |
1.1 |
} |
734 |
|
|
|
735 |
|
|
|
736 |
rpandey |
1.23 |
/**Resets necessary class fields |
737 |
|
|
* |
738 |
|
|
*/ |
739 |
rpandey |
1.1 |
protected void resetAllFields() { |
740 |
|
|
this.deletedDocs = null; |
741 |
rpandey |
1.26 |
this.noHarvestedDocs = 0; |
742 |
rpandey |
1.6 |
this.urlResource = null; |
743 |
rpandey |
1.22 |
resetRecordCaptureFields(false); |
744 |
rpandey |
1.10 |
deleteTempDir(); |
745 |
rpandey |
1.14 |
this.tempDir = null; |
746 |
rpandey |
1.1 |
} |
747 |
|
|
|
748 |
rpandey |
1.23 |
|
749 |
|
|
/**Builds simple index parameters for indexation of |
750 |
|
|
* oai records into the undelryi |
751 |
|
|
* @return |
752 |
|
|
*/ |
753 |
rpandey |
1.1 |
protected IndexParameters getIndexParameters() { |
754 |
|
|
IndexParameters params = new IndexParameters(); |
755 |
|
|
params.setSendIndexationEvents(IndexParameters.SEND_ALL_EVENTS); |
756 |
rpandey |
1.23 |
params.setBatchMax(this.noRecordsPerBatch); |
757 |
|
|
if (this.pipe != null && this.docbase != null) { |
758 |
|
|
this.pipe.setConsumer(this.docbase.getIndexationPipeline()); |
759 |
rpandey |
1.1 |
params.setPipeline(this.pipe); |
760 |
|
|
} |
761 |
rpandey |
1.17 |
params.setPipelineParams(super.getHarvestParameters()); |
762 |
rpandey |
1.1 |
return params; |
763 |
|
|
} |
764 |
|
|
|
765 |
rpandey |
1.23 |
/**Sends the details of stored harvesting requests |
766 |
|
|
* to the current consumer |
767 |
|
|
* |
768 |
|
|
* @throws SAXException |
769 |
|
|
*/ |
770 |
rpandey |
1.6 |
public void sendStoredHarvestingRequests() throws SAXException { |
771 |
rpandey |
1.4 |
try { |
772 |
rpandey |
1.6 |
if (this.storedRequests != null && this.storedRequests.size() > 0) { |
773 |
rpandey |
1.4 |
this.acquire(); |
774 |
|
|
super.acquireSynchronizedXMLConsumer(); |
775 |
|
|
super.startElement(Framework.SDXNamespaceURI, |
776 |
|
|
fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS, |
777 |
|
|
Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS, |
778 |
|
|
null); |
779 |
|
|
Enumeration requests = storedRequests.elements(); |
780 |
|
|
if (requests != null) { |
781 |
|
|
while (requests.hasMoreElements()) { |
782 |
|
|
OAIRequest request = (OAIRequest) requests.nextElement(); |
783 |
|
|
if (request != null) |
784 |
|
|
request.toSAX(this); |
785 |
|
|
} |
786 |
rpandey |
1.1 |
} |
787 |
rpandey |
1.4 |
|
788 |
|
|
super.endElement(Framework.SDXNamespaceURI, |
789 |
|
|
fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS, Framework.SDXNamespacePrefix + ":" + |
790 |
|
|
fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS); |
791 |
rpandey |
1.1 |
} |
792 |
rpandey |
1.4 |
} catch (InterruptedException e) { |
793 |
|
|
throw new SAXException(e.getMessage(), e); |
794 |
|
|
} finally { |
795 |
|
|
super.releaseSynchronizedXMLConsumer(); |
796 |
|
|
this.release(); |
797 |
rpandey |
1.1 |
} |
798 |
|
|
|
799 |
|
|
} |
800 |
|
|
|
801 |
|
|
|
802 |
rpandey |
1.23 |
/**Triggers a oai request to a repository based |
803 |
|
|
* upon a trigger name (also a request url) |
804 |
|
|
* |
805 |
|
|
* @param triggerName |
806 |
|
|
*/ |
807 |
rpandey |
1.1 |
public synchronized void targetTriggered(String triggerName) { |
808 |
rpandey |
1.3 |
//DEBUG System.out.println("i was triggered :" + triggerName); |
809 |
rpandey |
1.23 |
//verify that we have this trigger defined |
810 |
rpandey |
1.1 |
OAIRequest request = (OAIRequest) this.storedRequests.get(triggerName); |
811 |
|
|
if (request != null) { |
812 |
|
|
String requestUrl = request.getRequestURL(); |
813 |
rpandey |
1.17 |
/*TODO: query datastructs and get laste update time for this url so |
814 |
|
|
*that we harvest only what may be changed and add this time to the request |
815 |
|
|
*object |
816 |
|
|
*TODO:? but should allow force complete re-harvest? |
817 |
|
|
*/ |
818 |
uid67066 |
1.18 |
this.receiveSynchronizedRequest(requestUrl); |
819 |
rpandey |
1.3 |
//DEBUG System.out.println("i was requested :" + requestUrl); |
820 |
rpandey |
1.1 |
} |
821 |
|
|
|
822 |
|
|
|
823 |
|
|
} |
824 |
|
|
|
825 |
rpandey |
1.3 |
|
826 |
|
|
public void startElement(String s, String s1, String s2, Attributes attributes) throws SAXException { |
827 |
|
|
if (Utilities.checkString(this.identifierName)) { |
828 |
|
|
/*looking for an element name or the VALUE of an attribute with the name "name" matching super.indentifierName |
829 |
jlecren |
1.43 |
*so one could create a metadata identifier element like |
830 |
rpandey |
1.3 |
*<super.identifierName>myIdentifierValue</super.identifierName> |
831 |
|
|
*or |
832 |
|
|
*<anyElementName name="super.identifierName">myIdentifierValue</anyElementName> |
833 |
|
|
*/ |
834 |
rpandey |
1.4 |
if (attributes != null && this.identifierName.equals(attributes.getValue(fr.gouv.culture.sdx.utils.constants.Node.Name.NAME))) |
835 |
rpandey |
1.3 |
this.captureElemContent = true; |
836 |
|
|
} |
837 |
|
|
super.startElement(s, s1, s2, attributes); |
838 |
|
|
} |
839 |
|
|
|
840 |
|
|
public void endElement(String s, String s1, String s2) throws SAXException { |
841 |
|
|
if (super.sBuff != null && super.sBuff.length() > 0) { |
842 |
|
|
String content = super.sBuff.toString(); |
843 |
|
|
|
844 |
|
|
if (!OAIObject.Node.Xmlns.OAI_2_0.equals(s)) { |
845 |
|
|
if (fr.gouv.culture.sdx.utils.constants.Node.Name.FIELD.equals(s1)) { |
846 |
|
|
super.currentMetadtaUrlIdentifier = content; |
847 |
|
|
try { |
848 |
rpandey |
1.12 |
prepareResourceFromUrlIdentifierCapture(); |
849 |
rpandey |
1.3 |
} catch (Exception e) { |
850 |
|
|
//if we can't build the document we don't just fail completely we will continue |
851 |
rpandey |
1.26 |
LoggingUtils.logException(logger, e); |
852 |
rpandey |
1.3 |
} |
853 |
|
|
} |
854 |
|
|
} |
855 |
|
|
|
856 |
|
|
} |
857 |
|
|
super.endElement(s, s1, s2); |
858 |
rpandey |
1.22 |
} |
859 |
|
|
|
860 |
rpandey |
1.23 |
/**Querys the underlying data structures |
861 |
|
|
* based upon current sax flow |
862 |
|
|
* position/set class fields and |
863 |
|
|
* determines whether an oai record should be |
864 |
|
|
* harvested |
865 |
|
|
* |
866 |
|
|
* @return boolean indicates whether the record should be handled |
867 |
|
|
*/ |
868 |
rpandey |
1.22 |
protected boolean shouldHarvestDocument() { |
869 |
|
|
boolean ret = true; |
870 |
|
|
try { |
871 |
|
|
String verb = super.requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, null); |
872 |
|
|
Parameters params = new Parameters(); |
873 |
|
|
params.setParameter(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL, super.repoUrl); |
874 |
|
|
params.setParameter(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb); |
875 |
rpandey |
1.33 |
String[] dbes = this._database.search(params); |
876 |
rpandey |
1.22 |
// bq.add(new TermQuery(new Term(OAI_VERB, super.requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, OAIRequest.VERB_STRING_LIST_RECORDS))), false, false); |
877 |
|
|
// bq.add(new TermQuery(new Term(OAI_VERB, super.requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, OAIRequest.VERB_STRING_GET_RECORD))), false, false); |
878 |
|
|
//this harvest is a result of a resumptionToken request |
879 |
|
|
// if (Utilities.checkString(resTok)) |
880 |
|
|
// bq.add(new TermQuery(new Term(OAI_HARVESTER_RESUMPTION_TOKEN, resTok)), true, false); |
881 |
|
|
// Hits hits = getSearchIndex().search(bq); |
882 |
|
|
if (dbes == null || dbes.length == 0) |
883 |
|
|
return ret;//no harvest executed as yet |
884 |
|
|
else {//we'll do comparisons on harvested data |
885 |
|
|
for (int i = 0; i < dbes.length; i++) { |
886 |
|
|
String dbeId = dbes[i]; |
887 |
rpandey |
1.33 |
DatabaseEntity dbe = this._database.getEntity(dbeId); |
888 |
rpandey |
1.22 |
/*TODO: determine if this is really necessary as the following comparisons, in theory, are sufficient |
889 |
|
|
//first comparison is based on a previously received resumption token |
890 |
|
|
String resTok = super.requestParams.getParameter(OAIRequest.URL_PARAM_NAME_RESUMPTION_TOKEN, ""); |
891 |
|
|
String dbeResTok = dbe.getProperty(OAI_HARVESTER_RESUMPTION_TOKEN); |
892 |
|
|
if (Utilities.checkString(dbeResTok) && Utilities.checkString(resTok)) { |
893 |
|
|
if (resTok.equals(dbeResTok)) |
894 |
|
|
return ret; |
895 |
|
|
}*/ |
896 |
|
|
|
897 |
jlecren |
1.43 |
//second comparison: we make a query to see if a document with this identifier exists |
898 |
rpandey |
1.22 |
if (Utilities.checkString(super.currentOaiIdentifier)) { |
899 |
rpandey |
1.23 |
DatabaseEntity docDbe = ((SDXDocumentBase) this.docbase).getDatabase().getEntity(super.currentOaiIdentifier); |
900 |
rpandey |
1.22 |
if (docDbe == null) |
901 |
|
|
return ret;//no document with this id exists, so we should harvest it |
902 |
|
|
} |
903 |
|
|
|
904 |
|
|
//third comparison will be based upon the timestamp of the current document vs. the time of the last harvest |
905 |
|
|
//for ListRecords or GetRecords |
906 |
|
|
String formattedDate = dbe.getProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED); |
907 |
|
|
java.util.Date lastUpdatedDate = fr.gouv.culture.sdx.utils.Date.parseDate(formattedDate); |
908 |
|
|
java.util.Date currentDocumentDate = fr.gouv.culture.sdx.utils.Date.parseDate(super.currentDatestamp); |
909 |
|
|
if (currentDocumentDate.getTime() > lastUpdatedDate.getTime()) |
910 |
|
|
return ret; |
911 |
|
|
|
912 |
|
|
/*document is being delivered via a previously received resumpToken |
913 |
|
|
*or |
914 |
|
|
*document has not changed since last harvest withing the respective repository |
915 |
|
|
*or |
916 |
|
|
*the document exists and is upToDate |
917 |
|
|
*/ |
918 |
rpandey |
1.33 |
|
919 |
rpandey |
1.22 |
ret = false; |
920 |
|
|
} |
921 |
|
|
} |
922 |
|
|
} catch (SDXException e) { |
923 |
rpandey |
1.26 |
LoggingUtils.logException(logger, e); |
924 |
rpandey |
1.22 |
} |
925 |
rpandey |
1.33 |
|
926 |
|
|
return ret; |
927 |
rpandey |
1.22 |
} |
928 |
|
|
|
929 |
rpandey |
1.23 |
/**Saves critical data about a harvest |
930 |
|
|
* |
931 |
|
|
* @param dataHarvested |
932 |
|
|
* @throws SAXException |
933 |
|
|
*/ |
934 |
rpandey |
1.22 |
protected void saveCriticalFields(boolean dataHarvested) throws SAXException { |
935 |
|
|
if (dataHarvested) {//if a harvest took place we store the details |
936 |
|
|
try { |
937 |
|
|
String harvestId = generateNewHarvestId(); |
938 |
|
|
|
939 |
|
|
DatabaseEntity dbe = new DatabaseEntity(harvestId);//this will be changed later |
940 |
|
|
|
941 |
|
|
if (Utilities.checkString(super.repoUrl)) |
942 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_REQUEST_URL, super.requestUrl); |
943 |
|
|
|
944 |
|
|
if (Utilities.checkString(super.repoUrl)) |
945 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL, super.repoUrl); |
946 |
|
|
|
947 |
|
|
if (requestParams != null) { |
948 |
|
|
|
949 |
|
|
String verb = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, null); |
950 |
|
|
if (Utilities.checkString(verb)) { |
951 |
|
|
|
952 |
|
|
String mdPrefix = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_METADATA_PREFIX, null); |
953 |
|
|
|
954 |
|
|
if (verb.equals(OAIRequest.VERB_STRING_GET_RECORD)) { |
955 |
|
|
|
956 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb); |
957 |
|
|
|
958 |
|
|
if (Utilities.checkString(mdPrefix)) |
959 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_METADATA_PREFIX, mdPrefix); |
960 |
|
|
|
961 |
|
|
String id = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_IDENTIFIER, null); |
962 |
|
|
if (Utilities.checkString(id)) |
963 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_IDENTIFIER, id); |
964 |
|
|
} |
965 |
|
|
|
966 |
|
|
if (verb.equals(OAIRequest.VERB_STRING_LIST_RECORDS)) { |
967 |
|
|
|
968 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb); |
969 |
|
|
|
970 |
|
|
if (Utilities.checkString(mdPrefix)) |
971 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_METADATA_PREFIX, mdPrefix); |
972 |
|
|
|
973 |
|
|
String from = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_FROM, null); |
974 |
|
|
if (Utilities.checkString(from)) |
975 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_FROM, from); |
976 |
|
|
|
977 |
|
|
String until = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_UNTIL, null); |
978 |
|
|
if (Utilities.checkString(until)) |
979 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_UNTIL, until); |
980 |
|
|
|
981 |
|
|
String set = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_SET, null); |
982 |
|
|
if (Utilities.checkString(set)) |
983 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_SET, set); |
984 |
|
|
|
985 |
|
|
} |
986 |
|
|
|
987 |
|
|
} |
988 |
|
|
} |
989 |
|
|
|
990 |
|
|
|
991 |
|
|
dbe.setId(harvestId); |
992 |
|
|
//deleting old info. |
993 |
|
|
//adding latest info. |
994 |
rpandey |
1.23 |
if (Utilities.checkString(super.responseDate)) |
995 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED, super.responseDate); |
996 |
rpandey |
1.22 |
if (Utilities.checkString(super.resumptionToken)) |
997 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_RESUMPTION_TOKEN, super.resumptionToken); |
998 |
|
|
if (deletedDocs != null) |
999 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.NO_DOCS_DELETED, Integer.toString(this.deletedDocs.size())); |
1000 |
rpandey |
1.26 |
dbe.addProperty(LuceneDocumentBaseOAIHarvester.NO_DOCS_HARVESTED, Integer.toString(this.noHarvestedDocs)); |
1001 |
rpandey |
1.22 |
|
1002 |
rpandey |
1.33 |
this._database.update(dbe); |
1003 |
|
|
this._database.optimize(); |
1004 |
rpandey |
1.22 |
|
1005 |
|
|
} catch (Exception e) { |
1006 |
|
|
throw new SAXException(e.getMessage(), e); |
1007 |
|
|
} |
1008 |
|
|
} |
1009 |
|
|
} |
1010 |
|
|
|
1011 |
rpandey |
1.23 |
/**Generates an id to associate |
1012 |
|
|
* with a harvest |
1013 |
|
|
* |
1014 |
|
|
* @return |
1015 |
|
|
*/ |
1016 |
rpandey |
1.22 |
protected String generateNewHarvestId() { |
1017 |
|
|
return this.harvesterIdGen.generate(); |
1018 |
|
|
} |
1019 |
|
|
|
1020 |
rpandey |
1.23 |
/**Sends sax events to the current consumer |
1021 |
|
|
* with summary details of the all the past harvests |
1022 |
|
|
* |
1023 |
|
|
* @throws SAXException |
1024 |
|
|
*/ |
1025 |
rpandey |
1.22 |
public void sendPastHarvestsSummary() throws SAXException { |
1026 |
|
|
//build a query get the doc |
1027 |
|
|
try { |
1028 |
rpandey |
1.33 |
DatabaseEntity[] dbes = this._database.getEntities(); |
1029 |
rpandey |
1.22 |
if (dbes != null && dbes.length > 0) { |
1030 |
|
|
this.acquire(); |
1031 |
|
|
acquireSynchronizedXMLConsumer(); |
1032 |
|
|
super.startElement(Framework.SDXNamespaceURI, fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS, |
1033 |
|
|
Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS, null); |
1034 |
|
|
for (int i = 0; i < dbes.length; i++) { |
1035 |
rpandey |
1.24 |
DatabaseEntity dbe = dbes[i]; |
1036 |
rpandey |
1.22 |
AttributesImpl atts = new AttributesImpl(); |
1037 |
|
|
String repoUrl = null; |
1038 |
|
|
if (dbe != null) { |
1039 |
|
|
Property[] props = dbe.getProperties(); |
1040 |
|
|
if (props != null) { |
1041 |
|
|
for (int j = 0; j < props.length; j++) { |
1042 |
|
|
Property prop = props[j]; |
1043 |
|
|
if (prop != null) { |
1044 |
|
|
String propName = prop.getName(); |
1045 |
|
|
if (Utilities.checkString(propName) && !LuceneDocumentBaseOAIHarvester.OAI_HARVEST_ID.equals(propName)) { |
1046 |
|
|
String propVal = prop.getValue(); |
1047 |
|
|
if (propName.equals(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL)) { |
1048 |
|
|
repoUrl = propVal; |
1049 |
|
|
} else { |
1050 |
|
|
if (Utilities.checkString(propVal)) |
1051 |
|
|
atts.addAttribute("", propName, propName, OAIObject.Node.Type.CDATA, propVal); |
1052 |
|
|
} |
1053 |
|
|
} |
1054 |
|
|
} |
1055 |
|
|
|
1056 |
|
|
} |
1057 |
|
|
} |
1058 |
|
|
|
1059 |
|
|
} |
1060 |
|
|
sendElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.REQUEST, OAIObject.Node.Name.REQUEST, |
1061 |
|
|
atts, repoUrl); |
1062 |
|
|
} |
1063 |
|
|
super.endElement(Framework.SDXNamespaceURI, fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS, |
1064 |
|
|
Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS); |
1065 |
|
|
} |
1066 |
|
|
|
1067 |
|
|
} catch (/*IO*/Exception e) { |
1068 |
|
|
throw new SAXException(e.getMessage(), e); |
1069 |
rpandey |
1.27 |
} /* Utilities.logException(super.getLog(), e);TODO: what is the best way to handle this? |
1070 |
rpandey |
1.22 |
} catch (SDXException e) { |
1071 |
|
|
try { |
1072 |
|
|
e.toSAX(this); |
1073 |
|
|
} catch (ProcessingException e1) { |
1074 |
rpandey |
1.27 |
Utilities.logException(super.getLog(), e1); |
1075 |
rpandey |
1.22 |
} |
1076 |
|
|
} */ finally { |
1077 |
|
|
releaseSynchronizedXMLConsumer(); |
1078 |
|
|
this.release(); |
1079 |
|
|
} |
1080 |
|
|
|
1081 |
|
|
|
1082 |
|
|
} |
1083 |
|
|
|
1084 |
rpandey |
1.23 |
/**Retrieves the time when the harvester was last updated |
1085 |
|
|
* |
1086 |
|
|
* @return |
1087 |
|
|
*/ |
1088 |
rpandey |
1.22 |
public Date lastUpdated() { |
1089 |
rpandey |
1.23 |
return this.docbase.lastModificationDate(); |
1090 |
rpandey |
1.22 |
} |
1091 |
|
|
|
1092 |
rpandey |
1.23 |
/**Destroys all summary data pertaining to past harvests |
1093 |
|
|
* but not the actual oai records harvested |
1094 |
|
|
* |
1095 |
|
|
*/ |
1096 |
rpandey |
1.22 |
public void purgePastHarvestsData() { |
1097 |
|
|
try { |
1098 |
rpandey |
1.33 |
DatabaseEntity[] dbes = this._database.getEntities(); |
1099 |
rpandey |
1.22 |
for (int i = 0; i < dbes.length; i++) { |
1100 |
rpandey |
1.24 |
DatabaseEntity dbe = dbes[i]; |
1101 |
rpandey |
1.33 |
this._database.delete(dbe); |
1102 |
rpandey |
1.22 |
} |
1103 |
rpandey |
1.33 |
this._database.optimize(); |
1104 |
rpandey |
1.22 |
} catch (SDXException e) { |
1105 |
rpandey |
1.26 |
LoggingUtils.logException(logger, e); |
1106 |
rpandey |
1.24 |
} |
1107 |
|
|
} |
1108 |
|
|
|
1109 |
|
|
/**Stores data about harvesting failures caused |
1110 |
|
|
* by problems other than oai errors sent from |
1111 |
|
|
* a queried repository |
1112 |
|
|
* |
1113 |
|
|
* @param e |
1114 |
|
|
*/ |
1115 |
|
|
protected void storeFailedHarvestData(Exception e) { |
1116 |
|
|
try { |
1117 |
|
|
//TODO: call this method when harvesting fails/major exception thrown |
1118 |
|
|
//create a special property "failedHarvest" |
1119 |
|
|
DatabaseEntity dbe = new DatabaseEntity(generateNewHarvestId()); |
1120 |
|
|
String message = "noMessage"; |
1121 |
|
|
if (e != null) message = e.getMessage(); |
1122 |
|
|
dbe.addProperty(OAI_FAILED_HARVEST, message); |
1123 |
rpandey |
1.33 |
//create a _database entity and store all relevant class fields |
1124 |
rpandey |
1.24 |
dbe.addProperty(OAI_REQUEST_URL, this.requestUrl); |
1125 |
|
|
//save it to the database |
1126 |
rpandey |
1.33 |
this._database.update(dbe); |
1127 |
rpandey |
1.37 |
this._database.optimize(); |
1128 |
rpandey |
1.24 |
//TODO:later perhaps we can try to auto-reexecute these failed request, keying on the OAI_FAILED_HARVEST property ??? |
1129 |
|
|
} catch (SDXException e1) { |
1130 |
rpandey |
1.26 |
LoggingUtils.logException(logger, e1); |
1131 |
rpandey |
1.22 |
} |
1132 |
rpandey |
1.1 |
} |
1133 |
|
|
|
1134 |
|
|
|
1135 |
|
|
} |
1136 |
|
|
|
1137 |
|
|
|