1 |
rpandey |
1.1 |
/* |
2 |
|
|
SDX: Documentary System in XML. |
3 |
|
|
Copyright (C) 2000, 2001, 2002 Ministere de la culture et de la communication (France), AJLSM |
4 |
|
|
|
5 |
|
|
Ministere de la culture et de la communication, |
6 |
|
|
Mission de la recherche et de la technologie |
7 |
|
|
3 rue de Valois, 75042 Paris Cedex 01 (France) |
8 |
|
|
mrt@culture.fr, michel.bottin@culture.fr |
9 |
|
|
|
10 |
|
|
AJLSM, 17, rue Vital Carles, 33000 Bordeaux (France) |
11 |
|
|
sevigny@ajlsm.com |
12 |
|
|
|
13 |
|
|
This program is free software; you can redistribute it and/or |
14 |
|
|
modify it under the terms of the GNU General Public License |
15 |
|
|
as published by the Free Software Foundation; either version 2 |
16 |
|
|
of the License, or (at your option) any later version. |
17 |
|
|
|
18 |
|
|
This program is distributed in the hope that it will be useful, |
19 |
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of |
20 |
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
21 |
|
|
See the GNU General Public License for more details. |
22 |
|
|
|
23 |
|
|
You should have received a copy of the GNU General Public License |
24 |
|
|
along with this program; if not, write to the |
25 |
|
|
Free Software Foundation, Inc. |
26 |
|
|
59 Temple Place - Suite 330, Boston, MA 02111-1307, USA |
27 |
|
|
or connect to: |
28 |
|
|
http://www.fsf.org/copyleft/gpl.html |
29 |
|
|
*/ |
30 |
|
|
package fr.gouv.culture.oai; |
31 |
|
|
|
32 |
rpandey |
1.13 |
import fr.gouv.culture.oai.util.OAIUtilities; |
33 |
rpandey |
1.15 |
import fr.gouv.culture.util.apache.cocoon.components.source.URLSource; |
34 |
rpandey |
1.10 |
import org.apache.avalon.framework.component.ComponentException; |
35 |
|
|
import org.apache.avalon.framework.component.ComponentManager; |
36 |
|
|
import org.apache.avalon.framework.component.Composable; |
37 |
rpandey |
1.9 |
import org.apache.avalon.framework.parameters.Parameters; |
38 |
rpandey |
1.10 |
import org.apache.cocoon.xml.EmbeddedXMLPipe; |
39 |
rpandey |
1.9 |
import org.apache.cocoon.xml.XMLConsumer; |
40 |
rpandey |
1.10 |
import org.apache.excalibur.source.SourceParameters; |
41 |
rpandey |
1.9 |
import org.xml.sax.Attributes; |
42 |
rpandey |
1.1 |
import org.xml.sax.ContentHandler; |
43 |
|
|
import org.xml.sax.SAXException; |
44 |
|
|
|
45 |
rpandey |
1.10 |
import java.net.URL; |
46 |
rpandey |
1.9 |
|
47 |
rpandey |
1.1 |
/** |
48 |
|
|
* Created by IntelliJ IDEA. |
49 |
|
|
* User: rpandey |
50 |
|
|
* Date: Apr 24, 2003 |
51 |
|
|
* Time: 4:57:04 PM |
52 |
|
|
* To change this template use Options | File Templates. |
53 |
|
|
*/ |
54 |
rpandey |
1.14 |
public abstract class AbstractOAIHarvester extends SynchronizedOAIObjectImpl implements OAIHarvester, Composable { |
55 |
rpandey |
1.28 |
|
56 |
|
|
public static final String OAI_REQUEST_URL = "oaiRequestUrl"; |
57 |
|
|
public static final String OAI_REPOSITORY_URL = "oaiRepositoryUrl"; |
58 |
|
|
|
59 |
|
|
/**Component manager for the object*/ |
60 |
|
|
protected ComponentManager manager = null; |
61 |
|
|
|
62 |
|
|
/**List of email address strings for |
63 |
|
|
* administrators of this harvester |
64 |
|
|
* |
65 |
|
|
*/ |
66 |
uid67066 |
1.26 |
protected String[] adminEmails = null; |
67 |
rpandey |
1.28 |
|
68 |
|
|
/**User agent value to send with request*/ |
69 |
rpandey |
1.10 |
protected String userAgent = ""; |
70 |
|
|
|
71 |
rpandey |
1.28 |
/**Variable to hold the url of the request to be sent*/ |
72 |
|
|
protected String requestUrl = null; |
73 |
malo_pichot |
1.29.2.4 |
/**The new URL to resolve the next resumptionToken*/ |
74 |
|
|
protected String newRequestUrl = null; |
75 |
rpandey |
1.28 |
/**The parameters for the request sent for which a response is being received*/ |
76 |
|
|
protected Parameters requestParams = null; |
77 |
|
|
/**buffer for data collection from sax stream*/ |
78 |
|
|
protected StringBuffer sBuff = null; |
79 |
|
|
/**flag for sax event handling indicating that an element's content should be captured in the endElement method*/ |
80 |
|
|
protected boolean captureElemContent = false; |
81 |
|
|
/**flag for sax event handling indicating that a record should be capture*/ |
82 |
|
|
protected boolean captureRecord = false; |
83 |
malo_pichot |
1.29.2.1 |
/**flag for sax event handling indicating that a record should be delete*/ |
84 |
|
|
protected boolean deleteRecord = false; |
85 |
rpandey |
1.28 |
|
86 |
rpandey |
1.10 |
|
87 |
rpandey |
1.28 |
/**Variable to hold the url of the repository from which a response is being received*/ |
88 |
|
|
protected String repoUrl = null; |
89 |
|
|
/**Variable to hold the datestamp of the response of the repository from which a response is being received*/ |
90 |
|
|
protected String responseDate = null; |
91 |
|
|
/**Variable to hold the resumptionToken of the response of the repository from which a response is being received*/ |
92 |
rpandey |
1.3 |
protected String resumptionToken = null; |
93 |
rpandey |
1.28 |
|
94 |
|
|
/**The oai identifier for the current record from the stream*/ |
95 |
rpandey |
1.9 |
protected String currentOaiIdentifier = null; |
96 |
rpandey |
1.28 |
/**The datestamp for the current record from the stream*/ |
97 |
rpandey |
1.24 |
protected String currentDatestamp = null; |
98 |
malo_pichot |
1.29.2.2 |
/**The oai status for the current record from the stream*/ |
99 |
|
|
protected String currentOaiStatus = null; |
100 |
malo_pichot |
1.29.2.5 |
|
101 |
rpandey |
1.28 |
/**if a identifier name is provided we will attempt |
102 |
|
|
* to take the value of the element named as such |
103 |
|
|
* outside of the OAI2.0 namespace and retrieve an |
104 |
|
|
* underlying XML document assuming the value is |
105 |
|
|
* a valid url identifier and will incorporate the |
106 |
|
|
* XML content into the oai-record |
107 |
|
|
*/ |
108 |
|
|
protected String identifierName = "";//Defaulted for nothing |
109 |
|
|
/**Variable to hold any value retrieved based on |
110 |
|
|
* @see #identifierName |
111 |
|
|
*/ |
112 |
rpandey |
1.9 |
protected String currentMetadtaUrlIdentifier = null; |
113 |
|
|
|
114 |
|
|
|
115 |
rpandey |
1.28 |
/**The first externally provided xml consumer. |
116 |
|
|
* As serialization of the sax stream must be done |
117 |
|
|
* the consumer of this object has to be dynamically |
118 |
|
|
* changed and therefore we need a reference to the |
119 |
|
|
* first externall provided consumer so that we may |
120 |
|
|
* continue to supply it with sax events. |
121 |
|
|
*/ |
122 |
rpandey |
1.9 |
protected XMLConsumer firstXmlConsumer = null; |
123 |
|
|
|
124 |
|
|
|
125 |
rpandey |
1.28 |
/**The component manager for the object |
126 |
|
|
* |
127 |
|
|
* @param componentManager |
128 |
|
|
* @throws ComponentException |
129 |
|
|
*/ |
130 |
rpandey |
1.10 |
public void compose(ComponentManager componentManager) throws ComponentException { |
131 |
|
|
this.manager = componentManager; |
132 |
|
|
} |
133 |
|
|
|
134 |
rpandey |
1.28 |
/**Set's the consumer of this object's events |
135 |
|
|
* and will attempt to establish our |
136 |
|
|
* <code>firstXmlConsumer</code> |
137 |
|
|
* |
138 |
|
|
* @param consumer |
139 |
|
|
*/ |
140 |
rpandey |
1.9 |
public void setConsumer(XMLConsumer consumer) { |
141 |
rpandey |
1.14 |
if (consumer != null && this.firstXmlConsumer == null) { |
142 |
|
|
consumer = new EmbeddedXMLPipe(consumer); |
143 |
|
|
this.firstXmlConsumer = consumer; |
144 |
|
|
} |
145 |
|
|
super.setConsumer(consumer); |
146 |
rpandey |
1.9 |
} |
147 |
|
|
|
148 |
rpandey |
1.1 |
|
149 |
rpandey |
1.28 |
/**Retrieves the list of |
150 |
|
|
* adminstrator email addresses |
151 |
|
|
* |
152 |
|
|
* @return |
153 |
|
|
*/ |
154 |
uid67066 |
1.26 |
public String[] getAdminEmails() { |
155 |
|
|
return adminEmails; |
156 |
rpandey |
1.3 |
} |
157 |
|
|
|
158 |
rpandey |
1.28 |
/**Establishes the list of |
159 |
|
|
* adminstrator email addresses |
160 |
|
|
*/ |
161 |
|
|
public void setAdminEmails(String[] adminEmails) { |
162 |
uid67066 |
1.26 |
this.adminEmails = adminEmails; |
163 |
rpandey |
1.1 |
} |
164 |
|
|
|
165 |
rpandey |
1.28 |
/**Establishes the identifier class field |
166 |
|
|
* |
167 |
|
|
* @param name |
168 |
|
|
* @see #identifierName |
169 |
|
|
*/ |
170 |
rpandey |
1.4 |
public void setIdentifierName(String name) { |
171 |
|
|
this.identifierName = name; |
172 |
rpandey |
1.1 |
} |
173 |
|
|
|
174 |
rpandey |
1.6 |
/**Currently does nothing TODO: |
175 |
|
|
* |
176 |
|
|
* @param contentHandler |
177 |
|
|
* @throws SAXException |
178 |
|
|
*/ |
179 |
rpandey |
1.3 |
public void toSAX(ContentHandler contentHandler) throws SAXException { |
180 |
rpandey |
1.1 |
} |
181 |
rpandey |
1.3 |
|
182 |
rpandey |
1.9 |
|
183 |
|
|
public void startElement(String s, String s1, String s2, Attributes attributes) throws SAXException { |
184 |
|
|
//read the datestamp from the sax stream and set the class variable |
185 |
|
|
//we have either getRecord response or list records |
186 |
|
|
//we read the events and index the results |
187 |
|
|
//if one specifies a field which contains an identifier, we try to build full documents from the URL's provided |
188 |
|
|
//we have a resumption token |
189 |
|
|
//should send incoming data to a pipeline so that we can index the metadata and try to reach the actual data |
190 |
|
|
if (OAIObject.Node.Xmlns.OAI_2_0.equals(s)) { |
191 |
|
|
if (OAIObject.Node.Name.RESPONSE_DATE.equals(s1) || OAIObject.Node.Name.RESUMPTION_TOKEN.equals(s1) |
192 |
malo_pichot |
1.29.2.2 |
|| OAIObject.Node.Name.IDENTIFIER.equals(s1) || OAIObject.Node.Name.DATESTAMP.equals(s1) ) |
193 |
rpandey |
1.9 |
this.captureElemContent = true; |
194 |
rpandey |
1.22 |
else if (OAIObject.Node.Name.RECORD.equals(s1)) { |
195 |
rpandey |
1.28 |
this.captureRecord = true; |
196 |
rpandey |
1.22 |
prepareRecordCapture(); |
197 |
|
|
super.startDocument();//sending start document for the metadata serializer see prepareRecordCapture() |
198 |
rpandey |
1.19 |
super.startPrefixMapping(OAIObject.Node.Prefix.XSI, OAIObject.Node.Xmlns.XSI); |
199 |
rpandey |
1.9 |
} else if (OAIObject.Node.Name.REQUEST.equals(s1)) { |
200 |
|
|
this.captureElemContent = true; |
201 |
|
|
if (attributes != null) { |
202 |
|
|
int attsLen = attributes.getLength(); |
203 |
|
|
for (int i = 0; i < attsLen; i++) { |
204 |
|
|
if (this.requestParams == null) |
205 |
|
|
this.requestParams = new Parameters(); |
206 |
|
|
String paramName = attributes.getLocalName(i); |
207 |
|
|
String paramVal = attributes.getValue(i); |
208 |
rpandey |
1.13 |
if (OAIUtilities.checkString(paramName) && OAIUtilities.checkString(paramVal)) |
209 |
rpandey |
1.9 |
this.requestParams.setParameter(paramName, paramVal); |
210 |
|
|
} |
211 |
|
|
} |
212 |
|
|
} else if (OAIObject.Node.Name.HEADER.equals((s1))) { |
213 |
|
|
if (attributes != null) { |
214 |
|
|
String status = attributes.getValue(OAIObject.Node.Name.STATUS); |
215 |
rpandey |
1.13 |
if (OAIObject.Node.Value.DELETED.equalsIgnoreCase(status)) { |
216 |
malo_pichot |
1.29.2.2 |
this.currentOaiStatus = status;//memorize the status of the current record ; we need that for the indexation |
217 |
|
|
this.deleteRecord = true;//indicate that the current record must be deleted |
218 |
|
|
this.captureRecord = true;//we do not need to "capture" the record ; we just want to capture its meta-informations |
219 |
rpandey |
1.9 |
} |
220 |
|
|
} |
221 |
|
|
} else if (OAIObject.Node.Name.ERROR.equals(s1)) { |
222 |
|
|
this.captureElemContent = true; |
223 |
|
|
if (attributes != null) { |
224 |
|
|
String errorCode = attributes.getValue(OAIObject.Node.Name.CODE); |
225 |
rpandey |
1.13 |
if (OAIUtilities.checkString(errorCode)) |
226 |
|
|
OAIUtilities.logError(super.logger, errorCode, null); |
227 |
rpandey |
1.9 |
} |
228 |
|
|
|
229 |
|
|
} |
230 |
rpandey |
1.13 |
} else if (OAIUtilities.checkString(this.identifierName)) { |
231 |
rpandey |
1.9 |
/*looking for an element name or the VALUE of an attribute with the name "name" matching super.indentifierName |
232 |
|
|
*so one could create a metadata identifier element like |
233 |
|
|
*<super.identifierName>myIdentifierValue</super.identifierName> |
234 |
|
|
*or |
235 |
|
|
*<anyElementName name="super.identifierName">myIdentifierValue</anyElementName> |
236 |
|
|
*/ |
237 |
rpandey |
1.13 |
if (this.identifierName.equals(s1)) |
238 |
rpandey |
1.9 |
this.captureElemContent = true; |
239 |
|
|
} |
240 |
|
|
|
241 |
|
|
super.startElement(s, s1, s2, attributes); |
242 |
|
|
|
243 |
|
|
|
244 |
|
|
} |
245 |
|
|
|
246 |
|
|
public void characters(char[] chars, int relation, int relation1) throws SAXException { |
247 |
|
|
if (this.captureElemContent) { |
248 |
|
|
if (this.sBuff == null) this.sBuff = new StringBuffer(); |
249 |
|
|
this.sBuff.append(chars, relation, relation1); |
250 |
|
|
} |
251 |
|
|
|
252 |
|
|
super.characters(chars, relation, relation1); |
253 |
|
|
} |
254 |
|
|
|
255 |
|
|
public void endElement(String s, String s1, String s2) throws SAXException { |
256 |
|
|
super.endElement(s, s1, s2); |
257 |
|
|
if (this.sBuff != null && this.sBuff.length() > 0) { |
258 |
|
|
String content = this.sBuff.toString(); |
259 |
|
|
|
260 |
|
|
if (OAIObject.Node.Xmlns.OAI_2_0.equals(s)) { |
261 |
|
|
if (OAIObject.Node.Name.RESPONSE_DATE.equals(s1)) |
262 |
rpandey |
1.28 |
this.responseDate = content; |
263 |
rpandey |
1.9 |
else if (OAIObject.Node.Name.RESUMPTION_TOKEN.equals(s1)) |
264 |
|
|
this.resumptionToken = content; |
265 |
|
|
else if (OAIObject.Node.Name.IDENTIFIER.equals(s1)) |
266 |
|
|
this.currentOaiIdentifier = content; |
267 |
rpandey |
1.24 |
else if (OAIObject.Node.Name.DATESTAMP.equals(s1)) |
268 |
|
|
this.currentDatestamp = content; |
269 |
rpandey |
1.9 |
else if (OAIObject.Node.Name.REQUEST.equals(s1)) |
270 |
|
|
this.repoUrl = content; |
271 |
|
|
else if (OAIObject.Node.Name.ERROR.equals(s1)) |
272 |
|
|
handleErrors(content); |
273 |
|
|
} |
274 |
|
|
|
275 |
|
|
|
276 |
|
|
if (!OAIObject.Node.Xmlns.OAI_2_0.equals(s)) { |
277 |
rpandey |
1.13 |
if (OAIUtilities.checkString(this.identifierName) && this.identifierName.equals(s1)) { |
278 |
rpandey |
1.9 |
this.currentMetadtaUrlIdentifier = content; |
279 |
|
|
try { |
280 |
rpandey |
1.22 |
prepareResourceFromUrlIdentifierCapture(); |
281 |
rpandey |
1.9 |
} catch (Exception e) { |
282 |
|
|
//if we can't build the document we don't just fail completely we will continue |
283 |
rpandey |
1.20 |
OAIUtilities.logException(logger, e); |
284 |
rpandey |
1.9 |
} |
285 |
|
|
} |
286 |
|
|
} |
287 |
|
|
|
288 |
|
|
} |
289 |
|
|
if (OAIObject.Node.Xmlns.OAI_2_0.equals(s)) { |
290 |
|
|
|
291 |
malo_pichot |
1.29.2.8 |
// boolean shoulHarvestCurrentDocument = true; |
292 |
|
|
|
293 |
rpandey |
1.27 |
if (OAIObject.Node.Name.HEADER.equals(s1)) { |
294 |
malo_pichot |
1.29.2.1 |
|
295 |
malo_pichot |
1.29.2.8 |
// shoulHarvestCurrentDocument = shouldHarvestDocument();//we want to know if it's a good idea to harvest this document vs the docs we have in the document base |
296 |
|
|
// if (!shoulHarvestCurrentDocument)//we do not need to harvest this doc, see shoulHarvestCurrentDocument() for details |
297 |
|
|
// abortRecordCapture(); |
298 |
malo_pichot |
1.29.2.1 |
|
299 |
|
|
if (this.deleteRecord)//the record must be deleted |
300 |
|
|
prepareRecordForDeletion(); |
301 |
|
|
|
302 |
rpandey |
1.27 |
} |
303 |
|
|
|
304 |
rpandey |
1.28 |
if (this.captureRecord) { |
305 |
rpandey |
1.9 |
|
306 |
rpandey |
1.22 |
if (OAIObject.Node.Name.RECORD.equals(s1)) { |
307 |
rpandey |
1.9 |
try { |
308 |
rpandey |
1.28 |
this.captureRecord = false; |
309 |
rpandey |
1.22 |
captureResourceFromUrlIdentifier(); |
310 |
rpandey |
1.19 |
super.endPrefixMapping(OAIObject.Node.Prefix.XSI); |
311 |
rpandey |
1.22 |
super.endDocument();//sending end document for the metadata serializer see captureRecord() implementations |
312 |
malo_pichot |
1.29.2.8 |
// if (!shoulHarvestCurrentDocument) abortRecordCapture(); |
313 |
|
|
// else captureRecord(); |
314 |
rpandey |
1.22 |
captureRecord(); |
315 |
rpandey |
1.9 |
} catch (Exception e) { |
316 |
|
|
//if we can't build the document we don't just fail completely we will continue |
317 |
rpandey |
1.20 |
OAIUtilities.logException(logger, e); |
318 |
rpandey |
1.9 |
} |
319 |
rpandey |
1.10 |
super.recycle();//nullifying the consumer |
320 |
|
|
super.setConsumer(firstXmlConsumer);//resetting the consumer to the first external consumer provided if any |
321 |
rpandey |
1.14 |
acquireSynchronizedXMLConsumer(); |
322 |
rpandey |
1.9 |
} |
323 |
|
|
} else if (OAIObject.Node.Name.OAI_PMH.equals(s1)) { |
324 |
|
|
try { |
325 |
malo_pichot |
1.29.2.5 |
|
326 |
rpandey |
1.21 |
boolean storedData = storeHarvestedData(); |
327 |
malo_pichot |
1.29.2.5 |
|
328 |
malo_pichot |
1.29.2.2 |
//before saving critical fields, we have to check if the harvest is finish, if there's not resumptionToken |
329 |
malo_pichot |
1.29.2.6 |
if (this.resumptionToken == null) saveCriticalFields(storedData); |
330 |
malo_pichot |
1.29.2.5 |
|
331 |
malo_pichot |
1.29.2.3 |
//if we have a resumptionToken we continue to harvest |
332 |
|
|
else handleResumptionToken(); |
333 |
malo_pichot |
1.29.2.5 |
|
334 |
rpandey |
1.9 |
} catch (Exception e) { |
335 |
rpandey |
1.23 |
OAIUtilities.logException(logger, e); |
336 |
rpandey |
1.9 |
throw new SAXException(e.getMessage(), e); |
337 |
malo_pichot |
1.29.2.6 |
} finally{ |
338 |
malo_pichot |
1.29.2.7 |
resetAllFieldsFinally(); |
339 |
malo_pichot |
1.29.2.6 |
} |
340 |
rpandey |
1.9 |
} |
341 |
|
|
|
342 |
|
|
} |
343 |
|
|
|
344 |
|
|
//resetting the string buffer |
345 |
|
|
this.sBuff = null; |
346 |
|
|
//resetting the class field |
347 |
|
|
this.captureElemContent = false; |
348 |
|
|
|
349 |
rpandey |
1.17 |
|
350 |
rpandey |
1.14 |
} |
351 |
|
|
|
352 |
rpandey |
1.28 |
/**Stops any record capture currently |
353 |
|
|
* being executed and sends a flag to |
354 |
|
|
* the called method telling it to |
355 |
|
|
* delete any document saved to any media |
356 |
|
|
*/ |
357 |
rpandey |
1.27 |
protected void abortRecordCapture() { |
358 |
|
|
resetRecordCaptureFields(true);//so we can cancel the harvest of the currrent record if necessary |
359 |
|
|
} |
360 |
|
|
|
361 |
rpandey |
1.28 |
/**Logs error messages, and the request parameters |
362 |
|
|
* for that were sent to the repository which |
363 |
|
|
* may have caused the error state |
364 |
|
|
* |
365 |
|
|
* @param errorMsg |
366 |
|
|
*/ |
367 |
rpandey |
1.9 |
protected void handleErrors(String errorMsg) { |
368 |
rpandey |
1.13 |
OAIUtilities.logError(logger, "An error occured during harvesting, please see below : ", null); |
369 |
|
|
OAIUtilities.logError(logger, this.repoUrl, null); |
370 |
rpandey |
1.9 |
if (this.requestParams != null) { |
371 |
|
|
String[] paramNames = this.requestParams.getNames(); |
372 |
|
|
for (int i = 0; i < paramNames.length; i++) { |
373 |
|
|
String paramName = paramNames[i]; |
374 |
|
|
String paramVal = ""; |
375 |
rpandey |
1.13 |
if (OAIUtilities.checkString(paramName)) |
376 |
rpandey |
1.9 |
paramVal = this.requestParams.getParameter(paramName, ""); |
377 |
rpandey |
1.13 |
OAIUtilities.logError(logger, "parameter : " + paramName + " value : " + paramVal, null); |
378 |
rpandey |
1.9 |
|
379 |
|
|
} |
380 |
|
|
|
381 |
|
|
} |
382 |
rpandey |
1.13 |
OAIUtilities.logError(logger, errorMsg, null); |
383 |
|
|
OAIUtilities.logError(logger, "End Harvest Error", null); |
384 |
|
|
|
385 |
rpandey |
1.9 |
|
386 |
|
|
} |
387 |
|
|
|
388 |
rpandey |
1.28 |
/**Prepares resources for capturing an oai record |
389 |
|
|
* |
390 |
|
|
* @throws SAXException |
391 |
|
|
*/ |
392 |
rpandey |
1.22 |
protected abstract void prepareRecordCapture() throws SAXException; |
393 |
rpandey |
1.21 |
|
394 |
rpandey |
1.28 |
/**Querys underlying data structures do |
395 |
|
|
* determine whether the current oai record |
396 |
|
|
* should be harvested based on the state of the |
397 |
|
|
* harvester (ie. past harvests, presence or lack or record |
398 |
|
|
* in harvester data structures) |
399 |
|
|
* |
400 |
|
|
* @return |
401 |
|
|
*/ |
402 |
rpandey |
1.24 |
protected abstract boolean shouldHarvestDocument(); |
403 |
|
|
|
404 |
rpandey |
1.28 |
/**When a complete record is received, this method |
405 |
|
|
* takes the necessary steps to save the record |
406 |
|
|
* to any underlying media, or pre-media |
407 |
|
|
* |
408 |
|
|
* @throws Exception |
409 |
|
|
*/ |
410 |
|
|
protected abstract void captureRecord() throws Exception; |
411 |
|
|
|
412 |
|
|
/**After receiving a header@status="deleted" for a record, |
413 |
|
|
* this method makes the necessary preparations to delete |
414 |
|
|
* the record from the harvester |
415 |
|
|
* |
416 |
|
|
*/ |
417 |
rpandey |
1.22 |
protected abstract void prepareRecordForDeletion(); |
418 |
rpandey |
1.9 |
|
419 |
|
|
|
420 |
rpandey |
1.28 |
/**Prepares resources for capturing the underlying document |
421 |
|
|
* available via a url described by the oai record |
422 |
|
|
* |
423 |
|
|
* @see #currentMetadtaUrlIdentifier, #identifierName |
424 |
|
|
*/ |
425 |
|
|
protected abstract void prepareResourceFromUrlIdentifierCapture(); |
426 |
rpandey |
1.17 |
|
427 |
rpandey |
1.28 |
/**When a complete "underlying document" is received, this method |
428 |
|
|
* takes the necessary steps to save the document |
429 |
|
|
* to any underlying media, or pre-media |
430 |
|
|
* |
431 |
|
|
* @see #currentMetadtaUrlIdentifier, #identifierName |
432 |
|
|
*/ |
433 |
|
|
protected abstract void captureResourceFromUrlIdentifier(); |
434 |
rpandey |
1.9 |
|
435 |
rpandey |
1.28 |
/**This method saves all harvested records to a particular media*/ |
436 |
rpandey |
1.11 |
protected abstract boolean storeHarvestedData() throws Exception; |
437 |
rpandey |
1.9 |
|
438 |
rpandey |
1.29 |
/**This method stores information about a failed |
439 |
|
|
* (internal failure not external error from OAI repository) |
440 |
|
|
* harvest request, so that the valid request may be reexecuted |
441 |
|
|
* by the proper mechanism. |
442 |
|
|
*/ |
443 |
|
|
protected abstract void storeFailedHarvestData(Exception e); |
444 |
|
|
|
445 |
rpandey |
1.28 |
/**This method handles and reissues a new request using any resumption token received*/ |
446 |
rpandey |
1.9 |
protected abstract void handleResumptionToken(); |
447 |
|
|
|
448 |
rpandey |
1.28 |
/**If data has been harvested, this method |
449 |
|
|
* saves the any/all details of the harvest |
450 |
|
|
* |
451 |
|
|
* @param dataHarvested boolean indicating data was harvested |
452 |
|
|
* @throws SAXException |
453 |
|
|
*/ |
454 |
rpandey |
1.11 |
protected abstract void saveCriticalFields(boolean dataHarvested) throws SAXException; |
455 |
rpandey |
1.9 |
|
456 |
rpandey |
1.28 |
|
457 |
|
|
/**Stops any record capture currently |
458 |
|
|
* being executed, resets the corresponding |
459 |
|
|
* class fields and potentially |
460 |
|
|
* deletes any document saved to any media |
461 |
|
|
* @param deleteDoc |
462 |
|
|
*/ |
463 |
rpandey |
1.27 |
protected abstract void resetRecordCaptureFields(boolean deleteDoc); |
464 |
|
|
|
465 |
rpandey |
1.28 |
/**Resets the necessary class fields |
466 |
|
|
* |
467 |
|
|
*/ |
468 |
rpandey |
1.9 |
protected void resetAllFields() { |
469 |
rpandey |
1.11 |
this.resetResumptionToken(); |
470 |
rpandey |
1.17 |
this.identifierName = null; |
471 |
rpandey |
1.9 |
this.requestParams = null; |
472 |
malo_pichot |
1.29.2.4 |
this.repoUrl = this.currentMetadtaUrlIdentifier = this.currentOaiIdentifier = null; |
473 |
rpandey |
1.9 |
this.sBuff = null; |
474 |
rpandey |
1.28 |
this.captureRecord = this.captureElemContent = false; |
475 |
rpandey |
1.17 |
super.synchronizedXmlConsumerAcquired = false; |
476 |
rpandey |
1.11 |
} |
477 |
|
|
|
478 |
malo_pichot |
1.29.2.6 |
/**Final resets the necessary class fields |
479 |
|
|
* |
480 |
|
|
*/ |
481 |
|
|
protected void resetAllFieldsFinally() { |
482 |
|
|
resetAllFields(); |
483 |
|
|
} |
484 |
|
|
|
485 |
rpandey |
1.28 |
/**Clears any consumers provided to this object*/ |
486 |
rpandey |
1.14 |
public void recycle() { |
487 |
|
|
this.firstXmlConsumer = null; |
488 |
|
|
super.recycle(); |
489 |
|
|
} |
490 |
|
|
|
491 |
rpandey |
1.11 |
protected void resetResumptionToken() { |
492 |
|
|
this.resumptionToken = null; |
493 |
rpandey |
1.10 |
} |
494 |
malo_pichot |
1.29.2.5 |
|
495 |
malo_pichot |
1.29.2.4 |
public synchronized void receiveSynchronizedRequest(String url) { |
496 |
|
|
receiveSynchronizedRequest(url, url); |
497 |
|
|
} |
498 |
rpandey |
1.10 |
|
499 |
malo_pichot |
1.29.2.4 |
public synchronized void receiveSynchronizedRequest(String url, String originalRequestUrl) { |
500 |
|
|
if (OAIUtilities.checkString(url)) { |
501 |
|
|
requestUrl = originalRequestUrl; |
502 |
|
|
String currentUrl = url; |
503 |
|
|
String oldRequestUrl = null; |
504 |
|
|
while ( currentUrl != null && !currentUrl.equals(oldRequestUrl) ) { |
505 |
malo_pichot |
1.29.2.3 |
try { |
506 |
|
|
this.acquire(); |
507 |
|
|
this.receiveRequest(currentUrl); |
508 |
|
|
} catch (Exception e) { |
509 |
|
|
OAIUtilities.logError(logger, currentUrl, e); |
510 |
|
|
newRequestUrl = null; |
511 |
malo_pichot |
1.29.2.6 |
} finally { |
512 |
|
|
this.release(); |
513 |
|
|
} |
514 |
malo_pichot |
1.29.2.4 |
oldRequestUrl = currentUrl; |
515 |
malo_pichot |
1.29.2.3 |
currentUrl = newRequestUrl; |
516 |
|
|
} |
517 |
malo_pichot |
1.29.2.4 |
currentUrl = oldRequestUrl = newRequestUrl = requestUrl = url = originalRequestUrl = null; |
518 |
rpandey |
1.22 |
} |
519 |
|
|
} |
520 |
|
|
|
521 |
|
|
/**Internal receive request method that by passes synchronization of this object |
522 |
|
|
* as it may have already been synchronized elsewhere in the processing. It is |
523 |
|
|
* useful when handling resumption tokens |
524 |
|
|
* |
525 |
malo_pichot |
1.29.2.5 |
* @param url |
526 |
rpandey |
1.22 |
*/ |
527 |
malo_pichot |
1.29.2.4 |
public synchronized void receiveRequest(String url) { |
528 |
|
|
if (OAIUtilities.checkString(url)) { |
529 |
rpandey |
1.22 |
try { |
530 |
malo_pichot |
1.29.2.4 |
URL rUrl = new URL(url); |
531 |
rpandey |
1.22 |
URLSource source = new URLSource(rUrl, this.manager); |
532 |
|
|
SourceParameters sp = new SourceParameters(); |
533 |
|
|
sp.setParameter(HTTP_HEADER_NAME_USER_AGENT, this.userAgent); |
534 |
uid67066 |
1.26 |
for (int i = 0; i < adminEmails.length; i++) { |
535 |
|
|
String adminEmail = adminEmails[i]; |
536 |
|
|
sp.setParameter(HTTP_HEADER_NAME_FROM, adminEmail);//using adminEmail in following harvester implementation guidlines |
537 |
|
|
} |
538 |
rpandey |
1.22 |
source.setRequestProperties(sp); |
539 |
malo_pichot |
1.29.2.4 |
OAIUtilities.logInfo(logger, "Begin reception of OAI response from url : " + url); |
540 |
malo_pichot |
1.29.2.6 |
source.toSAX(/*handler*/this);//consume the sax events |
541 |
rpandey |
1.10 |
} catch (Exception e) { |
542 |
malo_pichot |
1.29.2.4 |
OAIUtilities.logError(logger, url, e); |
543 |
malo_pichot |
1.29.2.8 |
} finally{ |
544 |
|
|
OAIUtilities.logInfo(logger, "End reception of OAI response from url : " + url); |
545 |
rpandey |
1.10 |
} |
546 |
|
|
} |
547 |
malo_pichot |
1.29.2.5 |
|
548 |
rpandey |
1.8 |
} |
549 |
rpandey |
1.25 |
|
550 |
|
|
/**This method returns the parameters for the request |
551 |
|
|
* sent by this harvester as well as the "repository url", |
552 |
|
|
* "request url", and the "harvester admin email". |
553 |
|
|
* |
554 |
|
|
* This info will be useful when harvested records are processed. |
555 |
|
|
* |
556 |
|
|
* @return |
557 |
|
|
*/ |
558 |
uid67066 |
1.26 |
protected Parameters getHarvestParameters() { |
559 |
rpandey |
1.25 |
Parameters ret = new Parameters(); |
560 |
|
|
if (this.requestParams != null) |
561 |
|
|
ret.merge(this.requestParams); |
562 |
rpandey |
1.28 |
ret.setParameter(OAI_REPOSITORY_URL, this.repoUrl); |
563 |
|
|
ret.setParameter(OAI_REQUEST_URL, this.requestUrl); |
564 |
uid67066 |
1.26 |
for (int i = 0; i < adminEmails.length; i++) { |
565 |
|
|
String adminEmail = adminEmails[i]; |
566 |
rpandey |
1.28 |
ret.setParameter(OAIObject.Node.Name.ADMIN_EMAIL + "_" + Integer.toString(i), adminEmail); |
567 |
uid67066 |
1.26 |
} |
568 |
rpandey |
1.25 |
return ret; |
569 |
|
|
} |
570 |
rpandey |
1.8 |
|
571 |
rpandey |
1.3 |
|
572 |
rpandey |
1.1 |
} |
573 |
rpandey |
1.9 |
|