1 |
rpandey |
1.1 |
/* |
2 |
|
|
SDX: Documentary System in XML. |
3 |
|
|
Copyright (C) 2000, 2001, 2002 Ministere de la culture et de la communication (France), AJLSM |
4 |
|
|
|
5 |
|
|
Ministere de la culture et de la communication, |
6 |
|
|
Mission de la recherche et de la technologie |
7 |
|
|
3 rue de Valois, 75042 Paris Cedex 01 (France) |
8 |
|
|
mrt@culture.fr, michel.bottin@culture.fr |
9 |
|
|
|
10 |
|
|
AJLSM, 17, rue Vital Carles, 33000 Bordeaux (France) |
11 |
|
|
sevigny@ajlsm.com |
12 |
|
|
|
13 |
|
|
This program is free software; you can redistribute it and/or |
14 |
|
|
modify it under the terms of the GNU General Public License |
15 |
|
|
as published by the Free Software Foundation; either version 2 |
16 |
|
|
of the License, or (at your option) any later version. |
17 |
|
|
|
18 |
|
|
This program is distributed in the hope that it will be useful, |
19 |
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of |
20 |
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
21 |
|
|
See the GNU General Public License for more details. |
22 |
|
|
|
23 |
|
|
You should have received a copy of the GNU General Public License |
24 |
|
|
along with this program; if not, write to the |
25 |
|
|
Free Software Foundation, Inc. |
26 |
|
|
59 Temple Place - Suite 330, Boston, MA 02111-1307, USA |
27 |
|
|
or connect to: |
28 |
|
|
http://www.fsf.org/copyleft/gpl.html |
29 |
|
|
*/ |
30 |
|
|
package fr.gouv.culture.oai; |
31 |
|
|
|
32 |
rpandey |
1.13 |
import fr.gouv.culture.oai.util.OAIUtilities; |
33 |
larcaini |
1.34 |
import fr.gouv.culture.sdx.documentbase.DocumentBase; |
34 |
|
|
|
35 |
jlecren |
1.32 |
import org.apache.cocoon.components.source.SourceUtil; |
36 |
|
|
import org.apache.excalibur.source.impl.URLSource; |
37 |
|
|
import org.apache.avalon.framework.service.ServiceException; |
38 |
|
|
import org.apache.avalon.framework.service.ServiceManager; |
39 |
|
|
import org.apache.avalon.framework.service.Serviceable; |
40 |
rpandey |
1.9 |
import org.apache.avalon.framework.parameters.Parameters; |
41 |
rpandey |
1.10 |
import org.apache.cocoon.xml.EmbeddedXMLPipe; |
42 |
jlecren |
1.32 |
import java.util.HashMap; |
43 |
rpandey |
1.9 |
import org.apache.cocoon.xml.XMLConsumer; |
44 |
|
|
import org.xml.sax.Attributes; |
45 |
larcaini |
1.34 |
import org.xml.sax.helpers.AttributesImpl; |
46 |
rpandey |
1.1 |
import org.xml.sax.ContentHandler; |
47 |
|
|
import org.xml.sax.SAXException; |
48 |
|
|
|
49 |
larcaini |
1.34 |
|
50 |
rpandey |
1.10 |
import java.net.URL; |
51 |
rpandey |
1.9 |
|
52 |
rpandey |
1.1 |
/** |
53 |
|
|
* Created by IntelliJ IDEA. |
54 |
|
|
* User: rpandey |
55 |
|
|
* Date: Apr 24, 2003 |
56 |
|
|
* Time: 4:57:04 PM |
57 |
|
|
* To change this template use Options | File Templates. |
58 |
|
|
*/ |
59 |
jlecren |
1.32 |
public abstract class AbstractOAIHarvester extends SynchronizedOAIObjectImpl implements OAIHarvester, Serviceable { |
60 |
rpandey |
1.28 |
|
61 |
|
|
public static final String OAI_REQUEST_URL = "oaiRequestUrl"; |
62 |
|
|
public static final String OAI_REPOSITORY_URL = "oaiRepositoryUrl"; |
63 |
|
|
|
64 |
jlecren |
1.32 |
/**Service manager for the object*/ |
65 |
|
|
protected ServiceManager manager = null; |
66 |
rpandey |
1.28 |
|
67 |
|
|
/**List of email address strings for |
68 |
|
|
* administrators of this harvester |
69 |
|
|
* |
70 |
|
|
*/ |
71 |
uid67066 |
1.26 |
protected String[] adminEmails = null; |
72 |
rpandey |
1.28 |
|
73 |
|
|
/**User agent value to send with request*/ |
74 |
rpandey |
1.10 |
protected String userAgent = ""; |
75 |
|
|
|
76 |
rpandey |
1.28 |
/**Variable to hold the url of the request to be sent*/ |
77 |
|
|
protected String requestUrl = null; |
78 |
malo_pichot |
1.38 |
/**The new URL to resolve the next resumptionToken*/ |
79 |
|
|
protected String newRequestUrl = null; |
80 |
rpandey |
1.28 |
/**The parameters for the request sent for which a response is being received*/ |
81 |
|
|
protected Parameters requestParams = null; |
82 |
|
|
/**buffer for data collection from sax stream*/ |
83 |
|
|
protected StringBuffer sBuff = null; |
84 |
|
|
/**flag for sax event handling indicating that an element's content should be captured in the endElement method*/ |
85 |
|
|
protected boolean captureElemContent = false; |
86 |
|
|
/**flag for sax event handling indicating that a record should be capture*/ |
87 |
|
|
protected boolean captureRecord = false; |
88 |
malo_pichot |
1.36 |
/**flag for sax event handling indicating that a record should be delete*/ |
89 |
|
|
protected boolean deleteRecord = false; |
90 |
rpandey |
1.28 |
|
91 |
rpandey |
1.10 |
|
92 |
rpandey |
1.28 |
/**Variable to hold the url of the repository from which a response is being received*/ |
93 |
|
|
protected String repoUrl = null; |
94 |
rpandey |
1.30 |
/**Variable to hold the _datestamp of the response of the repository from which a response is being received*/ |
95 |
rpandey |
1.28 |
protected String responseDate = null; |
96 |
|
|
/**Variable to hold the resumptionToken of the response of the repository from which a response is being received*/ |
97 |
rpandey |
1.3 |
protected String resumptionToken = null; |
98 |
rpandey |
1.30 |
/**Variable to hold cursor information from a request using resumptionTokens to return an entire set by multiple parts*/ |
99 |
|
|
protected int cursor = -1; |
100 |
rpandey |
1.28 |
|
101 |
jlecren |
1.35 |
/**The oai identifier for the current record from the stream*/ |
102 |
rpandey |
1.9 |
protected String currentOaiIdentifier = null; |
103 |
rpandey |
1.30 |
/**The _datestamp for the current record from the stream*/ |
104 |
rpandey |
1.24 |
protected String currentDatestamp = null; |
105 |
malo_pichot |
1.37 |
/**The oai status for the current record from the stream*/ |
106 |
|
|
protected String currentOaiStatus = null; |
107 |
rpandey |
1.28 |
|
108 |
|
|
|
109 |
jlecren |
1.35 |
/**if a identifier name is provided we will attempt |
110 |
rpandey |
1.28 |
* to take the value of the element named as such |
111 |
|
|
* outside of the OAI2.0 namespace and retrieve an |
112 |
|
|
* underlying XML document assuming the value is |
113 |
jlecren |
1.35 |
* a valid url identifier and will incorporate the |
114 |
rpandey |
1.28 |
* XML content into the oai-record |
115 |
|
|
*/ |
116 |
|
|
protected String identifierName = "";//Defaulted for nothing |
117 |
|
|
/**Variable to hold any value retrieved based on |
118 |
|
|
* @see #identifierName |
119 |
|
|
*/ |
120 |
rpandey |
1.9 |
protected String currentMetadtaUrlIdentifier = null; |
121 |
|
|
|
122 |
|
|
|
123 |
rpandey |
1.28 |
/**The first externally provided xml consumer. |
124 |
|
|
* As serialization of the sax stream must be done |
125 |
|
|
* the consumer of this object has to be dynamically |
126 |
|
|
* changed and therefore we need a reference to the |
127 |
|
|
* first externall provided consumer so that we may |
128 |
|
|
* continue to supply it with sax events. |
129 |
|
|
*/ |
130 |
rpandey |
1.9 |
protected XMLConsumer firstXmlConsumer = null; |
131 |
|
|
|
132 |
|
|
|
133 |
jlecren |
1.32 |
/**The service manager for the object |
134 |
rpandey |
1.28 |
* |
135 |
jlecren |
1.32 |
* @param serviceManager |
136 |
|
|
* @throws ServiceException |
137 |
rpandey |
1.28 |
*/ |
138 |
jlecren |
1.32 |
public void service(ServiceManager serviceManager) throws ServiceException { |
139 |
|
|
this.manager = serviceManager; |
140 |
rpandey |
1.10 |
} |
141 |
|
|
|
142 |
rpandey |
1.28 |
/**Set's the consumer of this object's events |
143 |
|
|
* and will attempt to establish our |
144 |
|
|
* <code>firstXmlConsumer</code> |
145 |
|
|
* |
146 |
|
|
* @param consumer |
147 |
|
|
*/ |
148 |
rpandey |
1.9 |
public void setConsumer(XMLConsumer consumer) { |
149 |
rpandey |
1.14 |
if (consumer != null && this.firstXmlConsumer == null) { |
150 |
|
|
consumer = new EmbeddedXMLPipe(consumer); |
151 |
|
|
this.firstXmlConsumer = consumer; |
152 |
|
|
} |
153 |
|
|
super.setConsumer(consumer); |
154 |
rpandey |
1.9 |
} |
155 |
|
|
|
156 |
rpandey |
1.1 |
|
157 |
rpandey |
1.28 |
/**Retrieves the list of |
158 |
|
|
* adminstrator email addresses |
159 |
|
|
* |
160 |
|
|
* @return |
161 |
|
|
*/ |
162 |
uid67066 |
1.26 |
public String[] getAdminEmails() { |
163 |
|
|
return adminEmails; |
164 |
rpandey |
1.3 |
} |
165 |
|
|
|
166 |
rpandey |
1.28 |
/**Establishes the list of |
167 |
|
|
* adminstrator email addresses |
168 |
|
|
*/ |
169 |
|
|
public void setAdminEmails(String[] adminEmails) { |
170 |
uid67066 |
1.26 |
this.adminEmails = adminEmails; |
171 |
rpandey |
1.1 |
} |
172 |
|
|
|
173 |
jlecren |
1.35 |
/**Establishes the identifier class field |
174 |
rpandey |
1.28 |
* |
175 |
|
|
* @param name |
176 |
|
|
* @see #identifierName |
177 |
|
|
*/ |
178 |
rpandey |
1.4 |
public void setIdentifierName(String name) { |
179 |
|
|
this.identifierName = name; |
180 |
rpandey |
1.1 |
} |
181 |
|
|
|
182 |
rpandey |
1.6 |
/**Currently does nothing TODO: |
183 |
|
|
* |
184 |
|
|
* @param contentHandler |
185 |
|
|
* @throws SAXException |
186 |
|
|
*/ |
187 |
rpandey |
1.3 |
public void toSAX(ContentHandler contentHandler) throws SAXException { |
188 |
malo_pichot |
1.36 |
String xmlns=OAIObject.Node.Xmlns.OAI_2_0; |
189 |
|
|
String localname=DocumentBase.ConfigurationNode.OAI_HARVESTER; |
190 |
|
|
String fullname=OAIObject.Node.Prefix.OAI +":"+localname; |
191 |
|
|
AttributesImpl att= new AttributesImpl(); |
192 |
|
|
att.addAttribute("","","",Node.Type.CDATA,""); |
193 |
|
|
contentHandler.startElement(xmlns,localname,fullname,att); |
194 |
rpandey |
1.1 |
} |
195 |
rpandey |
1.3 |
|
196 |
rpandey |
1.9 |
|
197 |
|
|
public void startElement(String s, String s1, String s2, Attributes attributes) throws SAXException { |
198 |
rpandey |
1.30 |
//read the _datestamp from the sax stream and set the class variable |
199 |
rpandey |
1.9 |
//we have either getRecord response or list records |
200 |
|
|
//we read the events and index the results |
201 |
jlecren |
1.35 |
//if one specifies a field which contains an identifier, we try to build full documents from the URL's provided |
202 |
rpandey |
1.9 |
//we have a resumption token |
203 |
|
|
//should send incoming data to a pipeline so that we can index the metadata and try to reach the actual data |
204 |
|
|
if (OAIObject.Node.Xmlns.OAI_2_0.equals(s)) { |
205 |
|
|
if (OAIObject.Node.Name.RESPONSE_DATE.equals(s1) || OAIObject.Node.Name.RESUMPTION_TOKEN.equals(s1) |
206 |
rpandey |
1.24 |
|| OAIObject.Node.Name.IDENTIFIER.equals(s1) || OAIObject.Node.Name.DATESTAMP.equals(s1)) |
207 |
rpandey |
1.9 |
this.captureElemContent = true; |
208 |
rpandey |
1.22 |
else if (OAIObject.Node.Name.RECORD.equals(s1)) { |
209 |
rpandey |
1.28 |
this.captureRecord = true; |
210 |
rpandey |
1.22 |
prepareRecordCapture(); |
211 |
|
|
super.startDocument();//sending start document for the metadata serializer see prepareRecordCapture() |
212 |
rpandey |
1.19 |
super.startPrefixMapping(OAIObject.Node.Prefix.XSI, OAIObject.Node.Xmlns.XSI); |
213 |
rpandey |
1.9 |
} else if (OAIObject.Node.Name.REQUEST.equals(s1)) { |
214 |
|
|
this.captureElemContent = true; |
215 |
|
|
if (attributes != null) { |
216 |
|
|
int attsLen = attributes.getLength(); |
217 |
|
|
for (int i = 0; i < attsLen; i++) { |
218 |
|
|
if (this.requestParams == null) |
219 |
|
|
this.requestParams = new Parameters(); |
220 |
|
|
String paramName = attributes.getLocalName(i); |
221 |
|
|
String paramVal = attributes.getValue(i); |
222 |
rpandey |
1.13 |
if (OAIUtilities.checkString(paramName) && OAIUtilities.checkString(paramVal)) |
223 |
rpandey |
1.9 |
this.requestParams.setParameter(paramName, paramVal); |
224 |
|
|
} |
225 |
|
|
} |
226 |
|
|
} else if (OAIObject.Node.Name.HEADER.equals((s1))) { |
227 |
|
|
if (attributes != null) { |
228 |
|
|
String status = attributes.getValue(OAIObject.Node.Name.STATUS); |
229 |
rpandey |
1.13 |
if (OAIObject.Node.Value.DELETED.equalsIgnoreCase(status)) { |
230 |
malo_pichot |
1.37 |
this.currentOaiStatus = status;//memorize the status of the current record ; we need that for the indexation |
231 |
|
|
this.deleteRecord = true;//indicate that the current record must be deleted |
232 |
|
|
this.captureRecord = true;//we do not need to "capture" the record ; we just want to capture its meta-informations |
233 |
rpandey |
1.9 |
} |
234 |
|
|
} |
235 |
|
|
} else if (OAIObject.Node.Name.ERROR.equals(s1)) { |
236 |
|
|
this.captureElemContent = true; |
237 |
|
|
if (attributes != null) { |
238 |
|
|
String errorCode = attributes.getValue(OAIObject.Node.Name.CODE); |
239 |
rpandey |
1.13 |
if (OAIUtilities.checkString(errorCode)) |
240 |
|
|
OAIUtilities.logError(super.logger, errorCode, null); |
241 |
rpandey |
1.9 |
} |
242 |
|
|
|
243 |
|
|
} |
244 |
rpandey |
1.13 |
} else if (OAIUtilities.checkString(this.identifierName)) { |
245 |
rpandey |
1.9 |
/*looking for an element name or the VALUE of an attribute with the name "name" matching super.indentifierName |
246 |
jlecren |
1.35 |
*so one could create a metadata identifier element like |
247 |
rpandey |
1.9 |
*<super.identifierName>myIdentifierValue</super.identifierName> |
248 |
|
|
*or |
249 |
|
|
*<anyElementName name="super.identifierName">myIdentifierValue</anyElementName> |
250 |
|
|
*/ |
251 |
rpandey |
1.13 |
if (this.identifierName.equals(s1)) |
252 |
rpandey |
1.9 |
this.captureElemContent = true; |
253 |
|
|
} |
254 |
|
|
|
255 |
|
|
super.startElement(s, s1, s2, attributes); |
256 |
|
|
|
257 |
|
|
|
258 |
|
|
} |
259 |
|
|
|
260 |
|
|
public void characters(char[] chars, int relation, int relation1) throws SAXException { |
261 |
|
|
if (this.captureElemContent) { |
262 |
|
|
if (this.sBuff == null) this.sBuff = new StringBuffer(); |
263 |
|
|
this.sBuff.append(chars, relation, relation1); |
264 |
|
|
} |
265 |
|
|
|
266 |
|
|
super.characters(chars, relation, relation1); |
267 |
|
|
} |
268 |
|
|
|
269 |
|
|
public void endElement(String s, String s1, String s2) throws SAXException { |
270 |
|
|
super.endElement(s, s1, s2); |
271 |
|
|
if (this.sBuff != null && this.sBuff.length() > 0) { |
272 |
|
|
String content = this.sBuff.toString(); |
273 |
|
|
|
274 |
|
|
if (OAIObject.Node.Xmlns.OAI_2_0.equals(s)) { |
275 |
|
|
if (OAIObject.Node.Name.RESPONSE_DATE.equals(s1)) |
276 |
rpandey |
1.28 |
this.responseDate = content; |
277 |
rpandey |
1.9 |
else if (OAIObject.Node.Name.RESUMPTION_TOKEN.equals(s1)) |
278 |
|
|
this.resumptionToken = content; |
279 |
|
|
else if (OAIObject.Node.Name.IDENTIFIER.equals(s1)) |
280 |
|
|
this.currentOaiIdentifier = content; |
281 |
rpandey |
1.24 |
else if (OAIObject.Node.Name.DATESTAMP.equals(s1)) |
282 |
|
|
this.currentDatestamp = content; |
283 |
rpandey |
1.9 |
else if (OAIObject.Node.Name.REQUEST.equals(s1)) |
284 |
|
|
this.repoUrl = content; |
285 |
|
|
else if (OAIObject.Node.Name.ERROR.equals(s1)) |
286 |
|
|
handleErrors(content); |
287 |
|
|
} |
288 |
|
|
|
289 |
|
|
|
290 |
|
|
if (!OAIObject.Node.Xmlns.OAI_2_0.equals(s)) { |
291 |
rpandey |
1.13 |
if (OAIUtilities.checkString(this.identifierName) && this.identifierName.equals(s1)) { |
292 |
rpandey |
1.9 |
this.currentMetadtaUrlIdentifier = content; |
293 |
|
|
try { |
294 |
rpandey |
1.22 |
prepareResourceFromUrlIdentifierCapture(); |
295 |
rpandey |
1.9 |
} catch (Exception e) { |
296 |
|
|
//if we can't build the document we don't just fail completely we will continue |
297 |
rpandey |
1.20 |
OAIUtilities.logException(logger, e); |
298 |
rpandey |
1.9 |
} |
299 |
|
|
} |
300 |
|
|
} |
301 |
|
|
|
302 |
|
|
} |
303 |
|
|
if (OAIObject.Node.Xmlns.OAI_2_0.equals(s)) { |
304 |
|
|
|
305 |
rpandey |
1.27 |
if (OAIObject.Node.Name.HEADER.equals(s1)) { |
306 |
malo_pichot |
1.36 |
|
307 |
|
|
if (!shouldHarvestDocument())//we have an identifier and datestamp |
308 |
rpandey |
1.27 |
abortRecordCapture(); |
309 |
malo_pichot |
1.36 |
|
310 |
|
|
if (this.deleteRecord)//the record must be deleted |
311 |
|
|
prepareRecordForDeletion(); |
312 |
|
|
|
313 |
rpandey |
1.27 |
} |
314 |
|
|
|
315 |
rpandey |
1.28 |
if (this.captureRecord) { |
316 |
rpandey |
1.9 |
|
317 |
rpandey |
1.22 |
if (OAIObject.Node.Name.RECORD.equals(s1)) { |
318 |
rpandey |
1.9 |
try { |
319 |
rpandey |
1.28 |
this.captureRecord = false; |
320 |
rpandey |
1.22 |
captureResourceFromUrlIdentifier(); |
321 |
rpandey |
1.19 |
super.endPrefixMapping(OAIObject.Node.Prefix.XSI); |
322 |
rpandey |
1.22 |
super.endDocument();//sending end document for the metadata serializer see captureRecord() implementations |
323 |
|
|
captureRecord(); |
324 |
rpandey |
1.9 |
} catch (Exception e) { |
325 |
|
|
//if we can't build the document we don't just fail completely we will continue |
326 |
rpandey |
1.20 |
OAIUtilities.logException(logger, e); |
327 |
rpandey |
1.9 |
} |
328 |
rpandey |
1.10 |
super.recycle();//nullifying the consumer |
329 |
|
|
super.setConsumer(firstXmlConsumer);//resetting the consumer to the first external consumer provided if any |
330 |
rpandey |
1.14 |
acquireSynchronizedXMLConsumer(); |
331 |
rpandey |
1.9 |
} |
332 |
|
|
} else if (OAIObject.Node.Name.OAI_PMH.equals(s1)) { |
333 |
|
|
try { |
334 |
rpandey |
1.21 |
boolean storedData = storeHarvestedData(); |
335 |
malo_pichot |
1.37 |
//before saving critical fields, we have to check if the harvest is finish, if there's not resumptionToken |
336 |
|
|
if (this.resumptionToken == null) |
337 |
|
|
saveCriticalFields(storedData);//we only do this if we were able to index some documents |
338 |
|
|
else handleResumptionToken();//if we have a resumptionToken we continue to harvest |
339 |
rpandey |
1.21 |
resetAllFields();//calling any overiding methods |
340 |
rpandey |
1.9 |
} catch (Exception e) { |
341 |
rpandey |
1.23 |
OAIUtilities.logException(logger, e); |
342 |
rpandey |
1.9 |
throw new SAXException(e.getMessage(), e); |
343 |
|
|
} finally { |
344 |
rpandey |
1.21 |
this.resetAllFields();//call the method in this class specfically |
345 |
rpandey |
1.9 |
} |
346 |
|
|
} |
347 |
|
|
|
348 |
|
|
} |
349 |
|
|
|
350 |
|
|
//resetting the string buffer |
351 |
|
|
this.sBuff = null; |
352 |
|
|
//resetting the class field |
353 |
|
|
this.captureElemContent = false; |
354 |
|
|
|
355 |
rpandey |
1.17 |
|
356 |
rpandey |
1.14 |
} |
357 |
|
|
|
358 |
rpandey |
1.28 |
/**Stops any record capture currently |
359 |
|
|
* being executed and sends a flag to |
360 |
|
|
* the called method telling it to |
361 |
|
|
* delete any document saved to any media |
362 |
|
|
*/ |
363 |
rpandey |
1.27 |
protected void abortRecordCapture() { |
364 |
|
|
resetRecordCaptureFields(true);//so we can cancel the harvest of the currrent record if necessary |
365 |
|
|
} |
366 |
|
|
|
367 |
rpandey |
1.28 |
/**Logs error messages, and the request parameters |
368 |
|
|
* for that were sent to the repository which |
369 |
|
|
* may have caused the error state |
370 |
|
|
* |
371 |
|
|
* @param errorMsg |
372 |
|
|
*/ |
373 |
rpandey |
1.9 |
protected void handleErrors(String errorMsg) { |
374 |
rpandey |
1.13 |
OAIUtilities.logError(logger, "An error occured during harvesting, please see below : ", null); |
375 |
|
|
OAIUtilities.logError(logger, this.repoUrl, null); |
376 |
rpandey |
1.9 |
if (this.requestParams != null) { |
377 |
|
|
String[] paramNames = this.requestParams.getNames(); |
378 |
|
|
for (int i = 0; i < paramNames.length; i++) { |
379 |
|
|
String paramName = paramNames[i]; |
380 |
|
|
String paramVal = ""; |
381 |
rpandey |
1.13 |
if (OAIUtilities.checkString(paramName)) |
382 |
rpandey |
1.9 |
paramVal = this.requestParams.getParameter(paramName, ""); |
383 |
rpandey |
1.13 |
OAIUtilities.logError(logger, "parameter : " + paramName + " value : " + paramVal, null); |
384 |
rpandey |
1.9 |
|
385 |
|
|
} |
386 |
|
|
|
387 |
|
|
} |
388 |
rpandey |
1.13 |
OAIUtilities.logError(logger, errorMsg, null); |
389 |
|
|
OAIUtilities.logError(logger, "End Harvest Error", null); |
390 |
|
|
|
391 |
rpandey |
1.9 |
|
392 |
|
|
} |
393 |
|
|
|
394 |
rpandey |
1.28 |
/**Prepares resources for capturing an oai record |
395 |
|
|
* |
396 |
|
|
* @throws SAXException |
397 |
|
|
*/ |
398 |
rpandey |
1.22 |
protected abstract void prepareRecordCapture() throws SAXException; |
399 |
rpandey |
1.21 |
|
400 |
rpandey |
1.28 |
/**Querys underlying data structures do |
401 |
|
|
* determine whether the current oai record |
402 |
|
|
* should be harvested based on the state of the |
403 |
|
|
* harvester (ie. past harvests, presence or lack or record |
404 |
|
|
* in harvester data structures) |
405 |
|
|
* |
406 |
|
|
* @return |
407 |
|
|
*/ |
408 |
rpandey |
1.24 |
protected abstract boolean shouldHarvestDocument(); |
409 |
|
|
|
410 |
rpandey |
1.28 |
/**When a complete record is received, this method |
411 |
|
|
* takes the necessary steps to save the record |
412 |
|
|
* to any underlying media, or pre-media |
413 |
|
|
* |
414 |
|
|
* @throws Exception |
415 |
|
|
*/ |
416 |
|
|
protected abstract void captureRecord() throws Exception; |
417 |
|
|
|
418 |
|
|
/**After receiving a header@status="deleted" for a record, |
419 |
|
|
* this method makes the necessary preparations to delete |
420 |
|
|
* the record from the harvester |
421 |
|
|
* |
422 |
|
|
*/ |
423 |
rpandey |
1.22 |
protected abstract void prepareRecordForDeletion(); |
424 |
rpandey |
1.9 |
|
425 |
|
|
|
426 |
rpandey |
1.28 |
/**Prepares resources for capturing the underlying document |
427 |
|
|
* available via a url described by the oai record |
428 |
|
|
* |
429 |
|
|
* @see #currentMetadtaUrlIdentifier, #identifierName |
430 |
|
|
*/ |
431 |
|
|
protected abstract void prepareResourceFromUrlIdentifierCapture(); |
432 |
rpandey |
1.17 |
|
433 |
rpandey |
1.28 |
/**When a complete "underlying document" is received, this method |
434 |
|
|
* takes the necessary steps to save the document |
435 |
|
|
* to any underlying media, or pre-media |
436 |
|
|
* |
437 |
|
|
* @see #currentMetadtaUrlIdentifier, #identifierName |
438 |
|
|
*/ |
439 |
|
|
protected abstract void captureResourceFromUrlIdentifier(); |
440 |
rpandey |
1.9 |
|
441 |
rpandey |
1.28 |
/**This method saves all harvested records to a particular media*/ |
442 |
rpandey |
1.11 |
protected abstract boolean storeHarvestedData() throws Exception; |
443 |
rpandey |
1.9 |
|
444 |
rpandey |
1.29 |
/**This method stores information about a failed |
445 |
|
|
* (internal failure not external error from OAI repository) |
446 |
|
|
* harvest request, so that the valid request may be reexecuted |
447 |
|
|
* by the proper mechanism. |
448 |
|
|
*/ |
449 |
|
|
protected abstract void storeFailedHarvestData(Exception e); |
450 |
|
|
|
451 |
rpandey |
1.28 |
/**This method handles and reissues a new request using any resumption token received*/ |
452 |
rpandey |
1.9 |
protected abstract void handleResumptionToken(); |
453 |
|
|
|
454 |
rpandey |
1.28 |
/**If data has been harvested, this method |
455 |
|
|
* saves the any/all details of the harvest |
456 |
|
|
* |
457 |
|
|
* @param dataHarvested boolean indicating data was harvested |
458 |
|
|
* @throws SAXException |
459 |
|
|
*/ |
460 |
rpandey |
1.11 |
protected abstract void saveCriticalFields(boolean dataHarvested) throws SAXException; |
461 |
rpandey |
1.9 |
|
462 |
rpandey |
1.28 |
|
463 |
|
|
/**Stops any record capture currently |
464 |
|
|
* being executed, resets the corresponding |
465 |
|
|
* class fields and potentially |
466 |
|
|
* deletes any document saved to any media |
467 |
|
|
* @param deleteDoc |
468 |
|
|
*/ |
469 |
rpandey |
1.27 |
protected abstract void resetRecordCaptureFields(boolean deleteDoc); |
470 |
|
|
|
471 |
rpandey |
1.28 |
/**Resets the necessary class fields |
472 |
|
|
* |
473 |
|
|
*/ |
474 |
rpandey |
1.9 |
protected void resetAllFields() { |
475 |
rpandey |
1.11 |
this.resetResumptionToken(); |
476 |
rpandey |
1.17 |
this.identifierName = null; |
477 |
rpandey |
1.9 |
this.requestParams = null; |
478 |
rpandey |
1.25 |
this.requestUrl = this.repoUrl = this.currentMetadtaUrlIdentifier = this.currentOaiIdentifier = null; |
479 |
rpandey |
1.9 |
this.sBuff = null; |
480 |
rpandey |
1.28 |
this.captureRecord = this.captureElemContent = false; |
481 |
rpandey |
1.17 |
super.synchronizedXmlConsumerAcquired = false; |
482 |
rpandey |
1.11 |
} |
483 |
|
|
|
484 |
rpandey |
1.28 |
/**Clears any consumers provided to this object*/ |
485 |
rpandey |
1.14 |
public void recycle() { |
486 |
|
|
this.firstXmlConsumer = null; |
487 |
|
|
super.recycle(); |
488 |
|
|
} |
489 |
|
|
|
490 |
rpandey |
1.11 |
protected void resetResumptionToken() { |
491 |
|
|
this.resumptionToken = null; |
492 |
rpandey |
1.10 |
} |
493 |
|
|
|
494 |
uid67066 |
1.26 |
public synchronized void receiveSynchronizedRequest(String requestUrl) { |
495 |
rpandey |
1.10 |
if (OAIUtilities.checkString(requestUrl)) { |
496 |
malo_pichot |
1.38 |
String currentUrl = requestUrl; |
497 |
|
|
while ( currentUrl != null ) { |
498 |
|
|
try { |
499 |
|
|
this.acquire(); |
500 |
|
|
this.receiveRequest(currentUrl); |
501 |
|
|
this.release(); |
502 |
|
|
} catch (Exception e) { |
503 |
|
|
OAIUtilities.logError(logger, currentUrl, e); |
504 |
|
|
newRequestUrl = null; |
505 |
|
|
} |
506 |
|
|
currentUrl = newRequestUrl; |
507 |
|
|
} |
508 |
rpandey |
1.22 |
} |
509 |
|
|
} |
510 |
|
|
|
511 |
|
|
/**Internal receive request method that by passes synchronization of this object |
512 |
|
|
* as it may have already been synchronized elsewhere in the processing. It is |
513 |
|
|
* useful when handling resumption tokens |
514 |
|
|
* |
515 |
|
|
* @param requestUrl |
516 |
|
|
*/ |
517 |
uid67066 |
1.26 |
public synchronized void receiveRequest(String requestUrl) { |
518 |
rpandey |
1.22 |
if (OAIUtilities.checkString(requestUrl)) { |
519 |
|
|
try { |
520 |
rpandey |
1.24 |
this.requestUrl = requestUrl; |
521 |
rpandey |
1.22 |
URL rUrl = new URL(requestUrl); |
522 |
jlecren |
1.32 |
URLSource source = new URLSource(); |
523 |
|
|
HashMap sp = new HashMap(); |
524 |
|
|
sp.put(HTTP_HEADER_NAME_USER_AGENT, this.userAgent); |
525 |
uid67066 |
1.26 |
for (int i = 0; i < adminEmails.length; i++) { |
526 |
|
|
String adminEmail = adminEmails[i]; |
527 |
jlecren |
1.32 |
sp.put(HTTP_HEADER_NAME_FROM, adminEmail);//using adminEmail in following harvester implementation guidlines |
528 |
uid67066 |
1.26 |
} |
529 |
malo_pichot |
1.36 |
source.init(rUrl,sp); |
530 |
rpandey |
1.27 |
OAIUtilities.logInfo(logger, "Begin reception of OAI response from url : " + requestUrl); |
531 |
jlecren |
1.32 |
SourceUtil.toSAX(source,this);//consume the sax events |
532 |
rpandey |
1.27 |
OAIUtilities.logInfo(logger, "End reception of OAI response from url : " + requestUrl); |
533 |
rpandey |
1.10 |
} catch (Exception e) { |
534 |
rpandey |
1.21 |
OAIUtilities.logError(logger, requestUrl, e); |
535 |
rpandey |
1.10 |
} |
536 |
|
|
} |
537 |
|
|
|
538 |
rpandey |
1.8 |
} |
539 |
rpandey |
1.25 |
|
540 |
|
|
/**This method returns the parameters for the request |
541 |
|
|
* sent by this harvester as well as the "repository url", |
542 |
|
|
* "request url", and the "harvester admin email". |
543 |
|
|
* |
544 |
|
|
* This info will be useful when harvested records are processed. |
545 |
|
|
* |
546 |
|
|
* @return |
547 |
|
|
*/ |
548 |
uid67066 |
1.26 |
protected Parameters getHarvestParameters() { |
549 |
rpandey |
1.25 |
Parameters ret = new Parameters(); |
550 |
|
|
if (this.requestParams != null) |
551 |
|
|
ret.merge(this.requestParams); |
552 |
rpandey |
1.28 |
ret.setParameter(OAI_REPOSITORY_URL, this.repoUrl); |
553 |
|
|
ret.setParameter(OAI_REQUEST_URL, this.requestUrl); |
554 |
uid67066 |
1.26 |
for (int i = 0; i < adminEmails.length; i++) { |
555 |
|
|
String adminEmail = adminEmails[i]; |
556 |
rpandey |
1.28 |
ret.setParameter(OAIObject.Node.Name.ADMIN_EMAIL + "_" + Integer.toString(i), adminEmail); |
557 |
uid67066 |
1.26 |
} |
558 |
rpandey |
1.25 |
return ret; |
559 |
|
|
} |
560 |
rpandey |
1.8 |
|
561 |
rpandey |
1.3 |
|
562 |
rpandey |
1.1 |
} |