1 |
rpandey |
1.1 |
/* |
2 |
|
|
SDX: Documentary System in XML. |
3 |
|
|
Copyright (C) 2000, 2001, 2002 Ministere de la culture et de la communication (France), AJLSM |
4 |
|
|
|
5 |
|
|
Ministere de la culture et de la communication, |
6 |
|
|
Mission de la recherche et de la technologie |
7 |
|
|
3 rue de Valois, 75042 Paris Cedex 01 (France) |
8 |
|
|
mrt@culture.fr, michel.bottin@culture.fr |
9 |
|
|
|
10 |
|
|
AJLSM, 17, rue Vital Carles, 33000 Bordeaux (France) |
11 |
|
|
sevigny@ajlsm.com |
12 |
|
|
|
13 |
|
|
This program is free software; you can redistribute it and/or |
14 |
|
|
modify it under the terms of the GNU General Public License |
15 |
|
|
as published by the Free Software Foundation; either version 2 |
16 |
|
|
of the License, or (at your option) any later version. |
17 |
|
|
|
18 |
|
|
This program is distributed in the hope that it will be useful, |
19 |
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of |
20 |
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
21 |
|
|
See the GNU General Public License for more details. |
22 |
|
|
|
23 |
|
|
You should have received a copy of the GNU General Public License |
24 |
|
|
along with this program; if not, write to the |
25 |
|
|
Free Software Foundation, Inc. |
26 |
|
|
59 Temple Place - Suite 330, Boston, MA 02111-1307, USA |
27 |
|
|
or connect to: |
28 |
|
|
http://www.fsf.org/copyleft/gpl.html |
29 |
|
|
*/ |
30 |
|
|
package fr.gouv.culture.sdx.oai; |
31 |
|
|
|
32 |
malo_pichot |
1.24.2.3 |
import java.io.File; |
33 |
|
|
import java.io.FileOutputStream; |
34 |
|
|
import java.io.IOException; |
35 |
|
|
import java.net.MalformedURLException; |
36 |
|
|
import java.net.URL; |
37 |
|
|
import java.util.ArrayList; |
38 |
|
|
import java.util.Date; |
39 |
|
|
import java.util.Enumeration; |
40 |
|
|
import java.util.Hashtable; |
41 |
malo_pichot |
1.24.2.6 |
import java.util.Properties; |
42 |
malo_pichot |
1.24.2.3 |
|
43 |
malo_pichot |
1.24.2.4 |
import org.apache.avalon.excalibur.io.FileUtil; |
44 |
malo_pichot |
1.24.2.3 |
import org.apache.avalon.excalibur.xml.Parser; |
45 |
|
|
import org.apache.avalon.framework.component.ComponentException; |
46 |
|
|
import org.apache.avalon.framework.configuration.Configuration; |
47 |
|
|
import org.apache.avalon.framework.configuration.ConfigurationException; |
48 |
|
|
import org.apache.cocoon.Constants; |
49 |
|
|
import org.apache.cocoon.ProcessingException; |
50 |
|
|
import org.apache.cocoon.serialization.XMLSerializer; |
51 |
|
|
import org.apache.cocoon.xml.IncludeXMLConsumer; |
52 |
|
|
import org.apache.cocoon.xml.XMLConsumer; |
53 |
|
|
import org.apache.cocoon.xml.XMLMulticaster; |
54 |
|
|
import org.apache.cocoon.xml.XMLPipe; |
55 |
|
|
import org.xml.sax.Attributes; |
56 |
|
|
import org.xml.sax.SAXException; |
57 |
|
|
import org.xml.sax.helpers.AttributesImpl; |
58 |
|
|
|
59 |
rpandey |
1.3 |
import fr.gouv.culture.oai.AbstractOAIHarvester; |
60 |
|
|
import fr.gouv.culture.oai.OAIObject; |
61 |
|
|
import fr.gouv.culture.oai.OAIRequest; |
62 |
|
|
import fr.gouv.culture.oai.OAIRequestImpl; |
63 |
malo_pichot |
1.24.2.6 |
import fr.gouv.culture.oai.util.OAIUtilities; |
64 |
rpandey |
1.1 |
import fr.gouv.culture.sdx.application.Application; |
65 |
|
|
import fr.gouv.culture.sdx.document.IndexableDocument; |
66 |
malo_pichot |
1.24.2.3 |
import fr.gouv.culture.sdx.document.OAIDocument; |
67 |
rpandey |
1.1 |
import fr.gouv.culture.sdx.document.XMLDocument; |
68 |
malo_pichot |
1.24.2.3 |
import fr.gouv.culture.sdx.documentbase.AbstractDocumentBase; |
69 |
|
|
import fr.gouv.culture.sdx.documentbase.DocumentBase; |
70 |
|
|
import fr.gouv.culture.sdx.documentbase.IDGenerator; |
71 |
|
|
import fr.gouv.culture.sdx.documentbase.IndexParameters; |
72 |
rpandey |
1.1 |
import fr.gouv.culture.sdx.exception.SDXException; |
73 |
|
|
import fr.gouv.culture.sdx.exception.SDXExceptionCode; |
74 |
|
|
import fr.gouv.culture.sdx.framework.Framework; |
75 |
rpandey |
1.10 |
import fr.gouv.culture.sdx.framework.FrameworkImpl; |
76 |
rpandey |
1.1 |
import fr.gouv.culture.sdx.pipeline.GenericPipeline; |
77 |
|
|
import fr.gouv.culture.sdx.pipeline.Pipeline; |
78 |
|
|
import fr.gouv.culture.sdx.repository.Repository; |
79 |
malo_pichot |
1.24.2.3 |
import fr.gouv.culture.sdx.search.lucene.query.ComplexQuery; |
80 |
malo_pichot |
1.24.2.15 |
import fr.gouv.culture.sdx.search.lucene.query.DateIntervalQuery; |
81 |
malo_pichot |
1.24.2.3 |
import fr.gouv.culture.sdx.search.lucene.query.FieldQuery; |
82 |
|
|
import fr.gouv.culture.sdx.search.lucene.query.SearchLocations; |
83 |
rpandey |
1.21 |
import fr.gouv.culture.sdx.utils.ConfigurationUtilities; |
84 |
rpandey |
1.1 |
import fr.gouv.culture.sdx.utils.Utilities; |
85 |
rpandey |
1.19 |
import fr.gouv.culture.sdx.utils.database.Database; |
86 |
|
|
import fr.gouv.culture.sdx.utils.database.DatabaseBacked; |
87 |
rpandey |
1.22 |
import fr.gouv.culture.sdx.utils.database.DatabaseEntity; |
88 |
|
|
import fr.gouv.culture.sdx.utils.database.Property; |
89 |
malo_pichot |
1.24.2.3 |
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.SimpleTimeScheduler; |
90 |
|
|
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.Target; |
91 |
|
|
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeScheduler; |
92 |
|
|
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeTrigger; |
93 |
|
|
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeTriggerFactory; |
94 |
rpandey |
1.10 |
import fr.gouv.culture.util.apache.cocoon.xml.SynchronizedXMLConsumerWrapper; |
95 |
rpandey |
1.1 |
|
96 |
|
|
/** |
97 |
|
|
* Created by IntelliJ IDEA. |
98 |
|
|
* User: rpandey |
99 |
|
|
* Date: May 12, 2003 |
100 |
|
|
* Time: 12:32:16 PM |
101 |
|
|
*/ |
102 |
|
|
public abstract class AbstractDocumentBaseOAIHarvester extends AbstractOAIHarvester implements Target { |
103 |
|
|
|
104 |
|
|
protected String ELEMENT_NAME_PIPELINE = "pipeline"; |
105 |
|
|
|
106 |
rpandey |
1.23 |
/**The underlying document base*/ |
107 |
|
|
protected DocumentBase docbase = null; |
108 |
|
|
/**Id of the underlying document base*/ |
109 |
|
|
protected String docbaseId = ""; |
110 |
|
|
/**Properties from the document base*/ |
111 |
|
|
protected Hashtable docBaseProps = null; |
112 |
|
|
|
113 |
|
|
/**Pre-indexation pipeline*/ |
114 |
rpandey |
1.1 |
protected Pipeline pipe = null; |
115 |
rpandey |
1.23 |
/**Underlying database to store any info*/ |
116 |
rpandey |
1.19 |
protected Database database = null; |
117 |
rpandey |
1.23 |
/**Requests in application.xconf*/ |
118 |
rpandey |
1.1 |
protected Hashtable storedRequests = null; |
119 |
rpandey |
1.23 |
/**References to the underlying documentbase's/application's repositories*/ |
120 |
rpandey |
1.1 |
protected Hashtable storeRepositoriesRefs = null; |
121 |
rpandey |
1.23 |
/**Time scheduler for stored requests*/ |
122 |
rpandey |
1.1 |
protected TimeScheduler scheduler = null; |
123 |
rpandey |
1.23 |
/**IDGenerator for this object*/ |
124 |
|
|
protected IDGenerator harvesterIdGen = null; |
125 |
|
|
|
126 |
|
|
//variables for sax stream handling |
127 |
|
|
protected String TEMPFILE_SUFFIX = ".sdx"; |
128 |
rpandey |
1.10 |
protected File tempDir = null; |
129 |
|
|
protected File harvestDoc = null; |
130 |
|
|
protected FileOutputStream fileOs = null; |
131 |
rpandey |
1.6 |
protected XMLDocument urlResource = null; |
132 |
rpandey |
1.1 |
protected ArrayList deletedDocs = null; |
133 |
rpandey |
1.19 |
protected ArrayList harvestedDocs = null; |
134 |
malo_pichot |
1.24.2.10 |
protected int noDocsHarvested = 0; |
135 |
|
|
protected int noDocsDeleted = 0; |
136 |
rpandey |
1.19 |
|
137 |
rpandey |
1.23 |
protected boolean keepDeletedRecords = false;//Defaulted, we will delete "deletedRecords" keeping our harvester in sync with source repository |
138 |
|
|
protected int noRecordsPerBatch = OAIObject.NUMBER_RECORDS_PER_RESPONSE; |
139 |
|
|
|
140 |
|
|
//configuration node names |
141 |
|
|
/**Configuration node name*/ |
142 |
|
|
protected static final String ELEMENT_NAME_OAI_DATA_PROVIDERS = "oai-data-providers"; |
143 |
|
|
/**Configuration node name*/ |
144 |
|
|
protected static final String ELEMENT_NAME_OAI_VERB = "oai-verb"; |
145 |
|
|
/**Configuration node name*/ |
146 |
|
|
protected static final String ELEMENT_NAME_OAI_IDENTIFIER = "oai-identifier"; |
147 |
|
|
/**Configuration node name*/ |
148 |
|
|
protected static final String ATTRIBUTE_NAME_NAME = "name"; |
149 |
|
|
/**Configuration node name*/ |
150 |
|
|
protected static final String ATTRIBUTE_NAME_ADMIN_EMAIL = OAIObject.Node.Name.ADMIN_EMAIL; |
151 |
|
|
/**Configuration node name*/ |
152 |
|
|
protected static final String ATTRIBUTE_NAME_USER_AGENT = "userAgent"; |
153 |
|
|
/**Configuration node name*/ |
154 |
|
|
protected static final String ATTRIBUTE_NAME_URL = "url"; |
155 |
|
|
/**Configuration node name*/ |
156 |
|
|
protected static final String ATTRIBUTE_NAME_UPDATE = "update"; |
157 |
|
|
/**Configuration node name*/ |
158 |
|
|
protected static final String ATTRIBUTE_NAME_METADATA_PREFIX = OAIObject.Node.Name.METADATA_PREFIX; |
159 |
|
|
/**Configuration node name*/ |
160 |
|
|
protected static final String ATTRIBUTE_NAME_SDX_REPOSITORY = "sdxRepository"; |
161 |
|
|
/**Configuration node name*/ |
162 |
|
|
protected static final String ATTRIBUTE_NAME_FROM = "from"; |
163 |
|
|
/**Configuration node name*/ |
164 |
malo_pichot |
1.24.2.10 |
protected static final String ATTRIBUTE_USE_LAST_HARVEST_DATE = "useLastHarvestDate"; |
165 |
|
|
/**Configuration node name*/ |
166 |
rpandey |
1.23 |
protected static final String ATTRIBUTE_NAME_UNTIL = "until"; |
167 |
|
|
/**Configuration node name*/ |
168 |
|
|
protected static final String ATTRIBUTE_NAME_SET = "set"; |
169 |
|
|
protected static final String ATTRIBUTE_NAME_KEEP_DELETED_RECORD = "keepDeletedRecords"; |
170 |
|
|
/**Configuration node name*/ |
171 |
|
|
protected static final String ATTRIBUTE_NO_RECORDS_PER_BATCH = "noRecordsPerBatch"; |
172 |
|
|
|
173 |
|
|
|
174 |
|
|
//strings for internal database properties and sax output |
175 |
rpandey |
1.22 |
protected static final String OAI_HARVEST_ID = "oaiHarvestId"; |
176 |
rpandey |
1.24 |
//internal database property for failed harvests caused by internal errors( errors in this implementation and not OAI repository errors) |
177 |
|
|
protected static final String OAI_FAILED_HARVEST = "oaiFailedHarvest"; |
178 |
rpandey |
1.22 |
protected static final String OAI_HARVESTER_LAST_UPDATED = "oaiHarvesterLastUpdated"; |
179 |
|
|
protected static final String OAI_HARVESTER_RESUMPTION_TOKEN = "oaiHarvesterResumptionToken"; |
180 |
|
|
/*Field names for common request parameters*/ |
181 |
|
|
protected static final String OAI_VERB = "oaiVerb"; |
182 |
|
|
protected static final String OAI_IDENTIFIER = "oaiIdentifier"; |
183 |
|
|
protected static final String OAI_METADATA_PREFIX = "oaiMetadataPrefix"; |
184 |
|
|
protected static final String OAI_FROM = "oaiFrom"; |
185 |
|
|
protected static final String OAI_UNTIL = "oaiUntil"; |
186 |
|
|
protected static final String OAI_SET = "oaiSet"; |
187 |
|
|
protected static final String NO_DOCS_DELETED = "noDocDeleted"; |
188 |
|
|
protected static final String NO_DOCS_HARVESTED = "noDocHarvested"; |
189 |
rpandey |
1.1 |
|
190 |
malo_pichot |
1.24.2.6 |
/**List OAI files with OAI properties*/ |
191 |
|
|
protected Hashtable filesProperties = null; |
192 |
|
|
|
193 |
rpandey |
1.1 |
|
194 |
rpandey |
1.23 |
/**Basic constructor*/ |
195 |
rpandey |
1.1 |
public AbstractDocumentBaseOAIHarvester(DocumentBase base) { |
196 |
rpandey |
1.23 |
this.docbase = base; |
197 |
|
|
if (this.docbase != null) |
198 |
|
|
this.docbaseId = this.docbase.getId(); |
199 |
rpandey |
1.1 |
} |
200 |
|
|
|
201 |
rpandey |
1.23 |
/**Set's the properties for this object*/ |
202 |
rpandey |
1.1 |
public void setProperties(Hashtable props) { |
203 |
uid67066 |
1.18 |
this.docBaseProps = props; |
204 |
rpandey |
1.1 |
} |
205 |
|
|
|
206 |
|
|
|
207 |
rpandey |
1.23 |
/**Configures this object*/ |
208 |
rpandey |
1.1 |
public void configure(Configuration configuration) throws ConfigurationException { |
209 |
|
|
|
210 |
uid67066 |
1.18 |
super.userAgent = configuration.getAttribute(ATTRIBUTE_NAME_USER_AGENT, "SDX OAI Harvester"); |
211 |
rpandey |
1.20 |
this.keepDeletedRecords = configuration.getAttributeAsBoolean(ATTRIBUTE_NAME_KEEP_DELETED_RECORD, this.keepDeletedRecords); |
212 |
rpandey |
1.23 |
this.noRecordsPerBatch = configuration.getAttributeAsInteger(ATTRIBUTE_NO_RECORDS_PER_BATCH, this.noRecordsPerBatch); |
213 |
uid67066 |
1.18 |
configureAdminEmails(configuration); |
214 |
rpandey |
1.1 |
this.configureDataProviders(configuration); |
215 |
|
|
this.configurePipeline(configuration); |
216 |
rpandey |
1.19 |
configureDatabase(configuration); |
217 |
rpandey |
1.21 |
configureHarvestIDGenerator(configuration); |
218 |
rpandey |
1.19 |
} |
219 |
|
|
|
220 |
rpandey |
1.21 |
|
221 |
rpandey |
1.23 |
/**Configures the internal database*/ |
222 |
rpandey |
1.19 |
protected void configureDatabase(Configuration configuration) throws ConfigurationException { |
223 |
|
|
DatabaseBacked internalDb = new DatabaseBacked(); |
224 |
|
|
try { |
225 |
|
|
internalDb.setId(getHarvesterId()); |
226 |
rpandey |
1.24.2.1 |
internalDb.compose(this.manager); |
227 |
rpandey |
1.19 |
internalDb.setProperties(this.docBaseProps); |
228 |
|
|
internalDb.configure(configuration); |
229 |
|
|
internalDb.init(); |
230 |
|
|
this.database = internalDb.getDatabase(); |
231 |
|
|
} catch (SDXException e) { |
232 |
rpandey |
1.24.2.1 |
throw new ConfigurationException(e.getMessage(), e); |
233 |
|
|
} catch (ComponentException e) { |
234 |
rpandey |
1.19 |
throw new ConfigurationException(e.getMessage(), e); |
235 |
|
|
} |
236 |
|
|
|
237 |
|
|
} |
238 |
|
|
|
239 |
rpandey |
1.23 |
/**Configures the id generator for harvests*/ |
240 |
rpandey |
1.21 |
protected void configureHarvestIDGenerator(Configuration configuration) throws ConfigurationException { |
241 |
|
|
this.harvesterIdGen = ConfigurationUtilities.configureIDGenerator(this.logger, configuration); |
242 |
|
|
this.harvesterIdGen.setDatabase(this.database); |
243 |
|
|
} |
244 |
|
|
|
245 |
|
|
|
246 |
rpandey |
1.23 |
/**Returns an id for this harvester based upon the underlying document base id*/ |
247 |
rpandey |
1.19 |
protected String getHarvesterId() { |
248 |
rpandey |
1.23 |
String hid = ""; |
249 |
|
|
hid += Framework.SDXNamespacePrefix + "_"; |
250 |
|
|
hid += OAIObject.Node.Prefix.OAI + "_"; |
251 |
|
|
hid += "harvester" + "_"; |
252 |
|
|
hid += this.docbaseId; |
253 |
|
|
return hid; |
254 |
uid67066 |
1.18 |
} |
255 |
|
|
|
256 |
rpandey |
1.23 |
/**Configures a list of admin emails |
257 |
|
|
* can be sub-elements, a single attribute, |
258 |
|
|
* or both |
259 |
|
|
* |
260 |
|
|
* @param configuration |
261 |
|
|
* @throws ConfigurationException |
262 |
|
|
*/ |
263 |
uid67066 |
1.18 |
protected void configureAdminEmails(Configuration configuration) throws ConfigurationException { |
264 |
|
|
//configure the admin email |
265 |
|
|
ArrayList locAdminEmailsList = new ArrayList(); |
266 |
|
|
String firstAdminEmail = configuration.getAttribute(ATTRIBUTE_NAME_ADMIN_EMAIL, null); |
267 |
|
|
Configuration[] locAdminEmails = configuration.getChildren(ATTRIBUTE_NAME_ADMIN_EMAIL); |
268 |
|
|
if (Utilities.checkString(firstAdminEmail)) |
269 |
|
|
locAdminEmailsList.add(firstAdminEmail); |
270 |
|
|
for (int i = 0; i < locAdminEmails.length; i++) { |
271 |
|
|
Configuration locAdminEmail = locAdminEmails[i]; |
272 |
|
|
if (locAdminEmail != null) { |
273 |
|
|
String value = locAdminEmail.getValue(); |
274 |
|
|
if (Utilities.checkString(value)) |
275 |
|
|
locAdminEmailsList.add(value); |
276 |
|
|
} |
277 |
|
|
} |
278 |
|
|
if (locAdminEmailsList.size() <= 0)//no admin email throw an error TODO:make this exception better |
279 |
|
|
Utilities.checkConfAttributeValue(ATTRIBUTE_NAME_ADMIN_EMAIL, null, configuration.getLocation()); |
280 |
|
|
|
281 |
|
|
super.adminEmails = (String[]) locAdminEmailsList.toArray(new String[0]); |
282 |
|
|
//releasing resources |
283 |
|
|
locAdminEmailsList.clear(); |
284 |
|
|
locAdminEmailsList = null; |
285 |
|
|
|
286 |
rpandey |
1.1 |
} |
287 |
|
|
|
288 |
rpandey |
1.23 |
/**Configures data providers info that can be reused |
289 |
|
|
* and from which requests can be automatically executed |
290 |
|
|
* |
291 |
|
|
* @param configuration |
292 |
|
|
* @throws ConfigurationException |
293 |
|
|
* @see #storedRequests |
294 |
|
|
*/ |
295 |
rpandey |
1.1 |
protected void configureDataProviders(Configuration configuration) throws ConfigurationException { |
296 |
|
|
if (configuration != null) { |
297 |
rpandey |
1.3 |
Configuration dataProvidersConf = configuration.getChild(ELEMENT_NAME_OAI_DATA_PROVIDERS, false); |
298 |
rpandey |
1.1 |
if (dataProvidersConf != null) { |
299 |
rpandey |
1.3 |
Configuration[] repoRequestConfs = dataProvidersConf.getChildren(AbstractDocumentBase.ELEMENT_NAME_OAI_REPOSITORY); |
300 |
rpandey |
1.1 |
if (repoRequestConfs != null) { |
301 |
rpandey |
1.21 |
for (int x = 0; x < repoRequestConfs.length; x++) { |
302 |
|
|
Configuration repoRequestConf = repoRequestConfs[x]; |
303 |
rpandey |
1.1 |
if (repoRequestConf != null) { |
304 |
|
|
if (storedRequests == null) storedRequests = new Hashtable(); |
305 |
rpandey |
1.3 |
String repoUrl = repoRequestConf.getAttribute(ATTRIBUTE_NAME_URL); |
306 |
rpandey |
1.1 |
configureStoreRepositories(repoUrl, repoRequestConf); |
307 |
rpandey |
1.3 |
Configuration updateConf = repoRequestConf.getChild(ATTRIBUTE_NAME_UPDATE, false); |
308 |
|
|
Configuration[] verbConfs = repoRequestConf.getChildren(ELEMENT_NAME_OAI_VERB); |
309 |
rpandey |
1.1 |
if (verbConfs != null) { |
310 |
rpandey |
1.21 |
for (int y = 0; y < verbConfs.length; y++) { |
311 |
rpandey |
1.1 |
OAIRequest request = null; |
312 |
rpandey |
1.21 |
Configuration verbConf = verbConfs[y]; |
313 |
rpandey |
1.1 |
if (verbConf != null) { |
314 |
rpandey |
1.3 |
String verb = verbConf.getAttribute(ATTRIBUTE_NAME_NAME); |
315 |
|
|
String mdPrefix = verbConf.getAttribute(ATTRIBUTE_NAME_METADATA_PREFIX); |
316 |
|
|
Utilities.checkConfAttributeValue(ATTRIBUTE_NAME_METADATA_PREFIX, mdPrefix, verbConf.getLocation()); |
317 |
malo_pichot |
1.24.2.10 |
String verbId = verbConf.getAttribute("id"); |
318 |
|
|
Utilities.checkConfAttributeValue("id", verbId, verbConf.getLocation()); |
319 |
rpandey |
1.1 |
if (verb.equalsIgnoreCase(OAIRequest.VERB_STRING_GET_RECORD)) { |
320 |
|
|
verb = OAIRequest.VERB_STRING_GET_RECORD; |
321 |
rpandey |
1.3 |
Configuration[] idsConf = verbConf.getChildren(ELEMENT_NAME_OAI_IDENTIFIER); |
322 |
rpandey |
1.1 |
if (idsConf != null) { |
323 |
rpandey |
1.21 |
for (int z = 0; z < idsConf.length; z++) { |
324 |
|
|
Configuration idConf = idsConf[z]; |
325 |
rpandey |
1.1 |
if (idConf != null) { |
326 |
|
|
String id = idConf.getValue(); |
327 |
|
|
request = new OAIRequestImpl(); |
328 |
|
|
request.enableLogging(this.logger); |
329 |
|
|
request.setRepositoryURL(repoUrl); |
330 |
|
|
request.setVerbString(verb); |
331 |
|
|
request.setMetadataPrefix(mdPrefix); |
332 |
malo_pichot |
1.24.2.10 |
request.setVerbId(verbId); |
333 |
rpandey |
1.1 |
request.setIdentifier(id); |
334 |
|
|
storedRequests.put(request.getRequestURL(), request); |
335 |
|
|
configureUpdateTriggers(request.getRequestURL(), updateConf); |
336 |
|
|
} |
337 |
|
|
|
338 |
|
|
} |
339 |
|
|
} |
340 |
|
|
|
341 |
|
|
} else if (verb.equalsIgnoreCase(OAIRequest.VERB_STRING_LIST_RECORDS)) { |
342 |
|
|
verb = OAIRequest.VERB_STRING_LIST_RECORDS; |
343 |
rpandey |
1.3 |
String from = verbConf.getAttribute(ATTRIBUTE_NAME_FROM, null); |
344 |
malo_pichot |
1.24.2.16 |
boolean useLastHarvestDate = verbConf.getAttributeAsBoolean(ATTRIBUTE_USE_LAST_HARVEST_DATE, /*true by default*/true); |
345 |
rpandey |
1.3 |
String until = verbConf.getAttribute(ATTRIBUTE_NAME_UNTIL, null); |
346 |
|
|
String set = verbConf.getAttribute(ATTRIBUTE_NAME_SET, null); |
347 |
rpandey |
1.1 |
request = new OAIRequestImpl(); |
348 |
|
|
request.enableLogging(this.logger); |
349 |
|
|
request.setRepositoryURL(repoUrl); |
350 |
|
|
request.setVerbString(verb); |
351 |
|
|
request.setMetadataPrefix(mdPrefix); |
352 |
malo_pichot |
1.24.2.10 |
request.setVerbId(verbId); |
353 |
rpandey |
1.1 |
request.setFrom(from); |
354 |
malo_pichot |
1.24.2.10 |
request.setUseLastHarvestDate(useLastHarvestDate); |
355 |
rpandey |
1.1 |
request.setUntil(until); |
356 |
|
|
request.setSetIdentifier(set); |
357 |
|
|
this.storedRequests.put(request.getRequestURL(), request); |
358 |
|
|
configureUpdateTriggers(request.getRequestURL(), updateConf); |
359 |
|
|
} else//TODOException: |
360 |
|
|
throw new ConfigurationException("this verb action is not supported for harvesting : " + verb); |
361 |
|
|
} |
362 |
|
|
|
363 |
|
|
|
364 |
|
|
} |
365 |
|
|
} |
366 |
|
|
|
367 |
malo_pichot |
1.24.2.3 |
// now we can configure indexation pipeline for these repository |
368 |
|
|
this.configurePipeline(repoRequestConf); |
369 |
rpandey |
1.1 |
|
370 |
malo_pichot |
1.24.2.3 |
} |
371 |
rpandey |
1.1 |
|
372 |
|
|
} |
373 |
|
|
if (this.scheduler != null) this.scheduler.start(); |
374 |
|
|
} |
375 |
|
|
|
376 |
|
|
} |
377 |
|
|
|
378 |
|
|
} |
379 |
|
|
} |
380 |
|
|
|
381 |
rpandey |
1.23 |
/**Configures time triggers for |
382 |
|
|
* stored requests |
383 |
|
|
* |
384 |
|
|
* @param requestUrl The request url |
385 |
|
|
* @param updateConf The configuration for updates |
386 |
|
|
* @throws ConfigurationException |
387 |
|
|
* @see #scheduler, #storedRequests |
388 |
|
|
*/ |
389 |
rpandey |
1.1 |
protected void configureUpdateTriggers(String requestUrl, Configuration updateConf) throws ConfigurationException { |
390 |
|
|
if (Utilities.checkString(requestUrl) && updateConf != null) { |
391 |
|
|
TimeTrigger trigger = new TimeTriggerFactory().createTimeTrigger(updateConf); |
392 |
|
|
if (trigger != null) { |
393 |
|
|
if (this.scheduler == null) this.scheduler = new SimpleTimeScheduler(); |
394 |
|
|
this.scheduler.addTrigger(requestUrl, trigger, this); |
395 |
|
|
} |
396 |
|
|
} |
397 |
|
|
} |
398 |
|
|
|
399 |
rpandey |
1.23 |
|
400 |
|
|
/**Configures the repositories |
401 |
|
|
* to which data will be stored |
402 |
|
|
* based upon their repository url |
403 |
|
|
* |
404 |
|
|
* @param repoUrl The repository/data provider url |
405 |
|
|
* @param oaiRepoConf The configuration |
406 |
|
|
* @throws ConfigurationException |
407 |
|
|
*/ |
408 |
rpandey |
1.1 |
protected void configureStoreRepositories(String repoUrl, Configuration oaiRepoConf) throws ConfigurationException { |
409 |
|
|
if (Utilities.checkString(repoUrl)) { |
410 |
rpandey |
1.3 |
String ref = oaiRepoConf.getAttribute(ATTRIBUTE_NAME_SDX_REPOSITORY, null); |
411 |
malo_pichot |
1.24.2.3 |
/*check for the sdxrepository attribute, if it exists, get the repository object and add it to the local hashtable*/ |
412 |
|
|
if (Utilities.checkString(ref)) { |
413 |
rpandey |
1.1 |
Repository repo = null; |
414 |
uid67066 |
1.18 |
Hashtable appRepos = (Hashtable) this.docBaseProps.get(Application.APPLICATION_REPOSITORIES); |
415 |
rpandey |
1.1 |
if (appRepos != null) |
416 |
|
|
repo = (Repository) appRepos.get(ref); |
417 |
|
|
if (repo == null) |
418 |
|
|
try { |
419 |
rpandey |
1.23 |
repo = this.docbase.getRepository(ref); |
420 |
rpandey |
1.1 |
} catch (SDXException e) { |
421 |
|
|
String[] args = new String[1]; |
422 |
|
|
args[0] = ref; |
423 |
|
|
SDXException sdxE = new SDXException(logger, SDXExceptionCode.ERROR_LOAD_REFERENCED_REPO, args, null); |
424 |
|
|
throw new ConfigurationException(sdxE.getMessage(), sdxE); |
425 |
|
|
} |
426 |
|
|
if (this.storeRepositoriesRefs == null) this.storeRepositoriesRefs = new Hashtable(); |
427 |
|
|
//populating the hashtable |
428 |
|
|
storeRepositoriesRefs.put(repoUrl, repo); |
429 |
|
|
} |
430 |
|
|
} |
431 |
|
|
|
432 |
|
|
|
433 |
|
|
} |
434 |
|
|
|
435 |
|
|
|
436 |
rpandey |
1.23 |
/**Configures the preIndexation pipeline |
437 |
|
|
* |
438 |
|
|
* @param configuration |
439 |
|
|
* @throws ConfigurationException |
440 |
|
|
* @see #pipe |
441 |
|
|
*/ |
442 |
rpandey |
1.1 |
protected void configurePipeline(Configuration configuration) throws ConfigurationException { |
443 |
|
|
//at this point, we should have a <sdx:pipeline> element |
444 |
|
|
Configuration pipeConf = configuration.getChild(ELEMENT_NAME_PIPELINE, false); |
445 |
|
|
//testing if we have something |
446 |
|
|
if (pipeConf != null) { |
447 |
|
|
//creating the pipeline and assigning the class field |
448 |
|
|
this.pipe = new GenericPipeline(); |
449 |
|
|
//setting the logger for the pipeline object |
450 |
|
|
this.pipe.enableLogging(this.logger); |
451 |
|
|
//giving the object the cocoon component manager |
452 |
|
|
try { |
453 |
|
|
this.pipe.compose(this.manager); |
454 |
|
|
} catch (ComponentException e) { |
455 |
|
|
throw new ConfigurationException(e.getMessage(), e); |
456 |
|
|
} |
457 |
|
|
|
458 |
malo_pichot |
1.24.2.3 |
//pass OAI base properties to the pipe |
459 |
|
|
this.pipe.setProperties(docBaseProps); |
460 |
|
|
|
461 |
rpandey |
1.1 |
//configuring the pipeline object from 'application.xconf' |
462 |
|
|
this.pipe.configure(pipeConf); |
463 |
|
|
} |
464 |
|
|
} |
465 |
|
|
|
466 |
rpandey |
1.23 |
/**Creates a new temporary directory for |
467 |
|
|
* writing harvested records before the will |
468 |
|
|
* be indexed |
469 |
|
|
* @return |
470 |
|
|
* @throws SDXException |
471 |
|
|
* @throws IOException |
472 |
|
|
*/ |
473 |
|
|
protected File getNewTempDir() throws SDXException, IOException { |
474 |
|
|
File ret = (File) this.docBaseProps.get(Constants.CONTEXT_UPLOAD_DIR); |
475 |
malo_pichot |
1.24.2.3 |
String childDir = Utilities.getStringFromHashtable(FrameworkImpl.APP_PATH_NAME, this.docBaseProps) + "_oaiHarvests" + File.separator + this.docbaseId + File.separator + "harvest-" + Utilities.encodeURL(Long.toString(new Date().getTime()), null); |
476 |
rpandey |
1.23 |
if (Utilities.checkString(super.resumptionToken)) |
477 |
|
|
childDir += "-" + OAIObject.Node.Name.RESUMPTION_TOKEN + "-" + super.resumptionToken; |
478 |
|
|
if (ret.canWrite()) |
479 |
|
|
ret = new File(ret, childDir); |
480 |
|
|
else |
481 |
|
|
ret = new File(Utilities.getSystemTempDir(), childDir); |
482 |
|
|
Utilities.checkDirectory(ret.getCanonicalPath(), logger); |
483 |
|
|
return ret; |
484 |
|
|
} |
485 |
|
|
|
486 |
|
|
/**Deletes the directory |
487 |
|
|
* represented by the tempDir |
488 |
|
|
* class field |
489 |
|
|
* |
490 |
|
|
*/ |
491 |
|
|
protected void deleteTempDir() { |
492 |
|
|
if (tempDir != null) { |
493 |
|
|
try { |
494 |
|
|
FileUtil.deleteDirectory(this.tempDir);//deleting the old tempdir if possible |
495 |
|
|
} catch (IOException e) { |
496 |
|
|
Utilities.logWarn(logger, e.getMessage(), e); |
497 |
|
|
} |
498 |
|
|
} |
499 |
|
|
} |
500 |
|
|
|
501 |
|
|
/**Establishes the tempDir class field |
502 |
|
|
* |
503 |
|
|
* @throws SDXException |
504 |
|
|
* @throws IOException |
505 |
|
|
*/ |
506 |
|
|
protected void initTempDir() throws SDXException, IOException { |
507 |
|
|
this.tempDir = getNewTempDir(); |
508 |
|
|
} |
509 |
|
|
|
510 |
|
|
/**Get's the current date in iso8601 format |
511 |
|
|
* |
512 |
|
|
* @return |
513 |
|
|
*/ |
514 |
|
|
protected String getIsoDate() { |
515 |
|
|
return fr.gouv.culture.sdx.utils.Date.formatDate(fr.gouv.culture.sdx.utils.Date.getUtcIso8601Date()); |
516 |
|
|
} |
517 |
|
|
|
518 |
rpandey |
1.1 |
|
519 |
rpandey |
1.23 |
/**Sets up resources to capture an oai record |
520 |
|
|
* |
521 |
|
|
* |
522 |
|
|
* @throws SAXException |
523 |
|
|
*/ |
524 |
rpandey |
1.12 |
protected void prepareRecordCapture() throws SAXException { |
525 |
rpandey |
1.22 |
try { |
526 |
|
|
if (this.tempDir == null || !this.tempDir.exists()) initTempDir(); |
527 |
rpandey |
1.23 |
this.harvestDoc = File.createTempFile("oaiHarvestedRecord", TEMPFILE_SUFFIX, tempDir); |
528 |
rpandey |
1.22 |
this.fileOs = new FileOutputStream(this.harvestDoc); |
529 |
malo_pichot |
1.24.2.6 |
|
530 |
|
|
//Initialise OAI properties for the record |
531 |
|
|
if (filesProperties == null) |
532 |
|
|
filesProperties = new Hashtable(); |
533 |
|
|
Properties props = new Properties(); |
534 |
|
|
filesProperties.put(this.harvestDoc.getName(), props);//we can work with file name, automatically generated and unique |
535 |
malo_pichot |
1.24.2.7 |
this.currentDatestamp = this.currentMetadtaUrlIdentifier = this.currentOaiIdentifier = this.currentOaiStatus = null; |
536 |
malo_pichot |
1.24.2.15 |
this.deleteRecord=false; |
537 |
rpandey |
1.22 |
XMLSerializer cBytes = new XMLSerializer(); |
538 |
|
|
cBytes.setOutputStream(this.fileOs); |
539 |
malo_pichot |
1.24.2.3 |
// We must strip the OAI envelop |
540 |
|
|
XMLPipe oaiStripper = new OAIEnvelopStripper(); |
541 |
|
|
oaiStripper.setConsumer(cBytes); |
542 |
rpandey |
1.22 |
XMLConsumer newConsumer = null; |
543 |
|
|
if (firstXmlConsumer != null) |
544 |
malo_pichot |
1.24.2.3 |
newConsumer = new XMLMulticaster(oaiStripper, firstXmlConsumer); |
545 |
rpandey |
1.22 |
else |
546 |
malo_pichot |
1.24.2.3 |
newConsumer = oaiStripper; |
547 |
rpandey |
1.23 |
|
548 |
|
|
super.setSynchronizedConsumer(new SynchronizedXMLConsumerWrapper(newConsumer)); |
549 |
malo_pichot |
1.24.2.3 |
|
550 |
rpandey |
1.22 |
} catch (IOException e) { |
551 |
|
|
throw new SAXException(e.getMessage(), e); |
552 |
|
|
} catch (SDXException e) { |
553 |
|
|
throw new SAXException(e.getMessage(), e); |
554 |
rpandey |
1.10 |
} |
555 |
rpandey |
1.22 |
|
556 |
rpandey |
1.10 |
} |
557 |
rpandey |
1.16 |
|
558 |
malo_pichot |
1.24.2.6 |
/**Ends the capture of an oai record. |
559 |
|
|
* Store properties (dateStamp and OAI identifier) needed for indexation. |
560 |
rpandey |
1.23 |
* |
561 |
|
|
* @throws Exception |
562 |
|
|
*/ |
563 |
rpandey |
1.12 |
protected void captureRecord() throws Exception { |
564 |
rpandey |
1.22 |
if (this.fileOs != null && this.harvestDoc != null) { |
565 |
malo_pichot |
1.24.2.6 |
|
566 |
rpandey |
1.10 |
this.fileOs.flush(); |
567 |
|
|
this.fileOs.close(); |
568 |
malo_pichot |
1.24.2.14 |
if (OAIObject.Node.Value.DELETED.equalsIgnoreCase(currentOaiStatus)) |
569 |
|
|
resetRecordCaptureFields(true); |
570 |
|
|
else{ |
571 |
|
|
/*Setting OAI properties (dateStamp, OAI identifier, OAI status) for the current record. |
572 |
|
|
Those informations will be used for indexation*/ |
573 |
|
|
Properties oaiProps = (Properties)filesProperties.get(this.harvestDoc.getName());//TODO: do we need to prevent the case where properties are null ? -mp |
574 |
|
|
if (this.currentDatestamp != null) |
575 |
|
|
oaiProps.setProperty("datestamp", this.currentDatestamp); |
576 |
|
|
else oaiProps.setProperty("datestamp", ""); |
577 |
|
|
if (this.currentOaiIdentifier != null) |
578 |
|
|
oaiProps.setProperty("oaiid", this.currentOaiIdentifier); |
579 |
|
|
else oaiProps.setProperty("oaiid", ""); |
580 |
|
|
if (this.currentOaiStatus != null) |
581 |
|
|
oaiProps.setProperty("oaistatus", this.currentOaiStatus); |
582 |
|
|
else oaiProps.setProperty("oaistatus", ""); |
583 |
|
|
resetRecordCaptureFields(false); |
584 |
|
|
} |
585 |
rpandey |
1.22 |
} |
586 |
|
|
} |
587 |
|
|
|
588 |
rpandey |
1.23 |
/**Resets the class fields for record capture |
589 |
|
|
* possibility deleting the current <code>harvetDoc</code> |
590 |
|
|
* object underlying file |
591 |
|
|
* |
592 |
|
|
* @param deleteDoc flag for deletion of actual file |
593 |
|
|
*/ |
594 |
rpandey |
1.22 |
protected void resetRecordCaptureFields(boolean deleteDoc) { |
595 |
|
|
if (this.fileOs != null) { |
596 |
|
|
try { |
597 |
|
|
this.fileOs.flush(); |
598 |
|
|
this.fileOs.close(); |
599 |
|
|
this.fileOs = null; |
600 |
|
|
} catch (IOException e) { |
601 |
|
|
Utilities.logException(logger, e); |
602 |
|
|
} |
603 |
|
|
} |
604 |
|
|
|
605 |
|
|
if (this.harvestDoc != null) { |
606 |
|
|
if (deleteDoc) |
607 |
|
|
this.harvestDoc.delete(); |
608 |
rpandey |
1.13 |
this.harvestDoc = null; |
609 |
rpandey |
1.1 |
} |
610 |
rpandey |
1.22 |
|
611 |
|
|
super.setConsumer(super.firstXmlConsumer);//resetting the consumer to the first external consumer |
612 |
rpandey |
1.1 |
} |
613 |
|
|
|
614 |
rpandey |
1.23 |
|
615 |
|
|
/**Sets up resources to delete an oai record |
616 |
malo_pichot |
1.24.2.3 |
*Add the record to the list of the records to removed |
617 |
rpandey |
1.23 |
*/ |
618 |
|
|
protected void prepareRecordForDeletion() { |
619 |
|
|
if (this.deletedDocs == null) this.deletedDocs = new ArrayList(); |
620 |
|
|
if (Utilities.checkString(this.currentOaiIdentifier)) { |
621 |
|
|
XMLDocument deleteDoc = null; |
622 |
|
|
try { |
623 |
malo_pichot |
1.24.2.3 |
deleteDoc = new XMLDocument(this.currentOaiIdentifier);//constructs an XMLDocument. We should pass the sdxdocid, not the sdxoaiid |
624 |
rpandey |
1.23 |
} catch (SDXException e) { |
625 |
|
|
//do nothing here |
626 |
|
|
} |
627 |
|
|
if (deleteDoc != null) this.deletedDocs.add(deleteDoc); |
628 |
|
|
} |
629 |
|
|
} |
630 |
|
|
|
631 |
|
|
|
632 |
|
|
/**Reads the documents from <code>tempDir</code> |
633 |
|
|
* and indexes them in the corresponding document |
634 |
|
|
* base, any marked deletions will be carried out |
635 |
|
|
* as well |
636 |
|
|
* |
637 |
|
|
* @return |
638 |
malo_pichot |
1.24.2.3 |
* @throws ProcessingException |
639 |
rpandey |
1.23 |
* @throws SDXException |
640 |
|
|
* @throws SAXException |
641 |
malo_pichot |
1.24.2.3 |
* @throws IOException |
642 |
malo_pichot |
1.24.2.6 |
* @see fr.gouv.culture.oai.AbstractOAIHarvester#storeHarvestedData() |
643 |
rpandey |
1.23 |
*/ |
644 |
malo_pichot |
1.24.2.3 |
protected boolean storeHarvestedData() throws ProcessingException, IOException, SDXException, SAXException { |
645 |
rpandey |
1.1 |
boolean dataHarvested = false; |
646 |
rpandey |
1.23 |
if (docbase != null) { |
647 |
malo_pichot |
1.24.2.15 |
|
648 |
rpandey |
1.11 |
//deleting any docs which have been removed from the repo |
649 |
malo_pichot |
1.24.2.15 |
if (this.deletedDocs != null && this.deletedDocs.size() > 0 ) |
650 |
malo_pichot |
1.24.2.3 |
deleteOAIDocuments(); |
651 |
malo_pichot |
1.24.2.15 |
|
652 |
rpandey |
1.1 |
IndexableDocument[] indexDocs = null; |
653 |
rpandey |
1.11 |
//creating our docs from the disk |
654 |
rpandey |
1.10 |
if (this.tempDir != null) { |
655 |
|
|
String[] files = this.tempDir.list(); |
656 |
rpandey |
1.19 |
this.harvestedDocs = new ArrayList(); |
657 |
rpandey |
1.21 |
ArrayList docs = new ArrayList(); |
658 |
|
|
IndexParameters indexParams = getIndexParameters(); |
659 |
rpandey |
1.10 |
for (int i = 0; i < files.length; i++) { |
660 |
malo_pichot |
1.24.2.15 |
|
661 |
rpandey |
1.10 |
String fileName = files[i]; |
662 |
malo_pichot |
1.24.2.15 |
|
663 |
malo_pichot |
1.24.2.6 |
//Recovers OAI properties (dateStamp, OAI identifier) for the current OAI record |
664 |
|
|
Properties oaiProps = (Properties)filesProperties.get(fileName); |
665 |
|
|
this.currentDatestamp = oaiProps.getProperty("datestamp"); |
666 |
|
|
this.currentOaiIdentifier = oaiProps.getProperty("oaiid"); |
667 |
|
|
this.currentOaiStatus = oaiProps.getProperty("oaistatus"); |
668 |
malo_pichot |
1.24.2.15 |
|
669 |
malo_pichot |
1.24.2.6 |
/*We don't want to indexe document that must be deleted*/ |
670 |
|
|
if (this.currentOaiStatus != null && !OAIObject.Node.Value.DELETED.equalsIgnoreCase(this.currentOaiStatus)) { |
671 |
malo_pichot |
1.24.2.15 |
|
672 |
malo_pichot |
1.24.2.6 |
//Constructs OAIDocument wich will be indexed |
673 |
|
|
OAIDocument metadataDoc = new OAIDocument(); |
674 |
|
|
metadataDoc.setDateString(this.currentDatestamp); |
675 |
|
|
metadataDoc.setIdentifier(this.currentOaiIdentifier); |
676 |
malo_pichot |
1.24.2.15 |
|
677 |
malo_pichot |
1.24.2.6 |
try { |
678 |
|
|
metadataDoc.setContent(new File(tempDir, fileName).toURL()); |
679 |
|
|
if (docs == null) docs = new ArrayList(); |
680 |
|
|
docs.add(metadataDoc); |
681 |
|
|
this.harvestedDocs.add(metadataDoc);//keeping track of all additions |
682 |
|
|
} catch (MalformedURLException e) { |
683 |
|
|
Utilities.logException(logger, e); |
684 |
|
|
} |
685 |
malo_pichot |
1.24.2.15 |
} |
686 |
|
|
|
687 |
|
|
//at every 1000th document we will index into the base and/or at the end of the group |
688 |
|
|
int modulusTopLimit = indexParams.getBatchMax() - 1; |
689 |
|
|
if ( i == (files.length - 1) || ( i > 0 && (i % modulusTopLimit) == 0 ) ) { |
690 |
|
|
indexDocs = (IndexableDocument[]) docs.toArray(new IndexableDocument[0]); |
691 |
|
|
if (indexDocs != null) { |
692 |
|
|
Repository repo = null; |
693 |
|
|
if (Utilities.checkString(this.repoUrl) && storeRepositoriesRefs != null) |
694 |
|
|
repo = (Repository) this.storeRepositoriesRefs.get(this.repoUrl); |
695 |
|
|
this.docbase.index(indexDocs, repo, indexParams, /*this.contentHandler*/this); |
696 |
|
|
dataHarvested = true; |
697 |
|
|
docs = null; |
698 |
|
|
indexDocs = null; |
699 |
rpandey |
1.11 |
} |
700 |
|
|
} |
701 |
rpandey |
1.10 |
} |
702 |
malo_pichot |
1.24.2.15 |
|
703 |
malo_pichot |
1.24.2.10 |
this.noDocsHarvested = this.noDocsHarvested + this.harvestedDocs.size(); |
704 |
rpandey |
1.10 |
} |
705 |
malo_pichot |
1.24.2.15 |
|
706 |
|
|
|
707 |
rpandey |
1.1 |
} |
708 |
|
|
return dataHarvested; |
709 |
|
|
} |
710 |
|
|
|
711 |
malo_pichot |
1.24.2.3 |
/**Delete OAI documents from the current document base. |
712 |
|
|
* |
713 |
|
|
* @throws IOException |
714 |
|
|
* @throws ProcessingException |
715 |
|
|
* @throws SDXException |
716 |
|
|
* @throws SAXException |
717 |
|
|
*/ |
718 |
|
|
protected void deleteOAIDocuments() throws IOException, ProcessingException, SDXException, SAXException { |
719 |
|
|
|
720 |
|
|
if (this.deletedDocs != null && this.deletedDocs.size() > 0){ |
721 |
|
|
|
722 |
|
|
//Construct the SearchLocations for the query |
723 |
|
|
SearchLocations slocs = new SearchLocations(); |
724 |
|
|
slocs.enableLogging(this.logger); |
725 |
|
|
slocs.addIndex(this.docbase.getIndex()); |
726 |
|
|
|
727 |
|
|
ComplexQuery cq = new ComplexQuery(); |
728 |
|
|
cq.enableLogging(this.logger); |
729 |
|
|
cq.setUp(slocs, fr.gouv.culture.sdx.search.lucene.query.Query.OPERATOR_OR); |
730 |
|
|
|
731 |
malo_pichot |
1.24.2.11 |
//boucle sur les documents a supprimer |
732 |
malo_pichot |
1.24.2.3 |
for( int i = 0; i < this.deletedDocs.size(); i++ ){ |
733 |
|
|
fr.gouv.culture.sdx.document.XMLDocument doc = (XMLDocument) this.deletedDocs.get(i); |
734 |
|
|
String sdxoaiid = doc.getId(); |
735 |
|
|
FieldQuery fq = new FieldQuery(); |
736 |
|
|
fq.setUp(slocs, /*sdxoaiid*/sdxoaiid, /*fieldname: sdxoaiid*/OAIDocument.INTERNAL_FIELD_NAME_SDXOAIID); |
737 |
|
|
cq.addComponent(fq); |
738 |
|
|
} |
739 |
|
|
|
740 |
|
|
cq.prepare();//prepare the query |
741 |
|
|
|
742 |
|
|
fr.gouv.culture.sdx.search.lucene.query.Results res = cq.execute();//execute the query |
743 |
|
|
|
744 |
|
|
//we have results |
745 |
malo_pichot |
1.24.2.15 |
if ( res != null && res.count() > 0 ) { |
746 |
malo_pichot |
1.24.2.3 |
String[] ids = res.getDocIds(); |
747 |
|
|
XMLDocument[] docs = new XMLDocument[ids.length]; |
748 |
|
|
for ( int j = 0; j < docs.length ; j++ ){ |
749 |
|
|
docs[j] = new XMLDocument(ids[j]); |
750 |
|
|
} |
751 |
|
|
//delete the documents |
752 |
|
|
this.docbase.delete(docs, null); |
753 |
malo_pichot |
1.24.2.10 |
noDocsDeleted = noDocsDeleted + docs.length; |
754 |
malo_pichot |
1.24.2.3 |
} |
755 |
|
|
} |
756 |
|
|
|
757 |
|
|
} |
758 |
|
|
|
759 |
rpandey |
1.23 |
/**Handles the resumption token by issuing another request |
760 |
|
|
* based upon the request from which the resumption token was received. |
761 |
|
|
*/ |
762 |
rpandey |
1.1 |
protected void handleResumptionToken() { |
763 |
|
|
if (Utilities.checkString(super.resumptionToken) && Utilities.checkString(super.repoUrl)) { |
764 |
malo_pichot |
1.24.2.12 |
String verb = ""; |
765 |
|
|
if (this.requestParams!=null) |
766 |
|
|
verb = this.requestParams.getParameter(OAIObject.Node.Name.VERB, ""); |
767 |
|
|
else if (this.storedRequests.size() > 0){ |
768 |
|
|
verb = ( (OAIRequest)this.storedRequests.get(super.requestUrl) ).getVerbString(); |
769 |
|
|
} |
770 |
|
|
|
771 |
|
|
try{ |
772 |
|
|
if (Utilities.checkString(verb)) { |
773 |
|
|
newRequestUrl = this.repoUrl + OAIRequest.URL_CHARACTER_QUESTION_MARK + OAIObject.Node.Name.VERB + OAIRequest.URL_CHARACTER_EQUALS + verb + OAIRequest.URL_CHARACTER_AMPERSAND + OAIObject.Node.Name.RESUMPTION_TOKEN + OAIRequest.URL_CHARACTER_EQUALS + super.resumptionToken; |
774 |
|
|
this.resetAllFields(); |
775 |
|
|
try { |
776 |
|
|
this.initTempDir();//trying to build a tempDir with the resumptionToken in the dir name |
777 |
|
|
} catch (SDXException e) { |
778 |
|
|
Utilities.logException(logger, e);//if this fails we log it as there will be another opportunity during the harvest |
779 |
|
|
} catch (IOException e) { |
780 |
|
|
Utilities.logException(logger, e); |
781 |
|
|
} |
782 |
|
|
} |
783 |
|
|
} catch (java.lang.NullPointerException npe){ |
784 |
|
|
Utilities.logError(logger, "Request parameters have been reset or expired..", npe); |
785 |
|
|
} finally{ |
786 |
|
|
super.resetResumptionToken();//resetting the resumption token here so we don't loop |
787 |
|
|
} |
788 |
rpandey |
1.1 |
} |
789 |
|
|
} |
790 |
|
|
|
791 |
rpandey |
1.23 |
//TODO-TEST: this should be preparing for a merge with the metadata, done |
792 |
|
|
/**Prepares to read a url value from an oai record and |
793 |
|
|
* retrieve the XML behind. |
794 |
|
|
*@see #identifierName, #currentMetadtaUrlIdentifier |
795 |
|
|
*/ |
796 |
|
|
protected void prepareResourceFromUrlIdentifierCapture() { |
797 |
rpandey |
1.1 |
if (Utilities.checkString(this.currentMetadtaUrlIdentifier)) { |
798 |
rpandey |
1.23 |
try { |
799 |
|
|
URL resourceUrl = new URL(this.currentMetadtaUrlIdentifier); |
800 |
|
|
XMLDocument resource = new XMLDocument(); |
801 |
|
|
resource.setId(this.currentMetadtaUrlIdentifier); |
802 |
|
|
resource.setContent(resourceUrl); |
803 |
|
|
this.urlResource = resource; |
804 |
|
|
} catch (MalformedURLException e) { |
805 |
|
|
Utilities.logException(logger, e); |
806 |
|
|
} catch (SDXException e) { |
807 |
|
|
Utilities.logException(logger, e); |
808 |
|
|
} |
809 |
rpandey |
1.6 |
/* |
810 |
rpandey |
1.1 |
if (docs == null) docs = new ArrayList(); |
811 |
|
|
docs.add(resource); |
812 |
rpandey |
1.6 |
*/ |
813 |
|
|
|
814 |
|
|
/*InputStream contents = resource.openStream(); |
815 |
|
|
byte[] chars = new byte[contents.available()]; |
816 |
|
|
contents.read(chars, 0, chars.length); |
817 |
|
|
contents.close(); |
818 |
|
|
docBytes. |
819 |
|
|
docBytes.write(chars);*/ |
820 |
|
|
|
821 |
|
|
} |
822 |
|
|
} |
823 |
|
|
|
824 |
rpandey |
1.23 |
/**Captures the xml from a url taken from an oai record and adds |
825 |
|
|
* it to the oai-record as a sibling of the <metadata/> element |
826 |
|
|
* |
827 |
|
|
*/ |
828 |
|
|
protected void captureResourceFromUrlIdentifier() { |
829 |
rpandey |
1.6 |
if (this.urlResource != null) { |
830 |
|
|
try { |
831 |
rpandey |
1.23 |
IncludeXMLConsumer include = new IncludeXMLConsumer(super.synchronizedXmlConsumer); |
832 |
|
|
urlResource.setConsumer(include); |
833 |
|
|
include.startElement(Framework.SDXNamespaceURI, "urlResource", "sdx:urlResource", new AttributesImpl()); |
834 |
|
|
Parser parser = null; |
835 |
|
|
try { |
836 |
|
|
parser = (Parser) this.manager.lookup(Parser.ROLE); |
837 |
|
|
urlResource.parse(parser); |
838 |
|
|
} finally { |
839 |
|
|
if (parser != null) |
840 |
|
|
this.manager.release(parser); |
841 |
|
|
this.urlResource = null; |
842 |
|
|
include.endElement(Framework.SDXNamespaceURI, "urlResource", "sdx:urlResource"); |
843 |
|
|
} |
844 |
|
|
} catch (SAXException e) { |
845 |
|
|
Utilities.logException(logger, e); |
846 |
|
|
} catch (ComponentException e) { |
847 |
|
|
Utilities.logException(logger, e); |
848 |
|
|
} catch (SDXException e) { |
849 |
|
|
Utilities.logException(logger, e); |
850 |
rpandey |
1.6 |
} |
851 |
rpandey |
1.1 |
} |
852 |
rpandey |
1.6 |
|
853 |
rpandey |
1.1 |
} |
854 |
|
|
|
855 |
|
|
|
856 |
rpandey |
1.23 |
/**Resets necessary class fields |
857 |
|
|
* |
858 |
|
|
*/ |
859 |
rpandey |
1.1 |
protected void resetAllFields() { |
860 |
malo_pichot |
1.24.2.13 |
noDocsHarvested = noDocsDeleted = 0; |
861 |
|
|
deletedDocs = harvestedDocs = null; |
862 |
rpandey |
1.6 |
this.urlResource = null; |
863 |
rpandey |
1.22 |
resetRecordCaptureFields(false); |
864 |
rpandey |
1.10 |
deleteTempDir(); |
865 |
rpandey |
1.14 |
this.tempDir = null; |
866 |
malo_pichot |
1.24.2.12 |
super.resetAllFields(); |
867 |
rpandey |
1.1 |
} |
868 |
|
|
|
869 |
rpandey |
1.23 |
|
870 |
|
|
/**Builds simple index parameters for indexation of |
871 |
|
|
* oai records into the undelryi |
872 |
|
|
* @return |
873 |
|
|
*/ |
874 |
rpandey |
1.1 |
protected IndexParameters getIndexParameters() { |
875 |
|
|
IndexParameters params = new IndexParameters(); |
876 |
|
|
params.setSendIndexationEvents(IndexParameters.SEND_ALL_EVENTS); |
877 |
rpandey |
1.23 |
params.setBatchMax(this.noRecordsPerBatch); |
878 |
malo_pichot |
1.24.2.3 |
//if (this.pipe != null && this.docbase != null) { |
879 |
|
|
// this.pipe.setConsumer(this.docbase.getIndexationPipeline()); |
880 |
|
|
// params.setPipeline(this.pipe); |
881 |
|
|
//} |
882 |
|
|
if ( this.pipe != null ) params.setPipeline(this.pipe); |
883 |
|
|
else if ( this.docbase != null ) params.setPipeline(this.docbase.getIndexationPipeline()); |
884 |
rpandey |
1.17 |
params.setPipelineParams(super.getHarvestParameters()); |
885 |
rpandey |
1.1 |
return params; |
886 |
|
|
} |
887 |
|
|
|
888 |
rpandey |
1.23 |
/**Sends the details of stored harvesting requests |
889 |
|
|
* to the current consumer |
890 |
|
|
* |
891 |
|
|
* @throws SAXException |
892 |
|
|
*/ |
893 |
rpandey |
1.6 |
public void sendStoredHarvestingRequests() throws SAXException { |
894 |
rpandey |
1.4 |
try { |
895 |
rpandey |
1.6 |
if (this.storedRequests != null && this.storedRequests.size() > 0) { |
896 |
rpandey |
1.4 |
this.acquire(); |
897 |
|
|
super.acquireSynchronizedXMLConsumer(); |
898 |
|
|
super.startElement(Framework.SDXNamespaceURI, |
899 |
|
|
fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS, |
900 |
|
|
Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS, |
901 |
|
|
null); |
902 |
|
|
Enumeration requests = storedRequests.elements(); |
903 |
|
|
if (requests != null) { |
904 |
|
|
while (requests.hasMoreElements()) { |
905 |
|
|
OAIRequest request = (OAIRequest) requests.nextElement(); |
906 |
|
|
if (request != null) |
907 |
|
|
request.toSAX(this); |
908 |
|
|
} |
909 |
rpandey |
1.1 |
} |
910 |
rpandey |
1.4 |
|
911 |
|
|
super.endElement(Framework.SDXNamespaceURI, |
912 |
|
|
fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS, Framework.SDXNamespacePrefix + ":" + |
913 |
|
|
fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS); |
914 |
rpandey |
1.1 |
} |
915 |
rpandey |
1.4 |
} catch (InterruptedException e) { |
916 |
|
|
throw new SAXException(e.getMessage(), e); |
917 |
|
|
} finally { |
918 |
|
|
super.releaseSynchronizedXMLConsumer(); |
919 |
|
|
this.release(); |
920 |
rpandey |
1.1 |
} |
921 |
|
|
|
922 |
|
|
} |
923 |
|
|
|
924 |
|
|
|
925 |
rpandey |
1.23 |
/**Triggers a oai request to a repository based |
926 |
|
|
* upon a trigger name (also a request url) |
927 |
|
|
* |
928 |
|
|
* @param triggerName |
929 |
|
|
*/ |
930 |
rpandey |
1.1 |
public synchronized void targetTriggered(String triggerName) { |
931 |
rpandey |
1.3 |
//DEBUG System.out.println("i was triggered :" + triggerName); |
932 |
malo_pichot |
1.24.2.11 |
|
933 |
rpandey |
1.23 |
//verify that we have this trigger defined |
934 |
rpandey |
1.1 |
OAIRequest request = (OAIRequest) this.storedRequests.get(triggerName); |
935 |
malo_pichot |
1.24.2.11 |
|
936 |
rpandey |
1.1 |
if (request != null) { |
937 |
malo_pichot |
1.24.2.11 |
|
938 |
|
|
// Test if we have to force the use of the from parameter (in application.xconf) instead of the last haverst date |
939 |
malo_pichot |
1.24.2.10 |
boolean useLastHarvestDate = request.getUseLastHarvestDate(); |
940 |
|
|
if ( useLastHarvestDate ){ |
941 |
|
|
try { |
942 |
|
|
DatabaseEntity dbe = this.database.getEntity(request.getVerbId()); |
943 |
|
|
if ( dbe != null ){ |
944 |
|
|
String lastHarvestDate = dbe.getProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED); |
945 |
|
|
request.setFrom(lastHarvestDate); |
946 |
|
|
} |
947 |
|
|
} catch (SDXException e) { |
948 |
|
|
Utilities.logException(logger, e); |
949 |
|
|
} |
950 |
|
|
} |
951 |
malo_pichot |
1.24.2.11 |
|
952 |
rpandey |
1.1 |
String requestUrl = request.getRequestURL(); |
953 |
malo_pichot |
1.24.2.11 |
|
954 |
malo_pichot |
1.24.2.10 |
this.receiveSynchronizedRequest(requestUrl, triggerName); |
955 |
malo_pichot |
1.24.2.11 |
|
956 |
rpandey |
1.3 |
//DEBUG System.out.println("i was requested :" + requestUrl); |
957 |
rpandey |
1.1 |
} |
958 |
|
|
} |
959 |
|
|
|
960 |
rpandey |
1.3 |
|
961 |
|
|
public void startElement(String s, String s1, String s2, Attributes attributes) throws SAXException { |
962 |
|
|
if (Utilities.checkString(this.identifierName)) { |
963 |
|
|
/*looking for an element name or the VALUE of an attribute with the name "name" matching super.indentifierName |
964 |
|
|
*so one could create a metadata identifier element like |
965 |
|
|
*<super.identifierName>myIdentifierValue</super.identifierName> |
966 |
|
|
*or |
967 |
|
|
*<anyElementName name="super.identifierName">myIdentifierValue</anyElementName> |
968 |
|
|
*/ |
969 |
rpandey |
1.4 |
if (attributes != null && this.identifierName.equals(attributes.getValue(fr.gouv.culture.sdx.utils.constants.Node.Name.NAME))) |
970 |
rpandey |
1.3 |
this.captureElemContent = true; |
971 |
|
|
} |
972 |
|
|
super.startElement(s, s1, s2, attributes); |
973 |
|
|
} |
974 |
|
|
|
975 |
|
|
public void endElement(String s, String s1, String s2) throws SAXException { |
976 |
|
|
if (super.sBuff != null && super.sBuff.length() > 0) { |
977 |
|
|
String content = super.sBuff.toString(); |
978 |
|
|
|
979 |
|
|
if (!OAIObject.Node.Xmlns.OAI_2_0.equals(s)) { |
980 |
|
|
if (fr.gouv.culture.sdx.utils.constants.Node.Name.FIELD.equals(s1)) { |
981 |
|
|
super.currentMetadtaUrlIdentifier = content; |
982 |
|
|
try { |
983 |
rpandey |
1.12 |
prepareResourceFromUrlIdentifierCapture(); |
984 |
rpandey |
1.3 |
} catch (Exception e) { |
985 |
|
|
//if we can't build the document we don't just fail completely we will continue |
986 |
rpandey |
1.9 |
Utilities.logException(logger, e); |
987 |
rpandey |
1.3 |
} |
988 |
|
|
} |
989 |
|
|
} |
990 |
|
|
|
991 |
|
|
} |
992 |
|
|
super.endElement(s, s1, s2); |
993 |
rpandey |
1.22 |
} |
994 |
|
|
|
995 |
rpandey |
1.23 |
/**Querys the underlying data structures |
996 |
|
|
* based upon current sax flow |
997 |
|
|
* position/set class fields and |
998 |
|
|
* determines whether an oai record should be |
999 |
|
|
* harvested |
1000 |
|
|
* |
1001 |
|
|
* @return boolean indicates whether the record should be handled |
1002 |
|
|
*/ |
1003 |
rpandey |
1.22 |
protected boolean shouldHarvestDocument() { |
1004 |
|
|
boolean ret = true; |
1005 |
|
|
try { |
1006 |
malo_pichot |
1.24.2.14 |
|
1007 |
malo_pichot |
1.24.2.16 |
//First control: do we have a previous harvest for this provider/verb ? If so, look for the datestamp of the current oai doc vs. the time of the last harvest |
1008 |
malo_pichot |
1.24.2.14 |
OAIRequest request = (OAIRequest) this.storedRequests.get(super.requestUrl); |
1009 |
malo_pichot |
1.24.2.16 |
if (request != null && request.getVerbId()!=null && !request.getVerbId().equals("") && this.database.entityExists(request.getVerbId())){ |
1010 |
|
|
DatabaseEntity dbe = this.database.getEntity(request.getVerbId()); |
1011 |
|
|
String formattedDate = dbe.getProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED); |
1012 |
|
|
java.util.Date lastUpdatedDate = fr.gouv.culture.sdx.utils.Date.parseDate(formattedDate); |
1013 |
|
|
java.util.Date currentDocumentDate = fr.gouv.culture.sdx.utils.Date.parseDate(super.currentDatestamp); |
1014 |
|
|
if (currentDocumentDate.getTime() > lastUpdatedDate.getTime()) |
1015 |
|
|
return ret;//the timestamp of the current doc is more recent than the last harvest, we should harvest it |
1016 |
malo_pichot |
1.24.2.14 |
} |
1017 |
|
|
|
1018 |
malo_pichot |
1.24.2.16 |
//Second control: search for an sdxdocument which correspond to the current oai id |
1019 |
|
|
else if (Utilities.checkString(super.currentOaiIdentifier)) { |
1020 |
malo_pichot |
1.24.2.14 |
|
1021 |
|
|
//The complexQuery |
1022 |
|
|
ComplexQuery cq = new ComplexQuery(); |
1023 |
|
|
SearchLocations slocs = new SearchLocations(); |
1024 |
|
|
slocs.enableLogging(this.logger); |
1025 |
malo_pichot |
1.24.2.15 |
slocs.addIndex(this.docbase.getIndex()); |
1026 |
malo_pichot |
1.24.2.14 |
cq.enableLogging(this.logger); |
1027 |
|
|
cq.setUp(slocs,fr.gouv.culture.sdx.search.lucene.query.Query.OPERATOR_AND); |
1028 |
|
|
//The fieldQuery |
1029 |
|
|
FieldQuery fq = new FieldQuery(); |
1030 |
|
|
fq.setUp(slocs, /*sdxoaiid*/super.currentOaiIdentifier, /*fieldname: sdxoaiid*/OAIDocument.INTERNAL_FIELD_NAME_SDXOAIID); |
1031 |
|
|
cq.addComponent(fq); |
1032 |
|
|
//The dateQuery |
1033 |
|
|
DateIntervalQuery dq = new DateIntervalQuery(); |
1034 |
|
|
java.util.Date currentOaiDocumentDate= fr.gouv.culture.sdx.utils.Date.parseDate(super.currentDatestamp); |
1035 |
malo_pichot |
1.24.2.16 |
dq.setUp(slocs, /*sdxoaidate*/OAIDocument.INTERNAL_FIELD_NAME_SDXOAIDATE, /*from*/currentOaiDocumentDate, /*to*/null, /*bounds inclusive*/true); |
1036 |
malo_pichot |
1.24.2.14 |
cq.addComponent(dq); |
1037 |
|
|
|
1038 |
|
|
cq.prepare(); |
1039 |
|
|
fr.gouv.culture.sdx.search.lucene.query.Results res = cq.execute(); |
1040 |
|
|
|
1041 |
|
|
//we have result(s) |
1042 |
|
|
if (res != null && res.count() == 1 ){//1 result cool ! |
1043 |
|
|
//String docid = res.getDocIds()[0];//we need to compare the timestamp of this sdxdoc and the current oaidoc |
1044 |
|
|
ret = false; |
1045 |
|
|
} |
1046 |
|
|
else if(res != null && res.count() > 1) {//More than 1 result: error |
1047 |
|
|
OAIUtilities.logError(logger, "A problem occured during harvesting: "+res.count()+" documents correspond to the OAI identifier: "+super.currentOaiIdentifier, null); |
1048 |
|
|
} |
1049 |
|
|
else return ret;//no document with this id exists, so we should harvest it |
1050 |
|
|
} |
1051 |
|
|
} catch (SDXException e) { |
1052 |
malo_pichot |
1.24.2.15 |
Utilities.logException(logger, e); |
1053 |
malo_pichot |
1.24.2.14 |
} |
1054 |
|
|
|
1055 |
|
|
return ret; |
1056 |
rpandey |
1.22 |
} |
1057 |
|
|
|
1058 |
rpandey |
1.23 |
/**Saves critical data about a harvest |
1059 |
|
|
* |
1060 |
|
|
* @param dataHarvested |
1061 |
|
|
* @throws SAXException |
1062 |
|
|
*/ |
1063 |
rpandey |
1.22 |
protected void saveCriticalFields(boolean dataHarvested) throws SAXException { |
1064 |
malo_pichot |
1.24.2.11 |
|
1065 |
malo_pichot |
1.24.2.10 |
try { |
1066 |
malo_pichot |
1.24.2.11 |
|
1067 |
malo_pichot |
1.24.2.10 |
OAIRequest request = (OAIRequest) this.storedRequests.get(super.requestUrl); |
1068 |
malo_pichot |
1.24.2.11 |
|
1069 |
|
|
if (request != null && request.getVerbId() != null && !request.getVerbId().equals("")) { |
1070 |
|
|
|
1071 |
malo_pichot |
1.24.2.10 |
DatabaseEntity dbe = this.database.getEntity(request.getVerbId()); |
1072 |
malo_pichot |
1.24.2.11 |
if (dbe == null) dbe = new DatabaseEntity(request.getVerbId()); |
1073 |
|
|
|
1074 |
malo_pichot |
1.24.2.10 |
//Now we have to clear the database |
1075 |
|
|
Property[] props = dbe.getProperties(); |
1076 |
|
|
if (props.length > 0){ |
1077 |
|
|
for(int i=0 ; i < props.length; i++){ |
1078 |
|
|
Property prop = props[i]; |
1079 |
|
|
dbe.deleteProperty(props[i].getName()); |
1080 |
|
|
} |
1081 |
|
|
} |
1082 |
malo_pichot |
1.24.2.11 |
|
1083 |
malo_pichot |
1.24.2.10 |
if (Utilities.checkString(super.repoUrl)) |
1084 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_REQUEST_URL, super.requestUrl); |
1085 |
malo_pichot |
1.24.2.11 |
|
1086 |
malo_pichot |
1.24.2.10 |
if (Utilities.checkString(super.repoUrl)) |
1087 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL, super.repoUrl); |
1088 |
malo_pichot |
1.24.2.11 |
|
1089 |
malo_pichot |
1.24.2.10 |
if (requestParams != null) { |
1090 |
malo_pichot |
1.24.2.11 |
|
1091 |
malo_pichot |
1.24.2.10 |
String verb = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_VERB, null); |
1092 |
|
|
if (Utilities.checkString(verb)) { |
1093 |
malo_pichot |
1.24.2.11 |
|
1094 |
malo_pichot |
1.24.2.10 |
String mdPrefix = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_METADATA_PREFIX, null); |
1095 |
malo_pichot |
1.24.2.11 |
|
1096 |
malo_pichot |
1.24.2.10 |
if (verb.equals(OAIRequest.VERB_STRING_GET_RECORD)) { |
1097 |
malo_pichot |
1.24.2.11 |
|
1098 |
malo_pichot |
1.24.2.10 |
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb); |
1099 |
malo_pichot |
1.24.2.11 |
|
1100 |
malo_pichot |
1.24.2.10 |
if (Utilities.checkString(mdPrefix)) |
1101 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_METADATA_PREFIX, mdPrefix); |
1102 |
malo_pichot |
1.24.2.11 |
|
1103 |
malo_pichot |
1.24.2.10 |
String id = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_IDENTIFIER, null); |
1104 |
|
|
if (Utilities.checkString(id)) |
1105 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_IDENTIFIER, id); |
1106 |
|
|
} |
1107 |
malo_pichot |
1.24.2.11 |
|
1108 |
malo_pichot |
1.24.2.10 |
if (verb.equals(OAIRequest.VERB_STRING_LIST_RECORDS)) { |
1109 |
malo_pichot |
1.24.2.11 |
|
1110 |
malo_pichot |
1.24.2.10 |
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb); |
1111 |
malo_pichot |
1.24.2.11 |
|
1112 |
malo_pichot |
1.24.2.10 |
if (Utilities.checkString(mdPrefix)) |
1113 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_METADATA_PREFIX, mdPrefix); |
1114 |
malo_pichot |
1.24.2.11 |
|
1115 |
malo_pichot |
1.24.2.10 |
String from = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_FROM, null); |
1116 |
|
|
if (Utilities.checkString(from)) |
1117 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_FROM, from); |
1118 |
malo_pichot |
1.24.2.11 |
|
1119 |
malo_pichot |
1.24.2.10 |
String until = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_UNTIL, null); |
1120 |
|
|
if (Utilities.checkString(until)) |
1121 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_UNTIL, until); |
1122 |
malo_pichot |
1.24.2.11 |
|
1123 |
malo_pichot |
1.24.2.10 |
String set = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_SET, null); |
1124 |
|
|
if (Utilities.checkString(set)) |
1125 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_SET, set); |
1126 |
malo_pichot |
1.24.2.11 |
|
1127 |
malo_pichot |
1.24.2.10 |
} |
1128 |
malo_pichot |
1.24.2.11 |
|
1129 |
malo_pichot |
1.24.2.10 |
} |
1130 |
|
|
} |
1131 |
malo_pichot |
1.24.2.11 |
|
1132 |
|
|
|
1133 |
|
|
dbe.setId(request.getVerbId()); |
1134 |
malo_pichot |
1.24.2.10 |
//deleting old info. |
1135 |
|
|
//adding latest info. |
1136 |
|
|
if (Utilities.checkString(super.responseDate)) |
1137 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED, super.responseDate); |
1138 |
|
|
if (Utilities.checkString(super.resumptionToken)) |
1139 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_RESUMPTION_TOKEN, super.resumptionToken); |
1140 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.NO_DOCS_DELETED, Integer.toString(noDocsDeleted)); |
1141 |
|
|
dbe.addProperty(LuceneDocumentBaseOAIHarvester.NO_DOCS_HARVESTED, Integer.toString(noDocsHarvested)); |
1142 |
malo_pichot |
1.24.2.11 |
|
1143 |
malo_pichot |
1.24.2.10 |
this.database.update(dbe); |
1144 |
|
|
this.database.optimize(); |
1145 |
malo_pichot |
1.24.2.11 |
|
1146 |
malo_pichot |
1.24.2.10 |
} |
1147 |
malo_pichot |
1.24.2.11 |
|
1148 |
malo_pichot |
1.24.2.10 |
} catch (Exception e) { |
1149 |
|
|
throw new SAXException(e.getMessage(), e); |
1150 |
|
|
} |
1151 |
|
|
} |
1152 |
rpandey |
1.22 |
|
1153 |
rpandey |
1.23 |
/**Generates an id to associate |
1154 |
|
|
* with a harvest |
1155 |
|
|
* |
1156 |
|
|
* @return |
1157 |
|
|
*/ |
1158 |
rpandey |
1.22 |
protected String generateNewHarvestId() { |
1159 |
|
|
return this.harvesterIdGen.generate(); |
1160 |
malo_pichot |
1.24.2.10 |
//Here we must generate a known ID to work with later |
1161 |
rpandey |
1.22 |
} |
1162 |
|
|
|
1163 |
rpandey |
1.23 |
/**Sends sax events to the current consumer |
1164 |
|
|
* with summary details of the all the past harvests |
1165 |
|
|
* |
1166 |
|
|
* @throws SAXException |
1167 |
|
|
*/ |
1168 |
rpandey |
1.22 |
public void sendPastHarvestsSummary() throws SAXException { |
1169 |
|
|
//build a query get the doc |
1170 |
|
|
try { |
1171 |
rpandey |
1.24 |
DatabaseEntity[] dbes = this.database.getEntities(); |
1172 |
rpandey |
1.22 |
if (dbes != null && dbes.length > 0) { |
1173 |
|
|
this.acquire(); |
1174 |
|
|
acquireSynchronizedXMLConsumer(); |
1175 |
|
|
super.startElement(Framework.SDXNamespaceURI, fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS, |
1176 |
|
|
Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS, null); |
1177 |
|
|
for (int i = 0; i < dbes.length; i++) { |
1178 |
rpandey |
1.24 |
DatabaseEntity dbe = dbes[i]; |
1179 |
rpandey |
1.22 |
AttributesImpl atts = new AttributesImpl(); |
1180 |
|
|
String repoUrl = null; |
1181 |
|
|
if (dbe != null) { |
1182 |
|
|
Property[] props = dbe.getProperties(); |
1183 |
|
|
if (props != null) { |
1184 |
|
|
for (int j = 0; j < props.length; j++) { |
1185 |
|
|
Property prop = props[j]; |
1186 |
|
|
if (prop != null) { |
1187 |
|
|
String propName = prop.getName(); |
1188 |
|
|
if (Utilities.checkString(propName) && !LuceneDocumentBaseOAIHarvester.OAI_HARVEST_ID.equals(propName)) { |
1189 |
|
|
String propVal = prop.getValue(); |
1190 |
|
|
if (propName.equals(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL)) { |
1191 |
|
|
repoUrl = propVal; |
1192 |
|
|
} else { |
1193 |
|
|
if (Utilities.checkString(propVal)) |
1194 |
|
|
atts.addAttribute("", propName, propName, OAIObject.Node.Type.CDATA, propVal); |
1195 |
|
|
} |
1196 |
|
|
} |
1197 |
|
|
} |
1198 |
|
|
|
1199 |
|
|
} |
1200 |
|
|
} |
1201 |
|
|
|
1202 |
|
|
} |
1203 |
|
|
sendElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.REQUEST, OAIObject.Node.Name.REQUEST, |
1204 |
|
|
atts, repoUrl); |
1205 |
|
|
} |
1206 |
|
|
super.endElement(Framework.SDXNamespaceURI, fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS, |
1207 |
|
|
Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS); |
1208 |
|
|
} |
1209 |
|
|
|
1210 |
|
|
} catch (/*IO*/Exception e) { |
1211 |
|
|
throw new SAXException(e.getMessage(), e); |
1212 |
|
|
} /* Utilities.logException(this.logger, e);TODO: what is the best way to handle this? |
1213 |
|
|
} catch (SDXException e) { |
1214 |
|
|
try { |
1215 |
|
|
e.toSAX(this); |
1216 |
|
|
} catch (ProcessingException e1) { |
1217 |
|
|
Utilities.logException(this.logger, e1); |
1218 |
|
|
} |
1219 |
|
|
} */ finally { |
1220 |
|
|
releaseSynchronizedXMLConsumer(); |
1221 |
|
|
this.release(); |
1222 |
|
|
} |
1223 |
|
|
|
1224 |
|
|
|
1225 |
|
|
} |
1226 |
|
|
|
1227 |
rpandey |
1.23 |
/**Retrieves the time when the harvester was last updated |
1228 |
|
|
* |
1229 |
|
|
* @return |
1230 |
|
|
*/ |
1231 |
rpandey |
1.22 |
public Date lastUpdated() { |
1232 |
rpandey |
1.23 |
return this.docbase.lastModificationDate(); |
1233 |
rpandey |
1.22 |
} |
1234 |
|
|
|
1235 |
rpandey |
1.23 |
/**Destroys all summary data pertaining to past harvests |
1236 |
|
|
* but not the actual oai records harvested |
1237 |
|
|
* |
1238 |
|
|
*/ |
1239 |
rpandey |
1.22 |
public void purgePastHarvestsData() { |
1240 |
|
|
try { |
1241 |
rpandey |
1.24 |
DatabaseEntity[] dbes = this.database.getEntities(); |
1242 |
rpandey |
1.22 |
for (int i = 0; i < dbes.length; i++) { |
1243 |
rpandey |
1.24 |
DatabaseEntity dbe = dbes[i]; |
1244 |
rpandey |
1.22 |
this.database.delete(dbe); |
1245 |
|
|
} |
1246 |
|
|
this.database.optimize(); |
1247 |
|
|
} catch (SDXException e) { |
1248 |
|
|
Utilities.logException(logger, e); |
1249 |
rpandey |
1.24 |
} |
1250 |
|
|
} |
1251 |
|
|
|
1252 |
|
|
/**Stores data about harvesting failures caused |
1253 |
|
|
* by problems other than oai errors sent from |
1254 |
|
|
* a queried repository |
1255 |
|
|
* |
1256 |
|
|
* @param e |
1257 |
|
|
*/ |
1258 |
|
|
protected void storeFailedHarvestData(Exception e) { |
1259 |
|
|
try { |
1260 |
|
|
//TODO: call this method when harvesting fails/major exception thrown |
1261 |
|
|
//create a special property "failedHarvest" |
1262 |
|
|
DatabaseEntity dbe = new DatabaseEntity(generateNewHarvestId()); |
1263 |
|
|
String message = "noMessage"; |
1264 |
|
|
if (e != null) message = e.getMessage(); |
1265 |
|
|
dbe.addProperty(OAI_FAILED_HARVEST, message); |
1266 |
|
|
//create a database entity and store all relevant class fields |
1267 |
|
|
dbe.addProperty(OAI_REQUEST_URL, this.requestUrl); |
1268 |
|
|
//save it to the database |
1269 |
|
|
this.database.update(dbe); |
1270 |
|
|
//TODO:later perhaps we can try to auto-reexecute these failed request, keying on the OAI_FAILED_HARVEST property ??? |
1271 |
|
|
} catch (SDXException e1) { |
1272 |
|
|
Utilities.logException(logger, e1); |
1273 |
rpandey |
1.22 |
} |
1274 |
rpandey |
1.1 |
} |
1275 |
|
|
|
1276 |
|
|
|
1277 |
malo_pichot |
1.24.2.6 |
} |