From d917704fdf120a03520cdd861c98da257f396bf6 Mon Sep 17 00:00:00 2001
From: mbrodhu <brodhun@sub.uni-goettingen.de>
Date: Fri, 18 Nov 2016 13:24:47 +0100
Subject: [PATCH] better restoken handling by end of list
---
oaipmh-core/pom.xml | 2 +-
.../info/textgrid/middleware/OAIPMHImpl.java | 25 ++++---
.../middleware/RecordListDeliverer.java | 72 +++++++++++--------
.../info/textgrid/middleware/TGConstants.java | 1 +
.../info/textgrid/middleware/OaiPmhTest.java | 20 +++---
oaipmh-webapp/pom.xml | 2 +-
pom.xml | 2 +-
7 files changed, 72 insertions(+), 52 deletions(-)
diff --git a/oaipmh-core/pom.xml b/oaipmh-core/pom.xml
index 7bbedd62..a8782453 100644
--- a/oaipmh-core/pom.xml
+++ b/oaipmh-core/pom.xml
@@ -4,7 +4,7 @@
<parent>
<artifactId>oaipmh</artifactId>
<groupId>info.textgrid.middleware</groupId>
- <version>2.5.12-SNAPSHOT</version>
+ <version>2.5.13-SNAPSHOT</version>
</parent>
<groupId>info.textgrid.middleware</groupId>
<artifactId>oaipmh-core</artifactId>
diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/OAIPMHImpl.java b/oaipmh-core/src/main/java/info/textgrid/middleware/OAIPMHImpl.java
index 89b121f2..3ef29e26 100644
--- a/oaipmh-core/src/main/java/info/textgrid/middleware/OAIPMHImpl.java
+++ b/oaipmh-core/src/main/java/info/textgrid/middleware/OAIPMHImpl.java
@@ -307,16 +307,21 @@ public class OAIPMHImpl implements OAIPMHProducer {
}
else{
System.out.println("bla1000");
- lit = identifierList.processIdentifierList(request.getFrom(), request.getUntil(), request.getSet(), request.getResumptionToken());
- if (lit != null) {
- System.out.println("bla2000");
- oai.setListIdentifiers(lit);
- } else {
- error.setError(TGConstants.OAI_NO_RECORD_MATCH).setValue("The combination of the values "
- + request.getFrom() + " and "
- + request.getUntil()
- + " results in an empty list");
- oai.getError().add(error.setError(TGConstants.OAI_NO_RECORD_MATCH));
+ if(!RecordListDeliverer.cursorCollector.containsKey(request.getResumptionToken())){
+ error.setError(TGConstants.OAI_BAD_RESUMPTION_TOKEN).setValue("The value of the resumptionToken argument is invalid or expired");
+ oai.getError().add(error.setError(TGConstants.OAI_BAD_RESUMPTION_TOKEN));
+ }else {
+ lit = identifierList.processIdentifierList(request.getFrom(), request.getUntil(), request.getSet(), request.getResumptionToken());
+ if (lit != null) {
+ System.out.println("bla2000");
+ oai.setListIdentifiers(lit);
+ } else {
+ error.setError(TGConstants.OAI_NO_RECORD_MATCH).setValue("The combination of the values "
+ + request.getFrom() + " and "
+ + request.getUntil()
+ + " results in an empty list");
+ oai.getError().add(error.setError(TGConstants.OAI_NO_RECORD_MATCH));
+ }
}
}
diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDeliverer.java b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDeliverer.java
index 35a6e05e..fc061ca8 100644
--- a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDeliverer.java
+++ b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDeliverer.java
@@ -54,8 +54,8 @@ public class RecordListDeliverer {
private OAI_ESClient oaiEsClient;
private String[] fields;
private String[] workFields;
- private boolean textgrid;
- private boolean dariah;
+ private boolean textgrid;
+ private boolean dariah;
private String formatField;
private String formatToFilter;
private String dateOfObjectCreation;
@@ -64,7 +64,7 @@ public class RecordListDeliverer {
private String rangeField;
private String modifiedField;
private String identifierField;
- private boolean foundItems;
+ private boolean foundItems;
@@ -106,7 +106,7 @@ public class RecordListDeliverer {
private String[] typeList;
private String modifiedValue;
- private static Map<String, Integer> cursorCollector = new Hashtable<String, Integer >();
+ public static Map<String, Integer> cursorCollector = new Hashtable<String, Integer >();
private int searchResponseSize;
@@ -132,21 +132,23 @@ public class RecordListDeliverer {
* @throws UnsupportedEncodingException
*/
public GetResponse furtherDCElements(String id, Client client) {
-
- try {
- id = URLDecoder.decode(id, "UTF-8");
- } catch (UnsupportedEncodingException e) {
- e.printStackTrace();
- }
-
- GetRequestBuilder getWorkValues = OAI_ESClient.getOaiESClient()
- .prepareGet().setIndex(OAI_ESClient.getEsIndex())
- .setType(OAI_ESClient.getEsType()).setId(id)
- .setFields(this.workFields);
+ if(id != null){
+ try {
+ id = URLDecoder.decode(id, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
- GetResponse responseWorkValues = getWorkValues.execute().actionGet();
+ GetRequestBuilder getWorkValues = OAI_ESClient.getOaiESClient()
+ .prepareGet().setIndex(OAI_ESClient.getEsIndex())
+ .setType(OAI_ESClient.getEsType()).setId(id)
+ .setFields(this.workFields);
+
+ GetResponse responseWorkValues = getWorkValues.execute().actionGet();
+ return responseWorkValues;
+ }
- return responseWorkValues;
+ return null;
}
@@ -182,7 +184,7 @@ public class RecordListDeliverer {
if(resumptionToken == null){
scrollResp = OAI_ESClient.getOaiESClient().prepareSearch(OAI_ESClient.getEsIndex())
- .setScroll(new TimeValue(60000))
+ .setScroll(new TimeValue(10000))
.setTypes(OAI_ESClient.getEsType())
.addFields(this.fields)
.setQuery(recordFilter)
@@ -210,8 +212,10 @@ public class RecordListDeliverer {
//resTokenForResponse.setCursor((BigInteger.valueOf(searchResposeSize)));
//resTokenForResponse.setValue(scrollID);
int i =0;
- //System.out.println("YEAH" + scrollResp.getHits().getHits());
- for (SearchHit hit : scrollResp.getHits().getHits()) {
+ System.out.println("YEAH");
+
+ for (SearchHit hit : scrollResp.getHits().getHits()) {
+ System.out.println("YEAH FOR EACH");
i++;
System.out.println("TADA");
if (hit != null && hit.getFields() != null) {
@@ -232,12 +236,15 @@ public class RecordListDeliverer {
//System.out.println("MODIEFED: " + modifiedValue);
// TextGrid search!
- if (this.textgrid == true && hit.getFields().get(formatField).values().get(0).toString().equals(formatToFilter)) {
- String workUri = hit.getFields().get(this.relationToFurtherMetadataObject).values().get(0).toString()
- .substring(this.repositoryObjectURIPrefix.length());
-
- if (!workUri.endsWith(".0")) {
- workUri = workUri.concat(".0");
+ String workUri = "";
+ if (this.textgrid == true && hit.getFields().get(formatField).values().get(0).toString().equals(formatToFilter)) {
+ if( hit.getFields().get(this.relationToFurtherMetadataObject) != null){
+ workUri = hit.getFields().get(this.relationToFurtherMetadataObject).values().get(0).toString()
+ .substring(this.repositoryObjectURIPrefix.length());
+
+ if (!workUri.endsWith(".0")) {
+ workUri = workUri.concat(".0");
+ }
}
putContentIntoDCFieldLists(hit, furtherDCElements(workUri, OAI_ESClient.getOaiESClient()));
@@ -256,12 +263,13 @@ public class RecordListDeliverer {
}
-
+ System.out.println("BLABLABLABLA");
int cursor;
if(resumptionToken != null && cursorCollector.containsKey(resumptionToken)){
cursor = cursorCollector.get(resumptionToken).intValue() + i;
resTokenForResponse.setCursor(BigInteger.valueOf((long) cursor));
cursorCollector.put(scrollResp.getScrollId(), cursor);
+ cursorCollector.remove(resumptionToken);
}else {
resTokenForResponse.setCursor(BigInteger.valueOf(searchResponseSize));
@@ -270,12 +278,14 @@ public class RecordListDeliverer {
}
- if(cursor < scrollResp.getHits().getTotalHits()){
+
// System.out.println("JUHU");
+ if(cursor >= scrollResp.getHits().getTotalHits()){
resTokenForResponse.setValue(scrollID);
- resTokenForResponse.setCompleteListSize(BigInteger.valueOf(scrollResp.getHits().getTotalHits()));
- recordList.setResumptionToken(resTokenForResponse);
- }
+ }
+ resTokenForResponse.setCompleteListSize(BigInteger.valueOf(scrollResp.getHits().getTotalHits()));
+ recordList.setResumptionToken(resTokenForResponse);
+
} else {
setFoundItems(false);
diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/TGConstants.java b/oaipmh-core/src/main/java/info/textgrid/middleware/TGConstants.java
index f7ee4c1d..e6c0effb 100644
--- a/oaipmh-core/src/main/java/info/textgrid/middleware/TGConstants.java
+++ b/oaipmh-core/src/main/java/info/textgrid/middleware/TGConstants.java
@@ -38,6 +38,7 @@ public final class TGConstants {
public static final String OAI_METADATA_FORMAT_ERROR = "FormatError";
public static final String OAI_NO_RECORD_MATCH = "RecordMatchError";
public static final String OAI_NO_SET_HIERARCHY = "SetHierarchyError";
+ public static final String OAI_BAD_RESUMPTION_TOKEN = "badResumptionToken";
/*
* TextGrid Metadata Fields for OAI-PMH Request (Mapping to DC)
diff --git a/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTest.java b/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTest.java
index b9d2e80e..594b5214 100644
--- a/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTest.java
+++ b/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTest.java
@@ -40,17 +40,17 @@ public class OaiPmhTest {
private RecordDeliverer record = new RecordDeliverer(oaiEsClient, TGConstants.TGFields, true, false);
- //private RecordListDeliverer recordList = new RecordListDeliverer(oaiEsClient,TGConstants.TGFields, TGConstants.TGWorkFields, true, false);
- private RecordListDeliverer recordList = new RecordListDeliverer(oaiEsClient, DARIAHConstants.DARIAHFields, DARIAHConstants.DARIAHFields, false, true);
+ private RecordListDeliverer recordList = new RecordListDeliverer(oaiEsClient,TGConstants.TGFields, TGConstants.TGWorkFields, true, false);
+ //private RecordListDeliverer recordList = new RecordListDeliverer(oaiEsClient, DARIAHConstants.DARIAHFields, DARIAHConstants.DARIAHFields, false, true);
// TGConstants.TGWorkFields, true, false);
private MetadataFormatListDeliverer metadataFormatList = new MetadataFormatListDeliverer(oaiEsClient);
private SetDeliverer setListDARIAH = new SetDeliverer(oaiEsClient, false, true);
private SetDeliverer setListTextGrid = new SetDeliverer(oaiEsClient, true, false);
- private IdentifierListDeliverer identifierList = new IdentifierListDeliverer(oaiEsClient, false, true);
- //private IdentifierListDeliverer identifierList = new IdentifierListDeliverer(oaiEsClient, true, false);
- private OAIPMHImpl request = new OAIPMHImpl(oaiEsClient, this.rep, this.record, this.recordList, this.metadataFormatList, this.setListDARIAH, this.identifierList);
+ //private IdentifierListDeliverer identifierList = new IdentifierListDeliverer(oaiEsClient, false, true);
+ private IdentifierListDeliverer identifierList = new IdentifierListDeliverer(oaiEsClient, true, false);
+ private OAIPMHImpl request = new OAIPMHImpl(oaiEsClient, this.rep, this.record, this.recordList, this.metadataFormatList, this.setListTextGrid, this.identifierList);
/**
* @throws Exception
@@ -59,8 +59,8 @@ public class OaiPmhTest {
public static void setUp() throws Exception {
//sc = new SesameClient("http://localhost:9091/openrdf-workbench/repositories/textgrid-public");
- //oaiEsClient = new OAI_ESClient("localhost", "9302", "tg-esx1-dev-instance");
- oaiEsClient = new OAI_ESClient("localhost", "9302", "dariah-repository-prototype");
+ oaiEsClient = new OAI_ESClient("localhost", "9302", "tg-esx1-dev-instance");
+ //oaiEsClient = new OAI_ESClient("localhost", "9302", "dariah-repository-prototype");
OAI_ESClient.setEsIndex("textgrid-public");
OAI_ESClient.setEsType("metadata");
//oaiEsClient.setEsIndex("dariahreptest");
@@ -337,6 +337,8 @@ public class OaiPmhTest {
@Test
@Ignore
public void testListRecords2() throws ParseException {
+
+ System.out.println("TG");
this.recordList.setContributor(TGConstants.CONTRIBUTOR_LIST);
this.recordList.setCoverage(TGConstants.COVERAGE_LIST);
@@ -361,8 +363,10 @@ public class OaiPmhTest {
this.recordList.setRepositoryObjectURIPrefix(TGConstants.ITEM_IDENTIFIER_PREFIX);
this.recordList.setFieldForRange(TGConstants.CREATED);
this.recordList.setModifiedField(TGConstants.CREATED);
+ this.recordList.setIdentifierField("textgridUri");
+ this.recordList.setSearchResponseSize("100");
- JAXBElement<OAIPMHType> r = this.request.getRequest("ListRecords", "","oai_dc", "", "", "", "");
+ JAXBElement<OAIPMHType> r = this.request.getRequest("ListRecords", "","oai_dc", "project:TGPR-355b6bcd-668d-5f79-e661-582305346ed1", "", "", "");
JAXB.marshal(r, System.out);
System.out.println("-----------------------------------\n");
}
diff --git a/oaipmh-webapp/pom.xml b/oaipmh-webapp/pom.xml
index 86cff4f2..1e840361 100644
--- a/oaipmh-webapp/pom.xml
+++ b/oaipmh-webapp/pom.xml
@@ -4,7 +4,7 @@
<parent>
<artifactId>oaipmh</artifactId>
<groupId>info.textgrid.middleware</groupId>
- <version>2.5.12-SNAPSHOT</version>
+ <version>2.5.13-SNAPSHOT</version>
</parent>
<groupId>info.textgrid.middleware</groupId>
<artifactId>oaipmh-webapp</artifactId>
diff --git a/pom.xml b/pom.xml
index 8af28314..875c5ff5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>info.textgrid.middleware</groupId>
<artifactId>oaipmh</artifactId>
- <version>2.5.12-SNAPSHOT</version>
+ <version>2.5.13-SNAPSHOT</version>
<packaging>pom</packaging>
<name>DARIAHDE :: OAI-PMH DataProvider</name>
<properties>
--
GitLab