From ef7e9889d7c2fde04d5e9365adf3dcdc9d3cf56c Mon Sep 17 00:00:00 2001
From: mbrodhu <brodhun@sub.uni-goettingen.de>
Date: Mon, 26 Sep 2016 18:32:49 +0200
Subject: [PATCH] ListIdentifiers with ResumptionToken
---
.../middleware/IdentifierListDeliverer.java | 152 +++++++++---------
.../middleware/RecordListDeliverer.java | 3 +
.../info/textgrid/middleware/OaiPmhTest.java | 3 +-
.../src/main/webapp/WEB-INF/beans.xml | 3 +-
4 files changed, 80 insertions(+), 81 deletions(-)
diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDeliverer.java b/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDeliverer.java
index 8da3a90e..92aa578a 100644
--- a/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDeliverer.java
+++ b/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDeliverer.java
@@ -9,6 +9,8 @@ import info.textgrid.middleware.oaipmh.ResumptionTokenType;
import java.math.BigInteger;
import java.text.ParseException;
+import java.util.Hashtable;
+import java.util.Map;
import javax.xml.datatype.DatatypeConfigurationException;
@@ -49,9 +51,12 @@ public class IdentifierListDeliverer {
private String repositoryObjectURIPrefix; //
private String identifierField;
- long start = 0;
- int searchResponseSize = 100;
- BigInteger cursorToAdd = BigInteger.valueOf(searchResponseSize);
+ private static Map<String, Integer> cursorCollector = new Hashtable<String, Integer >();
+
+ private int searchResponseSize;
+ //BigInteger valueToAdd = BigInteger.valueOf(searchResponseSize);
+
+ ResumptionTokenType resTokenForResponse = new ResumptionTokenType();
@@ -71,8 +76,8 @@ public class IdentifierListDeliverer {
this.textgrid = textgrid;
this.dariah = dariah;
this.oaiEsClient = oaiEsClient;
- //System.out.println("DARIAH boolean is set to: " + dariah);
- //System.out.println("TEXTGRID boolean is set to: " + textgrid);
+ System.out.println("DARIAH boolean is set to: " + dariah);
+ System.out.println("TEXTGRID boolean is set to: " + textgrid);
}
@@ -114,84 +119,45 @@ public class IdentifierListDeliverer {
// FIXME combine it with with getIdentifierListWithSet and check for empty set
- public ListIdentifiersType processIdentifierList(String from, String to, String set, String resumptionToken)
- throws ParseException {
- //System.out.println("CHECK1");
+ public ListIdentifiersType processIdentifierList(String from, String to, String set, String resumptionToken) throws ParseException {
+ System.out.println("JUHU");
ListIdentifiersType lit = new ListIdentifiersType();
- ResumptionTokenType resTokenForResponse = new ResumptionTokenType();
- String identifier = "";
- QueryBuilder query = setOrNot(set, from, to);
- SearchRequestBuilder getListIdentifiersValuesBuilder = null;
- SearchScrollRequestBuilder getListIdentifiersValuesBuilderWithRT = null;
- SearchResponse listListIdentiferValues = null;
- String scrollID = "";
-
- /*resTokenForResponse.setValue(resumptionToken);
- System.out.println("ResToken:");
- System.out.println(resumptionToken);
- System.out.println("CHECK2");
- if(lit.getResumptionToken().getValue() == null)
- System.out.println("Woho");*/
-
-
-
- //System.out.println("Value of resToken: " + lit.getResumptionToken().getValue());
-
+ QueryBuilder query = setOrNot(set, from, to);
+ SearchResponse listListIdentiferValues = null;
+ System.out.println("JUHU2");
if(resumptionToken != null){
- //System.out.println("Here should I not be");
- getListIdentifiersValuesBuilderWithRT = OAI_ESClient.getOaiESClient().prepareSearchScroll(resumptionToken).setScroll(new TimeValue(6000));
- listListIdentiferValues = getListIdentifiersValuesBuilderWithRT.execute().actionGet();
- scrollID = listListIdentiferValues.getScrollId();
+ System.out.println("BLA");
+ listListIdentiferValues = OAI_ESClient.getOaiESClient().prepareSearchScroll(resumptionToken).setScroll(new TimeValue(6000)).execute().actionGet();
}
else{
- //System.out.println("Here should I be");
- getListIdentifiersValuesBuilder = OAI_ESClient
- .getOaiESClient().prepareSearch(OAI_ESClient.getEsIndex())
- .setScroll(new TimeValue(60000))
+ System.out.println("JUHU3: " + searchResponseSize);
+ System.out.println(OAI_ESClient.getClusterName());
+ listListIdentiferValues = OAI_ESClient
+ .getOaiESClient().prepareSearch(OAI_ESClient.getEsIndex())
.setTypes(OAI_ESClient.getEsType())
+ .setScroll(new TimeValue(60000))
.addFields(identifierListFields)
.setQuery(query)
- .setSize(searchResponseSize);
- //System.out.println("AFTER QUERY");
-
- listListIdentiferValues = getListIdentifiersValuesBuilder.execute().actionGet();
- //System.out.println("AFTER EXECUTE");
- scrollID = listListIdentiferValues.getScrollId();
- //System.out.println(getListIdentifiersValuesBuilder);
- }
-
- //System.out.println("WO");
- BigInteger completeListSize = BigInteger.valueOf(listListIdentiferValues.getHits().getTotalHits()); //Number of Total Hits as BigInteger (required by oaipmh ResumptionToken)
- BigInteger cursor = BigInteger.valueOf(start); //start Value of the list. Important to increase the the cursor value of the resumption token
- cursor = cursor.add(cursorToAdd);
-
- //System.out.println("hier? " + listListIdentiferValues.getHits().getTotalHits());
-
- if (listListIdentiferValues.getHits().getTotalHits() > 0) {
- //System.out.println("TADA");
- listListIdentiferValues = hitHandling(listListIdentiferValues, identifier, lit, set);
- start=start+searchResponseSize-start;
- resTokenForResponse.setCompleteListSize(completeListSize);
- resTokenForResponse.setCursor(cursor);
- resTokenForResponse.setValue(scrollID);
- lit.setResumptionToken(resTokenForResponse);
- //lit.setResumptionToken(setResToken(bi2, bi1, listListIdentiferValues.getScrollId()));
-
- }
-
+ .setSize(searchResponseSize)
+ .execute()
+ .actionGet();
+ System.out.println("DAMDAMDAM");
+
+ }
+ System.out.println("JUHU3.5");
+ listListIdentiferValues = hitHandling(listListIdentiferValues, lit, set, listListIdentiferValues.getScrollId());
return lit;
}
+
- public SearchResponse hitHandling(SearchResponse listFurtherValues, String identifier, ListIdentifiersType lit, String set){
+ public SearchResponse hitHandling(SearchResponse listFurtherValues, ListIdentifiersType lit, String set, String resumptionToken){
+ System.out.println("JUHU4");
System.out.println("FOR SET: " + set);
+ int i = 0;
for (SearchHit hit : listFurtherValues.getHits().getHits()) {
- System.out.println("INSIDE FOR LOOP IN LIST_IDENTIFIER");
- if (this.textgrid ) {
- //System.out.println("blablubb");
- //System.out.println(dateOfObjectCreation);
- //System.out.println(hit.getFields().get(dateOfObjectCreation).getValue().toString());
-
+
+ if (this.textgrid ) {
String datestamp = hit.getFields().get(dateOfObjectCreation).getValue().toString();
//System.out.println("IF");
@@ -202,17 +168,15 @@ public class IdentifierListDeliverer {
} catch (DatatypeConfigurationException e1) {
log.debug(e1);
}
+ String identifier = hit.getFields().get(identifierField).getValue().toString();
- //System.out.println("TÜDELÜ: " + identifierField);
- identifier = hit.getFields().get(identifierField).getValue().toString();
- //System.out.println(identifier);
lit = setListIdentifierHeader(datestamp, identifier, lit, set);
}
if (this.dariah) {
- System.out.println("DARIAH! " + dateOfObjectCreation);
+ System.out.println("BLA");
datestamp = hit.getFields().get(dateOfObjectCreation).getValue().toString();
- System.out.println("AFTER DATESTAMP: " + datestamp);
+
try {
datestamp = hit.getFields().get(this.dateOfObjectCreation).getValue().toString();
datestamp = OAIPMHUtillities.convertDateFormat(datestamp).toXMLFormat();
@@ -221,13 +185,34 @@ public class IdentifierListDeliverer {
} catch (DatatypeConfigurationException e1) {
log.debug(e1);
}
- System.out.println("AFTER TRY: " + identifierField);
- System.out.println("AFTER TRY: " + hit.getFields().get(identifierField).getValue().toString());
-
- identifier = hit.getFields().get(identifierField).getValue().toString();
+
+ String identifier = hit.getFields().get(identifierField).getValue().toString();
lit = setListIdentifierHeader(datestamp, identifier, lit, set);
}
+ i++;
}
+
+
+ int cursor;
+ if(resumptionToken != null && cursorCollector.containsKey(resumptionToken)){
+ cursor = cursorCollector.get(resumptionToken).intValue() + i;
+ resTokenForResponse.setCursor(BigInteger.valueOf((long) cursor));
+ cursorCollector.put(listFurtherValues.getScrollId(), cursor);
+
+ }else {
+ resTokenForResponse.setCursor(BigInteger.valueOf(searchResponseSize));
+ cursorCollector.put(listFurtherValues.getScrollId(), searchResponseSize);
+ cursor = 0;
+ }
+
+
+ if(cursor <= listFurtherValues.getHits().getTotalHits()){
+ resTokenForResponse.setValue(listFurtherValues.getScrollId());
+ resTokenForResponse.setCompleteListSize(BigInteger.valueOf(listFurtherValues.getHits().getTotalHits()));
+ lit.setResumptionToken(resTokenForResponse);
+ }
+
+
return listFurtherValues;
}
@@ -370,4 +355,13 @@ public class IdentifierListDeliverer {
public void setIdentifierField(String identifierPrefix) {
this.identifierField = identifierPrefix;
}
+ public int getSearchResponseSize() {
+ return searchResponseSize;
+ }
+
+ public void setSearchResponseSize(int searchResponseSize) {
+ System.out.println(searchResponseSize);
+ this.searchResponseSize = searchResponseSize;
+ System.out.println(this.searchResponseSize);
+ }
}
diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDeliverer.java b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDeliverer.java
index 63130e18..f7871d1f 100644
--- a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDeliverer.java
+++ b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDeliverer.java
@@ -171,6 +171,7 @@ public class RecordListDeliverer {
QueryBuilder recordFilter;
if(this.dariah == true){
+
recordFilter = query;
}else {
recordFilter = QueryBuilders.boolQuery().must(query).must(QueryBuilders.matchPhrasePrefixQuery(formatField, formatToFilter));
@@ -191,6 +192,8 @@ public class RecordListDeliverer {
.execute()
.actionGet();
+ System.out.println("Clustername: " + OAI_ESClient.getClusterName());
+
}else{
scrollResp = OAI_ESClient.getOaiESClient().prepareSearchScroll(resumptionToken).setScroll(new TimeValue(600000)).execute().actionGet();
}
diff --git a/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTest.java b/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTest.java
index d90feeda..5e591a41 100644
--- a/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTest.java
+++ b/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTest.java
@@ -494,7 +494,8 @@ public class OaiPmhTest {
this.identifierList.setDateOfObjectCreation("administrativeMetadata.dcterms:modified");
this.identifierList.setRepositoryObjectURIPrefix("hdl:");
this.identifierList.setIdentifierField("administrativeMetadata.dcterms:identifier");
-
+ this.identifierList.setSearchResponseSize(100);
+ //System.out.println("BLABLUBB");
JAXBElement<OAIPMHType> r = this.request.getRequest("ListIdentifiers","", "oai_dc", "", "", "", "");
JAXB.marshal(r, System.out);
System.out.println("-----------------------------------\n");
diff --git a/oaipmh-webapp/src/main/webapp/WEB-INF/beans.xml b/oaipmh-webapp/src/main/webapp/WEB-INF/beans.xml
index 979d384f..4be7fdf4 100644
--- a/oaipmh-webapp/src/main/webapp/WEB-INF/beans.xml
+++ b/oaipmh-webapp/src/main/webapp/WEB-INF/beans.xml
@@ -162,7 +162,8 @@
<property name="repositoryObjectURIPrefix" value="${repositoryObjectURIPrefix}" />
<property name="identifierListFields" value="${identifierListFields}"></property>
<property name="FieldForRange" value="${rangeField}" />
- <property name="identifierField" value="${identifierField}" />
+ <property name="identifierField" value="${identifierField}" />
+ <property name="searchResponseSize" value="${searchResponseSize}" />
</bean>
<bean id="SetDeliverer" class="info.textgrid.middleware.SetDeliverer">
--
GitLab