From db59004caf5b2d8ec6bf7bc98217ac74f53d14aa Mon Sep 17 00:00:00 2001
From: "Stefan E. Funk" <funk@sub.uni-goettingen.de>
Date: Wed, 20 Oct 2021 17:40:52 +0200
Subject: [PATCH] Add configuration for idiom special cursor width
---
.../IdentifierListDelivererAbstract.java | 15 +++++++++
.../IdentifierListDelivererIDIOM.java | 4 +--
.../textgrid/middleware/OAIPMHUtilities.java | 32 ++++++++++---------
.../RecordListDelivererAbstract.java | 17 +++++++++-
.../middleware/RecordListDelivererIDIOM.java | 22 ++++++-------
.../middleware/OaiPmhTextgridOnlineTests.java | 11 ++++---
.../src/main/webapp/WEB-INF/.gitignore | 1 +
.../src/main/webapp/WEB-INF/beans.xml | 5 +++
.../webapp/WEB-INF/oaipmh.dariah.properties | 2 +-
.../webapp/WEB-INF/oaipmh.textgrid.properties | 14 +++++---
10 files changed, 85 insertions(+), 38 deletions(-)
create mode 100644 oaipmh-webapp/src/main/webapp/WEB-INF/.gitignore
diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDelivererAbstract.java b/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDelivererAbstract.java
index 2c817d91..f7477c8a 100644
--- a/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDelivererAbstract.java
+++ b/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDelivererAbstract.java
@@ -66,6 +66,7 @@ public abstract class IdentifierListDelivererAbstract implements IdentifierListD
protected String formatToFilter;
protected String specField;
protected String specFieldPrefix;
+ protected int theVerySpecialIDIOMCursorWidth;
/**
* @param textgrid
@@ -556,4 +557,18 @@ public abstract class IdentifierListDelivererAbstract implements IdentifierListD
this.specFieldPrefix = specFieldPrefix;
}
+ /**
+ * @return
+ */
+ public int getTheVerySpecialIDIOMCursorWidth() {
+ return this.theVerySpecialIDIOMCursorWidth;
+ }
+
+ /**
+ * @param theVerySpecialIDIOMCursorWidth
+ */
+ public void setTheVerySpecialIDIOMCursorWidth(int theVerySpecialIDIOMCursorWidth) {
+ this.theVerySpecialIDIOMCursorWidth = theVerySpecialIDIOMCursorWidth;
+ }
+
}
diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDelivererIDIOM.java b/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDelivererIDIOM.java
index 63cb5731..078aea0f 100644
--- a/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDelivererIDIOM.java
+++ b/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDelivererIDIOM.java
@@ -69,7 +69,7 @@ public class IdentifierListDelivererIDIOM extends IdentifierListDelivererAbstrac
scrollResp = null;
searchSourceBuilder.query(recordFilterForClassicMayan);
- searchSourceBuilder.size(30);
+ searchSourceBuilder.size(this.theVerySpecialIDIOMCursorWidth);
searchRequest.source(searchSourceBuilder);
if (resumptionToken != null) {
@@ -125,7 +125,7 @@ public class IdentifierListDelivererIDIOM extends IdentifierListDelivererAbstrac
// Check the need for a resumption token!
ResumptionTokenType resTokenForResponse = OAIPMHUtilities.getResumptionToken(completeListSize,
- resumptionToken, cursorCollector, scrollID, 30, i);
+ resumptionToken, cursorCollector, scrollID, this.theVerySpecialIDIOMCursorWidth, i);
if (resTokenForResponse != null) {
identifierList.setResumptionToken(resTokenForResponse);
}
diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/OAIPMHUtilities.java b/oaipmh-core/src/main/java/info/textgrid/middleware/OAIPMHUtilities.java
index 2475161b..95a5a960 100644
--- a/oaipmh-core/src/main/java/info/textgrid/middleware/OAIPMHUtilities.java
+++ b/oaipmh-core/src/main/java/info/textgrid/middleware/OAIPMHUtilities.java
@@ -271,8 +271,9 @@ public class OAIPMHUtilities {
*/
public static ResumptionTokenType getResumptionToken(final long completeListSize,
final String resumptionToken, Map<String, Integer> cursorCollector, final String scrollID,
- int searchResponseSize, final int i) {
- //TODO: searchResponseSize to final and configure it via propertiesFile
+ final int searchResponseSize, final int i) {
+
+ // TODO: searchResponseSize to final and configure it via propertiesFile
log.info("Creating a ResumptionToken:\n " +
"CompleteListSize: " + completeListSize + "\n" +
"ResumptionToken: " + resumptionToken + "\n" +
@@ -288,31 +289,32 @@ public class OAIPMHUtilities {
// check hash map!
// 2. Complete list size is > searchResponseSize and we have no token: we do need one!
// 3. Complete list size is <= searchResponseSize (we do not need a token! do nothing!)
- if(searchResponseSize < 100){
- //Just for idiom. Better implementation by config is TODO
- searchResponseSize = 30;
- }
+ // if(searchResponseSize < 100){
+ // //Just for idiom. Better implementation by config is TODO
+ // searchResponseSize = 30;
+ // }
if (completeListSize > searchResponseSize) {
ResumptionTokenType resTokenForResponse = new ResumptionTokenType();
if (resumptionToken != null && cursorCollector.containsKey(resumptionToken)) {
cursor = cursorCollector.get(resumptionToken).intValue() + i;
- if (cursor == 100) {
- cursor = 200;
+ if (cursor == searchResponseSize) {
+ // TODO Why not "cursor += searchResponseSize;" ??
+ cursor = 2 * searchResponseSize;
}
resTokenForResponse.setCursor(BigInteger.valueOf((long) cursor));
cursorCollector.put(scrollID, cursor);
} else {
resTokenForResponse.setCursor(BigInteger.valueOf(searchResponseSize));
cursorCollector.put(scrollID, searchResponseSize);
- if(searchResponseSize == 30){
- //AGAIN IDIOM CASE
- cursor = 30;
- }else{
- cursor = 100;
- }
- resTokenForResponse.setCursor(BigInteger.valueOf((long) cursor));
+ // if (searchResponseSize == 30) {
+ // // AGAIN IDIOM CASE
+ // cursor = 30;
+ // } else {
+ cursor = searchResponseSize;
+ // }
+ resTokenForResponse.setCursor(BigInteger.valueOf((long) cursor));
}
// Set resumption token string if cursor is less then complete list size: More objects can be
diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererAbstract.java b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererAbstract.java
index 216d0fe9..88566eb9 100644
--- a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererAbstract.java
+++ b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererAbstract.java
@@ -45,6 +45,7 @@ public abstract class RecordListDelivererAbstract implements RecordListDeliverer
protected String specFieldPrefix;
protected String modifiedValue;
+ protected int theVerySpecialIDIOMCursorWidth;
private long resultSize;
private boolean foundItems;
@@ -89,7 +90,7 @@ public abstract class RecordListDelivererAbstract implements RecordListDeliverer
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(recordFilter);
- searchSourceBuilder.size(30);
+ searchSourceBuilder.size(this.theVerySpecialIDIOMCursorWidth);
searchRequest.source(searchSourceBuilder);
SearchResponse scrollResp = new SearchResponse();
@@ -549,4 +550,18 @@ public abstract class RecordListDelivererAbstract implements RecordListDeliverer
this.specFieldPrefix = specFieldPrefix;
}
+ /**
+ * @return
+ */
+ public int getTheVerySpecialIDIOMCursorWidth() {
+ return this.theVerySpecialIDIOMCursorWidth;
+ }
+
+ /**
+ * @param theVerySpecialIDIOMCursorWidth
+ */
+ public void setTheVerySpecialIDIOMCursorWidth(int theVerySpecialIDIOMCursorWidth) {
+ this.theVerySpecialIDIOMCursorWidth = theVerySpecialIDIOMCursorWidth;
+ }
+
}
diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererIDIOM.java b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererIDIOM.java
index 7bb839c3..bb389542 100644
--- a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererIDIOM.java
+++ b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererIDIOM.java
@@ -51,10 +51,10 @@ public class RecordListDelivererIDIOM extends RecordListDelivererAbstract {
recordFilterForClassicMayan = QueryBuilders.boolQuery()
.must(rangeQuery)
- .must(QueryBuilders.matchPhraseQuery("project.id","TGPR-0e926f53-1aba-d415-ecf6-539edcd8a318"))
+ .must(QueryBuilders.matchPhraseQuery("project.id",
+ "TGPR-0e926f53-1aba-d415-ecf6-539edcd8a318"))
.must(QueryBuilders.matchPhraseQuery("format", "text/tg.inputform+rdf+xml"))
- .must(QueryBuilders.matchPhraseQuery("notes", "ARTEFACT"))
- ;
+ .must(QueryBuilders.matchPhraseQuery("notes", "ARTEFACT"));
SearchResponse scrollResp;
@@ -64,7 +64,7 @@ public class RecordListDelivererIDIOM extends RecordListDelivererAbstract {
scrollResp = null;
searchSourceBuilder.query(recordFilterForClassicMayan);
- searchSourceBuilder.size(30);
+ searchSourceBuilder.size(this.theVerySpecialIDIOMCursorWidth);
searchRequest.source(searchSourceBuilder);
log.info("resToken: " + resumptionToken);
if (resumptionToken != null) {
@@ -94,28 +94,28 @@ public class RecordListDelivererIDIOM extends RecordListDelivererAbstract {
if (completeListSize > 0) {
setFoundItems(true);
int i = 0;
-
+
for (SearchHit hit : scrollResp.getHits().getHits()) {
i++;
String textgridURI =
OAIPMHUtilities.fieldLoader(new JSONObject(hit.getSourceAsMap()), "textgridUri");
- //log.debug("IDIOM URIs: " + textgridURI);
+ // log.debug("IDIOM URIs: " + textgridURI);
// hit.getFields().get("textgridUri").getValue().toString().replace(".0", "");
- //log.debug("Processing: " + textgridURI);
-
+ // log.debug("Processing: " + textgridURI);
+
log.info(i + " Records are processed");
RecordDelivererIDIOM idiomRecord = new RecordDelivererIDIOM(true, false);
recordList.getRecord()
.add(idiomRecord.getRecordById(textgridURI.replace(".0", "")).getRecord());
}
// Check the need for a resumption token!
-
- ResumptionTokenType resTokenForResponse = OAIPMHUtilities.getResumptionToken(
- completeListSize, resumptionToken, cursorCollector, scrollID, 30, i);
+
+ ResumptionTokenType resTokenForResponse = OAIPMHUtilities.getResumptionToken(completeListSize,
+ resumptionToken, cursorCollector, scrollID, this.theVerySpecialIDIOMCursorWidth, i);
if (resTokenForResponse != null) {
recordList.setResumptionToken(resTokenForResponse);
}
diff --git a/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTextgridOnlineTests.java b/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTextgridOnlineTests.java
index f6c89b2f..b2067d82 100644
--- a/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTextgridOnlineTests.java
+++ b/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTextgridOnlineTests.java
@@ -26,7 +26,7 @@ import info.textgrid.utils.httpclient.TGHttpResponse;
*
* @author Stefan E. Funk, SUB Göttingen
*/
-@Ignore
+// @Ignore
public class OaiPmhTextgridOnlineTests {
private static final String OAI_DC_PREFIX = "oai_dc";
@@ -48,8 +48,8 @@ public class OaiPmhTextgridOnlineTests {
private static final String VERB_LIST_SETS = "ListSets";
// The OAIPMH host to be tested.
- // private static String host = "http://dev.textgridlab.org/1.0/tgoaipmh/";
- private static String host = "http://textgridlab.org/1.0/tgoaipmh/";
+ private static String host = "http://dev.textgridlab.org/1.0/tgoaipmh/";
+ // private static String host = "http://textgridlab.org/1.0/tgoaipmh/";
// NOTE Test with "mvn jetty:run" in oaipmh-webapp module folder and SSH tunneling "ssh -L
// 9302:localhost:9302 -l username textgrid-esx2.gwdg.de", please use original OAIPMH config file
@@ -453,7 +453,7 @@ public class OaiPmhTextgridOnlineTests {
System.out.println(TESTING + "testListRecordsIDIOMMETSMorePages()");
- testList(VERB_LIST_RECORDS, NO_SET, OAI_IDIOMMETS_PREFIX, 100, 30, NO_THREAD_NAME);
+ testList(VERB_LIST_RECORDS, NO_SET, OAI_IDIOMMETS_PREFIX, 120, 30, NO_THREAD_NAME);
System.out.println(OK);
}
@@ -1005,6 +1005,9 @@ public class OaiPmhTextgridOnlineTests {
if (!restok.isEmpty()) {
synchronized (OaiPmhTextgridOnlineTests.class) {
// Check <record> or <header> count, must be recordsExpectedPerRequest!
+
+ System.out.println(" ## " + recordCount + " " + recordsExpectedPerRequest);
+
if (recordCount != recordsExpectedPerRequest) {
String message = ERROR + ": " + recordOrHeader + " count mismatch, must be "
+ recordsExpectedPerRequest + " if token is provided, but is " + recordCount;
diff --git a/oaipmh-webapp/src/main/webapp/WEB-INF/.gitignore b/oaipmh-webapp/src/main/webapp/WEB-INF/.gitignore
new file mode 100644
index 00000000..47065ee3
--- /dev/null
+++ b/oaipmh-webapp/src/main/webapp/WEB-INF/.gitignore
@@ -0,0 +1 @@
+/local.properties
diff --git a/oaipmh-webapp/src/main/webapp/WEB-INF/beans.xml b/oaipmh-webapp/src/main/webapp/WEB-INF/beans.xml
index 37c18fa1..5f02bbb3 100644
--- a/oaipmh-webapp/src/main/webapp/WEB-INF/beans.xml
+++ b/oaipmh-webapp/src/main/webapp/WEB-INF/beans.xml
@@ -280,6 +280,9 @@
<constructor-arg index="1" value="${dariah}" />
<!-- TODO Why we don't need any property definitions here? -->
+
+ <property name="theVerySpecialIDIOMCursorWidth"
+ value="${theVerySpecialIDIOMCursorWidth}" />
</bean>
<bean id="RecordListDATACITE"
@@ -346,6 +349,8 @@
value="${identifierListFields}"></property>
<property name="identifierField" value="${identifierField}" />
<property name="specFieldPrefix" value="${specFieldPrefix}" />
+ <property name="theVerySpecialIDIOMCursorWidth"
+ value="${theVerySpecialIDIOMCursorWidth}" />
</bean>
<bean id="ListIdentifierDATACITE"
diff --git a/oaipmh-webapp/src/main/webapp/WEB-INF/oaipmh.dariah.properties b/oaipmh-webapp/src/main/webapp/WEB-INF/oaipmh.dariah.properties
index e9827e77..b055113c 100644
--- a/oaipmh-webapp/src/main/webapp/WEB-INF/oaipmh.dariah.properties
+++ b/oaipmh-webapp/src/main/webapp/WEB-INF/oaipmh.dariah.properties
@@ -9,7 +9,7 @@ RS_ENDPOINT = https://repository.de.dariah.eu/1.0/oaipmh
############################
elasticSearch.url = 127.0.0.1
-elasticSearch.port = 9202 9203
+elasticSearch.port = 9202, 9203
elasticSearch.clusterName = ***
elasticSearch.type = metadata
elasticSearch.itemLimit = 100
diff --git a/oaipmh-webapp/src/main/webapp/WEB-INF/oaipmh.textgrid.properties b/oaipmh-webapp/src/main/webapp/WEB-INF/oaipmh.textgrid.properties
index ab0df658..e11c7aac 100644
--- a/oaipmh-webapp/src/main/webapp/WEB-INF/oaipmh.textgrid.properties
+++ b/oaipmh-webapp/src/main/webapp/WEB-INF/oaipmh.textgrid.properties
@@ -1,6 +1,6 @@
-##################################
-## OAI-PMH Configuration File ##
-##################################
+###############################################################################
+## OAI-PMH Configuration File ##
+###############################################################################
RS_ENDPOINT = https://dev.textgridlab.org/1.0/tgoaipmh
@@ -9,7 +9,7 @@ RS_ENDPOINT = https://dev.textgridlab.org/1.0/tgoaipmh
############################
elasticSearch.url = 127.0.0.1
-elasticSearch.ports = 9202 9203
+elasticSearch.ports = 9202, 9203
elasticSearch.index = ***
elasticSearch.type = metadata
elasticSearch.itemLimit = 100
@@ -111,3 +111,9 @@ searchResponseSize = 100
specField = project.id
specFieldPrefix = project:
+
+########################################
+### IDIOM Metadata Format Settings ###
+########################################
+
+theVerySpecialIDIOMCursorWidth = 30
--
GitLab