Skip to content
Snippets Groups Projects
Commit a8c0ad3c authored by mbrodhu's avatar mbrodhu
Browse files

adapted ListIdentifiers and ListRecords to ES6

parent 18d635d0
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 110 additions and 216 deletions
oaipmh-core/src/main/java/info/textgrid/middleware/DublinCoreFieldLoader.java
+ 51
− 156
View file @ a8c0ad3c
......@@ -39,14 +39,14 @@ public class DublinCoreFieldLoader {
List<String> contributors = new ArrayList<String>();
for (String field : fields) {
/* for (String field : fields) {
if (hit.getFields().get(field) != null) {
contributors.add(hit.getFields().get(field).getValues().get(0)
.toString());
}
}
}*/
return contributors;
return fillList(hit, fields);
}
/**
......@@ -79,16 +79,7 @@ public class DublinCoreFieldLoader {
*/
public static List<String> setCreator(SearchHit hit, String[] fields) {
List<String> creators = new ArrayList<String>();
for (String field : fields) {
if (hit.getFields().get(field) != null) {
creators.add(hit.getFields().get(field).getValues().get(0)
.toString());
}
}
return creators;
return fillList(hit, fields);
}
/**
......@@ -106,17 +97,7 @@ public class DublinCoreFieldLoader {
* @return all elements mapped from TetxGrid metadata scheme to Dublin Core
*/
public static List<String> setCoverage(SearchHit hit, String[] fields) {
List<String> coverages = new ArrayList<String>();
for (String field : fields) {
if (hit.getFields().get(field) != null) {
coverages.add(hit.getFields().get(field).getValues().get(0)
.toString());
}
}
return coverages;
return fillList(hit, fields);
}
/**
......@@ -137,7 +118,7 @@ public class DublinCoreFieldLoader {
public static List<String> setDate(SearchHit hit, String[] fields) {
List<String> dates = new ArrayList<String>();
System.out.println("TADA");
for (String field : fields) {
if (hit.getFields().get(field) != null) {
......@@ -173,17 +154,7 @@ public class DublinCoreFieldLoader {
* @return all elements mapped from TetxGrid metadata scheme to Dublin Core
*/
public static List<String> setDescription(SearchHit hit, String[] fields) {
List<String> descriptions = new ArrayList<String>();
for (String field : fields) {
if (hit.getFields().get(field) != null) {
descriptions.add(hit.getFields().get(field).getValues().get(0)
.toString());
}
}
return descriptions;
return fillList(hit, fields);
}
/**
......@@ -192,17 +163,7 @@ public class DublinCoreFieldLoader {
* @return all elements mapped from TetxGrid metadata scheme to Dublin Core
*/
public static List<String> setFormat(SearchHit hit, String[] fields) {
List<String> formats = new ArrayList<String>();
for (String field : fields) {
if (hit.getFields().get(field) != null) {
formats.add(hit.getFields().get(field).getValues().get(0)
.toString());
}
}
return formats;
return fillList(hit, fields);
}
/**
......@@ -221,25 +182,7 @@ public class DublinCoreFieldLoader {
* @return all elements mapped from TetxGrid metadata scheme to Dublin Core
*/
public static List<String> setIdentifier(SearchHit hit, String[] fields) {
List<String> identifiers = new ArrayList<String>();
String identifier = "";
for (String field : fields) {
if (hit.getFields().get(field) != null) {
for (int i = 0; i < hit.getFields().get(field).getValues().size(); i++) {
if (hit.getFields().get(field).getValues().get(i).toString().startsWith("textgrid")) {
identifier = hit.getFields().get(field).getValues().get(i).toString();
identifier = identifier.replace(identifier, "https://textgridrep.org/" + identifier);
identifiers.add(identifier);
} else {
identifiers.add(hit.getFields().get(field).getValues().get(i).toString());
}
}
}
}
return identifiers;
return fillList(hit, fields);
}
/**
......@@ -258,16 +201,7 @@ public class DublinCoreFieldLoader {
* @return all elements mapped from TetxGrid metadata scheme to Dublin Core
*/
public static List<String> setLanguage(SearchHit hit, String[] fields) {
List<String> languages = new ArrayList<String>();
for (String field : fields) {
if (hit.getFields().get(field) != null) {
languages.add(hit.getFields().get(field).getValues().get(0).toString());
}
}
return languages;
return fillList(hit, fields);
}
/**
......@@ -286,16 +220,7 @@ public class DublinCoreFieldLoader {
* @return all elements mapped from TetxGrid metadata scheme to Dublin Core
*/
public static List<String> setPublisher(SearchHit hit, String[] fields) {
List<String> publishers = new ArrayList<String>();
for (String field : fields) {
if (hit.getFields().get(field) != null) {
publishers.add(hit.getFields().get(field).getValues().get(0).toString());
}
}
return publishers;
return fillList(hit, fields);
}
/**
......@@ -314,18 +239,7 @@ public class DublinCoreFieldLoader {
* @return all elements mapped from TetxGrid metadata scheme to Dublin Core
*/
public static List<String> setRelation(SearchHit hit, String[] fields) {
List<String> relations = new ArrayList<String>();
for (String field : fields) {
if (hit.getFields().get(field) != null) {
for (int i = 0; i < hit.getFields().get(field).getValues().size(); i++) {
relations.add(hit.getFields().get(field).getValues().get(i).toString());
}
}
}
return relations;
return fillList(hit, fields);
}
/**
......@@ -345,9 +259,9 @@ public class DublinCoreFieldLoader {
*/
public static List<String> setRelationForWork(GetResponse responseWorkValues, String[] fields) {
for(String field : fields) {
/*for(String field : fields) {
System.out.println(field);
}
}*/
return fillList(responseWorkValues, fields);
}
......@@ -358,16 +272,7 @@ public class DublinCoreFieldLoader {
* @return all elements mapped from TetxGrid metadata scheme to Dublin Core
*/
public static List<String> setRights(SearchHit hit, String[] fields) {
List<String> rights = new ArrayList<String>();
for (String field : fields) {
if (hit.getFields().get(field) != null) {
rights.add(hit.getFields().get(field).getValues().get(0).toString());
}
}
return rights;
return fillList(hit, fields);
}
/**
......@@ -376,7 +281,6 @@ public class DublinCoreFieldLoader {
* @return all elements mapped from TetxGrid metadata scheme to Dublin Core
*/
public static List<String> setRights(GetResponse responseWorkValues, String[] fields) {
return fillList(responseWorkValues, fields);
}
......@@ -386,16 +290,7 @@ public class DublinCoreFieldLoader {
* @return all elements mapped from TetxGrid metadata scheme to Dublin Core
*/
public static List<String> setSources(SearchHit hit, String[] fields) {
List<String> sources = new ArrayList<String>();
for (String field : fields) {
if (hit.getFields().get(field) != null) {
sources.add(hit.getFields().get(field).getValues().get(0).toString());
}
}
return sources;
return fillList(hit, fields);
}
/**
......@@ -424,16 +319,7 @@ public class DublinCoreFieldLoader {
* @return all elements mapped from TetxGrid metadata scheme to Dublin Core
*/
public static List<String> setSubject(SearchHit hit, String[] fields) {
List<String> subjects = new ArrayList<String>();
for (String field : fields) {
if (hit.getFields().get(field) != null) {
subjects.add(hit.getFields().get(field).getValues().get(0).toString());
}
}
return subjects;
return fillList(hit, fields);
}
/**
......@@ -442,18 +328,7 @@ public class DublinCoreFieldLoader {
* @return all elements mapped from TetxGrid metadata scheme to Dublin Core
*/
public static List<String> setTitle(SearchHit hit, String[] fields) {
List<String> titles = new ArrayList<String>();
for (String field : fields) {
if (hit.getFields().get(field) != null) {
for (int i = 0; i < hit.getFields().get(field).getValues().size(); i++) {
titles.add(hit.getFields().get(field).getValues().get(i).toString());
}
}
}
return titles;
return fillList(hit, fields);
}
/**
......@@ -485,18 +360,7 @@ public class DublinCoreFieldLoader {
* @return all elements mapped from TetxGrid metadata scheme to Dublin Core
*/
public static List<String> setType(SearchHit hit, String[] fields) {
List<String> types = new ArrayList<String>();
for (String field : fields) {
if (hit.getFields().get(field) != null) {
for (int i = 0; i < hit.getFields().get(field).getValues().size(); i++) {
types.add(hit.getFields().get(field).getValues().get(i).toString());
}
}
}
return types;
return fillList(hit, fields);
}
public static List<String> fillList(GetResponse responseWorkValues, String[] fields){
......@@ -518,7 +382,6 @@ public class DublinCoreFieldLoader {
nestedMap2!=null && nestedMap2.get(requestedField[i])!=null ) {
valueOfRequestedField =nestedMap2.get(requestedField[i]).toString();
if(valueOfRequestedField.length()>0) {
System.out.println("VALUE: " + valueOfRequestedField);
list.add(valueOfRequestedField);
}
}
......@@ -534,5 +397,37 @@ public class DublinCoreFieldLoader {
return list;
}
public static List<String> fillList(SearchHit hit, String[] fields){
List<String> list = new ArrayList<String>();
if (fields != null) {
for (String field : fields) {
if(hit.getSourceAsMap().get(field)==null ) {
String[] requestedField = field.split("\\.");
Map<String, Object> nestedMap = hit.getSourceAsMap();
String valueOfRequestedField=null;
Map<String, Object> nestedMap2 = null;
for(int i=0; i<requestedField.length; i++) {
if(i<requestedField.length-1 && nestedMap!=null && nestedMap.get(requestedField[i])!=null) {
nestedMap2 = (Map<String, Object>)nestedMap.get(requestedField[i]);
}
if(i==requestedField.length-1 &&
nestedMap2!=null && nestedMap2.get(requestedField[i])!=null ) {
valueOfRequestedField =nestedMap2.get(requestedField[i]).toString();
if(valueOfRequestedField.length()>0) {
list.add(valueOfRequestedField);
}
}
nestedMap= nestedMap2;
}
}else {
list.add(hit.getSourceAsMap().get(field).toString());
}
}
}
return list;
}
}
oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDelivererDC.java
+ 21
− 8
View file @ a8c0ad3c
......@@ -14,12 +14,15 @@ import org.apache.commons.logging.LogFactory;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequest;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
......@@ -44,7 +47,7 @@ public class IdentifierListDelivererDC extends IdentifierListDelivererAbstract {
private String dateOfObjectCreation; // Field for the object creation in the repository
private String repositoryObjectURIPrefix; //
private String identifierField;
private static final int lifeTimeResToken = 10000;
private static final int lifeTimeResToken = 100;
public static Map<String, Integer> cursorCollector = new Hashtable<String, Integer>();
......@@ -109,14 +112,20 @@ public class IdentifierListDelivererDC extends IdentifierListDelivererAbstract {
ListIdentifiersType lit = new ListIdentifiersType();
QueryBuilder query = setOrNot(set, from, to);
SearchResponse listListIdentiferValues;
SearchResponse listListIdentiferValues;
String[] includes = this.identifierListFields;
String[] excludes = Strings.EMPTY_ARRAY;
FetchSourceContext fetchSourceContext =
new FetchSourceContext(true, includes, excludes);
SearchRequest searchRequest = new SearchRequest(OAI_ESClient.getEsIndex());
SearchRequest searchRequest = new SearchRequest(OAI_ESClient.getEsIndex()).searchType(SearchType.QUERY_THEN_FETCH);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(query);
searchSourceBuilder.size(this.searchResponseSize);
searchSourceBuilder.fetchSource(includes, excludes);
if (resumptionToken != null) {
......@@ -128,7 +137,11 @@ public class IdentifierListDelivererDC extends IdentifierListDelivererAbstract {
} else {
searchRequest.source(searchSourceBuilder);
searchRequest.scroll(TimeValue.timeValueMinutes(lifeTimeResToken));
listListIdentiferValues = OAI_ESClient.getEsClient().search(searchRequest, RequestOptions.DEFAULT);
listListIdentiferValues = OAI_ESClient.getEsClient()
.search(
searchRequest,
RequestOptions.DEFAULT
);
}
listListIdentiferValues =
hitHandling(listListIdentiferValues, lit, set, listListIdentiferValues.getScrollId());
......@@ -150,8 +163,8 @@ public class IdentifierListDelivererDC extends IdentifierListDelivererAbstract {
for (SearchHit hit : listFurtherValues.getHits().getHits()) {
i++;
if (this.textgrid) {
String datestamp = hit.getFields().get(this.dateOfObjectCreation).getValue().toString();
//hit.getSourceAsMap().get(this.dateOfObjectCreation).toString();
String datestamp = hit.getSourceAsMap().get(this.dateOfObjectCreation).toString();
try {
datestamp = OAIPMHUtillities.convertDateFormat(datestamp).toXMLFormat();
} catch (ParseException e1) {
......@@ -159,7 +172,7 @@ public class IdentifierListDelivererDC extends IdentifierListDelivererAbstract {
} catch (DatatypeConfigurationException e1) {
log.debug(e1);
}
String identifier = hit.getFields().get(this.identifierField).getValue().toString();
String identifier = hit.getSourceAsMap().get(this.identifierField).toString();
lit = setListIdentifierHeader(datestamp, identifier, lit, set);
}
......
oaipmh-core/src/main/java/info/textgrid/middleware/RecordDelivererDC.java
+ 2
− 6
View file @ a8c0ad3c
......@@ -158,13 +158,9 @@ public class RecordDelivererDC extends RecordDelivererAbstract {
}
if(i==requestedField.length-1) {
valueOfRequestedField =nestedMap2.get(requestedField[i]).toString();
}
}
}
//Map<String, Object> nestedMap2 = (Map<String, Object>)nestedMap.get("edition");
String workUri = nestedMap2.get("isEditionOf").toString();
......@@ -269,7 +265,7 @@ public class RecordDelivererDC extends RecordDelivererAbstract {
// Set DublinCore lists with content from elastic search results due to configuration.
result.setContributor(
DublinCoreFieldLoader.setContributor(responseWorkValues, this.contributorList));
DublinCoreFieldLoader.fillList(responseWorkValues, this.contributorList));
result.setCoverage(DublinCoreFieldLoader.setCoverage(responseWorkValues, this.coverageList));
result.setCreator(DublinCoreFieldLoader.setCreator(relatedWorkObject, this.creatorList));
result.setDate(DublinCoreFieldLoader.setDate(relatedWorkObject, this.dateList));
......
oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererDC.java
+ 30
− 40
View file @ a8c0ad3c
......@@ -80,7 +80,7 @@ public class RecordListDelivererDC extends RecordListDelivererAbstract {
private String[] titleList;
private String[] typeList;
private String modifiedValue;
private static final int lifeTimeResToken = 60000;
private static final int lifeTimeResToken = 600;
static Map<String, Integer> cursorCollector = new Hashtable<String, Integer>();
......@@ -112,27 +112,18 @@ public class RecordListDelivererDC extends RecordListDelivererAbstract {
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
String[] includes = this.workFields;
String[] excludes = Strings.EMPTY_ARRAY;
FetchSourceContext fetchSourceContext =
new FetchSourceContext(true, includes, excludes);
String[] includes = this.workFields;
String[] excludes = Strings.EMPTY_ARRAY;
FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes);
GetRequest getRequest = new GetRequest(OAI_ESClient.getEsIndex(), OAI_ESClient.getEsType(), id).fetchSourceContext(fetchSourceContext);
/*GetRequestBuilder getWorkValues =
OAI_ESClient.getOaiESClient()
.prepareGet()
.setIndex(OAI_ESClient.getEsIndex())
.setType(OAI_ESClient.getEsType())
.setId(id)
.setFields();*/
GetResponse responseWorkValues=null;
try {
try {
responseWorkValues = OAI_ESClient.getEsClient().get(getRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return responseWorkValues;
}
......@@ -169,21 +160,21 @@ public class RecordListDelivererDC extends RecordListDelivererAbstract {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
scrollResp=null;
if (resumptionToken == null) {
searchSourceBuilder.query(recordFilter);
searchSourceBuilder.size(this.searchResponseSize);
searchRequest.source(searchSourceBuilder);
searchSourceBuilder.query(recordFilter);
searchSourceBuilder.size(this.searchResponseSize);
searchRequest.source(searchSourceBuilder);
if (resumptionToken != null) {
SearchScrollRequest scrollRequest = new SearchScrollRequest(resumptionToken);
scrollRequest.scroll(TimeValue.timeValueSeconds(lifeTimeResToken));
try {
try {
scrollResp = OAI_ESClient.getEsClient().scroll(scrollRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
} else {
......@@ -213,10 +204,12 @@ public class RecordListDelivererDC extends RecordListDelivererAbstract {
if (hit != null && hit.getFields() != null) {
try {
if (hit.getFields().get(this.dateOfObjectCreation) != null) {
if (hit.getSourceAsMap().get(this.dateOfObjectCreation) != null) {
String datestamp =
hit.getFields().get(this.dateOfObjectCreation).getValue().toString();
hit.getSourceAsMap().get(this.dateOfObjectCreation).toString();
datestamp = OAIPMHUtillities.convertDateFormat(datestamp).toXMLFormat();
}
} catch (ParseException e1) {
log.debug(e1);
......@@ -224,27 +217,26 @@ public class RecordListDelivererDC extends RecordListDelivererAbstract {
log.debug(e1);
}
this.modifiedValue = hit.getFields().get(this.modifiedField).getValue().toString();
this.modifiedValue = hit.getSourceAsMap().get(this.modifiedField).toString();
// TextGrid search!
String workUri = "";
if (this.textgrid == true && hit.getFields().get(this.formatField).getValues().get(0)
String[] blaFields = new String[] {this.relationToFurtherMetadataObject};
if (this.textgrid == true && hit.getSourceAsMap().get(this.formatField)
.toString().equals(this.formatToFilter)) {
if (hit.getFields().get(this.relationToFurtherMetadataObject) != null) {
workUri = hit.getFields().get(this.relationToFurtherMetadataObject).getValues().get(0)
.toString().substring(this.repositoryObjectURIPrefix.length());
if (DublinCoreFieldLoader.fillList(hit, blaFields).get(0)!=null) {
workUri = DublinCoreFieldLoader.fillList(hit, blaFields).get(0).substring(this.repositoryObjectURIPrefix.length());
if (!workUri.endsWith(".0")) {
workUri = workUri.concat(".0");
}
}
dublinCoreBuilder = putContentIntoDCFieldLists(hit,
furtherDCElements(workUri, OAI_ESClient.getEsClient()));
buildRecord(recordList, set,
hit.getFields().get(this.identifierField).getValue().toString(), dublinCoreBuilder);
hit.getSourceAsMap().get(this.identifierField).toString(), dublinCoreBuilder);
}
if (this.dariah == true) {
......@@ -273,13 +265,11 @@ public class RecordListDelivererDC extends RecordListDelivererAbstract {
* @param responseWorkValues
* @return
*/
public DublinCoreBuilder putContentIntoDCFieldLists(SearchHit hit,
GetResponse responseWorkValues) {
public DublinCoreBuilder putContentIntoDCFieldLists(SearchHit hit, GetResponse responseWorkValues) {
DublinCoreBuilder result = new DublinCoreBuilder();
result.setContributor(
DublinCoreFieldLoader.setContributor(responseWorkValues, this.contributorList));
System.out.println("BLA");
result.setContributor(DublinCoreFieldLoader.setContributor(responseWorkValues, this.contributorList));
result.setCoverage(DublinCoreFieldLoader.setCoverage(responseWorkValues, this.coverageList));
result.setCreator(DublinCoreFieldLoader.setCreator(responseWorkValues, this.creatorList));
result.setDate(DublinCoreFieldLoader.setDate(responseWorkValues, this.dateList));
......@@ -310,7 +300,7 @@ public class RecordListDelivererDC extends RecordListDelivererAbstract {
GetResponse responseWorkValues) {
DublinCoreBuilder result = new DublinCoreBuilder();
result.setContributor(DublinCoreFieldLoader.setContributor(hit, this.contributorList));
result.setCoverage(DublinCoreFieldLoader.setCoverage(hit, this.coverageList));
result.setCreator(DublinCoreFieldLoader.setCreator(hit, this.creatorList));
......
oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTest.java
+ 6
− 6
View file @ a8c0ad3c
......@@ -168,7 +168,7 @@ public class OaiPmhTest {
* @throws ParseException
*/
@Test
// @Ignore
@Ignore
public void testGetRequestGetRecordTextGrid() throws ParseException {
record.setContributor(TGConstants.CONTRIBUTOR_LIST);
......@@ -236,7 +236,7 @@ public class OaiPmhTest {
* @throws ParseException
*/
@Test
//@Ignore
@Ignore
public void testListIdentifierSetsDC() throws ParseException {
OaiPmhTest.identifierList.setFieldForRange(TGConstants.RANGE_FIELD);
......@@ -247,7 +247,7 @@ public class OaiPmhTest {
OaiPmhTest.identifierList.setSearchResponseSize("100");
System.out.println("Test for the verb \"ListIdentifiers\" with succesfull response");
JAXBElement<OAIPMHType> p = this.request.getRequest("ListIdentifiers", "", "oai_dc",
"project:TGPR-26236625-1acc-b921-a5fa-53567c3eeb80", "", "", "");
"", "", "", "");
JAXB.marshal(p, System.out);
System.out.println("-----------------------------------\n");
}
......@@ -356,7 +356,7 @@ public class OaiPmhTest {
this.request.getRequest("ListRecords", "", "oai_dc", "", "", "", "");
JAXB.marshal(p, System.out);
String resToken ="";
/*String resToken ="";
for(Map.Entry<String, Integer> entry : RecordListDelivererDC.cursorCollector.entrySet()) {
resToken = entry.getKey();
}
......@@ -365,7 +365,7 @@ public class OaiPmhTest {
"", "", "", resToken);
JAXB.marshal(p2, System.out);
System.out.println("HASH MAP AFTER: ");
System.out.println(RecordListDelivererDC.cursorCollector);
System.out.println(RecordListDelivererDC.cursorCollector);*/
System.out.println("-----------------------------------\n");
......@@ -406,7 +406,7 @@ public class OaiPmhTest {
public void testListRecords() throws ParseException {
JAXBElement<OAIPMHType> r =
this.request.getRequest("ListRecords", "", "oai_dc", "", "2011-02-06T20:48:39", "", "");
this.request.getRequest("ListRecords", "", "oai_dc", "", "", "", "");
JAXB.marshal(r, System.out);
System.out.println("-----------------------------------\n");
}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment