From 1af26b1a1e8eb167e7f1cb3bc601bdcbd49d0601 Mon Sep 17 00:00:00 2001 From: "Stefan E. Funk" <funk@sub.uni-goettingen.de> Date: Thu, 1 Jul 2021 16:05:54 +0200 Subject: [PATCH] Add some method javadoc Add some this. qualifiers --- .../middleware/DublinCoreBuilder.java | 11 +- .../IdentifierListDelivererIDIOM.java | 175 +- .../middleware/OpenAireIdentifierList.java | 71 +- .../textgrid/middleware/OpenAireRecord.java | 1402 +++++++++-------- .../middleware/OpenAireRecordList.java | 289 ++-- .../middleware/RecordDelivererIDIOM.java | 4 +- .../RecordListDelivererAbstract.java | 452 +++--- .../middleware/RecordListDelivererIDIOM.java | 1 - .../textgrid/middleware/OaiPmhTestTG.java | 3 +- 9 files changed, 1312 insertions(+), 1096 deletions(-) diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/DublinCoreBuilder.java b/oaipmh-core/src/main/java/info/textgrid/middleware/DublinCoreBuilder.java index 17386deb..f1d98b18 100644 --- a/oaipmh-core/src/main/java/info/textgrid/middleware/DublinCoreBuilder.java +++ b/oaipmh-core/src/main/java/info/textgrid/middleware/DublinCoreBuilder.java @@ -4,7 +4,6 @@ import java.text.ParseException; import java.util.List; import javax.xml.bind.JAXBElement; import javax.xml.datatype.DatatypeConfigurationException; - import info.textgrid.middleware.oaipmh.ElementType; import info.textgrid.middleware.oaipmh.MetadataType; import info.textgrid.middleware.oaipmh.OaiDcType; @@ -108,8 +107,8 @@ public final class DublinCoreBuilder { * </p> * * @param dates - * @throws DatatypeConfigurationException - * @throws ParseException + * @throws DatatypeConfigurationException + * @throws ParseException */ public void setDate(List<String> dates) throws ParseException, DatatypeConfigurationException { for (String dcdate : dates) { @@ -163,11 +162,11 @@ public final class DublinCoreBuilder { * @param identifiers */ public void setIdentifier(List<String> identifiers) { - + for (String dcidentifier : identifiers) { // System.out.println("hooray"); - + ElementType identifierElement = new ElementType(); identifierElement.setValue(dcidentifier); JAXBElement<ElementType> dcCoreIdentifier = oaiDcObj.createIdentifier(identifierElement); @@ -339,7 +338,7 @@ public final class DublinCoreBuilder { * @deprecated */ @Deprecated - public ObjectFactory getOaiDcObj() { + public static ObjectFactory getOaiDcObj() { return oaiDcObj; } diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDelivererIDIOM.java b/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDelivererIDIOM.java index cbc53c52..9117941c 100644 --- a/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDelivererIDIOM.java +++ b/oaipmh-core/src/main/java/info/textgrid/middleware/IdentifierListDelivererIDIOM.java @@ -2,24 +2,18 @@ package info.textgrid.middleware; import java.io.IOException; import java.text.ParseException; -import java.util.ArrayList; import java.util.Hashtable; -import java.util.List; import java.util.Map; -import org.classicmayan.tools.Queries; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.RangeQueryBuilder; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; - import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; - import info.textgrid.middleware.oaipmh.ListIdentifiersType; import info.textgrid.middleware.oaipmh.ResumptionTokenType; @@ -30,6 +24,7 @@ public class IdentifierListDelivererIDIOM extends IdentifierListDelivererAbstrac static Map<String, Integer> cursorCollector = new Hashtable<String, Integer>(); private boolean foundItems; + /** * @param textgrid * @param dariah @@ -49,104 +44,114 @@ public class IdentifierListDelivererIDIOM extends IdentifierListDelivererAbstrac String resTokenValue = resumptionToken; - //QueryBuilder recordFilterForClassicMayan; - //QueryBuilder rangeQuery; - - //rangeQuery = QueryBuilders.rangeQuery("lastModified").from(from).to(to); - /*recordFilterForClassicMayan = QueryBuilders.boolQuery().must(rangeQuery) - .must(QueryBuilders.matchPhraseQuery("project.id", "TGPR-0e926f53-1aba-d415-ecf6-539edcd8a318")) - .must(QueryBuilders.matchPhraseQuery("format", "text/tg.inputform+rdf+xml")) - .must(QueryBuilders.matchPhraseQuery("notes", "ARTEFACT"));*/ - + // QueryBuilder recordFilterForClassicMayan; + // QueryBuilder rangeQuery; + + // rangeQuery = QueryBuilders.rangeQuery("lastModified").from(from).to(to); + /* + * recordFilterForClassicMayan = QueryBuilders.boolQuery().must(rangeQuery) + * .must(QueryBuilders.matchPhraseQuery("project.id", + * "TGPR-0e926f53-1aba-d415-ecf6-539edcd8a318")) .must(QueryBuilders.matchPhraseQuery("format", + * "text/tg.inputform+rdf+xml")) .must(QueryBuilders.matchPhraseQuery("notes", "ARTEFACT")); + */ + BoolQueryBuilder recordFilterForClassicMayan; RangeQueryBuilder rangeQuery; - + rangeQuery = QueryBuilders.rangeQuery("lastModified").from(from).to(to); - + recordFilterForClassicMayan = QueryBuilders.boolQuery().must(rangeQuery) - .must(QueryBuilders.matchPhraseQuery("project.id", - "TGPR-0e926f53-1aba-d415-ecf6-539edcd8a318")) - .must(QueryBuilders.matchPhraseQuery("format", "text/tg.inputform+rdf+xml")) - .must(QueryBuilders.matchPhraseQuery("notes", "ARTEFACT")); - - + .must(QueryBuilders.matchPhraseQuery("project.id", + "TGPR-0e926f53-1aba-d415-ecf6-539edcd8a318")) + .must(QueryBuilders.matchPhraseQuery("format", "text/tg.inputform+rdf+xml")) + .must(QueryBuilders.matchPhraseQuery("notes", "ARTEFACT")); + + ListIdentifiersType identifierList = new ListIdentifiersType(); - //List<String> artefactURIs = new ArrayList<String>(); - //Queries queries = new Queries(); - - + // List<String> artefactURIs = new ArrayList<String>(); + // Queries queries = new Queries(); + + SearchResponse scrollResp; - + SearchRequest searchRequest = new SearchRequest(OAI_ESClient.getEsIndex()); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - + scrollResp = null; - + searchSourceBuilder.query(recordFilterForClassicMayan); searchSourceBuilder.size(30); searchRequest.source(searchSourceBuilder); - + if (resumptionToken == null) { - - SearchScrollRequest scrollRequest = new SearchScrollRequest(resumptionToken); - scrollRequest.scroll(TimeValue.timeValueHours(24L)); - try { - scrollResp = OAI_ESClient.getEsClient().scroll(scrollRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } else { + + SearchScrollRequest scrollRequest = new SearchScrollRequest(resumptionToken); + scrollRequest.scroll(TimeValue.timeValueHours(24L)); + try { + scrollResp = OAI_ESClient.getEsClient().scroll(scrollRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } else { // Use given resumption token here. - searchRequest.source(searchSourceBuilder); - searchRequest.scroll(TimeValue.timeValueHours(24L)); - // scrollResp = OAI_ESClient.getEsClient().prepareSearchScroll(resumptionToken) - // .setScroll(TimeValue.timeValueHours(24L)).execute().actionGet(); + searchRequest.source(searchSourceBuilder); + searchRequest.scroll(TimeValue.timeValueHours(24L)); + // scrollResp = OAI_ESClient.getEsClient().prepareSearchScroll(resumptionToken) + // .setScroll(TimeValue.timeValueHours(24L)).execute().actionGet(); } - String scrollID = scrollResp.getScrollId(); - - long completeListSize = scrollResp.getHits().totalHits; - - //long listSize = Queries.getAmountOfArtefacts(); - if (completeListSize > 0) { - setFoundItems(true); - int i = 0; - - - for (SearchHit hit : scrollResp.getHits().getHits()) { - i++; - String textgridURI = hit.getFields().get("textgridUri").getValue().toString().replace(".0", ""); - RecordDelivererIDIOM idiomRecord = new RecordDelivererIDIOM(true, false); - identifierList.getHeader() - .add(idiomRecord.getRecordById(textgridURI).getRecord().getHeader()); - } - /* for (String artefactURI : artefactURIs) { - RecordDelivererIDIOM idiomRecord = new RecordDelivererIDIOM(true, false); - identifierList.getHeader() - .add(idiomRecord.getRecordById(artefactURI).getRecord().getHeader()); - i++; - }*/ - - - // Check the need for a resumption token! - ResumptionTokenType resTokenForResponse = OAIPMHUtilities.getResumptionToken(completeListSize, - resumptionToken, cursorCollector, scrollID, 30, i); - if (resTokenForResponse != null) { - identifierList.setResumptionToken(resTokenForResponse); + String scrollID = scrollResp.getScrollId(); + + long completeListSize = scrollResp.getHits().totalHits; + + // long listSize = Queries.getAmountOfArtefacts(); + if (completeListSize > 0) { + setFoundItems(true); + int i = 0; + + for (SearchHit hit : scrollResp.getHits().getHits()) { + i++; + String textgridURI = + hit.getFields().get("textgridUri").getValue().toString().replace(".0", ""); + RecordDelivererIDIOM idiomRecord = new RecordDelivererIDIOM(true, false); + identifierList.getHeader() + .add(idiomRecord.getRecordById(textgridURI).getRecord().getHeader()); + } + /* + * for (String artefactURI : artefactURIs) { RecordDelivererIDIOM idiomRecord = new + * RecordDelivererIDIOM(true, false); identifierList.getHeader() + * .add(idiomRecord.getRecordById(artefactURI).getRecord().getHeader()); i++; } + */ + + // Check the need for a resumption token! + ResumptionTokenType resTokenForResponse = OAIPMHUtilities.getResumptionToken(completeListSize, + resumptionToken, cursorCollector, scrollID, 30, i); + if (resTokenForResponse != null) { + identifierList.setResumptionToken(resTokenForResponse); + } + } else { + setFoundItems(false); } - }else { - setFoundItems(false); - } return identifierList; } - + + /* + * (non-Javadoc) + * + * @see info.textgrid.middleware.IdentifierListDelivererAbstract#isFoundItems() + */ public boolean isFoundItems() { - return foundItems; - } + return this.foundItems; + } - public void setFoundItems(boolean foundItems) { - this.foundItems = foundItems; - } + /* + * (non-Javadoc) + * + * @see info.textgrid.middleware.IdentifierListDelivererAbstract#setFoundItems(boolean) + */ + public void setFoundItems(boolean foundItems) { + this.foundItems = foundItems; + } } diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/OpenAireIdentifierList.java b/oaipmh-core/src/main/java/info/textgrid/middleware/OpenAireIdentifierList.java index 8eb058ad..d17e42ee 100644 --- a/oaipmh-core/src/main/java/info/textgrid/middleware/OpenAireIdentifierList.java +++ b/oaipmh-core/src/main/java/info/textgrid/middleware/OpenAireIdentifierList.java @@ -2,48 +2,45 @@ package info.textgrid.middleware; import java.io.IOException; import java.text.ParseException; -import java.util.ArrayList; -import java.util.List; - import javax.xml.datatype.DatatypeConfigurationException; - import info.textgrid.middleware.oaipmh.HeaderType; import info.textgrid.middleware.oaipmh.ListIdentifiersType; import info.textgrid.middleware.oaipmh.ResumptionTokenType; public class OpenAireIdentifierList extends IdentifierListDelivererAbstract { - public ResumptionTokenType resTokenForResponse; - - public OpenAireIdentifierList(boolean textgrid, boolean dariah) { - super(textgrid, dariah); - // TODO Auto-generated constructor stub - } - - public ListIdentifiersType processIdentifierList(String from, String to, String set, String resumptionToken) throws IOException { - - ListIdentifiersType openAireIddentifiersList = new ListIdentifiersType(); - //List<String> uriList = new ArrayList<String>(); - //OAIPMHUtilities oaipmhUtilities = new OAIPMHUtilities(); - - for(String uri : getUriListOfAllEditions(from, to, set, resumptionToken)) { - - OpenAireRecord openAireRecord = new OpenAireRecord(true, false); - HeaderType header = new HeaderType(); - - try { - header = openAireRecord.getRecordById(uri.replace("textgrid:", "")).getRecord().getHeader(); - } catch (ParseException | DatatypeConfigurationException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - openAireIddentifiersList.getHeader().add(header); - } - - - //if(oaipmhUtilities.getResTokenForResponse()!=null) { - openAireIddentifiersList.setResumptionToken(getResTokenForResponse()); - //} - return openAireIddentifiersList; - } + public ResumptionTokenType resTokenForResponse; + + public OpenAireIdentifierList(boolean textgrid, boolean dariah) { + super(textgrid, dariah); + // TODO Auto-generated constructor stub + } + + public ListIdentifiersType processIdentifierList(String from, String to, String set, + String resumptionToken) throws IOException { + + ListIdentifiersType openAireIddentifiersList = new ListIdentifiersType(); + // List<String> uriList = new ArrayList<String>(); + // OAIPMHUtilities oaipmhUtilities = new OAIPMHUtilities(); + + for (String uri : getUriListOfAllEditions(from, to, set, resumptionToken)) { + + OpenAireRecord openAireRecord = new OpenAireRecord(true, false); + HeaderType header = new HeaderType(); + + try { + header = openAireRecord.getRecordById(uri.replace("textgrid:", "")).getRecord().getHeader(); + } catch (ParseException | DatatypeConfigurationException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + openAireIddentifiersList.getHeader().add(header); + } + + + // if(oaipmhUtilities.getResTokenForResponse()!=null) { + openAireIddentifiersList.setResumptionToken(getResTokenForResponse()); + // } + return openAireIddentifiersList; + } } diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/OpenAireRecord.java b/oaipmh-core/src/main/java/info/textgrid/middleware/OpenAireRecord.java index c510f393..20a33704 100644 --- a/oaipmh-core/src/main/java/info/textgrid/middleware/OpenAireRecord.java +++ b/oaipmh-core/src/main/java/info/textgrid/middleware/OpenAireRecord.java @@ -6,17 +6,14 @@ import java.io.IOException; import java.io.InputStream; import java.text.ParseException; import java.util.Properties; - import javax.xml.bind.JAXBElement; import javax.xml.bind.JAXBException; import javax.xml.datatype.DatatypeConfigurationException; - import org.apache.commons.logging.LogFactory; import org.elasticsearch.common.Strings; import org.json.JSONException; import org.json.JSONObject; import org.springframework.stereotype.Component; - import info.textgrid.middleware.oaipmh.ContributorType; import info.textgrid.middleware.oaipmh.DateType; import info.textgrid.middleware.oaipmh.GetRecordType; @@ -31,7 +28,7 @@ import info.textgrid.middleware.oaipmh.Resource.AlternateIdentifiers; import info.textgrid.middleware.oaipmh.Resource.AlternateIdentifiers.AlternateIdentifier; import info.textgrid.middleware.oaipmh.Resource.Contributors; import info.textgrid.middleware.oaipmh.Resource.Contributors.Contributor; -//import info.textgrid.middleware.oaipmh.Resource.Contributors.Contributor.NameIdentifier; +// import info.textgrid.middleware.oaipmh.Resource.Contributors.Contributor.NameIdentifier; import info.textgrid.middleware.oaipmh.Resource.Creators; import info.textgrid.middleware.oaipmh.Resource.Creators.Creator; import info.textgrid.middleware.oaipmh.Resource.Creators.Creator.NameIdentifier; @@ -46,7 +43,6 @@ import info.textgrid.middleware.oaipmh.Resource.Identifier; import info.textgrid.middleware.oaipmh.Resource.RelatedIdentifiers; import info.textgrid.middleware.oaipmh.Resource.RelatedIdentifiers.RelatedIdentifier; import info.textgrid.middleware.oaipmh.Resource.ResourceType; - import info.textgrid.middleware.oaipmh.Resource.RightsList; import info.textgrid.middleware.oaipmh.Resource.RightsList.Rights; import info.textgrid.middleware.oaipmh.Resource.Sizes; @@ -55,11 +51,7 @@ import info.textgrid.middleware.oaipmh.Resource.Subjects.Subject; import info.textgrid.middleware.oaipmh.Resource.Titles; import info.textgrid.middleware.oaipmh.Resource.Titles.Title; import info.textgrid.middleware.oaipmh.TitleType; - - -//import info.textgrid.middleware.oaipmh.ResourceType; - -//import info.textgrid.middleware.oaipmh.ResourceType; +// import info.textgrid.middleware.oaipmh.ResourceType; /** * @@ -72,638 +64,772 @@ import info.textgrid.middleware.oaipmh.TitleType; @Component public class OpenAireRecord extends RecordDelivererAbstract { - //Logger for errors and info - private static org.apache.commons.logging.Log log = LogFactory.getLog(OpenAireRecord.class); - - //Array containing all fields to get from the edition dataset in elasticsearch - private String[] fields; - - //Container object for the result of the elasticsearch request. This object contains the source of the response - private JSONObject jsonObj = new JSONObject(); - - //This variables contains the field requested in the ElasticSearch Index. The content will be given by a configuration file - private String[] titleFields; - private String[] dateFields; - private String[] contributorFields; - private String[] languageFields; - private String[] alternateIdentifierFields; - private String[] formatFields; - private String[] rightsFields; - private String[] descriptionFields; - private String[] relatedIdentifierFields; - private String[] workFields; - private String[] geoLocationFields; - private String[] versionFields; - private String[] subjectFields; - private String relationToWorkObject; - private String identifierField; - private String formatField; - private String handle; - private String sizeField; - - //Contains the setting to get connection to the ElasticSearch index. The content will be given by a configuration file - private OAI_ESClient oaiEsClient; - - //This variable contains the XML-content for a whole OpenAire file - private Resource resource = new Resource(); - - /** - * - * @param textgrid flag to indicate that the content of the record is context of the TextGrid repo - * @param dariah flag to indicate that the content of the record is context of the DARIAH repo - * @throws IOException - */ - public OpenAireRecord(boolean textgrid, boolean dariah) throws IOException { - super(textgrid, dariah); - - //Read the fields to request from elasticsearch from content file - InputStream input = null; - try { - input = new FileInputStream("/etc/dhrep/oaipmh/oaipmh.properties"); - } catch (FileNotFoundException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - Properties prop = new Properties(); - prop.load(input); - this.setHandle(prop.getProperty("oar.handle")); - this.setFields(OAIPMHUtilities.fetchArrayFromPropFile("fields", prop)[0]); - this.setFormatField(prop.getProperty("formatField")); - this.setTitleFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.titleFields", prop)[0]); - this.setDateFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.dateFields", prop)[0]); - this.setContributorFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.contributorFields", prop)[0]); - this.setLanguageFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.languageFields", prop)[0]); - this.setAlternateIdentifierFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.alternateIdentifierFields", prop)[0]); - this.setFormatFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.formatFields", prop)[0]); - this.setRightsFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.rightsFields", prop)[0]); - this.setDescriptionFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.descriptionFields", prop)[0]); - this.setRelationToWorkObject(prop.getProperty("oar.relationToWorkObject")); - this.setRelatedIdentifierFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.relatedIdentifierFields", prop)[0]); - this.setGeoLocationFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.geoLocationFields", prop)[0]); - this.setVersionFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.versionFields", prop)[0]); - this.setSubjectFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.subjectFields", prop)[0]); - this.setSizeField(prop.getProperty("oar.sizeField")); - } - - /** - * building the record XML object for the OAI-PMH response - * @throws ParseException - * @throws IOException - */ - public GetRecordType getRecordById(String idInElasticSearchIndex) throws ParseException, DatatypeConfigurationException, IOException { - jsonObj = new JSONObject(OAIPMHUtilities.getRcordByIDFromElasticSearch(idInElasticSearchIndex.replace("textgrid:", ""), this.fields, Strings.EMPTY_ARRAY).getSource()); - - HeaderType header = new HeaderType(); - header.setDatestamp(OAIPMHUtilities.datestampAsString(OAIPMHUtilities.fieldLoader(jsonObj, TGConstants.CREATED))); - - if(!idInElasticSearchIndex.startsWith(TGConstants.ITEM_IDENTIFIER_PREFIX)) { - header.setIdentifier(TGConstants.ITEM_IDENTIFIER_PREFIX + idInElasticSearchIndex); - }else { - header.setIdentifier(idInElasticSearchIndex); - } - - GetRecordType grt = OAIPMHUtilities.getRecordType(this.setOpenAireRecord(), header); - - return grt; - } - - /** - * Put the record from open-aire into the JAXBElement container to add it to an oai-pmh object - * @param idInElasticSearchIndex - * @return - * @throws ParseException - * @throws JAXBException - * @throws DatatypeConfigurationException - * @throws IOException - */ - public JAXBElement<OAIPMHtype> getRecord(String idInElasticSearchIndex) throws ParseException, JAXBException, DatatypeConfigurationException, IOException { - ObjectFactory obf = new ObjectFactory(); - JAXBElement<OAIPMHtype> oaijaxb; - OAIPMHtype oai = new OAIPMHtype(); - HeaderType header = new HeaderType(); - - jsonObj = new JSONObject(OAIPMHUtilities.getRcordByIDFromElasticSearch(idInElasticSearchIndex, this.fields, Strings.EMPTY_ARRAY).getSource()); - - //Setting the header for the oaipmh response - header.setDatestamp(OAIPMHUtilities.datestampAsString(OAIPMHUtilities.fieldLoader(jsonObj, TGConstants.CREATED))); - //The id for the response has to be starts with "textgrid:". The id in elasticsearch is missing. In consequence is optional in oaipmh - //request to write textgrid:1233.0 or jsut 1233.0. For the response the prefix has to be added - if(!idInElasticSearchIndex.startsWith(TGConstants.ITEM_IDENTIFIER_PREFIX)) { - header.setIdentifier(TGConstants.ITEM_IDENTIFIER_PREFIX + idInElasticSearchIndex); - }else { - header.setIdentifier(idInElasticSearchIndex); - } - //It is necessary to add the GetRecordType to the OAIPMH object. - GetRecordType grt = OAIPMHUtilities.getRecordType(this.setOpenAireRecord(), header); - oai.setGetRecord(grt); - oaijaxb = obf.createOAIPMH(oai); - oaijaxb.setValue(oai); - - return oaijaxb; - } - - /** - * Adding the open aire record fields to the openaire record object - * @param idInElasticSearchIndex - * @return - * @throws ParseException - * @throws DatatypeConfigurationException - * @throws IOException - */ - private Resource setOpenAireRecord() throws ParseException, DatatypeConfigurationException, IOException { - - resource.setTitles(this.addTitles()); - resource.setPublisher(this.addPublisher()); - resource.setPublicationYear(this.addPublicationYear()); - resource.setDates(this.addDates()); - resource.setContributors(this.addContributors()); - resource.setLanguage(this.addLanguae()); - resource.setAlternateIdentifiers(this.addAlternateIdentifier()); - resource.setFormats(this.addFormats()); - resource.setRightsList(this.addRights()); - resource.setDescriptions(this.addDescriptions()); - resource.setRelatedIdentifiers(this.addRelatedIdentifiers()); - resource.setIdentifier(this.addIdentifier()); - resource.setCreators(this.addCreators()); - this.addGeoLocation(); - resource.setResourceType(this.addResourceType()); - resource.setVersion(this.addVersion()); - - if(addSubjects().getSubject().get(0).getValue()!=null) { - resource.setSubjects(this.addSubjects()); //TODO what if subjects == null? - } - - resource.setSizes(this.addSize()); - - - return resource; - } - - private Sizes addSize() { - Sizes sizes = new Sizes(); - - sizes.getSize().add(OAIPMHUtilities.fieldLoader(jsonObj, sizeField) + " Bytes"); - return sizes; - } - - - private Subjects addSubjects() { - - Subjects subjects = new Subjects(); - - for(String subjectField : subjectFields) { - Subject subject = new Subject(); - subject.setValue(OAIPMHUtilities.fieldLoader(jsonObj, subjectField + ".value")); - subject.setSchemeURI(OAIPMHUtilities.fieldLoader(jsonObj, subjectField + ".id.type")); - subject.setSubjectScheme(OAIPMHUtilities.fieldLoader(jsonObj, subjectField + ".id.value")); - subjects.getSubject().add(subject); - } - if(subjects.getSubject().get(0)!=null) { - return subjects; - }else { - return null; - } - - } - private String addVersion() { - String version = ""; - for(String versionField : versionFields) { - version = OAIPMHUtilities.fieldLoader(jsonObj, versionField); - } - - return version; - } - - private ResourceType addResourceType() { - ResourceType resourceType = new ResourceType(); - - resourceType.setResourceTypeGeneral(info.textgrid.middleware.oaipmh.ResourceType.DATASET); - - return resourceType; - } - - private void addGeoLocation() { - GeoLocations geoLocations = new GeoLocations(); - for(String geoLocationField : geoLocationFields) { - GeoLocation geoLocation = new GeoLocation(); - geoLocation.setGeoLocationPlace(OAIPMHUtilities.fieldLoader(jsonObj, geoLocationField + ".value")); - geoLocations.getGeoLocation().add(geoLocation); - } - if(geoLocations.getGeoLocation().get(0).getGeoLocationPlace()!=null) { - resource.setGeoLocations(geoLocations); - } - //return geoLocations; - } - - private RelatedIdentifiers addRelatedIdentifiers() throws IOException { - RelatedIdentifiers relatedIdentifiers = new RelatedIdentifiers(); - - for (String relatedIdentifierField: relatedIdentifierFields) { - String[] workFields = {handle}; - RelatedIdentifier relatedIdentifier = new RelatedIdentifier(); - relatedIdentifier.setRelatedIdentifierType(RelatedIdentifierType.HANDLE); - //relatedIdentifier.setRelatedMetadataScheme(TGConstants.TEXTGRID_METADATASCHEME); - relatedIdentifier.setRelationType(RelationType.IS_PART_OF); - //relatedIdentifier.setSchemeType("XSD"); - //relatedIdentifier.setSchemeURI(TGConstants.TEXTGRID_METADATASCHEME_URI); - - String uriForWork = OAIPMHUtilities.fieldLoader(jsonObj, relatedIdentifierField).replace("textgrid:", ""); - relatedIdentifier.setValue(OAIPMHUtilities.fieldLoader(new JSONObject(OAIPMHUtilities.getRcordByIDFromElasticSearch(uriForWork, workFields, Strings.EMPTY_ARRAY).getSource()), handle)); - relatedIdentifiers.getRelatedIdentifier().add(relatedIdentifier); - } - - return relatedIdentifiers; - } - - private Descriptions addDescriptions() throws IOException { - Descriptions descriptions = new Descriptions(); - - for (String descriptionField: descriptionFields) { - Description description = new Description(); - //description.setOpenaireDescriptionType(OpenaireDescriptionType.ABSTRACT); - //description.setOpenaireDescriptionType(OpenaireDescriptionType.ABSTRACT); - description.setDescriptionType(OpenaireDescriptionType.ABSTRACT); - String idForWorkObject = OAIPMHUtilities.fieldLoader(jsonObj, relationToWorkObject).replace("textgrid:", ""); - JSONObject resultOfFurtherObject = new JSONObject(OAIPMHUtilities.getRcordByIDFromElasticSearch(idForWorkObject, this.workFields, Strings.EMPTY_ARRAY).getSource()); - String abstractFoFWork = - OAIPMHUtilities.fieldLoader( - resultOfFurtherObject, - descriptionField); - description.setLang(abstractFoFWork); - descriptions.getDescription().add(description); - } - return descriptions; - } - - private RightsList addRights() { - RightsList rightsList = new RightsList(); - for (String rightsField: rightsFields) { - Rights rights = new Rights(); - rights.setRightsURI(OAIPMHUtilities.fieldLoader(jsonObj, rightsField + ".licenseUri")); - rights.setValue(OAIPMHUtilities.fieldLoader(jsonObj, rightsField + ".value")); - rightsList.getRights().add(rights); - } - if(rightsList.getRights().get(0).getValue()!=null) { - Rights openAccesRight = new Rights(); - openAccesRight.setRightsURI("info:eu-repo/semantics/openAccess"); - rightsList.getRights().add(openAccesRight); - } - - return rightsList; - } - - private Formats addFormats() { - Formats formats = new Formats(); - for (String formatField: formatFields) { - formats.getFormat().add(OAIPMHUtilities.fieldLoader(jsonObj, formatField)); - } - return formats; - } - - private AlternateIdentifiers addAlternateIdentifier() { - - AlternateIdentifiers alternateIdentifiers = new AlternateIdentifiers(); - for (String alternateIdentifierField: alternateIdentifierFields) { - AlternateIdentifier alternateIdentifier = new AlternateIdentifier(); - alternateIdentifier.setValue(OAIPMHUtilities.fieldLoader(jsonObj, alternateIdentifierField)); - alternateIdentifier.setAlternateIdentifierType(TGConstants.TEXTGRIDURI_IDENTIIER_TYPE); - alternateIdentifiers.getAlternateIdentifier().add(alternateIdentifier); - } - - return alternateIdentifiers; - } - - private String addLanguae() { - - String language=null; - for (String languageField: languageFields) { - language = OAIPMHUtilities.fieldLoader(jsonObj, languageField); - } - - return language; - } - - private Contributors addContributors() { - Contributors contributors = new Contributors(); - for (String contributorField: contributorFields) { - Contributor contributorInOpenAireRecord = new Contributor(); - if(contributorField.equals("project")) { - contributorInOpenAireRecord.setContributorName(OAIPMHUtilities.fieldLoader(jsonObj, contributorField + ".value")); - contributorInOpenAireRecord.setContributorType(ContributorType.OTHER); - info.textgrid.middleware.oaipmh.Resource.Contributors.Contributor.NameIdentifier name = new info.textgrid.middleware.oaipmh.Resource.Contributors.Contributor.NameIdentifier(); - name.setNameIdentifierScheme("textgrid"); - name.setValue(OAIPMHUtilities.fieldLoader(jsonObj, contributorField + ".id")); - name.setSchemeURI("http://www.textgridlab.org/schema/textgrid-metadata_2010.xsd"); - contributorInOpenAireRecord.setNameIdentifier(name); - - }else { - contributorInOpenAireRecord.setContributorName(OAIPMHUtilities.fieldLoader(jsonObj, contributorField)); - contributorInOpenAireRecord.setContributorType(ContributorType.DATA_MANAGER); - /*info.textgrid.middleware.oaipmh.Resource.Contributors.Contributor.NameIdentifier name = new info.textgrid.middleware.oaipmh.Resource.Contributors.Contributor.NameIdentifier(); - name.setNameIdentifierScheme("textgrid"); - name.setValue(""); - name.setSchemeURI("http://www.textgridlab.org/schema/textgrid-metadata_2010.xsd"); - contributorInOpenAireRecord.setNameIdentifier(name);*/ - } - contributors.getContributor().add(contributorInOpenAireRecord); - } - - return contributors; - } - - - private Dates addDates() { - - Dates dates = new Dates(); - - for (String dateField: dateFields) { - Date dateInOpenAireRecord = new Date(); - //System.out.println(dateField); - try { - dateInOpenAireRecord.setValue(OAIPMHUtilities.datestampAsString(OAIPMHUtilities.fieldLoader(jsonObj, dateField))); - } catch (ParseException e) { - // TODO Auto-generated catch block - log.error("Could not parse date field"); - } - if (dateField.equals("created")) { - dateInOpenAireRecord.setDateType(DateType.CREATED); - } - if (dateField.equals("issued")) { - dateInOpenAireRecord.setDateType(DateType.ISSUED); - } - if (dateField.equals("lastModified")) { - dateInOpenAireRecord.setDateType(DateType.UPDATED); - } - dates.getDate().add(dateInOpenAireRecord); - } - - return dates; - } - - private Identifier addIdentifier() { - Identifier identifier = new Identifier(); - identifier.setIdentifierType("Handle"); - //System.out.println(jsonObj.get("textgridUri").toString()); - try { - identifier.setValue(jsonObj.getJSONObject("pid").get("value").toString()); - - }catch(JSONException fieldNotFound) { - identifier.setValue(jsonObj.get("textgridUri").toString()); - } - - - return identifier; - } - - private Creators addCreators() { - - Creators creators = new Creators(); - /** - * Creators Field + // Logger for errors and info + private static org.apache.commons.logging.Log log = LogFactory.getLog(OpenAireRecord.class); + + // Array containing all fields to get from the edition dataset in elasticsearch + private String[] fields; + + // Container object for the result of the elasticsearch request. This object contains the source + // of the response + private JSONObject jsonObj = new JSONObject(); + + // This variables contains the field requested in the ElasticSearch Index. The content will be + // given by a configuration file + private String[] titleFields; + private String[] dateFields; + private String[] contributorFields; + private String[] languageFields; + private String[] alternateIdentifierFields; + private String[] formatFields; + private String[] rightsFields; + private String[] descriptionFields; + private String[] relatedIdentifierFields; + private String[] workFields; + private String[] geoLocationFields; + private String[] versionFields; + private String[] subjectFields; + private String relationToWorkObject; + private String identifierField; + private String formatField; + private String handle; + private String sizeField; + + // Contains the setting to get connection to the ElasticSearch index. The content will be given by + // a configuration file + private OAI_ESClient oaiEsClient; + + // This variable contains the XML-content for a whole OpenAire file + private Resource resource = new Resource(); + + /** + * + * @param textgrid flag to indicate that the content of the record is context of the TextGrid repo + * @param dariah flag to indicate that the content of the record is context of the DARIAH repo + * @throws IOException + */ + public OpenAireRecord(boolean textgrid, boolean dariah) throws IOException { + super(textgrid, dariah); + + // Read the fields to request from elasticsearch from content file + InputStream input = null; + try { + input = new FileInputStream("/etc/dhrep/oaipmh/oaipmh.properties"); + } catch (FileNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + Properties prop = new Properties(); + prop.load(input); + this.setHandle(prop.getProperty("oar.handle")); + this.setFields(OAIPMHUtilities.fetchArrayFromPropFile("fields", prop)[0]); + this.setFormatField(prop.getProperty("formatField")); + this.setTitleFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.titleFields", prop)[0]); + this.setDateFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.dateFields", prop)[0]); + this.setContributorFields( + OAIPMHUtilities.fetchArrayFromPropFile("oar.contributorFields", prop)[0]); + this.setLanguageFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.languageFields", prop)[0]); + this.setAlternateIdentifierFields( + OAIPMHUtilities.fetchArrayFromPropFile("oar.alternateIdentifierFields", prop)[0]); + this.setFormatFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.formatFields", prop)[0]); + this.setRightsFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.rightsFields", prop)[0]); + this.setDescriptionFields( + OAIPMHUtilities.fetchArrayFromPropFile("oar.descriptionFields", prop)[0]); + this.setRelationToWorkObject(prop.getProperty("oar.relationToWorkObject")); + this.setRelatedIdentifierFields( + OAIPMHUtilities.fetchArrayFromPropFile("oar.relatedIdentifierFields", prop)[0]); + this.setGeoLocationFields( + OAIPMHUtilities.fetchArrayFromPropFile("oar.geoLocationFields", prop)[0]); + this.setVersionFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.versionFields", prop)[0]); + this.setSubjectFields(OAIPMHUtilities.fetchArrayFromPropFile("oar.subjectFields", prop)[0]); + this.setSizeField(prop.getProperty("oar.sizeField")); + } + + /** + * building the record XML object for the OAI-PMH response + * + * @throws ParseException + * @throws IOException + */ + public GetRecordType getRecordById(String idInElasticSearchIndex) + throws ParseException, DatatypeConfigurationException, IOException { + this.jsonObj = new JSONObject(OAIPMHUtilities + .getRcordByIDFromElasticSearch(idInElasticSearchIndex.replace("textgrid:", ""), this.fields, + Strings.EMPTY_ARRAY) + .getSource()); + + HeaderType header = new HeaderType(); + header.setDatestamp(OAIPMHUtilities + .datestampAsString(OAIPMHUtilities.fieldLoader(this.jsonObj, TGConstants.CREATED))); + + if (!idInElasticSearchIndex.startsWith(TGConstants.ITEM_IDENTIFIER_PREFIX)) { + header.setIdentifier(TGConstants.ITEM_IDENTIFIER_PREFIX + idInElasticSearchIndex); + } else { + header.setIdentifier(idInElasticSearchIndex); + } + + GetRecordType grt = OAIPMHUtilities.getRecordType(this.setOpenAireRecord(), header); + + return grt; + } + + /** + * Put the record from open-aire into the JAXBElement container to add it to an oai-pmh object + * + * @param idInElasticSearchIndex + * @return + * @throws ParseException + * @throws JAXBException + * @throws DatatypeConfigurationException + * @throws IOException + */ + public JAXBElement<OAIPMHtype> getRecord(String idInElasticSearchIndex) + throws ParseException, JAXBException, DatatypeConfigurationException, IOException { + ObjectFactory obf = new ObjectFactory(); + JAXBElement<OAIPMHtype> oaijaxb; + OAIPMHtype oai = new OAIPMHtype(); + HeaderType header = new HeaderType(); + + this.jsonObj = new JSONObject(OAIPMHUtilities + .getRcordByIDFromElasticSearch(idInElasticSearchIndex, this.fields, Strings.EMPTY_ARRAY) + .getSource()); + + // Setting the header for the oaipmh response + header.setDatestamp(OAIPMHUtilities + .datestampAsString(OAIPMHUtilities.fieldLoader(this.jsonObj, TGConstants.CREATED))); + // The id for the response has to be starts with "textgrid:". The id in elasticsearch is + // missing. In consequence is optional in oaipmh + // request to write textgrid:1233.0 or jsut 1233.0. For the response the prefix has to be added + if (!idInElasticSearchIndex.startsWith(TGConstants.ITEM_IDENTIFIER_PREFIX)) { + header.setIdentifier(TGConstants.ITEM_IDENTIFIER_PREFIX + idInElasticSearchIndex); + } else { + header.setIdentifier(idInElasticSearchIndex); + } + // It is necessary to add the GetRecordType to the OAIPMH object. + GetRecordType grt = OAIPMHUtilities.getRecordType(this.setOpenAireRecord(), header); + oai.setGetRecord(grt); + oaijaxb = obf.createOAIPMH(oai); + oaijaxb.setValue(oai); + + return oaijaxb; + } + + /** + * Adding the open aire record fields to the openaire record object + * + * @param idInElasticSearchIndex + * @return + * @throws ParseException + * @throws DatatypeConfigurationException + * @throws IOException + */ + private Resource setOpenAireRecord() + throws ParseException, DatatypeConfigurationException, IOException { + + this.resource.setTitles(this.addTitles()); + this.resource.setPublisher(this.addPublisher()); + this.resource.setPublicationYear(this.addPublicationYear()); + this.resource.setDates(this.addDates()); + this.resource.setContributors(this.addContributors()); + this.resource.setLanguage(this.addLanguae()); + this.resource.setAlternateIdentifiers(this.addAlternateIdentifier()); + this.resource.setFormats(this.addFormats()); + this.resource.setRightsList(this.addRights()); + this.resource.setDescriptions(this.addDescriptions()); + this.resource.setRelatedIdentifiers(this.addRelatedIdentifiers()); + this.resource.setIdentifier(this.addIdentifier()); + this.resource.setCreators(this.addCreators()); + this.addGeoLocation(); + this.resource.setResourceType(this.addResourceType()); + this.resource.setVersion(this.addVersion()); + + if (addSubjects().getSubject().get(0).getValue() != null) { + this.resource.setSubjects(this.addSubjects()); // TODO what if subjects == null? + } + + this.resource.setSizes(this.addSize()); + + + return this.resource; + } + + private Sizes addSize() { + Sizes sizes = new Sizes(); + + sizes.getSize().add(OAIPMHUtilities.fieldLoader(this.jsonObj, this.sizeField) + " Bytes"); + return sizes; + } + + + private Subjects addSubjects() { + + Subjects subjects = new Subjects(); + + for (String subjectField : this.subjectFields) { + Subject subject = new Subject(); + subject.setValue(OAIPMHUtilities.fieldLoader(this.jsonObj, subjectField + ".value")); + subject.setSchemeURI(OAIPMHUtilities.fieldLoader(this.jsonObj, subjectField + ".id.type")); + subject + .setSubjectScheme(OAIPMHUtilities.fieldLoader(this.jsonObj, subjectField + ".id.value")); + subjects.getSubject().add(subject); + } + if (subjects.getSubject().get(0) != null) { + return subjects; + } else { + return null; + } + + } + + /** + * @return + */ + private String addVersion() { + String version = ""; + for (String versionField : this.versionFields) { + version = OAIPMHUtilities.fieldLoader(this.jsonObj, versionField); + } + + return version; + } + + /** + * @return + */ + private static ResourceType addResourceType() { + ResourceType resourceType = new ResourceType(); + + resourceType.setResourceTypeGeneral(info.textgrid.middleware.oaipmh.ResourceType.DATASET); + + return resourceType; + } + + /** + * + */ + private void addGeoLocation() { + GeoLocations geoLocations = new GeoLocations(); + for (String geoLocationField : this.geoLocationFields) { + GeoLocation geoLocation = new GeoLocation(); + geoLocation + .setGeoLocationPlace( + OAIPMHUtilities.fieldLoader(this.jsonObj, geoLocationField + ".value")); + geoLocations.getGeoLocation().add(geoLocation); + } + if (geoLocations.getGeoLocation().get(0).getGeoLocationPlace() != null) { + this.resource.setGeoLocations(geoLocations); + } + // return geoLocations; + } + + /** + * @return + * @throws IOException + */ + private RelatedIdentifiers addRelatedIdentifiers() throws IOException { + RelatedIdentifiers relatedIdentifiers = new RelatedIdentifiers(); + + for (String relatedIdentifierField : this.relatedIdentifierFields) { + String[] workFields = {this.handle}; + RelatedIdentifier relatedIdentifier = new RelatedIdentifier(); + relatedIdentifier.setRelatedIdentifierType(RelatedIdentifierType.HANDLE); + // relatedIdentifier.setRelatedMetadataScheme(TGConstants.TEXTGRID_METADATASCHEME); + relatedIdentifier.setRelationType(RelationType.IS_PART_OF); + // relatedIdentifier.setSchemeType("XSD"); + // relatedIdentifier.setSchemeURI(TGConstants.TEXTGRID_METADATASCHEME_URI); + + String uriForWork = + OAIPMHUtilities.fieldLoader(this.jsonObj, relatedIdentifierField).replace("textgrid:", + ""); + relatedIdentifier.setValue(OAIPMHUtilities.fieldLoader(new JSONObject(OAIPMHUtilities + .getRcordByIDFromElasticSearch(uriForWork, workFields, Strings.EMPTY_ARRAY).getSource()), + this.handle)); + relatedIdentifiers.getRelatedIdentifier().add(relatedIdentifier); + } + + return relatedIdentifiers; + } + + /** + * @return + * @throws IOException + */ + private Descriptions addDescriptions() throws IOException { + Descriptions descriptions = new Descriptions(); + + for (String descriptionField : this.descriptionFields) { + Description description = new Description(); + // description.setOpenaireDescriptionType(OpenaireDescriptionType.ABSTRACT); + // description.setOpenaireDescriptionType(OpenaireDescriptionType.ABSTRACT); + description.setDescriptionType(OpenaireDescriptionType.ABSTRACT); + String idForWorkObject = + OAIPMHUtilities.fieldLoader(this.jsonObj, this.relationToWorkObject).replace("textgrid:", + ""); + JSONObject resultOfFurtherObject = new JSONObject(OAIPMHUtilities + .getRcordByIDFromElasticSearch(idForWorkObject, this.workFields, Strings.EMPTY_ARRAY) + .getSource()); + String abstractFoFWork = + OAIPMHUtilities.fieldLoader( + resultOfFurtherObject, + descriptionField); + description.setLang(abstractFoFWork); + descriptions.getDescription().add(description); + } + return descriptions; + } + + /** + * @return + */ + private RightsList addRights() { + RightsList rightsList = new RightsList(); + for (String rightsField : this.rightsFields) { + Rights rights = new Rights(); + rights.setRightsURI(OAIPMHUtilities.fieldLoader(this.jsonObj, rightsField + ".licenseUri")); + rights.setValue(OAIPMHUtilities.fieldLoader(this.jsonObj, rightsField + ".value")); + rightsList.getRights().add(rights); + } + if (rightsList.getRights().get(0).getValue() != null) { + Rights openAccesRight = new Rights(); + openAccesRight.setRightsURI("info:eu-repo/semantics/openAccess"); + rightsList.getRights().add(openAccesRight); + } + + return rightsList; + } + + /** + * @return + */ + private Formats addFormats() { + Formats formats = new Formats(); + for (String formatField : this.formatFields) { + formats.getFormat().add(OAIPMHUtilities.fieldLoader(this.jsonObj, formatField)); + } + return formats; + } + + /** + * @return + */ + private AlternateIdentifiers addAlternateIdentifier() { + + AlternateIdentifiers alternateIdentifiers = new AlternateIdentifiers(); + for (String alternateIdentifierField : this.alternateIdentifierFields) { + AlternateIdentifier alternateIdentifier = new AlternateIdentifier(); + alternateIdentifier + .setValue(OAIPMHUtilities.fieldLoader(this.jsonObj, alternateIdentifierField)); + alternateIdentifier.setAlternateIdentifierType(TGConstants.TEXTGRIDURI_IDENTIIER_TYPE); + alternateIdentifiers.getAlternateIdentifier().add(alternateIdentifier); + } + + return alternateIdentifiers; + } + + /** + * @return + */ + private String addLanguae() { + + String language = null; + for (String languageField : this.languageFields) { + language = OAIPMHUtilities.fieldLoader(this.jsonObj, languageField); + } + + return language; + } + + /** + * @return + */ + private Contributors addContributors() { + Contributors contributors = new Contributors(); + for (String contributorField : this.contributorFields) { + Contributor contributorInOpenAireRecord = new Contributor(); + if (contributorField.equals("project")) { + contributorInOpenAireRecord + .setContributorName( + OAIPMHUtilities.fieldLoader(this.jsonObj, contributorField + ".value")); + contributorInOpenAireRecord.setContributorType(ContributorType.OTHER); + info.textgrid.middleware.oaipmh.Resource.Contributors.Contributor.NameIdentifier name = + new info.textgrid.middleware.oaipmh.Resource.Contributors.Contributor.NameIdentifier(); + name.setNameIdentifierScheme("textgrid"); + name.setValue(OAIPMHUtilities.fieldLoader(this.jsonObj, contributorField + ".id")); + name.setSchemeURI("http://www.textgridlab.org/schema/textgrid-metadata_2010.xsd"); + contributorInOpenAireRecord.setNameIdentifier(name); + + } else { + contributorInOpenAireRecord + .setContributorName(OAIPMHUtilities.fieldLoader(this.jsonObj, contributorField)); + contributorInOpenAireRecord.setContributorType(ContributorType.DATA_MANAGER); + /* + * info.textgrid.middleware.oaipmh.Resource.Contributors.Contributor.NameIdentifier name = + * new info.textgrid.middleware.oaipmh.Resource.Contributors.Contributor.NameIdentifier(); + * name.setNameIdentifierScheme("textgrid"); name.setValue(""); + * name.setSchemeURI("http://www.textgridlab.org/schema/textgrid-metadata_2010.xsd"); + * contributorInOpenAireRecord.setNameIdentifier(name); */ - //TODO: Fields from configFile - Creator creator = new Creator(); - - creator.setCreatorName(OAIPMHUtilities.fieldLoader(jsonObj, "edition.agent.value")); - NameIdentifier nameIdentifier = new NameIdentifier(); - //TODO: creatorID Field from configfile - String creatorID = OAIPMHUtilities.fieldLoader(jsonObj, "edition.source.bibliographicCitation.author.id"); - - if (creatorID != null && creatorID.contains(":")) { - nameIdentifier.setValue(creatorID.split(":")[1]); - nameIdentifier.setNameIdentifierScheme(creatorID.split(":")[0]); - /* - * TODO Check if the scheme URI is correct - */ - nameIdentifier.setSchemeURI("https://ref.de.dariah.eu/pndsearch/pndquery.xql?id="); - }else { - nameIdentifier.setValue(creatorID); - } - creator.setNameIdentifier(nameIdentifier); - nameIdentifier.setSchemeURI("https://ref.de.dariah.eu/pndsearch/pndquery.xql?id="); - creators.getCreator().add(creator); - - return creators; - - } - - private Titles addTitles() { - Titles titles = new Titles(); - for (String titleField: titleFields) { - - Title tileInOpenAireRecord = new Title(); - tileInOpenAireRecord.setValue(OAIPMHUtilities.fieldLoader(jsonObj, titleField)); - tileInOpenAireRecord.setTitleType(TitleType.OTHER); - titles.getTitle().add(tileInOpenAireRecord); - } - - return titles; - } - - - private String addPublisher() { - /** - * Publisher fields - */ - //TODO: fields from config file - //TODO: for each field - //return OAIPMHUtilities.fieldLoader(jsonObj, "edition.source.bibliographicCitation.publisher.value"); - return "TextGrid"; - - } - - private String addPublicationYear() throws ParseException, DatatypeConfigurationException { - /** - * Publication Year fields - */ - //TODO: fields from config file - - return Integer.toString(OAIPMHUtilities.convertDateFormat(OAIPMHUtilities.fieldLoader(jsonObj, "created")).getYear()); - } - /** - * @param formatField - */ - public void setFormatField(String formatField) { - this.formatField = formatField; - } - - public String getFormatField() { - return formatField; - } - /** - * @param fields - */ - public void setFields(String[] fields) { - this.fields = fields; - } - + } + contributors.getContributor().add(contributorInOpenAireRecord); + } + + return contributors; + } + + /** + * @return + */ + private Dates addDates() { + + Dates dates = new Dates(); + + for (String dateField : this.dateFields) { + Date dateInOpenAireRecord = new Date(); + // System.out.println(dateField); + try { + dateInOpenAireRecord.setValue( + OAIPMHUtilities + .datestampAsString(OAIPMHUtilities.fieldLoader(this.jsonObj, dateField))); + } catch (ParseException e) { + // TODO Auto-generated catch block + log.error("Could not parse date field"); + } + if (dateField.equals("created")) { + dateInOpenAireRecord.setDateType(DateType.CREATED); + } + if (dateField.equals("issued")) { + dateInOpenAireRecord.setDateType(DateType.ISSUED); + } + if (dateField.equals("lastModified")) { + dateInOpenAireRecord.setDateType(DateType.UPDATED); + } + dates.getDate().add(dateInOpenAireRecord); + } + + return dates; + } + + /** + * @return + */ + private Identifier addIdentifier() { + Identifier identifier = new Identifier(); + identifier.setIdentifierType("Handle"); + // System.out.println(jsonObj.get("textgridUri").toString()); + try { + identifier.setValue(this.jsonObj.getJSONObject("pid").get("value").toString()); + + } catch (JSONException fieldNotFound) { + identifier.setValue(this.jsonObj.get("textgridUri").toString()); + } + + return identifier; + } + + /** + * @return + */ + private Creators addCreators() { + + Creators creators = new Creators(); /** - * @return + * Creators Field */ - public String[] getFields() { - return this.fields; - } - - public OAI_ESClient getOaiEsClient() { - return oaiEsClient; - } - - public void setOaiEsClient(OAI_ESClient oaiEsClient) { - this.oaiEsClient = oaiEsClient; - } - - public String[] getTitleFields() { - return titleFields; - } - - public void setTitleFields(String[] titleFields) { - this.titleFields = titleFields; - } - - public Resource getResource() { - return resource; - } - - public void setResource(Resource resource) { - this.resource = resource; - } - - public String[] getDateFields() { - return dateFields; - } - - public void setDateFields(String[] dateFields) { - this.dateFields = dateFields; - } - - public String[] getContributorFields() { - return contributorFields; - } - - public void setContributorFields(String[] contributorFields) { - this.contributorFields = contributorFields; - } - - public String[] getLanguageFields() { - return languageFields; - } - - public void setLanguageFields(String[] languageFields) { - this.languageFields = languageFields; - } - - public String[] getAlternateIdentifierFields() { - return alternateIdentifierFields; - } - - public void setAlternateIdentifierFields(String[] alternateIdentifierFields) { - this.alternateIdentifierFields = alternateIdentifierFields; - } - - public String[] getFormatFields() { - return formatFields; - } - - public void setFormatFields(String[] formatFields) { - this.formatFields = formatFields; - } - - - - public String[] getRightsFields() { - return rightsFields; - } - - public void setRightsFields(String[] rightsFields) { - this.rightsFields = rightsFields; - } - - public String[] getDescriptionFields() { - return descriptionFields; - } - - public void setDescriptionFields(String[] descriptionFields) { - this.descriptionFields = descriptionFields; - } - - public String[] getRelatedIdentifierFields() { - return relatedIdentifierFields; - } - - public void setRelatedIdentifierFields(String[] relatedIdentifierFields) { - this.relatedIdentifierFields = relatedIdentifierFields; - } - - public String getRelationToWorkObject() { - return relationToWorkObject; - } - - public void setRelationToWorkObject(String relationToWorkObject) { - this.relationToWorkObject = relationToWorkObject; - } - + // TODO: Fields from configFile + Creator creator = new Creator(); + + creator.setCreatorName(OAIPMHUtilities.fieldLoader(this.jsonObj, "edition.agent.value")); + NameIdentifier nameIdentifier = new NameIdentifier(); + // TODO: creatorID Field from configfile + String creatorID = + OAIPMHUtilities.fieldLoader(this.jsonObj, "edition.source.bibliographicCitation.author.id"); + + if (creatorID != null && creatorID.contains(":")) { + nameIdentifier.setValue(creatorID.split(":")[1]); + nameIdentifier.setNameIdentifierScheme(creatorID.split(":")[0]); + /* + * TODO Check if the scheme URI is correct + */ + nameIdentifier.setSchemeURI("https://ref.de.dariah.eu/pndsearch/pndquery.xql?id="); + } else { + nameIdentifier.setValue(creatorID); + } + creator.setNameIdentifier(nameIdentifier); + nameIdentifier.setSchemeURI("https://ref.de.dariah.eu/pndsearch/pndquery.xql?id="); + creators.getCreator().add(creator); + + return creators; + + } + + /** + * @return + */ + private Titles addTitles() { + Titles titles = new Titles(); + for (String titleField : this.titleFields) { + + Title tileInOpenAireRecord = new Title(); + tileInOpenAireRecord.setValue(OAIPMHUtilities.fieldLoader(this.jsonObj, titleField)); + tileInOpenAireRecord.setTitleType(TitleType.OTHER); + titles.getTitle().add(tileInOpenAireRecord); + } + + return titles; + } + + /** + * @return + */ + private static String addPublisher() { /** - * @return + * Publisher fields */ - public String[] getWorkFields() { - return this.workFields; - } - + // TODO: fields from config file + // TODO: for each field + // return OAIPMHUtilities.fieldLoader(jsonObj, + // "edition.source.bibliographicCitation.publisher.value"); + return "TextGrid"; + + } + + /** + * @return + * @throws ParseException + * @throws DatatypeConfigurationException + */ + private String addPublicationYear() throws ParseException, DatatypeConfigurationException { /** - * @param workFields + * Publication Year fields */ - public void setWorkFields(String[] workFields) { - this.workFields = workFields; - } - - public String getIdentifierField() { - return identifierField; - } - - public void setIdentifierField(String identifierField) { - this.identifierField = identifierField; - } - - public String[] getGeoLocationFields() { - return geoLocationFields; - } - - public void setGeoLocationFields(String[] geoLocationFields) { - this.geoLocationFields = geoLocationFields; - } - - public String getHandle() { - return handle; - } - - public void setHandle(String handle) { - this.handle = handle; - } - - public String[] getVersionFields() { - return versionFields; - } - - public void setVersionFields(String[] versionFields) { - this.versionFields = versionFields; - } - - public String[] getSubjectFields() { - return subjectFields; - } - - public void setSubjectFields(String[] subjectFields) { - this.subjectFields = subjectFields; - } - - public String getSizeField() { - return sizeField; - } - - public void setSizeField(String size) { - this.sizeField = size; - } - - - -} \ No newline at end of file + // TODO: fields from config file + + return Integer.toString(OAIPMHUtilities + .convertDateFormat(OAIPMHUtilities.fieldLoader(this.jsonObj, "created")).getYear()); + } + + /** + * @param formatField + */ + public void setFormatField(String formatField) { + this.formatField = formatField; + } + + /** + * @return + */ + public String getFormatField() { + return this.formatField; + } + + /** + * @param fields + */ + public void setFields(String[] fields) { + this.fields = fields; + } + + /** + * @return + */ + public String[] getFields() { + return this.fields; + } + + public OAI_ESClient getOaiEsClient() { + return this.oaiEsClient; + } + + public void setOaiEsClient(OAI_ESClient oaiEsClient) { + this.oaiEsClient = oaiEsClient; + } + + public String[] getTitleFields() { + return this.titleFields; + } + + public void setTitleFields(String[] titleFields) { + this.titleFields = titleFields; + } + + public Resource getResource() { + return this.resource; + } + + public void setResource(Resource resource) { + this.resource = resource; + } + + public String[] getDateFields() { + return this.dateFields; + } + + public void setDateFields(String[] dateFields) { + this.dateFields = dateFields; + } + + public String[] getContributorFields() { + return this.contributorFields; + } + + public void setContributorFields(String[] contributorFields) { + this.contributorFields = contributorFields; + } + + public String[] getLanguageFields() { + return this.languageFields; + } + + public void setLanguageFields(String[] languageFields) { + this.languageFields = languageFields; + } + + public String[] getAlternateIdentifierFields() { + return this.alternateIdentifierFields; + } + + public void setAlternateIdentifierFields(String[] alternateIdentifierFields) { + this.alternateIdentifierFields = alternateIdentifierFields; + } + + public String[] getFormatFields() { + return this.formatFields; + } + + public void setFormatFields(String[] formatFields) { + this.formatFields = formatFields; + } + + + + public String[] getRightsFields() { + return this.rightsFields; + } + + public void setRightsFields(String[] rightsFields) { + this.rightsFields = rightsFields; + } + + public String[] getDescriptionFields() { + return this.descriptionFields; + } + + public void setDescriptionFields(String[] descriptionFields) { + this.descriptionFields = descriptionFields; + } + + public String[] getRelatedIdentifierFields() { + return this.relatedIdentifierFields; + } + + public void setRelatedIdentifierFields(String[] relatedIdentifierFields) { + this.relatedIdentifierFields = relatedIdentifierFields; + } + + public String getRelationToWorkObject() { + return this.relationToWorkObject; + } + + public void setRelationToWorkObject(String relationToWorkObject) { + this.relationToWorkObject = relationToWorkObject; + } + + /** + * @return + */ + public String[] getWorkFields() { + return this.workFields; + } + + /** + * @param workFields + */ + public void setWorkFields(String[] workFields) { + this.workFields = workFields; + } + + /** + * @return + */ + public String getIdentifierField() { + return this.identifierField; + } + + /** + * @param identifierField + */ + public void setIdentifierField(String identifierField) { + this.identifierField = identifierField; + } + + /** + * @return + */ + public String[] getGeoLocationFields() { + return this.geoLocationFields; + } + + /** + * @param geoLocationFields + */ + public void setGeoLocationFields(String[] geoLocationFields) { + this.geoLocationFields = geoLocationFields; + } + + /** + * @return + */ + public String getHandle() { + return this.handle; + } + + /** + * @param handle + */ + public void setHandle(String handle) { + this.handle = handle; + } + + /** + * @return + */ + public String[] getVersionFields() { + return this.versionFields; + } + + /** + * @param versionFields + */ + public void setVersionFields(String[] versionFields) { + this.versionFields = versionFields; + } + + /** + * @return + */ + public String[] getSubjectFields() { + return this.subjectFields; + } + + /** + * @param subjectFields + */ + public void setSubjectFields(String[] subjectFields) { + this.subjectFields = subjectFields; + } + + /** + * @return + */ + public String getSizeField() { + return this.sizeField; + } + + /** + * @param size + */ + public void setSizeField(String size) { + this.sizeField = size; + } + +} diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/OpenAireRecordList.java b/oaipmh-core/src/main/java/info/textgrid/middleware/OpenAireRecordList.java index dcc08069..fbff014e 100644 --- a/oaipmh-core/src/main/java/info/textgrid/middleware/OpenAireRecordList.java +++ b/oaipmh-core/src/main/java/info/textgrid/middleware/OpenAireRecordList.java @@ -2,133 +2,180 @@ package info.textgrid.middleware; import java.io.IOException; import java.text.ParseException; -import java.util.Hashtable; -import java.util.Map; - import javax.xml.datatype.DatatypeConfigurationException; - import info.textgrid.middleware.oaipmh.GetRecordType; import info.textgrid.middleware.oaipmh.HeaderType; import info.textgrid.middleware.oaipmh.ListRecordsType; import info.textgrid.middleware.oaipmh.ResumptionTokenType; public class OpenAireRecordList extends RecordListDelivererAbstract { - - //protected static Map<String, Integer> cursorCollector = new Hashtable<String, Integer>(); - - - private String rangeField; - private String formatField; - private String formatToFilter; - private int searchResponseSize; - private long resultSize; - private boolean foundItems; - public ResumptionTokenType resTokenForResponse; - - public OpenAireRecordList(boolean textgrid, boolean dariah) { - super(textgrid, dariah); - } - - - public ListRecordsType getRecords(String from, String to, String set, String resumptionToken){ - ListRecordsType openAireRecordList = new ListRecordsType(); - OpenAireRecord openAireRecord; - - for(String uri : getUriList(from, to, set, resumptionToken)) { - try { - try { - openAireRecord = new OpenAireRecord(true, false); - GetRecordType grt = openAireRecord.getRecordById(uri.replace("textgrid:", "")); - openAireRecordList.getRecord().add(grt.getRecord()); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - } catch (ParseException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - - } catch (DatatypeConfigurationException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - //if (resTokenForResponse != null) { - //System.out.println("RESTOKEN: "); - //System.out.println(getResTokenForResponse().getValue()); - //System.out.println(getResumptionToken()); - //resTokenForResponse = OAIPMHUtilities.getResumptionToken( - // 94146, resumptionToken, cursorCollector, scrollID, this.searchResponseSize, i) - openAireRecordList.setResumptionToken(getResTokenForResponse()); - //} - - return openAireRecordList; - } - - @Override - public HeaderType setHeader(String set, String headerIdentifier, String setSpec) { - // TODO Auto-generated method stub - return null; - } - - //GETTER AND SETTER - - public String getRangeField() { - return rangeField; - } - - public void setRangeField(String rangeField) { - this.rangeField = rangeField; - } - - - /** - * @param formatToFilter - */ - public void setFormatToFilter(String formatToFilter) { - this.formatToFilter = formatToFilter; - } - - /** - * @param formatField - */ - public void setFormatField(String formatField) { - this.formatField = formatField; - } - - public int getSearchResponseSize() { - return searchResponseSize; - } - - public void setSearchResponseSize(int searchResponseSize) { - this.searchResponseSize = searchResponseSize; - } - - public long getResultSize() { - return resultSize; - } - - public void setResultSize(long resultSize) { - this.resultSize = resultSize; - } - - public boolean isFoundItems() { - return foundItems; - } - - public void setFoundItems(boolean foundItems) { - this.foundItems = foundItems; - } - - - /*public ResumptionTokenType getResTokenForResponse() { - return resTokenForResponse; - } - - public void setResTokenForResponse(ResumptionTokenType resTokenForResponse) { - this.resTokenForResponse = resTokenForResponse; - }*/ + + // protected static Map<String, Integer> cursorCollector = new Hashtable<String, Integer>(); + + private String rangeField; + private String formatField; + private String formatToFilter; + private int searchResponseSize; + private long resultSize; + private boolean foundItems; + public ResumptionTokenType resTokenForResponse; + + /** + * @param textgrid + * @param dariah + */ + public OpenAireRecordList(boolean textgrid, boolean dariah) { + super(textgrid, dariah); + } + + /* + * (non-Javadoc) + * + * @see info.textgrid.middleware.RecordListDelivererInterface#getRecords(java.lang.String, + * java.lang.String, java.lang.String, java.lang.String) + */ + public ListRecordsType getRecords(String from, String to, String set, String resumptionToken) { + ListRecordsType openAireRecordList = new ListRecordsType(); + OpenAireRecord openAireRecord; + + for (String uri : getUriList(from, to, set, resumptionToken)) { + try { + try { + openAireRecord = new OpenAireRecord(true, false); + GetRecordType grt = openAireRecord.getRecordById(uri.replace("textgrid:", "")); + openAireRecordList.getRecord().add(grt.getRecord()); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } catch (ParseException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + + } catch (DatatypeConfigurationException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + // if (resTokenForResponse != null) { + // System.out.println("RESTOKEN: "); + // System.out.println(getResTokenForResponse().getValue()); + // System.out.println(getResumptionToken()); + // resTokenForResponse = OAIPMHUtilities.getResumptionToken( + // 94146, resumptionToken, cursorCollector, scrollID, this.searchResponseSize, i) + openAireRecordList.setResumptionToken(getResTokenForResponse()); + // } + + return openAireRecordList; + } + + /* + * (non-Javadoc) + * + * @see info.textgrid.middleware.RecordListDelivererInterface#setHeader(java.lang.String, + * java.lang.String, java.lang.String) + */ + @Override + public HeaderType setHeader(String set, String headerIdentifier, String setSpec) { + // TODO Auto-generated method stub + return null; + } + + // GETTER AND SETTER + + /* + * (non-Javadoc) + * + * @see info.textgrid.middleware.RecordListDelivererAbstract#getRangeField() + */ + public String getRangeField() { + return this.rangeField; + } + + /* + * (non-Javadoc) + * + * @see info.textgrid.middleware.RecordListDelivererAbstract#setRangeField(java.lang.String) + */ + public void setRangeField(String rangeField) { + this.rangeField = rangeField; + } + + /** + * @param formatToFilter + */ + public void setFormatToFilter(String formatToFilter) { + this.formatToFilter = formatToFilter; + } + + /** + * @param formatField + */ + public void setFormatField(String formatField) { + this.formatField = formatField; + } + + /* + * (non-Javadoc) + * + * @see info.textgrid.middleware.RecordListDelivererAbstract#getSearchResponseSize() + */ + public int getSearchResponseSize() { + return this.searchResponseSize; + } + + /* + * (non-Javadoc) + * + * @see info.textgrid.middleware.RecordListDelivererAbstract#setSearchResponseSize(int) + */ + public void setSearchResponseSize(int searchResponseSize) { + this.searchResponseSize = searchResponseSize; + } + + /* + * (non-Javadoc) + * + * @see info.textgrid.middleware.RecordListDelivererAbstract#getResultSize() + */ + public long getResultSize() { + return this.resultSize; + } + + /* + * (non-Javadoc) + * + * @see info.textgrid.middleware.RecordListDelivererAbstract#setResultSize(long) + */ + public void setResultSize(long resultSize) { + this.resultSize = resultSize; + } + + /* + * (non-Javadoc) + * + * @see info.textgrid.middleware.RecordListDelivererAbstract#isFoundItems() + */ + public boolean isFoundItems() { + return this.foundItems; + } + + /* + * (non-Javadoc) + * + * @see info.textgrid.middleware.RecordListDelivererAbstract#setFoundItems(boolean) + */ + public void setFoundItems(boolean foundItems) { + this.foundItems = foundItems; + } + + /* + * public ResumptionTokenType getResTokenForResponse() { return resTokenForResponse; } + * + * public void setResTokenForResponse(ResumptionTokenType resTokenForResponse) { + * this.resTokenForResponse = resTokenForResponse; } + */ } diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordDelivererIDIOM.java b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordDelivererIDIOM.java index 93855416..de1c1208 100644 --- a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordDelivererIDIOM.java +++ b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordDelivererIDIOM.java @@ -150,7 +150,7 @@ public class RecordDelivererIDIOM extends RecordDelivererAbstract { textgridBaseURI_FromID = id; } log.info("ID for generating the METS/MODS file is: " + textgridBaseURI_FromID); - metsmods = new ClassicMayanMetsMods( + this.metsmods = new ClassicMayanMetsMods( textgridBaseURI_FromID, this.dateOfObjectCreation, this.dateOfLastOblectModification); @@ -197,7 +197,7 @@ public class RecordDelivererIDIOM extends RecordDelivererAbstract { try { builder = factory.newDocumentBuilder(); - Document doc = builder.parse(new InputSource(new StringReader(metsmods.getMets()))); + Document doc = builder.parse(new InputSource(new StringReader(this.metsmods.getMets()))); metadataMets.setAny(doc.getDocumentElement()); } catch (Exception e) { e.printStackTrace(); diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererAbstract.java b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererAbstract.java index 41c047dd..3a2db81b 100644 --- a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererAbstract.java +++ b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererAbstract.java @@ -5,9 +5,7 @@ import java.util.ArrayList; import java.util.Hashtable; import java.util.List; import java.util.Map; - import org.apache.commons.logging.LogFactory; -import org.apache.jena.atlas.logging.Log; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; @@ -17,7 +15,6 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; - import info.textgrid.middleware.oaipmh.RequestType; import info.textgrid.middleware.oaipmh.ResumptionTokenType; @@ -26,8 +23,8 @@ import info.textgrid.middleware.oaipmh.ResumptionTokenType; */ public abstract class RecordListDelivererAbstract implements RecordListDelivererInterface { - private OAI_ESClient oaiEsClient; - + private OAI_ESClient oaiEsClient; + protected boolean textgrid; protected boolean dariah; @@ -38,12 +35,13 @@ public abstract class RecordListDelivererAbstract implements RecordListDeliverer private long resultSize; private boolean foundItems; public ResumptionTokenType resTokenForResponse; - + private static final int LIFETIME_RES_TOKEN = 600; protected static Map<String, Integer> cursorCollector = new Hashtable<String, Integer>(); - - private static org.apache.commons.logging.Log log = LogFactory.getLog(RecordListDelivererAbstract.class); - + + private static org.apache.commons.logging.Log log = + LogFactory.getLog(RecordListDelivererAbstract.class); + /** * @param textgrid * @param dariah @@ -53,117 +51,131 @@ public abstract class RecordListDelivererAbstract implements RecordListDeliverer this.dariah = dariah; } + /** + * @param from + * @param to + * @param set + * @param resumptionToken + * @return + */ public List<String> getUriList(String from, String to, String set, String resumptionToken) { QueryBuilder query; - //System.out.println("RANGEFIELD: " + "created"); + // System.out.println("RANGEFIELD: " + "created"); QueryBuilder rangeQuery = QueryBuilders.rangeQuery("created").from(from).to(to); - + QueryBuilder filterSandBox = QueryBuilders.matchPhraseQuery("nearlyPublished", "true"); - - if(set!=null && !set.equals("openaire_data")) { - String[] setParts = set.split(":"); - - String queryField = ""; - String valueField = ""; - - if (setParts[0].equals(TGConstants.SET_FIELD_FOR_TEXTGRID)) { - queryField = TGConstants.PROJECT_ID_FOR_TEXTGRID; - valueField = setParts[1]; - } - - QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery(queryField, valueField); - QueryBuilder boolQuery = QueryBuilders.boolQuery().must(rangeQuery).must(matchQuery).mustNot(filterSandBox); - - query = boolQuery; - }else { - //query = rangeQuery; - query = QueryBuilders.boolQuery().must(rangeQuery).mustNot(filterSandBox); - } - - return getFieldsFromESIndex(query, resumptionToken, set); + + if (set != null && !set.equals("openaire_data")) { + String[] setParts = set.split(":"); + + String queryField = ""; + String valueField = ""; + + if (setParts[0].equals(TGConstants.SET_FIELD_FOR_TEXTGRID)) { + queryField = TGConstants.PROJECT_ID_FOR_TEXTGRID; + valueField = setParts[1]; + } + + QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery(queryField, valueField); + QueryBuilder boolQuery = + QueryBuilders.boolQuery().must(rangeQuery).must(matchQuery).mustNot(filterSandBox); + + query = boolQuery; + } else { + // query = rangeQuery; + query = QueryBuilders.boolQuery().must(rangeQuery).mustNot(filterSandBox); + } + + return getFieldsFromESIndex(query, resumptionToken, set); } - - + + /** + * @param query + * @param resumptionToken + * @param set + * @return + */ public List<String> getFieldsFromESIndex(QueryBuilder query, String resumptionToken, - String set) { - - List<String> uriList = new ArrayList<String>(); - QueryBuilder recordFilter = QueryBuilders.boolQuery().must(query) - .must(QueryBuilders.matchPhraseQuery("format", "text/tg.edition+tg.aggregation+xml")); - - SearchRequest searchRequest = new SearchRequest(OAI_ESClient.getEsIndex()); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - - - searchSourceBuilder.query(recordFilter); - searchSourceBuilder.size(100); - searchRequest.source(searchSourceBuilder); - - SearchResponse scrollResp = new SearchResponse(); - - if (resumptionToken != null) { - - SearchScrollRequest scrollRequest = new SearchScrollRequest(resumptionToken); - scrollRequest.scroll(TimeValue.timeValueHours(24L)); - - try { - - scrollResp = OAI_ESClient.getEsClient().scroll(scrollRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } else { - searchRequest.source(searchSourceBuilder); - searchRequest.scroll(TimeValue.timeValueHours(24L)); - try { - scrollResp = OAI_ESClient.getEsClient().search(searchRequest,RequestOptions.DEFAULT); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - String scrollID = scrollResp.getScrollId(); - long completeListSize = scrollResp.getHits().totalHits; - setResultSize(completeListSize); - - if (completeListSize > 0) { - - setFoundItems(true); - int i=0; - - for (SearchHit hit : scrollResp.getHits().getHits()) { - i++; - if (hit != null && hit.getFields() != null) { - uriList.add(hit.getSourceAsMap().get(TGConstants.URI).toString()); - } - } - if(resumptionToken!=null && resTokenForResponse.getCursor().intValue()>=completeListSize) { - try{ - cursorCollector.remove(resumptionToken); - }catch(NullPointerException couldNotRemove) { - log.info("Could net remove hash value: " + resumptionToken + " from hash map"); - } - resTokenForResponse.setValue(""); - }else { - resTokenForResponse = OAIPMHUtilities.getResumptionToken( - completeListSize, resumptionToken, cursorCollector, scrollID, this.searchResponseSize, i); - } - - // System.out.println("TADA: "); - //System.out.println(cursorCollector); - - }else { - setFoundItems(false); - } - //System.out.println(resTokenForResponse.getValue()); - //System.out.println(resTokenForResponse.getCursor()); - return uriList; - } - - + String set) { + + List<String> uriList = new ArrayList<String>(); + QueryBuilder recordFilter = QueryBuilders.boolQuery().must(query) + .must(QueryBuilders.matchPhraseQuery("format", "text/tg.edition+tg.aggregation+xml")); + + SearchRequest searchRequest = new SearchRequest(OAI_ESClient.getEsIndex()); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + + + searchSourceBuilder.query(recordFilter); + searchSourceBuilder.size(100); + searchRequest.source(searchSourceBuilder); + + SearchResponse scrollResp = new SearchResponse(); + + if (resumptionToken != null) { + + SearchScrollRequest scrollRequest = new SearchScrollRequest(resumptionToken); + scrollRequest.scroll(TimeValue.timeValueHours(24L)); + + try { + + scrollResp = OAI_ESClient.getEsClient().scroll(scrollRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } else { + searchRequest.source(searchSourceBuilder); + searchRequest.scroll(TimeValue.timeValueHours(24L)); + try { + scrollResp = OAI_ESClient.getEsClient().search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + String scrollID = scrollResp.getScrollId(); + long completeListSize = scrollResp.getHits().totalHits; + setResultSize(completeListSize); + + if (completeListSize > 0) { + + setFoundItems(true); + int i = 0; + + for (SearchHit hit : scrollResp.getHits().getHits()) { + i++; + if (hit != null && hit.getFields() != null) { + uriList.add(hit.getSourceAsMap().get(TGConstants.URI).toString()); + } + } + if (resumptionToken != null + && this.resTokenForResponse.getCursor().intValue() >= completeListSize) { + try { + cursorCollector.remove(resumptionToken); + } catch (NullPointerException couldNotRemove) { + log.info("Could net remove hash value: " + resumptionToken + " from hash map"); + } + this.resTokenForResponse.setValue(""); + } else { + this.resTokenForResponse = OAIPMHUtilities.getResumptionToken( + completeListSize, resumptionToken, cursorCollector, scrollID, this.searchResponseSize, + i); + } + + // System.out.println("TADA: "); + // System.out.println(cursorCollector); + + } else { + setFoundItems(false); + } + // System.out.println(resTokenForResponse.getValue()); + // System.out.println(resTokenForResponse.getCursor()); + return uriList; + } + /** * @param request * @return @@ -171,7 +183,7 @@ public abstract class RecordListDelivererAbstract implements RecordListDeliverer public static ErrorHandler requestChecker(RequestType request) { ErrorHandler result = new ErrorHandler(); - + // Check if metadata prefix is existing and valid. if (request.getMetadataPrefix() != null && !request.getMetadataPrefix().equals(OAIPMHUtilities.OAIDC_PREFIX) @@ -185,21 +197,19 @@ public abstract class RecordListDelivererAbstract implements RecordListDeliverer // Check if resumptionToken is invalid or existing. if (request.getResumptionToken() != null) { - boolean restokIDIOMExisting = RecordListDelivererIDIOM.cursorCollector != null - && RecordListDelivererIDIOM.cursorCollector.containsKey(request.getResumptionToken()); - - boolean restokDCExisting = RecordListDelivererDC.cursorCollector != null - && RecordListDelivererDC.cursorCollector.containsKey(request.getResumptionToken()); - boolean restok = cursorCollector != null - && cursorCollector.containsKey(request.getResumptionToken()); - - if (!restokDCExisting && !restokIDIOMExisting && !restok) { - result.setError(TGConstants.OAI_BAD_RESUMPTION_TOKEN, "The value of the " - + request.getResumptionToken() + " argument is invalid or expired."); - } + boolean restokIDIOMExisting = RecordListDelivererIDIOM.cursorCollector != null + && RecordListDelivererIDIOM.cursorCollector.containsKey(request.getResumptionToken()); + + boolean restokDCExisting = RecordListDelivererDC.cursorCollector != null + && RecordListDelivererDC.cursorCollector.containsKey(request.getResumptionToken()); + boolean restok = cursorCollector != null + && cursorCollector.containsKey(request.getResumptionToken()); + + if (!restokDCExisting && !restokIDIOMExisting && !restok) { + result.setError(TGConstants.OAI_BAD_RESUMPTION_TOKEN, "The value of the " + + request.getResumptionToken() + " argument is invalid or expired."); } - - + } // Check params in general. List<String> errorValues = new ArrayList<String>(); @@ -210,15 +220,16 @@ public abstract class RecordListDelivererAbstract implements RecordListDeliverer if (request.getIdentifier() != null) { errorValues.add("identifier"); } - /* if(request.getFrom()!= null && (!OAIPMHUtilities.isThisDateValid(request.getFrom()) && !OAIPMHUtilities.isThisDateValidToOtherTimeStamp(request.getFrom()))) { - errorValues.add("from"); - } - if(request.getUntil() != null && (!OAIPMHUtilities.isThisDateValid(request.getUntil()) && !OAIPMHUtilities.isThisDateValidToOtherTimeStamp(request.getUntil()))) { - errorValues.add("until"); - } - if(request.getFrom()!= null && request.getUntil()!=null && !OAIPMHUtilities.getFormatOfDate(request.getFrom()).equals(OAIPMHUtilities.getFormatOfDate(request.getFrom()))) { - errorValues.add("until, from"); - }*/ + /* + * if(request.getFrom()!= null && (!OAIPMHUtilities.isThisDateValid(request.getFrom()) && + * !OAIPMHUtilities.isThisDateValidToOtherTimeStamp(request.getFrom()))) { + * errorValues.add("from"); } if(request.getUntil() != null && + * (!OAIPMHUtilities.isThisDateValid(request.getUntil()) && + * !OAIPMHUtilities.isThisDateValidToOtherTimeStamp(request.getUntil()))) { + * errorValues.add("until"); } if(request.getFrom()!= null && request.getUntil()!=null && + * !OAIPMHUtilities.getFormatOfDate(request.getFrom()).equals(OAIPMHUtilities.getFormatOfDate( + * request.getFrom()))) { errorValues.add("until, from"); } + */ if (errorValues.size() > 0) { result.setError(TGConstants.OAI_BAD_ARGUMENT, "The request includes illegal arguments " + "or is missing required arguments: " + errorValues); @@ -227,72 +238,105 @@ public abstract class RecordListDelivererAbstract implements RecordListDeliverer return result; } - - - //GETTER AND SETTER - - public String getRangeField() { - return rangeField; - } - - public void setRangeField(String rangeField) { - this.rangeField = rangeField; - } - - - /** - * @param formatToFilter - */ - public void setFormatToFilter(String formatToFilter) { - this.formatToFilter = formatToFilter; - } - - /** - * @param formatField - */ - public void setFormatField(String formatField) { - this.formatField = formatField; - } - - public int getSearchResponseSize() { - return searchResponseSize; - } - - public void setSearchResponseSize(int searchResponseSize) { - //System.out.println("SearchResponseSize: " + searchResponseSize); - this.searchResponseSize = searchResponseSize; - } - - public long getResultSize() { - return resultSize; - } - - public void setResultSize(long resultSize) { - this.resultSize = resultSize; - } - - public boolean isFoundItems() { - return foundItems; - } - - public void setFoundItems(boolean foundItems) { - this.foundItems = foundItems; - } - - public OAI_ESClient getOaiEsClient() { - return oaiEsClient; - } - - public void setOaiEsClient(OAI_ESClient oaiEsClient) { - this.oaiEsClient = oaiEsClient; - } - - public ResumptionTokenType getResTokenForResponse() { - return resTokenForResponse; - } - - public void setResTokenForResponse(ResumptionTokenType resTokenForResponse) { - this.resTokenForResponse = resTokenForResponse; - } + // GETTER AND SETTER + + /** + * @return + */ + public String getRangeField() { + return this.rangeField; + } + + /** + * @param rangeField + */ + public void setRangeField(String rangeField) { + this.rangeField = rangeField; + } + + /** + * @param formatToFilter + */ + public void setFormatToFilter(String formatToFilter) { + this.formatToFilter = formatToFilter; + } + + /** + * @param formatField + */ + public void setFormatField(String formatField) { + this.formatField = formatField; + } + + /** + * @return + */ + public int getSearchResponseSize() { + return this.searchResponseSize; + } + + /** + * @param searchResponseSize + */ + public void setSearchResponseSize(int searchResponseSize) { + // System.out.println("SearchResponseSize: " + searchResponseSize); + this.searchResponseSize = searchResponseSize; + } + + /** + * @return + */ + public long getResultSize() { + return this.resultSize; + } + + /** + * @param resultSize + */ + public void setResultSize(long resultSize) { + this.resultSize = resultSize; + } + + /** + * @return + */ + public boolean isFoundItems() { + return this.foundItems; + } + + /** + * @param foundItems + */ + public void setFoundItems(boolean foundItems) { + this.foundItems = foundItems; + } + + /** + * @return + */ + public OAI_ESClient getOaiEsClient() { + return this.oaiEsClient; + } + + /** + * @param oaiEsClient + */ + public void setOaiEsClient(OAI_ESClient oaiEsClient) { + this.oaiEsClient = oaiEsClient; + } + + /** + * @return + */ + public ResumptionTokenType getResTokenForResponse() { + return this.resTokenForResponse; + } + + /** + * @param resTokenForResponse + */ + public void setResTokenForResponse(ResumptionTokenType resTokenForResponse) { + this.resTokenForResponse = resTokenForResponse; + } } diff --git a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererIDIOM.java b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererIDIOM.java index 4bbe6ae6..d1ea00a3 100644 --- a/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererIDIOM.java +++ b/oaipmh-core/src/main/java/info/textgrid/middleware/RecordListDelivererIDIOM.java @@ -64,7 +64,6 @@ public class RecordListDelivererIDIOM extends RecordListDelivererAbstract { * @see info.textgrid.middleware.RecordListDelivererInterface#getRecords(java.lang.String, * java.lang.String, java.lang.String, java.lang.String) */ - @Override public ListRecordsType getRecords(final String from, final String to, final String set, diff --git a/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTestTG.java b/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTestTG.java index b23c9434..0726fe60 100644 --- a/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTestTG.java +++ b/oaipmh-core/src/test/java/info/textgrid/middleware/OaiPmhTestTG.java @@ -8,14 +8,13 @@ import java.util.Map; import javax.xml.bind.JAXBException; import javax.xml.datatype.DatatypeConfigurationException; import javax.xml.datatype.XMLGregorianCalendar; - import org.json.JSONObject; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; +import info.textgrid.middleware.common.TextGridMimetypes; import info.textgrid.middleware.oaipmh.DeletedRecordType; import info.textgrid.middleware.oaipmh.GranularityType; -import info.textgrid.middleware.common.TextGridMimetypes; /** * @author Maximilian Brodhun, SUB Göttingen -- GitLab