From 02c0e0d2800d6a68c071ce95533b07ce0121b496 Mon Sep 17 00:00:00 2001 From: "Stefan E. Funk" <funk@sub.uni-goettingen.de> Date: Fri, 25 Nov 2022 16:21:36 +0100 Subject: [PATCH 1/5] docs: remove unused link --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 8d09428..19a1e09 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,6 @@ New ready-to-go images from ConedaKor shall be published in the TextGrid Reposit #### ConedaKOR queries -* Documentation - * <> * OAI-PMH * <https://classicmayan.kor.de.dariah.eu/oai-pmh/entities?verb=ListIdentifiers> * REST -- GitLab From 7589b1820cf17c9088400d68c16756ffce0d8db9 Mon Sep 17 00:00:00 2001 From: "Stefan E. Funk" <funk@sub.uni-goettingen.de> Date: Fri, 25 Nov 2022 17:33:57 +0100 Subject: [PATCH 2/5] docs: add some pom doc --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 19a1e09..c602db5 100644 --- a/README.md +++ b/README.md @@ -70,4 +70,4 @@ New ready-to-go images from ConedaKor shall be published in the TextGrid Reposit ## Releasing a new version -For releasing a new version of TG-crud or DH-crud, please have a look at the [DARIAH-DE Release Management Page](https://wiki.de.dariah.eu/display/DARIAH3/DARIAH-DE+Release+Management#DARIAHDEReleaseManagement-Gitlabflow/Gitflow(develop,main,featurebranchesundtags)) or see the [Gitlab CI file](.gitlab-ci.yml). +For releasing a new version of Queries Collection, please have a look at the [DARIAH-DE Release Management Page](https://wiki.de.dariah.eu/display/DARIAH3/DARIAH-DE+Release+Management#DARIAHDEReleaseManagement-Gitlabflow/Gitflow(develop,main,featurebranchesundtags)) or see the [Gitlab CI file](.gitlab-ci.yml). diff --git a/pom.xml b/pom.xml index 3b6771f..e6b808d 100644 --- a/pom.xml +++ b/pom.xml @@ -10,6 +10,7 @@ <url>http://maven.apache.org</url> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <const.version>4.4.1</const.version> <cyclonedx-maven-plugin.version>2.7.0</cyclonedx-maven-plugin.version> <javax-mail.version>1.4.7</javax-mail.version> <jaxws-api.version>2.2.6</jaxws-api.version> @@ -24,7 +25,6 @@ <orientdb.version>3.0.4</orientdb.version> <slf4j.version>1.7.25</slf4j.version> <tgclient.version>4.4.3</tgclient.version> - <const.version>4.4.1</const.version> </properties> <dependencies> <dependency> -- GitLab From 0397cf6d6fb0c500265338409e160bcde6caf988 Mon Sep 17 00:00:00 2001 From: "Stefan E. Funk" <funk@sub.uni-goettingen.de> Date: Wed, 30 Nov 2022 17:41:27 +0100 Subject: [PATCH 3/5] fix: improve update --- .../tools/ConedaKor2TextGridRep.java | 8 ++-- .../classicmayan/tools/IdiomConstants.java | 6 +-- .../tools/IdiomTextGridObject.java | 4 +- .../classicmayan/tools/TGSearchQueries.java | 38 +++++++++++++++++-- 4 files changed, 43 insertions(+), 13 deletions(-) rename src/{test => main}/java/org/classicmayan/tools/ConedaKor2TextGridRep.java (98%) diff --git a/src/test/java/org/classicmayan/tools/ConedaKor2TextGridRep.java b/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java similarity index 98% rename from src/test/java/org/classicmayan/tools/ConedaKor2TextGridRep.java rename to src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java index 3b467ab..f2ce9e6 100644 --- a/src/test/java/org/classicmayan/tools/ConedaKor2TextGridRep.java +++ b/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java @@ -11,7 +11,7 @@ * * @copyright Göttingen State and University Library * @license GNU Lesser General Public License v3 (http://www.gnu.org/licenses/lgpl-3.0.txt) - * @author Maximilian Behnert-Brodhun (behnert-brodhun@sub.uni-goettingen.de) + * @author Stefan E. Funk (funk@sub.uni-goettingen.de) */ package org.classicmayan.tools; @@ -424,7 +424,7 @@ public class ConedaKor2TextGridRep { System.out.println(); System.out.println("ConedaKOR --> TextGridRep configuration"); System.out.println(LINES); - System.out.println("\tSCOPE: " + idiomTextGridObject.getScope()); + System.out.println("\tSCOPE: " + idiomTextGridObject.getScope()); System.out.println("\tDRY_RUN: " + idiomTextGridObject.isDryRun()); System.out.println(LINES); System.out @@ -471,8 +471,8 @@ public class ConedaKor2TextGridRep { } catch (NotFoundException e) { errorList.add(korID); - System.out.println( - "!WARNING! " + e.getMessage() + " Please check <notes> tag for KOR ID [" + korID + "]"); + System.out.println("!ERROR! " + e.getMessage() + + " Please check <notes> and <title> tag for KOR ID [" + korID + "]"); } } diff --git a/src/main/java/org/classicmayan/tools/IdiomConstants.java b/src/main/java/org/classicmayan/tools/IdiomConstants.java index 29193b9..0179802 100644 --- a/src/main/java/org/classicmayan/tools/IdiomConstants.java +++ b/src/main/java/org/classicmayan/tools/IdiomConstants.java @@ -22,11 +22,11 @@ package org.classicmayan.tools; public final class IdiomConstants { // CHANGE PROPERTIES FILE HERE! - public static final String PROPERTIES_FILE = "src/main/resources/idiomQueries.properties"; + // public static final String PROPERTIES_FILE = "src/main/resources/idiomQueries.properties"; // public static final String PROPERTIES_FILE = // "src/main/resources/idiomQueries-SECRET-dev.properties"; - // public static final String PROPERTIES_FILE = - // "src/main/resources/idiomQueries-SECRET-prod.properties"; + public static final String PROPERTIES_FILE = + "src/main/resources/idiomQueries-SECRET-prod.properties"; // CHANGE PROPERTIES FILE HERE! public static final String DEFAULT_TGURL = "https://textgridlab.org/1.0/"; diff --git a/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java b/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java index 72a9fbe..8e90ae9 100644 --- a/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java +++ b/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java @@ -303,9 +303,9 @@ public class IdiomTextGridObject { .setObject(tgObject) .execute(); - System.out.println(this.textgridURI + " [UPDATED]"); + System.out.println(" [UPDATED]"); } else { - System.out.println(this.textgridURI + " [DRYRUN]"); + System.out.println(" [DRYRUN]"); } } diff --git a/src/main/java/org/classicmayan/tools/TGSearchQueries.java b/src/main/java/org/classicmayan/tools/TGSearchQueries.java index 3c28ecd..6054b5d 100644 --- a/src/main/java/org/classicmayan/tools/TGSearchQueries.java +++ b/src/main/java/org/classicmayan/tools/TGSearchQueries.java @@ -257,19 +257,49 @@ public class TGSearchQueries { String result = ""; + // Check title. + String titleURI = ""; + String titleQuery = "title:\"[" + korID + "]\""; + List<ResultType> titleList = searchClient.searchQuery() + .setSid(this.getSessionID()) + .setQuery("project.id:" + this.getProjectID()) + .setQuery(titleQuery) + .execute() + .getResult(); + if (titleList.size() != 0) { + titleURI = + titleList.get(0).getObject().getGeneric().getGenerated().getTextgridUri().getValue(); + } + + // Check notes. + String notesURI = ""; String notesQuery = "(notes:\"ConedaKorMediumData " + korID + "\")"; - List<ResultType> resultList = searchClient.searchQuery() + List<ResultType> notesList = searchClient.searchQuery() .setSid(this.getSessionID()) .setQuery("project.id:" + this.getProjectID()) .setQuery(notesQuery) .execute() .getResult(); + if (notesList.size() != 0) { + notesURI = + titleList.get(0).getObject().getGeneric().getGenerated().getTextgridUri().getValue(); + } - if (resultList.size() == 0) { - throw new NotFoundException("NOTHING FOUND!"); + // Check for empty node lists. + if (notesURI.isEmpty() && titleURI.isEmpty()) { + throw new NotFoundException("KOR ID not found in notes and title!"); } - result = resultList.get(0).getObject().getGeneric().getGenerated().getTextgridUri().getValue(); + // Check for equality. + if (notesURI.equals(titleURI)) { + result = notesURI; + } else if (notesURI.isEmpty()) { + System.out.print("[WARNING! No TextGrid URI in <title> for " + korID + "] "); + result = titleURI; + } else { + System.out.print("[WARNING! No TextGrid URI in <notes> for " + korID + "] "); + result = notesURI; + } return result; } -- GitLab From 40b7c12688d11bcc8f3883e1e05117df41dc9f69 Mon Sep 17 00:00:00 2001 From: "Stefan E. Funk" <funk@sub.uni-goettingen.de> Date: Thu, 1 Dec 2022 16:01:13 +0100 Subject: [PATCH 4/5] fix: add title checks on update --- .../tools/ConedaKor2TextGridRep.java | 5 ++--- .../tools/IdiomTextGridObject.java | 18 +++++++++++++++--- .../classicmayan/tools/TGSearchQueries.java | 4 +++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java b/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java index f2ce9e6..5297551 100644 --- a/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java +++ b/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java @@ -477,9 +477,8 @@ public class ConedaKor2TextGridRep { } if (!errorList.isEmpty()) { - String message = - "---> !ERROR! " + errorList.size() + " warning" + (errorList.size() != 1 ? "s" : "") - + " in KOR ID mapping! Please check IDs " + errorList.toString() + " <---"; + String message = "---> !ERROR! Please check KOR ID" + (errorList.size() != 1 ? "s" : "") + + ": " + errorList.toString() + " <---"; System.err.println(); System.err.println(message); System.err.println(); diff --git a/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java b/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java index 8e90ae9..7334533 100644 --- a/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java +++ b/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java @@ -284,11 +284,23 @@ public class IdiomTextGridObject { public void updateToTextGrid(JSONObject json, String title) throws IOException, CrudClientException { - // Get existing metadata from TG-crud, update title! + // Get existing metadata from TG-crud. MetadataContainerType metadata = IdiomTextGridObject.crudClient.readMetadata() .setSid(this.sessionID).setTextgridUri(this.textgridURI).execute(); - metadata.getObject().getGeneric().getProvided().getTitle().remove(0); - metadata.getObject().getGeneric().getProvided().getTitle().add(title); + + // Update title, if KOR ID in title is the same as in JSON metadata. + // TODO Check for more titles? + String oldTitle = metadata.getObject().getGeneric().getProvided().getTitle().get(0); + String korID = String.valueOf(json.getInt("id")); + if (oldTitle.startsWith("[" + korID + "]")) { + metadata.getObject().getGeneric().getProvided().getTitle().remove(0); + metadata.getObject().getGeneric().getProvided().getTitle().add(title); + } else { + System.err.println(); + System.err.println("TG Object title: '" + oldTitle + "'"); + System.err.println("New KOR title: '" + title + "'"); + throw new IOException("Old TG object title does not fit KOR title!"); + } String jsonData = json.toString(2); DataHandler objectData = diff --git a/src/main/java/org/classicmayan/tools/TGSearchQueries.java b/src/main/java/org/classicmayan/tools/TGSearchQueries.java index 6054b5d..8576a64 100644 --- a/src/main/java/org/classicmayan/tools/TGSearchQueries.java +++ b/src/main/java/org/classicmayan/tools/TGSearchQueries.java @@ -215,7 +215,9 @@ public class TGSearchQueries { .getTextgridUri(); // Get latest object as result. - result = uriList.get(uriList.size() - 1); + if (!uriList.isEmpty()) { + result = uriList.get(uriList.size() - 1); + } return result; } -- GitLab From 7a39911fee0af2d13b9091cbadfaec1bb7c434b7 Mon Sep 17 00:00:00 2001 From: "Stefan E. Funk" <funk@sub.uni-goettingen.de> Date: Fri, 2 Dec 2022 15:18:13 +0100 Subject: [PATCH 5/5] fix: fix some small issues for first update and import of new files --- .../tools/ConedaKor2TextGridRep.java | 10 +++++-- .../tools/IdiomTextGridObject.java | 30 +++++++++++++++++-- .../classicmayan/tools/TGSearchQueries.java | 2 ++ 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java b/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java index 5297551..472b5d1 100644 --- a/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java +++ b/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java @@ -304,6 +304,12 @@ public class ConedaKor2TextGridRep { JSONArray updatedMediaList = ConedaKorQueries .getImageMetadataUpdates(this.timestamp, OBJECTS_PER_PAGE).getJSONArray("records"); + // Test only the first... used for productive testing only... :-D + // + // JSONArray urg = new JSONArray(); + // urg.put(updatedMediaList.get(0)); + // updatedMediaList = urg; + System.out.println(); System.out.println("\tFound " + updatedMediaList.length() + " modified images!"); @@ -371,8 +377,8 @@ public class ConedaKor2TextGridRep { if (uriList.size() <= this.amountOfEntriesLeftInLastCollection) { System.out - .println("\t" + uriList.size() + " new objects fit in last collection with space for " - + this.amountOfEntriesLeftInLastCollection + " objects)"); + .println("\t" + uriList.size() + " new objects fit in latest collection with space for " + + this.amountOfEntriesLeftInLastCollection + " objects"); // Add to latest subcollection only. this.itgo.addUriListToAggregation(uriList, this.latestSubcollectionURI); diff --git a/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java b/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java index 7334533..2341862 100644 --- a/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java +++ b/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java @@ -348,8 +348,11 @@ public class IdiomTextGridObject { .setSid(this.sessionID) .setTextgridUri(collectionURI) .execute(); - String aggregation = IOUtils.readStringFromStream(tgobj.getData()); + + // Check amount of entries. + int entriesBefore = getResourceCount(aggregation); + for (String uri : objectURIs) { aggregation = aggregation.replace("</rdf:Description>", " <ore:aggregates rdf:resource=\"" @@ -357,7 +360,13 @@ public class IdiomTextGridObject { + " </rdf:Description>"); } - // System.out.println("COLLECTION: " + aggregation); + // Check amount of entries. + int entriesAfter = getResourceCount(aggregation); + if (entriesAfter != entriesBefore + objectURIs.size()) { + System.err.println("!ERROR! WRONG ENTRIES COUNT IN COLLECTION " + collectionURI); + System.err + .println("!ERROR! " + entriesAfter + " != " + entriesBefore + " + " + objectURIs.size()); + } tgobj.setData(new ByteArrayInputStream(aggregation.getBytes())); @@ -547,6 +556,23 @@ public class IdiomTextGridObject { this.textgridURI = uuid + ".0"; } + /** + * @param theAggregation + * @return + */ + private static int getResourceCount(String theAggregation) { + + int result = 0; + + String entries[] = theAggregation.split("<ore:aggregates rdf:resource="); + + if (entries.length > 0) { + result = entries.length - 1; + } + + return result; + } + // ** // GETTERS & SETTERS // ** diff --git a/src/main/java/org/classicmayan/tools/TGSearchQueries.java b/src/main/java/org/classicmayan/tools/TGSearchQueries.java index 8576a64..5f5577e 100644 --- a/src/main/java/org/classicmayan/tools/TGSearchQueries.java +++ b/src/main/java/org/classicmayan/tools/TGSearchQueries.java @@ -189,6 +189,8 @@ public class TGSearchQueries { // Get all entries of collection, uses // <https://dev.textgridlab.org/1.0/tgsearch/info/textgrid:42j3z.0/children> + // NOTE We have to use children here, not navigation, we need every object from the aggregation, + // not only the displayed ones (such as deleted or the forbidden ones)! List<String> uriList = searchClient.infoQuery() .setSid(this.sessionID) .getChildren(collectionURI) -- GitLab