From 02c0e0d2800d6a68c071ce95533b07ce0121b496 Mon Sep 17 00:00:00 2001
From: "Stefan E. Funk" <funk@sub.uni-goettingen.de>
Date: Fri, 25 Nov 2022 16:21:36 +0100
Subject: [PATCH 1/5] docs: remove unused link

---
 README.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/README.md b/README.md
index 8d09428..19a1e09 100644
--- a/README.md
+++ b/README.md
@@ -58,8 +58,6 @@ New ready-to-go images from ConedaKor shall be published in the TextGrid Reposit
 
 #### ConedaKOR queries
 
-* Documentation
-    * <>
 * OAI-PMH
     * <https://classicmayan.kor.de.dariah.eu/oai-pmh/entities?verb=ListIdentifiers>
 * REST
-- 
GitLab


From 7589b1820cf17c9088400d68c16756ffce0d8db9 Mon Sep 17 00:00:00 2001
From: "Stefan E. Funk" <funk@sub.uni-goettingen.de>
Date: Fri, 25 Nov 2022 17:33:57 +0100
Subject: [PATCH 2/5] docs: add some pom doc

---
 README.md | 2 +-
 pom.xml   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 19a1e09..c602db5 100644
--- a/README.md
+++ b/README.md
@@ -70,4 +70,4 @@ New ready-to-go images from ConedaKor shall be published in the TextGrid Reposit
 
 ## Releasing a new version
 
-For releasing a new version of TG-crud or DH-crud, please have a look at the [DARIAH-DE Release Management Page](https://wiki.de.dariah.eu/display/DARIAH3/DARIAH-DE+Release+Management#DARIAHDEReleaseManagement-Gitlabflow/Gitflow(develop,main,featurebranchesundtags)) or see the [Gitlab CI file](.gitlab-ci.yml).
+For releasing a new version of Queries Collection, please have a look at the [DARIAH-DE Release Management Page](https://wiki.de.dariah.eu/display/DARIAH3/DARIAH-DE+Release+Management#DARIAHDEReleaseManagement-Gitlabflow/Gitflow(develop,main,featurebranchesundtags)) or see the [Gitlab CI file](.gitlab-ci.yml).
diff --git a/pom.xml b/pom.xml
index 3b6771f..e6b808d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -10,6 +10,7 @@
 	<url>http://maven.apache.org</url>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<const.version>4.4.1</const.version>
 		<cyclonedx-maven-plugin.version>2.7.0</cyclonedx-maven-plugin.version>
 		<javax-mail.version>1.4.7</javax-mail.version>
 		<jaxws-api.version>2.2.6</jaxws-api.version>
@@ -24,7 +25,6 @@
 		<orientdb.version>3.0.4</orientdb.version>
 		<slf4j.version>1.7.25</slf4j.version>
 		<tgclient.version>4.4.3</tgclient.version>
-		<const.version>4.4.1</const.version>
 	</properties>
 	<dependencies>
 		<dependency>
-- 
GitLab


From 0397cf6d6fb0c500265338409e160bcde6caf988 Mon Sep 17 00:00:00 2001
From: "Stefan E. Funk" <funk@sub.uni-goettingen.de>
Date: Wed, 30 Nov 2022 17:41:27 +0100
Subject: [PATCH 3/5] fix: improve update

---
 .../tools/ConedaKor2TextGridRep.java          |  8 ++--
 .../classicmayan/tools/IdiomConstants.java    |  6 +--
 .../tools/IdiomTextGridObject.java            |  4 +-
 .../classicmayan/tools/TGSearchQueries.java   | 38 +++++++++++++++++--
 4 files changed, 43 insertions(+), 13 deletions(-)
 rename src/{test => main}/java/org/classicmayan/tools/ConedaKor2TextGridRep.java (98%)

diff --git a/src/test/java/org/classicmayan/tools/ConedaKor2TextGridRep.java b/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java
similarity index 98%
rename from src/test/java/org/classicmayan/tools/ConedaKor2TextGridRep.java
rename to src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java
index 3b467ab..f2ce9e6 100644
--- a/src/test/java/org/classicmayan/tools/ConedaKor2TextGridRep.java
+++ b/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java
@@ -11,7 +11,7 @@
  *
  * @copyright Göttingen State and University Library
  * @license GNU Lesser General Public License v3 (http://www.gnu.org/licenses/lgpl-3.0.txt)
- * @author Maximilian Behnert-Brodhun (behnert-brodhun@sub.uni-goettingen.de)
+ * @author Stefan E. Funk (funk@sub.uni-goettingen.de)
  */
 
 package org.classicmayan.tools;
@@ -424,7 +424,7 @@ public class ConedaKor2TextGridRep {
     System.out.println();
     System.out.println("ConedaKOR --> TextGridRep configuration");
     System.out.println(LINES);
-    System.out.println("\tSCOPE:   " + idiomTextGridObject.getScope());
+    System.out.println("\tSCOPE: " + idiomTextGridObject.getScope());
     System.out.println("\tDRY_RUN: " + idiomTextGridObject.isDryRun());
     System.out.println(LINES);
     System.out
@@ -471,8 +471,8 @@ public class ConedaKor2TextGridRep {
 
       } catch (NotFoundException e) {
         errorList.add(korID);
-        System.out.println(
-            "!WARNING! " + e.getMessage() + " Please check <notes> tag for KOR ID [" + korID + "]");
+        System.out.println("!ERROR! " + e.getMessage()
+            + " Please check <notes> and <title> tag for KOR ID [" + korID + "]");
       }
     }
 
diff --git a/src/main/java/org/classicmayan/tools/IdiomConstants.java b/src/main/java/org/classicmayan/tools/IdiomConstants.java
index 29193b9..0179802 100644
--- a/src/main/java/org/classicmayan/tools/IdiomConstants.java
+++ b/src/main/java/org/classicmayan/tools/IdiomConstants.java
@@ -22,11 +22,11 @@ package org.classicmayan.tools;
 public final class IdiomConstants {
 
   // CHANGE PROPERTIES FILE HERE!
-  public static final String PROPERTIES_FILE = "src/main/resources/idiomQueries.properties";
+  // public static final String PROPERTIES_FILE = "src/main/resources/idiomQueries.properties";
   // public static final String PROPERTIES_FILE =
   // "src/main/resources/idiomQueries-SECRET-dev.properties";
-  // public static final String PROPERTIES_FILE =
-  // "src/main/resources/idiomQueries-SECRET-prod.properties";
+  public static final String PROPERTIES_FILE =
+      "src/main/resources/idiomQueries-SECRET-prod.properties";
   // CHANGE PROPERTIES FILE HERE!
 
   public static final String DEFAULT_TGURL = "https://textgridlab.org/1.0/";
diff --git a/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java b/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java
index 72a9fbe..8e90ae9 100644
--- a/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java
+++ b/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java
@@ -303,9 +303,9 @@ public class IdiomTextGridObject {
           .setObject(tgObject)
           .execute();
 
-      System.out.println(this.textgridURI + " [UPDATED]");
+      System.out.println(" [UPDATED]");
     } else {
-      System.out.println(this.textgridURI + " [DRYRUN]");
+      System.out.println(" [DRYRUN]");
     }
 
   }
diff --git a/src/main/java/org/classicmayan/tools/TGSearchQueries.java b/src/main/java/org/classicmayan/tools/TGSearchQueries.java
index 3c28ecd..6054b5d 100644
--- a/src/main/java/org/classicmayan/tools/TGSearchQueries.java
+++ b/src/main/java/org/classicmayan/tools/TGSearchQueries.java
@@ -257,19 +257,49 @@ public class TGSearchQueries {
 
     String result = "";
 
+    // Check title.
+    String titleURI = "";
+    String titleQuery = "title:\"[" + korID + "]\"";
+    List<ResultType> titleList = searchClient.searchQuery()
+        .setSid(this.getSessionID())
+        .setQuery("project.id:" + this.getProjectID())
+        .setQuery(titleQuery)
+        .execute()
+        .getResult();
+    if (titleList.size() != 0) {
+      titleURI =
+          titleList.get(0).getObject().getGeneric().getGenerated().getTextgridUri().getValue();
+    }
+
+    // Check notes.
+    String notesURI = "";
     String notesQuery = "(notes:\"ConedaKorMediumData " + korID + "\")";
-    List<ResultType> resultList = searchClient.searchQuery()
+    List<ResultType> notesList = searchClient.searchQuery()
         .setSid(this.getSessionID())
         .setQuery("project.id:" + this.getProjectID())
         .setQuery(notesQuery)
         .execute()
         .getResult();
+    if (notesList.size() != 0) {
+      notesURI =
+          titleList.get(0).getObject().getGeneric().getGenerated().getTextgridUri().getValue();
+    }
 
-    if (resultList.size() == 0) {
-      throw new NotFoundException("NOTHING FOUND!");
+    // Check for empty node lists.
+    if (notesURI.isEmpty() && titleURI.isEmpty()) {
+      throw new NotFoundException("KOR ID not found in notes and title!");
     }
 
-    result = resultList.get(0).getObject().getGeneric().getGenerated().getTextgridUri().getValue();
+    // Check for equality.
+    if (notesURI.equals(titleURI)) {
+      result = notesURI;
+    } else if (notesURI.isEmpty()) {
+      System.out.print("[WARNING! No TextGrid URI in <title> for " + korID + "] ");
+      result = titleURI;
+    } else {
+      System.out.print("[WARNING! No TextGrid URI in <notes> for " + korID + "] ");
+      result = notesURI;
+    }
 
     return result;
   }
-- 
GitLab


From 40b7c12688d11bcc8f3883e1e05117df41dc9f69 Mon Sep 17 00:00:00 2001
From: "Stefan E. Funk" <funk@sub.uni-goettingen.de>
Date: Thu, 1 Dec 2022 16:01:13 +0100
Subject: [PATCH 4/5] fix: add title checks on update

---
 .../tools/ConedaKor2TextGridRep.java           |  5 ++---
 .../tools/IdiomTextGridObject.java             | 18 +++++++++++++++---
 .../classicmayan/tools/TGSearchQueries.java    |  4 +++-
 3 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java b/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java
index f2ce9e6..5297551 100644
--- a/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java
+++ b/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java
@@ -477,9 +477,8 @@ public class ConedaKor2TextGridRep {
     }
 
     if (!errorList.isEmpty()) {
-      String message =
-          "--->  !ERROR! " + errorList.size() + " warning" + (errorList.size() != 1 ? "s" : "")
-              + " in KOR ID mapping! Please check IDs " + errorList.toString() + "  <---";
+      String message = "--->  !ERROR! Please check KOR ID" + (errorList.size() != 1 ? "s" : "")
+          + ": " + errorList.toString() + "  <---";
       System.err.println();
       System.err.println(message);
       System.err.println();
diff --git a/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java b/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java
index 8e90ae9..7334533 100644
--- a/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java
+++ b/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java
@@ -284,11 +284,23 @@ public class IdiomTextGridObject {
   public void updateToTextGrid(JSONObject json, String title)
       throws IOException, CrudClientException {
 
-    // Get existing metadata from TG-crud, update title!
+    // Get existing metadata from TG-crud.
     MetadataContainerType metadata = IdiomTextGridObject.crudClient.readMetadata()
         .setSid(this.sessionID).setTextgridUri(this.textgridURI).execute();
-    metadata.getObject().getGeneric().getProvided().getTitle().remove(0);
-    metadata.getObject().getGeneric().getProvided().getTitle().add(title);
+
+    // Update title, if KOR ID in title is the same as in JSON metadata.
+    // TODO Check for more titles?
+    String oldTitle = metadata.getObject().getGeneric().getProvided().getTitle().get(0);
+    String korID = String.valueOf(json.getInt("id"));
+    if (oldTitle.startsWith("[" + korID + "]")) {
+      metadata.getObject().getGeneric().getProvided().getTitle().remove(0);
+      metadata.getObject().getGeneric().getProvided().getTitle().add(title);
+    } else {
+      System.err.println();
+      System.err.println("TG Object title: '" + oldTitle + "'");
+      System.err.println("New KOR title:   '" + title + "'");
+      throw new IOException("Old TG object title does not fit KOR title!");
+    }
 
     String jsonData = json.toString(2);
     DataHandler objectData =
diff --git a/src/main/java/org/classicmayan/tools/TGSearchQueries.java b/src/main/java/org/classicmayan/tools/TGSearchQueries.java
index 6054b5d..8576a64 100644
--- a/src/main/java/org/classicmayan/tools/TGSearchQueries.java
+++ b/src/main/java/org/classicmayan/tools/TGSearchQueries.java
@@ -215,7 +215,9 @@ public class TGSearchQueries {
         .getTextgridUri();
 
     // Get latest object as result.
-    result = uriList.get(uriList.size() - 1);
+    if (!uriList.isEmpty()) {
+      result = uriList.get(uriList.size() - 1);
+    }
 
     return result;
   }
-- 
GitLab


From 7a39911fee0af2d13b9091cbadfaec1bb7c434b7 Mon Sep 17 00:00:00 2001
From: "Stefan E. Funk" <funk@sub.uni-goettingen.de>
Date: Fri, 2 Dec 2022 15:18:13 +0100
Subject: [PATCH 5/5] fix: fix some small issues for first update and import of
 new files

---
 .../tools/ConedaKor2TextGridRep.java          | 10 +++++--
 .../tools/IdiomTextGridObject.java            | 30 +++++++++++++++++--
 .../classicmayan/tools/TGSearchQueries.java   |  2 ++
 3 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java b/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java
index 5297551..472b5d1 100644
--- a/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java
+++ b/src/main/java/org/classicmayan/tools/ConedaKor2TextGridRep.java
@@ -304,6 +304,12 @@ public class ConedaKor2TextGridRep {
     JSONArray updatedMediaList = ConedaKorQueries
         .getImageMetadataUpdates(this.timestamp, OBJECTS_PER_PAGE).getJSONArray("records");
 
+    // Test only the first... used for productive testing only... :-D
+    //
+    // JSONArray urg = new JSONArray();
+    // urg.put(updatedMediaList.get(0));
+    // updatedMediaList = urg;
+
     System.out.println();
     System.out.println("\tFound " + updatedMediaList.length() + " modified images!");
 
@@ -371,8 +377,8 @@ public class ConedaKor2TextGridRep {
     if (uriList.size() <= this.amountOfEntriesLeftInLastCollection) {
 
       System.out
-          .println("\t" + uriList.size() + " new objects fit in last collection with space for "
-              + this.amountOfEntriesLeftInLastCollection + " objects)");
+          .println("\t" + uriList.size() + " new objects fit in latest collection with space for "
+              + this.amountOfEntriesLeftInLastCollection + " objects");
 
       // Add to latest subcollection only.
       this.itgo.addUriListToAggregation(uriList, this.latestSubcollectionURI);
diff --git a/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java b/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java
index 7334533..2341862 100644
--- a/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java
+++ b/src/main/java/org/classicmayan/tools/IdiomTextGridObject.java
@@ -348,8 +348,11 @@ public class IdiomTextGridObject {
         .setSid(this.sessionID)
         .setTextgridUri(collectionURI)
         .execute();
-
     String aggregation = IOUtils.readStringFromStream(tgobj.getData());
+
+    // Check amount of entries.
+    int entriesBefore = getResourceCount(aggregation);
+
     for (String uri : objectURIs) {
       aggregation = aggregation.replace("</rdf:Description>",
           "        <ore:aggregates rdf:resource=\""
@@ -357,7 +360,13 @@ public class IdiomTextGridObject {
               + "    </rdf:Description>");
     }
 
-    // System.out.println("COLLECTION: " + aggregation);
+    // Check amount of entries.
+    int entriesAfter = getResourceCount(aggregation);
+    if (entriesAfter != entriesBefore + objectURIs.size()) {
+      System.err.println("!ERROR! WRONG ENTRIES COUNT IN COLLECTION " + collectionURI);
+      System.err
+          .println("!ERROR! " + entriesAfter + " != " + entriesBefore + " + " + objectURIs.size());
+    }
 
     tgobj.setData(new ByteArrayInputStream(aggregation.getBytes()));
 
@@ -547,6 +556,23 @@ public class IdiomTextGridObject {
     this.textgridURI = uuid + ".0";
   }
 
+  /**
+   * @param theAggregation
+   * @return
+   */
+  private static int getResourceCount(String theAggregation) {
+
+    int result = 0;
+
+    String entries[] = theAggregation.split("<ore:aggregates rdf:resource=");
+
+    if (entries.length > 0) {
+      result = entries.length - 1;
+    }
+
+    return result;
+  }
+
   // **
   // GETTERS & SETTERS
   // **
diff --git a/src/main/java/org/classicmayan/tools/TGSearchQueries.java b/src/main/java/org/classicmayan/tools/TGSearchQueries.java
index 8576a64..5f5577e 100644
--- a/src/main/java/org/classicmayan/tools/TGSearchQueries.java
+++ b/src/main/java/org/classicmayan/tools/TGSearchQueries.java
@@ -189,6 +189,8 @@ public class TGSearchQueries {
 
     // Get all entries of collection, uses
     // <https://dev.textgridlab.org/1.0/tgsearch/info/textgrid:42j3z.0/children>
+    // NOTE We have to use children here, not navigation, we need every object from the aggregation,
+    // not only the displayed ones (such as deleted or the forbidden ones)!
     List<String> uriList = searchClient.infoQuery()
         .setSid(this.sessionID)
         .getChildren(collectionURI)
-- 
GitLab