Merge branch 'staging' into 'feature/column-transformation'

# Conflicts:
#   modules/architrave/transform.xqm
#   templates/css/custom.css

.gitignore

 build/
 test/
 expath-pkg.xml
+
+search/node_modules
+search/public
\ No newline at end of file

build.xml

@@ -28,7 +28,7 @@
       </copy>
       <mkdir dir="${build.dir}"/>
       <zip basedir="." destfile="${destfile}" defaultexcludes="no"
-          excludes=".git/,${build.dir}/,${test.dir}/"/>
+          excludes=".git/,${build.dir}/,${test.dir}/,search/node_modules/,search/public/"/>
   </target>
 
   <target name="cleanup">

controller.xql

@@ -76,6 +76,14 @@ else if(ends-with($exist:resource, ".md")) then
     <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
         <redirect url="../content.html?id=docs/{$exist:resource}"/>
     </dispatch>
+else if ($exist:path eq "/api.json") then
+    <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
+        <forward url="{$exist:controller}/modules/api.xq" method="get" />
+    </dispatch>
+else if ($exist:path eq "/lang.xml") then
+    <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
+        <forward url="lang.xml"/>
+    </dispatch>
 else if($exist:resource = "sitemap.xml") then
     <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
         {for $html in xmldb:get-child-resources( $config:app-root || "/templates")

helpers/query.xq

+xquery version "3.1";
+
+declare namespace tei="http://www.tei-c.org/ns/1.0";
+declare namespace xhtml="http://www.w3.org/1999/xhtml";
+
+let $docs := collection('/db/apps/sade-architrave/templates/register')//tei:item
+let $pages := collection('/db/apps/sade-architrave/templates/34zmq')/xhtml:div
+let $more := collection('/db/apps/sade-architrave/templates/3q4rq')/xhtml:div
+return $more//xhtml:h4[1]
+(:let $hits := $pages[ft:query(., 'e*')]:)
+(:return $hits:)

helpers/redindex.xq

+xquery version "3.1";
+(:xmldb:reindex('/db/apps/sade-architrave'):)
+import module namespace elastic="http://elastic.io" at "../modules/elastic.xqm";
+
+let $r := elastic:setup()
+return $r

lang.xml

@@ -9,6 +9,11 @@
         <lang key="en">Results of Search</lang>
         <lang key="fr">Résultats</lang>
     </word>
+    <word key="search">
+        <lang key="de">suchen</lang>
+        <lang key="en">search</lang>
+        <lang key="fr">rechercher</lang>
+    </word>
     <word key="Language">
         <lang key="de">Deutsch</lang>
         <lang key="en">English</lang>
@@ -341,11 +346,6 @@
         <lang key="en">pages</lang>
         <lang key="fr">Pages</lang>
     </word>
-    <word key="page">
-        <lang key="de">Seite</lang>
-        <lang key="en">pages</lang>
-        <lang key="fr">Page</lang>
-    </word>
     <word key="legendDates">
         <lang key="de">Datum</lang>
         <lang key="en">dates</lang>
@@ -578,18 +578,18 @@
     </word>
     <word key="register_persons">
         <lang key="de">Personen</lang>
-        <lang key="en">persons</lang>
-        <lang key="fr">personnes</lang>
+        <lang key="en">Persons</lang>
+        <lang key="fr">Personnes</lang>
     </word>
     <word key="register_places">
         <lang key="de">Orte</lang>
-        <lang key="en">places</lang>
-        <lang key="fr">les lieux</lang>
+        <lang key="en">Places</lang>
+        <lang key="fr">Lieux</lang>
     </word>
     <word key="register_works">
         <lang key="de">Werke</lang>
-        <lang key="en">works</lang>
-        <lang key="fr">les œuvres</lang>
+        <lang key="en">Works</lang>
+        <lang key="fr">Œuvres</lang>
     </word>
     <word key="button_other">
         <lang key="de">andere</lang>
@@ -601,4 +601,34 @@
         <lang key="en">not identified</lang>
         <lang key="fr">non identifié</lang>
     </word>
+    <word key="page">
+        <lang key="de">Seite</lang>
+        <lang key="en">page</lang>
+        <lang key="fr">page</lang>
+    </word>
+    <word key="no_results">
+        <lang key="de">keine Ergebnisse</lang>
+        <lang key="en">no results</lang>
+        <lang key="fr">aucun résultat</lang>
+    </word>
+    <word key="show_results_in_other_languages">
+        <lang key="de">auch Ergebnisse in anderen Sprachen anzeigen</lang>
+        <lang key="en">also show results in other languages</lang>
+        <lang key="fr">afficher les résultats dans d'autres langues</lang>
+    </word>
+    <word key="german">
+        <lang key="de">deutsch</lang>
+        <lang key="en">German</lang>
+        <lang key="fr">allemand</lang>
+    </word>
+    <word key="english">
+        <lang key="de">Englisch</lang>
+        <lang key="en">English</lang>
+        <lang key="fr">anglais</lang>
+    </word>
+    <word key="french">
+        <lang key="de">französisch</lang>
+        <lang key="en">French</lang>
+        <lang key="fr">français</lang>
+    </word>
 </translation>
\ No newline at end of file

modules/api.xq

+xquery version "3.1";
+
+(:import module namespace config="https://sade.textgrid.de/ns/config" at "config.xqm";:)
+(:import module namespace kwic="http://exist-db.org/xquery/kwic";:)
+import module namespace elastic="http://elastic.io" at "elastic.xqm";
+
+declare namespace output="http://www.w3.org/2010/xslt-xquery-serialization";
+(:declare namespace tei="http://www.tei-c.org/ns/1.0";:)
+(:declare namespace xhtml="http://www.w3.org/1999/xhtml";:)
+
+declare option output:method "json";
+declare option output:media-type "application/json";
+
+declare variable $action := request:get-parameter('action', 'search');
+declare variable $terms := request:get-parameter('terms', '');
+declare variable $page := xs:integer(request:get-parameter('page', 1));
+declare variable $per_page := xs:integer(request:get-parameter('per_page', 10));
+declare variable $locale := request:get-parameter('locale', 'any');
+declare variable $app-root := '/db/apps/sade-architrave';
+
+declare function local:search-editions-elastic() {
+    let $filters := if ($locale eq 'any') then [] else [
+        map {"term": map {"locale": $locale}}
+    ]
+    let $response := elastic:search("edition_pages", map {
+        "from": ($page - 1) * $per_page,
+        "size": $per_page,
+        "query": map {
+            "bool": map {
+                "must": [
+                    map {
+                        "query_string": map {
+                            "query": $terms
+                        }
+                    }
+                ],
+                "filter": $filters
+            }
+        },
+        "highlight": map {
+            "fragment_size": 100,
+            "require_field_match": fn:false(),
+            "pre_tags": ['<span class="hl">'],
+            "post_tags": ['</span>'],
+            "type": "plain",
+            "highlight_query": map {
+                "query_string": map {
+                    "query": $terms
+                }
+            },
+            "fields": map {
+              "name": map {"number_of_fragments": 0},
+              "search_data": map {}
+            }
+        }
+    })
+    return $response
+};
+
+declare function local:search-pages-elastic() {
+    let $filters := if ($locale eq 'any') then [] else [
+        map {"term": map {"locale": $locale}}
+    ]
+    let $response := elastic:search("wiki_pages", map {
+        "from": ($page - 1) * $per_page,
+        "size": $per_page,
+        "query": map {
+            "bool": map {
+                "must": [
+                    map {
+                        "query_string": map {
+                            "query": $terms
+                        }
+                    }
+                ],
+                "filter": $filters
+            }
+        },
+        "highlight": map {
+            "fragment_size": 100,
+            "require_field_match": fn:false(),
+            "pre_tags": ['<span class="hl">'],
+            "post_tags": ['</span>'],
+            "type": "plain",
+            "highlight_query":  map {
+                "query_string": map {
+                    "query": $terms
+                }
+            },
+            "fields": map {
+              "search_data": map {}
+            }
+        }
+    })
+    return $response
+};
+
+declare function local:search-register-elastic($type) {
+    let $fields := if ($locale eq 'de') then
+        ['search_data.de']
+    else if ($locale eq 'fr') then
+        ['search_data.fr']
+    else
+        ['search_data.de', 'search_data.fr']
+    let $response := elastic:search($type, map {
+        "from": ($page - 1) * $per_page,
+        "size": $per_page,
+        "query": map {
+            "query_string": map {
+                "query": $terms,
+                "fields": $fields
+            }
+        },
+        "highlight": map {
+            "fragment_size": 100,
+            "require_field_match": fn:false(),
+            "pre_tags": ['<span class="hl">'],
+            "post_tags": ['</span>'],
+            "type": "plain",
+            "fields": map {
+                "name.de": map {"number_of_fragments": 0},
+                "name.fr": map {"number_of_fragments": 0},
+                "search_data.de": map {},
+                "search_data.fr": map {}
+            }
+        }
+    })
+    return $response
+};
+
+declare function local:error($message as xs:string, $status as xs:integer) {
+    let $x := response:set-status-code($status)
+    return
+        map {'message': $message}
+};
+
+declare function local:route() {
+    if ($action eq 'search-people') then
+        local:search-register-elastic('people')
+    else if ($action eq 'search-works') then
+        local:search-register-elastic('works')
+    else if ($action eq 'search-places') then
+        local:search-register-elastic('places')
+    else if ($action eq 'search-editions') then
+        local:search-editions-elastic()
+    else if ($action eq 'search-pages') then
+        local:search-pages-elastic()
+    else
+        local:error('unknown action', 400)
+};
+
+local:route()

modules/architrave/transform_register.xq

@@ -71,7 +71,7 @@ declare variable $letter_query :=
 if ($letter_value = "button_other") then "^[^A-Za-zÄäÖöÜüÉéÎîÈè]"
 else "^[" || $letter_value || "]";
 
-<div>
+<div id="register_ul_css">
     
 <ul>
 <h2>{string($letter_text)}</h2>

modules/elastic.xqm

+xquery version "3.1";
+
+module namespace elastic="http://elastic.io";
+
+import module namespace config="https://sade.textgrid.de/ns/config" at "config.xqm";
+
+declare namespace tei="http://www.tei-c.org/ns/1.0";
+declare namespace xhtml="http://www.w3.org/1999/xhtml";
+declare namespace http="http://expath.org/ns/http-client";
+
+declare namespace elastic="http://elastic.io";
+
+declare variable $elastic:uri := 'http://127.0.0.1:9200';
+declare variable $elastic:prefix := 'at_';
+
+declare function elastic:search($index, $query) {
+  let $json := elastic:to_json($query)
+  let $response := http:send-request(
+      <http:request method="POST" href="{$elastic:uri}/{$elastic:prefix}{$index}/_search">
+          <http:body media-type="application/json" method="text">{$json}</http:body>
+      </http:request>
+  )
+  let $str := util:base64-decode($response[2])
+  let $data := parse-json($str)
+  return $data
+};
+
+declare function elastic:drop-index($index) {
+    let $response := http:send-request(
+        <http:request method="DELETE" href="{$elastic:uri}/{$elastic:prefix}{$index}" />
+    )
+    return $response
+};
+
+declare function elastic:create-index($index, $data) {
+    let $json := elastic:to_json($data)
+    let $response := http:send-request(
+        <http:request method="PUT" href="{$elastic:uri}/{$elastic:prefix}{$index}">
+            <http:body media-type="application/json" method="text">{$json}</http:body>
+        </http:request>
+    )
+    let $ok := elastic:require_ok($response)
+    return $response
+};
+
+declare function elastic:index-doc($index, $doc) {
+    let $json := elastic:to_json($doc)
+    let $response := http:send-request(
+        <http:request method="POST" href="{$elastic:uri}/{$elastic:prefix}{$index}/_doc">
+            <http:body media-type="application/json" method="text">{$json}</http:body>
+        </http:request>
+    )
+    let $ok := elastic:require_ok($response)
+    return $response
+};
+
+declare function elastic:index-doc($index, $id, $doc) {
+    let $json := elastic:to_json($doc)
+    let $response := http:send-request(
+        <http:request method="POST" href="{$elastic:uri}/{$elastic:prefix}{$index}/_doc/{$id}">
+            <http:body media-type="application/json" method="text">{$json}</http:body>
+        </http:request>
+    )
+    let $ok := elastic:require_ok($response)
+    return $response
+};
+
+declare function elastic:refresh() {
+    let $response := http:send-request(
+        <http:request method="GET" href="{$elastic:uri}/_refresh" />
+    )
+    let $ok := elastic:require_ok($response)
+    return $response
+};
+
+declare function elastic:index-person($person) {
+    let $doc := map {
+        "id": string($person/@xml:id),
+        "name": map {
+            "de": data($person/tei:persName[@xml:lang='de']),
+            "fr": data($person/tei:persName[@xml:lang='fra'])
+        },
+        "search_data": map {
+            "de": $person/*[@xml:lang='de']/text(),
+            "fr": $person/*[@xml:lang='fra']/text()
+        }
+    }
+    return elastic:index-doc('people', $doc)
+};
+
+declare function elastic:index-place($place) {
+    let $x := fn:trace($place, "Bla")
+    let $doc := map {
+        "id": string($place/@xml:id),
+        "name": map {
+            "de": data($place/tei:placeName[@xml:lang='de']),
+            "fr": data($place/tei:placeName[@xml:lang='fra'])
+        },
+        "search_data": map {
+            "de": data($place/*[@xml:lang='de']),
+            "fr": data($place/*[@xml:lang='fra'])
+        }
+    }
+    return elastic:index-doc('places', $doc)
+};
+
+declare function elastic:index-work($work) {
+    let $doc := map {
+        "id": string($work/@xml:id),
+        "name": map {
+            "de": data($work/tei:name[@xml:lang='de']),
+            "fr": data($work/tei:name[@xml:lang='fra'])
+        },
+        "search_data": map {
+            "de": data($work/*[@xml:lang='de']),
+            "fr": data($work/*[@xml:lang='fra'])
+        }
+    }
+    return elastic:index-doc('works', $doc)
+};
+
+declare function elastic:index-edition-page($page, $locale) {
+    let $uri := string($page//xhtml:span[@id='tei-meta-textGridURI']/text())
+    let $edition := fn:tokenize($uri, '[:\.]')[2]
+    let $page_number := xs:integer($page//xhtml:span[@class='pb']/text())
+    let $doc := map {
+        "id": concat('edition-', $edition, '-', $page_number),
+        "edition": $edition,
+        "page_number": xs:integer($page//xhtml:span[@class='pb']/text()),
+        "locale": $locale,
+        "name": string($page//xhtml:h4[starts-with(@id, 'tei-title-main-')][1]),
+        "search_data": fn:normalize-space(data($page))
+    }
+    return elastic:index-doc('edition_pages', $doc)
+};
+
+declare function elastic:index-wiki-page($page, $locale) {
+    let $uri := string(base-uri($page))
+    let $doc_name := fn:tokenize($uri, '/')[6]
+    let $parts := fn:tokenize($doc_name, '[_\.]')
+    let $id := $parts[1]
+    let $locale := $parts[2]
+    let $elastic_doc_id := concat($id, '_', $locale)
+    let $doc := map {
+        "uri": $uri,
+        "id": $id,
+        "locale": $locale,
+        "name": data($page//h1),
+        "search_data": fn:normalize-space(data($page))
+    }
+    return elastic:index-doc('wiki_pages', $elastic_doc_id, $doc)
+};
+
+declare function elastic:require_ok($response) {
+    let $code := xs:integer($response[1]/@status)
+    return 
+        if ($code < 200 or $code > 299) then
+            let $body := util:base64-decode($response[2])
+            let $name := QName('http://datypic.com/err', 'ProdNumReq')
+            let $msg := concat("(", string($code), ") elasticsearch responded with:", $body)
+            return fn:error($name, $msg)
+        else
+            fn:true()
+};
+
+declare function elastic:to_json($data) {
+  let $result := fn:serialize($data, map { 'method': 'json'})
+  return $result
+};
+
+declare function elastic:setup() {
+    let $editions_de := collection(
+        concat($config:app-root, '/templates/34zmq'), (: harrach :)
+        concat($config:app-root, '/templates/34zs7'), (: sturm :)
+        concat($config:app-root, '/templates/3ptwg'), (: corfey :)
+        concat($config:app-root, '/templates/3qr4f'), (: neumann :)
+        concat($config:app-root, '/templates/34znb'), (: pitzler :)
+        concat($config:app-root, '/templates/3c0m2')  (: knesebeck :)
+    )
+    let $editions_fr := collection(
+        concat($config:app-root, '/templates/3czfj'), (: harrach :)
+        concat($config:app-root, '/templates/3q4rq'), (: sturm :)
+        concat($config:app-root, '/templates/3r0fv'), (: corfey :)
+        concat($config:app-root, '/templates/3r3nn'), (: neumann :)
+        concat($config:app-root, '/templates/350mg'), (: pitzler :)
+        concat($config:app-root, '/templates/3czn9')  (: knesebeck :)
+    )
+    let $register_mappings := map {
+        "_doc": map {
+            "properties": map {
+                "id": map {"type": "keyword", "index": fn:false()},
+                "name": map {
+                    "properties": map {
+                        "de": map {"type": "text", "index": fn:false()},
+                        "fr": map {"type": "text", "index": fn:false()}
+                    }
+                },
+                "search_data": map {
+                    "properties": map {
+                        "de": map {"type": "text"},
+                        "fr": map {"type": "text"}
+                    }
+                }
+            }
+        }
+    }
+    let $r := [
+        elastic:drop-index("people"),
+        elastic:drop-index("places"),
+        elastic:drop-index("works"),
+        elastic:drop-index("wiki_pages"),
+        elastic:drop-index("edition_pages"),
+        
+        elastic:create-index("people", map {"mappings": $register_mappings}),
+        elastic:create-index("places", map {"mappings": $register_mappings}),
+        elastic:create-index("works", map {"mappings": $register_mappings}),
+        elastic:create-index("wiki_pages", map {"mappings": map {
+            "_doc": map {
+                "properties": map {
+                    "id": map {"type": "keyword", "index": fn:false()},
+                    "locale": map {"type": "keyword"},
+                    "name": map {"type": "text"},
+                    "search_data": map {"type": "text"}
+                }
+            }
+        }}),
+        elastic:create-index("edition_pages", map {"mappings": map {
+            "_doc": map {
+                "properties": map {
+                    "id": map {"type": "keyword", "index": fn:false()},
+                    "edition": map {"type": "keyword", "index": fn:false()},
+                    "page_number": map {"type": "integer"},
+                    "locale": map {"type": "keyword"},
+                    "name": map {"type": "text"},
+                    "search_data": map {"type": "text"}
+                }
+            }
+        }}),
+        
+        for $doc in collection(concat($config:app-root, '/templates/register'))//tei:person
+            return elastic:index-person($doc),
+            
+        for $doc in collection(concat($config:app-root, '/templates/register'))//tei:place
+            return elastic:index-place($doc),
+            
+        for $doc in collection(concat($config:app-root, '/templates/register'))//tei:item
+            return elastic:index-work($doc),
+    
+        for $doc in $editions_de/xhtml:div
+            return elastic:index-edition-page($doc, "de"),
+          
+        for $doc in $editions_fr/xhtml:div
+            return elastic:index-edition-page($doc, "fr"),
+        
+        for $doc in collection(concat($config:app-root, '/docs'))//div[@id='wiki']
+            let $uri := string(base-uri($doc))
+            let $doc_name := fn:tokenize($uri, '/')[6]
+            let $parts := fn:tokenize($doc_name, '[_\.]')
+            let $locale := $parts[2]
+            let $revision := xs:integer(replace($parts[3], 'rev', ''))
+            order by $revision
+            return elastic:index-wiki-page($doc, $locale),
+        
+        elastic:refresh()
+    ]
+    
+    return $r
+};

modules/wiki/confluence.xqm

@@ -12,6 +12,7 @@ import module namespace config="https://sade.textgrid.de/ns/config" at "../confi
 import module namespace app="https://sade.textgrid.de/ns/app" at "../app.xqm";
 import module namespace functx="http://www.functx.com";
 import module namespace console="http://exist-db.org/xquery/console";
+import module namespace elastic="http://elastic.io" at "../elastic.xqm";
 
 declare namespace cf="https://sade.textgrid.de/ns/configfile";
 declare namespace http="http://expath.org/ns/http-client";
@@ -62,6 +63,7 @@ if( $revisionInWiki = $lastRevinDb )
             )
 
         let $result := doc( $forLoop( $lang ) )
+        let $indexed := elastic:index-wiki-page($result, $lang)
         return ($result)
     )
 

post-install.xq

 xquery version "3.1";
 
 import module namespace transformXML="https://sade.textgrid.de/ns/transform" at "modules/architrave/transform.xqm";
+import module namespace elastic="http://elastic.io" at "modules/elastic.xqm";
 
 declare namespace repo="http://exist-db.org/xquery/repo";
 declare namespace xconf="http://exist-db.org/collection-config/1.0";
@@ -95,7 +96,7 @@ let $data-xconf := $target || "/textgrid/data/collection.xconf"
 let $store-xconf := xmldb:store("/db/system/config" || $target || "/textgrid/data", "collection.xconf", doc( $data-xconf ))
 let $reindex := xmldb:reindex($target || "/textgrid/data")
 
-(: import XML files from TGRep and transform them into HTML 
+(: import XML files from TGRep and transform them into HTML :)
 let $log := util:log-system-out("Importing XML files from TGRep")
 let $harrach := transformXML:importFromTGRep('34zmq.5','34zmq')
 let $neumannFR := transformXML:importFromTGRep('3r3nn.1','3r3nn')
@@ -110,7 +111,6 @@ let $pitzlerFR := transformXML:importFromTGRep('350mg.3','350mg')
 let $knesebeck := transformXML:importFromTGRep('3c0m2.3','3c0m2')
 let $knesebeckFR := transformXML:importFromTGRep('3czn9.4','3czn9')
 let $log := util:log-system-out("DONE: Importing XML files from TGRep")
-:)
 
 let $store-xconf := xmldb:store("/db/system/config"||$target||"/textgrid/data", "collection.xconf", doc( $data-xconf ))
 let $reindex := xmldb:reindex($target||"/textgrid/data")
@@ -119,6 +119,8 @@ let $path := $system-path
 let $user := util:system-property("user.name")
 let $message1 := $path || " is not available. Create it and make sure " || $user || " can write there."
 let $message2 := "Could not write to " || $path || "."
+let $indexing_msg := util:log-system-out("indexing edition pages and registers ...")
+let $indexing := elastic:setup()
 return (
 if
     (file:is-directory($path))
@@ -148,3 +150,5 @@ let $parameters :=
 	  return
 	    util:log-system-out($file-name || " saved.")
 )
+
+

search/package.json

+{
+  "name": "architrave-search",
+  "version": "1.0.0",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"