Skip to content

GitLab

Commit ecf84b17 authored by sendler's avatar sendler 🐢
Browse files

Merge branch 'staging' into 'feature/column-transformation' 

# Conflicts:
#   modules/architrave/transform.xqm
#   templates/css/custom.css
parents adb879e4 c37863d9
Expand all Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ ecf84b17
build/
test/
expath-pkg.xml
search/node_modules
search/public
\ No newline at end of file
build.xml
View file @ ecf84b17
......@@ -28,7 +28,7 @@
</copy>
<mkdir dir="${build.dir}"/>
<zip basedir="." destfile="${destfile}" defaultexcludes="no"
excludes=".git/,${build.dir}/,${test.dir}/"/>
excludes=".git/,${build.dir}/,${test.dir}/,search/node_modules/,search/public/"/>
</target>
<target name="cleanup">
......
controller.xql
View file @ ecf84b17
......@@ -76,6 +76,14 @@ else if(ends-with($exist:resource, ".md")) then
<dispatch xmlns="http://exist.sourceforge.net/NS/exist">
<redirect url="../content.html?id=docs/{$exist:resource}"/>
</dispatch>
else if ($exist:path eq "/api.json") then
<dispatch xmlns="http://exist.sourceforge.net/NS/exist">
<forward url="{$exist:controller}/modules/api.xq" method="get" />
</dispatch>
else if ($exist:path eq "/lang.xml") then
<dispatch xmlns="http://exist.sourceforge.net/NS/exist">
<forward url="lang.xml"/>
</dispatch>
else if($exist:resource = "sitemap.xml") then
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
{for $html in xmldb:get-child-resources( $config:app-root || "/templates")
......
helpers/query.xq 0 → 100644
View file @ ecf84b17
xquery version "3.1";
declare namespace tei="http://www.tei-c.org/ns/1.0";
declare namespace xhtml="http://www.w3.org/1999/xhtml";
let $docs := collection('/db/apps/sade-architrave/templates/register')//tei:item
let $pages := collection('/db/apps/sade-architrave/templates/34zmq')/xhtml:div
let $more := collection('/db/apps/sade-architrave/templates/3q4rq')/xhtml:div
return $more//xhtml:h4[1]
(:let $hits := $pages[ft:query(., 'e*')]:)
(:return $hits:)
helpers/redindex.xq 0 → 100644
View file @ ecf84b17
xquery version "3.1";
(:xmldb:reindex('/db/apps/sade-architrave'):)
import module namespace elastic="http://elastic.io" at "../modules/elastic.xqm";
let $r := elastic:setup()
return $r
lang.xml
View file @ ecf84b17
......@@ -9,6 +9,11 @@
<lang key="en">Results of Search</lang>
<lang key="fr">Résultats</lang>
</word>
<word key="search">
<lang key="de">suchen</lang>
<lang key="en">search</lang>
<lang key="fr">rechercher</lang>
</word>
<word key="Language">
<lang key="de">Deutsch</lang>
<lang key="en">English</lang>
......@@ -341,11 +346,6 @@
<lang key="en">pages</lang>
<lang key="fr">Pages</lang>
</word>
<word key="page">
<lang key="de">Seite</lang>
<lang key="en">pages</lang>
<lang key="fr">Page</lang>
</word>
<word key="legendDates">
<lang key="de">Datum</lang>
<lang key="en">dates</lang>
......@@ -578,18 +578,18 @@
</word>
<word key="register_persons">
<lang key="de">Personen</lang>
<lang key="en">persons</lang>
<lang key="fr">personnes</lang>
<lang key="en">Persons</lang>
<lang key="fr">Personnes</lang>
</word>
<word key="register_places">
<lang key="de">Orte</lang>
<lang key="en">places</lang>
<lang key="fr">les lieux</lang>
<lang key="en">Places</lang>
<lang key="fr">Lieux</lang>
</word>
<word key="register_works">
<lang key="de">Werke</lang>
<lang key="en">works</lang>
<lang key="fr">les œuvres</lang>
<lang key="en">Works</lang>
<lang key="fr">Œuvres</lang>
</word>
<word key="button_other">
<lang key="de">andere</lang>
......@@ -601,4 +601,34 @@
<lang key="en">not identified</lang>
<lang key="fr">non identifié</lang>
</word>
<word key="page">
<lang key="de">Seite</lang>
<lang key="en">page</lang>
<lang key="fr">page</lang>
</word>
<word key="no_results">
<lang key="de">keine Ergebnisse</lang>
<lang key="en">no results</lang>
<lang key="fr">aucun résultat</lang>
</word>
<word key="show_results_in_other_languages">
<lang key="de">auch Ergebnisse in anderen Sprachen anzeigen</lang>
<lang key="en">also show results in other languages</lang>
<lang key="fr">afficher les résultats dans d'autres langues</lang>
</word>
<word key="german">
<lang key="de">deutsch</lang>
<lang key="en">German</lang>
<lang key="fr">allemand</lang>
</word>
<word key="english">
<lang key="de">Englisch</lang>
<lang key="en">English</lang>
<lang key="fr">anglais</lang>
</word>
<word key="french">
<lang key="de">französisch</lang>
<lang key="en">French</lang>
<lang key="fr">français</lang>
</word>
</translation>
\ No newline at end of file
modules/api.xq 0 → 100644
View file @ ecf84b17
xquery version "3.1";
(:import module namespace config="https://sade.textgrid.de/ns/config" at "config.xqm";:)
(:import module namespace kwic="http://exist-db.org/xquery/kwic";:)
import module namespace elastic="http://elastic.io" at "elastic.xqm";
declare namespace output="http://www.w3.org/2010/xslt-xquery-serialization";
(:declare namespace tei="http://www.tei-c.org/ns/1.0";:)
(:declare namespace xhtml="http://www.w3.org/1999/xhtml";:)
declare option output:method "json";
declare option output:media-type "application/json";
declare variable $action := request:get-parameter('action', 'search');
declare variable $terms := request:get-parameter('terms', '');
declare variable $page := xs:integer(request:get-parameter('page', 1));
declare variable $per_page := xs:integer(request:get-parameter('per_page', 10));
declare variable $locale := request:get-parameter('locale', 'any');
declare variable $app-root := '/db/apps/sade-architrave';
declare function local:search-editions-elastic() {
let $filters := if ($locale eq 'any') then [] else [
map {"term": map {"locale": $locale}}
]
let $response := elastic:search("edition_pages", map {
"from": ($page - 1) * $per_page,
"size": $per_page,
"query": map {
"bool": map {
"must": [
map {
"query_string": map {
"query": $terms
}
}
],
"filter": $filters
}
},
"highlight": map {
"fragment_size": 100,
"require_field_match": fn:false(),
"pre_tags": ['<span class="hl">'],
"post_tags": ['</span>'],
"type": "plain",
"highlight_query": map {
"query_string": map {
"query": $terms
}
},
"fields": map {
"name": map {"number_of_fragments": 0},
"search_data": map {}
}
}
})
return $response
};
declare function local:search-pages-elastic() {
let $filters := if ($locale eq 'any') then [] else [
map {"term": map {"locale": $locale}}
]
let $response := elastic:search("wiki_pages", map {
"from": ($page - 1) * $per_page,
"size": $per_page,
"query": map {
"bool": map {
"must": [
map {
"query_string": map {
"query": $terms
}
}
],
"filter": $filters
}
},
"highlight": map {
"fragment_size": 100,
"require_field_match": fn:false(),
"pre_tags": ['<span class="hl">'],
"post_tags": ['</span>'],
"type": "plain",
"highlight_query": map {
"query_string": map {
"query": $terms
}
},
"fields": map {
"search_data": map {}
}
}
})
return $response
};
declare function local:search-register-elastic($type) {
let $fields := if ($locale eq 'de') then
['search_data.de']
else if ($locale eq 'fr') then
['search_data.fr']
else
['search_data.de', 'search_data.fr']
let $response := elastic:search($type, map {
"from": ($page - 1) * $per_page,
"size": $per_page,
"query": map {
"query_string": map {
"query": $terms,
"fields": $fields
}
},
"highlight": map {
"fragment_size": 100,
"require_field_match": fn:false(),
"pre_tags": ['<span class="hl">'],
"post_tags": ['</span>'],
"type": "plain",
"fields": map {
"name.de": map {"number_of_fragments": 0},
"name.fr": map {"number_of_fragments": 0},
"search_data.de": map {},
"search_data.fr": map {}
}
}
})
return $response
};
declare function local:error($message as xs:string, $status as xs:integer) {
let $x := response:set-status-code($status)
return
map {'message': $message}
};
declare function local:route() {
if ($action eq 'search-people') then
local:search-register-elastic('people')
else if ($action eq 'search-works') then
local:search-register-elastic('works')
else if ($action eq 'search-places') then
local:search-register-elastic('places')
else if ($action eq 'search-editions') then
local:search-editions-elastic()
else if ($action eq 'search-pages') then
local:search-pages-elastic()
else
local:error('unknown action', 400)
};
local:route()
modules/architrave/transform_register.xq
View file @ ecf84b17
......@@ -71,7 +71,7 @@ declare variable $letter_query :=
if ($letter_value = "button_other") then "^[^A-Za-zÄäÖöÜüÉéÎîÈè]"
else "^[" || $letter_value || "]";
<div>
<div id="register_ul_css">
<ul>
<h2>{string($letter_text)}</h2>
......
modules/elastic.xqm 0 → 100644
View file @ ecf84b17
xquery version "3.1";
module namespace elastic="http://elastic.io";
import module namespace config="https://sade.textgrid.de/ns/config" at "config.xqm";
declare namespace tei="http://www.tei-c.org/ns/1.0";
declare namespace xhtml="http://www.w3.org/1999/xhtml";
declare namespace http="http://expath.org/ns/http-client";
declare namespace elastic="http://elastic.io";
declare variable $elastic:uri := 'http://127.0.0.1:9200';
declare variable $elastic:prefix := 'at_';
declare function elastic:search($index, $query) {
let $json := elastic:to_json($query)
let $response := http:send-request(
<http:request method="POST" href="{$elastic:uri}/{$elastic:prefix}{$index}/_search">
<http:body media-type="application/json" method="text">{$json}</http:body>
</http:request>
)
let $str := util:base64-decode($response[2])
let $data := parse-json($str)
return $data
};
declare function elastic:drop-index($index) {
let $response := http:send-request(
<http:request method="DELETE" href="{$elastic:uri}/{$elastic:prefix}{$index}" />
)
return $response
};
declare function elastic:create-index($index, $data) {
let $json := elastic:to_json($data)
let $response := http:send-request(
<http:request method="PUT" href="{$elastic:uri}/{$elastic:prefix}{$index}">
<http:body media-type="application/json" method="text">{$json}</http:body>
</http:request>
)
let $ok := elastic:require_ok($response)
return $response
};
declare function elastic:index-doc($index, $doc) {
let $json := elastic:to_json($doc)
let $response := http:send-request(
<http:request method="POST" href="{$elastic:uri}/{$elastic:prefix}{$index}/_doc">
<http:body media-type="application/json" method="text">{$json}</http:body>
</http:request>
)
let $ok := elastic:require_ok($response)
return $response
};
declare function elastic:index-doc($index, $id, $doc) {
let $json := elastic:to_json($doc)
let $response := http:send-request(
<http:request method="POST" href="{$elastic:uri}/{$elastic:prefix}{$index}/_doc/{$id}">
<http:body media-type="application/json" method="text">{$json}</http:body>
</http:request>
)
let $ok := elastic:require_ok($response)
return $response
};
declare function elastic:refresh() {
let $response := http:send-request(
<http:request method="GET" href="{$elastic:uri}/_refresh" />
)
let $ok := elastic:require_ok($response)
return $response
};
declare function elastic:index-person($person) {
let $doc := map {
"id": string($person/@xml:id),
"name": map {
"de": data($person/tei:persName[@xml:lang='de']),
"fr": data($person/tei:persName[@xml:lang='fra'])
},
"search_data": map {
"de": $person/*[@xml:lang='de']/text(),
"fr": $person/*[@xml:lang='fra']/text()
}
}
return elastic:index-doc('people', $doc)
};
declare function elastic:index-place($place) {
let $x := fn:trace($place, "Bla")
let $doc := map {
"id": string($place/@xml:id),
"name": map {
"de": data($place/tei:placeName[@xml:lang='de']),
"fr": data($place/tei:placeName[@xml:lang='fra'])
},
"search_data": map {
"de": data($place/*[@xml:lang='de']),
"fr": data($place/*[@xml:lang='fra'])
}
}
return elastic:index-doc('places', $doc)
};
declare function elastic:index-work($work) {
let $doc := map {
"id": string($work/@xml:id),
"name": map {
"de": data($work/tei:name[@xml:lang='de']),
"fr": data($work/tei:name[@xml:lang='fra'])
},
"search_data": map {
"de": data($work/*[@xml:lang='de']),
"fr": data($work/*[@xml:lang='fra'])
}
}
return elastic:index-doc('works', $doc)
};
declare function elastic:index-edition-page($page, $locale) {
let $uri := string($page//xhtml:span[@id='tei-meta-textGridURI']/text())
let $edition := fn:tokenize($uri, '[:\.]')[2]
let $page_number := xs:integer($page//xhtml:span[@class='pb']/text())
let $doc := map {
"id": concat('edition-', $edition, '-', $page_number),
"edition": $edition,
"page_number": xs:integer($page//xhtml:span[@class='pb']/text()),
"locale": $locale,
"name": string($page//xhtml:h4[starts-with(@id, 'tei-title-main-')][1]),
"search_data": fn:normalize-space(data($page))
}
return elastic:index-doc('edition_pages', $doc)
};
declare function elastic:index-wiki-page($page, $locale) {
let $uri := string(base-uri($page))
let $doc_name := fn:tokenize($uri, '/')[6]
let $parts := fn:tokenize($doc_name, '[_\.]')
let $id := $parts[1]
let $locale := $parts[2]
let $elastic_doc_id := concat($id, '_', $locale)
let $doc := map {
"uri": $uri,
"id": $id,
"locale": $locale,
"name": data($page//h1),
"search_data": fn:normalize-space(data($page))
}
return elastic:index-doc('wiki_pages', $elastic_doc_id, $doc)
};
declare function elastic:require_ok($response) {
let $code := xs:integer($response[1]/@status)
return
if ($code < 200 or $code > 299) then
let $body := util:base64-decode($response[2])
let $name := QName('http://datypic.com/err', 'ProdNumReq')
let $msg := concat("(", string($code), ") elasticsearch responded with:", $body)
return fn:error($name, $msg)
else
fn:true()
};
declare function elastic:to_json($data) {
let $result := fn:serialize($data, map { 'method': 'json'})
return $result
};
declare function elastic:setup() {
let $editions_de := collection(
concat($config:app-root, '/templates/34zmq'), (: harrach :)
concat($config:app-root, '/templates/34zs7'), (: sturm :)
concat($config:app-root, '/templates/3ptwg'), (: corfey :)
concat($config:app-root, '/templates/3qr4f'), (: neumann :)
concat($config:app-root, '/templates/34znb'), (: pitzler :)
concat($config:app-root, '/templates/3c0m2') (: knesebeck :)
)
let $editions_fr := collection(
concat($config:app-root, '/templates/3czfj'), (: harrach :)
concat($config:app-root, '/templates/3q4rq'), (: sturm :)
concat($config:app-root, '/templates/3r0fv'), (: corfey :)
concat($config:app-root, '/templates/3r3nn'), (: neumann :)
concat($config:app-root, '/templates/350mg'), (: pitzler :)
concat($config:app-root, '/templates/3czn9') (: knesebeck :)
)
let $register_mappings := map {
"_doc": map {
"properties": map {
"id": map {"type": "keyword", "index": fn:false()},
"name": map {
"properties": map {
"de": map {"type": "text", "index": fn:false()},
"fr": map {"type": "text", "index": fn:false()}
}
},
"search_data": map {
"properties": map {
"de": map {"type": "text"},
"fr": map {"type": "text"}
}
}
}
}
}
let $r := [
elastic:drop-index("people"),
elastic:drop-index("places"),
elastic:drop-index("works"),
elastic:drop-index("wiki_pages"),
elastic:drop-index("edition_pages"),
elastic:create-index("people", map {"mappings": $register_mappings}),
elastic:create-index("places", map {"mappings": $register_mappings}),
elastic:create-index("works", map {"mappings": $register_mappings}),
elastic:create-index("wiki_pages", map {"mappings": map {
"_doc": map {
"properties": map {
"id": map {"type": "keyword", "index": fn:false()},
"locale": map {"type": "keyword"},
"name": map {"type": "text"},
"search_data": map {"type": "text"}
}
}
}}),
elastic:create-index("edition_pages", map {"mappings": map {
"_doc": map {
"properties": map {
"id": map {"type": "keyword", "index": fn:false()},
"edition": map {"type": "keyword", "index": fn:false()},
"page_number": map {"type": "integer"},
"locale": map {"type": "keyword"},
"name": map {"type": "text"},
"search_data": map {"type": "text"}
}
}
}}),
for $doc in collection(concat($config:app-root, '/templates/register'))//tei:person
return elastic:index-person($doc),
for $doc in collection(concat($config:app-root, '/templates/register'))//tei:place
return elastic:index-place($doc),
for $doc in collection(concat($config:app-root, '/templates/register'))//tei:item
return elastic:index-work($doc),
for $doc in $editions_de/xhtml:div
return elastic:index-edition-page($doc, "de"),
for $doc in $editions_fr/xhtml:div
return elastic:index-edition-page($doc, "fr"),
for $doc in collection(concat($config:app-root, '/docs'))//div[@id='wiki']
let $uri := string(base-uri($doc))
let $doc_name := fn:tokenize($uri, '/')[6]
let $parts := fn:tokenize($doc_name, '[_\.]')
let $locale := $parts[2]
let $revision := xs:integer(replace($parts[3], 'rev', ''))
order by $revision
return elastic:index-wiki-page($doc, $locale),
elastic:refresh()
]
return $r
};
modules/wiki/confluence.xqm
View file @ ecf84b17
......@@ -12,6 +12,7 @@ import module namespace config="https://sade.textgrid.de/ns/config" at "../confi
import module namespace app="https://sade.textgrid.de/ns/app" at "../app.xqm";
import module namespace functx="http://www.functx.com";
import module namespace console="http://exist-db.org/xquery/console";
import module namespace elastic="http://elastic.io" at "../elastic.xqm";
declare namespace cf="https://sade.textgrid.de/ns/configfile";
declare namespace http="http://expath.org/ns/http-client";
......@@ -62,6 +63,7 @@ if( $revisionInWiki = $lastRevinDb )
)
let $result := doc( $forLoop( $lang ) )
let $indexed := elastic:index-wiki-page($result, $lang)
return ($result)
)
......
post-install.xq
View file @ ecf84b17
xquery version "3.1";
import module namespace transformXML="https://sade.textgrid.de/ns/transform" at "modules/architrave/transform.xqm";
import module namespace elastic="http://elastic.io" at "modules/elastic.xqm";
declare namespace repo="http://exist-db.org/xquery/repo";
declare namespace xconf="http://exist-db.org/collection-config/1.0";
......@@ -95,7 +96,7 @@ let $data-xconf := $target || "/textgrid/data/collection.xconf"
let $store-xconf := xmldb:store("/db/system/config" || $target || "/textgrid/data", "collection.xconf", doc( $data-xconf ))
let $reindex := xmldb:reindex($target || "/textgrid/data")
(: import XML files from TGRep and transform them into HTML
(: import XML files from TGRep and transform them into HTML :)
let $log := util:log-system-out("Importing XML files from TGRep")
let $harrach := transformXML:importFromTGRep('34zmq.5','34zmq')
let $neumannFR := transformXML:importFromTGRep('3r3nn.1','3r3nn')
......@@ -110,7 +111,6 @@ let $pitzlerFR := transformXML:importFromTGRep('350mg.3','350mg')
let $knesebeck := transformXML:importFromTGRep('3c0m2.3','3c0m2')
let $knesebeckFR := transformXML:importFromTGRep('3czn9.4','3czn9')
let $log := util:log-system-out("DONE: Importing XML files from TGRep")
:)
let $store-xconf := xmldb:store("/db/system/config"||$target||"/textgrid/data", "collection.xconf", doc( $data-xconf ))
let $reindex := xmldb:reindex($target||"/textgrid/data")
......@@ -119,6 +119,8 @@ let $path := $system-path
let $user := util:system-property("user.name")
let $message1 := $path || " is not available. Create it and make sure " || $user || " can write there."
let $message2 := "Could not write to " || $path || "."
let $indexing_msg := util:log-system-out("indexing edition pages and registers ...")
let $indexing := elastic:setup()
return (
if
(file:is-directory($path))
......@@ -148,3 +150,5 @@ let $parameters :=
return
util:log-system-out($file-name || " saved.")
)
search/package-lock.json 0 → 100644
View file @ ecf84b17
This diff is collapsed.
search/package.json 0 → 100644
View file @ ecf84b17
{
"name": "architrave-search",
"version": "1.0.0",