elastic.xqm

xquery version "3.1";

module namespace elastic="http://elastic.io";

import module namespace config="https://sade.textgrid.de/ns/config" at "config.xqm";

declare namespace tei="http://www.tei-c.org/ns/1.0";
declare namespace xhtml="http://www.w3.org/1999/xhtml";
declare namespace http="http://expath.org/ns/http-client";

declare namespace elastic="http://elastic.io";

declare variable $elastic:uri := 'http://127.0.0.1:9200';
declare variable $elastic:prefix := 'at_';

declare function elastic:search($index, $query) {
  let $json := elastic:to_json($query)
  let $response := http:send-request(
      <http:request method="POST" href="{$elastic:uri}/{$elastic:prefix}{$index}/_search">
          <http:body media-type="application/json" method="text">{$json}</http:body>
      </http:request>
  )
  let $str := util:base64-decode($response[2])
  let $data := parse-json($str)
  return $data
};

declare function elastic:drop-index($index) {
    let $response := http:send-request(
        <http:request method="DELETE" href="{$elastic:uri}/{$elastic:prefix}{$index}" />
    )
    let $ok := elastic:require_ok($response)
    return $response
};

declare function elastic:create-index($index, $data) {
    let $json := elastic:to_json($data)
    let $response := http:send-request(
        <http:request method="PUT" href="{$elastic:uri}/{$elastic:prefix}{$index}">
            <http:body media-type="application/json" method="text">{$json}</http:body>
        </http:request>
    )
    let $ok := elastic:require_ok($response)
    return $response
};

declare function elastic:index-doc($index, $doc) {
    let $json := elastic:to_json($doc)
    let $response := http:send-request(
        <http:request method="POST" href="{$elastic:uri}/{$elastic:prefix}{$index}/_doc">
            <http:body media-type="application/json" method="text">{$json}</http:body>
        </http:request>
    )
    let $ok := elastic:require_ok($response)
    return $response
};

declare function elastic:index-doc($index, $id, $doc) {
    let $json := elastic:to_json($doc)
    let $response := http:send-request(
        <http:request method="POST" href="{$elastic:uri}/{$elastic:prefix}{$index}/_doc/{$id}">
            <http:body media-type="application/json" method="text">{$json}</http:body>
        </http:request>
    )
    let $ok := elastic:require_ok($response)
    return $response
};

declare function elastic:refresh() {
    let $response := http:send-request(
        <http:request method="GET" href="{$elastic:uri}/_refresh" />
    )
    let $ok := elastic:require_ok($response)
    return $response
};

declare function elastic:index-person($person) {
    let $doc := map {
        "id": string($person/@xml:id),
        "name": map {
            "de": data($person/tei:persName[@xml:lang='de']),
            "fr": data($person/tei:persName[@xml:lang='fra'])
        },
        "search_data": map {
            "de": $person/*[@xml:lang='de']/text(),
            "fr": $person/*[@xml:lang='fra']/text()
        }
    }
    return elastic:index-doc('people', $doc)
};

declare function elastic:index-place($place) {
    let $x := fn:trace($place, "Bla")
    let $doc := map {
        "id": string($place/@xml:id),
        "name": map {
            "de": data($place/tei:placeName[@xml:lang='de']),
            "fr": data($place/tei:placeName[@xml:lang='fra'])
        },
        "search_data": map {
            "de": data($place/*[@xml:lang='de']),
            "fr": data($place/*[@xml:lang='fra'])
        }
    }
    return elastic:index-doc('places', $doc)
};

declare function elastic:index-work($work) {
    let $doc := map {
        "id": string($work/@xml:id),
        "name": map {
            "de": data($work/tei:name[@xml:lang='de']),
            "fr": data($work/tei:name[@xml:lang='fra'])
        },
        "search_data": map {
            "de": data($work/*[@xml:lang='de']),
            "fr": data($work/*[@xml:lang='fra'])
        }
    }
    return elastic:index-doc('works', $doc)
};

declare function elastic:index-edition-page($page, $locale) {
    let $uri := string($page//xhtml:span[@id='tei-meta-textGridURI']/text())
    let $edition := fn:tokenize($uri, '[:\.]')[2]
    let $page_number := xs:integer($page//xhtml:span[@class='pb']/text())
    let $doc := map {
        "id": concat('edition-', $edition, '-', $page_number),
        "edition": $edition,
        "page_number": xs:integer($page//xhtml:span[@class='pb']/text()),
        "locale": $locale,
        "name": string($page//xhtml:h4[starts-with(@id, 'tei-title-main-')][1]),
        "search_data": fn:normalize-space(data($page))
    }
    return elastic:index-doc('edition_pages', $doc)
};

declare function elastic:index-wiki-page($page, $locale) {
    let $uri := string(base-uri($page))
    let $doc_name := fn:tokenize($uri, '/')[6]
    let $parts := fn:tokenize($doc_name, '[_\.]')
    let $id := $parts[1]
    let $locale := $parts[2]
    let $elastic_doc_id := concat($id, '_', $locale)
    let $doc := map {
        "uri": $uri,
        "id": $id,
        "locale": $locale,
        "name": data($page//h1),
        "search_data": fn:normalize-space(data($page))
    }
    return elastic:index-doc('wiki_pages', $elastic_doc_id, $doc)
};

declare function elastic:require_ok($response) {
    let $code := xs:integer($response[1]/@status)
    return 
        if ($code < 200 or $code > 299) then
            let $body := util:base64-decode($response[2])
            let $name := QName('http://datypic.com/err', 'ProdNumReq')
            let $msg := concat("(", string($code), ") elasticsearch responded with:", $body)
            return fn:error($name, $msg)
        else
            fn:true()
};

declare function elastic:to_json($data) {
  let $result := fn:serialize($data, map { 'method': 'json'})
  return $result
};

declare function elastic:setup() {
    let $editions_de := collection(
        concat($config:app-root, '/templates/34zmq'), (: harrach :)
        concat($config:app-root, '/templates/34zs7'), (: sturm :)
        concat($config:app-root, '/templates/3ptwg'), (: corfey :)
        concat($config:app-root, '/templates/3qr4f'), (: neumann :)
        concat($config:app-root, '/templates/34znb'), (: pitzler :)
        concat($config:app-root, '/templates/3c0m2')  (: knesebeck :)
    )
    let $editions_fr := collection(
        concat($config:app-root, '/templates/3czfj'), (: harrach :)
        concat($config:app-root, '/templates/3q4rq'), (: sturm :)
        concat($config:app-root, '/templates/3r0fv'), (: corfey :)
        concat($config:app-root, '/templates/3r3nn'), (: neumann :)
        concat($config:app-root, '/templates/350mg'), (: pitzler :)
        concat($config:app-root, '/templates/3czn9')  (: knesebeck :)
    )
    let $register_mappings := map {
        "_doc": map {
            "properties": map {
                "id": map {"type": "keyword", "index": fn:false()},
                "name": map {
                    "properties": map {
                        "de": map {"type": "text", "index": fn:false()},
                        "fr": map {"type": "text", "index": fn:false()}
                    }
                },
                "search_data": map {
                    "properties": map {
                        "de": map {"type": "text"},
                        "fr": map {"type": "text"}
                    }
                }
            }
        }
    }
    let $r := [
        elastic:drop-index("people"),
        elastic:drop-index("places"),
        elastic:drop-index("works"),
        elastic:drop-index("wiki_pages"),
        elastic:drop-index("edition_pages"),
        
        elastic:create-index("people", map {"mappings": $register_mappings}),
        elastic:create-index("places", map {"mappings": $register_mappings}),
        elastic:create-index("works", map {"mappings": $register_mappings}),
        elastic:create-index("wiki_pages", map {"mappings": map {
            "_doc": map {
                "properties": map {
                    "id": map {"type": "keyword", "index": fn:false()},
                    "locale": map {"type": "keyword"},
                    "name": map {"type": "text"},
                    "search_data": map {"type": "text"}
                }
            }
        }}),
        elastic:create-index("edition_pages", map {"mappings": map {
            "_doc": map {
                "properties": map {
                    "id": map {"type": "keyword", "index": fn:false()},
                    "edition": map {"type": "keyword", "index": fn:false()},
                    "page_number": map {"type": "integer"},
                    "locale": map {"type": "keyword"},
                    "name": map {"type": "text"},
                    "search_data": map {"type": "text"}
                }
            }
        }}),
        
        for $doc in collection(concat($config:app-root, '/templates/register'))//tei:person
            return elastic:index-person($doc),
            
        for $doc in collection(concat($config:app-root, '/templates/register'))//tei:place
            return elastic:index-place($doc),
            
        for $doc in collection(concat($config:app-root, '/templates/register'))//tei:item
            return elastic:index-work($doc),
    
        for $doc in $editions_de/xhtml:div
            return elastic:index-edition-page($doc, "de"),
          
        for $doc in $editions_fr/xhtml:div
            return elastic:index-edition-page($doc, "fr"),
        
        for $doc in collection(concat($config:app-root, '/docs'))//div[@id='wiki']
            let $uri := string(base-uri($doc))
            let $doc_name := fn:tokenize($uri, '/')[6]
            let $parts := fn:tokenize($doc_name, '[_\.]')
            let $locale := $parts[2]
            let $revision := xs:integer(replace($parts[3], 'rev', ''))
            order by $revision
            return elastic:index-wiki-page($doc, $locale),
        
        elastic:refresh()
    ]
    
    return $r
};