Commit cf372c85 authored by Mathias Goebel's avatar Mathias Goebel 🎠
Browse files

Merge branch 'release/3.0.0'

parents c05ac8e2 e864dfeb
image: frekele/ant
- build
- deploy
stage: build
- echo "starting ant build"
- ant
- build/*.xar
stage: deploy
- FILENAME=$(ls build/*.xar)
- curl -u ci:${EXIST_UPLOAD_PW} -X POST -F file=@${FILENAME}
stage: deploy
- "curl -X POST -F token=${GITLAB_TRIGGER_TOKEN} -F ref=develop"
......@@ -10,6 +10,8 @@
<param key="textgrid.authZinstance"></param>
<param key="textgrid.public-triplestore"></param>
<param key="textgrid.nonpublic-triplestore"></param>
<param key=""></param>
<param key=""></param>
<!-- if you want to distinguish between develop and productive instance,
set sade.develop "true"
......@@ -45,12 +47,22 @@
<module key="wiki">
<param key="confluence.url"></param>
<param key="confluence.lang" description="if the multilanguage plugin is enabled, used languages specified here">
<!-- sample configuration for Confluence -->
<param key="confluence.url"></param>
<param key="confluence.lang" description="if the multilanguage plugin is enabled, used languages specified here">
<lang code="de" name="german"/>
<lang code="en" name="english"/>
<lang code="fr" name="french"/>
<!-- sample configuration for DokuWiki -->
<param key="dokuwiki.url"></param>
<param key="">fontane</param>
<param key="dokuwiki.whitelist">
<module key="multiviewer">
......@@ -65,20 +77,20 @@
<module key="faceted-search">
<param key="viewer-html">content.html?id=/data/</param>
<!-- how much kwic hits to show per document -->
<param key="hits-per-page">10</param>
<param key="kwic-hits">3</param>
<param key="kwic-width">20</param>
<param key="thumbnail">false</param>
<param key="hits-per-page">10</param>
<param key="result-xslt">search-hits.xslt</param>
<param key="query-root">
<param key="result-title">
<param key="result-xslt">search-hits.xslt</param>
<param key="thumbnail">false</param>
<param key="viewer-html">content.html?id=/data/</param>
<param key="facets">
<facet key="authors" title="Authors">
......@@ -12,6 +12,8 @@ declare variable $exist:controller external;
declare variable $exist:prefix external;
declare variable $exist:root external;
let $exist:resource := replace($exist:resource, "%3A", ":")
if ($exist:path eq '') then
<dispatch xmlns="">
<redirect url="{request:get-uri()}/"/>
......@@ -27,14 +29,14 @@ else if (ends-with($exist:resource, ".html")) then
<dispatch xmlns="">
<forward url="{$exist:controller}/templates{$exist:path}" method="get"/>
<forward url="{$exist:controller}/modules/view.xq">
<add-parameter name="exist-resource" value="{$exist:resource}" />
<forward url="{$exist:controller}/templates/error.html" method="get"/>
<forward url="{$exist:controller}/modules/view.xq"/>
<forward url="{$exist:controller}/modules/view.xq">
<add-parameter name="exist-resource" value="{$exist:resource}" />
<forward url="{$exist:controller}/templates/error.html" method="get"/>
<forward url="{$exist:controller}/modules/view.xq"/>
(: Resource paths starting with ~assets are loaded from the sade-assets app :)
else if (contains($exist:path, "/~assets/")) then
......@@ -43,13 +45,13 @@ else if (contains($exist:path, "/~assets/")) then
<set-header name="Cache-Control" value="max-age=3600, must-revalidate"/>
(: redirect textgrid URIs to their renderings :)
else if(starts-with($exist:resource, "textgrid")) then
let $preserveRevision := config:get("textgrid.preserveRevision") = "true"
let $id := if( contains($exist:resource, ".") and $preserveRevision )
then $exist:resource
else tokenize($exist:resource, "\.")[1]
let $isImage := doc( $config:data-root || "/images.xml" )//image/starts-with(@uri, $id) = true()
let $isImage := doc( $config:data-root || "/images.xml" )//image/starts-with(@uri, $id) = true()
......@@ -73,16 +75,22 @@ else if(starts-with($exist:resource, "textgrid")) then
(: redirect markdown files from documentation to the content.html :)
else if(ends-with($exist:resource, ".md")) then
<dispatch xmlns="">
<redirect url="../content.html?id=docs/{$exist:resource}"/>
if(starts-with($exist:path, "/docs"))
<dispatch xmlns="">
<redirect url="../content.html?id=docs/{$exist:resource}"/>
<dispatch xmlns="">
<redirect url="content.html?id=docs/{$exist:resource}"/>
else if($exist:resource = "sitemap.xml") then
<urlset xmlns="">
{for $html in xmldb:get-child-resources( $config:app-root || "/templates")
where not( contains($html, "_") )
<loc>{request:get-url() => substring-before("sitemap.xml")}{$html}</loc>
<loc>{ request:get-url() => substring-before("sitemap.xml") }{ $html }</loc>
......@@ -13,27 +13,31 @@ Institute for the History of Science (MPI-WG), the Cologne Center for
eHumanities (CCEH) and the Göttingen State and University Library (SUB; as
part of the project TextGrid).
The software is based on the XML database eXist and its up to date version
itself is a customization. It was developed for the TextGrid infrastructure.
Over the years a few components became obsolete, so the current version is a
completely rewritten one.
The software is based on the XML database [eXist]( and its
up to date version itself is a customization. It was developed for the TextGrid
infrastructure. Over the years a few components became obsolete, so the current
version is a completely rewritten one.
## Features
- [faceted search](
- [preconfigured indexes](
- [TextGrid clients](
- transfer data from the Lab
- transfer data from the TextGrid Repository
- validation against RelaxNG schema on publish
- store data in the Lab
- [fork/download projects](
- [Lucene string analyzer](
- made for TEI
- easy (as in easy) configuration of charmaps and synonyms
- easy (as in easy) to configure menu bar (frontend)
- store data in the TextGrid Repository
- [fork/download SADE projects](
- [preconfigured indexes](
- Lucene string analyzer
- configured for TEI
- easy (as in easy) configuration/maintenance of charmaps and synonyms
- Range Index
- frontend
- easy (as in easy) to configure menu bar
- professional template
- [wiki parser](
- synchronize selected wiki pages
- [Confluence](
- [Dokuwiki](
- professional template
- multi-language support
- [prepared for optionally prerendering](
### Third party addons
......@@ -45,7 +49,15 @@ By default the eXist database offers a RESTful interface to all of your data,
but with the help of a little XQuery, you can provide on-the-fly transformations
via REST as well.
## TextGrid
This version is prepared to present data transferred via the [TextGrid](
Laboratory. You have to use the SADE-Publisher Plug-In in the Lab.
## Development
Get an overview of the complete architecture and an entry point to start
developing right now at the [Developers Guide](
You can find our git repositories at
[GWDG's Gitlab](
......@@ -3,14 +3,14 @@ SADE was build to fit the needs of many projects creating digital editions.
For a complete customization, a adaption of some XQuery modules may becomes
necessary. But a few things are made to set up by any user, as long as she is
familiar with the basics of XML. When you want to start preparing your own
instance, look at the file config.xml in the root collection of the application.
instance, look at the file `config.xml` in the root collection of the application.
There are many things you can set up.
## Top Menu
Edit the top menu (navbar) by entering your preferred structure in the file
<item label="Ein Text" link="view.html?"/>
<submenu label="Links">
......@@ -18,24 +18,3 @@ Edit the top menu (navbar) by entering your preferred structure in the file
Mit dieser Konfiguration wird ein minimales Menü aufgebaut, das ein Aufklappbares Menü mit dem Namen "Links" und einem Unterpunkt mit externem Link aufbaut, obendrein einen Link auf der Hauptebene zu "Ein Text", in der Darstellung sieht das dann so aus:
![Menü Screenshot](~assets/docs/sade-dropdown.png)
Sie können einfach zum Tutorialprojekt neue Menüpunkte hinzufügen, diese umbenennen, oder das Menü umstellen, probieren Sie es aus!
[Menü bearbeiten](/exist/apps/eXide/index.html?open=/db/sade-projects/tutorial/navigation.xml)
Schauen Sie das Ergebnis an: [Tutorialprojekt](/exist/apps/SADE/tutorial/)
Sie können natürlich auch eigene Dokumente in das Menü aufnhemen, z.B. publizierte TEI Daten .
# Anpassen des Projekttitels
Auch der Projekttitel lässt sich anpassen, er ist vermerkt in der config.xml als ``<param key="project-title">Mein Projekt</param>``
[Projektkonfiguration bearbeiten](/exist/apps/eXide/index.html?open=/db/sade-projects/tutorial/config.xml).
1. Erstellen Sie einen neuen Link im Menü
2. Ändern Sie den Text von "Text1"
3. Ändern Sie den Projekttitel
4. Verlinken Sie ein selbtpubliziertes TEI Dokument im Menü (hierzu "[Publikation eigener Inhalte](view.html?" lesen)
# Development
## Architecture
This is image includes data.
## Packages
SADE consists of three EXPath packages to run within the eXist database
+ SADE.xar - the main application ([git]( | [built](
+ assets.xar - all static resources ([git]( | [built](
+ search engine - to configure the Lucene index ([git]( | [built](
## Dependencies and Requirements
Within eXist the following packages are required:
+ Markdown Parser
+ Function Documentation
+ Shared Resources
Further dependencies may defined by [eXist](
Mainly this is Java 8 and sufficient amount of memory 2GiB should be available
for exclusive usage (`Xmx`).
## Build
There are `ant` tasks available in a separate [git repo](
. This build script generate a complete package containing the eXist database and all required components. Configure this build via the `` file.
Every package can be built separate by calling `ant` in the root directory. The
result will be a `build\*.xar` file.
## Physical Structure
The file structure is pretty usual for EXPath application within eXist.
To read more on this, the chapter about the Model View Controller in the eXist
book is highly recommended.
All basic files are stored in the root directory, especially the main configuration
and the controller.xql (the main router in the MVC model), index configuration
and package descriptions.
|-- docs
|-- modules
|-- templates
|-- textgrid
## Request Processing by Example
All requests for HTML pages are handled by the controller.xql
# DokuwikiParser
To be able to maintain the pages containing content about project, documentation
and other stuff an comfortable editor and a system that stores the history of a
document is very helpful. In many cases projects using wikis for communication,
coordination and as a general knowledge base. By adding selected pages to a
white-list, the wiki may be become the place for editing the textual content of
the SADE webpage.
The dokuwikiParser was developed by the project on Fontane-Notizbücher.
All the code is in `modules/wiki/wiki.xqm`.
# Faceted-Search
## Important Notes
Always use a root node that is specified in the xconf.
## Konfiguration des Moduls
Die Konfiguration für das Faceted-Search Modul befindet sich in der config.xml
<module key="faceted-search">
<param key="viewer-html">index.html?id=/xml/data/</param>
[Konfiguration bearbeiten](/exist/apps/eXide/index.html?open=/db/sade-projects/textgrid/config.xml)
## Ändern der Ergebnisdarstellung
Mithilfe der Konfigurations-Option "result-xslt" kann auf ein XSLT Stylesheet verwiesen werden,
welches aus den Metadaten im TEI-Dokument des Einzeltreffers einzelne Suchtreffer darstellt.
<param key="result-xslt">/sade-projects/textgrid/xslt/search-hit.xslt</param>
Eine einfaches XSLT Stylesheet zur Ergebnisanzeige ist vorhanden und kann an eigene TEI-Metadaten angepasst werden.
[XSLT bearbeiten](/exist/apps/eXide/index.html?open=/db/sade-projects/textgrid/xslt/search-hit.xslt). Eine solche
XSLT Transformation für Einzelergebnisse kann im Template definiert werden:
<div data-template="fsearch:result-xslt"/>
## Konfiguration der Facetten
In der Konfiguration des Faceted-Search Moduls können die dargestellten Facetten mithilfe eines XPATH Ausdrucks festgelegt werden.
relations to other documents:
+ [index configuration](
## Configuration – Base
As always the configuration is stored at `config.xml` in the applications root
collection. Please look for `<module key="faceted-search">`.
Besides the facets there are several options and endpoints to configure:
|hits-per-page|hits pagination|
|kwic-hits|KWIC hits to show per document|
|kwic-width|how many left/right neighbors to show for every hit|
|query-root|XML node the query is send to; **always** use one that is specified in your data collections `collection.xconf` (and so available for Lucene)|
|result-title|title for a document hit|
|result-xslt|XSLT to apply to the hits|
|thumbnail|!LEGACY show/hide thumbnails for a document|
|viewer-html|the link that every result points to|
## Configuration – Facets
To set up own facets you have to provide a `@key` (for internal usage and
debugging), a `@title` (used as headline) and a corresponding XPath that is
applied to all hits on a `descendant-or-self` axis.
When a namespace is needed that is not specified in `modules/faceted-search.xqm`,
there are three possible options:
+ specify namespace there
+ use a wildcard namespace `*:` (untested)
+ a valid explicit QName notation may be added here (untested).
<param key="facets">
<facet key="persons" title="Person">
<facet key="authors" title="Authors">
<facet key="keywords" title="Schlagwort">
<facet key="dates" title="Zeit">
[Konfiguration bearbeiten](/exist/apps/eXide/index.html?open=/db/sade-projects/textgrid/config.xml)
......@@ -3,11 +3,11 @@ When working with a SADE instance users usually start manipulating the output,
preparing own transformations or want to use a different configuration for the
lucene index. These changes should be tracked in a Version Control System like
*git* and also they should be deployed to a standalone application that does not
interfere with others. Thats why SADE offers a mechanism to fork any instance at
interfere with others. That's why SADE offers a mechanism to fork any instance at
any state. Developers can use this to continue development and to new users it
offers a playground for testing purposes.
At the SADE GUI within the Lab enter a new project name at the input field in
the first panel (1).
At the SADE Publisher within the Lab enter a new project name at the input field
in the first panel (1).
Click on "create" (2) and wait until you get redirected to your own instance.
......@@ -18,16 +18,20 @@ To continue you have to enter new credentials to the SADE Plugin configuration.
> User: ProjectName
> Password:
> Password:
The username will be the same as the project name and you have to enter the
formerly typed password here as well.
# Download
# Download & Tracking changes
You can download any application by clicking on the download button at the SADE
Publihser GUI. This gives you a XAR package according to the EXPath package
standard which is in fact a ZIP archive.
When any further development is planned, fork the offical SADE repo from
When any further development is planned, fork the offical SADE repo from
[]( clone the
repo to your local machine and replace the files with those from the XAR archive.
This is for complete upstream compatibility.
\ No newline at end of file
This is for complete upstream compatibility.
When you want to publish your fork in the SADE group (for example to make it
part of the [reference instance]( you have to rename
the repository.
......@@ -2,8 +2,11 @@
SADE ships with preconfigured indexes to be found in the files `collection.xconf`.
For larger data sets a good index configuration helps to provide adequate load
**Attention**: To take effect changes must be done in the corresponding resources at `/db/system/config/`. The the post installation script copies the files over there. This is just to preserve the configuration in the EXPath package of this
**Attention**: To take effect changes must be done in the corresponding
resources at `/db/system/config/`. The the post installation script copies the
files over there. This is just to preserve the configuration in the EXPath
package of this application (and the git repo).
## Range Index
The range index is used for fast key/value lookups. You will find more information in the eXist-db documentation [website](
......@@ -13,13 +16,15 @@ Queries are *may* optimized by automatic index lookups when they provide paths.
For all text nodes in the XML resources, Lucene provides an index to query for
words and phrases. It is used by the [faceted search module](
With a standard installation of SADE, a customized and configurable [String Analyzer]( is available. It is installed as a XAR library.
For more information on the built-in lucene, please look at the [eXist
## Current Configurations
### `/collection.xconf`
Within the root collection, the configuration contains the RESTXQ trigger only,
that enables REST annotations for function calls to provide paths and convert
folders to parameter.
**This currently not used by SADE.** All pathes additional to the collections and
**This currently not used by SADE.** All paths additional to the collections and
resources stored in the database are provided by the URL rewriter at `/controller.xql`.
### `textgrid/agg/collection.xconf` and `textgrid/rdf/collection.xconf`
......@@ -30,12 +35,22 @@ Enables the RDF index. This is an optional feature that requires the
This configures the range index for objects generated with the text-image-link editor. Indexes are configured for link targets and shapes.
### `textgrid/meta/collection.xconf`
Within this collection all textgrid metadata is stored. Usually queries regard the
Within this collection all textgrid metadata is stored. Usually queries are on the
complete URI in the `tgmd:textgridUri` element and the title from `tgmd:title`.
### `textgrid/data/collection.xconf`
This configures the Lucene index and a range index.
#### Lucene
Character mappings and a synonym list are stored in a separate files on the file system. The publish GUI puts text files (text/plain) there.
Character mappings and a synonym list are stored in separate files on the file system.
The publish GUI puts text files (text/plain) there. This means that you can maintain
these files from outside the database, via the Lab or any other external tool.
##### Charmaps
When an object named `charmap.txt` comes in via the Publisher, this file will be
stored on the system at `$EXIST_HOME`. It have to comply to the Lucene standard.
##### Synonyms
Same applies to a `synonyms.txt`.
##### TEI data
Define the nodes a text index should be prepared with `<text qname="tei:TEI" analyzer="fontane"/>`. To index words that are divided in different text nodes because of inline elements, these elements should be defined with `<inline qname="tei:g"/>` (optional). Text nodes in elements may not appear in the index can be defined with `<ignore qname="tei:del"/>` (optional).
#### Range
The entries here are just for example. They are used within the Fontane-Notizbücher
project but should usually not interfere with custom data.
# Prerendering
When the amount of data increases there will be a trade-off between boosting
the transformation with more efficient queries together with a corresponding
index configuration and the option to disable on-the-fly processing by caching
transformation results. It will speed up the page loading time.
The preprocessed items can be added to the database and may be available via REST
in addition to the original dataset.
When dealing with objects transferred form the TextGrid Repository (e.g. via the
SADE Publisher) there is exactly one place to insert a switch to the preferred
routines: within the `/modules/textgrid/connect.xqm` you can add instructions
based on the MIME-type of the source starting at [line 118](
and with a placeholder at [line 135](
Prerendering is disabled by default, as it is hard to guess what transformation
should be done and where to store.
......@@ -2,47 +2,50 @@
Within the TextGridLab the SADE Publish-Plugin is needed. It is available via the [Marketplace](
## Installation
![Marketplace Screenshot](~assets/docs/marketplace.png)
### Einrichten des "SADE Publish Tool"
Nach dem Neustart aus dem Menu "Fenster->Benutzervorgaben" auswählen. Hier den Eintrag "Sade Publisher" auswählen und eintragen:
## Configuration
After the installation process is done, the plugin needs a pointer to a SADE
server. Add the configuration via menu "Window" -> "Preferences"
![Preferences Screenshot](~assets/docs/preferences.png)
> __SADE Url:__ http://localhost:8080/exist/apps/textgrid-connect/publish/tutorial/
> __Authorized User:__ sade
> __Password:__ test
![Preferences Screenshot](~assets/docs/preferences.png)
> __Password:__
(leave the password empty)
## Eine erste Publikation
## Usage
To publish data from the Lab to a SADE instance the SADE server must be up and
### Öffnen der SADE-Publish Perspektive
Falls noch nicht geschehen, nun bitte im TextGridLab anmelden. Den "Welcome-Screen" schliessen, und dann aus der Tool-Leiste die SADE-Publish-Perspektive asuwählen. Es bietet sich die Folgende Ansicht:
### Start the Publish tool
To start the SADE publisher use on of the following options:
+ click the icon ![icon](~assets/docs/publisher-icon.png),
+ via menu `Tools` -> `SADE Publish` or
+ hit `[CTRL] + 3` and type `SADE` and select `SADE Perspective` from the list.
The view should change to something like this:
![SADE-Publish Perspective Screenshot](~assets/docs/sade-publish2.png)
Der Knopf "Reset" ist zum Zurücksetzen der Ansicht, der Knopf "Publish" löst den Publikationsvorgang aus, dazu mehr im nächsten Abschnitt.
### Hinzufügen eigener Objekte
Nun können Objekte aus einem bestehenden Projekt publiziert werden. Im Navigator links werden geeignete Objekte ausgewählt, dabei ist auch eine Mehrfachauswahl mithilfe der 'Shift' oder 'Strg' Taste möglich. Über den Kontextmenüintrag "Publish to SADE" im Navigator können die ausgewählten Objekte in die rechte Publikationsauswahl einefügt werden, wie im folgenden Screenshot zu sehen.
![SADE-Publish - Objekte hinzufügen](~assets/docs/sade-publish3.png)
Im Publikationsbereich erscheint nun der Name des zu publizierenden Objektes, davor ein Icon das die Art des Objektes anzeigt und ein kleines [x], mit dem einzelne Objekte wieder aus der Liste gelöscht werden können.
### Publikation starten
Ein Klick auf "Publish" löst den Publikationsvorgang aus. Wenn der Publikatiosvorgang eines Objektes abgeschlossen ist erscheint ein grünes Häckchen hinter dem jeweiligen Eintrag.
When using the [Navigators](TGMANUAL on Navigator) context menu entry `Publish to SADE` when the tool is not active,
the item to be published will not be recognized by the Publisher.
![SADE-Publish - OK](~assets/docs/sade-publish-done.png)
### Add items to publish
A context menu entry is added to the [Navigator](TGMANUAL on Navigator). So you
can right-click on the item you want to publish and select `Publish to SADE`.
ProTipp: By holding `[CTRL]` or `[Shift]` you can select multiple items.
Die Publikation war erfolgreich. Wie die publizierten Daten nun betrachtet werden können ist im Abschnitt "[Betrachten eigener Inhalte](view.html?" zu lesen.
Linux and Mac user can also take advantage of a drag'n'drop feature. Simply drop
the items you want to publish at the central space within the publishers view.
#Betrachten eigener Inhalte
When an aggregation is added to the Publisher, its content (and all of its content)
will be sent to SADE.
## Suche
Nun kann durch Eingabe eines Suchwortes oben rechts im Suchschlitz nach den eigenen Inhalten gesucht werden [Beispiel](results.html?q=und). Klick auf den Titel können diese angezeigt werden.
### Starting the process
Click on the "Publish" button. Successfully transferred items will be highlighted
green. The "Log" contains all information to the process.
# TextGrid Clients
\ No newline at end of file
There are XQuery functions to talk to the REST and SOAP interfaces of TextGrid
available in `/modules/textgrid/`. They are mainly used to READ data from any
(public|private) part of the repository – see []( –,
but can be used to do any other operation, eg. manipulating data, creating data
and more. A possible use case may is the
logging of information in a file available in TextGrid - so it will be
independent from you installation and will survive an `apt purge` on the server.
You can use SADE to cleanup data, rewrite files, put some information about
validation to the project you and your team is working on.
For description of the available functions, see the XQdocs annotations or have
a look at the visualization on any running server at [`fundocs.html`](http://localhost:8080/exist/apps/sade/fundocs.html).
......@@ -5,5 +5,5 @@ the somewhat weird formats to plain HTML and stores these resources in `docs`.
To setup one or both modules, a specified configuration is needed. Read on at the
specific documentation:
* **[Conluence Wiki](**
* **[Confluence Wiki](**
* **[DokuWiki](**
<package xmlns="" name="" abbrev="SADE" version="2.9.1" spec="1.0">
<package xmlns="" name="" abbrev="SADE" version="3.0.0" spec="1.0">
<title>Scalable Architecture for Digital Editions powered by TextGrid</title>
<dependency package=""/>
<dependency package=""/>
<dependency package=""/>
<dependency package=""/>
......@@ -13,6 +13,7 @@ import module namespace functx="";
declare namespace rdf="";
declare namespace tei="";
declare namespace test="";
declare namespace tgmd="";
declare namespace tgrel="";
declare namespace xqdoc="";
......@@ -60,8 +61,7 @@ return