handle bibl elements outside footnotes

8985e985 · Christian Boulanger · 9ceadad4 · 8985e985 · 8985e985 · 8985e985
Commit 8985e985 authored 8 months ago by Christian Boulanger
--- a/convert-anystyle-data/anystyle-to-tei.ipynb
+++ b/convert-anystyle-data/anystyle-to-tei.ipynb
@@ -319,8 +319,8 @@
  {
   "metadata": {
    "ExecuteTime": {
-     "end_time": "2024-07-30T13:21:35.263603300Z",
+     "end_time": "2024-07-30T14:03:11.482551100Z",
-     "start_time": "2024-07-30T13:21:34.796357300Z"
+     "start_time": "2024-07-30T14:03:11.388100200Z"
    }
   },
   "cell_type": "code",
@@ -374,16 +374,26 @@
    "        return start, end\n",
    "    raise ValueError(f\"Could not find '{string}' in '{container}'\")\n",
    "\n",
-    "def add_node(parent, tag, text=\"\", attributes=None, clean_func=None):\n",
+    "def add_node(parent, tag, text=\"\", attributes=None, clean_func=None, preserve=False):\n",
-    "    cleaned_text = clean_func(text) if clean_func else text\n",
+    "    \"\"\"\n",
-    "    start, end = find_string(cleaned_text, text)\n",
+    "    Adds a child node to the parent, optionally adding text and attributes. \n",
-    "    prefix, suffix = text[:start], text[end:]\n",
+    "    If a clean_func is passed, the text is set after applying the function to it.\n",
-    "    if prefix !=\"\" and len(parent) > 0:\n",
+    "    If the `preserve` flag is True, the removed preceding or trailing text is preserved in the xml,\n",
-    "        parent[-1].tail = prefix\n",
+    "    outside of the node content \n",
+    "    \"\"\"\n",
    "    node = ET.SubElement(parent, tag, (attributes or {}))\n",
-    "    node.text = cleaned_text\n",
+    "    if clean_func:\n",
-    "    if suffix != \"\":\n",
+    "        cleaned_text = clean_func(text)\n",
-    "        node.tail = suffix\n",
+    "        if preserve:\n",
+    "            start, end = find_string(cleaned_text, text)\n",
+    "            prefix, suffix = text[:start], text[end:]\n",
+    "            if prefix !=\"\" and len(parent) > 0:\n",
+    "                parent[-1].tail = prefix\n",
+    "            node.text = cleaned_text\n",
+    "            if suffix != \"\":\n",
+    "                node.tail = suffix\n",
+    "    else:\n",
+    "        node.text = text\n",
    "    return node\n",
    "\n",
    "def create_tei_root():\n",
@@ -392,41 +402,46 @@
    "    })\n",
    "\n",
    "def create_tei_header(tei_root, title):\n",
-    "    tei_header = ET.SubElement(tei_root, 'teiHeader')\n",
+    "    tei_header = add_node(tei_root, 'teiHeader')\n",
-    "    file_desc = ET.SubElement(tei_header, 'fileDesc')\n",
+    "    file_desc = add_node(tei_header, 'fileDesc')\n",
-    "    title_stmt = ET.SubElement(file_desc, 'titleStmt')\n",
+    "    title_stmt = add_node(file_desc, 'titleStmt')\n",
-    "    ET.SubElement(title_stmt, 'title').text = title\n",
+    "    add_node(title_stmt, 'title', title)\n",
-    "    publication_stmt = ET.SubElement(file_desc, 'publicationStmt')\n",
+    "    publication_stmt = add_node(file_desc, 'publicationStmt')\n",
-    "    ET.SubElement(publication_stmt, 'publisher').text = \"mpilhlt\"\n",
+    "    add_node(publication_stmt, 'publisher', 'mpilhlt')\n",
-    "    source_desc = ET.SubElement(file_desc, 'sourceDesc')\n",
+    "    source_desc = add_node(file_desc, 'sourceDesc')\n",
-    "    ET.SubElement(source_desc, 'p').text = title\n",
+    "    add_node(source_desc, 'p', title)\n",
    "    return tei_header\n",
    "\n",
    "def create_body(text_root):\n",
    "    body = ET.SubElement(text_root, 'body')\n",
-    "    ET.SubElement(body, 'p').text = 'The article text is not part of this document'\n",
+    "    add_node(body, 'p', 'The article text is not part of this document')\n",
    "    return body\n",
    "\n",
-    "def anystyle_to_tei(input_xml_path, id):\n",
+    "def anystyle_to_tei(input_xml_path, id, preserve=False):\n",
    "    anystyle_root = ET.parse(input_xml_path).getroot()\n",
    "    tei_root = create_tei_root()\n",
    "    create_tei_header(tei_root, title=id)\n",
-    "    text_root = ET.SubElement(tei_root, 'text')\n",
+    "    text_root = add_node(tei_root, 'text')\n",
-    "    create_body(text_root)\n",
+    "    body = create_body(text_root)\n",
+    "    # <listBibl> element for <bibl> elements that are not in footnotes, such as a bibliography\n",
+    "    listBibl = add_node(body, 'listBibl')\n",
+    "    # iterate over all sequences (=footnotes) and translate into TEI equivalents\n",
    "    for sequence in anystyle_root.findall('sequence'):\n",
+    "        # if the sequence contains a citation-number, create a new <note> to add <bibl> elements to \n",
    "        if (cn:= sequence.findall('citation-number')):\n",
    "            attributes = {\n",
    "                'n': cn[0].text,\n",
    "                'place': 'bottom'\n",
    "            }\n",
-    "            node = add_node(text_root, 'note', attributes=attributes, clean_func=remove_punctuation)\n",
+    "            node = add_node(text_root, 'note', attributes=attributes, clean_func=remove_punctuation, preserve=preserve)\n",
    "        else:\n",
-    "            node = text_root\n",
+    "            # otherwise add to <listBibl> element\n",
+    "            node = listBibl\n",
    "        bibl = None\n",
    "        for child in sequence:\n",
    "            tag = child.tag\n",
    "            text = child.text\n",
-    "            if tag == \"citation-number\": continue\n",
+    "            if tag == \"citation-number\": continue # this has already been taken care of\n",
    "            if (bibl is None # if we do not have a bibl element yet\n",
    "                or bibl.find(tag) and tag != \"note\" # or tag already exists in the current element\n",
    "                or tag in ['signal', 'legal-ref'] # or tag belongs to a specific groups that signal a separate reference\n",
@@ -435,38 +450,38 @@
    "                bibl = ET.SubElement(node, 'bibl')\n",
    "            match tag:\n",
    "                case 'author':\n",
-    "                    add_node(bibl, 'author', text, clean_func=remove_punctuation)\n",
+    "                    add_node(bibl, 'author', text, clean_func=remove_punctuation, preserve=preserve)\n",
    "                case 'backref':\n",
-    "                    add_node(bibl, 'ref', text, clean_func=remove_punctuation)\n",
+    "                    add_node(bibl, 'ref', text, clean_func=remove_punctuation, preserve=preserve)\n",
    "                case 'container-title':\n",
-    "                    add_node(bibl, 'title', text, {'level': 'm'}, clean_func= clean_container)\n",
+    "                    add_node(bibl, 'title', text, {'level': 'm'}, clean_func= clean_container, preserve=preserve)\n",
    "                case 'collection-title':\n",
-    "                    add_node(bibl, 'title', text, {'level': 's'}, clean_func= clean_container)\n",
+    "                    add_node(bibl, 'title', text, {'level': 's'}, clean_func= clean_container, preserve=preserve)\n",
    "                case 'date':\n",
-    "                    add_node(bibl, 'date', text, clean_func= extract_year)\n",
+    "                    add_node(bibl, 'date', text, clean_func= extract_year, preserve=preserve)\n",
    "                case 'editor':\n",
-    "                    add_node(bibl, 'editor', text, clean_func=clean_editor)\n",
+    "                    add_node(bibl, 'editor', text, clean_func=clean_editor, preserve=preserve)\n",
    "                case 'note':\n",
    "                    add_node(bibl, 'note', text)                    \n",
    "                case 'journal':\n",
-    "                    add_node(bibl, 'title', text, {'level': 'j'}, clean_func= clean_container)\n",
+    "                    add_node(bibl, 'title', text, {'level': 'j'}, clean_func= clean_container, preserve=preserve)\n",
    "                case 'legal-ref':\n",
-    "                    add_node(bibl, 'ref', text, {'type': 'legal'}, clean_func = remove_punctuation)\n",
+    "                    add_node(bibl, 'ref', text, {'type': 'legal'}, clean_func = remove_punctuation, preserve=preserve)\n",
    "                case 'pages':\n",
-    "                    add_node(bibl, 'biblScope', text, {'unit': 'pp'}, clean_func= clean_pages)\n",
+    "                    add_node(bibl, 'biblScope', text, {'unit': 'pp'}, clean_func= clean_pages, preserve=preserve)\n",
    "                case 'signal':\n",
-    "                    add_node(bibl, 'note', text, {'type': 'signal'}, clean_func=remove_punctuation)\n",
+    "                    add_node(bibl, 'note', text, {'type': 'signal'}, clean_func=remove_punctuation, preserve=preserve)\n",
    "                case 'title':\n",
-    "                    add_node(bibl, 'title', text, {'level': 'a'}, clean_func=remove_punctuation)\n",
+    "                    add_node(bibl, 'title', text, {'level': 'a'}, clean_func=remove_punctuation, preserve=preserve)\n",
    "                case 'volume':\n",
-    "                    add_node(bibl, 'biblScope', text, {'unit': 'vol'}, clean_func = remove_punctuation)\n",
+    "                    add_node(bibl, 'biblScope', text, {'unit': 'vol'}, clean_func = remove_punctuation, preserve=preserve)\n",
    "    return prettify(tei_root)\n",
    "\n",
    "for input_path in glob.glob('anystyle/*.xml'):\n",
    "    base_name = os.path.basename(input_path)\n",
    "    id = os.path.splitext(base_name)[0]\n",
    "    output_path = f'tei/{id}.xml'\n",
-    "    output_xml = anystyle_to_tei(input_path, id)\n",
+    "    output_xml = anystyle_to_tei(input_path, id, preserve=True)\n",
    "    print(f'Writing {output_path}')\n",
    "    with open(output_path, 'w', encoding='utf-8') as output_file:\n",
    "        output_file.write(output_xml)"
@@ -484,7 +499,7 @@
     ]
    }
   ],
-   "execution_count": 15
+   "execution_count": 17
  },
  {
   "metadata": {},

 %% Cell type:markdown id:4c77ab592c98dfd tags:
 # Conversion to TEI (`<bibl>`)
 References:
 - https://vault.tei-c.de/P5/3.0.0/doc/tei-p5-doc/en/html/CO.html#COBI (Overview)
 - https://vault.tei-c.de/P5/3.0.0/doc/tei-p5-doc/en/html/CO.html#COBIOT (Mapping to other bibliographic formats)
 - https://vault.tei-c.de/P5/3.0.0/doc/tei-p5-doc/en/html/ref-bibl.html (`<bibl>`)
 - https://vault.tei-c.de/P5/3.0.0/doc/tei-p5-doc/en/html/ref-biblStruct.html (`biblStruct`)
 - https://epidoc.stoa.org/gl/latest/supp-bibliography.html (Examples)
 - https://quod.lib.umich.edu/cgi/t/tei/tei-idx?type=HTML&rgn=DIV2&byte=647051
 - https://grobid.readthedocs.io/en/latest/training/Bibliographical-references/ (Grobid examples using `<bibl>`)
 We use `<bibl>` here instead of `<biblStruct>` because it is more loosely-structured and allows for a more flat datastructure.
 ## Collect metadata on TEI `<bibl>` tags
 %% Cell type:code id:ff140f40df428a8f tags:
 ``` python
 import xmlschema
 import os
 # cache for local use
 if not os.path.isdir("schema/tei"):
    schema = xmlschema.XMLSchema("https://www.tei-c.org/release/xml/tei/custom/schema/xsd/tei_all.xsd")
    schema.export(target='schema/tei', save_remote=True)
 ```
 %% Cell type:code id:572f566fc9784238 tags:
 ``` python
 import xml.etree.ElementTree as ET
 import pandas as pd
 import requests
 from bs4 import BeautifulSoup
 import re
 from tqdm.notebook import tqdm
 # written by GPT-4
 def extract_headings_and_links(tag, doc_heading, doc_base_url):
    # Extract heading numbers from the document
    heading_numbers = re.findall(r'\d+(?:\.\d+)*', doc_heading)
    # Download the HTML page
    url = f"{doc_base_url}/ref-{tag}.html"
    response = requests.get(url)
    soup = BeautifulSoup(response.content, 'html.parser')
    # Extract the links associated with each heading number
    links = {}
    for link in soup.find_all('a', class_='link_ptr'):
        heading_value = link.find('span', class_='headingNumber').text.strip()
        link_url = link.get('href')
        links[heading_value] = f"{doc_base_url}/{link_url}"
    return {heading: link_url for heading, link_url in zip(heading_numbers, links.values()) if
            heading in heading_numbers}
 def generate_tag_docs(xsd_path):
    namespaces = {'xs': 'http://www.w3.org/2001/XMLSchema'}
    doc_base_url = "https://vault.tei-c.de/P5/3.0.0/doc/tei-p5-doc/en/html"
    tree = ET.parse('schema/tei/tei_all.xsd')
    root = tree.getroot()
    schema = xmlschema.XMLSchema(xsd_path)
    bibl_schema = schema.find("tei:bibl")
    data_list = []
    #names = [child_element.local_name for child_element in bibl_schema.iterchildren()]
    names = ['author', 'biblScope', 'citedRange', 'date', 'edition', 'editor', 'idno', 'location', 'note', 'orgName',
             'publisher', 'pubPlace', 'ptr', 'series', 'span', 'title', 'volume', 'issue']
    for name in tqdm(names, desc="Processing TEI tags"):
        doc_node = root.find(f".//xs:element[@name='{name}']/xs:annotation/xs:documentation", namespaces=namespaces)
        if doc_node is not None:
            matches = re.search(r'^(.*)\[(.*)]$', doc_node.text)
            if matches is None: continue
            description = matches.group(1)
            doc_heading = matches.group(2)
            doc_urls = extract_headings_and_links(name, doc_heading, doc_base_url)
            data_list.append({'name': name, 'description': description, 'documentation': doc_heading, 'urls': doc_urls})
    return pd.DataFrame(data_list)
 cache_file = "schema/tei/tei-tags-documentation.json"
 if not os.path.isfile(cache_file):
    df = generate_tag_docs("schema/tei/tei_all.xsd")
    df.to_json(cache_file, index=False, orient='records')
 else:
    df = pd.read_json(cache_file)
 df
 ```
 %% Output
              name                                        description  \
    0       author  (author) in a bibliographic reference, contain...
    1    biblScope  (scope of bibliographic reference) defines the...
    2   citedRange  (cited range) defines the range of cited conte...
    3         date             (date) contains a date in any format.
    4      edition  (edition) describes the particularities of one...
    5       editor  contains a secondary statement of responsibili...
    6     location  (location) defines the location of a place as ...
    7         note             (note) contains a note or annotation.
    8    publisher  (publisher) provides the name of the organizat...
    9     pubPlace  (publication place) contains the name of the p...
    10      series  (series information) contains information abou...
    11        span  associates an interpretative annotation direct...
    12       title    (title) contains a title for any kind of work.
                                            documentation  \
    0   3.12.2.2. Titles, Authors, and Editors 2.2.1. ...
    1   3.12.2.5. Scopes and Ranges in Bibliographic C...
    2   3.12.2.5. Scopes and Ranges in Bibliographic C...
    3   3.6.4. Dates and Times 2.2.4. Publication, Dis...
    4                        2.2.2. The Edition Statement
    5              3.12.2.2. Titles, Authors, and Editors
    6                                      14.3.4. Places
    7   3.9.1. Notes and Simple Annotation 2.2.6. The ...
    8   3.12.2.4. Imprint, Size of a Document, and Rep...
    9   3.12.2.4. Imprint, Size of a Document, and Rep...
    10  3.12.2.1. Analytic, Monographic, and Series Le...
    11                    18.3. Spans and Interpretations
    12  3.12.2.2. Titles, Authors, and Editors 2.2.1. ...
                                                     urls
    0   {'3.12.2.2': 'https://vault.tei-c.de/P5/3.0.0/...
    1   {'3.12.2.5': 'https://vault.tei-c.de/P5/3.0.0/...
    2   {'3.12.2.5': 'https://vault.tei-c.de/P5/3.0.0/...
    3   {'3.6.4': 'https://vault.tei-c.de/P5/3.0.0/doc...
    4   {'2.2.2': 'https://vault.tei-c.de/P5/3.0.0/doc...
    5   {'3.12.2.2': 'https://vault.tei-c.de/P5/3.0.0/...
    6   {'14.3.4': 'https://vault.tei-c.de/P5/3.0.0/do...
    7   {'3.9.1': 'https://vault.tei-c.de/P5/3.0.0/doc...
    8   {'3.12.2.4': 'https://vault.tei-c.de/P5/3.0.0/...
    9   {'3.12.2.4': 'https://vault.tei-c.de/P5/3.0.0/...
    10  {'3.12.2.1': 'https://vault.tei-c.de/P5/3.0.0/...
    11  {'18.3': 'https://vault.tei-c.de/P5/3.0.0/doc/...
    12  {'3.12.2.2': 'https://vault.tei-c.de/P5/3.0.0/...
 %% Cell type:markdown id:aaf43ee43bb6d4d tags:
 ## Convert Groundd Truth to TEI
 %% Cell type:code id:b3ee84984b88f24a tags:
 ``` python
 import xml.etree.ElementTree as ET
 import regex as re
 import glob
 import os
 import xml.dom.minidom
 def prettify(elem, indentation="  "):
    """Return a pretty-printed XML string for the Element."""
    return xml.dom.minidom.parseString(ET.tostring(elem, 'unicode')).toprettyxml(indent=indentation)
 def even_num_brackets(string: str):
    """
    Simple heuristic to determine if string contains an even number of round and square brackets,
    so that if not, trailing or leading brackets will be removed.
    """
    return ((string.endswith(")") and string.count(")") == string.count("("))
            or (string.endswith("]") and string.count("]") == string.count("[")))
 def remove_punctuation(text):
    """This removes leading and trailing punctuation using very simple rules for German and English"""
    start, end = 0, len(text)
    while start < len(text) and re.match("\p{P}", text[start]) and text[end - 1]:
        start += 1
    while end > start and re.match("\p{P}", text[end - 1]) and not even_num_brackets(text[start:end]) and text[end - 1] not in "?!":
        end -= 1
    return text[start:end].strip()
 def clean_editor(text):
    text = re.sub(r'^in(:| )', '', remove_punctuation(text), flags=re.IGNORECASE)
    text = re.sub(r'\(?(hrsg\. v\.|hg\. v|hrsg\.|ed\.|eds\.)\)?', '', text, flags=re.IGNORECASE)
    return text
 def clean_container(text):
    return remove_punctuation(re.sub(r'^(in|aus|from)(:| )', '', text.strip(), flags=re.IGNORECASE))
 def clean_pages(text):
    return remove_punctuation(re.sub(r'^(S\.|p\.|pp\.|ff?\.||seqq?\.)', '', text.strip(), flags=re.IGNORECASE))
 def extract_year(text):
    m = re.search( r'[12][0-9]{3}', text)
    return m.group(0) if m else None
 def find_string(string, container):
    start = container.find(string)
    if start > -1:
        end = start + len(string)
        return start, end
    raise ValueError(f"Could not find '{string}' in '{container}'")
-def add_node(parent, tag, text="", attributes=None, clean_func=None):
+def add_node(parent, tag, text="", attributes=None, clean_func=None, preserve=False):
-    cleaned_text = clean_func(text) if clean_func else text
+    """
-    start, end = find_string(cleaned_text, text)
+    Adds a child node to the parent, optionally adding text and attributes.
-    prefix, suffix = text[:start], text[end:]
+    If a clean_func is passed, the text is set after applying the function to it.
-    if prefix !="" and len(parent) > 0:
+    If the `preserve` flag is True, the removed preceding or trailing text is preserved in the xml,
-        parent[-1].tail = prefix
+    outside of the node content
+    """
    node = ET.SubElement(parent, tag, (attributes or {}))
-    node.text = cleaned_text
+    if clean_func:
-    if suffix != "":
+        cleaned_text = clean_func(text)
-        node.tail = suffix
+        if preserve:
+            start, end = find_string(cleaned_text, text)
+            prefix, suffix = text[:start], text[end:]
+            if prefix !="" and len(parent) > 0:
+                parent[-1].tail = prefix
+            node.text = cleaned_text
+            if suffix != "":
+                node.tail = suffix
+    else:
+        node.text = text
    return node
 def create_tei_root():
    return ET.Element('TEI', {
        'xmlns': "http://www.tei-c.org/ns/1.0"
    })
 def create_tei_header(tei_root, title):
-    tei_header = ET.SubElement(tei_root, 'teiHeader')
+    tei_header = add_node(tei_root, 'teiHeader')
-    file_desc = ET.SubElement(tei_header, 'fileDesc')
+    file_desc = add_node(tei_header, 'fileDesc')
-    title_stmt = ET.SubElement(file_desc, 'titleStmt')
+    title_stmt = add_node(file_desc, 'titleStmt')
-    ET.SubElement(title_stmt, 'title').text = title
+    add_node(title_stmt, 'title', title)
-    publication_stmt = ET.SubElement(file_desc, 'publicationStmt')
+    publication_stmt = add_node(file_desc, 'publicationStmt')
-    ET.SubElement(publication_stmt, 'publisher').text = "mpilhlt"
+    add_node(publication_stmt, 'publisher', 'mpilhlt')
-    source_desc = ET.SubElement(file_desc, 'sourceDesc')
+    source_desc = add_node(file_desc, 'sourceDesc')
-    ET.SubElement(source_desc, 'p').text = title
+    add_node(source_desc, 'p', title)
    return tei_header
 def create_body(text_root):
    body = ET.SubElement(text_root, 'body')
-    ET.SubElement(body, 'p').text = 'The article text is not part of this document'
+    add_node(body, 'p', 'The article text is not part of this document')
    return body
-def anystyle_to_tei(input_xml_path, id):
+def anystyle_to_tei(input_xml_path, id, preserve=False):
    anystyle_root = ET.parse(input_xml_path).getroot()
    tei_root = create_tei_root()
    create_tei_header(tei_root, title=id)
-    text_root = ET.SubElement(tei_root, 'text')
+    text_root = add_node(tei_root, 'text')
-    create_body(text_root)
+    body = create_body(text_root)
+    # <listBibl> element for <bibl> elements that are not in footnotes, such as a bibliography
+    listBibl = add_node(body, 'listBibl')
+    # iterate over all sequences (=footnotes) and translate into TEI equivalents
    for sequence in anystyle_root.findall('sequence'):
+        # if the sequence contains a citation-number, create a new <note> to add <bibl> elements to
        if (cn:= sequence.findall('citation-number')):
            attributes = {
                'n': cn[0].text,
                'place': 'bottom'
            }
-            node = add_node(text_root, 'note', attributes=attributes, clean_func=remove_punctuation)
+            node = add_node(text_root, 'note', attributes=attributes, clean_func=remove_punctuation, preserve=preserve)
        else:
-            node = text_root
+            # otherwise add to <listBibl> element
+            node = listBibl
        bibl = None
        for child in sequence:
            tag = child.tag
            text = child.text
-            if tag == "citation-number": continue
+            if tag == "citation-number": continue # this has already been taken care of
            if (bibl is None # if we do not have a bibl element yet
                or bibl.find(tag) and tag != "note" # or tag already exists in the current element
                or tag in ['signal', 'legal-ref'] # or tag belongs to a specific groups that signal a separate reference
                or (tag in ["author", "editor", "authority"] and bibl.find('date'))): # or specific tags follow a date field
                # then create a new bibl element
                bibl = ET.SubElement(node, 'bibl')
            match tag:
                case 'author':
-                    add_node(bibl, 'author', text, clean_func=remove_punctuation)
+                    add_node(bibl, 'author', text, clean_func=remove_punctuation, preserve=preserve)
                case 'backref':
-                    add_node(bibl, 'ref', text, clean_func=remove_punctuation)
+                    add_node(bibl, 'ref', text, clean_func=remove_punctuation, preserve=preserve)
                case 'container-title':
-                    add_node(bibl, 'title', text, {'level': 'm'}, clean_func= clean_container)
+                    add_node(bibl, 'title', text, {'level': 'm'}, clean_func= clean_container, preserve=preserve)
                case 'collection-title':
-                    add_node(bibl, 'title', text, {'level': 's'}, clean_func= clean_container)
+                    add_node(bibl, 'title', text, {'level': 's'}, clean_func= clean_container, preserve=preserve)
                case 'date':
-                    add_node(bibl, 'date', text, clean_func= extract_year)
+                    add_node(bibl, 'date', text, clean_func= extract_year, preserve=preserve)
                case 'editor':
-                    add_node(bibl, 'editor', text, clean_func=clean_editor)
+                    add_node(bibl, 'editor', text, clean_func=clean_editor, preserve=preserve)
                case 'note':
                    add_node(bibl, 'note', text)
                case 'journal':
-                    add_node(bibl, 'title', text, {'level': 'j'}, clean_func= clean_container)
+                    add_node(bibl, 'title', text, {'level': 'j'}, clean_func= clean_container, preserve=preserve)
                case 'legal-ref':
-                    add_node(bibl, 'ref', text, {'type': 'legal'}, clean_func = remove_punctuation)
+                    add_node(bibl, 'ref', text, {'type': 'legal'}, clean_func = remove_punctuation, preserve=preserve)
                case 'pages':
-                    add_node(bibl, 'biblScope', text, {'unit': 'pp'}, clean_func= clean_pages)
+                    add_node(bibl, 'biblScope', text, {'unit': 'pp'}, clean_func= clean_pages, preserve=preserve)
                case 'signal':
-                    add_node(bibl, 'note', text, {'type': 'signal'}, clean_func=remove_punctuation)
+                    add_node(bibl, 'note', text, {'type': 'signal'}, clean_func=remove_punctuation, preserve=preserve)
                case 'title':
-                    add_node(bibl, 'title', text, {'level': 'a'}, clean_func=remove_punctuation)
+                    add_node(bibl, 'title', text, {'level': 'a'}, clean_func=remove_punctuation, preserve=preserve)
                case 'volume':
-                    add_node(bibl, 'biblScope', text, {'unit': 'vol'}, clean_func = remove_punctuation)
+                    add_node(bibl, 'biblScope', text, {'unit': 'vol'}, clean_func = remove_punctuation, preserve=preserve)
    return prettify(tei_root)
 for input_path in glob.glob('anystyle/*.xml'):
    base_name = os.path.basename(input_path)
    id = os.path.splitext(base_name)[0]
    output_path = f'tei/{id}.xml'
-    output_xml = anystyle_to_tei(input_path, id)
+    output_xml = anystyle_to_tei(input_path, id, preserve=True)
    print(f'Writing {output_path}')
    with open(output_path, 'w', encoding='utf-8') as output_file:
        output_file.write(output_xml)
 ```
 %% Output
    Writing tei/10.1111_1467-6478.00057.xml
    Writing tei/10.1111_1467-6478.00080.xml
    Writing tei/10.1515_zfrs-1980-0103.xml
    Writing tei/10.1515_zfrs-1980-0104.xml
 %% Cell type:markdown id:b0a231dc7bdd8b01 tags:
 ## Create LinkML schema from TEI XSD
 %% Cell type:markdown id:aa86435960e61937 tags:

--- a/convert-anystyle-data/anystyle/10.1515_zfrs-1980-0103.xml
+++ b/convert-anystyle-data/anystyle/10.1515_zfrs-1980-0103.xml
@@ -28,9 +28,9 @@
        <location>Freiburg</location>
        <date>1978.</date>
        <signal>Der ausführliche Forschungsbericht von </signal>
-        <author>Richard  Rosellen</author>
+        <author>Richard Rosellen</author>
        <note> erscheint in Kürze unter dem Titel</note>
-        <title>.Private Verbrechenskontrolle — eine empirische Untersuchung zur Anzeigeerstattung',</title>
+        <title>'Private Verbrechenskontrolle — eine empirische Untersuchung zur Anzeigeerstattung',</title>
        <location>Berlin,</location>
        <date>voraussichtlich 1980.</date>
    </sequence>
@@ -56,7 +56,7 @@
    </sequence>
    <sequence>
        <citation-number>7</citation-number>
-        <signal>Zum Konzept der .Thematisierung von Recht' vgl.</signal>
+        <signal>Zum Konzept der 'Thematisierung von Recht' vgl.</signal>
        <author>Luhmann</author>
        <date>1980,</date>
        <pages>S. 99—112.</pages>
@@ -133,7 +133,7 @@
        <citation-number>18</citation-number>
        <title>Projektbericht ,Rechtshilfebedürfnisse sozial Schwacher*,</title>
        <author>(Blankenburg/Gorges/Reifner; Ticmann). </author>
-        <note>Befragt  wurden im Januar bis März 1979 je eine Person in 835 Haushalten Westberlins. Veröffentlichung ist vorgesehen für </note>
+        <note>Befragt wurden im Januar bis März 1979 je eine Person in 835 Haushalten Westberlins. Veröffentlichung ist vorgesehen für </note>
        <date>Ende 1980. </date>
        <note>Quelle: Eigene Befragung in West-Berlin 1979, Zufallsauswahl aus allen Haushalten mit Deutschen. Mehrfachnennungen, % jeweils bezogen auf Personen, die im jeweiligen Bereich ein Problem nennen.</note>
    </sequence>

--- a/convert-anystyle-data/anystyle/10.1515_zfrs-1980-0104.xml
+++ b/convert-anystyle-data/anystyle/10.1515_zfrs-1980-0104.xml
@@ -3,7 +3,7 @@
    <sequence>
        <ignore>Abgekürzt werden zitiert:</ignore>
        <editor>Armer/Grimshaw (Hrsg.),</editor>
-        <title>Comparative Social ResearchMethodological Problems and Strategies</title>
+        <title>Comparative Social Research - Methodological Problems and Strategies</title>
        <location>(New York, London, Sydney, Tokio</location>
        <date>1973);</date>
        <editor>Drobnig/ Rebbinder (Hrsg.),</editor>

--- a/convert-anystyle-data/tei/10.1111_1467-6478.00057.xml
+++ b/convert-anystyle-data/tei/10.1111_1467-6478.00057.xml
--- a/convert-anystyle-data/tei/10.1111_1467-6478.00080.xml
+++ b/convert-anystyle-data/tei/10.1111_1467-6478.00080.xml
@@ -16,18 +16,19 @@
  <text>
    <body>
      <p>The article text is not part of this document</p>
+      <listBibl/>
    </body>
    <note n="1" place="bottom">
      <bibl>
        <note type="signal">For a contrary view, see</note>
        <author>G. Jones</author>
-        ‘ 
+        ,
        <title level="a">“Traditional” Legal Scholarship: a Personal View</title>
-        in 
+        ’
        <title level="m">What Are Law Schools For?</title>
-        ed.
+        ,
        <editor> P. Birks</editor>
-        (
+        ed.
        <date>1996</date>
        )
        <biblScope unit="pp">14</biblScope>
@@ -37,9 +38,9 @@
    <note n="3" place="bottom">
      <bibl>
        <author>R. Goff</author>
-        ‘
+        ,
        <title level="a">The Search for Principle</title>
-        (
+        ’
        <date>1983</date>
        )
        <title level="j">Proceeedings of the British Academy</title>
@@ -51,7 +52,6 @@
        <author>A. Dicey</author>
        ,
        <title level="a">Can English Law be taught at the Universities?</title>
-        (
        <date>1883</date>
        )
        <biblScope unit="pp">20</biblScope>
@@ -63,7 +63,6 @@
        <author>J. Smith</author>
        ,
        <title level="a">The Law of Contract</title>
-        (
        <date>1989</date>
        )
      </bibl>
@@ -73,9 +72,9 @@
        <note type="signal">See, for example</note>
        ,
        <author>D. Kennedy</author>
-        ‘
+        ,
        <title level="a">Form and substance in Private Law Ajudication</title>
-        (
+        ’
        <date>1976</date>
        )
        <biblScope unit="vol">89</biblScope>
@@ -88,9 +87,9 @@
    <note n="7" place="bottom">
      <bibl>
        <author>B. Hepple</author>
-        ‘
+        ,
        <title level="a">The Renewal of the Liberal Law Degree</title>
-        (
+        ’
        <date>1996</date>
        )
        <title level="j">Cambridge Law J</title>
@@ -102,13 +101,13 @@
    <note n="8" place="bottom">
      <bibl>
        <author>P.A. Thomas</author>
-        ‘
+        ,
        <title level="a">Introduction</title>
-        in 
+        ’
        <title level="m">Socio-Legal Studies</title>
-        ed.
+        ,
        <editor> P.A. Thomas</editor>
-        (
+        ed.
        <date>1997</date>
        )
        <biblScope unit="pp">19</biblScope>
@@ -120,7 +119,6 @@
        <author>R. Cotterrell</author>
        ,
        <title level="a">Law’s Community</title>
-        (
        <date>1995</date>
        )
        <biblScope unit="pp">296</biblScope>
@@ -130,9 +128,8 @@
    <note n="10" place="bottom">
      <bibl>
        <note>Socio-legal studies has been defined in many different ways. In this essay the term is taken to indicate the use of ideas ‘from other disciplines primarily but not exclusively from within the social science and humanities fields’.</note>
-        ‘
        <title level="a">Company Law</title>
-         in 
+        ’
        <editor>Thomas</editor>
        ,
        <ref>op. cit., n. 8</ref>
@@ -146,15 +143,12 @@
        <note>Some fail wholly. It is difficult to see any effect on academic legal education that resulted from</note>
        <author>Lady Marre’s</author>
        <title level="a">report A Time for Change</title>
-        (
        <date>1988</date>
        ).
        <note>The Jarratt report on universities produced for the Committee of Vice-Chancellors and Principals (CVCP), Report of the Steering Committee for Efficiency studies in Universities</note>
-        (
        <date>1988</date>
        ),
        <note>produced much comment but little action. Even those that are thought of as being a success are not wholly implemented. Despite Ormrod’s recommendations no Institute of Profesional Legal Studies was set up and the universities and colleges of higher education did not take sole responsibility for vocational legal training</note>
-        (
        <title level="a">Report of the Committee on Legal Education</title>
        (
        <date>1971</date>
@@ -167,7 +161,6 @@
        ).
        <note>There were also other recommendations that were not implemented.</note>
        <title level="a">The Robbins report on higher education, Higher Education</title>
-        (
        <date>1963</date>
        ;
      </bibl>
@@ -175,7 +168,6 @@
        <ref type="legal">Cmnd. 2154</ref>
        )
        <note>took it is axiomatic that ‘courses of higher education should be available for all those who are qualified by ability and attainment to pursue them and wish to do so’</note>
-        (
        <biblScope unit="pp">para. 31</biblScope>
        ).
        <note>This has yet to happen.</note>
@@ -186,14 +178,12 @@
        <author>National Committee of Inquiry into Higher Education</author>
        ,
        <title level="a">Higher Education in the learning society</title>
-        (
        <date>1997</date>
        )
        <note>(the Dearing report);</note>
        <author>ACLEC</author>
        ,
        <title level="a">First Report on Legal Education and Training</title>
-        (
        <date>1996</date>
        ).
        <note>The Government’s White Paper on further and higher education had not been published at the time of writing this essay.</note>
@@ -211,10 +201,11 @@
      </bibl>
      <bibl>
        <note type="signal">See</note>
+        (
        <author>A. Bradney and F. Cownie</author>
-        ‘
+        ,
        <title level="a">Working on the Chain Gang?</title>
-        (
+        ’
        <date>1996</date>
        )
        <biblScope unit="vol">2</biblScope>
@@ -226,9 +217,9 @@
    <note n="15" place="bottom">
      <bibl>
        <author>J. MacFarlane, M. Jeeves, and A. Boon</author>
-        ‘
+        ,
        <title level="a">Education for Life or for Work?</title>
-        (
+        ’
        <date>1987</date>
        )
        <biblScope unit="vol">137</biblScope>
@@ -241,13 +232,12 @@
    <note n="16" place="bottom">
      <bibl>
        <author>T.H. Huxley</author>
-        ‘
+        ,
        <title level="a">Universities: Actual and Ideal</title>
-        in 
+        ’
        <title level="m">T.H. Huxley, Collected Essays</title>
        :
        <biblScope unit="vol">Volume III</biblScope>
-        (
        <date>1905</date>
        )
        <biblScope unit="pp">215</biblScope>
@@ -257,14 +247,15 @@
    <note n="17" place="bottom">
      <bibl>
        <author>J.S. Mill</author>
-        ‘
+        ,
        <title level="a">Inaugural address to the University of St Andrews</title>
-        in 
+        ’
        <title level="m">Collected Work of John Stuart Mill: Volume</title>
+        in 
        <biblScope unit="vol">XXI</biblScope>
-        ed.
+        ,
        <editor> J.M. Robson</editor>
-        (
+        ed.
        <date>1984</date>
        )
        <biblScope unit="pp">218</biblScope>
@@ -294,7 +285,6 @@
        <author>F.R. Leavis</author>
        ,
        <title level="a">Education and the University</title>
-        (
        <date>1948</date>
        )
        <biblScope unit="vol">28</biblScope>
@@ -307,9 +297,9 @@
        <note type="signal">See, further</note>
        ,
        <author>A. Bradney</author>
-        ‘
+        ,
        <title level="a">Liberalising Legal Education</title>
-        in 
+        ’
        <title level="m">The Law School: Global Issues, Local Questions</title>
        ,
        <note>ed. F. Cownie (forthcoming).</note>
@@ -322,10 +312,10 @@
        <title level="a">Of Blackstone’s Tower: Metephors of Distance and Histories of the English Law School</title>
        ’
        <ref>in Birks, op. cit., n. 1</ref>
-        p. 
+        ,
        <biblScope unit="pp">59. 23</biblScope>
+        p. 
        <title level="j">S. Turow, One L</title>
-        (
        <date>1977</date>
        )
        <biblScope unit="pp">106</biblScope>
@@ -335,9 +325,9 @@
    <note n="24" place="bottom">
      <bibl>
        <author>O. Kahn-Freund</author>
-        ‘
+        ,
        <title level="a">Reflections on Legal Education</title>
-        (
+        ’
        <date>1966</date>
        )
        <biblScope unit="vol">29</biblScope>
@@ -350,7 +340,6 @@
    <note n="25" place="bottom">
      <bibl>
        <note>Kahn-Freund believed ... legal argument</note>
-        (
        <author>Kahn-Freund</author>
        ,
        <ref>id</ref>
@@ -362,7 +351,7 @@
        <author>Leavis</author>
        ,
        <ref>op. cit., n. 20</ref>
-        p. 
+        ,
        <biblScope unit="pp">120</biblScope>
        .
      </bibl>
@@ -377,7 +366,6 @@
        <author>M. King</author>
        ,
        <title level="a">The New English Literatures</title>
-        (
        <date>1980</date>
        )
        <biblScope unit="pp">at 216–17</biblScope>
@@ -388,7 +376,7 @@
    <note n="30" place="bottom">
      <bibl>
        <title level="a">Jurisprudence by Sir John Salmond</title>
-        ed.
+        ,
        <editor> G. Willliams (10th ed</editor>
        .,
        <date>1947</date>
@@ -406,7 +394,6 @@
        ,
        <author>E. Durkheim</author>
        <title level="a">The Division of Labour in Society</title>
-        (
        <date>1933</date>
        )
        <biblScope unit="pp">68</biblScope>
@@ -418,16 +405,14 @@
        <author>M. Le Brun and R. Johnstone</author>
        ,
        <title level="a">The Quiet Revolution: Improving Student Learning in Law</title>
-        (
        <date>1994</date>
        )
        <note>65. On the effect on women students,</note>
      </bibl>
      <bibl>
        <note type="signal">see</note>
-        ‘
        <title level="a">Define and Empower: Women Students Consider Feminist Learning</title>
-        (
+        ’
        <date>1990</date>
        )
        <biblScope unit="vol">I</biblScope>
@@ -438,9 +423,9 @@
      <bibl>
        <note type="signal">For a survey of CLS and feminist literature on this general point, see</note>
        <author>W. Conklin</author>
-        ‘
+        ,
        <title level="a">The Invisible Author of Legal Authority</title>
-        (
+        ’
        <date>1996</date>
        )
        <biblScope unit="vol">VII</biblScope>
@@ -452,9 +437,9 @@
    <note n="33" place="bottom">
      <bibl>
        <author>R. Collier</author>
-        ‘
+        ,
        <title level="a">Masculinism, Law and Law Teaching</title>
-        (
+        ’
        <date>1991</date>
        )
        <biblScope unit="vol">19</biblScope>
@@ -466,9 +451,9 @@
    <note n="34" place="bottom">
      <bibl>
        <author>P. McAuslan</author>
-        ‘
+        ,
        <title level="a">Administrative Law, Collective Consumption and Judicial Policy</title>
-        (
+        ’
        <date>1983</date>
        )
        <biblScope unit="vol">46</biblScope>
@@ -483,7 +468,7 @@
        <author>Le Brun and Johnstone</author>
        ,
        <ref>op. cit, n. 32</ref>
-        pp. 
+        ,
        <biblScope unit="pp">71–5</biblScope>
        .
      </bibl>
@@ -499,9 +484,9 @@
    <note n="39" place="bottom">
      <bibl>
        <author>P. Samuelson</author>
-        ‘
+        ,
        <title level="a">The Convergence of the Law School and the University</title>
-        (
+        ’
        <date>1975</date>
        )
        <biblScope unit="vol">44</biblScope>
@@ -513,9 +498,8 @@
    <note n="40" place="bottom">
      <bibl>
        <author>P. Harris and M. Jones</author>
-        ‘
        <title level="a">A Survey of Law Schools in the United Kingdom, 1996</title>
-        (
+        ’
        <date>1997</date>
        )
        <biblScope unit="vol">31</biblScope>
@@ -527,9 +511,9 @@
    <note n="41" place="bottom">
      <bibl>
        <author>J. Wilson</author>
-        ‘
+         ,
        <title level="a">A third survey of university legal education</title>
-        (
+        ’
        <date>1993</date>
        )
        <biblScope unit="vol">13</biblScope>
@@ -544,6 +528,7 @@
      </bibl>
      <bibl>
        <note type="signal">Harris and</note>
+        (
        <author>Jones</author>
        ,
        <ref>op. cit., n. 40</ref>
@@ -557,7 +542,6 @@
        <author>P. Leighton, T. Mortimer, and N. Whatley</author>
        ,
        <title level="a">Law Teachers: Lawyers or Academics?</title>
-        (
        <date>1995</date>
        )
      </bibl>
@@ -573,9 +557,9 @@
    <note n="45" place="bottom">
      <bibl>
        <author>L. Skwarok</author>
-        ‘
+        ,
        <title level="a">Business Law for Non-Lawyers: Setting the Stage for Teaching, Learning and Assessment at Hong Kong Polytechnic University</title>
-        (
+        ’
        <date>1995</date>
        )
        <biblScope unit="vol">29</biblScope>
@@ -587,9 +571,9 @@
    <note n="46" place="bottom">
      <bibl>
        <author>N. Bastin</author>
-        ‘
+        ,
        <title level="a">Law, Law Staff and CNAA Business Studies Degree Courses</title>
-        (
+        ’
        <date>1985</date>
        )
        <biblScope unit="vol">19</biblScope>
@@ -601,9 +585,9 @@
    <note n="47" place="bottom">
      <bibl>
        <author>A. Ridley</author>
-        ‘
+        ,
        <title level="a">Legal Skills for Non-Law Students: Added Value or Irrelevant Diversion?</title>
-        (
+        ’
        <date>1994</date>
        )
        <biblScope unit="vol">28</biblScope>
@@ -615,9 +599,9 @@
    <note n="48" place="bottom">
      <bibl>
        <author>G. Cartan and T. Vilkinas</author>
-        ‘
+        ,
        <title level="a">Legal Literacy for Managers: The Role of the Educator</title>
-        (
+        ’
        <date>1990</date>
        )
        <biblScope unit="vol">24</biblScope>
@@ -644,13 +628,13 @@
    <note n="51" place="bottom">
      <bibl>
        <author>P. Birks</author>
-        ‘
+        ,
        <title level="a">Short Cuts</title>
-        in 
+        ’
        <title level="m">Pressing Problems in the Law</title>
-        ed.
+        ,
        <editor> P. Birks</editor>
-        (
+        ed.
        <date>1994</date>
        )
        <biblScope unit="pp">10–24</biblScope>
@@ -662,7 +646,7 @@
        <author>Ridley</author>
        ,
        <ref>op. cit., n. 47</ref>
-        p. 
+        ,
        <biblScope unit="pp">283</biblScope>
        .
      </bibl>
@@ -672,7 +656,7 @@
        <author>Cartan and Vilkinas</author>
        ,
        <ref>op. cit., n. 48</ref>
-        p. 
+        ,
        <biblScope unit="pp">248</biblScope>
        .
      </bibl>
@@ -680,9 +664,9 @@
    <note n="54" place="bottom">
      <bibl>
        <author>P. Harris</author>
-        ‘
+        ,
        <title level="a">Curriculum Development in Legal Studies</title>
-        (
+        ’
        <date>1986</date>
        )
        <biblScope unit="vol">20</biblScope>
@@ -706,7 +690,6 @@
        <author>G. Steiner</author>
        ,
        <title level="a">Errata: An Examined Life</title>
-        (
        <date>1997</date>
        )
        <biblScope unit="pp">20</biblScope>

--- a/convert-anystyle-data/tei/10.1515_zfrs-1980-0103.xml
+++ b/convert-anystyle-data/tei/10.1515_zfrs-1980-0103.xml
--- a/convert-anystyle-data/tei/10.1515_zfrs-1980-0104.xml
+++ b/convert-anystyle-data/tei/10.1515_zfrs-1980-0104.xml