Added transformations TEI->biblStruct, TEI->MODS and MODS->RIS via bibutils

cf342253 · Christian Boulanger · 21504eca · cf342253 · cf342253
Commit cf342253 authored 7 months ago by Christian Boulanger
--- a/convert-anystyle-data/readme.md
+++ b/convert-anystyle-data/readme.md
 # Conversion of AnyStyle training data to other formats
-This subrepo contains code to convert the existing training data in the AnyStyle formats (XML, TTX) into other formats
+This subrepo contains code to convert the existing training data in the AnyStyle formats (XML, TTX) into other formats.
-that can be used with other tools like prodigy or which are more standardized (such as LinkML)
-Note: The automatic generation of a LinkML schema from the converted JSONL files using the schema-automator tool 
+Note: the requirements introduce a huge dependency tree - use a virtual environment to avoid cluttering your python installation.
-introduces a huge dependency tree - use a virtual environment to avoid cluttering your python installation.
-## Content of directories:
- `in`: AnyStyle Ground Truth for document-level (ttx) and footnote-level (xml) reference information
- `jsonl`: AnyStyle footnote GT converted to a JSONL objects with "in" (Complete footnote as a string) and "out"
-  (Structured data) fields
- `json`: json files containing a flat list of objects with the structured data of the references in the footnotes
- `schema`: LinkML schema, autogenerated from the json files, not yet annotated.
-## Resources
- https://prodi.gy/docs/api-interfaces#spans_manual
- https://linkml.io/linkml/index.html
- https://linkml.io/schema-automator/
\ No newline at end of file
--- a/convert-anystyle-data/tei-to-bibformats.ipynb
+++ b/convert-anystyle-data/tei-to-bibformats.ipynb
@@ -86,7 +86,6 @@
   "source": [
    "from lxml import etree\n",
    "import glob\n",
-    "from urllib.request import urlopen\n",
    "import requests\n",
    "\n",
    "def apply_xslt(xslt_path, xml_input_path, xml_output_path):\n",
@@ -185,30 +184,14 @@
   "source": [
    "## Convert MODS to RIS tagged file format\n",
    "\n",
-    "This requires the install the bibutils suite of executables https://sourceforge.net/p/bibutils/home/Bibutils/ \n",
+    "This requires the install the bibutils suite of executables https://sourceforge.net/p/bibutils/home/Bibutils/ available in most distros.\n",
-    "(in windows, install it to the standard WSL distro)"
+    "(in windows, you will need to install it to the standard WSL distro)"
   ],
   "id": "5e75488ae4379946"
  },
  {
-   "metadata": {
+   "metadata": {},
-    "ExecuteTime": {
-     "end_time": "2024-08-22T08:19:43.322537Z",
-     "start_time": "2024-08-22T08:19:43.087262Z"
-    }
-   },
   "cell_type": "code",
-   "source": [
-    "import subprocess\n",
-    "import platform\n",
-    "\n",
-    "cmd = ['bash', 'lib/xml2ris.sh']\n",
-    "if platform.system() == 'Windows':\n",
-    "    cmd = ['wsl.exe', '-e'] + cmd\n",
-    "output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)\n",
-    "print(output.decode())"
-   ],
-   "id": "fde37a9e4a182bad",
   "outputs": [
    {
     "name": "stdout",
@@ -226,28 +209,18 @@
     ]
    }
   ],
-   "execution_count": 83
+   "execution_count": 83,
-  },
+   "source": [
-  {
+    "import subprocess\n",
-   "metadata": {
+    "import platform\n",
-    "ExecuteTime": {
+    "\n",
-     "end_time": "2024-08-22T08:18:13.159229Z",
+    "cmd = ['bash', 'lib/xml2ris.sh']\n",
-     "start_time": "2024-08-22T08:18:13.145443Z"
+    "if platform.system() == 'Windows':\n",
-    }
+    "    cmd = ['wsl.exe', '-e'] + cmd\n",
-   },
+    "output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)\n",
-   "cell_type": "code",
+    "print(output.decode())"
-   "source": "",
+   ],
-   "id": "8a013a47766a81cc",
+   "id": "fde37a9e4a182bad"
-   "outputs": [],
-   "execution_count": 80
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "outputs": [],
-   "execution_count": null,
-   "source": "",
-   "id": "bf5722a2500cf1a"
  }
 ],
 "metadata": {

 %% Cell type:markdown id:2cdf8ba1eefa38e0 tags:
 # Convert the generated TEI to bibliographic formats
 %% Cell type:markdown id:db65c4065691c578 tags:
 ## Download required XSLT documents
 we use XSLT provided by https://github.com/OpenArabicPE/convert_tei-to-bibliographic-data
 %% Cell type:code id:1de7cedbb3514188 tags:
 ``` python
 import os
 from urllib.parse import urljoin
 import requests
 from lxml import etree
 def download_xslt(url, target_dir = 'lib/xslt'):
    """written by GPT-4"""
    response = requests.get(url)
    response.raise_for_status()
    doc = etree.fromstring(response.content)
    for elem in doc.xpath('//*[local-name() = "import"]'):
        import_url = urljoin(url, elem.get('href'))  # Construct a full URL based on the href attribute relative to the original url
        download_xslt(import_url, target_dir)
    os.makedirs(target_dir, exist_ok=True)
    with open(os.path.join(target_dir, os.path.basename(url)), 'wb') as f:
        f.write(response.content)
    print(f'Downloaded {os.path.basename(url)}')
 base_url = 'https://openarabicpe.github.io/convert_tei-to-bibliographic-data/xslt'
 xslt_docs = ['convert_tei-to-mods_bibl.xsl']
 for xslt_doc in xslt_docs:
    download_xslt(f'{base_url}/{xslt_doc}')
 ```
 %% Output
    Downloaded date-functions.xsl
    Downloaded parameters.xsl
    Downloaded functions.xsl
    Downloaded convert_tei-to-biblstruct_functions.xsl
    Downloaded convert_tei-to-mods_functions.xsl
    Downloaded convert_tei-to-mods_bibl.xsl
 %% Cell type:markdown id:d08d51f8767602c5 tags:
 ## Extract bibliographic data from TEI files using XSLT
 ### Using lxml - currently not working
 %% Cell type:code id:af437a5ab3cc41a3 tags:
 ``` python
 from lxml import etree
 import glob
-from urllib.request import urlopen
 import requests
 def apply_xslt(xslt_path, xml_input_path, xml_output_path):
    try:
        xslt_doc = etree.parse(xslt_path)
        xml_doc = etree.parse(xml_input_path)
        transformer = etree.XSLT(xslt_doc)
        new_xml = transformer(xml_doc)
        with open(xml_output_path, 'w', encoding='utf-8') as f:
            f.write(new_xml)
    except etree.XSLTParseError as e:
        print(f"Error parsing XSLT file at {xslt_path}: {e}")
 for input_path in glob.glob('tei/*.xml'):
    print(f'Converting {input_path}')
    base_name = os.path.basename(input_path)
    output_path = f'tmp/{base_name.replace(".xml", "-mods.xml")}'
    apply_xslt('lib/xslt/convert_tei-to-mods_bibl.xsl', input_path, output_path)
 ```
 %% Output
    Converting tei\10.1111_1467-6478.00057.xml
    Error parsing XSLT file at lib/xslt/convert_tei-to-mods_bibl.xsl: xsltParseStylesheetTop: ignoring misplaced import element
    Converting tei\10.1111_1467-6478.00080.xml
    Error parsing XSLT file at lib/xslt/convert_tei-to-mods_bibl.xsl: xsltParseStylesheetTop: ignoring misplaced import element
    Converting tei\10.1515_zfrs-1980-0103.xml
    Error parsing XSLT file at lib/xslt/convert_tei-to-mods_bibl.xsl: xsltParseStylesheetTop: ignoring misplaced import element
    Converting tei\10.1515_zfrs-1980-0104.xml
    Error parsing XSLT file at lib/xslt/convert_tei-to-mods_bibl.xsl: xsltParseStylesheetTop: ignoring misplaced import element
 %% Cell type:markdown id:781d0e0e7a9dd346 tags:
 ### Using Saxon:
 - download ZIP from https://github.com/Saxonica/Saxon-HE/releases/download/SaxonHE12-5/SaxonHE12-5J.zip
 - unpack in lib/SaxonHE12-5J
 %% Cell type:code id:34087ef2f498ffa6 tags:
 ``` python
 import subprocess
 import os
 def transform_tei(xslt_path, file_path='tei', output_path='.'):
    file_path = os.path.normpath(file_path)
    xslt_path = os.path.normpath(xslt_path)
    cmd = ['java', '-jar', 'lib/SaxonHE12-5J/saxon-he-12.5.jar',
           f'-s:{file_path}',
           f'-xsl:{xslt_path}',
           f'-o:{output_path}',
           'p_target-language=de', 'p_github-action=true']
    process = subprocess.run(cmd, capture_output=True, text=True)
    if process.returncode != 0:
        raise RuntimeError(process.stderr)
    return process
 transform_tei(xslt_path='lib/xslt/convert_tei-to-biblstruct_bibl.xsl', output_path='biblStruct')
 transform_tei(xslt_path='lib/xslt/convert_tei-to-mods_bibl.xsl', output_path='mods')
 ```
 %% Output
    CompletedProcess(args=['java', '-jar', 'lib/SaxonHE12-5J/saxon-he-12.5.jar', '-s:tei', '-xsl:lib\\xslt\\convert_tei-to-mods_bibl.xsl', '-o:mods', 'p_target-language=de', 'p_github-action=true'], returncode=0, stdout='', stderr='')
 %% Cell type:markdown id:5e75488ae4379946 tags:
 ## Convert MODS to RIS tagged file format
-This requires the install the bibutils suite of executables https://sourceforge.net/p/bibutils/home/Bibutils/
+This requires the install the bibutils suite of executables https://sourceforge.net/p/bibutils/home/Bibutils/ available in most distros.
-(in windows, install it to the standard WSL distro)
+(in windows, you will need to install it to the standard WSL distro)
 %% Cell type:code id:fde37a9e4a182bad tags:
 ``` python
 import subprocess
 import platform
 cmd = ['bash', 'lib/xml2ris.sh']
 if platform.system() == 'Windows':
    cmd = ['wsl.exe', '-e'] + cmd
 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 print(output.decode())
 ```
 %% Output
    Converted mods/metadata/10.1111_1467-6478.00057-bibl.MODS.xml to ris/10.1111_1467-6478.00057.ris
    xml2ris: Processed 68 references.
    Converted mods/metadata/10.1111_1467-6478.00080-bibl.MODS.xml to ris/10.1111_1467-6478.00080.ris
    xml2ris: Processed 40 references.
    Converted mods/metadata/10.1515_zfrs-1980-0103-bibl.MODS.xml to ris/10.1515_zfrs-1980-0103.ris
    xml2ris: Processed 36 references.
    Converted mods/metadata/10.1515_zfrs-1980-0104-bibl.MODS.xml to ris/10.1515_zfrs-1980-0104.ris
    xml2ris: Processed 82 references.
-%% Cell type:code id:8a013a47766a81cc tags:
-``` python
-```
-%% Cell type:code id:bf5722a2500cf1a tags:
-``` python
-```