Adding finder file conversion

48cb0887 · cboulanger · 10ee6ba0 · 48cb0887 · 48cb0887 · 48cb0887
Commit 48cb0887 authored 11 months ago by cboulanger
--- a/prodigy/anystyle-to-prodigy.ipynb
+++ b/prodigy/anystyle-to-prodigy.ipynb
 {
 "cells": [
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": [
+    "# Convert AnyStyle training files to the Prodigy format\n",
+    "\n",
+    "Converter functions written by GPT-4, see https://chat.openai.com/share/3f42ae1d-3066-4563-944d-3139b5e1e867"
+   ],
+   "id": "cb631197c03a0768"
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "## Parser files",
+   "id": "83bfc356ce72fe51"
+  },
  {
   "cell_type": "code",
   "id": "initial_id",
@@ -45,19 +61,153 @@
  {
   "metadata": {
    "ExecuteTime": {
-     "end_time": "2024-04-12T05:36:56.511933Z",
+     "end_time": "2024-04-12T06:23:46.935341Z",
-     "start_time": "2024-04-12T05:36:56.497198Z"
+     "start_time": "2024-04-12T06:23:46.920522Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from pathlib import Path\n",
    "for filename in [file.stem for file in Path('in').glob('*.xml')]:\n",
-    "    xml_to_jsonl(f'in/{filename}.xml', f'out/{filename}.jsonl')"
+    "    xml_to_jsonl(f'in/{filename}.xml', f'out/{filename}-parser.jsonl')"
   ],
   "id": "fa9f2c194697a6b7",
   "outputs": [],
-   "execution_count": 7
+   "execution_count": 8
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "## Finder files",
+   "id": "2988615a1be96bb"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-12T06:54:04.924138Z",
+     "start_time": "2024-04-12T06:54:04.913099Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import json\n",
+    "\n",
+    "def markup_to_jsonl(input_path, output_jsonl_path, break_tags_before=None, break_tags_after=None):\n",
+    "    if break_tags_before is None:\n",
+    "        break_tags_before = []\n",
+    "    if break_tags_after is None:\n",
+    "        break_tags_after = []\n",
+    "\n",
+    "    # Helper function to write blocks to JSONL, now takes a file handle\n",
+    "    def write_block(full_text, spans, file_handle):\n",
+    "        if full_text.strip():  # Ensure there's content to write\n",
+    "            result = {\n",
+    "                'text': full_text.rstrip(),  # Remove trailing spaces\n",
+    "                'spans': spans\n",
+    "            }\n",
+    "            file_handle.write(json.dumps(result) + '\\n')\n",
+    "\n",
+    "    # Open the output file once at the beginning\n",
+    "    with open(output_jsonl_path, 'w', encoding='utf-8') as f:\n",
+    "        with open(input_path, 'r', encoding='utf-8') as file:\n",
+    "            lines = file.readlines()\n",
+    "\n",
+    "        full_text = ''\n",
+    "        spans = []\n",
+    "        current_text = ''\n",
+    "        text_start_pos = 0\n",
+    "        current_tag = 'text'  # Default initial tag\n",
+    "\n",
+    "        # Iterate through lines\n",
+    "        for line in lines:\n",
+    "            parts = line.strip().split('|', 1)\n",
+    "            if len(parts) < 2:\n",
+    "                continue\n",
+    "\n",
+    "            tag = parts[0].strip()\n",
+    "            content = parts[1].strip()\n",
+    "\n",
+    "            if tag in break_tags_before:\n",
+    "                # Finish the current text block before starting new due to break tag\n",
+    "                if current_text:\n",
+    "                    full_text += current_text\n",
+    "                    spans.append({\n",
+    "                        'start': text_start_pos,\n",
+    "                        'end': text_start_pos + len(current_text),\n",
+    "                        'label': current_tag\n",
+    "                    })\n",
+    "                    write_block(full_text, spans, f)\n",
+    "                    full_text, spans, current_text = '', [], ''\n",
+    "                current_tag = tag  # Update to the new tag\n",
+    "                text_start_pos = len(full_text)\n",
+    "\n",
+    "            # Append content with maintaining line break if the previous content was not empty\n",
+    "            current_text += content + '\\n' if current_text else content\n",
+    "\n",
+    "            if tag:\n",
+    "                if current_text and current_text != content:  # Ensures not to add just processed content span\n",
+    "                    # Finish the current span\n",
+    "                    full_text += current_text\n",
+    "                    spans.append({\n",
+    "                        'start': text_start_pos,\n",
+    "                        'end': text_start_pos + len(current_text) - 1,  # -1 to not count the last \\n for span\n",
+    "                        'label': current_tag\n",
+    "                    })\n",
+    "                current_text = content + '\\n'  # Start new text content under the new tag\n",
+    "                current_tag = tag\n",
+    "                text_start_pos = len(full_text)\n",
+    "\n",
+    "            if tag in break_tags_after:\n",
+    "                # Finalize current text and write it due to break tag\n",
+    "                full_text += current_text\n",
+    "                spans.append({\n",
+    "                    'start': text_start_pos,\n",
+    "                    'end': text_start_pos + len(current_text) - 1,  # -1 as above\n",
+    "                    'label': current_tag\n",
+    "                })\n",
+    "                current_text = ''\n",
+    "                write_block(full_text, spans, f)\n",
+    "                full_text, spans = '', []\n",
+    "\n",
+    "        # Final write for any remaining text\n",
+    "        if current_text:\n",
+    "            full_text += current_text\n",
+    "            spans.append({\n",
+    "                'start': text_start_pos,\n",
+    "                'end': text_start_pos + len(current_text) - 1,  # -1 to adjust for the last newline character\n",
+    "                'label': current_tag\n",
+    "            })\n",
+    "        write_block(full_text, spans, f)\n",
+    "\n",
+    "# Example usage:\n",
+    "# markup_to_jsonl('input_text.txt', 'output.jsonl', ['header'], ['footer'])\n"
+   ],
+   "id": "7764c66e149abd4f",
+   "outputs": [],
+   "execution_count": 26
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "Since the finder files have no information on page breaks, document segmentation is done using the `meta` tag, which is a very simple heuristic and probably fails in many cases. ",
+   "id": "8923ed5011c1d65f"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-04-12T06:54:05.879064Z",
+     "start_time": "2024-04-12T06:54:05.870910Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "from pathlib import Path\n",
+    "for filename in [file.stem for file in Path('in').glob('*.xml')]:\n",
+    "    markup_to_jsonl(f'in/{filename}.ttx', f'out/{filename}-finder.jsonl', break_tags_before=['meta'])"
+   ],
+   "id": "2b0ca72bbf70c829",
+   "outputs": [],
+   "execution_count": 27
  },
  {
   "metadata": {},
@@ -65,7 +215,7 @@
   "outputs": [],
   "execution_count": null,
   "source": "",
-   "id": "7764c66e149abd4f"
+   "id": "46d7e63e53b441e3"
  }
 ],
 "metadata": {

+%% Cell type:markdown id:cb631197c03a0768 tags:
+# Convert AnyStyle training files to the Prodigy format
+Converter functions written by GPT-4, see https://chat.openai.com/share/3f42ae1d-3066-4563-944d-3139b5e1e867
+%% Cell type:markdown id:83bfc356ce72fe51 tags:
+## Parser files
 %% Cell type:code id:initial_id tags:
 ``` python
 import xml.etree.ElementTree as ET
 import json
 def xml_to_jsonl(input_xml_path, output_jsonl_path):
    tree = ET.parse(input_xml_path)
    root = tree.getroot()
    with open(output_jsonl_path, 'w', encoding='utf-8') as f:
        for sequence in root.findall('sequence'):
            text = ''
            spans = []
            current_pos = 0
            for element in sequence:
                content = (element.text or '').strip() + ' '  # Append a space for separation
                if content:
                    start = current_pos
                    end = start + len(content) - 1  # Subtract 1 to account for the space added
                    spans.append({'start': start, 'end': end, 'label': element.tag})
                    text += content
                    current_pos = end + 1  # Move current position to right after the last character
            # Store the reconstructed sequence and its spans in a dictionary
            entry = {'text': text.strip(), 'spans': spans}
            # Write the JSON formatted string to the file, ending with a newline
            f.write(json.dumps(entry) + '\n')
 ```
 %% Cell type:code id:fa9f2c194697a6b7 tags:
 ``` python
 from pathlib import Path
 for filename in [file.stem for file in Path('in').glob('*.xml')]:
-    xml_to_jsonl(f'in/{filename}.xml', f'out/{filename}.jsonl')
+    xml_to_jsonl(f'in/{filename}.xml', f'out/{filename}-parser.jsonl')
 ```
+%% Cell type:markdown id:2988615a1be96bb tags:
+## Finder files
 %% Cell type:code id:7764c66e149abd4f tags:
 ``` python
+import json
+def markup_to_jsonl(input_path, output_jsonl_path, break_tags_before=None, break_tags_after=None):
+    if break_tags_before is None:
+        break_tags_before = []
+    if break_tags_after is None:
+        break_tags_after = []
+    # Helper function to write blocks to JSONL, now takes a file handle
+    def write_block(full_text, spans, file_handle):
+        if full_text.strip():  # Ensure there's content to write
+            result = {
+                'text': full_text.rstrip(),  # Remove trailing spaces
+                'spans': spans
+            }
+            file_handle.write(json.dumps(result) + '\n')
+    # Open the output file once at the beginning
+    with open(output_jsonl_path, 'w', encoding='utf-8') as f:
+        with open(input_path, 'r', encoding='utf-8') as file:
+            lines = file.readlines()
+        full_text = ''
+        spans = []
+        current_text = ''
+        text_start_pos = 0
+        current_tag = 'text'  # Default initial tag
+        # Iterate through lines
+        for line in lines:
+            parts = line.strip().split('|', 1)
+            if len(parts) < 2:
+                continue
+            tag = parts[0].strip()
+            content = parts[1].strip()
+            if tag in break_tags_before:
+                # Finish the current text block before starting new due to break tag
+                if current_text:
+                    full_text += current_text
+                    spans.append({
+                        'start': text_start_pos,
+                        'end': text_start_pos + len(current_text),
+                        'label': current_tag
+                    })
+                    write_block(full_text, spans, f)
+                    full_text, spans, current_text = '', [], ''
+                current_tag = tag  # Update to the new tag
+                text_start_pos = len(full_text)
+            # Append content with maintaining line break if the previous content was not empty
+            current_text += content + '\n' if current_text else content
+            if tag:
+                if current_text and current_text != content:  # Ensures not to add just processed content span
+                    # Finish the current span
+                    full_text += current_text
+                    spans.append({
+                        'start': text_start_pos,
+                        'end': text_start_pos + len(current_text) - 1,  # -1 to not count the last \n for span
+                        'label': current_tag
+                    })
+                current_text = content + '\n'  # Start new text content under the new tag
+                current_tag = tag
+                text_start_pos = len(full_text)
+            if tag in break_tags_after:
+                # Finalize current text and write it due to break tag
+                full_text += current_text
+                spans.append({
+                    'start': text_start_pos,
+                    'end': text_start_pos + len(current_text) - 1,  # -1 as above
+                    'label': current_tag
+                })
+                current_text = ''
+                write_block(full_text, spans, f)
+                full_text, spans = '', []
+        # Final write for any remaining text
+        if current_text:
+            full_text += current_text
+            spans.append({
+                'start': text_start_pos,
+                'end': text_start_pos + len(current_text) - 1,  # -1 to adjust for the last newline character
+                'label': current_tag
+            })
+        write_block(full_text, spans, f)
+# Example usage:
+# markup_to_jsonl('input_text.txt', 'output.jsonl', ['header'], ['footer'])
+```
+%% Cell type:markdown id:8923ed5011c1d65f tags:
+Since the finder files have no information on page breaks, document segmentation is done using the `meta` tag, which is a very simple heuristic and probably fails in many cases. 
+%% Cell type:code id:2b0ca72bbf70c829 tags:
+``` python
+from pathlib import Path
+for filename in [file.stem for file in Path('in').glob('*.xml')]:
+    markup_to_jsonl(f'in/{filename}.ttx', f'out/{filename}-finder.jsonl', break_tags_before=['meta'])
+```
+%% Cell type:code id:46d7e63e53b441e3 tags:
+``` python
 ```

--- a/prodigy/in/10.1515_zfrs-1980-0103.ttx
+++ b/prodigy/in/10.1515_zfrs-1980-0103.ttx
--- a/prodigy/in/10.1515_zfrs-1980-0104.ttx
+++ b/prodigy/in/10.1515_zfrs-1980-0104.ttx
--- a/prodigy/in/10.1515_zfrs-1980-0104.xml
+++ b/prodigy/in/10.1515_zfrs-1980-0104.xml
--- a/prodigy/out/10.1515_zfrs-1980-0103-finder.jsonl
+++ b/prodigy/out/10.1515_zfrs-1980-0103-finder.jsonl
--- a/prodigy/out/10.1515_zfrs-1980-0103.jsonl
+++ b/prodigy/out/10.1515_zfrs-1980-0103.jsonl
--- a/prodigy/out/10.1515_zfrs-1980-0104-finder.jsonl
+++ b/prodigy/out/10.1515_zfrs-1980-0104-finder.jsonl
--- a/prodigy/out/10.1515_zfrs-1980-0104-parser.jsonl
+++ b/prodigy/out/10.1515_zfrs-1980-0104-parser.jsonl