full-extraction-workflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Download wikipedia pages as source of triple extraction\n",
    "\n",
    "This improves on [data-extraction notebook](./data-extraction.ipynb) by downloading the wikipedia article from which information is to be extracted "
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "9d6a10996bfdd3cf"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 1. Download raw Wikipedia page content for the list of scholars and save it"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "2ad235b62e2efc09"
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-03-19T16:26:12.465730900Z",
     "start_time": "2024-03-19T16:26:07.245012500Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No Wikipedia page exists for Wolfgang Kaupen.\n"
     ]
    }
   ],
   "source": [
    "import os.path\n",
    "\n",
    "from lib.wikidata import get_wikipedia_page_data\n",
    "from urllib.parse import unquote\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('scholars.csv')\n",
    "for index, row in df.iterrows():\n",
    "    fullName = row['fullName']\n",
    "    language_code = None\n",
    "    if pd.notna(row['wikipedia_de']):\n",
    "        pagetTitle = unquote(os.path.basename(row['wikipedia_de']))\n",
    "        language_code = 'de'\n",
    "    elif pd.notna(row['wikipedia_en']):\n",
    "        pagetTitle = unquote(os.path.basename(row['wikipedia_en']))\n",
    "        language_code = 'en'\n",
    "    else:\n",
    "        print(f'No Wikipedia page exists for {fullName}.')\n",
    "        continue\n",
    "\n",
    "    wikipedia_content_cache_path = f'input/{fullName}-wikipedia.txt'\n",
    "    if not os.path.isfile(wikipedia_content_cache_path):\n",
    "        page_data = get_wikipedia_page_data(pagetTitle, language_code)\n",
    "        if page_data and page_data['page'].exists: \n",
    "            file_content = f\"{page_data['url']}\\n\\n{page_data['content']}\"\n",
    "            with open(wikipedia_content_cache_path, 'w', encoding='utf-8') as file:\n",
    "                file.write(file_content)\n",
    "        else:\n",
    "            print(f'No page content could be retrieved for \"{fullName}\"')"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "e83e59e1974a6506"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 2. Reduce text size\n",
    "\n",
    "In order to remove unnecessary information and reduce the token count, edit the downloaded files to contain only the biographical parts from which to extract the information"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "997c82c5d3d72b7"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 3. Extract the information "
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "303ddc348c4a2887"
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompting with handmade-prompt-template.txt took 0 minutes and 28 seconds and extracted 13 triples.\n",
      "Prompting with gp4-optimized-prompt-template.txt took 0 minutes and 19 seconds and extracted 6 triples.\n"
     ]
    }
   ],
   "source": [
    "from lib.langchain import extract_to_csv\n",
    "from langchain_openai import ChatOpenAI\n",
    "from pathlib import Path\n",
    "import time\n",
    "\n",
    "fullName = \"Thilo Ramm\"\n",
    "qid=\"Q59533838\"\n",
    "model = ChatOpenAI(model_name=\"gpt-4\")\n",
    "website_text = Path(f'input/{fullName}-wikipedia.txt').read_text()\n",
    "\n",
    "for template_file in ['handmade-prompt-template.txt', 'gp4-optimized-prompt-template.txt']:\n",
    "    start_time = time.time()\n",
    "    template = Path(f'prompts/{template_file}').read_text()   \n",
    "    df = extract_to_csv(model, template, debug=False, fullName=fullName, qid=qid, website_text=website_text)\n",
    "    end_time = time.time()\n",
    "    execution_time = end_time - start_time  # In seconds\n",
    "    minutes, seconds = divmod(execution_time, 60)\n",
    "    print(f\"Prompting with {template_file} took {int(minutes)} minutes and {int(seconds)} seconds and extracted {len(df)} triples.\")\n",
    "    csv_path = f'output/{fullName}-{template_file.split(\"-\")[0]}-prompt.csv'\n",
    "    df.to_csv(csv_path, index=False)\n",
    "    \n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-19T17:19:30.764372400Z",
     "start_time": "2024-03-19T17:18:42.398193300Z"
    }
   },
   "id": "d904e502f8eff15d"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-03-19T16:26:16.823261600Z"
    }
   },
   "id": "2c11affcd6ffbff4"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}