download-wikipedia-pages.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Download wikipedia pages as source of triple extraction\n",
    "\n",
    "This improves on [data-extraction notebook](./data-extraction.ipynb) by downloading the wikipedia article from which information is to be extracted "
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "9d6a10996bfdd3cf"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 1. Download raw Wikipedia page content for the list of scholars and save it"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "2ad235b62e2efc09"
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-03-18T15:26:43.976993400Z",
     "start_time": "2024-03-18T15:26:43.935308500Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No Wikipedia page exists for Wolfgang Kaupen.\n"
     ]
    }
   ],
   "source": [
    "import os.path\n",
    "\n",
    "from lib.wikidata import get_wikipedia_page_data\n",
    "from urllib.parse import unquote\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('scholars.csv')\n",
    "for index, row in df.iterrows():\n",
    "    fullName = row['fullName']\n",
    "    language_code = None\n",
    "    if pd.notna(row['wikipedia_de']):\n",
    "        pagetTitle = unquote(os.path.basename(row['wikipedia_de']))\n",
    "        language_code = 'de'\n",
    "    elif pd.notna(row['wikipedia_en']):\n",
    "        pagetTitle = unquote(os.path.basename(row['wikipedia_en']))\n",
    "        language_code = 'en'\n",
    "    else:\n",
    "        print(f'No Wikipedia page exists for {fullName}.')\n",
    "        continue\n",
    "\n",
    "    wikipedia_content_cache_path = f'data/{fullName}-wikipedia.txt'\n",
    "    if not os.path.isfile(wikipedia_content_cache_path):\n",
    "        page_data = get_wikipedia_page_data(pagetTitle, language_code)\n",
    "        if page_data and page_data['page'].exists: \n",
    "            file_content = f\"{page_data['url']}\\n\\n{page_data['content']}\"\n",
    "            with open(wikipedia_content_cache_path, 'w', encoding='utf-8') as file:\n",
    "                file.write(file_content)\n",
    "        else:\n",
    "            print(f'No page content could be retrieved for \"{fullName}\"')"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "e83e59e1974a6506"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 2. Reduce text size\n",
    "\n",
    "In order to remove unnecessary information and reduce the token count, edit the downloaded files to contain only the biographical parts from which to extract the information"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "997c82c5d3d72b7"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 3. Extract the information "
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "303ddc348c4a2887"
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\boulanger\\AppData\\Local\\miniconda3\\Lib\\site-packages\\langchain_openai\\chat_models\\base.py:454: PydanticDeprecatedSince20: The `dict` method is deprecated; use `model_dump` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.5/migration/\n",
      "  response = response.dict()\n",
      "C:\\Users\\boulanger\\AppData\\Local\\miniconda3\\Lib\\site-packages\\pydantic\\main.py:979: PydanticDeprecatedSince20: The `dict` method is deprecated; use `model_dump` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.5/migration/\n",
      "  warnings.warn('The `dict` method is deprecated; use `model_dump` instead.', DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from lib.langchain import extract_to_csv\n",
    "from langchain_openai import ChatOpenAI\n",
    "from pathlib import Path\n",
    "fullName = \"Thilo Ramm\"\n",
    "qid=\"Q59533838\"\n",
    "model = ChatOpenAI(model_name=\"gpt-4\")\n",
    "template = Path('extraction-prompt.txt').read_text()\n",
    "website_text = Path(f'data/{fullName}-wikipedia.txt').read_text()\n",
    "csv_path = f'data/{fullName}.csv'\n",
    "df = extract_to_csv(model, template, csv_path, fullName=fullName, qid=qid, website_text=website_text)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-18T17:03:40.614971500Z",
     "start_time": "2024-03-18T17:03:11.400373800Z"
    }
   },
   "id": "d904e502f8eff15d"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from lib.wikidata import update_wikidata_from_csv"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "7e04f078c326387a"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}