extraction with GPT-4 and upload of the manually corrected data works

wikidata/.env.dist

+OPENAI_API_KEY=''
+HUGGINGFACEHUB_API_TOKEN=''
\ No newline at end of file

wikidata/.gitignore

-timeline_data.xlsx
\ No newline at end of file
+timeline_data.xlsx
+user-config.py
+.env
+apicache
+/throttle.ctrl
+data/*-chatgpt.csv
\ No newline at end of file

wikidata/data/Erhard Blankenburg.csv

+subject-label,subject-qid,predicate,pid,object,object-qid,start_time,end_time,reference_url
+Erhard Blankenburg,Q51595283,educated at,P69,University of Freiburg,Q153987,,,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,educated at,P69,Free University Berlin,Q153006,,,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,educated at,P69,University of Oregon,Q766145,,,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,educated at,P69,University of Basel,Q372608,,1965,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,academic degree,P512,Master of Arts,Q2091008,1965,1965,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,academic degree,P512,Doctor of Philosophy,Q752297,1966,1966,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,employer,P108,University of Freiburg,Q153987,1966,1968,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,employer,P108,Quickborner Team,Q124866772,1969,1971,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,employer,P108,Prognos AG,Q2112115,,1973,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,employer,P108,Max Planck Institute for Foreign and International Criminal Law,Q832780,1973,1974,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,academic degree,P512,habilitation,Q308678,1974,1974,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,employed by,P463,WZB Berlin Social Science Center,Q475602,1975,1980,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,academic appointment,P8413,Free University of Amsterdam,Q1065414,1980,,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,affiliated with,P1416,International Institute for the Sociology of Law,Q1459361,,,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,editor,P98,Zeitschrift fur Rechtssoziologie,Q96335163,,,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Zeitschrift fur Rechtssoziologie,Q65972149,founded by,P112,Erhard Blankenburg,Q51595283,,,https://de.wikipedia.org/wiki/Erhard_Blankenburg
+Erhard Blankenburg,Q51595283,academic appointment,P8413,Free University of Amsterdam,Q1065414,,2003,https://www.linkedin.com/in/erhard-blankenburg-63938058/

wikidata/readme.md

 # WikiData data retrieval
 
-`pip install openpyxl plotly lxml tabulate`
\ No newline at end of file
+
+## Dependencies
+
+For data upload to WikiData
+`pip install pywikibot requests-oauthlib`
+
+For timeline plots
+`pip install openpyxl plotly lxml tabulate`
+
+For data extraction using OpenAi models
+`pip install langchain_core langchain_openai`
\ No newline at end of file

wikidata/update-wikidata.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "source": [
+    "# Update WikiData\n"
+   ],
+   "metadata": {
+    "collapsed": false
+   },
+   "id": "8ab9671b6edfa9f"
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Retrieved wikidata item.\n",
+      "Claim P108 P580 {\n",
+      "    \"after\": 0,\n",
+      "    \"before\": 0,\n",
+      "    \"calendarmodel\": \"http://www.wikidata.org/entity/Q1985727\",\n",
+      "    \"precision\": 9,\n",
+      "    \"time\": \"+00000001980-01-01T00:00:00Z\",\n",
+      "    \"timezone\": 0\n",
+      "} already exists.\n",
+      "Claim P108 P582 {\n",
+      "    \"after\": 0,\n",
+      "    \"before\": 0,\n",
+      "    \"calendarmodel\": \"http://www.wikidata.org/entity/Q1985727\",\n",
+      "    \"precision\": 9,\n",
+      "    \"time\": \"+00000002003-01-01T00:00:00Z\",\n",
+      "    \"timezone\": 0\n",
+      "} already exists.\n",
+      "Reference P4656 https://de.wikipedia.org/wiki/Erhard_Blankenburg already exists for P108.\n",
+      "Reference P854 https://www.linkedin.com/in/erhard-blankenburg-63938058/ already exists for P108.\n",
+      "Modifications applied in an idempotent manner.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pywikibot\n",
+    "from pywikibot import Claim, WbTime\n",
+    "from datetime import datetime\n",
+    "\n",
+    "site = pywikibot.Site(\"wikidata\", \"wikidata\")\n",
+    "repo = site.data_repository()\n",
+    "\n",
+    "item = pywikibot.ItemPage(repo, 'Q51595283')\n",
+    "item.get()\n",
+    "\n",
+    "print(\"Retrieved wikidata item.\")\n",
+    "\n",
+    "# Function to check if a qualifier exists\n",
+    "def qualifier_exists(claim, qualifier_property, target_value):\n",
+    "    for qualifier in claim.qualifiers.get(qualifier_property, []):\n",
+    "        if qualifier.getTarget() == target_value:\n",
+    "            print(f'Claim {claim.getID()} {qualifier_property} {target_value} already exists.')\n",
+    "            return True\n",
+    "    return False\n",
+    "\n",
+    "# Function to check if a reference exists\n",
+    "def reference_exists(claim, source_property, target_url):\n",
+    "    for source in claim.sources:\n",
+    "        for prop_id, values in source.items():\n",
+    "            if prop_id == source_property:\n",
+    "                for value in values:\n",
+    "                    if value.getTarget() == target_url:\n",
+    "                        print(f'Reference {source_property} {target_url} already exists for {claim.getID()}.')\n",
+    "                        return True\n",
+    "    return False\n",
+    "\n",
+    "# Ensure employment claim is not duplicated\n",
+    "employment_claim_exists = False\n",
+    "for claim in item.claims.get('P108', []):  # P108 is 'employer'\n",
+    "    if claim.getTarget().getID() == 'Q1065414':  # University of Amsterdam\n",
+    "        employment_claim_exists = True\n",
+    "        break\n",
+    "\n",
+    "if not employment_claim_exists:\n",
+    "    claim = Claim(repo, 'P108')\n",
+    "    target = pywikibot.ItemPage(repo, 'Q1065414')\n",
+    "    claim.setTarget(target)\n",
+    "    item.addClaim(claim)\n",
+    "    print(f'Created new claim {claim}...')\n",
+    "\n",
+    "# Add start and end time qualifiers if they don't already exist\n",
+    "start_time = WbTime(year=1980)\n",
+    "if not qualifier_exists(claim, 'P580', start_time):\n",
+    "    start_qualifier = Claim(repo, 'P580')\n",
+    "    start_qualifier.setTarget(start_time)\n",
+    "    claim.addQualifier(start_qualifier)\n",
+    "    print(f'Added new qualifier {start_qualifier}...')\n",
+    "\n",
+    "end_time = WbTime(year=2003)\n",
+    "if not qualifier_exists(claim, 'P582', end_time):\n",
+    "    end_qualifier = Claim(repo, 'P582')\n",
+    "    end_qualifier.setTarget(end_time)\n",
+    "    claim.addQualifier(end_qualifier)\n",
+    "    print(f'Added new qualifier {end_qualifier}...')\n",
+    "\n",
+    "# Add references with 'retrieved at' qualifier\n",
+    "current_datetime = datetime.utcnow()\n",
+    "retrieved_at_datetime = WbTime(year=current_datetime.year, month=current_datetime.month, day=current_datetime.day)\n",
+    "\n",
+    "wikipedia_url = 'https://de.wikipedia.org/wiki/Erhard_Blankenburg'\n",
+    "linkedin_url = 'https://www.linkedin.com/in/erhard-blankenburg-63938058/'\n",
+    "\n",
+    "if not reference_exists(claim, 'P4656', wikipedia_url):\n",
+    "    # Add Wikipedia reference\n",
+    "    wikipedia_reference = Claim(repo, 'P4656')\n",
+    "    wikipedia_reference.setTarget(wikipedia_url)\n",
+    "    retrieved_at_claim_wiki = Claim(repo, 'P813')\n",
+    "    retrieved_at_claim_wiki.setTarget(retrieved_at_datetime)\n",
+    "    wikipedia_reference.addQualifier(retrieved_at_claim_wiki)\n",
+    "    claim.addSources([wikipedia_reference])\n",
+    "    print(f'Added new source {wikipedia_reference}...')\n",
+    "\n",
+    "if not reference_exists(claim, 'P854', linkedin_url):\n",
+    "    # Add LinkedIn reference\n",
+    "    linkedin_reference = Claim(repo, 'P854')\n",
+    "    linkedin_reference.setTarget(linkedin_url)\n",
+    "    retrieved_at_claim_linkedin = Claim(repo, 'P813')\n",
+    "    retrieved_at_claim_linkedin.setTarget(retrieved_at_datetime)\n",
+    "    linkedin_reference.addQualifier(retrieved_at_claim_linkedin)\n",
+    "    claim.addSources([linkedin_reference])\n",
+    "    print(f'Added new source {linkedin_reference}...')\n",
+    "\n",
+    "print('Modifications applied in an idempotent manner.')\n",
+    "\n"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-03-15T08:44:19.767213300Z",
+     "start_time": "2024-03-15T08:44:19.382460Z"
+    }
+   },
+   "id": "59d15dc93174e6ad"
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [],
+   "metadata": {
+    "collapsed": false
+   },
+   "id": "d702eb98f46957ca"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

wikidata/user-config.py.dist

+mylang = 'wikidata'
+family = 'wikidata'
+usernames['wikidata']['wikidata'] = '<name of bot>'
+authenticate['*.wikidata.org'] = ('Consumer token', 'Consumer secret',
+                                  'Access token', 'Access secret')
+console_encoding = 'utf-8'