python get_death_dates.py

Posted 2021-05-09

#!/usr/bin/python

import csv
import datetime
import re
from urllib.parse import unquote
from utils import load_data, save_data
from SPARQLWrapper import SPARQLWrapper, JSON

def run():
    sparql_endpoint = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql'
    s = SPARQLWrapper(sparql_endpoint)

    output = csv.writer(open("death_dates.csv", "w"))
    output.writerow(['bioguide', 'name', 'death'])

    # sparql query
    query = """
      PREFIX wd: <http://www.wikidata.org/entity/>
      PREFIX wdt: <http://www.wikidata.org/prop/direct/>
      PREFIX schema: <http://schema.org/>

      SELECT ?bio ?birth ?death ?subject
      WHERE {
        ?subject wdt:P1157 ?bio .
        ?subject wdt:P569 ?birth .
        ?subject wdt:P570 ?death .
        OPTIONAL {
            ?article schema:about ?subject .
            ?article schema:inLanguage "en" .
            ?article schema:isPartOf <https://en.wikipedia.org/> .
        }
      }

    """

    # run the query and stick everything in a hash keyed by bioguide id
    s.setQuery(query)
    s.setReturnFormat(JSON)
    results = s.query().convert()
    ret = {}

    y = load_data("legislators-historical.yaml")

    deaths_by_bioguide = {}
    for row in results['results']['bindings']:
        deaths_by_bioguide[row['bio']['value']] = row['death']['value']

    for m in y:
        try:
            ddate = deaths_by_bioguide[m['id']['bioguide']]
        except KeyError:
            continue

        try:
            ddate = datetime.datetime.strptime(ddate, "%Y-%m-%dT%H:%M:%SZ").date()
        except ValueError:
            continue

        last_term_end = datetime.datetime.strptime(m['terms'][-1]['end'], '%Y-%m-%d').date()

        output.writerow([m['id']['bioguide'], m['id']['wikipedia'], str(ddate)])


if __name__ == '__main__':
  run()