Reintroduce HoC votes about api.iscanadafair.ca

[bradbeattie]

The following has been useful in pre-caching large ranges of URLs relatively quickly. I can't seem to integrate it into the Django admin command as deadlocks occur (ugh). But caching for later use, it works swell.

from bs4 import BeautifulSoup
from federal_common.utils import fetch_url
import concurrent.futures


soup = BeautifulSoup(
    fetch_url("http://www.ourcommons.ca/Parliamentarians/en/HouseVotes/ExportVotes?sessionId=145&output=XML"),
    "lxml",
)
urls = []
for vote in soup.select("voteparticipant"):
    params = (vote.parliamentnumber.text, vote.sessionnumber.text, vote.decisiondivisionnumber.text)
    urls.extend([
        "http://www.ourcommons.ca/Parliamentarians/en/votes/{}/{}/{}/".format(*params),
        "http://www.noscommunes.ca/Parliamentarians/fr/votes/{}/{}/{}/".format(*params),
        "http://www.ourcommons.ca/Parliamentarians/en/HouseVotes/ExportDetailsVotes?output=XML&parliament={}&session={}&vote={}".format(*params),
    ])


with concurrent.futures.ThreadPoolExecutor(max_workers=25) as executor:
    future_to_url = {
        executor.submit(fetch_url, url): url
        for url in urls
    }
    for future in concurrent.futures.as_completed(future_to_url):
        url = future_to_url[future]
        print(url)
        future.result()