Use Shelve to store user and track dicts while scraping

Just in case something breaks mid-scrape

.gitignore

@@ -112,3 +112,6 @@ dmypy.json
 
 # Generated graph files
 *.gexf
+
+# Shelve
+shelf.db

scrape.py

@@ -22,6 +22,7 @@ STARTING_USERS = ['thomotron']
 ##### Import all the things #################################
 
 import time
+import shelve
 import networkx as nx
 from selenium import webdriver
 from selenium.webdriver.common.by import By
@@ -266,6 +267,12 @@ def Get_Follows(url, limit = 100):
 user_dict = {}
 track_dict = {}
 
+with shelve.open('shelf.db', 'r') as shelf:
+    if 'user_dict' in shelf.keys():
+        user_dict = shelf['user_dict']
+    if 'track_dict' in shelf.keys():
+        track_dict = shelf['track_dict']
+
 users_to_process = []
 users_to_process_next = STARTING_USERS
 
@@ -319,6 +326,11 @@ for i in range(MAX_DEGREE + 1):
         user.processed = True
         user_dict[user.url_username] = user
 
+        # Update the shelf with the new dictionaries
+        with shelve.open('shelf.db', 'c') as shelf:
+            shelf['user_dict'] = user_dict
+            shelf['track_dict'] = track_dict
+
 driver.quit()
 
 ##### Process what was scraped ##############################