From 161d24e238bdf15c9427a0a8ae87082508f0dd64 Mon Sep 17 00:00:00 2001
From: Thomas Wade <gigatech55@gmail.com>
Date: Wed, 26 Sep 2018 14:37:50 +0930
Subject: [PATCH] Use Shelve to store user and track dicts while scraping

Just in case something breaks mid-scrape
---
 .gitignore |  3 +++
 scrape.py  | 12 ++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/.gitignore b/.gitignore
index 16c9567..2c973ed 100644
--- a/.gitignore
+++ b/.gitignore
@@ -112,3 +112,6 @@ dmypy.json
 
 # Generated graph files
 *.gexf
+
+# Shelve
+shelf.db
diff --git a/scrape.py b/scrape.py
index 9ecb372..2fa06c2 100755
--- a/scrape.py
+++ b/scrape.py
@@ -22,6 +22,7 @@ STARTING_USERS = ['thomotron']
 ##### Import all the things #################################
 
 import time
+import shelve
 import networkx as nx
 from selenium import webdriver
 from selenium.webdriver.common.by import By
@@ -266,6 +267,12 @@ def Get_Follows(url, limit = 100):
 user_dict = {}
 track_dict = {}
 
+with shelve.open('shelf.db', 'r') as shelf:
+    if 'user_dict' in shelf.keys():
+        user_dict = shelf['user_dict']
+    if 'track_dict' in shelf.keys():
+        track_dict = shelf['track_dict']
+
 users_to_process = []
 users_to_process_next = STARTING_USERS
 
@@ -319,6 +326,11 @@ for i in range(MAX_DEGREE + 1):
         user.processed = True
         user_dict[user.url_username] = user
 
+        # Update the shelf with the new dictionaries
+        with shelve.open('shelf.db', 'c') as shelf:
+            shelf['user_dict'] = user_dict
+            shelf['track_dict'] = track_dict
+
 driver.quit()
 
 ##### Process what was scraped ##############################