diff --git a/.gitignore b/.gitignore index 16c9567..2c973ed 100644 --- a/.gitignore +++ b/.gitignore @@ -112,3 +112,6 @@ dmypy.json # Generated graph files *.gexf + +# Shelve +shelf.db diff --git a/scrape.py b/scrape.py index 9ecb372..2fa06c2 100755 --- a/scrape.py +++ b/scrape.py @@ -22,6 +22,7 @@ STARTING_USERS = ['thomotron'] ##### Import all the things ################################# import time +import shelve import networkx as nx from selenium import webdriver from selenium.webdriver.common.by import By @@ -266,6 +267,12 @@ def Get_Follows(url, limit = 100): user_dict = {} track_dict = {} +with shelve.open('shelf.db', 'r') as shelf: + if 'user_dict' in shelf.keys(): + user_dict = shelf['user_dict'] + if 'track_dict' in shelf.keys(): + track_dict = shelf['track_dict'] + users_to_process = [] users_to_process_next = STARTING_USERS @@ -319,6 +326,11 @@ for i in range(MAX_DEGREE + 1): user.processed = True user_dict[user.url_username] = user + # Update the shelf with the new dictionaries + with shelve.open('shelf.db', 'c') as shelf: + shelf['user_dict'] = user_dict + shelf['track_dict'] = track_dict + driver.quit() ##### Process what was scraped ##############################