From 2cc2016d86ce8ed76e4f05daf2f6d816e80bec75 Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Thu, 27 Sep 2018 19:23:12 +0930 Subject: [PATCH] Properly implement persistence This will help immensely with finding that /one/ bug that seems to occur several hours into scraping --- scrape.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/scrape.py b/scrape.py index 2e4275e..0558afc 100755 --- a/scrape.py +++ b/scrape.py @@ -279,21 +279,26 @@ def Get_Follows(url, limit = 100): user_dict = {} track_dict = {} +users_to_process = STARTING_USERS +users_to_process_next = [] +iterator = 0 with shelve.open('shelf.db') as shelf: if 'user_dict' in shelf.keys(): user_dict = shelf['user_dict'] if 'track_dict' in shelf.keys(): track_dict = shelf['track_dict'] + if 'users_to_process' in shelf.keys(): + users_to_process = shelf['users_to_process'] + if 'users_to_process_next' in shelf.keys(): + users_to_process_next = shelf['users_to_process_next'] + if 'iterator' in shelf.keys(): + iterator = shelf['iterator'] -users_to_process = [] -users_to_process_next = STARTING_USERS -for i in range(MAX_DEGREE + 1): - users_to_process = users_to_process_next - users_to_process_next = [] +while iterator <= MAX_DEGREE: for user_str in users_to_process: - print('Processing ' + user_str + ' (distance ' + str(i) + ')') + print('Processing ' + user_str + ' (distance ' + str(iterator) + ')') # Skip any users that have already been passed over to avoid infinite loops if user_str in user_dict.keys(): @@ -343,6 +348,13 @@ for i in range(MAX_DEGREE + 1): with shelve.open('shelf.db') as shelf: shelf['user_dict'] = user_dict shelf['track_dict'] = track_dict + shelf['users_to_process'] = users_to_process + shelf['users_to_process_next'] = users_to_process_next + shelf['iterator'] = iterator + + iterator += 1 + users_to_process = users_to_process_next + users_to_process_next = [] driver.quit()