tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Properly implement persistence

Browse Source 
This will help immensely with finding that /one/ bug that seems to occur several hours into scraping
This commit is contained in:
Thomas Wade 2018-09-27 19:23:12 +09:30
parent 16a9c5c638
commit 2cc2016d86
1 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
scrape.py
View File

@ -279,21 +279,26 @@ def Get_Follows(url, limit = 100):
user_dict = {} user_dict = {}
track_dict = {} track_dict = {}
users_to_process = STARTING_USERS
users_to_process_next = []
iterator = 0
with shelve.open('shelf.db') as shelf: with shelve.open('shelf.db') as shelf:
if 'user_dict' in shelf.keys(): if 'user_dict' in shelf.keys():
user_dict = shelf['user_dict'] user_dict = shelf['user_dict']
if 'track_dict' in shelf.keys(): if 'track_dict' in shelf.keys():
track_dict = shelf['track_dict'] track_dict = shelf['track_dict']
if 'users_to_process' in shelf.keys():
users_to_process = shelf['users_to_process']
if 'users_to_process_next' in shelf.keys():
users_to_process_next = shelf['users_to_process_next']
if 'iterator' in shelf.keys():
iterator = shelf['iterator']
users_to_process = []
users_to_process_next = STARTING_USERS
for i in range(MAX_DEGREE + 1): while iterator <= MAX_DEGREE:
users_to_process = users_to_process_next
users_to_process_next = []
for user_str in users_to_process: for user_str in users_to_process:
print('Processing ' + user_str + ' (distance ' + str(i) + ')') print('Processing ' + user_str + ' (distance ' + str(iterator) + ')')
# Skip any users that have already been passed over to avoid infinite loops # Skip any users that have already been passed over to avoid infinite loops
if user_str in user_dict.keys(): if user_str in user_dict.keys():
@ -343,6 +348,13 @@ for i in range(MAX_DEGREE + 1):
with shelve.open('shelf.db') as shelf: with shelve.open('shelf.db') as shelf:
shelf['user_dict'] = user_dict shelf['user_dict'] = user_dict
shelf['track_dict'] = track_dict shelf['track_dict'] = track_dict
shelf['users_to_process'] = users_to_process
shelf['users_to_process_next'] = users_to_process_next
shelf['iterator'] = iterator
iterator += 1
users_to_process = users_to_process_next
users_to_process_next = []
driver.quit() driver.quit()