tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use Shelve to store user and track dicts while scraping

Browse Source 
Just in case something breaks mid-scrape
This commit is contained in:
Thomas Wade 2018-09-26 14:37:50 +09:30
parent 82445f9d37
commit 161d24e238
2 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -112,3 +112,6 @@ dmypy.json
# Generated graph files # Generated graph files
*.gexf *.gexf
# Shelve
shelf.db

12
scrape.py
View File

@ -22,6 +22,7 @@ STARTING_USERS = ['thomotron']
##### Import all the things ################################# ##### Import all the things #################################
import time import time
import shelve
import networkx as nx import networkx as nx
from selenium import webdriver from selenium import webdriver
from selenium.webdriver.common.by import By from selenium.webdriver.common.by import By
@ -266,6 +267,12 @@ def Get_Follows(url, limit = 100):
user_dict = {} user_dict = {}
track_dict = {} track_dict = {}
with shelve.open('shelf.db', 'r') as shelf:
if 'user_dict' in shelf.keys():
user_dict = shelf['user_dict']
if 'track_dict' in shelf.keys():
track_dict = shelf['track_dict']
users_to_process = [] users_to_process = []
users_to_process_next = STARTING_USERS users_to_process_next = STARTING_USERS
@ -319,6 +326,11 @@ for i in range(MAX_DEGREE + 1):
user.processed = True user.processed = True
user_dict[user.url_username] = user user_dict[user.url_username] = user
# Update the shelf with the new dictionaries
with shelve.open('shelf.db', 'c') as shelf:
shelf['user_dict'] = user_dict
shelf['track_dict'] = track_dict
driver.quit() driver.quit()
##### Process what was scraped ############################## ##### Process what was scraped ##############################