Add argument to skip scraping and just spit out a graph
This commit is contained in:
Thomas Wade
parent
65a5a5c63b
commit
3588e5937f
129
scrape.py
129
scrape.py
@ -43,6 +43,16 @@ from selenium.webdriver.support import expected_conditions
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from bs4 import BeautifulSoup
|
||||
from collections import Counter
|
||||
from argparse import ArgumentParser
|
||||
|
||||
##### Parse arguments #######################################
|
||||
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument('-d', '--dump', help='skip scraping and process whatever is shelved', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.dump:
|
||||
print('Dump option specified, will skip scraping')
|
||||
|
||||
##### Set up NetworkX #######################################
|
||||
|
||||
@ -52,18 +62,19 @@ print('NetworkX initialised')
|
||||
|
||||
##### Set up WebDriver with Chrome ##########################
|
||||
|
||||
chrome_options = Options()
|
||||
if not args.dump:
|
||||
chrome_options = Options()
|
||||
|
||||
# Start headless so we can run on a server overnight
|
||||
chrome_options.add_argument('--headless')
|
||||
# Start headless so we can run on a server overnight
|
||||
chrome_options.add_argument('--headless')
|
||||
|
||||
# Disable image loading, courtesy of rocky qi
|
||||
# Found at https://stackoverflow.com/a/31581387, accessed on 2018/09/17 at 12:54 UTC
|
||||
chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images":2})
|
||||
# Disable image loading, courtesy of rocky qi
|
||||
# Found at https://stackoverflow.com/a/31581387, accessed on 2018/09/17 at 12:54 UTC
|
||||
chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images":2})
|
||||
|
||||
driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options)
|
||||
driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options)
|
||||
|
||||
print('WebDriver initialised')
|
||||
print('WebDriver initialised')
|
||||
|
||||
##### Define some classes to hold our data ##################
|
||||
|
||||
@ -343,68 +354,68 @@ with shelve.open('shelf.db') as shelf:
|
||||
if 'iterator' in shelf.keys():
|
||||
iterator = shelf['iterator']
|
||||
|
||||
if not args.dump:
|
||||
while iterator <= MAX_DEGREE:
|
||||
for user_str in users_to_process:
|
||||
print('Processing ' + user_str + ' (distance ' + str(iterator) + ')')
|
||||
|
||||
while iterator <= MAX_DEGREE:
|
||||
for user_str in users_to_process:
|
||||
print('Processing ' + user_str + ' (distance ' + str(iterator) + ')')
|
||||
# Skip any users that have already been passed over to avoid infinite loops
|
||||
if user_str in user_dict.keys():
|
||||
if user_dict[user_str].processed:
|
||||
print(' Already processed, skipping')
|
||||
continue
|
||||
|
||||
# Skip any users that have already been passed over to avoid infinite loops
|
||||
if user_str in user_dict.keys():
|
||||
if user_dict[user_str].processed:
|
||||
print(' Already processed, skipping')
|
||||
continue
|
||||
# Initialise our user object to store our values in
|
||||
user = User()
|
||||
user.url_username = user_str
|
||||
print(' URL Username: ' + user.url_username)
|
||||
|
||||
# Initialise our user object to store our values in
|
||||
user = User()
|
||||
user.url_username = user_str
|
||||
print(' URL Username: ' + user.url_username)
|
||||
# Get their basic info
|
||||
Get_Basic_Info(user)
|
||||
|
||||
# Get their basic info
|
||||
Get_Basic_Info(user)
|
||||
# Get their tracks
|
||||
for track in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', USER_MAX_TRACKS):
|
||||
if not track.url in track_dict.keys():
|
||||
track_dict[track.url] = track
|
||||
user.tracks.append(track.url)
|
||||
|
||||
# Get their tracks
|
||||
for track in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', USER_MAX_TRACKS):
|
||||
if not track.url in track_dict.keys():
|
||||
track_dict[track.url] = track
|
||||
user.tracks.append(track.url)
|
||||
# Get their likes
|
||||
for like in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', USER_MAX_LIKES):
|
||||
if not like.url in track_dict.keys():
|
||||
track_dict[like.url] = like
|
||||
user.likes.append(like.url)
|
||||
|
||||
# Get their likes
|
||||
for like in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', USER_MAX_LIKES):
|
||||
if not like.url in track_dict.keys():
|
||||
track_dict[like.url] = like
|
||||
user.likes.append(like.url)
|
||||
# Get who they follow
|
||||
for following in Get_Follows('https://soundcloud.com/' + user.url_username + '/following', USER_MAX_FOLLOWING):
|
||||
if not following.url_username in user_dict.keys():
|
||||
user_dict[following.url_username] = following
|
||||
user.following.append(following.url_username)
|
||||
users_to_process_next.append(following.url_username)
|
||||
|
||||
# Get who they follow
|
||||
for following in Get_Follows('https://soundcloud.com/' + user.url_username + '/following', USER_MAX_FOLLOWING):
|
||||
if not following.url_username in user_dict.keys():
|
||||
user_dict[following.url_username] = following
|
||||
user.following.append(following.url_username)
|
||||
users_to_process_next.append(following.url_username)
|
||||
# Get who follows them
|
||||
for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', USER_MAX_FOLLOWERS):
|
||||
if not follower.url_username in user_dict.keys():
|
||||
user_dict[follower.url_username] = follower
|
||||
user.followers.append(follower.url_username)
|
||||
users_to_process_next.append(follower.url_username)
|
||||
|
||||
# Get who follows them
|
||||
for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', USER_MAX_FOLLOWERS):
|
||||
if not follower.url_username in user_dict.keys():
|
||||
user_dict[follower.url_username] = follower
|
||||
user.followers.append(follower.url_username)
|
||||
users_to_process_next.append(follower.url_username)
|
||||
# Finally add the user to the dictionary and mark them as processed
|
||||
user.processed = True
|
||||
user_dict[user.url_username] = user
|
||||
|
||||
# Finally add the user to the dictionary and mark them as processed
|
||||
user.processed = True
|
||||
user_dict[user.url_username] = user
|
||||
# Update the shelf with the new dictionaries
|
||||
with shelve.open('shelf.db') as shelf:
|
||||
shelf['user_dict'] = user_dict
|
||||
shelf['track_dict'] = track_dict
|
||||
shelf['users_to_process'] = users_to_process
|
||||
shelf['users_to_process_next'] = users_to_process_next
|
||||
shelf['iterator'] = iterator
|
||||
|
||||
# Update the shelf with the new dictionaries
|
||||
with shelve.open('shelf.db') as shelf:
|
||||
shelf['user_dict'] = user_dict
|
||||
shelf['track_dict'] = track_dict
|
||||
shelf['users_to_process'] = users_to_process
|
||||
shelf['users_to_process_next'] = users_to_process_next
|
||||
shelf['iterator'] = iterator
|
||||
iterator += 1
|
||||
users_to_process = users_to_process_next
|
||||
users_to_process_next = []
|
||||
|
||||
iterator += 1
|
||||
users_to_process = users_to_process_next
|
||||
users_to_process_next = []
|
||||
|
||||
driver.quit()
|
||||
driver.quit()
|
||||
|
||||
##### Process what was scraped ##############################
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user