From 3588e5937f452bd01c1e271341af7268168a0b15 Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Sun, 30 Sep 2018 15:02:45 +0930 Subject: [PATCH] Add argument to skip scraping and just spit out a graph --- scrape.py | 129 +++++++++++++++++++++++++++++------------------------- 1 file changed, 70 insertions(+), 59 deletions(-) diff --git a/scrape.py b/scrape.py index 2aa6117..8788451 100755 --- a/scrape.py +++ b/scrape.py @@ -43,6 +43,16 @@ from selenium.webdriver.support import expected_conditions from selenium.webdriver.chrome.options import Options from bs4 import BeautifulSoup from collections import Counter +from argparse import ArgumentParser + +##### Parse arguments ####################################### + +parser = ArgumentParser() +parser.add_argument('-d', '--dump', help='skip scraping and process whatever is shelved', action='store_true') +args = parser.parse_args() + +if args.dump: + print('Dump option specified, will skip scraping') ##### Set up NetworkX ####################################### @@ -52,18 +62,19 @@ print('NetworkX initialised') ##### Set up WebDriver with Chrome ########################## -chrome_options = Options() +if not args.dump: + chrome_options = Options() -# Start headless so we can run on a server overnight -chrome_options.add_argument('--headless') + # Start headless so we can run on a server overnight + chrome_options.add_argument('--headless') -# Disable image loading, courtesy of rocky qi -# Found at https://stackoverflow.com/a/31581387, accessed on 2018/09/17 at 12:54 UTC -chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images":2}) + # Disable image loading, courtesy of rocky qi + # Found at https://stackoverflow.com/a/31581387, accessed on 2018/09/17 at 12:54 UTC + chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images":2}) -driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options) + driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options) -print('WebDriver initialised') + print('WebDriver initialised') ##### Define some classes to hold our data ################## @@ -343,68 +354,68 @@ with shelve.open('shelf.db') as shelf: if 'iterator' in shelf.keys(): iterator = shelf['iterator'] +if not args.dump: + while iterator <= MAX_DEGREE: + for user_str in users_to_process: + print('Processing ' + user_str + ' (distance ' + str(iterator) + ')') -while iterator <= MAX_DEGREE: - for user_str in users_to_process: - print('Processing ' + user_str + ' (distance ' + str(iterator) + ')') + # Skip any users that have already been passed over to avoid infinite loops + if user_str in user_dict.keys(): + if user_dict[user_str].processed: + print(' Already processed, skipping') + continue - # Skip any users that have already been passed over to avoid infinite loops - if user_str in user_dict.keys(): - if user_dict[user_str].processed: - print(' Already processed, skipping') - continue + # Initialise our user object to store our values in + user = User() + user.url_username = user_str + print(' URL Username: ' + user.url_username) - # Initialise our user object to store our values in - user = User() - user.url_username = user_str - print(' URL Username: ' + user.url_username) + # Get their basic info + Get_Basic_Info(user) - # Get their basic info - Get_Basic_Info(user) + # Get their tracks + for track in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', USER_MAX_TRACKS): + if not track.url in track_dict.keys(): + track_dict[track.url] = track + user.tracks.append(track.url) - # Get their tracks - for track in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', USER_MAX_TRACKS): - if not track.url in track_dict.keys(): - track_dict[track.url] = track - user.tracks.append(track.url) + # Get their likes + for like in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', USER_MAX_LIKES): + if not like.url in track_dict.keys(): + track_dict[like.url] = like + user.likes.append(like.url) - # Get their likes - for like in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', USER_MAX_LIKES): - if not like.url in track_dict.keys(): - track_dict[like.url] = like - user.likes.append(like.url) + # Get who they follow + for following in Get_Follows('https://soundcloud.com/' + user.url_username + '/following', USER_MAX_FOLLOWING): + if not following.url_username in user_dict.keys(): + user_dict[following.url_username] = following + user.following.append(following.url_username) + users_to_process_next.append(following.url_username) - # Get who they follow - for following in Get_Follows('https://soundcloud.com/' + user.url_username + '/following', USER_MAX_FOLLOWING): - if not following.url_username in user_dict.keys(): - user_dict[following.url_username] = following - user.following.append(following.url_username) - users_to_process_next.append(following.url_username) + # Get who follows them + for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', USER_MAX_FOLLOWERS): + if not follower.url_username in user_dict.keys(): + user_dict[follower.url_username] = follower + user.followers.append(follower.url_username) + users_to_process_next.append(follower.url_username) - # Get who follows them - for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', USER_MAX_FOLLOWERS): - if not follower.url_username in user_dict.keys(): - user_dict[follower.url_username] = follower - user.followers.append(follower.url_username) - users_to_process_next.append(follower.url_username) + # Finally add the user to the dictionary and mark them as processed + user.processed = True + user_dict[user.url_username] = user - # Finally add the user to the dictionary and mark them as processed - user.processed = True - user_dict[user.url_username] = user + # Update the shelf with the new dictionaries + with shelve.open('shelf.db') as shelf: + shelf['user_dict'] = user_dict + shelf['track_dict'] = track_dict + shelf['users_to_process'] = users_to_process + shelf['users_to_process_next'] = users_to_process_next + shelf['iterator'] = iterator - # Update the shelf with the new dictionaries - with shelve.open('shelf.db') as shelf: - shelf['user_dict'] = user_dict - shelf['track_dict'] = track_dict - shelf['users_to_process'] = users_to_process - shelf['users_to_process_next'] = users_to_process_next - shelf['iterator'] = iterator + iterator += 1 + users_to_process = users_to_process_next + users_to_process_next = [] - iterator += 1 - users_to_process = users_to_process_next - users_to_process_next = [] - -driver.quit() + driver.quit() ##### Process what was scraped ##############################