Add argument to skip scraping and just spit out a graph

scrape.py

@@ -43,6 +43,16 @@ from selenium.webdriver.support import expected_conditions
 from selenium.webdriver.chrome.options import Options
 from bs4 import BeautifulSoup
 from collections import Counter
+from argparse import ArgumentParser
+
+##### Parse arguments #######################################
+
+parser = ArgumentParser()
+parser.add_argument('-d', '--dump', help='skip scraping and process whatever is shelved', action='store_true')
+args = parser.parse_args()
+
+if args.dump:
+    print('Dump option specified, will skip scraping')
 
 ##### Set up NetworkX #######################################
 
@@ -52,18 +62,19 @@ print('NetworkX initialised')
 
 ##### Set up WebDriver with Chrome ##########################
 
-chrome_options = Options()
+if not args.dump:
+    chrome_options = Options()
 
-# Start headless so we can run on a server overnight
+    # Start headless so we can run on a server overnight
-chrome_options.add_argument('--headless')
+    chrome_options.add_argument('--headless')
 
-# Disable image loading, courtesy of rocky qi
+    # Disable image loading, courtesy of rocky qi
-# Found at https://stackoverflow.com/a/31581387, accessed on 2018/09/17 at 12:54 UTC
+    # Found at https://stackoverflow.com/a/31581387, accessed on 2018/09/17 at 12:54 UTC
-chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images":2})
+    chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images":2})
 
-driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options)
+    driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options)
 
-print('WebDriver initialised')
+    print('WebDriver initialised')
 
 ##### Define some classes to hold our data ##################
 
@@ -343,68 +354,68 @@ with shelve.open('shelf.db') as shelf:
     if 'iterator' in shelf.keys():
         iterator = shelf['iterator']
 
+if not args.dump:
+    while iterator <= MAX_DEGREE:
+        for user_str in users_to_process:
+            print('Processing ' + user_str + ' (distance ' + str(iterator) + ')')
 
-while iterator <= MAX_DEGREE:
+            # Skip any users that have already been passed over to avoid infinite loops
-    for user_str in users_to_process:
+            if user_str in user_dict.keys():
-        print('Processing ' + user_str + ' (distance ' + str(iterator) + ')')
+                if user_dict[user_str].processed:
+                    print('  Already processed, skipping')
+                    continue
 
-        # Skip any users that have already been passed over to avoid infinite loops
+            # Initialise our user object to store our values in
-        if user_str in user_dict.keys():
+            user = User()
-            if user_dict[user_str].processed:
+            user.url_username = user_str
-                print('  Already processed, skipping')
+            print('  URL Username: ' + user.url_username)
-                continue
 
-        # Initialise our user object to store our values in
+            # Get their basic info
-        user = User()
+            Get_Basic_Info(user)
-        user.url_username = user_str
-        print('  URL Username: ' + user.url_username)
 
-        # Get their basic info
+            # Get their tracks
-        Get_Basic_Info(user)
+            for track in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', USER_MAX_TRACKS):
+                if not track.url in track_dict.keys():
+                    track_dict[track.url] = track
+                user.tracks.append(track.url)
 
-        # Get their tracks
+            # Get their likes
-        for track in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', USER_MAX_TRACKS):
+            for like in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', USER_MAX_LIKES):
-            if not track.url in track_dict.keys():
+                if not like.url in track_dict.keys():
-                track_dict[track.url] = track
+                    track_dict[like.url] = like
-            user.tracks.append(track.url)
+                user.likes.append(like.url)
 
-        # Get their likes
+            # Get who they follow
-        for like in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', USER_MAX_LIKES):
+            for following in Get_Follows('https://soundcloud.com/' + user.url_username + '/following', USER_MAX_FOLLOWING):
-            if not like.url in track_dict.keys():
+                if not following.url_username in user_dict.keys():
-                track_dict[like.url] = like
+                    user_dict[following.url_username] = following
-            user.likes.append(like.url)
+                user.following.append(following.url_username)
+                users_to_process_next.append(following.url_username)
 
-        # Get who they follow
+            # Get who follows them
-        for following in Get_Follows('https://soundcloud.com/' + user.url_username + '/following', USER_MAX_FOLLOWING):
+            for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', USER_MAX_FOLLOWERS):
-            if not following.url_username in user_dict.keys():
+                if not follower.url_username in user_dict.keys():
-                user_dict[following.url_username] = following
+                    user_dict[follower.url_username] = follower
-            user.following.append(following.url_username)
+                user.followers.append(follower.url_username)
-            users_to_process_next.append(following.url_username)
+                users_to_process_next.append(follower.url_username)
 
-        # Get who follows them
+            # Finally add the user to the dictionary and mark them as processed
-        for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', USER_MAX_FOLLOWERS):
+            user.processed = True
-            if not follower.url_username in user_dict.keys():
+            user_dict[user.url_username] = user
-                user_dict[follower.url_username] = follower
-            user.followers.append(follower.url_username)
-            users_to_process_next.append(follower.url_username)
 
-        # Finally add the user to the dictionary and mark them as processed
+            # Update the shelf with the new dictionaries
-        user.processed = True
+            with shelve.open('shelf.db') as shelf:
-        user_dict[user.url_username] = user
+                shelf['user_dict'] = user_dict
+                shelf['track_dict'] = track_dict
+                shelf['users_to_process'] = users_to_process
+                shelf['users_to_process_next'] = users_to_process_next
+                shelf['iterator'] = iterator
 
-        # Update the shelf with the new dictionaries
+        iterator += 1
-        with shelve.open('shelf.db') as shelf:
+        users_to_process = users_to_process_next
-            shelf['user_dict'] = user_dict
+        users_to_process_next = []
-            shelf['track_dict'] = track_dict
-            shelf['users_to_process'] = users_to_process
-            shelf['users_to_process_next'] = users_to_process_next
-            shelf['iterator'] = iterator
 
-    iterator += 1
+    driver.quit()
-    users_to_process = users_to_process_next
-    users_to_process_next = []
-
-driver.quit()
 
 ##### Process what was scraped ##############################