diff --git a/scrape.py b/scrape.py index ae7e650..f87bed0 100755 --- a/scrape.py +++ b/scrape.py @@ -51,7 +51,7 @@ driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chro print('WebDriver initialised') -##### Do stuff ############################################## +##### Define some classes to hold our data ################## # User class, holds all the data we will be collecting from user pages # It's being used here as a more structured alternative to a dictionary @@ -89,6 +89,8 @@ class Track: reposts = None comments = None +##### Define some methods to do our scraping ################ + # Gets a user's username, track count, following count, follower count, and like count def Get_Basic_Info(user): # Get the user's profile page and soup it @@ -125,6 +127,7 @@ def Get_Basic_Info(user): user.like_count = like_count_elem.string.split(' ')[0] print(' Likes: ' + str(user.like_count)) +# Gets a list of fully-populated Track objects def Get_Tracks_Info(url, limit = 100): # Get the page and soup it print('GET: ' + url) @@ -216,6 +219,7 @@ def Get_Tracks_Info(url, limit = 100): # Strip off keys and return a track list return tracks.values() +# Gets a list of User objects from a badge list containing only their url_username def Get_Follows(url, limit = 100): # Get the page print('GET: ' + url) @@ -257,6 +261,7 @@ def Get_Follows(url, limit = 100): return users.values() +##### Scrape ################################################ user_dict = {} track_dict = {} @@ -316,6 +321,8 @@ for i in range(MAX_DEGREE + 1): driver.quit() +##### Process what was scraped ############################## + print('Done scraping, here\'s what we got') print('==================================')