tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add dividers and some missing comments

Browse Source
This commit is contained in:
Thomas Wade 2018-09-25 19:44:52 +09:30
parent 829374fc6e
commit 490a2ac22d
1 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scrape.py
View File

@ -51,7 +51,7 @@ driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chro
print('WebDriver initialised') print('WebDriver initialised')
##### Do stuff ############################################## ##### Define some classes to hold our data ##################
# User class, holds all the data we will be collecting from user pages # User class, holds all the data we will be collecting from user pages
# It's being used here as a more structured alternative to a dictionary # It's being used here as a more structured alternative to a dictionary
@ -89,6 +89,8 @@ class Track:
reposts = None reposts = None
comments = None comments = None
##### Define some methods to do our scraping ################
# Gets a user's username, track count, following count, follower count, and like count # Gets a user's username, track count, following count, follower count, and like count
def Get_Basic_Info(user): def Get_Basic_Info(user):
# Get the user's profile page and soup it # Get the user's profile page and soup it
@ -125,6 +127,7 @@ def Get_Basic_Info(user):
user.like_count = like_count_elem.string.split(' ')[0] user.like_count = like_count_elem.string.split(' ')[0]
print(' Likes: ' + str(user.like_count)) print(' Likes: ' + str(user.like_count))
# Gets a list of fully-populated Track objects
def Get_Tracks_Info(url, limit = 100): def Get_Tracks_Info(url, limit = 100):
# Get the page and soup it # Get the page and soup it
print('GET: ' + url) print('GET: ' + url)
@ -216,6 +219,7 @@ def Get_Tracks_Info(url, limit = 100):
# Strip off keys and return a track list # Strip off keys and return a track list
return tracks.values() return tracks.values()
# Gets a list of User objects from a badge list containing only their url_username
def Get_Follows(url, limit = 100): def Get_Follows(url, limit = 100):
# Get the page # Get the page
print('GET: ' + url) print('GET: ' + url)
@ -257,6 +261,7 @@ def Get_Follows(url, limit = 100):
return users.values() return users.values()
##### Scrape ################################################
user_dict = {} user_dict = {}
track_dict = {} track_dict = {}
@ -316,6 +321,8 @@ for i in range(MAX_DEGREE + 1):
driver.quit() driver.quit()
##### Process what was scraped ##############################
print('Done scraping, here\'s what we got') print('Done scraping, here\'s what we got')
print('==================================') print('==================================')