tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add some better structure to status prints

Browse Source
This commit is contained in:
Thomas Wade 2018-09-19 09:49:01 +09:30
parent 3f8aa7304f
commit 5781fab7bd
1 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
scrape.py
View File

@ -67,32 +67,34 @@ def Get_Basic_Info(user):
driver.get('https://soundcloud.com/' + user.url_username) driver.get('https://soundcloud.com/' + user.url_username)
soup = BeautifulSoup(driver.page_source, 'lxml') soup = BeautifulSoup(driver.page_source, 'lxml')
print(' Basic Info:')
# Grab their username # Grab their username
# Use stripped_strings generator as workaround for users with premium badge # Use stripped_strings generator as workaround for users with premium badge
username = next(soup.find(class_='profileHeaderInfo__userName').stripped_strings) username = next(soup.find(class_='profileHeaderInfo__userName').stripped_strings)
user.username = username user.username = username
print(' Username: ' + user.username) print(' Username: ' + user.username)
# Grab their track count # Grab their track count
track_count = soup.find('a', href='/' + user.url_username + '/tracks', class_='infoStats__statLink').div.string track_count = soup.find('a', href='/' + user.url_username + '/tracks', class_='infoStats__statLink').div.string
user.track_count = track_count user.track_count = track_count
print(' Tracks: ' + user.track_count) print(' Tracks: ' + user.track_count)
# Grab their following count # Grab their following count
following_count = soup.find('a', href='/' + user.url_username + '/following', class_='infoStats__statLink').div.string following_count = soup.find('a', href='/' + user.url_username + '/following', class_='infoStats__statLink').div.string
user.following_count = following_count user.following_count = following_count
print(' Following: ' + str(user.following_count)) print(' Following: ' + str(user.following_count))
# Grab their follower count # Grab their follower count
follower_count = soup.find('a', href='/' + user.url_username + '/followers', class_='infoStats__statLink').div.string follower_count = soup.find('a', href='/' + user.url_username + '/followers', class_='infoStats__statLink').div.string
user.follower_count = follower_count user.follower_count = follower_count
print(' Followers: ' + str(user.follower_count)) print(' Followers: ' + str(user.follower_count))
# Grab their like count # Grab their like count
like_count_elem = soup.find('a', href='/' + user.url_username + '/likes').find(class_='sidebarHeader__actualTitle') like_count_elem = soup.find('a', href='/' + user.url_username + '/likes').find(class_='sidebarHeader__actualTitle')
if like_count_elem: if like_count_elem:
user.like_count = like_count_elem.string.split(' ')[0] user.like_count = like_count_elem.string.split(' ')[0]
print(' Likes: ' + str(user.like_count)) print(' Likes: ' + str(user.like_count))
def Get_Tracks_Info(user): def Get_Tracks_Info(user):
# Get the user's tracks page and soup it # Get the user's tracks page and soup it
@ -100,6 +102,8 @@ def Get_Tracks_Info(user):
driver.get('https://soundcloud.com/' + user.url_username + '/tracks') driver.get('https://soundcloud.com/' + user.url_username + '/tracks')
soup = BeautifulSoup(driver.page_source, 'lxml') soup = BeautifulSoup(driver.page_source, 'lxml')
print(' Tracks:')
tracks = {} tracks = {}
track_elems = soup.find_all(class_='soundList__item') track_elems = soup.find_all(class_='soundList__item')
@ -150,6 +154,8 @@ def Get_Tracks_Info(user):
# Finally add the track to the dictionary # Finally add the track to the dictionary
tracks[track.artist + track.title] = track tracks[track.artist + track.title] = track
print(' ' + track.artist + ' - ' + track.title)
user.tracks = tracks user.tracks = tracks
user_dict = {} user_dict = {}