tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Convert human-readable numbers to ints

Browse Source
This commit is contained in:
Thomas Wade 2018-09-29 13:29:08 +09:30
parent 3e9a4b5f65
commit bb88bbeb24
1 changed files with 22 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
scrape.py
View File

@ -121,25 +121,25 @@ def Get_Basic_Info(user):
# Grab their track count # Grab their track count
track_count_elem = soup.find('a', href='/' + user.url_username + '/tracks', class_='infoStats__statLink') track_count_elem = soup.find('a', href='/' + user.url_username + '/tracks', class_='infoStats__statLink')
if track_count_elem: if track_count_elem:
user.track_count = str(track_count_elem.div.string) user.track_count = Human_Str_To_Int(str(track_count_elem.div.string))
print(' Tracks: ' + str(user.track_count)) print(' Tracks: ' + str(user.track_count))
# Grab their following count # Grab their following count
following_count_elem = soup.find('a', href='/' + user.url_username + '/following', class_='infoStats__statLink') following_count_elem = soup.find('a', href='/' + user.url_username + '/following', class_='infoStats__statLink')
if following_count_elem: if following_count_elem:
user.following_count = str(following_count_elem.div.string) user.following_count = Human_Str_To_Int(str(following_count_elem.div.string))
print(' Following: ' + str(user.following_count)) print(' Following: ' + str(user.following_count))
# Grab their follower count # Grab their follower count
follower_count_elem = soup.find('a', href='/' + user.url_username + '/followers', class_='infoStats__statLink') follower_count_elem = soup.find('a', href='/' + user.url_username + '/followers', class_='infoStats__statLink')
if follower_count_elem: if follower_count_elem:
user.follower_count = str(follower_count_elem.div.string) user.follower_count = Human_Str_To_Int(str(follower_count_elem.div.string))
print(' Followers: ' + str(user.follower_count)) print(' Followers: ' + str(user.follower_count))
# Grab their like count # Grab their like count
like_count_elem = soup.find(class_='sidebarHeader__actualTitle') like_count_elem = soup.find(class_='sidebarHeader__actualTitle')
if like_count_elem: if like_count_elem:
user.like_count = str(like_count_elem.string.split(' ')[0]) user.like_count = Human_Str_To_Int(str(like_count_elem.string.split(' ')[0]))
print(' Likes: ' + str(user.like_count)) print(' Likes: ' + str(user.like_count))
# Gets a list of fully-populated Track objects # Gets a list of fully-populated Track objects
@ -212,19 +212,19 @@ def Get_Tracks_Info(url, limit = 100):
for elem in play_comment_elems: for elem in play_comment_elems:
num, unit = str(elem['title']).split(' ') num, unit = str(elem['title']).split(' ')
if unit == 'plays': if unit == 'plays':
track.plays = num track.plays = Human_Str_To_Int(num)
elif unit == 'comments': elif unit == 'comments':
track.comments = num track.comments = Human_Str_To_Int(num)
# Grab the like count # Grab the like count
like_elem = track_elem.find(class_='sc-button-like') like_elem = track_elem.find(class_='sc-button-like')
if like_elem: if like_elem:
track.likes = str(like_elem.string.strip()) track.likes = Human_Str_To_Int(str(like_elem.string.strip()))
# Grab the repost count # Grab the repost count
repost_elem = track_elem.find(class_='sc-button-repost') repost_elem = track_elem.find(class_='sc-button-repost')
if repost_elem: if repost_elem:
track.reposts = str(repost_elem.string.strip()) track.reposts = Human_Str_To_Int(str(repost_elem.string.strip()))
# Finally add the track to the dictionary # Finally add the track to the dictionary
tracks[track.url] = track tracks[track.url] = track
@ -276,6 +276,20 @@ def Get_Follows(url, limit = 100):
return users.values() return users.values()
def Human_Str_To_Int(string):
num = None
string = string.lower().replace(',', '')
if 'k' in string:
string = string.replace('k', '')
num = int(float(string) * 1000)
elif 'm' in string:
string = string.replace('m', '')
num = int(float(string) * 1000000)
else:
num = int(string)
return num
##### Scrape ################################################ ##### Scrape ################################################
user_dict = {} user_dict = {}