tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Track users and tracks in a database-like fashion

Browse Source 
Rather than have several copies of a user/track object, collect them all in a single dictionary and give the keys to each user object. This will avoid duplication and make generating graphs easier.
This commit is contained in:
Thomas Wade 2018-09-25 19:09:05 +09:30
parent 6e5cfdfa0d
commit 295e3e3d12
1 changed files with 30 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
scrape.py
View File

@ -228,21 +228,24 @@ def Get_Follows(url, limit = 100):
print(' Users:') print(' Users:')
users = [] users = {}
badge_elems = soup.find_all(class_='badgeList__item')[:limit] badge_elems = soup.find_all(class_='badgeList__item')[:limit]
for badge_elem in badge_elems: for badge_elem in badge_elems:
username_elem = badge_elem.find('a', class_='userBadgeListItem__heading') username_elem = badge_elem.find('a', class_='userBadgeListItem__heading')
if username_elem: if username_elem:
url_username = username_elem['href'].strip('/') user = User()
users.append(url_username) user.url_username = username_elem['href'].strip('/')
users[user.url_username] = user
print(' ' + url_username) print(' ' + user.url_username)
return users return users.values()
user_dict = {} user_dict = {}
track_dict = {}
users_to_process = [] users_to_process = []
users_to_process_next = ['thomotron', 'lacheque', 'slynk', 'bossfightswe'] users_to_process_next = ['thomotron', 'lacheque', 'slynk', 'bossfightswe']
@ -267,18 +270,30 @@ for i in range(2):
Get_Basic_Info(user) Get_Basic_Info(user)
# Get their tracks # Get their tracks
user.tracks = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', 10) for track in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', 10):
if not track.url in track_dict.keys():
track_dict[track.url] = track
user.tracks.append(track.url)
# Get their likes # Get their likes
user.likes = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', 10) for like in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', 10):
if not like.url in track_dict.keys():
track_dict[like.url] = like
user.likes.append(like.url)
# Get who they follow # Get who they follow
user.following = Get_Follows('https://soundcloud.com/' + user.url_username + '/following', 10) for followed in Get_Follows('https://soundcloud.com/' + user.url_username + '/followed', 10):
users_to_process_next.extend(user.following) if not followed.url_username in user_dict.keys():
user_dict[followed.url_username] = followed
user.followed.append(followed.url_username)
users_to_process_next.append(followed.url_username)
# Get who follows them # Get who follows them
user.followers = Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', 10) for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', 10):
users_to_process_next.extend(user.followers) if not follower.url_username in user_dict.keys():
user_dict[follower.url_username] = follower
user.followers.append(follower.url_username)
users_to_process_next.append(follower.url_username)
# Finally add the user to the dictionary and mark them as processed # Finally add the user to the dictionary and mark them as processed
user.processed = True user.processed = True
@ -300,13 +315,13 @@ for _, user in user_dict.items():
print(' Followed by ' + str(user.follower_count)) print(' Followed by ' + str(user.follower_count))
print(' Tracks:') print(' Tracks:')
for track in user.tracks: for track in user.tracks:
print(' ' + str(track.title)) print(' ' + str(track))
print(' Likes:') print(' Likes:')
for track in user.likes: for track in user.likes:
print(' ' + str(track.title)) print(' ' + str(track))
print(' Followers:') print(' Followers:')
for follower in user.followers: for follower in user.followers:
print(' ' + str(follower.url_username)) print(' ' + str(follower))
print(' Following:') print(' Following:')
for followed in user.following: for followed in user.following:
print(' ' + str(followed.url_username)) print(' ' + str(followed))