diff --git a/scrape.py b/scrape.py index 8a07852..27472f5 100755 --- a/scrape.py +++ b/scrape.py @@ -228,21 +228,24 @@ def Get_Follows(url, limit = 100): print(' Users:') - users = [] + users = {} badge_elems = soup.find_all(class_='badgeList__item')[:limit] for badge_elem in badge_elems: username_elem = badge_elem.find('a', class_='userBadgeListItem__heading') if username_elem: - url_username = username_elem['href'].strip('/') - users.append(url_username) + user = User() + user.url_username = username_elem['href'].strip('/') + users[user.url_username] = user - print(' ' + url_username) + print(' ' + user.url_username) - return users + return users.values() user_dict = {} +track_dict = {} + users_to_process = [] users_to_process_next = ['thomotron', 'lacheque', 'slynk', 'bossfightswe'] @@ -267,18 +270,30 @@ for i in range(2): Get_Basic_Info(user) # Get their tracks - user.tracks = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', 10) + for track in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', 10): + if not track.url in track_dict.keys(): + track_dict[track.url] = track + user.tracks.append(track.url) # Get their likes - user.likes = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', 10) + for like in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', 10): + if not like.url in track_dict.keys(): + track_dict[like.url] = like + user.likes.append(like.url) # Get who they follow - user.following = Get_Follows('https://soundcloud.com/' + user.url_username + '/following', 10) - users_to_process_next.extend(user.following) + for followed in Get_Follows('https://soundcloud.com/' + user.url_username + '/followed', 10): + if not followed.url_username in user_dict.keys(): + user_dict[followed.url_username] = followed + user.followed.append(followed.url_username) + users_to_process_next.append(followed.url_username) # Get who follows them - user.followers = Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', 10) - users_to_process_next.extend(user.followers) + for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', 10): + if not follower.url_username in user_dict.keys(): + user_dict[follower.url_username] = follower + user.followers.append(follower.url_username) + users_to_process_next.append(follower.url_username) # Finally add the user to the dictionary and mark them as processed user.processed = True @@ -300,13 +315,13 @@ for _, user in user_dict.items(): print(' Followed by ' + str(user.follower_count)) print(' Tracks:') for track in user.tracks: - print(' ' + str(track.title)) + print(' ' + str(track)) print(' Likes:') for track in user.likes: - print(' ' + str(track.title)) + print(' ' + str(track)) print(' Followers:') for follower in user.followers: - print(' ' + str(follower.url_username)) + print(' ' + str(follower)) print(' Following:') for followed in user.following: - print(' ' + str(followed.url_username)) + print(' ' + str(followed))