diff --git a/scrape.py b/scrape.py index 7883ce7..64f90c5 100755 --- a/scrape.py +++ b/scrape.py @@ -51,6 +51,9 @@ class User: tracks = [] likes = [] + followers = [] + following = [] + class Track: title = None artist = None @@ -190,6 +193,48 @@ def Get_Tracks_Info(url, limit = 100): # Strip off keys and return a track list return tracks.values() +def Get_Follows(url, limit = 100): + # Get the page + print('GET: ' + url) + driver.get(url) + + # Loop until we reach the bottom of the page so all badges are loaded + soup = None + while True: + print('Scrolling...') + driver.execute_script('window.scrollTo(0, document.body.scrollHeight);') # Scroll to the bottom of the page + time.sleep(1) # Wait a second for the page to load + + # Soup what we have + soup = BeautifulSoup(driver.page_source, 'lxml') + loading_elem = soup.find(class_='loading') + track_elems = soup.find_all(class_='badgeList__item') + + # Check if there are more items or we have enough + if not loading_elem: + print('Reached end of page') + break + elif len(track_elems) >= limit: + print('Reached user limit') + break + + print(' Users:') + + users = {} + + badge_elems = soup.find_all(class_='badgeList__item') + for badge_elem in badge_elems: + username_elem = badge_elem.find('a', class_='userBadgeListItem__heading') + if username_elem: + user = User() + user.url_username = username_elem['href'].strip('/') + users[user.url_username] = user + + print(' ' + user.url_username) + + return users.values() + + user_dict = {} users_to_process = ['thomotron', 'lacheque', 'slynk', 'bossfightswe'] @@ -215,6 +260,14 @@ for user_str in users_to_process: # Get their likes user.likes = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes') + # Get who they follow and queue them up + user.following = Get_Follows('https://soundcloud.com/' + user.url_username + '/following') + #users_to_process.extend(user.following) + + # Do the same with who follows them + user.followers = Get_Follows('https://soundcloud.com/' + user.url_username + '/followers') + #users_to_process.extend(user.followers) + # Finally add the user to the dictionary user_dict[user.url_username] = user @@ -235,3 +288,9 @@ for _, user in user_dict.items(): print(' Likes:') for track in user.likes: print(' ' + str(track.title)) + print(' Followers:') + for follower in user.followers: + print(' ' + str(follower.url_username)) + print(' Following:') + for followed in user.following: + print(' ' + str(followed.url_username))