diff --git a/scrape.py b/scrape.py index 096aad1..eea4681 100755 --- a/scrape.py +++ b/scrape.py @@ -220,56 +220,59 @@ def Get_Follows(url, limit = 100): print(' Users:') - users = {} + users = [] badge_elems = soup.find_all(class_='badgeList__item') for badge_elem in badge_elems: username_elem = badge_elem.find('a', class_='userBadgeListItem__heading') if username_elem: - user = User() - user.url_username = username_elem['href'].strip('/') - users[user.url_username] = user + url_username = username_elem['href'].strip('/') + users.append(url_username) - print(' ' + user.url_username) + print(' ' + url_username) - return users.values() + return users user_dict = {} -users_to_process = [{'distance':0, 'url_username':'thomotron'}, {'distance':0, 'url_username':'lacheque'}, {'distance':0, 'url_username':'slynk'}, {'distance':0, 'url_username':'bossfightswe'}] +users_to_process = [] +users_to_process_next = ['thomotron', 'lacheque', 'slynk', 'bossfightswe'] -for user_str in users_to_process: - print('Processing ' + user_str) +for i in range(2): + users_to_process = users_to_process_next + users_to_process_next = [] + for user_str in users_to_process: + print('Processing ' + user_str) - # Skip any users that have already been passed over to avoid infinite loops - if user_str in user_dict.keys(): - print(' Already processed, skipping') - continue + # Skip any users that have already been passed over to avoid infinite loops + if user_str in user_dict.keys(): + print(' Already processed, skipping') + continue - # Initialise our user object to store our values in - user = User() - user.url_username = user_str - print(' URL Username: ' + user.url_username) + # Initialise our user object to store our values in + user = User() + user.url_username = user_str + print(' URL Username: ' + user.url_username) - # Get their basic info - Get_Basic_Info(user) + # Get their basic info + Get_Basic_Info(user) - # Get their tracks - user.tracks = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks') + # Get their tracks + user.tracks = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', 10) - # Get their likes - user.likes = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes') + # Get their likes + user.likes = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', 10) - # Get who they follow and queue them up - user.following = Get_Follows('https://soundcloud.com/' + user.url_username + '/following') - #users_to_process.extend(user.following) + # Get who they follow + user.following = Get_Follows('https://soundcloud.com/' + user.url_username + '/following', 10) + users_to_process_next.extend(user.following) - # Do the same with who follows them - user.followers = Get_Follows('https://soundcloud.com/' + user.url_username + '/followers') - #users_to_process.extend(user.followers) + # Get who follows them + user.followers = Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', 10) + users_to_process_next.extend(user.followers) - # Finally add the user to the dictionary - user_dict[user.url_username] = user + # Finally add the user to the dictionary + user_dict[user.url_username] = user driver.quit()