diff --git a/scrape.py b/scrape.py index 8427481..83ff91c 100755 --- a/scrape.py +++ b/scrape.py @@ -51,15 +51,20 @@ user_dict = {} users_to_process = ['thomotron', 'lacheque', 'slynk', 'bossfightswe'] for user_str in users_to_process: + print('Processing ' + user_str) + # Skip any users that have already been passed over to avoid infinite loops if user_str in user_dict.keys(): + print(' Already processed, skipping') continue # Initialise our user object to store our values in user = User() user.url_username = user_str + print(' URL Username: ' + user.url_username) # Get the user's profile page and soup it + print('GET: https://soundcloud.com/' + user.url_username) driver.get('https://soundcloud.com/' + user.url_username) soup = BeautifulSoup(driver.page_source, 'lxml') @@ -67,29 +72,37 @@ for user_str in users_to_process: # Use stripped_strings generator as workaround for users with premium badge username = next(soup.find(class_='profileHeaderInfo__userName').stripped_strings) user.username = username + print(' Username: ' + user.username) # Grab their track count track_count = soup.find('a', href='/' + user.url_username + '/tracks', class_='infoStats__statLink').div.string user.track_count = track_count + print(' Tracks: ' + user.track_count) # Grab their following count following_count = soup.find('a', href='/' + user.url_username + '/following', class_='infoStats__statLink').div.string user.following_count = following_count + print(' Following: ' + str(user.following_count)) # Grab their follower count follower_count = soup.find('a', href='/' + user.url_username + '/followers', class_='infoStats__statLink').div.string user.follower_count = follower_count + print(' Followers: ' + str(user.follower_count)) # Grab their like count like_count_elem = soup.find('a', href='/' + user.url_username + '/likes').find(class_='sidebarHeader__actualTitle') if like_count_elem: user.like_count = like_count_elem.string.split(' ')[0] + print(' Likes: ' + str(user.like_count)) # Finally add the user to the dictionary user_dict[user.url_username] = user driver.quit() +print('Done scraping, here\'s what we got') +print('==================================') + for _, user in user_dict.items(): print(user.username) print(' ' + str(user.track_count) + ' tracks')