From 3879d437e2c40aa3a84c7fa1a8405b1b71f21d87 Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Tue, 18 Sep 2018 15:54:37 +0930 Subject: [PATCH] Store collected data in user_dict --- scrape.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scrape.py b/scrape.py index df40c72..72d81c9 100755 --- a/scrape.py +++ b/scrape.py @@ -40,22 +40,31 @@ print('WebDriver initialised') user_dict = {} users_to_process = ['thomotron', 'lacheque', 'slynk'] -for user in users: +for user_str in users_to_process: # Skip any users that have already been passed over to avoid infinite loops - if user in user_dict.keys(): + if user_str in user_dict.keys(): continue - driver.get('https://soundcloud.com/' + user) + driver.get('https://soundcloud.com/' + user_str) soup = BeautifulSoup(driver.page_source, 'lxml') + # Initialise our user object to store our values in + user = {} + # Use stripped_strings generator as workaround for users with premium badge username = next(soup.find(class_='profileHeaderInfo__userName').stripped_strings) + user['username'] = username print(username) track_count = int(soup.find('a', href='/' + username.lower() + '/tracks', class_='infoStats__statLink').div.string) + user['track_count'] = track_count print(track_count) like_count = int(soup.find('a', href='/' + username.lower() + '/likes').find(class_='sidebarHeader__actualTitle').string.split(' ')[0]) + user['track_count'] = track_count print(like_count) + # Finally add the user to the dictionary + user_dict['user_str'] = user + driver.quit()