From 0eedc671449685724e30f0a2deccfc9a341926c0 Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Tue, 18 Sep 2018 17:38:48 +0930 Subject: [PATCH] Use User class instead of dictionary --- scrape.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/scrape.py b/scrape.py index f3177f8..d4c9cc9 100755 --- a/scrape.py +++ b/scrape.py @@ -37,6 +37,16 @@ print('WebDriver initialised') ##### Do stuff ############################################## +# User class, holds all the data we will be collecting from user pages +# It's being used here as a more structured alternative to a dictionary +class User: + url_username = '' + username = '' + track_count = '' + like_count = '' + following_count = '' + follower_count = '' + user_dict = {} users_to_process = ['thomotron', 'lacheque', 'slynk'] @@ -45,26 +55,29 @@ for user_str in users_to_process: if user_str in user_dict.keys(): continue + # Initialise our user object to store our values in + user = User() + user.url_username = user_str + + # Get the user's profile page and soup it driver.get('https://soundcloud.com/' + user_str) soup = BeautifulSoup(driver.page_source, 'lxml') - # Initialise our user object to store our values in - user = {} - + # Grab their username # Use stripped_strings generator as workaround for users with premium badge username = next(soup.find(class_='profileHeaderInfo__userName').stripped_strings) - user['username'] = username - print(username) + user.username = username - track_count = int(soup.find('a', href='/' + username.lower() + '/tracks', class_='infoStats__statLink').div.string) - user['track_count'] = track_count - print(track_count) + # Grab their track count + track_count = soup.find('a', href='/' + user.url_username + '/tracks', class_='infoStats__statLink').div.string + user.track_count = track_count + + # Grab their like count + like_count = soup.find('a', href='/' + user.url_username + '/likes').find(class_='sidebarHeader__actualTitle').string.split(' ')[0] + user.like_count = like_count - like_count = int(soup.find('a', href='/' + username.lower() + '/likes').find(class_='sidebarHeader__actualTitle').string.split(' ')[0]) - user['like_count'] = like_count - print(like_count) # Finally add the user to the dictionary - user_dict['user_str'] = user + user_dict[user.url_username] = user driver.quit()