tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Store collected data in user_dict

Browse Source
This commit is contained in:
Thomas Wade 2018-09-18 15:54:37 +09:30
parent c45391c436
commit 3879d437e2
1 changed files with 12 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
scrape.py
View File

@ -40,22 +40,31 @@ print('WebDriver initialised')
user_dict = {} user_dict = {}
users_to_process = ['thomotron', 'lacheque', 'slynk'] users_to_process = ['thomotron', 'lacheque', 'slynk']
for user in users: for user_str in users_to_process:
# Skip any users that have already been passed over to avoid infinite loops # Skip any users that have already been passed over to avoid infinite loops
if user in user_dict.keys(): if user_str in user_dict.keys():
continue continue
driver.get('https://soundcloud.com/' + user) driver.get('https://soundcloud.com/' + user_str)
soup = BeautifulSoup(driver.page_source, 'lxml') soup = BeautifulSoup(driver.page_source, 'lxml')
# Initialise our user object to store our values in
user = {}
# Use stripped_strings generator as workaround for users with premium badge # Use stripped_strings generator as workaround for users with premium badge
username = next(soup.find(class_='profileHeaderInfo__userName').stripped_strings) username = next(soup.find(class_='profileHeaderInfo__userName').stripped_strings)
user['username'] = username
print(username) print(username)
track_count = int(soup.find('a', href='/' + username.lower() + '/tracks', class_='infoStats__statLink').div.string) track_count = int(soup.find('a', href='/' + username.lower() + '/tracks', class_='infoStats__statLink').div.string)
user['track_count'] = track_count
print(track_count) print(track_count)
like_count = int(soup.find('a', href='/' + username.lower() + '/likes').find(class_='sidebarHeader__actualTitle').string.split(' ')[0]) like_count = int(soup.find('a', href='/' + username.lower() + '/likes').find(class_='sidebarHeader__actualTitle').string.split(' ')[0])
user['track_count'] = track_count
print(like_count) print(like_count)
# Finally add the user to the dictionary
user_dict['user_str'] = user
driver.quit() driver.quit()