tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Process follow(ers/ing) of each user

Browse Source
This commit is contained in:
Thomas Wade 2018-09-25 16:54:26 +09:30
parent fc1b2ca4cd
commit 8ef89e373b
1 changed files with 34 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
scrape.py
View File

@ -220,25 +220,28 @@ def Get_Follows(url, limit = 100):
print(' Users:') print(' Users:')
users = {} users = []
badge_elems = soup.find_all(class_='badgeList__item') badge_elems = soup.find_all(class_='badgeList__item')
for badge_elem in badge_elems: for badge_elem in badge_elems:
username_elem = badge_elem.find('a', class_='userBadgeListItem__heading') username_elem = badge_elem.find('a', class_='userBadgeListItem__heading')
if username_elem: if username_elem:
user = User() url_username = username_elem['href'].strip('/')
user.url_username = username_elem['href'].strip('/') users.append(url_username)
users[user.url_username] = user
print(' ' + user.url_username) print(' ' + url_username)
return users.values() return users
user_dict = {} user_dict = {}
users_to_process = [{'distance':0, 'url_username':'thomotron'}, {'distance':0, 'url_username':'lacheque'}, {'distance':0, 'url_username':'slynk'}, {'distance':0, 'url_username':'bossfightswe'}] users_to_process = []
users_to_process_next = ['thomotron', 'lacheque', 'slynk', 'bossfightswe']
for user_str in users_to_process: for i in range(2):
users_to_process = users_to_process_next
users_to_process_next = []
for user_str in users_to_process:
print('Processing ' + user_str) print('Processing ' + user_str)
# Skip any users that have already been passed over to avoid infinite loops # Skip any users that have already been passed over to avoid infinite loops
@ -255,18 +258,18 @@ for user_str in users_to_process:
Get_Basic_Info(user) Get_Basic_Info(user)
# Get their tracks # Get their tracks
user.tracks = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks') user.tracks = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', 10)
# Get their likes # Get their likes
user.likes = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes') user.likes = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', 10)
# Get who they follow and queue them up # Get who they follow
user.following = Get_Follows('https://soundcloud.com/' + user.url_username + '/following') user.following = Get_Follows('https://soundcloud.com/' + user.url_username + '/following', 10)
#users_to_process.extend(user.following) users_to_process_next.extend(user.following)
# Do the same with who follows them # Get who follows them
user.followers = Get_Follows('https://soundcloud.com/' + user.url_username + '/followers') user.followers = Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', 10)
#users_to_process.extend(user.followers) users_to_process_next.extend(user.followers)
# Finally add the user to the dictionary # Finally add the user to the dictionary
user_dict[user.url_username] = user user_dict[user.url_username] = user