tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Process follow(ers/ing) of each user

Browse Source
This commit is contained in:
Thomas Wade 2018-09-25 16:54:26 +09:30
parent fc1b2ca4cd
commit 8ef89e373b
1 changed files with 34 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
scrape.py
View File

@ -220,56 +220,59 @@ def Get_Follows(url, limit = 100):
print(' Users:')
users = {}
users = []
badge_elems = soup.find_all(class_='badgeList__item')
for badge_elem in badge_elems:
username_elem = badge_elem.find('a', class_='userBadgeListItem__heading')
if username_elem:
user = User()
user.url_username = username_elem['href'].strip('/')
users[user.url_username] = user
url_username = username_elem['href'].strip('/')
users.append(url_username)
print(' ' + user.url_username)
print(' ' + url_username)
return users.values()
return users
user_dict = {}
users_to_process = [{'distance':0, 'url_username':'thomotron'}, {'distance':0, 'url_username':'lacheque'}, {'distance':0, 'url_username':'slynk'}, {'distance':0, 'url_username':'bossfightswe'}]
users_to_process = []
users_to_process_next = ['thomotron', 'lacheque', 'slynk', 'bossfightswe']
for user_str in users_to_process:
print('Processing ' + user_str)
for i in range(2):
users_to_process = users_to_process_next
users_to_process_next = []
for user_str in users_to_process:
print('Processing ' + user_str)
# Skip any users that have already been passed over to avoid infinite loops
if user_str in user_dict.keys():
print(' Already processed, skipping')
continue
# Skip any users that have already been passed over to avoid infinite loops
if user_str in user_dict.keys():
print(' Already processed, skipping')
continue
# Initialise our user object to store our values in
user = User()
user.url_username = user_str
print(' URL Username: ' + user.url_username)
# Initialise our user object to store our values in
user = User()
user.url_username = user_str
print(' URL Username: ' + user.url_username)
# Get their basic info
Get_Basic_Info(user)
# Get their basic info
Get_Basic_Info(user)
# Get their tracks
user.tracks = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks')
# Get their tracks
user.tracks = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', 10)
# Get their likes
user.likes = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes')
# Get their likes
user.likes = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', 10)
# Get who they follow and queue them up
user.following = Get_Follows('https://soundcloud.com/' + user.url_username + '/following')
#users_to_process.extend(user.following)
# Get who they follow
user.following = Get_Follows('https://soundcloud.com/' + user.url_username + '/following', 10)
users_to_process_next.extend(user.following)
# Do the same with who follows them
user.followers = Get_Follows('https://soundcloud.com/' + user.url_username + '/followers')
#users_to_process.extend(user.followers)
# Get who follows them
user.followers = Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', 10)
users_to_process_next.extend(user.followers)
# Finally add the user to the dictionary
user_dict[user.url_username] = user
# Finally add the user to the dictionary
user_dict[user.url_username] = user
driver.quit()