tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Mark users as processed and check when printing

Browse Source
This commit is contained in:
Thomas Wade 2018-09-25 19:03:44 +09:30
parent 0c496ecd71
commit 7de170efc9
1 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scrape.py
View File

@ -54,6 +54,7 @@ class User:
followers = None followers = None
following = None following = None
processed = False
def __init__(self): def __init__(self):
self.tracks = [] self.tracks = []
@ -253,8 +254,9 @@ for i in range(2):
# Skip any users that have already been passed over to avoid infinite loops # Skip any users that have already been passed over to avoid infinite loops
if user_str in user_dict.keys(): if user_str in user_dict.keys():
print(' Already processed, skipping') if user_dict[user_str].processed:
continue print(' Already processed, skipping')
continue
# Initialise our user object to store our values in # Initialise our user object to store our values in
user = User() user = User()
@ -278,7 +280,8 @@ for i in range(2):
user.followers = Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', 10) user.followers = Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', 10)
users_to_process_next.extend(user.followers) users_to_process_next.extend(user.followers)
# Finally add the user to the dictionary # Finally add the user to the dictionary and mark them as processed
user.processed = True
user_dict[user.url_username] = user user_dict[user.url_username] = user
driver.quit() driver.quit()
@ -287,6 +290,9 @@ print('Done scraping, here\'s what we got')
print('==================================') print('==================================')
for _, user in user_dict.items(): for _, user in user_dict.items():
if not user.processed:
continue
print(user.username) print(user.username)
print(' ' + str(user.track_count) + ' tracks') print(' ' + str(user.track_count) + ' tracks')
print(' ' + str(user.like_count) + ' likes') print(' ' + str(user.like_count) + ' likes')