diff --git a/scrape.py b/scrape.py index 65dda64..27cf0ce 100755 --- a/scrape.py +++ b/scrape.py @@ -54,6 +54,7 @@ class User: followers = None following = None + processed = False def __init__(self): self.tracks = [] @@ -253,8 +254,9 @@ for i in range(2): # Skip any users that have already been passed over to avoid infinite loops if user_str in user_dict.keys(): - print(' Already processed, skipping') - continue + if user_dict[user_str].processed: + print(' Already processed, skipping') + continue # Initialise our user object to store our values in user = User() @@ -278,7 +280,8 @@ for i in range(2): user.followers = Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', 10) users_to_process_next.extend(user.followers) - # Finally add the user to the dictionary + # Finally add the user to the dictionary and mark them as processed + user.processed = True user_dict[user.url_username] = user driver.quit() @@ -287,6 +290,9 @@ print('Done scraping, here\'s what we got') print('==================================') for _, user in user_dict.items(): + if not user.processed: + continue + print(user.username) print(' ' + str(user.track_count) + ' tracks') print(' ' + str(user.like_count) + ' likes')