Scrape users' follower and following lists

scrape.py

@@ -51,6 +51,9 @@ class User:
     tracks = []
     likes = []
 
+    followers = []
+    following = []
+
 class Track:
     title = None
     artist = None
@@ -190,6 +193,48 @@ def Get_Tracks_Info(url, limit = 100):
     # Strip off keys and return a track list
     return tracks.values()
 
+def Get_Follows(url, limit = 100):
+    # Get the page
+    print('GET: ' + url)
+    driver.get(url)
+
+    # Loop until we reach the bottom of the page so all badges are loaded
+    soup = None
+    while True:
+        print('Scrolling...')
+        driver.execute_script('window.scrollTo(0, document.body.scrollHeight);')  # Scroll to the bottom of the page
+        time.sleep(1) # Wait a second for the page to load
+
+        # Soup what we have
+        soup = BeautifulSoup(driver.page_source, 'lxml')
+        loading_elem = soup.find(class_='loading')
+        track_elems = soup.find_all(class_='badgeList__item')
+
+        # Check if there are more items or we have enough
+        if not loading_elem:
+            print('Reached end of page')
+            break
+        elif len(track_elems) >= limit:
+            print('Reached user limit')
+            break
+
+    print('  Users:')
+
+    users = {}
+
+    badge_elems = soup.find_all(class_='badgeList__item')
+    for badge_elem in badge_elems:
+        username_elem = badge_elem.find('a', class_='userBadgeListItem__heading')
+        if username_elem:
+            user = User()
+            user.url_username = username_elem['href'].strip('/')
+            users[user.url_username] = user
+
+            print('    ' + user.url_username)
+
+    return users.values()
+
+
 user_dict = {}
 users_to_process = ['thomotron', 'lacheque', 'slynk', 'bossfightswe']
 
@@ -215,6 +260,14 @@ for user_str in users_to_process:
     # Get their likes
     user.likes = Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes')
 
+    # Get who they follow and queue them up
+    user.following = Get_Follows('https://soundcloud.com/' + user.url_username + '/following')
+    #users_to_process.extend(user.following)
+
+    # Do the same with who follows them
+    user.followers = Get_Follows('https://soundcloud.com/' + user.url_username + '/followers')
+    #users_to_process.extend(user.followers)
+
     # Finally add the user to the dictionary
     user_dict[user.url_username] = user
 
@@ -235,3 +288,9 @@ for _, user in user_dict.items():
     print('  Likes:')
     for track in user.likes:
         print('    ' + str(track.title))
+    print('  Followers:')
+    for follower in user.followers:
+        print('    ' + str(follower.url_username))
+    print('  Following:')
+    for followed in user.following:
+        print('    ' + str(followed.url_username))