Scroll until limit is reached

scrape.py

@@ -6,6 +6,7 @@
 # path of the ChromeDriver executable.
 CHROMEDRIVER_PATH = '/usr/bin/chromedriver'
 
+import time
 import networkx as nx
 from selenium import webdriver
 from selenium.webdriver.common.by import By
@@ -96,25 +97,30 @@ def Get_Basic_Info(user):
         user.like_count = like_count_elem.string.split(' ')[0]
     print('    Likes: ' + str(user.like_count))
 
-def Get_Tracks_Info(url):
+def Get_Tracks_Info(url, limit = 100):
     # Get the page and soup it
     print('GET: ' + url)
     driver.get(url)
 
     # Loop until we reach the bottom of the page so all tracks are loaded
+    soup = None
     while True:
-        driver.execute_script('window.scrollTo(0, document.body.scrollHeight);')  # Scroll to the bottom of the page
-
-        try: # Wait for the infiniscroll to load new content and check for the footer element
-            wait = WebDriverWait(driver, 2) # Two second wait, assumes a good internet connection
         print('Scrolling...')
-            if wait.until(expected_conditions.presence_of_element_located((By.CLASS_NAME, 'paging-eof'))):
+        driver.execute_script('window.scrollTo(0, document.body.scrollHeight);')  # Scroll to the bottom of the page
+        time.sleep(1) # Wait a second for the page to load
+
+        # Soup what we have
+        soup = BeautifulSoup(driver.page_source, 'lxml')
+        loading_elem = soup.find(class_='loading')
+        track_elems = soup.find_all(class_='soundList__item')
+
+        # Check if there are more items or we have enough
+        if not loading_elem:
             print('Reached end of page')
             break
-        except:
+        elif len(track_elems) >= limit:
-            continue
+            print('Reached track limit')
-
+            break
-    soup = BeautifulSoup(driver.page_source, 'lxml')
 
     print('  Tracks:')