diff --git a/scrape.py b/scrape.py index 3aa5c57..350fe96 100755 --- a/scrape.py +++ b/scrape.py @@ -100,6 +100,20 @@ def Get_Tracks_Info(user): # Get the user's tracks page and soup it print('GET: https://soundcloud.com/' + user.url_username + '/tracks') driver.get('https://soundcloud.com/' + user.url_username + '/tracks') + + # Loop until we reach the bottom of the page so all tracks are loaded + while True: + driver.execute_script('window.scrollTo(0, document.body.scrollHeight);') # Scroll to the bottom of the page + + try: # Wait for the infiniscroll to load new content and check for the footer element + wait = WebDriverWait(driver, 2) # Two second wait, assumes a good internet connection + print('Scrolling...') + if wait.until(expected_conditions.presence_of_element_located((By.CLASS_NAME, 'paging-eof'))): + print('Reached end of page') + break + except: + continue + soup = BeautifulSoup(driver.page_source, 'lxml') print(' Tracks:')