tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Scroll through tracks page

Browse Source
This commit is contained in:
Thomas Wade 2018-09-19 15:11:22 +09:30
parent 5781fab7bd
commit 8c8631df7c
1 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
scrape.py
View File

@ -100,6 +100,20 @@ def Get_Tracks_Info(user):
# Get the user's tracks page and soup it # Get the user's tracks page and soup it
print('GET: https://soundcloud.com/' + user.url_username + '/tracks') print('GET: https://soundcloud.com/' + user.url_username + '/tracks')
driver.get('https://soundcloud.com/' + user.url_username + '/tracks') driver.get('https://soundcloud.com/' + user.url_username + '/tracks')
# Loop until we reach the bottom of the page so all tracks are loaded
while True:
driver.execute_script('window.scrollTo(0, document.body.scrollHeight);') # Scroll to the bottom of the page
try: # Wait for the infiniscroll to load new content and check for the footer element
wait = WebDriverWait(driver, 2) # Two second wait, assumes a good internet connection
print('Scrolling...')
if wait.until(expected_conditions.presence_of_element_located((By.CLASS_NAME, 'paging-eof'))):
print('Reached end of page')
break
except:
continue
soup = BeautifulSoup(driver.page_source, 'lxml') soup = BeautifulSoup(driver.page_source, 'lxml')
print(' Tracks:') print(' Tracks:')