tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Scroll until limit is reached

Browse Source
This commit is contained in:
Thomas Wade 2018-09-20 00:54:30 +09:30
parent 3decee9b03
commit aff01870b5
1 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
scrape.py
View File

@ -6,6 +6,7 @@
# path of the ChromeDriver executable. # path of the ChromeDriver executable.
CHROMEDRIVER_PATH = '/usr/bin/chromedriver' CHROMEDRIVER_PATH = '/usr/bin/chromedriver'
import time
import networkx as nx import networkx as nx
from selenium import webdriver from selenium import webdriver
from selenium.webdriver.common.by import By from selenium.webdriver.common.by import By
@ -96,25 +97,30 @@ def Get_Basic_Info(user):
user.like_count = like_count_elem.string.split(' ')[0] user.like_count = like_count_elem.string.split(' ')[0]
print(' Likes: ' + str(user.like_count)) print(' Likes: ' + str(user.like_count))
def Get_Tracks_Info(url): def Get_Tracks_Info(url, limit = 100):
# Get the page and soup it # Get the page and soup it
print('GET: ' + url) print('GET: ' + url)
driver.get(url) driver.get(url)
# Loop until we reach the bottom of the page so all tracks are loaded # Loop until we reach the bottom of the page so all tracks are loaded
soup = None
while True: while True:
print('Scrolling...')
driver.execute_script('window.scrollTo(0, document.body.scrollHeight);') # Scroll to the bottom of the page driver.execute_script('window.scrollTo(0, document.body.scrollHeight);') # Scroll to the bottom of the page
time.sleep(1) # Wait a second for the page to load
try: # Wait for the infiniscroll to load new content and check for the footer element # Soup what we have
wait = WebDriverWait(driver, 2) # Two second wait, assumes a good internet connection soup = BeautifulSoup(driver.page_source, 'lxml')
print('Scrolling...') loading_elem = soup.find(class_='loading')
if wait.until(expected_conditions.presence_of_element_located((By.CLASS_NAME, 'paging-eof'))): track_elems = soup.find_all(class_='soundList__item')
print('Reached end of page')
break
except:
continue
soup = BeautifulSoup(driver.page_source, 'lxml') # Check if there are more items or we have enough
if not loading_elem:
print('Reached end of page')
break
elif len(track_elems) >= limit:
print('Reached track limit')
break
print(' Tracks:') print(' Tracks:')