From aff01870b58597379bbff2d605fb03e3a10b1e7e Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Thu, 20 Sep 2018 00:54:30 +0930 Subject: [PATCH] Scroll until limit is reached --- scrape.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/scrape.py b/scrape.py index 81cd18c..0e174f1 100755 --- a/scrape.py +++ b/scrape.py @@ -6,6 +6,7 @@ # path of the ChromeDriver executable. CHROMEDRIVER_PATH = '/usr/bin/chromedriver' +import time import networkx as nx from selenium import webdriver from selenium.webdriver.common.by import By @@ -96,25 +97,30 @@ def Get_Basic_Info(user): user.like_count = like_count_elem.string.split(' ')[0] print(' Likes: ' + str(user.like_count)) -def Get_Tracks_Info(url): +def Get_Tracks_Info(url, limit = 100): # Get the page and soup it print('GET: ' + url) driver.get(url) # Loop until we reach the bottom of the page so all tracks are loaded + soup = None while True: + print('Scrolling...') driver.execute_script('window.scrollTo(0, document.body.scrollHeight);') # Scroll to the bottom of the page + time.sleep(1) # Wait a second for the page to load - try: # Wait for the infiniscroll to load new content and check for the footer element - wait = WebDriverWait(driver, 2) # Two second wait, assumes a good internet connection - print('Scrolling...') - if wait.until(expected_conditions.presence_of_element_located((By.CLASS_NAME, 'paging-eof'))): - print('Reached end of page') - break - except: - continue + # Soup what we have + soup = BeautifulSoup(driver.page_source, 'lxml') + loading_elem = soup.find(class_='loading') + track_elems = soup.find_all(class_='soundList__item') - soup = BeautifulSoup(driver.page_source, 'lxml') + # Check if there are more items or we have enough + if not loading_elem: + print('Reached end of page') + break + elif len(track_elems) >= limit: + print('Reached track limit') + break print(' Tracks:')