tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add argument to skip scraping and just spit out a graph

Browse Source
This commit is contained in:
Thomas Wade 2018-09-30 15:02:45 +09:30
parent 65a5a5c63b
commit 3588e5937f
1 changed files with 70 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
scrape.py
View File

@ -43,6 +43,16 @@ from selenium.webdriver.support import expected_conditions
from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from collections import Counter from collections import Counter
from argparse import ArgumentParser
##### Parse arguments #######################################
parser = ArgumentParser()
parser.add_argument('-d', '--dump', help='skip scraping and process whatever is shelved', action='store_true')
args = parser.parse_args()
if args.dump:
print('Dump option specified, will skip scraping')
##### Set up NetworkX ####################################### ##### Set up NetworkX #######################################
@ -52,18 +62,19 @@ print('NetworkX initialised')
##### Set up WebDriver with Chrome ########################## ##### Set up WebDriver with Chrome ##########################
chrome_options = Options() if not args.dump:
chrome_options = Options()
# Start headless so we can run on a server overnight # Start headless so we can run on a server overnight
chrome_options.add_argument('--headless') chrome_options.add_argument('--headless')
# Disable image loading, courtesy of rocky qi # Disable image loading, courtesy of rocky qi
# Found at https://stackoverflow.com/a/31581387, accessed on 2018/09/17 at 12:54 UTC # Found at https://stackoverflow.com/a/31581387, accessed on 2018/09/17 at 12:54 UTC
chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images":2}) chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images":2})
driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options) driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options)
print('WebDriver initialised') print('WebDriver initialised')
##### Define some classes to hold our data ################## ##### Define some classes to hold our data ##################
@ -343,8 +354,8 @@ with shelve.open('shelf.db') as shelf:
if 'iterator' in shelf.keys(): if 'iterator' in shelf.keys():
iterator = shelf['iterator'] iterator = shelf['iterator']
if not args.dump:
while iterator <= MAX_DEGREE: while iterator <= MAX_DEGREE:
for user_str in users_to_process: for user_str in users_to_process:
print('Processing ' + user_str + ' (distance ' + str(iterator) + ')') print('Processing ' + user_str + ' (distance ' + str(iterator) + ')')
@ -404,7 +415,7 @@ while iterator <= MAX_DEGREE:
users_to_process = users_to_process_next users_to_process = users_to_process_next
users_to_process_next = [] users_to_process_next = []
driver.quit() driver.quit()
##### Process what was scraped ############################## ##### Process what was scraped ##############################