89 lines
2.9 KiB
Python
Executable File
89 lines
2.9 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
# It is assumed that you have installed the required packages in requirements.txt
|
|
# and have also installed ChromeDriver from your package manager.
|
|
# If you have installed ChromeDriver separately, change the line below to the
|
|
# path of the ChromeDriver executable.
|
|
CHROMEDRIVER_PATH = '/usr/bin/chromedriver'
|
|
|
|
import networkx as nx
|
|
from selenium import webdriver
|
|
from selenium.webdriver.common.by import By
|
|
from selenium.webdriver.support.ui import WebDriverWait
|
|
from selenium.webdriver.support import expected_conditions
|
|
from selenium.webdriver.chrome.options import Options
|
|
from bs4 import BeautifulSoup
|
|
|
|
##### Set up NetworkX #######################################
|
|
|
|
graph = nx.DiGraph()
|
|
|
|
print('NetworkX initialised')
|
|
|
|
##### Set up WebDriver with Chrome ##########################
|
|
|
|
chrome_options = Options()
|
|
|
|
# Start headless so we can run on a server overnight
|
|
chrome_options.add_argument('--headless')
|
|
|
|
# Disable image loading, courtesy of rocky qi
|
|
# Found at https://stackoverflow.com/a/31581387, accessed on 2018/09/17 at 12:54 UTC
|
|
chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images":2})
|
|
|
|
driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options)
|
|
|
|
print('WebDriver initialised')
|
|
|
|
##### Do stuff ##############################################
|
|
|
|
# User class, holds all the data we will be collecting from user pages
|
|
# It's being used here as a more structured alternative to a dictionary
|
|
class User:
|
|
url_username = ''
|
|
username = ''
|
|
track_count = ''
|
|
like_count = ''
|
|
following_count = ''
|
|
follower_count = ''
|
|
|
|
user_dict = {}
|
|
users_to_process = ['thomotron', 'lacheque', 'slynk', 'bossfightswe']
|
|
|
|
for user_str in users_to_process:
|
|
# Skip any users that have already been passed over to avoid infinite loops
|
|
if user_str in user_dict.keys():
|
|
continue
|
|
|
|
# Initialise our user object to store our values in
|
|
user = User()
|
|
user.url_username = user_str
|
|
|
|
# Get the user's profile page and soup it
|
|
driver.get('https://soundcloud.com/' + user_str)
|
|
soup = BeautifulSoup(driver.page_source, 'lxml')
|
|
|
|
# Grab their username
|
|
# Use stripped_strings generator as workaround for users with premium badge
|
|
username = next(soup.find(class_='profileHeaderInfo__userName').stripped_strings)
|
|
user.username = username
|
|
|
|
# Grab their track count
|
|
track_count = soup.find('a', href='/' + user.url_username + '/tracks', class_='infoStats__statLink').div.string
|
|
user.track_count = track_count
|
|
|
|
# Grab their like count
|
|
like_count = soup.find('a', href='/' + user.url_username + '/likes').find(class_='sidebarHeader__actualTitle').string.split(' ')[0]
|
|
user.like_count = like_count
|
|
|
|
|
|
# Finally add the user to the dictionary
|
|
user_dict[user.url_username] = user
|
|
|
|
driver.quit()
|
|
|
|
for _, user in user_dict.items():
|
|
print(user.username)
|
|
print(' ' + user.track_count + ' tracks')
|
|
print(' ' + user.like_count + ' likes')
|