soundcloud-scrape/scrape.py

#!/usr/bin/python3

# It is assumed that you have installed the required packages in requirements.txt
# and have also installed ChromeDriver from your package manager.
# If you have installed ChromeDriver separately, change the line below to the
# path of the ChromeDriver executable.
CHROMEDRIVER_PATH = '/usr/bin/chromedriver'

import networkx as nx
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup

##### Set up NetworkX #######################################

graph = nx.DiGraph()

print('NetworkX initialised')

##### Set up WebDriver with Chrome ##########################

chrome_options = Options()

# Start headless so we can run on a server overnight
chrome_options.add_argument('--headless')

# Disable image loading, courtesy of rocky qi
# Found at https://stackoverflow.com/a/31581387, accessed on 2018/09/17 at 12:54 UTC
chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images":2})

driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options)

print('WebDriver initialised')

##### Do stuff ##############################################

# User class, holds all the data we will be collecting from user pages
# It's being used here as a more structured alternative to a dictionary
class User:
    url_username = None
    username = None
    track_count = None
    like_count = None
    following_count = None
    follower_count = None

class Track:
    title = None
    artist = None
    date = None
    tag = None # We'll only use the first tag for now, saves opening each track

    plays = None
    likes = None
    reposts = None
    comments = None

    tracks = {}

# Gets a user's username, track count, following count, follower count, and like count
def Get_Basic_Info(user):
    # Get the user's profile page and soup it
    print('GET: https://soundcloud.com/' + user.url_username)
    driver.get('https://soundcloud.com/' + user.url_username)
    soup = BeautifulSoup(driver.page_source, 'lxml')

    # Grab their username
    # Use stripped_strings generator as workaround for users with premium badge
    username = next(soup.find(class_='profileHeaderInfo__userName').stripped_strings)
    user.username = username
    print('  Username: ' + user.username)

    # Grab their track count
    track_count = soup.find('a', href='/' + user.url_username + '/tracks', class_='infoStats__statLink').div.string
    user.track_count = track_count
    print('  Tracks: ' + user.track_count)

    # Grab their following count
    following_count = soup.find('a', href='/' + user.url_username + '/following', class_='infoStats__statLink').div.string
    user.following_count = following_count
    print('  Following: ' + str(user.following_count))

    # Grab their follower count
    follower_count = soup.find('a', href='/' + user.url_username + '/followers', class_='infoStats__statLink').div.string
    user.follower_count = follower_count
    print('  Followers: ' + str(user.follower_count))

    # Grab their like count
    like_count_elem = soup.find('a', href='/' + user.url_username + '/likes').find(class_='sidebarHeader__actualTitle')
    if like_count_elem:
        user.like_count = like_count_elem.string.split(' ')[0]
    print('  Likes: ' + str(user.like_count))

def Get_Tracks_Info(user):
    # Get the user's tracks page and soup it
    print('GET: https://soundcloud.com/' + user.url_username + '/tracks')
    driver.get('https://soundcloud.com/' + user.url_username + '/tracks')
    soup = BeautifulSoup(driver.page_source, 'lxml')

    tracks = {}

    track_elems = soup.find_all(class_='soundList__item')
    for track_elem in track_elems:
        # Initialise a track object to hold our data
        track = Track()

        # Grab the title
        title_elem = soup.find(class_='soundTitle__title')
        if title_elem:
            track.title = title_elem.span.string.strip()

        # Grab who uploaded it (usually the artist unless it's a label)
        artist_elem = soup.find(class_='soundTitle__usernameText')
        if artist_elem:
            track.artist = artist_elem.string.strip()

        # Check if we have already gotten this track before
        if track.artist + track.title in tracks.keys():
            continue

        # Grab when it was uploaded
        date_elem = soup.find(class_='soundTitle__uploadTime')
        if date_elem:
            track.date = date_elem.time['datetime']

        # Grab the first tag featured on the list item
        tag_elem = soup.find(class_='soundTitle__tagContent')
        if tag_elem:
            track.tag = tag_elem.string

        # Grab the play and comment counts (either of these may or may not be present)
        play_comment_elems = soup.find_all(class_='sc-ministats-item')
        if play_comment_elems:
            for elem in play_comment_elems:
                num, unit = elem['title'].split(' ')
                if unit == 'plays':
                    track.plays = num
                elif unit == 'comments':
                    track.comments = num

        # Grab the like count
        track.likes = soup.find(class_='sc-button-like').string.strip()

        # Grab the repost count
        track.reposts = soup.find(class_='sc-button-repost').string.strip()

        # Finally add the track to the dictionary
        tracks[track.artist + track.title] = track

    user.tracks = tracks

user_dict = {}
users_to_process = ['thomotron', 'lacheque', 'slynk', 'bossfightswe']

for user_str in users_to_process:
    print('Processing ' + user_str)

    # Skip any users that have already been passed over to avoid infinite loops
    if user_str in user_dict.keys():
        print('  Already processed, skipping')
        continue

    # Initialise our user object to store our values in
    user = User()
    user.url_username = user_str
    print('  URL Username: ' + user.url_username)

    # Get their basic info
    Get_Basic_Info(user)

    # Get their tracks
    Get_Tracks_Info(user)

    # Finally add the user to the dictionary
    user_dict[user.url_username] = user

driver.quit()

print('Done scraping, here\'s what we got')
print('==================================')

for _, user in user_dict.items():
    print(user.username)
    print('  ' + str(user.track_count) + ' tracks')
    print('  ' + str(user.like_count) + ' likes')
    print('  Following ' + str(user.following_count))
    print('  Followed by ' + str(user.follower_count))