diff --git a/scrape.py b/scrape.py index 4331853..2b8f79d 100755 --- a/scrape.py +++ b/scrape.py @@ -47,6 +47,19 @@ class User: following_count = None follower_count = None +class Track: + title = None + artist = None + date = None + tag = None # We'll only use the first tag for now, saves opening each track + + plays = None + likes = None + reposts = None + comments = None + + tracks = {} + # Gets a user's username, track count, following count, follower count, and like count def Get_Basic_Info(user): # Get the user's profile page and soup it @@ -81,6 +94,64 @@ def Get_Basic_Info(user): user.like_count = like_count_elem.string.split(' ')[0] print(' Likes: ' + str(user.like_count)) +def Get_Tracks_Info(user): + # Get the user's tracks page and soup it + print('GET: https://soundcloud.com/' + user.url_username + '/tracks') + driver.get('https://soundcloud.com/' + user.url_username + '/tracks') + soup = BeautifulSoup(driver.page_source, 'lxml') + + tracks = {} + + track_elems = soup.find_all(class_='soundList__item') + for track_elem in track_elems: + # Initialise a track object to hold our data + track = Track() + + # Grab the title + title_elem = soup.find(class_='soundTitle__title') + if title_elem: + track.title = title_elem.span.string.strip() + + # Grab who uploaded it (usually the artist unless it's a label) + artist_elem = soup.find(class_='soundTitle__usernameText') + if artist_elem: + track.artist = artist_elem.string.strip() + + # Check if we have already gotten this track before + if track.artist + track.title in tracks.keys(): + continue + + # Grab when it was uploaded + date_elem = soup.find(class_='soundTitle__uploadTime') + if date_elem: + track.date = date_elem.time['datetime'] + + # Grab the first tag featured on the list item + tag_elem = soup.find(class_='soundTitle__tagContent') + if tag_elem: + track.tag = tag_elem.string + + # Grab the play and comment counts (either of these may or may not be present) + play_comment_elems = soup.find_all(class_='sc-ministats-item') + if play_comment_elems: + for elem in play_comment_elems: + num, unit = elem['title'].split(' ') + if unit == 'plays': + track.plays = num + elif unit == 'comments': + track.comments = num + + # Grab the like count + track.likes = soup.find(class_='sc-button-like').string.strip() + + # Grab the repost count + track.reposts = soup.find(class_='sc-button-repost').string.strip() + + # Finally add the track to the dictionary + tracks[track.artist + track.title] = track + + user.tracks = tracks + user_dict = {} users_to_process = ['thomotron', 'lacheque', 'slynk', 'bossfightswe'] @@ -100,6 +171,9 @@ for user_str in users_to_process: # Get their basic info Get_Basic_Info(user) + # Get their tracks + Get_Tracks_Info(user) + # Finally add the user to the dictionary user_dict[user.url_username] = user