tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix issue where users' following lists would not be scraped

Browse Source 
Wrong URL ^^'
Also 'followed' was replaced with 'following' everywhere for clarity and consistency with the changes
This commit is contained in:
Thomas Wade 2018-09-29 12:39:37 +09:30
parent 2cc2016d86
commit 6ebaa47013
1 changed files with 11 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
scrape.py
View File

@ -7,6 +7,7 @@
CHROMEDRIVER_PATH = '/usr/bin/chromedriver' CHROMEDRIVER_PATH = '/usr/bin/chromedriver'
# The number of steps outward from each starting user the script will scrape. # The number of steps outward from each starting user the script will scrape.
# This works for both who the user follows and who follows the user.
# For example: # For example:
# User <-- Follower <-- Follower's Follower # User <-- Follower <-- Follower's Follower
# ^0 ^1 ^2 # ^0 ^1 ^2
@ -22,11 +23,11 @@ STARTING_USERS = ['thomotron']
# The below settings determine how many of each item will be collected per user. # The below settings determine how many of each item will be collected per user.
# Adjust these as you see fit. The higher they are, the larger the graph and the # Adjust these as you see fit. The higher they are, the larger the graph and the
# longer the scrape time. # longer the scrape time.
# Be careful when changing the followed and followers settings, these will, in # Be careful when changing the following and followers settings. These will, in
# combination with MAX_DEGREE, determine how long the scraping process will take. # combination with MAX_DEGREE, determine how long the scraping process will take.
USER_MAX_TRACKS = 100 USER_MAX_TRACKS = 100
USER_MAX_LIKES = 100 USER_MAX_LIKES = 100
USER_MAX_FOLLOWED = 25 USER_MAX_FOLLOWING = 25
USER_MAX_FOLLOWERS = 25 USER_MAX_FOLLOWERS = 25
##### Import all the things ################################# ##### Import all the things #################################
@ -327,11 +328,11 @@ while iterator <= MAX_DEGREE:
user.likes.append(like.url) user.likes.append(like.url)
# Get who they follow # Get who they follow
for followed in Get_Follows('https://soundcloud.com/' + user.url_username + '/followed', USER_MAX_FOLLOWED): for following in Get_Follows('https://soundcloud.com/' + user.url_username + '/following', USER_MAX_FOLLOWING):
if not followed.url_username in user_dict.keys(): if not following.url_username in user_dict.keys():
user_dict[followed.url_username] = followed user_dict[following.url_username] = following
user.followed.append(followed.url_username) user.following.append(following.url_username)
users_to_process_next.append(followed.url_username) users_to_process_next.append(following.url_username)
# Get who follows them # Get who follows them
for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', USER_MAX_FOLLOWERS): for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', USER_MAX_FOLLOWERS):
@ -422,11 +423,11 @@ for _, user in user_dict.items():
label='follows' label='follows'
) )
print(' Following:') print(' Following:')
for followed in user.following: for following in user.following:
print(' ' + str(followed)) print(' ' + str(following))
graph.add_edge( \ graph.add_edge( \
user.url_username, \ user.url_username, \
follower, \ following, \
label='follows' label='follows'
) )