tom/soundcloud-scrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add settings for max items per user

Browse Source
This commit is contained in:
Thomas Wade 2018-09-27 17:01:52 +09:30
parent f3d171a92b
commit babc4156b5
1 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
scrape.py
View File

@ -19,6 +19,16 @@ MAX_DEGREE = 0
# on how high the max degree is set to (see above). # on how high the max degree is set to (see above).
STARTING_USERS = ['thomotron'] STARTING_USERS = ['thomotron']
# The below settings determine how many of each item will be collected per user.
# Adjust these as you see fit. The higher they are, the larger the graph and the
# longer the scrape time.
# Be careful when changing the followed and followers settings, these will, in
# combination with MAX_DEGREE, determine how long the scraping process will take.
USER_MAX_TRACKS = 100
USER_MAX_LIKES = 100
USER_MAX_FOLLOWED = 25
USER_MAX_FOLLOWERS = 25
##### Import all the things ################################# ##### Import all the things #################################
import time import time
@ -300,26 +310,26 @@ for i in range(MAX_DEGREE + 1):
Get_Basic_Info(user) Get_Basic_Info(user)
# Get their tracks # Get their tracks
for track in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', 10): for track in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', USER_MAX_TRACKS):
if not track.url in track_dict.keys(): if not track.url in track_dict.keys():
track_dict[track.url] = track track_dict[track.url] = track
user.tracks.append(track.url) user.tracks.append(track.url)
# Get their likes # Get their likes
for like in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', 10): for like in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', USER_MAX_LIKES):
if not like.url in track_dict.keys(): if not like.url in track_dict.keys():
track_dict[like.url] = like track_dict[like.url] = like
user.likes.append(like.url) user.likes.append(like.url)
# Get who they follow # Get who they follow
for followed in Get_Follows('https://soundcloud.com/' + user.url_username + '/followed', 10): for followed in Get_Follows('https://soundcloud.com/' + user.url_username + '/followed', USER_MAX_FOLLOWED):
if not followed.url_username in user_dict.keys(): if not followed.url_username in user_dict.keys():
user_dict[followed.url_username] = followed user_dict[followed.url_username] = followed
user.followed.append(followed.url_username) user.followed.append(followed.url_username)
users_to_process_next.append(followed.url_username) users_to_process_next.append(followed.url_username)
# Get who follows them # Get who follows them
for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', 10): for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', USER_MAX_FOLLOWERS):
if not follower.url_username in user_dict.keys(): if not follower.url_username in user_dict.keys():
user_dict[follower.url_username] = follower user_dict[follower.url_username] = follower
user.followers.append(follower.url_username) user.followers.append(follower.url_username)