diff --git a/scrape.py b/scrape.py index 5403d96..0e9261e 100755 --- a/scrape.py +++ b/scrape.py @@ -19,6 +19,16 @@ MAX_DEGREE = 0 # on how high the max degree is set to (see above). STARTING_USERS = ['thomotron'] +# The below settings determine how many of each item will be collected per user. +# Adjust these as you see fit. The higher they are, the larger the graph and the +# longer the scrape time. +# Be careful when changing the followed and followers settings, these will, in +# combination with MAX_DEGREE, determine how long the scraping process will take. +USER_MAX_TRACKS = 100 +USER_MAX_LIKES = 100 +USER_MAX_FOLLOWED = 25 +USER_MAX_FOLLOWERS = 25 + ##### Import all the things ################################# import time @@ -300,26 +310,26 @@ for i in range(MAX_DEGREE + 1): Get_Basic_Info(user) # Get their tracks - for track in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', 10): + for track in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/tracks', USER_MAX_TRACKS): if not track.url in track_dict.keys(): track_dict[track.url] = track user.tracks.append(track.url) # Get their likes - for like in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', 10): + for like in Get_Tracks_Info('https://soundcloud.com/' + user.url_username + '/likes', USER_MAX_LIKES): if not like.url in track_dict.keys(): track_dict[like.url] = like user.likes.append(like.url) # Get who they follow - for followed in Get_Follows('https://soundcloud.com/' + user.url_username + '/followed', 10): + for followed in Get_Follows('https://soundcloud.com/' + user.url_username + '/followed', USER_MAX_FOLLOWED): if not followed.url_username in user_dict.keys(): user_dict[followed.url_username] = followed user.followed.append(followed.url_username) users_to_process_next.append(followed.url_username) # Get who follows them - for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', 10): + for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', USER_MAX_FOLLOWERS): if not follower.url_username in user_dict.keys(): user_dict[follower.url_username] = follower user.followers.append(follower.url_username)