From 6ebaa470136f5855c9ee81334235fb5dd9e9361c Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Sat, 29 Sep 2018 12:39:37 +0930 Subject: [PATCH] Fix issue where users' following lists would not be scraped Wrong URL ^^' Also 'followed' was replaced with 'following' everywhere for clarity and consistency with the changes --- scrape.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/scrape.py b/scrape.py index 0558afc..460b24e 100755 --- a/scrape.py +++ b/scrape.py @@ -7,6 +7,7 @@ CHROMEDRIVER_PATH = '/usr/bin/chromedriver' # The number of steps outward from each starting user the script will scrape. +# This works for both who the user follows and who follows the user. # For example: # User <-- Follower <-- Follower's Follower # ^0 ^1 ^2 @@ -22,11 +23,11 @@ STARTING_USERS = ['thomotron'] # The below settings determine how many of each item will be collected per user. # Adjust these as you see fit. The higher they are, the larger the graph and the # longer the scrape time. -# Be careful when changing the followed and followers settings, these will, in +# Be careful when changing the following and followers settings. These will, in # combination with MAX_DEGREE, determine how long the scraping process will take. USER_MAX_TRACKS = 100 USER_MAX_LIKES = 100 -USER_MAX_FOLLOWED = 25 +USER_MAX_FOLLOWING = 25 USER_MAX_FOLLOWERS = 25 ##### Import all the things ################################# @@ -327,11 +328,11 @@ while iterator <= MAX_DEGREE: user.likes.append(like.url) # Get who they follow - for followed in Get_Follows('https://soundcloud.com/' + user.url_username + '/followed', USER_MAX_FOLLOWED): - if not followed.url_username in user_dict.keys(): - user_dict[followed.url_username] = followed - user.followed.append(followed.url_username) - users_to_process_next.append(followed.url_username) + for following in Get_Follows('https://soundcloud.com/' + user.url_username + '/following', USER_MAX_FOLLOWING): + if not following.url_username in user_dict.keys(): + user_dict[following.url_username] = following + user.following.append(following.url_username) + users_to_process_next.append(following.url_username) # Get who follows them for follower in Get_Follows('https://soundcloud.com/' + user.url_username + '/followers', USER_MAX_FOLLOWERS): @@ -422,11 +423,11 @@ for _, user in user_dict.items(): label='follows' ) print(' Following:') - for followed in user.following: - print(' ' + str(followed)) + for following in user.following: + print(' ' + str(following)) graph.add_edge( \ user.url_username, \ - follower, \ + following, \ label='follows' )