From 30eebeef90740b55c198f23d76180f3bce182ca6 Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Tue, 18 Sep 2018 18:35:57 +0930 Subject: [PATCH] Extract basic info scraping into function --- scrape.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/scrape.py b/scrape.py index 83ff91c..4331853 100755 --- a/scrape.py +++ b/scrape.py @@ -47,22 +47,8 @@ class User: following_count = None follower_count = None -user_dict = {} -users_to_process = ['thomotron', 'lacheque', 'slynk', 'bossfightswe'] - -for user_str in users_to_process: - print('Processing ' + user_str) - - # Skip any users that have already been passed over to avoid infinite loops - if user_str in user_dict.keys(): - print(' Already processed, skipping') - continue - - # Initialise our user object to store our values in - user = User() - user.url_username = user_str - print(' URL Username: ' + user.url_username) - +# Gets a user's username, track count, following count, follower count, and like count +def Get_Basic_Info(user): # Get the user's profile page and soup it print('GET: https://soundcloud.com/' + user.url_username) driver.get('https://soundcloud.com/' + user.url_username) @@ -95,6 +81,25 @@ for user_str in users_to_process: user.like_count = like_count_elem.string.split(' ')[0] print(' Likes: ' + str(user.like_count)) +user_dict = {} +users_to_process = ['thomotron', 'lacheque', 'slynk', 'bossfightswe'] + +for user_str in users_to_process: + print('Processing ' + user_str) + + # Skip any users that have already been passed over to avoid infinite loops + if user_str in user_dict.keys(): + print(' Already processed, skipping') + continue + + # Initialise our user object to store our values in + user = User() + user.url_username = user_str + print(' URL Username: ' + user.url_username) + + # Get their basic info + Get_Basic_Info(user) + # Finally add the user to the dictionary user_dict[user.url_username] = user