Extract basic info scraping into function
This commit is contained in:
parent
e5b2e8861a
commit
30eebeef90
37
scrape.py
37
scrape.py
@ -47,22 +47,8 @@ class User:
|
|||||||
following_count = None
|
following_count = None
|
||||||
follower_count = None
|
follower_count = None
|
||||||
|
|
||||||
user_dict = {}
|
# Gets a user's username, track count, following count, follower count, and like count
|
||||||
users_to_process = ['thomotron', 'lacheque', 'slynk', 'bossfightswe']
|
def Get_Basic_Info(user):
|
||||||
|
|
||||||
for user_str in users_to_process:
|
|
||||||
print('Processing ' + user_str)
|
|
||||||
|
|
||||||
# Skip any users that have already been passed over to avoid infinite loops
|
|
||||||
if user_str in user_dict.keys():
|
|
||||||
print(' Already processed, skipping')
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Initialise our user object to store our values in
|
|
||||||
user = User()
|
|
||||||
user.url_username = user_str
|
|
||||||
print(' URL Username: ' + user.url_username)
|
|
||||||
|
|
||||||
# Get the user's profile page and soup it
|
# Get the user's profile page and soup it
|
||||||
print('GET: https://soundcloud.com/' + user.url_username)
|
print('GET: https://soundcloud.com/' + user.url_username)
|
||||||
driver.get('https://soundcloud.com/' + user.url_username)
|
driver.get('https://soundcloud.com/' + user.url_username)
|
||||||
@ -95,6 +81,25 @@ for user_str in users_to_process:
|
|||||||
user.like_count = like_count_elem.string.split(' ')[0]
|
user.like_count = like_count_elem.string.split(' ')[0]
|
||||||
print(' Likes: ' + str(user.like_count))
|
print(' Likes: ' + str(user.like_count))
|
||||||
|
|
||||||
|
user_dict = {}
|
||||||
|
users_to_process = ['thomotron', 'lacheque', 'slynk', 'bossfightswe']
|
||||||
|
|
||||||
|
for user_str in users_to_process:
|
||||||
|
print('Processing ' + user_str)
|
||||||
|
|
||||||
|
# Skip any users that have already been passed over to avoid infinite loops
|
||||||
|
if user_str in user_dict.keys():
|
||||||
|
print(' Already processed, skipping')
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Initialise our user object to store our values in
|
||||||
|
user = User()
|
||||||
|
user.url_username = user_str
|
||||||
|
print(' URL Username: ' + user.url_username)
|
||||||
|
|
||||||
|
# Get their basic info
|
||||||
|
Get_Basic_Info(user)
|
||||||
|
|
||||||
# Finally add the user to the dictionary
|
# Finally add the user to the dictionary
|
||||||
user_dict[user.url_username] = user
|
user_dict[user.url_username] = user
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user