From 829374fc6eb629f9e49dd2396d49c6b7aaa5b4b9 Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Tue, 25 Sep 2018 19:37:55 +0930 Subject: [PATCH] Add some configuration options near start of script --- scrape.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scrape.py b/scrape.py index 27472f5..ae7e650 100755 --- a/scrape.py +++ b/scrape.py @@ -6,6 +6,21 @@ # path of the ChromeDriver executable. CHROMEDRIVER_PATH = '/usr/bin/chromedriver' +# The number of steps outward from each starting user the script will scrape. +# For example: +# User <-- Follower <-- Follower's Follower +# ^0 ^1 ^2 +# User is zero hops away from itself, so it's degree is zero. +# Follower is one hop away from User, so it's degree is one, and so on. +MAX_DEGREE = 0 + +# The list of starting users. Can be as many or as few as you would like. +# Bear in mind that each additional user will greatly increase run time, depending +# on how high the max degree is set to (see above). +STARTING_USERS = ['thomotron'] + +##### Import all the things ################################# + import time import networkx as nx from selenium import webdriver @@ -247,13 +262,13 @@ user_dict = {} track_dict = {} users_to_process = [] -users_to_process_next = ['thomotron', 'lacheque', 'slynk', 'bossfightswe'] +users_to_process_next = STARTING_USERS -for i in range(2): +for i in range(MAX_DEGREE + 1): users_to_process = users_to_process_next users_to_process_next = [] for user_str in users_to_process: - print('Processing ' + user_str) + print('Processing ' + user_str + ' (distance ' + str(i) + ')') # Skip any users that have already been passed over to avoid infinite loops if user_str in user_dict.keys():