Add some configuration options near start of script
This commit is contained in:
parent
295e3e3d12
commit
829374fc6e
21
scrape.py
21
scrape.py
@ -6,6 +6,21 @@
|
||||
# path of the ChromeDriver executable.
|
||||
CHROMEDRIVER_PATH = '/usr/bin/chromedriver'
|
||||
|
||||
# The number of steps outward from each starting user the script will scrape.
|
||||
# For example:
|
||||
# User <-- Follower <-- Follower's Follower
|
||||
# ^0 ^1 ^2
|
||||
# User is zero hops away from itself, so it's degree is zero.
|
||||
# Follower is one hop away from User, so it's degree is one, and so on.
|
||||
MAX_DEGREE = 0
|
||||
|
||||
# The list of starting users. Can be as many or as few as you would like.
|
||||
# Bear in mind that each additional user will greatly increase run time, depending
|
||||
# on how high the max degree is set to (see above).
|
||||
STARTING_USERS = ['thomotron']
|
||||
|
||||
##### Import all the things #################################
|
||||
|
||||
import time
|
||||
import networkx as nx
|
||||
from selenium import webdriver
|
||||
@ -247,13 +262,13 @@ user_dict = {}
|
||||
track_dict = {}
|
||||
|
||||
users_to_process = []
|
||||
users_to_process_next = ['thomotron', 'lacheque', 'slynk', 'bossfightswe']
|
||||
users_to_process_next = STARTING_USERS
|
||||
|
||||
for i in range(2):
|
||||
for i in range(MAX_DEGREE + 1):
|
||||
users_to_process = users_to_process_next
|
||||
users_to_process_next = []
|
||||
for user_str in users_to_process:
|
||||
print('Processing ' + user_str)
|
||||
print('Processing ' + user_str + ' (distance ' + str(i) + ')')
|
||||
|
||||
# Skip any users that have already been passed over to avoid infinite loops
|
||||
if user_str in user_dict.keys():
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user