Add some configuration options near start of script
This commit is contained in:
parent
295e3e3d12
commit
829374fc6e
21
scrape.py
21
scrape.py
@ -6,6 +6,21 @@
|
|||||||
# path of the ChromeDriver executable.
|
# path of the ChromeDriver executable.
|
||||||
CHROMEDRIVER_PATH = '/usr/bin/chromedriver'
|
CHROMEDRIVER_PATH = '/usr/bin/chromedriver'
|
||||||
|
|
||||||
|
# The number of steps outward from each starting user the script will scrape.
|
||||||
|
# For example:
|
||||||
|
# User <-- Follower <-- Follower's Follower
|
||||||
|
# ^0 ^1 ^2
|
||||||
|
# User is zero hops away from itself, so it's degree is zero.
|
||||||
|
# Follower is one hop away from User, so it's degree is one, and so on.
|
||||||
|
MAX_DEGREE = 0
|
||||||
|
|
||||||
|
# The list of starting users. Can be as many or as few as you would like.
|
||||||
|
# Bear in mind that each additional user will greatly increase run time, depending
|
||||||
|
# on how high the max degree is set to (see above).
|
||||||
|
STARTING_USERS = ['thomotron']
|
||||||
|
|
||||||
|
##### Import all the things #################################
|
||||||
|
|
||||||
import time
|
import time
|
||||||
import networkx as nx
|
import networkx as nx
|
||||||
from selenium import webdriver
|
from selenium import webdriver
|
||||||
@ -247,13 +262,13 @@ user_dict = {}
|
|||||||
track_dict = {}
|
track_dict = {}
|
||||||
|
|
||||||
users_to_process = []
|
users_to_process = []
|
||||||
users_to_process_next = ['thomotron', 'lacheque', 'slynk', 'bossfightswe']
|
users_to_process_next = STARTING_USERS
|
||||||
|
|
||||||
for i in range(2):
|
for i in range(MAX_DEGREE + 1):
|
||||||
users_to_process = users_to_process_next
|
users_to_process = users_to_process_next
|
||||||
users_to_process_next = []
|
users_to_process_next = []
|
||||||
for user_str in users_to_process:
|
for user_str in users_to_process:
|
||||||
print('Processing ' + user_str)
|
print('Processing ' + user_str + ' (distance ' + str(i) + ')')
|
||||||
|
|
||||||
# Skip any users that have already been passed over to avoid infinite loops
|
# Skip any users that have already been passed over to avoid infinite loops
|
||||||
if user_str in user_dict.keys():
|
if user_str in user_dict.keys():
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user