From 7090b7f5287713ce6b998089de81421910ec5e02 Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Tue, 21 Apr 2020 18:30:54 +0930 Subject: [PATCH] Merge everything into one file Python is really loose in how it handles module naming, so it's easier if I just make a bundle of joy like this than separating everything out. --- Comment.py | 21 -- FAAPI.py | 68 ---- FAAPI/FAAPI.py | 461 +++++++++++++++++++++++++ FaveScraper.py => FAAPI/FaveScraper.py | 2 +- Submission.py | 232 ------------- User.py | 156 --------- 6 files changed, 462 insertions(+), 478 deletions(-) delete mode 100644 Comment.py delete mode 100644 FAAPI.py create mode 100644 FAAPI/FAAPI.py rename FaveScraper.py => FAAPI/FaveScraper.py (96%) delete mode 100644 Submission.py delete mode 100644 User.py diff --git a/Comment.py b/Comment.py deleted file mode 100644 index adc86dc..0000000 --- a/Comment.py +++ /dev/null @@ -1,21 +0,0 @@ -import datetime -from typing import List - -import User -import Submission - - -class Comment: - """ - A comment on a FurAffinity submission. - """ - - def __init__(self, submission: Submission, id: int): - self.id = id - self.author: User.User = None - self.timestamp: datetime = None - self.submission: Submission.Submission = submission - self.parent: Comment = None - self.replies: List[Comment] = None - self.hidden_by_page_owner = None - self.hidden_by_author = None diff --git a/FAAPI.py b/FAAPI.py deleted file mode 100644 index 90e822b..0000000 --- a/FAAPI.py +++ /dev/null @@ -1,68 +0,0 @@ -import requests -from bs4 import BeautifulSoup -from selenium import webdriver - -from User import User -from Submission import Submission -from Comment import Comment - -FA_BASE_URL = 'https://furaffinity.net' - - -class FAAPI: - """ - A basic FurAffinity API instance. - """ - - def __init__(self, use_webdriver: bool = True): - """ - Initialises a new FAAPI instance. - Beware that a Chrome instance is launched to bypass Cloudflare and log in during initialisation. Change use_webdriver to False to disable this. - """ - - if not use_webdriver: - # Can't do any actual API things, stop here - return - - # Try sign in to FA - self._driver = webdriver.Chrome() - self._driver.get(FA_BASE_URL + '/login') - - # Wait until our username is visible on the top banner - while not self._driver.find_elements_by_id('my-username'): - pass - - # Save the username of the account we've logged in as - self.username = self._driver.find_elements_by_css_selector('#my-username:not(.hideondesktop)')[0].text.strip() - - # Extract the cookies and stop the driver - self._cookies = self._driver.get_cookies() - self._driver.close() - - # Set up a Requests session with the cookies we just got - self._req = requests.session() - [self._req.cookies.set(cookie['name'], cookie['value']) for cookie in self._cookies] - - def get_user(self, username: str) -> User: - soup = self.get_soup('{}/user/{}/'.format(FA_BASE_URL, username)) - - try: - # Try extract the username, stripping the ~/∞/! in front - username = soup.select('div.username h2')[0].text.strip()[1:] - except IndexError: - # No username, give up - return None - - return User.new_or_cached(self, username) - - def get_soup(self, url: str) -> BeautifulSoup: - """ - Gets a webpage and wraps it in a BeautifulSoup instance. - :param url: Webpage URL - :return: Webpage wrapped in a BeautifulSoup instance - """ - res = self._req.get(url) - if res.status_code >= 400: - raise Exception( - 'Got status code {} when trying to get URL {}'.format(res.status_code, url)) - return BeautifulSoup(res.content, 'lxml') diff --git a/FAAPI/FAAPI.py b/FAAPI/FAAPI.py new file mode 100644 index 0000000..b2403ea --- /dev/null +++ b/FAAPI/FAAPI.py @@ -0,0 +1,461 @@ +from datetime import datetime +from typing import List + +import requests +from bs4 import BeautifulSoup +from selenium import webdriver + +FA_BASE_URL = 'https://furaffinity.net' + + +class Comment: + """ + A comment on a FurAffinity submission. + """ + + def __init__(self, submission, id: int): + self.id = id + self.author: User = None + self.timestamp: datetime = None + self.submission: Submission = submission + self.parent: Comment = None + self.replies: List[Comment] = None + self.hidden_by_page_owner = None + self.hidden_by_author = None + + +class Submission: + """ + A FurAffinity submission. + """ + + _submissioncache = {} + + def __init__(self, api, id: int): + self._api = api + self.id: int = id + self._title: str = None + self._author: User = None + self._description: str = None + self._tags: List[str] = None + self._timestamp: datetime = None + self._category: str = None + self._subcategory: str = None + self._theme: str = None + self._species: str = None + self._gender: str = None + self._faves: int = None + self._comments: List[Comment] = None + self._views: int = None + self._has_preview: bool = None + self._preview_url: str = None + self._download_url: str = None + self._faved: bool = None + self._fave_url: str = None + + @property + def url(self): + if self.id and self.id > 0: + return '{}/view/{}'.format(FA_BASE_URL, self.id) + else: + return None + + @property + def title(self): + if not self._title: + self._get_submission() + + return self._title + + @property + def author(self): + if not self._author: + self._get_submission() + + return self._author + + @property + def description(self): + if not self._description: + self._get_submission() + + return self._description + + @property + def tags(self): + if not self._tags: + self._get_submission() + + return self._tags + + @property + def timestamp(self): + if not self._timestamp: + self._get_submission() + + return self._timestamp + + @property + def category(self): + if not self._category: + self._get_submission() + + return self._category + + @property + def subcategory(self): + if not self._subcategory: + self._get_submission() + + return self._subcategory + + @property + def species(self): + if not self._species: + self._get_submission() + + return self._species + + @property + def gender(self): + if not self._gender: + self._get_submission() + + return self._gender + + @property + def faves(self): + if not self._faves: + self._get_submission() + + return self._faves + + @property + def comments(self): + if not self._comments: + self._get_submission() + + return self._comments + + @property + def views(self): + if not self._views: + self._get_submission() + + return self._views + + @property + def preview_url(self): + if (not self._preview_url and self._has_preview is True) or self._has_preview is None: + self._get_submission() + + return self._preview_url + + @property + def download_url(self): + if not self._download_url: + self._get_submission() + + return self._download_url + + @property + def faved(self): + if self._faved is None: + self._get_submission() + + return self._faved + + @staticmethod + def new_or_cached(api, id: int): + if id not in Submission._submissioncache: + Submission._submissioncache[id] = Submission(api, id) + + return Submission._submissioncache[id] + + def fave(self): + if self.faved is False: # Usage of the faved property is deliberate to ensure _get_submission() has run + self._toggle_fave() + + def unfave(self): + if self.faved is True: # Usage of the faved property is deliberate to ensure _get_submission() has run + self._toggle_fave() + + def _get_submission(self): + soup = self._api.get_soup('{}/view/{}'.format(FA_BASE_URL, self.id)) + + # Parse all the easy stuff + self._title = soup.select('.submission-title')[0].text.strip() + self._author = User.new_or_cached(self._api, soup.select('.submission-id-sub-container a')[0].text.strip()) + self._description = soup.select('.submission-description')[0].text.strip() + self._tags = [tag.text.strip() for tag in soup.select('.submission-sidebar .tags')] + self._timestamp = datetime.strptime(soup.select('.popup_date')[0].get('title'), '%b %d, %Y %H:%M %p') # e.g. Nov 26, 2019 02:47 PM + self._category = soup.select('.category-name')[0].text.strip() + self._subcategory = soup.select('.type-name')[0].text.strip() + self._species = soup.select('.info div:nth-of-type(2) span')[0].text.strip() + self._gender = soup.select('.info div:nth-of-type(3) span')[0].text.strip() + self._faves = int(soup.select('.favorites .font-large')[0].text.strip()) + self._views = int(soup.select('.views .font-large')[0].text.strip()) + self._download_url = 'https:' + soup.select('.download a')[0].get('href') + self._faved = '-' in soup.select('.fav')[0].text + self._fave_url = FA_BASE_URL + soup.select('.fav a')[0].get('href') + + # Parse preview + try: + self._preview_url = 'https:' + soup.select('#submissionImg')[0].get('data-preview-src') + except IndexError: + self._preview_url = None + finally: + self._has_preview = self._preview_url is not None + + # Parse comments + self._comments = [] + for comment_element in soup.select('.comment_container'): + # Parse all the easy stuff + comment = Comment(self, int(comment_element.select('.comment_anchor')[0].get('id')[4:])) # Skip the 'cid:' in the ID e.g. cid:142785397 + + # TODO: Comment ancestry + + # Toggle hidden status + if comment_element.select('.comment-deleted'): + # Deleted by its owner, set flags and skip remaining attributes + comment.hidden_by_author = True + comment.hidden_by_page_owner = False + + self._comments.append(comment) + continue + elif 'collapsed_height' in comment_element.get('class'): + # Deleted by the page owner, set flags and skip remaining attributes + comment.hidden_by_author = False + comment.hidden_by_page_owner = True + + self._comments.append(comment) + continue + else: + comment.hidden_by_author = False + comment.hidden_by_page_owner = False + + # Parse content from non-hidden comments + comment.author = User.new_or_cached(self._api, comment_element.select('.comment_username')[0].text.strip()) + comment.timestamp = datetime.fromtimestamp(int(comment_element.get('data-timestamp'))) # FIXME: Needs to account for timezone difference since these are server-local epochs. Seems to be hosted in New York's TZ + + self._comments.append(comment) + + def _toggle_fave(self): + soup = self._api.get_soup(self._fave_url) + + self._faved = '-' in soup.select('.fav')[0].text + self._fave_url = FA_BASE_URL + soup.select('.fav a')[0].get('href') + + def __str__(self): + return '{} - {} ({}/view/{})'.format(self.author, self.title, FA_BASE_URL, self.url) + + +class User: + """ + A FurAffinity user. + """ + + _usercache = {} + + def __init__(self, api, username: str): + self._api = api + self.username: str = username + self._gallery: List[Submission] = None + self._faves: List[Submission] = None + self._watching: List[User] = None + self._watchers: List[User] = None + + @property + def username_lower(self) -> str: + return self.username.lower() + + @property + def gallery(self) -> List[Submission]: + # Get the gallery if we don't have it already + if not self._gallery: + self._gallery = self._get_gallery() + + return self._gallery + + @property + def faves(self) -> List[Submission]: + # Get faves if we don't have them already + if not self._faves: + self._faves = self._get_faves() + + return self._faves + + @property + def watching(self): # -> List[User]: + # Get the list if we don't have it already + if not self._watching: + self._watching = self._get_watching() + + return self._watching + + @property + def watchers(self): # -> List[User]: + # Get the list if we don't have it already + if not self._watchers: + self._watchers = self._get_watchers() + + return self._watchers + + @staticmethod + def new_or_cached(api, username: str): # -> User: + if username not in User._usercache: + User._usercache[username] = User(api, username) + + return User._usercache[username] + + def _get_gallery(self) -> List[Submission]: + submissions = [] + next_page_url = '{}/gallery/{}/'.format(FA_BASE_URL, self.username) + while next_page_url: + soup = self._api.get_soup(next_page_url) + + # Try get the next page + try: + next_page_button = soup.select('.submission-list form button[type=submit]')[0] + next_page_url = FA_BASE_URL + next_page_button.parent().get('action') + except IndexError: + # No next page, null the url to break the loop + next_page_url = None + + # Get all the submissions on this page + for item in soup.select('.gallery figure'): + sub = Submission.new_or_cached(self._api, int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-' + submissions.append(sub) + + return submissions + + def _get_faves(self): + submissions = [] + next_page_url = '{}/favorites/{}/'.format(FA_BASE_URL, self.username) + while next_page_url: + soup = self._api.get_soup(next_page_url) + + # Try get the next page + try: + next_page_button = soup.select('.pagination a.button:last-child')[0] + next_page_url = FA_BASE_URL + next_page_button.get('href') + except IndexError: + # No next page, null the url to break the loop + next_page_url = None + + # Get all the submissions on this page + for item in soup.select('.gallery figure'): + sub = Submission.new_or_cached(self._api, int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-' + submissions.append(sub) + + return submissions + + def _get_watching(self): + watching = [] + next_page_url = '{}/watchlist/by/{}/'.format(FA_BASE_URL, self.username) + while True: + soup = self._api.get_soup(next_page_url) + + # Stop processing if there aren't any users left + users = soup.select('.watch-list .watch-list-items a') + if len(users) == 0: + break + + for item in users: + watching.append(User.new_or_cached(self._api, item.text.strip())) + + # Try get the next page + try: + next_page_button = soup.select('.floatright form button[type=submit]')[0] + next_page_url = FA_BASE_URL + next_page_button.parent().get('href') + except IndexError: + # No next page, break out + break + + return watching + + def _get_watchers(self): + watchers = [] + next_page_url = '{}/watchlist/to/{}/'.format(FA_BASE_URL, self.username) + while True: + soup = self._api.get_soup(next_page_url) + + # Stop processing if there aren't any users left + users = soup.select('.watch-list .watch-list-items a') + if len(users) == 0: + break + + for item in users: + watchers.append(User.new_or_cached(self._api, item.text.strip())) + + # Try get the next page + try: + next_page_button = soup.select('.floatright form button[type=submit]')[0] + next_page_url = FA_BASE_URL + next_page_button.parent().get('href') + except IndexError: + # No next page, break out + break + + return watchers + + def __str__(self): + return self.username + + +class FAAPI: + """ + A basic FurAffinity API instance. + """ + + def __init__(self, use_webdriver: bool = True): + """ + Initialises a new FAAPI instance. + Beware that a Chrome instance is launched to bypass Cloudflare and log in during initialisation. Change use_webdriver to False to disable this. + """ + + if not use_webdriver: + # Can't do any actual API things, stop here + return + + # Try sign in to FA + self._driver = webdriver.Chrome() + self._driver.get(FA_BASE_URL + '/login') + + # Wait until our username is visible on the top banner + while not self._driver.find_elements_by_id('my-username'): + pass + + # Save the username of the account we've logged in as + self.username = self._driver.find_elements_by_css_selector('#my-username:not(.hideondesktop)')[0].text.strip() + + # Extract the cookies and stop the driver + self._cookies = self._driver.get_cookies() + self._driver.close() + + # Set up a Requests session with the cookies we just got + self._req = requests.session() + [self._req.cookies.set(cookie['name'], cookie['value']) for cookie in self._cookies] + + def get_user(self, username: str) -> User: + soup = self.get_soup('{}/user/{}/'.format(FA_BASE_URL, username)) + + try: + # Try extract the username, stripping the ~/∞/! in front + username = soup.select('div.username h2')[0].text.strip()[1:] + except IndexError: + # No username, give up + return None + + return User.new_or_cached(self, username) + + def get_soup(self, url: str) -> BeautifulSoup: + """ + Gets a webpage and wraps it in a BeautifulSoup instance. + :param url: Webpage URL + :return: Webpage wrapped in a BeautifulSoup instance + """ + res = self._req.get(url) + if res.status_code >= 400: + raise Exception( + 'Got status code {} when trying to get URL {}'.format(res.status_code, url)) + return BeautifulSoup(res.content, 'lxml') diff --git a/FaveScraper.py b/FAAPI/FaveScraper.py similarity index 96% rename from FaveScraper.py rename to FAAPI/FaveScraper.py index 5b0dd0a..b046af8 100755 --- a/FaveScraper.py +++ b/FAAPI/FaveScraper.py @@ -4,7 +4,7 @@ import multiprocessing import requests -from FAAPI import FAAPI +from FAAPI.FAAPI import FAAPI def download(url: str): diff --git a/Submission.py b/Submission.py deleted file mode 100644 index 95ed6dd..0000000 --- a/Submission.py +++ /dev/null @@ -1,232 +0,0 @@ -import datetime -from typing import List - -import FAAPI -import Comment -import User - - -class Submission: - """ - A FurAffinity submission. - """ - - _submissioncache = {} - - def __init__(self, api, id: int): - self._api = api - self.id: int = id - self._title: str = None - self._author: User.User = None - self._description: str = None - self._tags: List[str] = None - self._timestamp: datetime = None - self._category: str = None - self._subcategory: str = None - self._theme: str = None - self._species: str = None - self._gender: str = None - self._faves: int = None - self._comments: List[Comment.Comment] = None - self._views: int = None - self._has_preview: bool = None - self._preview_url: str = None - self._download_url: str = None - self._faved: bool = None - self._fave_url: str = None - - @property - def url(self): - if self.id and self.id > 0: - return '{}/view/{}'.format(FAAPI.FA_BASE_URL, self.id) - else: - return None - - @property - def title(self): - if not self._title: - self._get_submission() - - return self._title - - @property - def author(self): - if not self._author: - self._get_submission() - - return self._author - - @property - def description(self): - if not self._description: - self._get_submission() - - return self._description - - @property - def tags(self): - if not self._tags: - self._get_submission() - - return self._tags - - @property - def timestamp(self): - if not self._timestamp: - self._get_submission() - - return self._timestamp - - @property - def category(self): - if not self._category: - self._get_submission() - - return self._category - - @property - def subcategory(self): - if not self._subcategory: - self._get_submission() - - return self._subcategory - - @property - def species(self): - if not self._species: - self._get_submission() - - return self._species - - @property - def gender(self): - if not self._gender: - self._get_submission() - - return self._gender - - @property - def faves(self): - if not self._faves: - self._get_submission() - - return self._faves - - @property - def comments(self): - if not self._comments: - self._get_submission() - - return self._comments - - @property - def views(self): - if not self._views: - self._get_submission() - - return self._views - - @property - def preview_url(self): - if (not self._preview_url and self._has_preview is True) or self._has_preview is None: - self._get_submission() - - return self._preview_url - - @property - def download_url(self): - if not self._download_url: - self._get_submission() - - return self._download_url - - @property - def faved(self): - if self._faved is None: - self._get_submission() - - return self._faved - - @staticmethod - def new_or_cached(api, id: int): - if id not in Submission._submissioncache: - Submission._submissioncache[id] = Submission(api, id) - - return Submission._submissioncache[id] - - def fave(self): - if self.faved is False: # Usage of the faved property is deliberate to ensure _get_submission() has run - self._toggle_fave() - - def unfave(self): - if self.faved is True: # Usage of the faved property is deliberate to ensure _get_submission() has run - self._toggle_fave() - - def _get_submission(self): - soup = self._api.get_soup('{}/view/{}'.format(FAAPI.FA_BASE_URL, self.id)) - - # Parse all the easy stuff - self._title = soup.select('.submission-title')[0].text.strip() - self._author = User.User.new_or_cached(self._api, soup.select('.submission-id-sub-container a')[0].text.strip()) - self._description = soup.select('.submission-description')[0].text.strip() - self._tags = [tag.text.strip() for tag in soup.select('.submission-sidebar .tags')] - self._timestamp = datetime.datetime.strptime(soup.select('.popup_date')[0].get('title'), '%b %d, %Y %H:%M %p') # e.g. Nov 26, 2019 02:47 PM - self._category = soup.select('.category-name')[0].text.strip() - self._subcategory = soup.select('.type-name')[0].text.strip() - self._species = soup.select('.info div:nth-of-type(2) span')[0].text.strip() - self._gender = soup.select('.info div:nth-of-type(3) span')[0].text.strip() - self._faves = int(soup.select('.favorites .font-large')[0].text.strip()) - self._views = int(soup.select('.views .font-large')[0].text.strip()) - self._download_url = 'https:' + soup.select('.download a')[0].get('href') - self._faved = '-' in soup.select('.fav')[0].text - self._fave_url = FAAPI.FA_BASE_URL + soup.select('.fav a')[0].get('href') - - # Parse preview - try: - self._preview_url = 'https:' + soup.select('#submissionImg')[0].get('data-preview-src') - except IndexError: - self._preview_url = None - finally: - self._has_preview = self._preview_url is not None - - # Parse comments - self._comments = [] - for comment_element in soup.select('.comment_container'): - # Parse all the easy stuff - comment = Comment.Comment(self, int(comment_element.select('.comment_anchor')[0].get('id')[4:])) # Skip the 'cid:' in the ID e.g. cid:142785397 - - # TODO: Comment ancestry - - # Toggle hidden status - if comment_element.select('.comment-deleted'): - # Deleted by its owner, set flags and skip remaining attributes - comment.hidden_by_author = True - comment.hidden_by_page_owner = False - - self._comments.append(comment) - continue - elif 'collapsed_height' in comment_element.get('class'): - # Deleted by the page owner, set flags and skip remaining attributes - comment.hidden_by_author = False - comment.hidden_by_page_owner = True - - self._comments.append(comment) - continue - else: - comment.hidden_by_author = False - comment.hidden_by_page_owner = False - - # Parse content from non-hidden comments - comment.author = User.User.new_or_cached(self._api, comment_element.select('.comment_username')[0].text.strip()) - comment.timestamp = datetime.datetime.fromtimestamp(int(comment_element.get('data-timestamp'))) # FIXME: Needs to account for timezone difference since these are server-local epochs. Seems to be hosted in New York's TZ - - self._comments.append(comment) - - def _toggle_fave(self): - soup = self._api.get_soup(self._fave_url) - - self._faved = '-' in soup.select('.fav')[0].text - self._fave_url = FAAPI.FA_BASE_URL + soup.select('.fav a')[0].get('href') - - def __str__(self): - return '{} - {} ({}/view/{})'.format(self.author, self.title, FAAPI.FA_BASE_URL, self.url) diff --git a/User.py b/User.py deleted file mode 100644 index a3e6f46..0000000 --- a/User.py +++ /dev/null @@ -1,156 +0,0 @@ -from typing import List, Dict - -import FAAPI -import Submission - - -class User: - """ - A FurAffinity user. - """ - - _usercache = {} - - def __init__(self, api, username: str): - self._api = api - self.username: str = username - self._gallery: List[Submission.Submission] = None - self._faves: List[Submission.Submission] = None - self._watching: List[User] = None - self._watchers: List[User] = None - - @property - def username_lower(self) -> str: - return self.username.lower() - - @property - def gallery(self) -> List[Submission.Submission]: - # Get the gallery if we don't have it already - if not self._gallery: - self._gallery = self._get_gallery() - - return self._gallery - - @property - def faves(self) -> List[Submission.Submission]: - # Get faves if we don't have them already - if not self._faves: - self._faves = self._get_faves() - - return self._faves - - @property - def watching(self): # -> List[User]: - # Get the list if we don't have it already - if not self._watching: - self._watching = self._get_watching() - - return self._watching - - @property - def watchers(self): # -> List[User]: - # Get the list if we don't have it already - if not self._watchers: - self._watchers = self._get_watchers() - - return self._watchers - - @staticmethod - def new_or_cached(api, username: str): # -> User: - if username not in User._usercache: - User._usercache[username] = User(api, username) - - return User._usercache[username] - - def _get_gallery(self) -> List[Submission.Submission]: - submissions = [] - next_page_url = '{}/gallery/{}/'.format(FAAPI.FA_BASE_URL, self.username) - while next_page_url: - soup = self._api.get_soup(next_page_url) - - # Try get the next page - try: - next_page_button = soup.select('.submission-list form button[type=submit]')[0] - next_page_url = FAAPI.FA_BASE_URL + next_page_button.parent().get('action') - except IndexError: - # No next page, null the url to break the loop - next_page_url = None - - # Get all the submissions on this page - for item in soup.select('.gallery figure'): - sub = Submission.Submission.new_or_cached(self._api, int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-' - submissions.append(sub) - - return submissions - - def _get_faves(self): - submissions = [] - next_page_url = '{}/favorites/{}/'.format(FAAPI.FA_BASE_URL, self.username) - while next_page_url: - soup = self._api.get_soup(next_page_url) - - # Try get the next page - try: - next_page_button = soup.select('.pagination a.button:last-child')[0] - next_page_url = FAAPI.FA_BASE_URL + next_page_button.get('href') - except IndexError: - # No next page, null the url to break the loop - next_page_url = None - - # Get all the submissions on this page - for item in soup.select('.gallery figure'): - sub = Submission.Submission.new_or_cached(self._api, int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-' - submissions.append(sub) - - return submissions - - def _get_watching(self): - watching = [] - next_page_url = '{}/watchlist/by/{}/'.format(FAAPI.FA_BASE_URL, self.username) - while True: - soup = self._api.get_soup(next_page_url) - - # Stop processing if there aren't any users left - users = soup.select('.watch-list .watch-list-items a') - if len(users) == 0: - break - - for item in users: - watching.append(User.new_or_cached(self._api, item.text.strip())) - - # Try get the next page - try: - next_page_button = soup.select('.floatright form button[type=submit]')[0] - next_page_url = FAAPI.FA_BASE_URL + next_page_button.parent().get('href') - except: - # No next page, break out - break - - return watching - - def _get_watchers(self): - watchers = [] - next_page_url = '{}/watchlist/to/{}/'.format(FAAPI.FA_BASE_URL, self.username) - while True: - soup = self._api.get_soup(next_page_url) - - # Stop processing if there aren't any users left - users = soup.select('.watch-list .watch-list-items a') - if len(users) == 0: - break - - for item in users: - watchers.append(User.new_or_cached(self._api, item.text.strip())) - - # Try get the next page - try: - next_page_button = soup.select('.floatright form button[type=submit]')[0] - next_page_url = FAAPI.FA_BASE_URL + next_page_button.parent().get('href') - except: - # No next page, break out - break - - return watchers - - def __str__(self): - return self.username