From 0b21cae8b9f3d73ab7fd2da0c49d7996b891011d Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Tue, 21 Apr 2020 02:50:26 +0930 Subject: [PATCH] Add initial implementation --- .gitignore | 3 + Comment.py | 18 ++++++ FAAPI.py | 63 +++++++++++++++++++ Submission.py | 37 +++++++++++ User.py | 157 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 4 ++ 6 files changed, 282 insertions(+) create mode 100644 .gitignore create mode 100644 Comment.py create mode 100644 FAAPI.py create mode 100644 Submission.py create mode 100644 User.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e207f32 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +venv/ +__pycache__/ +.idea/ \ No newline at end of file diff --git a/Comment.py b/Comment.py new file mode 100644 index 0000000..ffcbec8 --- /dev/null +++ b/Comment.py @@ -0,0 +1,18 @@ +import datetime +from typing import List + +import User +import Submission + + +class Comment: + """ + A comment on a FurAffinity submission. + """ + + def __init__(self): + self.author: User.User = None + self.timestamp: datetime = None + self.submission: Submission.Submission = None + self.parent: Comment = None + self.replies: List[Comment] = None diff --git a/FAAPI.py b/FAAPI.py new file mode 100644 index 0000000..67772bb --- /dev/null +++ b/FAAPI.py @@ -0,0 +1,63 @@ +import requests +from bs4 import BeautifulSoup +from selenium import webdriver + +from User import User + +FA_BASE_URL = 'https://furaffinity.net' + + +class FAAPI: + """ + A basic FurAffinity API instance. + """ + + def __init__(self, use_webdriver: bool = True): + """ + Initialises a new FAAPI instance. + Beware that a Chrome instance is launched to bypass Cloudflare and log in during initialisation. Change use_webdriver to False to disable this. + """ + + if not use_webdriver: + # Can't do any actual API things, stop here + return + + # Try sign in to FA + self._driver = webdriver.Chrome() + self._driver.get(FA_BASE_URL + '/login') + + # Wait until our username is visible on the top banner + while not self._driver.find_elements_by_id('my-username'): + pass + + # Extract the cookies and stop the driver + self._cookies = self._driver.get_cookies() + self._driver.close() + + # Set up a Requests session with the cookies we just got + self._req = requests.session() + [self._req.cookies.set(cookie['name'], cookie['value']) for cookie in self._cookies] + + def get_user(self, username: str) -> User: + soup = self.get_soup('{}/user/{}/'.format(FA_BASE_URL, username)) + + try: + # Try extract the username, stripping the ~/∞/! in front + username = soup.select('div.username h2')[0].text.strip()[1:] + except IndexError: + # No username, give up + return None + + return User.new_or_cached(self, username) + + def get_soup(self, url: str) -> BeautifulSoup: + """ + Gets a webpage and wraps it in a BeautifulSoup instance. + :param url: Webpage URL + :return: Webpage wrapped in a BeautifulSoup instance + """ + res = self._req.get(url) + if res.status_code >= 400: + raise Exception( + 'Got status code {} when trying to get URL {}'.format(res.status_code, url)) + return BeautifulSoup(res.content) diff --git a/Submission.py b/Submission.py new file mode 100644 index 0000000..6907beb --- /dev/null +++ b/Submission.py @@ -0,0 +1,37 @@ +import datetime +from typing import List + +import FAAPI +import Comment +import User + + +class Submission: + """ + A FurAffinity submission. + """ + + def __init__(self, id: int): + self.id: int = id + self.title: str = None + self.author: User.User = None + self.description: str = None + self.tags: List[str] = None + self.timestamp: datetime = None + self.category: str = None + self.theme: str = None + self.species: str = None + self.gender: str = None + self.faves: int = None + self.comments: List[Comment.Comment] = None + self.views: int = None + + @property + def url(self): + if self.id and self.id > 0: + return '{}/view/{}'.format(FAAPI.FA_BASE_URL, self.id) + else: + return None + + def __str__(self): + return '{} - {} ({}/view/{})'.format(self.author, self.title, FAAPI.FA_BASE_URL, self.id) diff --git a/User.py b/User.py new file mode 100644 index 0000000..1a6f361 --- /dev/null +++ b/User.py @@ -0,0 +1,157 @@ +from typing import List, Dict + +import FAAPI +import Submission + + +class User: + """ + A FurAffinity user. + """ + + _usercache = {} + + def __init__(self, api, username: str): + self._api = api + self.username: str = username + self._gallery: List[Submission.Submission] = None + self._faves: List[Submission.Submission] = None + self._watching: List[User] = None + self._watchers: List[User] = None + + @property + def username_lower(self) -> str: + return self.username.lower() + + @property + def gallery(self) -> List[Submission.Submission]: + # Get the gallery if we don't have it already + if not self._gallery: + self._gallery = self._get_gallery() + + return self._gallery + + @property + def faves(self) -> List[Submission.Submission]: + # Get faves if we don't have them already + if not self._faves: + self._faves = self._get_faves() + + return self._faves + + @property + def watching(self): # -> List[User]: + # Get the list if we don't have it already + if not self._watching: + self._watching = self._get_watching() + + return self._watching + + @property + def watchers(self): # -> List[User]: + # Get the list if we don't have it already + if not self._watchers: + self._watchers = self._get_watchers() + + return self._watchers + + @staticmethod + def new_or_cached(api, username: str): # -> User: + if username in User._usercache: + return User._usercache[username] + + return User(api, username) + + def _get_gallery(self) -> List[Submission.Submission]: + submissions = [] + next_page_url = '{}/gallery/{}/'.format(FAAPI.FA_BASE_URL, self.username) + while next_page_url: + soup = self._api.get_soup(next_page_url) + + # Try get the next page + try: + next_page_button = soup.select('.submission-list form button[type=submit]')[0] + next_page_url = FAAPI.FA_BASE_URL + next_page_button.parent().get('action') + except: + # No next page, null the url to break the loop + next_page_url = None + + # Get all the submissions on this page + for item in soup.select('.gallery figure'): + sub = Submission.Submission(item.get('id')[4:]) # IDs look like 'sid-35908275', so we just skip the 'sid-' + sub.author = self + submissions.append(sub) + + return submissions + + def _get_faves(self): + submissions = [] + next_page_url = '{}/favorites/{}/'.format(FAAPI.FA_BASE_URL, self.username) + while next_page_url: + soup = self._api.get_soup(next_page_url) + + # Try get the next page + try: + next_page_button = soup.select('.pagination a.button:last-child')[0] + next_page_url = FAAPI.FA_BASE_URL + next_page_button.get('href') + except: + # No next page, null the url to break the loop + next_page_url = None + + # Get all the submissions on this page + for item in soup.select('.gallery figure'): + sub = Submission.Submission(item.get('id')[4:]) # IDs look like 'sid-35908275', so we just skip the 'sid-' + submissions.append(sub) + + return submissions + + def _get_watching(self): + watching = [] + next_page_url = '{}/watchlist/by/{}/'.format(FAAPI.FA_BASE_URL, self.username) + while True: + soup = self._api.get_soup(next_page_url) + + # Stop processing if there aren't any users left + users = soup.select('.watch-list .watch-list-items a') + if len(users) == 0: + break + + for item in users: + watching.append(User.new_or_cached(self._api, item.text.strip())) + + # Try get the next page + try: + next_page_button = soup.select('.floatright form button[type=submit]')[0] + next_page_url = FAAPI.FA_BASE_URL + next_page_button.parent().get('href') + except: + # No next page, break out + break + + return watching + + def _get_watchers(self): + watchers = [] + next_page_url = '{}/watchlist/to/{}/'.format(FAAPI.FA_BASE_URL, self.username) + while True: + soup = self._api.get_soup(next_page_url) + + # Stop processing if there aren't any users left + users = soup.select('.watch-list .watch-list-items a') + if len(users) == 0: + break + + for item in users: + watchers.append(User.new_or_cached(self._api, item.text.strip())) + + # Try get the next page + try: + next_page_button = soup.select('.floatright form button[type=submit]')[0] + next_page_url = FAAPI.FA_BASE_URL + next_page_button.parent().get('href') + except: + # No next page, break out + break + + return watchers + + def __str__(self): + return self.username diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7c27742 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +requests~=2.23.0 +bs4~=0.0.1 +beautifulsoup4~=4.9.0 +selenium~=3.141.0 \ No newline at end of file