Merge everything into one file
Python is really loose in how it handles module naming, so it's easier if I just make a bundle of joy like this than separating everything out.
This commit is contained in:
Thomas Wade
parent
7a11a940d1
commit
7090b7f528
21
Comment.py
21
Comment.py
@ -1,21 +0,0 @@
|
||||
import datetime
|
||||
from typing import List
|
||||
|
||||
import User
|
||||
import Submission
|
||||
|
||||
|
||||
class Comment:
|
||||
"""
|
||||
A comment on a FurAffinity submission.
|
||||
"""
|
||||
|
||||
def __init__(self, submission: Submission, id: int):
|
||||
self.id = id
|
||||
self.author: User.User = None
|
||||
self.timestamp: datetime = None
|
||||
self.submission: Submission.Submission = submission
|
||||
self.parent: Comment = None
|
||||
self.replies: List[Comment] = None
|
||||
self.hidden_by_page_owner = None
|
||||
self.hidden_by_author = None
|
||||
68
FAAPI.py
68
FAAPI.py
@ -1,68 +0,0 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
|
||||
from User import User
|
||||
from Submission import Submission
|
||||
from Comment import Comment
|
||||
|
||||
FA_BASE_URL = 'https://furaffinity.net'
|
||||
|
||||
|
||||
class FAAPI:
|
||||
"""
|
||||
A basic FurAffinity API instance.
|
||||
"""
|
||||
|
||||
def __init__(self, use_webdriver: bool = True):
|
||||
"""
|
||||
Initialises a new FAAPI instance.
|
||||
Beware that a Chrome instance is launched to bypass Cloudflare and log in during initialisation. Change use_webdriver to False to disable this.
|
||||
"""
|
||||
|
||||
if not use_webdriver:
|
||||
# Can't do any actual API things, stop here
|
||||
return
|
||||
|
||||
# Try sign in to FA
|
||||
self._driver = webdriver.Chrome()
|
||||
self._driver.get(FA_BASE_URL + '/login')
|
||||
|
||||
# Wait until our username is visible on the top banner
|
||||
while not self._driver.find_elements_by_id('my-username'):
|
||||
pass
|
||||
|
||||
# Save the username of the account we've logged in as
|
||||
self.username = self._driver.find_elements_by_css_selector('#my-username:not(.hideondesktop)')[0].text.strip()
|
||||
|
||||
# Extract the cookies and stop the driver
|
||||
self._cookies = self._driver.get_cookies()
|
||||
self._driver.close()
|
||||
|
||||
# Set up a Requests session with the cookies we just got
|
||||
self._req = requests.session()
|
||||
[self._req.cookies.set(cookie['name'], cookie['value']) for cookie in self._cookies]
|
||||
|
||||
def get_user(self, username: str) -> User:
|
||||
soup = self.get_soup('{}/user/{}/'.format(FA_BASE_URL, username))
|
||||
|
||||
try:
|
||||
# Try extract the username, stripping the ~/∞/! in front
|
||||
username = soup.select('div.username h2')[0].text.strip()[1:]
|
||||
except IndexError:
|
||||
# No username, give up
|
||||
return None
|
||||
|
||||
return User.new_or_cached(self, username)
|
||||
|
||||
def get_soup(self, url: str) -> BeautifulSoup:
|
||||
"""
|
||||
Gets a webpage and wraps it in a BeautifulSoup instance.
|
||||
:param url: Webpage URL
|
||||
:return: Webpage wrapped in a BeautifulSoup instance
|
||||
"""
|
||||
res = self._req.get(url)
|
||||
if res.status_code >= 400:
|
||||
raise Exception(
|
||||
'Got status code {} when trying to get URL {}'.format(res.status_code, url))
|
||||
return BeautifulSoup(res.content, 'lxml')
|
||||
461
FAAPI/FAAPI.py
Normal file
461
FAAPI/FAAPI.py
Normal file
@ -0,0 +1,461 @@
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
|
||||
FA_BASE_URL = 'https://furaffinity.net'
|
||||
|
||||
|
||||
class Comment:
|
||||
"""
|
||||
A comment on a FurAffinity submission.
|
||||
"""
|
||||
|
||||
def __init__(self, submission, id: int):
|
||||
self.id = id
|
||||
self.author: User = None
|
||||
self.timestamp: datetime = None
|
||||
self.submission: Submission = submission
|
||||
self.parent: Comment = None
|
||||
self.replies: List[Comment] = None
|
||||
self.hidden_by_page_owner = None
|
||||
self.hidden_by_author = None
|
||||
|
||||
|
||||
class Submission:
|
||||
"""
|
||||
A FurAffinity submission.
|
||||
"""
|
||||
|
||||
_submissioncache = {}
|
||||
|
||||
def __init__(self, api, id: int):
|
||||
self._api = api
|
||||
self.id: int = id
|
||||
self._title: str = None
|
||||
self._author: User = None
|
||||
self._description: str = None
|
||||
self._tags: List[str] = None
|
||||
self._timestamp: datetime = None
|
||||
self._category: str = None
|
||||
self._subcategory: str = None
|
||||
self._theme: str = None
|
||||
self._species: str = None
|
||||
self._gender: str = None
|
||||
self._faves: int = None
|
||||
self._comments: List[Comment] = None
|
||||
self._views: int = None
|
||||
self._has_preview: bool = None
|
||||
self._preview_url: str = None
|
||||
self._download_url: str = None
|
||||
self._faved: bool = None
|
||||
self._fave_url: str = None
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
if self.id and self.id > 0:
|
||||
return '{}/view/{}'.format(FA_BASE_URL, self.id)
|
||||
else:
|
||||
return None
|
||||
|
||||
@property
|
||||
def title(self):
|
||||
if not self._title:
|
||||
self._get_submission()
|
||||
|
||||
return self._title
|
||||
|
||||
@property
|
||||
def author(self):
|
||||
if not self._author:
|
||||
self._get_submission()
|
||||
|
||||
return self._author
|
||||
|
||||
@property
|
||||
def description(self):
|
||||
if not self._description:
|
||||
self._get_submission()
|
||||
|
||||
return self._description
|
||||
|
||||
@property
|
||||
def tags(self):
|
||||
if not self._tags:
|
||||
self._get_submission()
|
||||
|
||||
return self._tags
|
||||
|
||||
@property
|
||||
def timestamp(self):
|
||||
if not self._timestamp:
|
||||
self._get_submission()
|
||||
|
||||
return self._timestamp
|
||||
|
||||
@property
|
||||
def category(self):
|
||||
if not self._category:
|
||||
self._get_submission()
|
||||
|
||||
return self._category
|
||||
|
||||
@property
|
||||
def subcategory(self):
|
||||
if not self._subcategory:
|
||||
self._get_submission()
|
||||
|
||||
return self._subcategory
|
||||
|
||||
@property
|
||||
def species(self):
|
||||
if not self._species:
|
||||
self._get_submission()
|
||||
|
||||
return self._species
|
||||
|
||||
@property
|
||||
def gender(self):
|
||||
if not self._gender:
|
||||
self._get_submission()
|
||||
|
||||
return self._gender
|
||||
|
||||
@property
|
||||
def faves(self):
|
||||
if not self._faves:
|
||||
self._get_submission()
|
||||
|
||||
return self._faves
|
||||
|
||||
@property
|
||||
def comments(self):
|
||||
if not self._comments:
|
||||
self._get_submission()
|
||||
|
||||
return self._comments
|
||||
|
||||
@property
|
||||
def views(self):
|
||||
if not self._views:
|
||||
self._get_submission()
|
||||
|
||||
return self._views
|
||||
|
||||
@property
|
||||
def preview_url(self):
|
||||
if (not self._preview_url and self._has_preview is True) or self._has_preview is None:
|
||||
self._get_submission()
|
||||
|
||||
return self._preview_url
|
||||
|
||||
@property
|
||||
def download_url(self):
|
||||
if not self._download_url:
|
||||
self._get_submission()
|
||||
|
||||
return self._download_url
|
||||
|
||||
@property
|
||||
def faved(self):
|
||||
if self._faved is None:
|
||||
self._get_submission()
|
||||
|
||||
return self._faved
|
||||
|
||||
@staticmethod
|
||||
def new_or_cached(api, id: int):
|
||||
if id not in Submission._submissioncache:
|
||||
Submission._submissioncache[id] = Submission(api, id)
|
||||
|
||||
return Submission._submissioncache[id]
|
||||
|
||||
def fave(self):
|
||||
if self.faved is False: # Usage of the faved property is deliberate to ensure _get_submission() has run
|
||||
self._toggle_fave()
|
||||
|
||||
def unfave(self):
|
||||
if self.faved is True: # Usage of the faved property is deliberate to ensure _get_submission() has run
|
||||
self._toggle_fave()
|
||||
|
||||
def _get_submission(self):
|
||||
soup = self._api.get_soup('{}/view/{}'.format(FA_BASE_URL, self.id))
|
||||
|
||||
# Parse all the easy stuff
|
||||
self._title = soup.select('.submission-title')[0].text.strip()
|
||||
self._author = User.new_or_cached(self._api, soup.select('.submission-id-sub-container a')[0].text.strip())
|
||||
self._description = soup.select('.submission-description')[0].text.strip()
|
||||
self._tags = [tag.text.strip() for tag in soup.select('.submission-sidebar .tags')]
|
||||
self._timestamp = datetime.strptime(soup.select('.popup_date')[0].get('title'), '%b %d, %Y %H:%M %p') # e.g. Nov 26, 2019 02:47 PM
|
||||
self._category = soup.select('.category-name')[0].text.strip()
|
||||
self._subcategory = soup.select('.type-name')[0].text.strip()
|
||||
self._species = soup.select('.info div:nth-of-type(2) span')[0].text.strip()
|
||||
self._gender = soup.select('.info div:nth-of-type(3) span')[0].text.strip()
|
||||
self._faves = int(soup.select('.favorites .font-large')[0].text.strip())
|
||||
self._views = int(soup.select('.views .font-large')[0].text.strip())
|
||||
self._download_url = 'https:' + soup.select('.download a')[0].get('href')
|
||||
self._faved = '-' in soup.select('.fav')[0].text
|
||||
self._fave_url = FA_BASE_URL + soup.select('.fav a')[0].get('href')
|
||||
|
||||
# Parse preview
|
||||
try:
|
||||
self._preview_url = 'https:' + soup.select('#submissionImg')[0].get('data-preview-src')
|
||||
except IndexError:
|
||||
self._preview_url = None
|
||||
finally:
|
||||
self._has_preview = self._preview_url is not None
|
||||
|
||||
# Parse comments
|
||||
self._comments = []
|
||||
for comment_element in soup.select('.comment_container'):
|
||||
# Parse all the easy stuff
|
||||
comment = Comment(self, int(comment_element.select('.comment_anchor')[0].get('id')[4:])) # Skip the 'cid:' in the ID e.g. cid:142785397
|
||||
|
||||
# TODO: Comment ancestry
|
||||
|
||||
# Toggle hidden status
|
||||
if comment_element.select('.comment-deleted'):
|
||||
# Deleted by its owner, set flags and skip remaining attributes
|
||||
comment.hidden_by_author = True
|
||||
comment.hidden_by_page_owner = False
|
||||
|
||||
self._comments.append(comment)
|
||||
continue
|
||||
elif 'collapsed_height' in comment_element.get('class'):
|
||||
# Deleted by the page owner, set flags and skip remaining attributes
|
||||
comment.hidden_by_author = False
|
||||
comment.hidden_by_page_owner = True
|
||||
|
||||
self._comments.append(comment)
|
||||
continue
|
||||
else:
|
||||
comment.hidden_by_author = False
|
||||
comment.hidden_by_page_owner = False
|
||||
|
||||
# Parse content from non-hidden comments
|
||||
comment.author = User.new_or_cached(self._api, comment_element.select('.comment_username')[0].text.strip())
|
||||
comment.timestamp = datetime.fromtimestamp(int(comment_element.get('data-timestamp'))) # FIXME: Needs to account for timezone difference since these are server-local epochs. Seems to be hosted in New York's TZ
|
||||
|
||||
self._comments.append(comment)
|
||||
|
||||
def _toggle_fave(self):
|
||||
soup = self._api.get_soup(self._fave_url)
|
||||
|
||||
self._faved = '-' in soup.select('.fav')[0].text
|
||||
self._fave_url = FA_BASE_URL + soup.select('.fav a')[0].get('href')
|
||||
|
||||
def __str__(self):
|
||||
return '{} - {} ({}/view/{})'.format(self.author, self.title, FA_BASE_URL, self.url)
|
||||
|
||||
|
||||
class User:
|
||||
"""
|
||||
A FurAffinity user.
|
||||
"""
|
||||
|
||||
_usercache = {}
|
||||
|
||||
def __init__(self, api, username: str):
|
||||
self._api = api
|
||||
self.username: str = username
|
||||
self._gallery: List[Submission] = None
|
||||
self._faves: List[Submission] = None
|
||||
self._watching: List[User] = None
|
||||
self._watchers: List[User] = None
|
||||
|
||||
@property
|
||||
def username_lower(self) -> str:
|
||||
return self.username.lower()
|
||||
|
||||
@property
|
||||
def gallery(self) -> List[Submission]:
|
||||
# Get the gallery if we don't have it already
|
||||
if not self._gallery:
|
||||
self._gallery = self._get_gallery()
|
||||
|
||||
return self._gallery
|
||||
|
||||
@property
|
||||
def faves(self) -> List[Submission]:
|
||||
# Get faves if we don't have them already
|
||||
if not self._faves:
|
||||
self._faves = self._get_faves()
|
||||
|
||||
return self._faves
|
||||
|
||||
@property
|
||||
def watching(self): # -> List[User]:
|
||||
# Get the list if we don't have it already
|
||||
if not self._watching:
|
||||
self._watching = self._get_watching()
|
||||
|
||||
return self._watching
|
||||
|
||||
@property
|
||||
def watchers(self): # -> List[User]:
|
||||
# Get the list if we don't have it already
|
||||
if not self._watchers:
|
||||
self._watchers = self._get_watchers()
|
||||
|
||||
return self._watchers
|
||||
|
||||
@staticmethod
|
||||
def new_or_cached(api, username: str): # -> User:
|
||||
if username not in User._usercache:
|
||||
User._usercache[username] = User(api, username)
|
||||
|
||||
return User._usercache[username]
|
||||
|
||||
def _get_gallery(self) -> List[Submission]:
|
||||
submissions = []
|
||||
next_page_url = '{}/gallery/{}/'.format(FA_BASE_URL, self.username)
|
||||
while next_page_url:
|
||||
soup = self._api.get_soup(next_page_url)
|
||||
|
||||
# Try get the next page
|
||||
try:
|
||||
next_page_button = soup.select('.submission-list form button[type=submit]')[0]
|
||||
next_page_url = FA_BASE_URL + next_page_button.parent().get('action')
|
||||
except IndexError:
|
||||
# No next page, null the url to break the loop
|
||||
next_page_url = None
|
||||
|
||||
# Get all the submissions on this page
|
||||
for item in soup.select('.gallery figure'):
|
||||
sub = Submission.new_or_cached(self._api, int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-'
|
||||
submissions.append(sub)
|
||||
|
||||
return submissions
|
||||
|
||||
def _get_faves(self):
|
||||
submissions = []
|
||||
next_page_url = '{}/favorites/{}/'.format(FA_BASE_URL, self.username)
|
||||
while next_page_url:
|
||||
soup = self._api.get_soup(next_page_url)
|
||||
|
||||
# Try get the next page
|
||||
try:
|
||||
next_page_button = soup.select('.pagination a.button:last-child')[0]
|
||||
next_page_url = FA_BASE_URL + next_page_button.get('href')
|
||||
except IndexError:
|
||||
# No next page, null the url to break the loop
|
||||
next_page_url = None
|
||||
|
||||
# Get all the submissions on this page
|
||||
for item in soup.select('.gallery figure'):
|
||||
sub = Submission.new_or_cached(self._api, int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-'
|
||||
submissions.append(sub)
|
||||
|
||||
return submissions
|
||||
|
||||
def _get_watching(self):
|
||||
watching = []
|
||||
next_page_url = '{}/watchlist/by/{}/'.format(FA_BASE_URL, self.username)
|
||||
while True:
|
||||
soup = self._api.get_soup(next_page_url)
|
||||
|
||||
# Stop processing if there aren't any users left
|
||||
users = soup.select('.watch-list .watch-list-items a')
|
||||
if len(users) == 0:
|
||||
break
|
||||
|
||||
for item in users:
|
||||
watching.append(User.new_or_cached(self._api, item.text.strip()))
|
||||
|
||||
# Try get the next page
|
||||
try:
|
||||
next_page_button = soup.select('.floatright form button[type=submit]')[0]
|
||||
next_page_url = FA_BASE_URL + next_page_button.parent().get('href')
|
||||
except IndexError:
|
||||
# No next page, break out
|
||||
break
|
||||
|
||||
return watching
|
||||
|
||||
def _get_watchers(self):
|
||||
watchers = []
|
||||
next_page_url = '{}/watchlist/to/{}/'.format(FA_BASE_URL, self.username)
|
||||
while True:
|
||||
soup = self._api.get_soup(next_page_url)
|
||||
|
||||
# Stop processing if there aren't any users left
|
||||
users = soup.select('.watch-list .watch-list-items a')
|
||||
if len(users) == 0:
|
||||
break
|
||||
|
||||
for item in users:
|
||||
watchers.append(User.new_or_cached(self._api, item.text.strip()))
|
||||
|
||||
# Try get the next page
|
||||
try:
|
||||
next_page_button = soup.select('.floatright form button[type=submit]')[0]
|
||||
next_page_url = FA_BASE_URL + next_page_button.parent().get('href')
|
||||
except IndexError:
|
||||
# No next page, break out
|
||||
break
|
||||
|
||||
return watchers
|
||||
|
||||
def __str__(self):
|
||||
return self.username
|
||||
|
||||
|
||||
class FAAPI:
|
||||
"""
|
||||
A basic FurAffinity API instance.
|
||||
"""
|
||||
|
||||
def __init__(self, use_webdriver: bool = True):
|
||||
"""
|
||||
Initialises a new FAAPI instance.
|
||||
Beware that a Chrome instance is launched to bypass Cloudflare and log in during initialisation. Change use_webdriver to False to disable this.
|
||||
"""
|
||||
|
||||
if not use_webdriver:
|
||||
# Can't do any actual API things, stop here
|
||||
return
|
||||
|
||||
# Try sign in to FA
|
||||
self._driver = webdriver.Chrome()
|
||||
self._driver.get(FA_BASE_URL + '/login')
|
||||
|
||||
# Wait until our username is visible on the top banner
|
||||
while not self._driver.find_elements_by_id('my-username'):
|
||||
pass
|
||||
|
||||
# Save the username of the account we've logged in as
|
||||
self.username = self._driver.find_elements_by_css_selector('#my-username:not(.hideondesktop)')[0].text.strip()
|
||||
|
||||
# Extract the cookies and stop the driver
|
||||
self._cookies = self._driver.get_cookies()
|
||||
self._driver.close()
|
||||
|
||||
# Set up a Requests session with the cookies we just got
|
||||
self._req = requests.session()
|
||||
[self._req.cookies.set(cookie['name'], cookie['value']) for cookie in self._cookies]
|
||||
|
||||
def get_user(self, username: str) -> User:
|
||||
soup = self.get_soup('{}/user/{}/'.format(FA_BASE_URL, username))
|
||||
|
||||
try:
|
||||
# Try extract the username, stripping the ~/∞/! in front
|
||||
username = soup.select('div.username h2')[0].text.strip()[1:]
|
||||
except IndexError:
|
||||
# No username, give up
|
||||
return None
|
||||
|
||||
return User.new_or_cached(self, username)
|
||||
|
||||
def get_soup(self, url: str) -> BeautifulSoup:
|
||||
"""
|
||||
Gets a webpage and wraps it in a BeautifulSoup instance.
|
||||
:param url: Webpage URL
|
||||
:return: Webpage wrapped in a BeautifulSoup instance
|
||||
"""
|
||||
res = self._req.get(url)
|
||||
if res.status_code >= 400:
|
||||
raise Exception(
|
||||
'Got status code {} when trying to get URL {}'.format(res.status_code, url))
|
||||
return BeautifulSoup(res.content, 'lxml')
|
||||
@ -4,7 +4,7 @@ import multiprocessing
|
||||
|
||||
import requests
|
||||
|
||||
from FAAPI import FAAPI
|
||||
from FAAPI.FAAPI import FAAPI
|
||||
|
||||
|
||||
def download(url: str):
|
||||
232
Submission.py
232
Submission.py
@ -1,232 +0,0 @@
|
||||
import datetime
|
||||
from typing import List
|
||||
|
||||
import FAAPI
|
||||
import Comment
|
||||
import User
|
||||
|
||||
|
||||
class Submission:
|
||||
"""
|
||||
A FurAffinity submission.
|
||||
"""
|
||||
|
||||
_submissioncache = {}
|
||||
|
||||
def __init__(self, api, id: int):
|
||||
self._api = api
|
||||
self.id: int = id
|
||||
self._title: str = None
|
||||
self._author: User.User = None
|
||||
self._description: str = None
|
||||
self._tags: List[str] = None
|
||||
self._timestamp: datetime = None
|
||||
self._category: str = None
|
||||
self._subcategory: str = None
|
||||
self._theme: str = None
|
||||
self._species: str = None
|
||||
self._gender: str = None
|
||||
self._faves: int = None
|
||||
self._comments: List[Comment.Comment] = None
|
||||
self._views: int = None
|
||||
self._has_preview: bool = None
|
||||
self._preview_url: str = None
|
||||
self._download_url: str = None
|
||||
self._faved: bool = None
|
||||
self._fave_url: str = None
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
if self.id and self.id > 0:
|
||||
return '{}/view/{}'.format(FAAPI.FA_BASE_URL, self.id)
|
||||
else:
|
||||
return None
|
||||
|
||||
@property
|
||||
def title(self):
|
||||
if not self._title:
|
||||
self._get_submission()
|
||||
|
||||
return self._title
|
||||
|
||||
@property
|
||||
def author(self):
|
||||
if not self._author:
|
||||
self._get_submission()
|
||||
|
||||
return self._author
|
||||
|
||||
@property
|
||||
def description(self):
|
||||
if not self._description:
|
||||
self._get_submission()
|
||||
|
||||
return self._description
|
||||
|
||||
@property
|
||||
def tags(self):
|
||||
if not self._tags:
|
||||
self._get_submission()
|
||||
|
||||
return self._tags
|
||||
|
||||
@property
|
||||
def timestamp(self):
|
||||
if not self._timestamp:
|
||||
self._get_submission()
|
||||
|
||||
return self._timestamp
|
||||
|
||||
@property
|
||||
def category(self):
|
||||
if not self._category:
|
||||
self._get_submission()
|
||||
|
||||
return self._category
|
||||
|
||||
@property
|
||||
def subcategory(self):
|
||||
if not self._subcategory:
|
||||
self._get_submission()
|
||||
|
||||
return self._subcategory
|
||||
|
||||
@property
|
||||
def species(self):
|
||||
if not self._species:
|
||||
self._get_submission()
|
||||
|
||||
return self._species
|
||||
|
||||
@property
|
||||
def gender(self):
|
||||
if not self._gender:
|
||||
self._get_submission()
|
||||
|
||||
return self._gender
|
||||
|
||||
@property
|
||||
def faves(self):
|
||||
if not self._faves:
|
||||
self._get_submission()
|
||||
|
||||
return self._faves
|
||||
|
||||
@property
|
||||
def comments(self):
|
||||
if not self._comments:
|
||||
self._get_submission()
|
||||
|
||||
return self._comments
|
||||
|
||||
@property
|
||||
def views(self):
|
||||
if not self._views:
|
||||
self._get_submission()
|
||||
|
||||
return self._views
|
||||
|
||||
@property
|
||||
def preview_url(self):
|
||||
if (not self._preview_url and self._has_preview is True) or self._has_preview is None:
|
||||
self._get_submission()
|
||||
|
||||
return self._preview_url
|
||||
|
||||
@property
|
||||
def download_url(self):
|
||||
if not self._download_url:
|
||||
self._get_submission()
|
||||
|
||||
return self._download_url
|
||||
|
||||
@property
|
||||
def faved(self):
|
||||
if self._faved is None:
|
||||
self._get_submission()
|
||||
|
||||
return self._faved
|
||||
|
||||
@staticmethod
|
||||
def new_or_cached(api, id: int):
|
||||
if id not in Submission._submissioncache:
|
||||
Submission._submissioncache[id] = Submission(api, id)
|
||||
|
||||
return Submission._submissioncache[id]
|
||||
|
||||
def fave(self):
|
||||
if self.faved is False: # Usage of the faved property is deliberate to ensure _get_submission() has run
|
||||
self._toggle_fave()
|
||||
|
||||
def unfave(self):
|
||||
if self.faved is True: # Usage of the faved property is deliberate to ensure _get_submission() has run
|
||||
self._toggle_fave()
|
||||
|
||||
def _get_submission(self):
|
||||
soup = self._api.get_soup('{}/view/{}'.format(FAAPI.FA_BASE_URL, self.id))
|
||||
|
||||
# Parse all the easy stuff
|
||||
self._title = soup.select('.submission-title')[0].text.strip()
|
||||
self._author = User.User.new_or_cached(self._api, soup.select('.submission-id-sub-container a')[0].text.strip())
|
||||
self._description = soup.select('.submission-description')[0].text.strip()
|
||||
self._tags = [tag.text.strip() for tag in soup.select('.submission-sidebar .tags')]
|
||||
self._timestamp = datetime.datetime.strptime(soup.select('.popup_date')[0].get('title'), '%b %d, %Y %H:%M %p') # e.g. Nov 26, 2019 02:47 PM
|
||||
self._category = soup.select('.category-name')[0].text.strip()
|
||||
self._subcategory = soup.select('.type-name')[0].text.strip()
|
||||
self._species = soup.select('.info div:nth-of-type(2) span')[0].text.strip()
|
||||
self._gender = soup.select('.info div:nth-of-type(3) span')[0].text.strip()
|
||||
self._faves = int(soup.select('.favorites .font-large')[0].text.strip())
|
||||
self._views = int(soup.select('.views .font-large')[0].text.strip())
|
||||
self._download_url = 'https:' + soup.select('.download a')[0].get('href')
|
||||
self._faved = '-' in soup.select('.fav')[0].text
|
||||
self._fave_url = FAAPI.FA_BASE_URL + soup.select('.fav a')[0].get('href')
|
||||
|
||||
# Parse preview
|
||||
try:
|
||||
self._preview_url = 'https:' + soup.select('#submissionImg')[0].get('data-preview-src')
|
||||
except IndexError:
|
||||
self._preview_url = None
|
||||
finally:
|
||||
self._has_preview = self._preview_url is not None
|
||||
|
||||
# Parse comments
|
||||
self._comments = []
|
||||
for comment_element in soup.select('.comment_container'):
|
||||
# Parse all the easy stuff
|
||||
comment = Comment.Comment(self, int(comment_element.select('.comment_anchor')[0].get('id')[4:])) # Skip the 'cid:' in the ID e.g. cid:142785397
|
||||
|
||||
# TODO: Comment ancestry
|
||||
|
||||
# Toggle hidden status
|
||||
if comment_element.select('.comment-deleted'):
|
||||
# Deleted by its owner, set flags and skip remaining attributes
|
||||
comment.hidden_by_author = True
|
||||
comment.hidden_by_page_owner = False
|
||||
|
||||
self._comments.append(comment)
|
||||
continue
|
||||
elif 'collapsed_height' in comment_element.get('class'):
|
||||
# Deleted by the page owner, set flags and skip remaining attributes
|
||||
comment.hidden_by_author = False
|
||||
comment.hidden_by_page_owner = True
|
||||
|
||||
self._comments.append(comment)
|
||||
continue
|
||||
else:
|
||||
comment.hidden_by_author = False
|
||||
comment.hidden_by_page_owner = False
|
||||
|
||||
# Parse content from non-hidden comments
|
||||
comment.author = User.User.new_or_cached(self._api, comment_element.select('.comment_username')[0].text.strip())
|
||||
comment.timestamp = datetime.datetime.fromtimestamp(int(comment_element.get('data-timestamp'))) # FIXME: Needs to account for timezone difference since these are server-local epochs. Seems to be hosted in New York's TZ
|
||||
|
||||
self._comments.append(comment)
|
||||
|
||||
def _toggle_fave(self):
|
||||
soup = self._api.get_soup(self._fave_url)
|
||||
|
||||
self._faved = '-' in soup.select('.fav')[0].text
|
||||
self._fave_url = FAAPI.FA_BASE_URL + soup.select('.fav a')[0].get('href')
|
||||
|
||||
def __str__(self):
|
||||
return '{} - {} ({}/view/{})'.format(self.author, self.title, FAAPI.FA_BASE_URL, self.url)
|
||||
156
User.py
156
User.py
@ -1,156 +0,0 @@
|
||||
from typing import List, Dict
|
||||
|
||||
import FAAPI
|
||||
import Submission
|
||||
|
||||
|
||||
class User:
|
||||
"""
|
||||
A FurAffinity user.
|
||||
"""
|
||||
|
||||
_usercache = {}
|
||||
|
||||
def __init__(self, api, username: str):
|
||||
self._api = api
|
||||
self.username: str = username
|
||||
self._gallery: List[Submission.Submission] = None
|
||||
self._faves: List[Submission.Submission] = None
|
||||
self._watching: List[User] = None
|
||||
self._watchers: List[User] = None
|
||||
|
||||
@property
|
||||
def username_lower(self) -> str:
|
||||
return self.username.lower()
|
||||
|
||||
@property
|
||||
def gallery(self) -> List[Submission.Submission]:
|
||||
# Get the gallery if we don't have it already
|
||||
if not self._gallery:
|
||||
self._gallery = self._get_gallery()
|
||||
|
||||
return self._gallery
|
||||
|
||||
@property
|
||||
def faves(self) -> List[Submission.Submission]:
|
||||
# Get faves if we don't have them already
|
||||
if not self._faves:
|
||||
self._faves = self._get_faves()
|
||||
|
||||
return self._faves
|
||||
|
||||
@property
|
||||
def watching(self): # -> List[User]:
|
||||
# Get the list if we don't have it already
|
||||
if not self._watching:
|
||||
self._watching = self._get_watching()
|
||||
|
||||
return self._watching
|
||||
|
||||
@property
|
||||
def watchers(self): # -> List[User]:
|
||||
# Get the list if we don't have it already
|
||||
if not self._watchers:
|
||||
self._watchers = self._get_watchers()
|
||||
|
||||
return self._watchers
|
||||
|
||||
@staticmethod
|
||||
def new_or_cached(api, username: str): # -> User:
|
||||
if username not in User._usercache:
|
||||
User._usercache[username] = User(api, username)
|
||||
|
||||
return User._usercache[username]
|
||||
|
||||
def _get_gallery(self) -> List[Submission.Submission]:
|
||||
submissions = []
|
||||
next_page_url = '{}/gallery/{}/'.format(FAAPI.FA_BASE_URL, self.username)
|
||||
while next_page_url:
|
||||
soup = self._api.get_soup(next_page_url)
|
||||
|
||||
# Try get the next page
|
||||
try:
|
||||
next_page_button = soup.select('.submission-list form button[type=submit]')[0]
|
||||
next_page_url = FAAPI.FA_BASE_URL + next_page_button.parent().get('action')
|
||||
except IndexError:
|
||||
# No next page, null the url to break the loop
|
||||
next_page_url = None
|
||||
|
||||
# Get all the submissions on this page
|
||||
for item in soup.select('.gallery figure'):
|
||||
sub = Submission.Submission.new_or_cached(self._api, int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-'
|
||||
submissions.append(sub)
|
||||
|
||||
return submissions
|
||||
|
||||
def _get_faves(self):
|
||||
submissions = []
|
||||
next_page_url = '{}/favorites/{}/'.format(FAAPI.FA_BASE_URL, self.username)
|
||||
while next_page_url:
|
||||
soup = self._api.get_soup(next_page_url)
|
||||
|
||||
# Try get the next page
|
||||
try:
|
||||
next_page_button = soup.select('.pagination a.button:last-child')[0]
|
||||
next_page_url = FAAPI.FA_BASE_URL + next_page_button.get('href')
|
||||
except IndexError:
|
||||
# No next page, null the url to break the loop
|
||||
next_page_url = None
|
||||
|
||||
# Get all the submissions on this page
|
||||
for item in soup.select('.gallery figure'):
|
||||
sub = Submission.Submission.new_or_cached(self._api, int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-'
|
||||
submissions.append(sub)
|
||||
|
||||
return submissions
|
||||
|
||||
def _get_watching(self):
|
||||
watching = []
|
||||
next_page_url = '{}/watchlist/by/{}/'.format(FAAPI.FA_BASE_URL, self.username)
|
||||
while True:
|
||||
soup = self._api.get_soup(next_page_url)
|
||||
|
||||
# Stop processing if there aren't any users left
|
||||
users = soup.select('.watch-list .watch-list-items a')
|
||||
if len(users) == 0:
|
||||
break
|
||||
|
||||
for item in users:
|
||||
watching.append(User.new_or_cached(self._api, item.text.strip()))
|
||||
|
||||
# Try get the next page
|
||||
try:
|
||||
next_page_button = soup.select('.floatright form button[type=submit]')[0]
|
||||
next_page_url = FAAPI.FA_BASE_URL + next_page_button.parent().get('href')
|
||||
except:
|
||||
# No next page, break out
|
||||
break
|
||||
|
||||
return watching
|
||||
|
||||
def _get_watchers(self):
|
||||
watchers = []
|
||||
next_page_url = '{}/watchlist/to/{}/'.format(FAAPI.FA_BASE_URL, self.username)
|
||||
while True:
|
||||
soup = self._api.get_soup(next_page_url)
|
||||
|
||||
# Stop processing if there aren't any users left
|
||||
users = soup.select('.watch-list .watch-list-items a')
|
||||
if len(users) == 0:
|
||||
break
|
||||
|
||||
for item in users:
|
||||
watchers.append(User.new_or_cached(self._api, item.text.strip()))
|
||||
|
||||
# Try get the next page
|
||||
try:
|
||||
next_page_button = soup.select('.floatright form button[type=submit]')[0]
|
||||
next_page_url = FAAPI.FA_BASE_URL + next_page_button.parent().get('href')
|
||||
except:
|
||||
# No next page, break out
|
||||
break
|
||||
|
||||
return watchers
|
||||
|
||||
def __str__(self):
|
||||
return self.username
|
||||
Loading…
x
Reference in New Issue
Block a user