Better cache handling
The root API instance will handle item caching to bring it closer to the external-facing side of things. This also means that caches are instance-based now, which may be seen as either a pro or con. Additionally, User and Submission classes will no longer serialise their API instances when pickling.
This commit is contained in:
Thomas Wade
parent
042f732c69
commit
d4fff8d61a
@ -29,8 +29,6 @@ class Submission:
|
||||
A FurAffinity submission.
|
||||
"""
|
||||
|
||||
_submissioncache = {}
|
||||
|
||||
def __init__(self, api, id: int):
|
||||
self._api = api
|
||||
self.id: int = id
|
||||
@ -173,13 +171,6 @@ class Submission:
|
||||
|
||||
return self._recommended
|
||||
|
||||
@staticmethod
|
||||
def new_or_cached(api, id: int):
|
||||
if id not in Submission._submissioncache:
|
||||
Submission._submissioncache[id] = Submission(api, id)
|
||||
|
||||
return Submission._submissioncache[id]
|
||||
|
||||
def fave(self):
|
||||
if self.faved is False: # Usage of the faved property is deliberate to ensure _get_submission() has run
|
||||
self._toggle_fave()
|
||||
@ -193,7 +184,7 @@ class Submission:
|
||||
|
||||
# Parse all the easy stuff
|
||||
self._title = soup.select('.submission-title')[0].text.strip()
|
||||
self._author = User.new_or_cached(self._api, soup.select('.submission-id-sub-container a')[0].text.strip())
|
||||
self._author = self._api.get_user(soup.select('.submission-id-sub-container a')[0].text.strip())
|
||||
self._description = soup.select('.submission-description')[0].text.strip()
|
||||
self._tags = [tag.text.strip() for tag in soup.select('.submission-sidebar .tags')]
|
||||
self._timestamp = datetime.strptime(soup.select('.popup_date')[0].get('title'), '%b %d, %Y %H:%M %p') # e.g. Nov 26, 2019 02:47 PM
|
||||
@ -243,7 +234,7 @@ class Submission:
|
||||
comment.hidden_by_page_owner = False
|
||||
|
||||
# Parse content from non-hidden comments
|
||||
comment.author = User.new_or_cached(self._api, comment_element.select('.comment_username')[0].text.strip())
|
||||
comment.author = self._api.get_user(comment_element.select('.comment_username')[0].text.strip())
|
||||
comment.timestamp = datetime.fromtimestamp(int(comment_element.get('data-timestamp'))) # FIXME: Needs to account for timezone difference since these are server-local epochs. Seems to be hosted in New York's TZ
|
||||
|
||||
self._comments.append(comment)
|
||||
@ -252,7 +243,7 @@ class Submission:
|
||||
self._recommended = []
|
||||
for recommended in soup.select('.preview-gallery-container a'):
|
||||
id = int(recommended.get('href')[6:-1]) # e.g. /view/35992314/
|
||||
self._recommended.append(Submission.new_or_cached(self._api, id))
|
||||
self._recommended.append(self._api.get_submission(id))
|
||||
|
||||
def _toggle_fave(self):
|
||||
soup = self._api.get_soup(self._fave_url)
|
||||
@ -260,6 +251,18 @@ class Submission:
|
||||
self._faved = '-' in soup.select('.fav')[0].text
|
||||
self._fave_url = FA_BASE_URL + soup.select('.fav a')[0].get('href')
|
||||
|
||||
def __getstate__(self):
|
||||
# Get a copy of the object state without the API instance
|
||||
state = self.__dict__.copy()
|
||||
del (state['_api'])
|
||||
|
||||
return state
|
||||
|
||||
def __setstate__(self, state):
|
||||
# Set up the object with the given state
|
||||
self.__dict__.update(state)
|
||||
self._api = None
|
||||
|
||||
def __str__(self):
|
||||
return '{} - {} ({}/view/{})'.format(self.author, self.title, FA_BASE_URL, self.url)
|
||||
|
||||
@ -269,8 +272,6 @@ class User:
|
||||
A FurAffinity user.
|
||||
"""
|
||||
|
||||
_usercache = {}
|
||||
|
||||
def __init__(self, api, username: str):
|
||||
self._api = api
|
||||
self.username: str = username
|
||||
@ -315,13 +316,6 @@ class User:
|
||||
|
||||
return self._watchers
|
||||
|
||||
@staticmethod
|
||||
def new_or_cached(api, username: str): # -> User:
|
||||
if username not in User._usercache:
|
||||
User._usercache[username] = User(api, username)
|
||||
|
||||
return User._usercache[username]
|
||||
|
||||
def _get_gallery(self) -> List[Submission]:
|
||||
submissions = []
|
||||
next_page_url = '{}/gallery/{}/'.format(FA_BASE_URL, self.username)
|
||||
@ -338,7 +332,7 @@ class User:
|
||||
|
||||
# Get all the submissions on this page
|
||||
for item in soup.select('.gallery figure'):
|
||||
sub = Submission.new_or_cached(self._api, int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-'
|
||||
sub = self._api.get_submission(int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-'
|
||||
|
||||
# Try cache the preview URL early
|
||||
try:
|
||||
@ -367,7 +361,7 @@ class User:
|
||||
|
||||
# Get all the submissions on this page
|
||||
for item in soup.select('.gallery figure'):
|
||||
sub = Submission.new_or_cached(self._api, int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-'
|
||||
sub = self._api.get_submission(int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-'
|
||||
|
||||
# Try cache the preview URL early
|
||||
try:
|
||||
@ -392,7 +386,7 @@ class User:
|
||||
break
|
||||
|
||||
for item in users:
|
||||
watching.append(User.new_or_cached(self._api, item.text.strip()))
|
||||
watching.append(self._api.get_user(item.text.strip()))
|
||||
|
||||
# Try get the next page
|
||||
try:
|
||||
@ -416,7 +410,7 @@ class User:
|
||||
break
|
||||
|
||||
for item in users:
|
||||
watchers.append(User.new_or_cached(self._api, item.text.strip()))
|
||||
watchers.append(self._api.get_user(item.text.strip()))
|
||||
|
||||
# Try get the next page
|
||||
try:
|
||||
@ -428,6 +422,18 @@ class User:
|
||||
|
||||
return watchers
|
||||
|
||||
def __getstate__(self):
|
||||
# Get a copy of the object state without the API instance
|
||||
state = self.__dict__.copy()
|
||||
del(state['_api'])
|
||||
|
||||
return state
|
||||
|
||||
def __setstate__(self, state):
|
||||
# Set up the object with the given state
|
||||
self.__dict__.update(state)
|
||||
self._api = None
|
||||
|
||||
def __str__(self):
|
||||
return self.username
|
||||
|
||||
@ -446,6 +452,8 @@ class FAAPI:
|
||||
self._driver = None
|
||||
self.logged_in = False
|
||||
self.username = None
|
||||
self._usercache = {}
|
||||
self._submissioncache = {}
|
||||
|
||||
def login(self, cookies=None) -> bool:
|
||||
"""
|
||||
@ -520,16 +528,17 @@ class FAAPI:
|
||||
:param username: Username
|
||||
:return: User
|
||||
"""
|
||||
soup = self.get_soup('{}/user/{}/'.format(FA_BASE_URL, username))
|
||||
# Return a new cached user instance if one doesn't exist already
|
||||
if username not in self._usercache:
|
||||
user = User(self, username)
|
||||
self._usercache[username] = user
|
||||
return user
|
||||
|
||||
try:
|
||||
# Try extract the username, stripping the ~/∞/! in front
|
||||
username = soup.select('div.username h2')[0].text.strip()[1:]
|
||||
except IndexError:
|
||||
# No username, give up
|
||||
return None
|
||||
|
||||
return User.new_or_cached(self, username)
|
||||
# Get the cached user, updating the API instance if necessary
|
||||
user = self._usercache[username]
|
||||
if not user._api:
|
||||
user._api = self
|
||||
return user
|
||||
|
||||
def get_self(self) -> User:
|
||||
"""
|
||||
@ -544,4 +553,14 @@ class FAAPI:
|
||||
:param id: Submission ID
|
||||
:return: Submission
|
||||
"""
|
||||
return Submission.new_or_cached(self, id)
|
||||
# Return a new cached submission instance if one doesn't exist already
|
||||
if id not in self._submissioncache:
|
||||
submission = Submission(self, id)
|
||||
self._submissioncache[id] = submission
|
||||
return submission
|
||||
|
||||
# Get the cached submission, updating the API instance if necessary
|
||||
submission = self._submissioncache[id]
|
||||
if not submission._api:
|
||||
submission._api = self
|
||||
return submission
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user