From d4fff8d61a71a700b74f68c60b7f04edf7538871 Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Wed, 22 Apr 2020 15:46:13 +0930 Subject: [PATCH 1/6] Better cache handling The root API instance will handle item caching to bring it closer to the external-facing side of things. This also means that caches are instance-based now, which may be seen as either a pro or con. Additionally, User and Submission classes will no longer serialise their API instances when pickling. --- FAAPI/FAAPI.py | 89 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 35 deletions(-) diff --git a/FAAPI/FAAPI.py b/FAAPI/FAAPI.py index 8cf7f29..938cb9e 100644 --- a/FAAPI/FAAPI.py +++ b/FAAPI/FAAPI.py @@ -29,8 +29,6 @@ class Submission: A FurAffinity submission. """ - _submissioncache = {} - def __init__(self, api, id: int): self._api = api self.id: int = id @@ -173,13 +171,6 @@ class Submission: return self._recommended - @staticmethod - def new_or_cached(api, id: int): - if id not in Submission._submissioncache: - Submission._submissioncache[id] = Submission(api, id) - - return Submission._submissioncache[id] - def fave(self): if self.faved is False: # Usage of the faved property is deliberate to ensure _get_submission() has run self._toggle_fave() @@ -193,7 +184,7 @@ class Submission: # Parse all the easy stuff self._title = soup.select('.submission-title')[0].text.strip() - self._author = User.new_or_cached(self._api, soup.select('.submission-id-sub-container a')[0].text.strip()) + self._author = self._api.get_user(soup.select('.submission-id-sub-container a')[0].text.strip()) self._description = soup.select('.submission-description')[0].text.strip() self._tags = [tag.text.strip() for tag in soup.select('.submission-sidebar .tags')] self._timestamp = datetime.strptime(soup.select('.popup_date')[0].get('title'), '%b %d, %Y %H:%M %p') # e.g. Nov 26, 2019 02:47 PM @@ -243,7 +234,7 @@ class Submission: comment.hidden_by_page_owner = False # Parse content from non-hidden comments - comment.author = User.new_or_cached(self._api, comment_element.select('.comment_username')[0].text.strip()) + comment.author = self._api.get_user(comment_element.select('.comment_username')[0].text.strip()) comment.timestamp = datetime.fromtimestamp(int(comment_element.get('data-timestamp'))) # FIXME: Needs to account for timezone difference since these are server-local epochs. Seems to be hosted in New York's TZ self._comments.append(comment) @@ -252,7 +243,7 @@ class Submission: self._recommended = [] for recommended in soup.select('.preview-gallery-container a'): id = int(recommended.get('href')[6:-1]) # e.g. /view/35992314/ - self._recommended.append(Submission.new_or_cached(self._api, id)) + self._recommended.append(self._api.get_submission(id)) def _toggle_fave(self): soup = self._api.get_soup(self._fave_url) @@ -260,6 +251,18 @@ class Submission: self._faved = '-' in soup.select('.fav')[0].text self._fave_url = FA_BASE_URL + soup.select('.fav a')[0].get('href') + def __getstate__(self): + # Get a copy of the object state without the API instance + state = self.__dict__.copy() + del (state['_api']) + + return state + + def __setstate__(self, state): + # Set up the object with the given state + self.__dict__.update(state) + self._api = None + def __str__(self): return '{} - {} ({}/view/{})'.format(self.author, self.title, FA_BASE_URL, self.url) @@ -269,8 +272,6 @@ class User: A FurAffinity user. """ - _usercache = {} - def __init__(self, api, username: str): self._api = api self.username: str = username @@ -315,13 +316,6 @@ class User: return self._watchers - @staticmethod - def new_or_cached(api, username: str): # -> User: - if username not in User._usercache: - User._usercache[username] = User(api, username) - - return User._usercache[username] - def _get_gallery(self) -> List[Submission]: submissions = [] next_page_url = '{}/gallery/{}/'.format(FA_BASE_URL, self.username) @@ -338,7 +332,7 @@ class User: # Get all the submissions on this page for item in soup.select('.gallery figure'): - sub = Submission.new_or_cached(self._api, int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-' + sub = self._api.get_submission(int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-' # Try cache the preview URL early try: @@ -367,7 +361,7 @@ class User: # Get all the submissions on this page for item in soup.select('.gallery figure'): - sub = Submission.new_or_cached(self._api, int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-' + sub = self._api.get_submission(int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-' # Try cache the preview URL early try: @@ -392,7 +386,7 @@ class User: break for item in users: - watching.append(User.new_or_cached(self._api, item.text.strip())) + watching.append(self._api.get_user(item.text.strip())) # Try get the next page try: @@ -416,7 +410,7 @@ class User: break for item in users: - watchers.append(User.new_or_cached(self._api, item.text.strip())) + watchers.append(self._api.get_user(item.text.strip())) # Try get the next page try: @@ -428,6 +422,18 @@ class User: return watchers + def __getstate__(self): + # Get a copy of the object state without the API instance + state = self.__dict__.copy() + del(state['_api']) + + return state + + def __setstate__(self, state): + # Set up the object with the given state + self.__dict__.update(state) + self._api = None + def __str__(self): return self.username @@ -446,6 +452,8 @@ class FAAPI: self._driver = None self.logged_in = False self.username = None + self._usercache = {} + self._submissioncache = {} def login(self, cookies=None) -> bool: """ @@ -520,16 +528,17 @@ class FAAPI: :param username: Username :return: User """ - soup = self.get_soup('{}/user/{}/'.format(FA_BASE_URL, username)) + # Return a new cached user instance if one doesn't exist already + if username not in self._usercache: + user = User(self, username) + self._usercache[username] = user + return user - try: - # Try extract the username, stripping the ~/∞/! in front - username = soup.select('div.username h2')[0].text.strip()[1:] - except IndexError: - # No username, give up - return None - - return User.new_or_cached(self, username) + # Get the cached user, updating the API instance if necessary + user = self._usercache[username] + if not user._api: + user._api = self + return user def get_self(self) -> User: """ @@ -544,4 +553,14 @@ class FAAPI: :param id: Submission ID :return: Submission """ - return Submission.new_or_cached(self, id) + # Return a new cached submission instance if one doesn't exist already + if id not in self._submissioncache: + submission = Submission(self, id) + self._submissioncache[id] = submission + return submission + + # Get the cached submission, updating the API instance if necessary + submission = self._submissioncache[id] + if not submission._api: + submission._api = self + return submission From b3a5b4acdb601e2433f1429cee5474c348d40cd6 Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Wed, 22 Apr 2020 16:29:06 +0930 Subject: [PATCH 2/6] Make root API serialisable --- FAAPI/FAAPI.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/FAAPI/FAAPI.py b/FAAPI/FAAPI.py index 938cb9e..3bbcda0 100644 --- a/FAAPI/FAAPI.py +++ b/FAAPI/FAAPI.py @@ -449,7 +449,6 @@ class FAAPI: """ self._cookies = None self._req = requests.session() - self._driver = None self.logged_in = False self.username = None self._usercache = {} @@ -468,16 +467,16 @@ class FAAPI: if not cookies: # Try sign in to FA - self._driver = webdriver.Chrome() - self._driver.get(FA_BASE_URL + '/login') + driver = webdriver.Chrome() + driver.get(FA_BASE_URL + '/login') # Wait until our username is visible on the top banner - while not self._driver.find_elements_by_id('my-username'): + while not driver.find_elements_by_id('my-username'): pass # Extract the cookies and stop the driver - self._cookies = self._driver.get_cookies() - self._driver.close() + self._cookies = driver.get_cookies() + driver.close() else: self._cookies = cookies @@ -564,3 +563,17 @@ class FAAPI: if not submission._api: submission._api = self return submission + + def __getstate__(self): + # Get a copy of the object state without the requests session + state = self.__dict__.copy() + del(state['req']) + + return state + + def __setstate__(self, state): + # Set up the object with the given state + self.__dict__.update(state) + + # Create a new requests session + self._req = requests.session() From 676a61977505e5cdd65aab4c87cbadf5e9c60b70 Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Wed, 22 Apr 2020 16:31:39 +0930 Subject: [PATCH 3/6] Use existing cookies by default They can still be overridden by supplying custom cookies though --- FAAPI/FAAPI.py | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/FAAPI/FAAPI.py b/FAAPI/FAAPI.py index 3bbcda0..63b1aa2 100644 --- a/FAAPI/FAAPI.py +++ b/FAAPI/FAAPI.py @@ -457,7 +457,7 @@ class FAAPI: def login(self, cookies=None) -> bool: """ Logs this instance into FurAffinity. - Beware that a Chrome instance is launched to bypass Cloudflare and log in. Specify valid cookies to bypass this. + Beware that a Chrome instance may be launched to bypass Cloudflare and log in. Specify cookies to bypass this. :param cookies: Cookies to use for this session. :return: Whether the login attempt was successful. """ @@ -465,7 +465,9 @@ class FAAPI: # No need to do anything if we're already logged in return True - if not cookies: + if cookies: + self._cookies = cookies + elif not self._cookies: # Try sign in to FA driver = webdriver.Chrome() driver.get(FA_BASE_URL + '/login') @@ -477,8 +479,6 @@ class FAAPI: # Extract the cookies and stop the driver self._cookies = driver.get_cookies() driver.close() - else: - self._cookies = cookies # Set up the Requests session with the cookies we just got [self._req.cookies.set(cookie['name'], cookie['value']) for cookie in self._cookies] diff --git a/setup.py b/setup.py index 7422050..74afcb6 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='FAAPI', - version='0.4.2', + version='0.5.0', packages=find_packages(), url='https://tem.party/gitea/tom/FAAPI', license='WTFPL', From abdc8fe67ae60e36ad848e6fd08c1ac9c4a3825f Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Wed, 22 Apr 2020 16:39:36 +0930 Subject: [PATCH 4/6] Fix misspelled dict entry --- FAAPI/FAAPI.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/FAAPI/FAAPI.py b/FAAPI/FAAPI.py index 63b1aa2..f679fca 100644 --- a/FAAPI/FAAPI.py +++ b/FAAPI/FAAPI.py @@ -567,7 +567,7 @@ class FAAPI: def __getstate__(self): # Get a copy of the object state without the requests session state = self.__dict__.copy() - del(state['req']) + del(state['_req']) return state diff --git a/setup.py b/setup.py index 74afcb6..5bc9eb2 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='FAAPI', - version='0.5.0', + version='0.5.1', packages=find_packages(), url='https://tem.party/gitea/tom/FAAPI', license='WTFPL', From e7aba9e10a4e359dac82edddd3369e2130292a2c Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Wed, 22 Apr 2020 16:46:24 +0930 Subject: [PATCH 5/6] Only handle fave-related stuff when logged in --- FAAPI/FAAPI.py | 18 ++++++++++++------ setup.py | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/FAAPI/FAAPI.py b/FAAPI/FAAPI.py index f679fca..1af2814 100644 --- a/FAAPI/FAAPI.py +++ b/FAAPI/FAAPI.py @@ -159,10 +159,10 @@ class Submission: @property def faved(self): - if self._faved is None: + if self._api.logged_in and self._faved is None: self._get_submission() - return self._faved + return self._faved if self._api.logged_in else None @property def recommended(self): @@ -172,11 +172,11 @@ class Submission: return self._recommended def fave(self): - if self.faved is False: # Usage of the faved property is deliberate to ensure _get_submission() has run + if self._api.logged_in and self.faved is False: # Usage of the faved property is deliberate to ensure _get_submission() has run self._toggle_fave() def unfave(self): - if self.faved is True: # Usage of the faved property is deliberate to ensure _get_submission() has run + if self._api.logged_in and self.faved is True: # Usage of the faved property is deliberate to ensure _get_submission() has run self._toggle_fave() def _get_submission(self): @@ -195,8 +195,14 @@ class Submission: self._faves = int(soup.select('.favorites .font-large')[0].text.strip()) self._views = int(soup.select('.views .font-large')[0].text.strip()) self._download_url = 'https:' + soup.select('.download a')[0].get('href') - self._faved = '-' in soup.select('.fav')[0].text - self._fave_url = FA_BASE_URL + soup.select('.fav a')[0].get('href') + + # Only parse the fave button if we're logged in + if self._api.logged_in: + self._faved = '-' in soup.select('.fav')[0].text + self._fave_url = FA_BASE_URL + soup.select('.fav a')[0].get('href') + else: + self._faved = None + self._fave_url = None # Parse preview try: diff --git a/setup.py b/setup.py index 5bc9eb2..0a1e966 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='FAAPI', - version='0.5.1', + version='0.5.2', packages=find_packages(), url='https://tem.party/gitea/tom/FAAPI', license='WTFPL', From c0b381aca7933a2ee34f929daad46323afc93aa5 Mon Sep 17 00:00:00 2001 From: Thomas Wade Date: Wed, 22 Apr 2020 16:52:57 +0930 Subject: [PATCH 6/6] Update cached object API instances when deserialising --- FAAPI/FAAPI.py | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/FAAPI/FAAPI.py b/FAAPI/FAAPI.py index 1af2814..673582b 100644 --- a/FAAPI/FAAPI.py +++ b/FAAPI/FAAPI.py @@ -583,3 +583,9 @@ class FAAPI: # Create a new requests session self._req = requests.session() + + # Reset API instances on our cached objects + for user in self._usercache.values(): + user._api = self + for sub in self._submissioncache.values(): + sub._api = self diff --git a/setup.py b/setup.py index 0a1e966..5157b4d 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='FAAPI', - version='0.5.2', + version='0.5.3', packages=find_packages(), url='https://tem.party/gitea/tom/FAAPI', license='WTFPL',