FAAPI/FAAPI/FAAPI.py

from datetime import datetime
from enum import Enum
from typing import List

import requests
from bs4 import BeautifulSoup
from selenium import webdriver

FA_BASE_URL = 'https://www.furaffinity.net'


class Comment:
    """
    A comment on a FurAffinity submission.
    """

    def __init__(self, submission, id: int):
        self.id = id
        self.author: User = None
        self.timestamp: datetime = None
        self.submission: Submission = submission
        self.parent: Comment = None
        self.replies: List[Comment] = None
        self.hidden_by_page_owner = None
        self.hidden_by_author = None


class Submission:
    """
    A FurAffinity submission.
    """

    def __init__(self, api, id: int):
        self._api = api
        self.id: int = id
        self._title: str = None
        self._author: User = None
        self._description: str = None
        self._tags: List[str] = None
        self._timestamp: datetime = None
        self._category: str = None
        self._subcategory: str = None
        self._theme: str = None
        self._species: str = None
        self._gender: str = None
        self._faves: int = None
        self._comments: List[Comment] = None
        self._views: int = None
        self._has_preview: bool = None
        self._preview_url: str = None
        self._download_url: str = None
        self._faved: bool = None
        self._fave_url: str = None
        self._recommended: List[Submission] = None

    @property
    def url(self):
        if self.id and self.id > 0:
            return '{}/view/{}'.format(FA_BASE_URL, self.id)
        else:
            return None

    @property
    def title(self):
        if not self._title:
            self._get_submission()

        return self._title

    @property
    def author(self):
        if not self._author:
            self._get_submission()

        return self._author

    @property
    def description(self):
        if not self._description:
            self._get_submission()

        return self._description

    @property
    def tags(self):
        if not self._tags:
            self._get_submission()

        return self._tags

    @property
    def timestamp(self):
        if not self._timestamp:
            self._get_submission()

        return self._timestamp

    @property
    def category(self):
        if not self._category:
            self._get_submission()

        return self._category

    @property
    def subcategory(self):
        if not self._subcategory:
            self._get_submission()

        return self._subcategory

    @property
    def species(self):
        if not self._species:
            self._get_submission()

        return self._species

    @property
    def gender(self):
        if not self._gender:
            self._get_submission()

        return self._gender

    @property
    def faves(self):
        if not self._faves:
            self._get_submission()

        return self._faves

    @property
    def comments(self):
        if not self._comments:
            self._get_submission()

        return self._comments

    @property
    def views(self):
        if not self._views:
            self._get_submission()

        return self._views

    @property
    def preview_url(self):
        if (not self._preview_url and self._has_preview is True) or self._has_preview is None:
            self._get_submission()

        return self._preview_url

    @property
    def download_url(self):
        if not self._download_url:
            self._get_submission()

        return self._download_url

    @property
    def faved(self):
        if self._api.logged_in and self._faved is None:
            self._get_submission()

        return self._faved if self._api.logged_in else None

    @property
    def recommended(self):
        if self._recommended is None:
            self._get_submission()

        return self._recommended

    def fave(self):
        if self._api.logged_in and self.faved is False:  # Usage of the faved property is deliberate to ensure _get_submission() has run
            self._toggle_fave()

    def unfave(self):
        if self._api.logged_in and self.faved is True:  # Usage of the faved property is deliberate to ensure _get_submission() has run
            self._toggle_fave()

    def _get_submission(self):
        soup = self._api.get_soup('{}/view/{}'.format(FA_BASE_URL, self.id))

        # Parse all the easy stuff
        self._title = soup.select('.submission-title')[0].text.strip()
        self._author = self._api.get_user(soup.select('.submission-id-sub-container a')[0].text.strip())
        self._description = soup.select('.submission-description')[0].text.strip()
        self._tags = [tag.text.strip() for tag in soup.select('.submission-sidebar .tags')]
        self._timestamp = datetime.strptime(soup.select('.popup_date')[0].get('title'), '%b %d, %Y %H:%M %p')  # e.g. Nov 26, 2019 02:47 PM
        self._category = soup.select('.category-name')[0].text.strip()
        self._subcategory = soup.select('.type-name')[0].text.strip()
        self._species = soup.select('.info div:nth-of-type(2) span')[0].text.strip()
        self._gender = soup.select('.info div:nth-of-type(3) span')[0].text.strip()
        self._faves = int(soup.select('.favorites .font-large')[0].text.strip())
        self._views = int(soup.select('.views .font-large')[0].text.strip())
        self._download_url = 'https:' + soup.select('.download a')[0].get('href')

        # Only parse the fave button if we're logged in
        if self._api.logged_in:
            self._faved = '-' in soup.select('.fav')[0].text
            self._fave_url = FA_BASE_URL + soup.select('.fav a')[0].get('href')
        else:
            self._faved = None
            self._fave_url = None

        # Parse preview
        try:
            self._preview_url = 'https:' + soup.select('#submissionImg')[0].get('data-preview-src')
        except IndexError:
            self._preview_url = None
        finally:
            self._has_preview = self._preview_url is not None

        # Parse comments
        self._comments = []
        for comment_element in soup.select('.comment_container'):
            # Parse all the easy stuff
            comment = Comment(self, int(comment_element.select('.comment_anchor')[0].get('id')[4:]))  # Skip the 'cid:' in the ID e.g. cid:142785397

            # TODO: Comment ancestry

            # Toggle hidden status
            if comment_element.select('.comment-deleted'):
                # Deleted by its owner, set flags and skip remaining attributes
                comment.hidden_by_author = True
                comment.hidden_by_page_owner = False

                self._comments.append(comment)
                continue
            elif 'collapsed_height' in comment_element.get('class'):
                # Deleted by the page owner, set flags and skip remaining attributes
                comment.hidden_by_author = False
                comment.hidden_by_page_owner = True

                self._comments.append(comment)
                continue
            else:
                comment.hidden_by_author = False
                comment.hidden_by_page_owner = False

            # Parse content from non-hidden comments
            comment.author = self._api.get_user(comment_element.select('.comment_username')[0].text.strip())
            comment.timestamp = datetime.fromtimestamp(int(comment_element.get('data-timestamp')))  # FIXME: Needs to account for timezone difference since these are server-local epochs. Seems to be hosted in New York's TZ

            self._comments.append(comment)

        # Parse recommended submissions
        self._recommended = []
        for recommended in soup.select('.preview-gallery-container a'):
            id = int(recommended.get('href')[6:-1])  # e.g. /view/35992314/
            self._recommended.append(self._api.get_submission(id))

    def _toggle_fave(self):
        soup = self._api.get_soup(self._fave_url)

        self._faved = '-' in soup.select('.fav')[0].text
        self._fave_url = FA_BASE_URL + soup.select('.fav a')[0].get('href')

    def __getstate__(self):
        # Get a copy of the object state without the API instance
        state = self.__dict__.copy()
        del (state['_api'])

        return state

    def __setstate__(self, state):
        # Set up the object with the given state
        self.__dict__.update(state)
        self._api = None

    def __str__(self):
        return '{} - {} ({}/view/{})'.format(self.author, self.title, FA_BASE_URL, self.url)


class User:
    """
    A FurAffinity user.
    """

    def __init__(self, api, username: str):
        self._api = api
        self.username: str = username
        self._gallery: List[Submission] = None
        self._faves: List[Submission] = None
        self._watching: List[User] = None
        self._watchers: List[User] = None

    @property
    def username_lower(self) -> str:
        return self.username.lower()

    @property
    def gallery(self) -> List[Submission]:
        # Get the gallery if we don't have it already
        if not self._gallery:
            self._gallery = self._get_gallery()

        return self._gallery

    @property
    def faves(self) -> List[Submission]:
        # Get faves if we don't have them already
        if not self._faves:
            self._faves = self._get_faves()

        return self._faves

    @property
    def watching(self):  # -> List[User]:
        # Get the list if we don't have it already
        if not self._watching:
            self._watching = self._get_watching()

        return self._watching

    @property
    def watchers(self):  # -> List[User]:
        # Get the list if we don't have it already
        if not self._watchers:
            self._watchers = self._get_watchers()

        return self._watchers

    def _get_gallery(self) -> List[Submission]:
        submissions = []
        next_page_url = '{}/gallery/{}/'.format(FA_BASE_URL, self.username)
        while next_page_url:
            soup = self._api.get_soup(next_page_url)

            # Try get the next page
            try:
                next_page_button = soup.select('.submission-list .inline:last-child form button[type=submit]')[0]
                next_page_url = FA_BASE_URL + next_page_button.parent.get('action')
            except IndexError:
                # No next page, null the url to break the loop
                next_page_url = None

            # Get all the submissions on this page
            for item in soup.select('.gallery figure'):
                sub = self._api.get_submission(int(item.get('id')[4:]))  # IDs look like 'sid-35908275', so we just skip the 'sid-'

                # Try cache the preview URL early
                try:
                    sub._preview_url = 'https:' + item.select('img')[0].get('src')
                    sub._has_preview = True
                except IndexError:
                    sub._has_preview = False

                submissions.append(sub)

        return submissions

    def _get_faves(self):
        submissions = []
        next_page_url = '{}/favorites/{}/'.format(FA_BASE_URL, self.username)
        while next_page_url:
            soup = self._api.get_soup(next_page_url)

            # Try get the next page
            try:
                next_page_button = soup.select('.pagination a.button:last-child')[0]
                next_page_url = FA_BASE_URL + next_page_button.get('href')
            except IndexError:
                # No next page, null the url to break the loop
                next_page_url = None

            # Get all the submissions on this page
            for item in soup.select('.gallery figure'):
                sub = self._api.get_submission(int(item.get('id')[4:]))  # IDs look like 'sid-35908275', so we just skip the 'sid-'

                # Try cache the preview URL early
                try:
                    sub._preview_url = 'https:' + item.select('img')[0].get('src')
                    sub._has_preview = True
                except IndexError:
                    sub._has_preview = False

                submissions.append(sub)

        return submissions

    def _get_watching(self):
        watching = []
        next_page_url = '{}/watchlist/by/{}/'.format(FA_BASE_URL, self.username)
        while True:
            soup = self._api.get_soup(next_page_url)

            # Stop processing if there aren't any users left
            users = soup.select('.watch-list .watch-list-items a')
            if len(users) == 0:
                break

            for item in users:
                watching.append(self._api.get_user(item.text.strip()))

            # Try get the next page
            try:
                next_page_button = soup.select('.floatright form button[type=submit]')[0]
                next_page_url = FA_BASE_URL + next_page_button.parent.get('href')
            except IndexError:
                # No next page, break out
                break

        return watching

    def _get_watchers(self):
        watchers = []
        next_page_url = '{}/watchlist/to/{}/'.format(FA_BASE_URL, self.username)
        while True:
            soup = self._api.get_soup(next_page_url)

            # Stop processing if there aren't any users left
            users = soup.select('.watch-list .watch-list-items a')
            if len(users) == 0:
                break

            for item in users:
                watchers.append(self._api.get_user(item.text.strip()))

            # Try get the next page
            try:
                next_page_button = soup.select('.floatright form button[type=submit]')[0]
                next_page_url = FA_BASE_URL + next_page_button.parent.get('href')
            except IndexError:
                # No next page, break out
                break

        return watchers

    def __getstate__(self):
        # Get a copy of the object state without the API instance
        state = self.__dict__.copy()
        del(state['_api'])

        return state

    def __setstate__(self, state):
        # Set up the object with the given state
        self.__dict__.update(state)
        self._api = None

    def __str__(self):
        return self.username


class OrderByEnum(Enum):
    relevancy = 'relevancy'
    date = 'date'
    popularity = 'popularity'


class OrderEnum(Enum):
    ascending = 'asc'
    descending = 'desc'


class RangeEnum(Enum):
    day = 'day'
    three_days = '3days'
    week = 'week'
    month = 'month'
    all_time = 'all'


class MatchModeEnum(Enum):
    all = 'all'
    any = 'any'
    extended = 'extended'


class SearchOpts:
    """
    A container for FurAffinity search options.
    """

    def __init__(self, order_by: OrderByEnum = OrderByEnum.relevancy,
                 order_direction: OrderEnum = OrderEnum.descending,
                 range: RangeEnum = RangeEnum.all_time,
                 match_mode: MatchModeEnum = MatchModeEnum.extended,
                 rating_general: bool = True,
                 rating_mature: bool = True,
                 rating_adult: bool = True,
                 type_art: bool = True,
                 type_music: bool = True,
                 type_flash: bool = True,
                 type_story: bool = True,
                 type_photo: bool = True,
                 type_poetry: bool = True):
        self.order_by: OrderByEnum = order_by
        self.order_direction: OrderEnum = order_direction
        self.range: RangeEnum = range
        self.match_mode: MatchModeEnum = match_mode
        self.rating_general: bool = rating_general
        self.rating_mature: bool = rating_mature
        self.rating_adult: bool = rating_adult
        self.type_art: bool = type_art
        self.type_music: bool = type_music
        self.type_flash: bool = type_flash
        self.type_story: bool = type_story
        self.type_photo: bool = type_photo
        self.type_poetry: bool = type_poetry


class FAAPI:
    """
    A basic FurAffinity API instance.
    """

    def __init__(self):
        """
        Initialises a new FAAPI instance.
        """
        self._cookies = None
        self._req = requests.session()
        self.logged_in = False
        self.username = None
        self._usercache = {}
        self._submissioncache = {}

    def login(self, cookies=None) -> bool:
        """
        Logs this instance into FurAffinity.
        Beware that a Chrome instance may be launched to bypass Cloudflare and log in. Specify cookies to bypass this.
        :param cookies: Cookies to use for this session.
        :return: Whether the login attempt was successful.
        """
        if cookies:
            self._cookies = cookies
        elif not self._cookies:
            # Try sign in to FA
            driver = webdriver.Chrome()
            driver.get(FA_BASE_URL + '/login')

            # Wait until our username is visible on the top banner
            while not driver.find_elements_by_id('my-username'):
                pass

            # Extract the cookies and stop the driver
            self._cookies = driver.get_cookies()
            driver.close()

        # Set up the Requests session with the cookies we just got
        [self._req.cookies.set(cookie['name'], cookie['value']) for cookie in self._cookies]

        # Verify login by getting our username
        soup = self.get_soup(FA_BASE_URL)
        try:
            self.username = soup.select('.loggedin_user_avatar')[0].get('alt').strip()  # This is more reliable than the user menu heading, don't ask why
        except IndexError:
            # Not logged in
            self.logged_in = False
            self._cookies = None
            return False

        # Mark this instance as logged in
        self.logged_in = True
        return True

    def logout(self):
        """
        Logs this instance out of FurAffinity.
        """
        if not self.logged_in:
            # No need to do anything if we're already logged out
            return

        # Reset our state and start a new session
        self.logged_in = False
        self._cookies = None
        self._req = requests.session()

    def get_soup(self, url: str) -> BeautifulSoup:
        """
        Gets a webpage and wraps it in a BeautifulSoup instance.
        :param url: Webpage URL
        :return: Webpage wrapped in a BeautifulSoup instance
        """
        res = self._req.get(url)
        if res.status_code >= 400:
            raise Exception(
                'Got status code {} when trying to get URL {}'.format(res.status_code, url))
        return BeautifulSoup(res.content, 'lxml')

    def get_user(self, username: str) -> User:
        """
        Gets a user by their username.
        :param username: Username
        :return: User
        """
        # Return a new cached user instance if one doesn't exist already
        if username not in self._usercache:
            user = User(self, username)
            self._usercache[username] = user
            return user

        # Get the cached user, updating the API instance if necessary
        user = self._usercache[username]
        if not user._api:
            user._api = self
        return user

    def get_self(self) -> User:
        """
        Gets the user that this instance is logged in as.
        :return: User that this instance is logged in as.
        """
        return self.get_user(self.username) if self.logged_in else None

    def get_submission(self, id: int) -> Submission:
        """
        Gets a submission by its numeric ID.
        :param id: Submission ID
        :return: Submission
        """
        # Return a new cached submission instance if one doesn't exist already
        if id not in self._submissioncache:
            submission = Submission(self, id)
            self._submissioncache[id] = submission
            return submission

        # Get the cached submission, updating the API instance if necessary
        submission = self._submissioncache[id]
        if not submission._api:
            submission._api = self
        return submission

    def search(self, search: str, results: int = 48, opts: SearchOpts = SearchOpts()) -> List[Submission]:
        """
        Searches FurAffinity with the given string.
        Results are
        :param search: Search term
        :param results: Number of results to return
        :param opts: Search options
        :return: List containing results
        """
        url = FA_BASE_URL + '/search/'
        form_data = {
            'page': 1,
            'q': search,
            'do_search': 'Search',
            'order-by': opts.order_by.value,
            'order-direction': opts.order_direction.value,
            'range': opts.range.value,
            'mode': opts.match_mode.value
        }

        # Set optional keys since they're judged on presence, not value
        if opts.rating_general: form_data['rating-general'] = 'on'
        if opts.rating_mature: form_data['rating-mature'] = 'on'
        if opts.rating_adult: form_data['rating-adult'] = 'on'
        if opts.type_art: form_data['type-art'] = 'on'
        if opts.type_music: form_data['type-music'] = 'on'
        if opts.type_flash: form_data['type-flash'] = 'on'
        if opts.type_story: form_data['type-story'] = 'on'
        if opts.type_photo: form_data['type-photo'] = 'on'
        if opts.type_poetry: form_data['type-poetry'] = 'on'

        submissions = []
        while len(submissions) < results:
            # Get our search results
            res = self._req.post(url, data=form_data)
            if res.status_code >= 400:
                raise Exception('Got status code {} when trying to get URL {}'.format(res.status_code, url))
            soup = BeautifulSoup(res.content, 'lxml')

            # Get all the submissions on this page
            items = soup.select('.gallery figure')
            for item in items:
                sub = self.get_submission(int(item.get('id')[4:]))  # IDs look like 'sid-35908275', so we just skip the 'sid-'

                # Try cache the preview URL early
                try:
                    sub._preview_url = 'https:' + item.select('img')[0].get('src')
                    sub._has_preview = True
                except IndexError:
                    sub._has_preview = False

                submissions.append(sub)

            if not soup.select('input[name=next_page]:not(.hidden)'):
                # No further pages, stop searching
                break

            # Increment our page number
            form_data['page'] = form_data['page'] + 1

        return submissions[:results]

    def __getstate__(self):
        # Get a copy of the object state without the requests session
        state = self.__dict__.copy()
        del(state['_req'])

        return state

    def __setstate__(self, state):
        # Set up the object with the given state
        self.__dict__.update(state)

        # Create a new requests session
        self._req = requests.session()

        # Reset API instances on our cached objects
        for user in self._usercache.values():
            user._api = self
        for sub in self._submissioncache.values():
            sub._api = self