Extract titles, descriptions, and author usernames from embedded JSON

To save some more time, the scraper will succ the JSON used for generating pop-up descriptions on each item.

FAAPI/FAAPI.py

@@ -1,3 +1,5 @@
+import json
+import re
 from datetime import datetime
 from enum import Enum
 from typing import List
@@ -338,10 +340,30 @@ class User:
                 # No next page, null the url to break the loop
                 next_page_url = None
 
+            # Extract the description JSON
+            json_submissions = {}
+            try:
+                description_script = soup.select('#site-content > script:first-of-type')[0].string
+                json_str = re.search(r'^[\s]*var descriptions = (.*);$', description_script, re.MULTILINE).group(1)
+                json_submissions = json.loads(json_str)
+            except IndexError as e:
+                # Failed to get the description script, don't do anything about it and leave the dict as-is
+                pass
+
             # Get all the submissions on this page
             for item in soup.select('.gallery figure'):
                 sub = self._api.get_submission(int(item.get('id')[4:]))  # IDs look like 'sid-35908275', so we just skip the 'sid-'
 
+                # Fill out everything from the JSON submission of this item if there is one
+                if json_submissions and json_submissions[str(sub.id)]:
+                    json_submission = json_submissions[str(sub.id)]
+                    sub._title = json_submission['title']
+                    sub._author = self._api.get_user(json_submission['username'])
+
+                    # Don't fill out the description if it's truncated
+                    if not json_submission['description'].endswith(' .......'):
+                        sub._description = json_submission['description']
+
                 # Try cache the preview URL early
                 try:
                     sub._preview_url = 'https:' + item.select('img')[0].get('src')
@@ -367,10 +389,30 @@ class User:
                 # No next page, null the url to break the loop
                 next_page_url = None
 
+            # Extract the description JSON
+            json_submissions = {}
+            try:
+                description_script = soup.select('#standardpage > script:first-of-type')[0].string
+                json_str = re.search(r'^[\s]*var descriptions = (.*);$', description_script, re.MULTILINE).group(1)
+                json_submissions = json.loads(json_str)
+            except IndexError as e:
+                # Failed to get the description script, don't do anything about it and leave the dict as-is
+                pass
+
             # Get all the submissions on this page
             for item in soup.select('.gallery figure'):
                 sub = self._api.get_submission(int(item.get('id')[4:]))  # IDs look like 'sid-35908275', so we just skip the 'sid-'
 
+                # Fill out everything from the JSON submission of this item if there is one
+                if json_submissions and json_submissions[str(sub.id)]:
+                    json_submission = json_submissions[str(sub.id)]
+                    sub._title = json_submission['title']
+                    sub._author = self._api.get_user(json_submission['username'])
+
+                    # Don't fill out the description if it's truncated
+                    if not json_submission['description'].endswith(' .......'):
+                        sub._description = json_submission['description']
+
                 # Try cache the preview URL early
                 try:
                     sub._preview_url = 'https:' + item.select('img')[0].get('src')

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='FAAPI',
-    version='0.6.4',
+    version='0.6.5',
     packages=find_packages(),
     url='https://tem.party/gitea/tom/FAAPI',
     license='WTFPL',