tom/FAAPI
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Extract titles, descriptions, and author usernames from embedded JSON

Browse Source 
To save some more time, the scraper will succ the JSON used for generating pop-up descriptions on each item.
This commit is contained in:
Thomas Wade 2020-06-28 00:47:50 +09:30
parent e14f6492ff
commit 38ba8a50df
2 changed files with 43 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
FAAPI/FAAPI.py
View File

@ -1,3 +1,5 @@
import json
import re
from datetime import datetime
from enum import Enum
from typing import List
@ -338,10 +340,30 @@ class User:
# No next page, null the url to break the loop
next_page_url = None
# Extract the description JSON
json_submissions = {}
try:
description_script = soup.select('#site-content > script:first-of-type')[0].string
json_str = re.search(r'^[\s]*var descriptions = (.*);$', description_script, re.MULTILINE).group(1)
json_submissions = json.loads(json_str)
except IndexError as e:
# Failed to get the description script, don't do anything about it and leave the dict as-is
pass
# Get all the submissions on this page
for item in soup.select('.gallery figure'):
sub = self._api.get_submission(int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-'
# Fill out everything from the JSON submission of this item if there is one
if json_submissions and json_submissions[str(sub.id)]:
json_submission = json_submissions[str(sub.id)]
sub._title = json_submission['title']
sub._author = self._api.get_user(json_submission['username'])
# Don't fill out the description if it's truncated
if not json_submission['description'].endswith(' .......'):
sub._description = json_submission['description']
# Try cache the preview URL early
try:
sub._preview_url = 'https:' + item.select('img')[0].get('src')
@ -367,10 +389,30 @@ class User:
# No next page, null the url to break the loop
next_page_url = None
# Extract the description JSON
json_submissions = {}
try:
description_script = soup.select('#standardpage > script:first-of-type')[0].string
json_str = re.search(r'^[\s]*var descriptions = (.*);$', description_script, re.MULTILINE).group(1)
json_submissions = json.loads(json_str)
except IndexError as e:
# Failed to get the description script, don't do anything about it and leave the dict as-is
pass
# Get all the submissions on this page
for item in soup.select('.gallery figure'):
sub = self._api.get_submission(int(item.get('id')[4:])) # IDs look like 'sid-35908275', so we just skip the 'sid-'
# Fill out everything from the JSON submission of this item if there is one
if json_submissions and json_submissions[str(sub.id)]:
json_submission = json_submissions[str(sub.id)]
sub._title = json_submission['title']
sub._author = self._api.get_user(json_submission['username'])
# Don't fill out the description if it's truncated
if not json_submission['description'].endswith(' .......'):
sub._description = json_submission['description']
# Try cache the preview URL early
try:
sub._preview_url = 'https:' + item.select('img')[0].get('src')

2
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='FAAPI',
version='0.6.4',
version='0.6.5',
packages=find_packages(),
url='https://tem.party/gitea/tom/FAAPI',
license='WTFPL',