Add scraper example

This commit is contained in:
Thomas Wade 2020-04-21 16:17:10 +09:30
parent 16312833a4
commit cf3676f461

35
FaveScraper.py Executable file
View File

@ -0,0 +1,35 @@
#!/usr/bin/python3
import multiprocessing
import requests
from FAAPI import FAAPI
def download(url: str):
print('Downloading ' + url)
with open(url.split('/')[-1], 'wb') as file:
res = requests.get(url)
file.write(res.content)
if __name__ == '__main__':
# Get an API instance
api = FAAPI()
# Get the user we've authenticated as
user = api.get_user(api.username)
if not user:
print('Failed to get user')
exit(1)
# Get the download URLs from each fave
download_urls = []
for fave in user.faves[0:50]:
print('Scraping ' + str(fave.id))
download_urls.append(fave.download_url)
# Download them all
pool = multiprocessing.Pool(processes=10)
pool.map(download, download_urls)