Add scraper example
This commit is contained in:
parent
16312833a4
commit
cf3676f461
35
FaveScraper.py
Executable file
35
FaveScraper.py
Executable file
@ -0,0 +1,35 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import multiprocessing
|
||||
|
||||
import requests
|
||||
|
||||
from FAAPI import FAAPI
|
||||
|
||||
|
||||
def download(url: str):
|
||||
print('Downloading ' + url)
|
||||
with open(url.split('/')[-1], 'wb') as file:
|
||||
res = requests.get(url)
|
||||
file.write(res.content)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Get an API instance
|
||||
api = FAAPI()
|
||||
|
||||
# Get the user we've authenticated as
|
||||
user = api.get_user(api.username)
|
||||
if not user:
|
||||
print('Failed to get user')
|
||||
exit(1)
|
||||
|
||||
# Get the download URLs from each fave
|
||||
download_urls = []
|
||||
for fave in user.faves[0:50]:
|
||||
print('Scraping ' + str(fave.id))
|
||||
download_urls.append(fave.download_url)
|
||||
|
||||
# Download them all
|
||||
pool = multiprocessing.Pool(processes=10)
|
||||
pool.map(download, download_urls)
|
||||
Loading…
x
Reference in New Issue
Block a user