commit 8faf429e0ceb39dfebb4fb3be0d8d50455a69218 Author: Thomas Wade Date: Mon Sep 17 19:02:00 2018 +0930 Initial commit So far just pulls out the name of the target user diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6f7a6d9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,111 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6781945 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +lxml +bs4 +selenium diff --git a/scrape.py b/scrape.py new file mode 100755 index 0000000..037f130 --- /dev/null +++ b/scrape.py @@ -0,0 +1,34 @@ +#!/usr/bin/python3 + +# It is assumed that you have installed the required packages in requirements.txt +# and have also installed Chromedriver from your package manager. +# If you have installed Chromedriver separately, change the line below to the +# path of the Chromedriver executable. +CHROMEDRIVER_PATH = '/usr/bin/chromedriver' + +import networkx as nx +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions +from selenium.webdriver.chrome.options import Options +from bs4 import BeautifulSoup + +##### Set up NetworkX ####################################### + +graph = nx.DiGraph() + +##### Set up WebDriver with Chrome and get the page ######### + +chrome_options = Options() +chrome_options.add_argument('--headless') # Start headless so we can run on a server overnight + +driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options) +driver.get("https://soundcloud.com/lacheque") +soup = BeautifulSoup(driver.page_source, "lxml") +driver.quit() + +##### Do stuff ############################################## + +username = soup.find(class_='profileHeaderInfo__userName') +print(username.string.strip())