Initial commit

So far just pulls out the name of the target user
This commit is contained in:
Thomas Wade 2018-09-17 19:02:00 +09:30
commit 8faf429e0c
3 changed files with 148 additions and 0 deletions

111
.gitignore vendored Normal file
View File

@ -0,0 +1,111 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json

3
requirements.txt Normal file
View File

@ -0,0 +1,3 @@
lxml
bs4
selenium

34
scrape.py Executable file
View File

@ -0,0 +1,34 @@
#!/usr/bin/python3
# It is assumed that you have installed the required packages in requirements.txt
# and have also installed Chromedriver from your package manager.
# If you have installed Chromedriver separately, change the line below to the
# path of the Chromedriver executable.
CHROMEDRIVER_PATH = '/usr/bin/chromedriver'
import networkx as nx
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup
##### Set up NetworkX #######################################
graph = nx.DiGraph()
##### Set up WebDriver with Chrome and get the page #########
chrome_options = Options()
chrome_options.add_argument('--headless') # Start headless so we can run on a server overnight
driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options)
driver.get("https://soundcloud.com/lacheque")
soup = BeautifulSoup(driver.page_source, "lxml")
driver.quit()
##### Do stuff ##############################################
username = soup.find(class_='profileHeaderInfo__userName')
print(username.string.strip())