python-scripts/09_basic_link_web_crawler.py
2015-10-15 22:51:07 +01:00

35 lines
616 B
Python
Executable File

import requests
import re
try:
from urllib.parse import urljoin
except ImportError:
from urlparse import urljoin
# regex
link_re = re.compile(r'href="(.*?)"')
def crawl(url):
req = requests.get(url)
# Check if successful
if(req.status_code != 200):
return []
# Find links
links = link_re.findall(req.text)
print("\nFound {} links".format(len(links)))
# Search links for emails
for link in links:
# Get an absolute URL for a link
link = urljoin(url, link)
print(link)
if __name__ == '__main__':
crawl('http://www.realpython.com')