35 lines
616 B
Python
35 lines
616 B
Python
import requests
|
|
import re
|
|
try:
|
|
from urllib.parse import urljoin
|
|
except ImportError:
|
|
from urlparse import urljoin
|
|
|
|
# regex
|
|
link_re = re.compile(r'href="(.*?)"')
|
|
|
|
|
|
def crawl(url):
|
|
|
|
req = requests.get(url)
|
|
|
|
# Check if successful
|
|
if(req.status_code != 200):
|
|
return []
|
|
|
|
# Find links
|
|
links = link_re.findall(req.text)
|
|
|
|
print("\nFound {} links".format(len(links)))
|
|
|
|
# Search links for emails
|
|
for link in links:
|
|
|
|
# Get an absolute URL for a link
|
|
link = urljoin(url, link)
|
|
|
|
print(link)
|
|
|
|
if __name__ == '__main__':
|
|
crawl('http://www.realpython.com')
|