34 lines
554 B
Python
34 lines
554 B
Python
import requests
|
|
import re
|
|
import urlparse
|
|
|
|
# regex
|
|
link_re = re.compile(r'href="(.*?)"')
|
|
|
|
|
|
def crawl(url):
|
|
|
|
req = requests.get(url)
|
|
|
|
# Check if successful
|
|
if(req.status_code != 200):
|
|
return []
|
|
|
|
# Find links
|
|
links = link_re.findall(req.text)
|
|
|
|
print "\nFound {} links".format(len(links))
|
|
|
|
# Search links for emails
|
|
for link in links:
|
|
|
|
# Get an absolute URL for a link
|
|
link = urlparse.urljoin(url, link)
|
|
|
|
print link
|
|
|
|
|
|
if __name__ == '__main__':
|
|
crawl('http://www.realpython.com')
|
|
|