python-scripts/09_basic_link_web_crawler.py

import requests
import re
import urlparse

# regex
link_re = re.compile(r'href="(.*?)"')


def crawl(url):

    req = requests.get(url)

    # Check if successful
    if(req.status_code != 200):
        return []

    # Find links
    links = link_re.findall(req.text)

    print "\nFound {} links".format(len(links))

    # Search links for emails
    for link in links:

        # Get an absolute URL for a link
        link = urlparse.urljoin(url, link)

        print link


if __name__ == '__main__':
    crawl('http://www.realpython.com')