added link scraper

This commit is contained in:
Michael Herman 2014-05-18 07:29:23 -07:00
parent 7409f60137
commit 776ed9c75c
2 changed files with 53 additions and 8 deletions

View File

@ -6,6 +6,7 @@ import urlparse
email_re = re.compile(r'([\w\.,]+@[\w\.,]+\.\w+)')
link_re = re.compile(r'href="(.*?)"')
def crawl(url, maxlevel):
result = set()

View File

@ -0,0 +1,44 @@
import requests
import re
import urlparse
# regex
link_re = re.compile(r'href="(.*?)"')
def crawl(url, maxlevel):
result = set()
while maxlevel > 0:
# Get the webpage
req = requests.get(url)
# Check if successful
if(req.status_code != 200):
return []
# Find and follow all the links
links = link_re.findall(req.text)
for link in links:
# Get an absolute URL for a link
link = urlparse.urljoin(url, link)
# add links to result set
result.update(link)
print "Crawled level: {}".format(maxlevel)
# new level
maxlevel -= 1
# recurse
crawl(link, maxlevel)
return result
emails = crawl('http://www.website_goes_here_dot_com', 2)
print "\nScrapped links:"
for link in links:
print link