keyword_density.py

import urllib.request as urllib2
import re

TAG_RE = re.compile(r'<[^>]+>')

def fetch_page(siteURL):
    # create a variable which will hold our desired web page as a string
    site= siteURL
    # create the approprriate headers for our http request so that we wont run
    # into any 403 forbidden errors. All of this will be available at the tutorial
    # page that I will link to in the description below
    hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
           'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
           'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
           'Accept-Encoding': 'none',
           'Accept-Language': 'en-US,en;q=0.8',
           'Connection': 'keep-alive'}

    # Perform a HTTP request by passing in our desired URL and setting our headers to equal
    # the headers that we've defined above.
    req = urllib2.Request(site, headers=hdr)

    #
    try:
        # here we are going to open our desired page using urllib2.urlopen
        # and passing in our request object as a parameter and as a means of protection we
        # will surround this with a try except so that, should the script run into any errors
        # it will fail gracefully instead of just crashing.
        page = urllib2.urlopen(req)
    except urllib2.HTTPError as e:
        # print out the HTTPError
        print (e.fp.read())

    # lastly we will want to read the response which was generated by opening
    # the url and store it under content
    content = page.read().decode('utf-8')
    # and then print out this page.
    return content

def remove_tags(text):
    return TAG_RE.sub('', text)


def kwmain(url):
    page = fetch_page(url)
    #print(page) # get web page as text value
    wordsNoTags = remove_tags(page)
    #print(wordsNoTags) # remove the html tags 

    word_list = {}

    for word in wordsNoTags:
        if not word in word_list:
            word_list[word] = 1
        else:
            word_list[word] += 1

    ret = (len(word_list))

    return ret

#unit test
#kwmain("https://zerrowtech.com")