0
"""
download html
"""
def download_html(url):
    html = ""
    try:
        time.sleep(random.randint(1, 3))
        req_header = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) \
        AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
        'Accept':'text/html;q=0.9,*/*;q=0.8',
        'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
        'Accept-Encoding':'gzip',
        'Connection':'close',
        'Referer':None
        }
        req_timeout = 5
        req = urllib2.Request(url, None, req_header)
        response = urllib2.urlopen(req, None, req_timeout)
        html = response.read()
        html = html.replace("&", "&")
        #html = html.replace("xmlns", "id")
        #html = html.replace("id:fb", "class")
        return html
    except:
        return ""


"""
parse html use xpath element
"""
def parse_xpath_value(xpath_element, html):
    f = StringIO.StringIO(html)
    tree = etree.parse(f)
    r = tree.xpath(xpath_element)
    print r


def main():
    url = "http://academic.research.microsoft.com/RankList?entitytype=4&topDomainID=1&subDomainID=0&last=0&start=1&end=100"
    html = download_html(url)
    journal_xpath = "//tbody/tr/td/a/text()"
    journal_list = parse_xpath_value(journal_xpath, html)

I want to parse journals from the url in my code. When I read code of the url, i notice xmlns and xmlns:fb in the html element. If I remove xmlns and xmlns:fb from the html, my code works well.

2
  • 1
    Parse the HTML as HTML with lxml.html.fromstring(html).
    – Blender
    Jun 18, 2015 at 2:59
  • @Blender,thank you for your answer.It help me a lot.I will read the lxml document carefully.
    – veil
    Jun 18, 2015 at 9:14

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Browse other questions tagged or ask your own question.