0

How should this parse this with beautiful soup 4 when it doesn't recognize td.font.unwrap()? Should I replace it or something? or unwrap it?

<td align="CENTER">
    <font size="+2">横</font>(F
    <font size="+2">橫</font>)
</td>

I'd like to just get the string of 横(F橫) What I'm getting right now is: 横(F�)

I'm calling into the td field just fine, but just not able to get the last character... This is how I'm calling it right now y = cols[1].text

cols is the <td> field and this is the second one within the row<tr>....

FULL CODE BELOW:

    # coding: utf8
from pysqlite2 import dbapi2 as sqlite3
import urllib2
from bs4 import BeautifulSoup
from string import *


conn = sqlite3.connect(':memory:')
cursor = conn.cursor()

# # create a table
def createTable():
    cursor.execute("""CREATE TABLE characters
                      (rank INTEGER, word TEXT, definition TEXT) 
                   """)


def insertChar(rank,word,definition):
    cursor.execute("""INSERT INTO characters (rank,word,definition)
                        VALUES (?,?,?)""",(rank,word,definition))


def main():
    createTable()

    # u = unicode("辣", "utf-8")

    # insertChar(1,u,"123123123")

    # content = "\n".join(response.readlines()[1:])
    soup = BeautifulSoup(urllib2.urlopen('http://www.zein.se/patrick/3000char.html').read())

    # print (html_doc.prettify())   

    tables = soup.blockquote.table

    # print tables



    rows = tables.find_all('tr')[1:]
    result=[]
    for tr in rows:
        # print tr
        cols = tr.find_all('td')
        character = []
        # col = cols.fonts.unwrap()
        # x = int (cols[0].string)
        x = 0
        y = cols[1].text
        # chars = y.find_all('font') 

        z = "11"
        print y
        # y = cols[1].string
        # z = cols[2].string

        # xx = unicode(x, "utf-8")
        # yy = unicode(y , "utf-8")
        # zz = unicode(z , "utf-8")
        insertChar(x,y,z)

    conn.commit() 

main()

I appreciate all your help! Thanks

2
  • Please post more code, I get u'\n\u6a2a(F\n \u6a6b)\n' from td.text just fine. Feb 26, 2013 at 0:17
  • @Pavel Anossov Thanks for helping out :)
    – user805981
    Feb 26, 2013 at 0:19

1 Answer 1

1

That site claims to be in gb2312, but it's not. This should fix it:

url = 'http://www.zein.se/patrick/3000char.html'
soup = bs4.BeautifulSoup(urllib2.urlopen(url).read(), from_encoding='gb18030')

or just

soup = bs4.BeautifulSoup(urllib2.urlopen(url).read(), from_encoding='gbk')

Your browser figured it out, but BeautifulSoup needs a hint.

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.