0

I am building a scraper for Yellow Pages and I am trying to figure out how to get the website for each listing in search results included in the CSV output. The Website is listed right next to each listing.

I am using beautiful soup 4 on Python. Any help would be appreciated.

import sys
reload(sys)
sys.setdefaultencoding('utf-8')

import requests
import csv
import httplib2

from bs4 import BeautifulSoup

url = "http://www.yellowpages.com/search?search_terms=coffee&geo_location_terms=Nashville%2C+TN"
r = requests.get(url)

soup = BeautifulSoup(r.content)

links = soup.find_all("a")

for link in links:
    print "<a href='%s'>%s</a>" %(link.get("href"), link.text)

g_data = soup.find_all("div", {"class": "info"})

yellowpages_list=[]

for item in g_data:
    business_name = item.contents[0].find_all("a", {"class": "business-name"})[0].text
    try:
        street_address = item.contents[1].find_all("span", {"itemprop": "streetAddress"})[0].text
    except:
        pass
    try:
        locality = item.contents[1].find_all("span", {"itemprop": "addressLocality"})[0].text.replace(',','')
    except:
        pass
    try:
        region = item.contents[1].find_all("span", {"itemprop": "addressRegion"})[0].text
    except:
        pass
    try:
        postalcode = item.contents[1].find_all("span", {"itemprop": "postalCode"})[0].text
    except:
        pass
    try:
        primary = item.contents[1].find_all("li", {"class": "primary"})[0].text
    except:
        pass

    business=[business_name,street_address,locality,region,postalcode]
    yellowpages_list.append(business)

with open ('filename.csv','wb') as file:
    writer=csv.writer(file)
    for row in yellowpages_list:
        writer.writerow(row)

1 Answer 1

1

It looks like YellowPages is using class="track-visit-website" on their visit-website links.

Try this:

try:
    url = item.find("a", {"class": "track-visit-website"})["href"]
except:
    pass

P.S. You don't need to use find_all() and then get the first element of the array. find() will do the same thing for you.

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.

Not the answer you're looking for? Browse other questions tagged or ask your own question.