I am building a scraper for Yellow Pages and I am trying to figure out how to get the website for each listing in search results included in the CSV output. The Website is listed right next to each listing.
I am using beautiful soup 4 on Python. Any help would be appreciated.
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import requests
import csv
import httplib2
from bs4 import BeautifulSoup
url = "http://www.yellowpages.com/search?search_terms=coffee&geo_location_terms=Nashville%2C+TN"
r = requests.get(url)
soup = BeautifulSoup(r.content)
links = soup.find_all("a")
for link in links:
print "<a href='%s'>%s</a>" %(link.get("href"), link.text)
g_data = soup.find_all("div", {"class": "info"})
yellowpages_list=[]
for item in g_data:
business_name = item.contents[0].find_all("a", {"class": "business-name"})[0].text
try:
street_address = item.contents[1].find_all("span", {"itemprop": "streetAddress"})[0].text
except:
pass
try:
locality = item.contents[1].find_all("span", {"itemprop": "addressLocality"})[0].text.replace(',','')
except:
pass
try:
region = item.contents[1].find_all("span", {"itemprop": "addressRegion"})[0].text
except:
pass
try:
postalcode = item.contents[1].find_all("span", {"itemprop": "postalCode"})[0].text
except:
pass
try:
primary = item.contents[1].find_all("li", {"class": "primary"})[0].text
except:
pass
business=[business_name,street_address,locality,region,postalcode]
yellowpages_list.append(business)
with open ('filename.csv','wb') as file:
writer=csv.writer(file)
for row in yellowpages_list:
writer.writerow(row)