0

I am scraping data from a webpage, pagination also works. I have a table with tr. For each tr, there is position, title, URL.

In function .map((index, element) => { i want to call for each tr "async url => { await page.goto(${url});" and go to details scrape data from details page and return to lists.

how can i do it? (call async within .map())

      const puppeteer = require("puppeteer");
      const cheerio = require("cheerio");


      async function scrapeListings(page) {
     await page.goto(
       "https://www.example.com/podcast-detail/nth28-2ef41/99%25-Invisible- 
      Podcast",
        { waitUntil: ["load", "domcontentloaded", "networkidle0"] }
    );

     let moreResults = true;

     while (moreResults) {
     console.log(moreResults);
      try {
      const html = await page.content();
      const $ = cheerio.load(html);
      const listings = $("#yw0 > table > tbody > tr")
        .map((index, element) => {
          const position = $(element)
            .find("td.id.tc")
            .text();
          const title = $(element)
            .find("a.title.listen-now")
            .text();
          const url = $(element)
            .find("a")
            .attr("href");
          const datetime = $(element)
            .find("span.datetime")
            .text();
          async url => {
            await page.goto(`${url}`);
            //Open URL for details
            //SCRAPE Details
            //return value and store to listing
          };
          return { position, title, datetime, url };
        })
        .get();

      await sleep(1000);
      element = await page.$x(`//a[contains(text(),'→')]`);
      await element[0].click();
      console.log(listings);
      //await page.waitForNavigation();
    } catch (error) {
      moreResults = false;
      }
      }
     }

     async function sleep(miliseconds) {
     return new Promise(resolve => setTimeout(resolve, miliseconds));
      }

    async function main() {
     //await connectToMongoDb();

    const browser = await puppeteer.launch({ headless: false });
    const page = await browser.newPage();

     await page.setViewport({ width: 1920, height: 1057 });

     listings = await scrapeListings(page);

    console.log(listings);
     //await browser.close();
   }

    main();

2 Answers 2

1

I try to recode your script to use only Puppeteer. No cheerio needed.

const puppeteer = require("puppeteer")

const pageURL = 'https://www.podbean.com/podcast-detail/nth28-2ef41/99%25-Invisible//page/25'
const uaString = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3239.108 Safari/537.36'

let podCastsAll = []

;(async () => {

    const browser = await puppeteer.launch ({
        headless: false,
        devtools: false
    })

    const [page] = await browser.pages ()

    page.setDefaultNavigationTimeout (0)
    page.setUserAgent ( uaString )
    page.setViewport ({ width: 1366, height: 768 })
    page.setRequestInterception ( true )

    page.on ( 'request', async request => {
        if ( request.resourceType () === 'image' || request.resourceType () === 'font' ) {
            request.abort ()
        } else {
            request.continue ()
        }
    })

    const getPodcast = async url => {
        const pagePod = await browser.newPage ()
        const openPod = await pagePod.goto ( url, { waitUntil: `networkidle0`, timeout: 0 })
        const waitPod = await pagePod.waitForSelector ('p.pod-name')

        const podName = await pagePod.evaluate ( () => document.querySelector('p.pod-name').innerText )
        const podTime = await pagePod.evaluate ( () => document.querySelector('.time > span').innerText )
        const podDesc = await pagePod.evaluate ( () => document.querySelector('#desc').innerText )
        const podLink = await pagePod.evaluate ( () => document.querySelector('.player iframe').src )

        const openLink = await pagePod.goto ( podLink, { waitUntil: `networkidle0`, timeout: 0 })
        const waitElem = await pagePod.waitForSelector ( 'audio[preload] > source[src]', { timeout: 0 })

        const podFile = await pagePod.evaluate ( () => document.querySelector('audio[preload] > source[src]').src )

        const closeIt = await pagePod.close ()

        return ({
            name : podName,
            time : podTime,
            desc : podDesc,
            link : podLink,
            file : podFile
        })
    }

    const start = await page.goto ( pageURL, { waitUntil: `networkidle2`, timeout: 0 })

    const getThisPage = async () => {
        const wait = await page.waitForSelector ('.pagination > ul', { timeout: 0 })
        const items = await page.waitForSelector ('#yw0 > table > tbody > tr')

        const [podcasts, podNumber, podTitle, podDate, podURL] = await page.evaluate ( () => {
            let podcasts = document.querySelectorAll('#yw0 > table > tbody > tr')

            let podNumber = []
            let podTitle = []
            let podDate = []
            let podURL = []

            podcasts.forEach( elem => {
                podNumber.push( elem.querySelector('td.id.tc').innerText )
                podTitle.push( elem.querySelector('a.title.listen-now').innerText )
                podDate.push( elem.querySelector('span.datetime').innerText )
                podURL.push( elem.querySelector('a[href]').href )
            })

            return [podcasts, podNumber, podTitle, podDate, podURL]
        })

        let podcastsArray = []
        let podcastDetail = []

        for ( let num in podcasts ) {
            podcastDetail = []

            podcastsArray[num] = {
                number: podNumber[num],
                title: podTitle[num],
                datetime: podDate[num],
                link: podURL[num]
            }

            podcastDetail[num] = await getPodcast ( podURL[num] )

            podcastsArray[num].desc = podcastDetail[num].desc
            podcastsArray[num].play = podcastDetail[num].link
            podcastsArray[num].source = podcastDetail[num].file

            await page.waitFor (1000)
        }

        podCastsAll = podCastsAll.concat( podcastsArray )

        var nextEnabled = await page.evaluate ( () => document.querySelector( '.pagination > ul > li.next.disabled' ) === null )
        console.log ('NEXT PAGE >>>')
        if (nextEnabled) {
            var thisPage = await page.evaluate ( () => document.querySelector( '.pagination > ul > li.active > a[href]' ).innerText )
            const next = await page.evaluate ( () => document.querySelector( '.pagination > ul > li.next > a' ).click() )

            while ( thisPage === await page.evaluate ( () =>  document.querySelector( '.pagination > ul > li.active > a[href]' ).innerText ) ) {
                await page.waitFor (100)
            }

            await getThisPage ()
        } else {
            console.log ('FINISHED!\n')
            console.log (podCastsAll)
        }
    }

    await getThisPage()

})()
1
  • This is great, thank you very much. It works really fine. Dec 13, 2019 at 15:16
0

if somebody need to click on link and open a new page instead to call url this is code: maybe some has a better solution, be free to share thanks a lot

const getPodcast = async url => {
    const pageTarget = page.target();
    await page.click(".table > .items > tr > td > .title");
    const newTarget = await browser.waitForTarget(
    target => target.opener() === pageTarget
    );
    const pagePod = await newTarget.page();
    console.log("current page count ", (await browser.pages()).length);
    //const pagePod = await browser.newPage();
    //const openPod = await pagePod.goto(url, {  waitUntil: `networkidle0`,   
    timeout: 0 });

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.