As an alternative to the excellent current answer which injects a MutationObserver using evaluate which forwards the data to an exposed Node function, Puppeteer offers a higher-level function called page.waitForFunction that blocks on an arbitrary predicate and uses either a MutationObserver or requestAnimationFrame under the hood to determine when to re-evaluate the predicate.
Calling page.waitForFunction in a loop might add overhead since each new call involves registering a fresh observer or RAF. You'd have to profile for your use case -- this isn't something I'd worry much about prematurely, though.
That said, the RAF option may provide tighter latency than MO for the cost of some extra CPU cycles to poll constantly.
Here's a minimal example on the following site that offers a periodically updating feed:
const wait = ms => new Promise(r => setTimeout(r, ms));
const r = (lo, hi) => ~~(Math.random() * (hi - lo) + lo);
const randomString = n =>
[...Array(n)].map(() => String.fromCharCode(r(97, 123))).join("")
;
(async () => {
for (let i = 0; i < 500; i++) {
const el = document.createElement("div");
document.body.appendChild(el);
el.innerText = randomString(r(5, 15));
await wait(r(1000, 5000));
}
})();
const puppeteer = require("puppeteer");
const html = `
<html><body><div class="container"></div><script>
const wait = ms => new Promise(r => setTimeout(r, ms));
const r = (lo, hi) => ~~(Math.random() * (hi - lo) + lo);
const randomString = n =>
[...Array(n)].map(() => String.fromCharCode(r(97, 123))).join("")
;
(async () => {
for (;;) {
const el = document.createElement("div");
document.querySelector(".container").appendChild(el);
el.innerText = randomString(r(5, 15));
await wait(r(1000, 5000));
}
})();
</script></body></html>
`;
let browser;
(async () => {
browser = await puppeteer.launch({headless: false});
const [page] = await browser.pages();
await page.setContent(html);
for (;;) {
await page.waitForFunction((el, oldLength) =>
el.children.length > oldLength, // predicate
{polling: "mutation" /* or: "raf" */, timeout: 0}, // wFF options
await page.$(".container"), // elem to watch
await page.$eval(".container", el => el.children.length), // oldLength
);
const selMostRecent = ".container div:last-child";
console.log(await page.$eval(selMostRecent, el => el.textContent));
}
})()
.catch(err => console.error(err))
.finally(() => browser?.close())
;
See also: