Use array of keywords and loop through script in Playwright

Question

So, I am trying to scrape a couple of searchengines with a couple of search phrases with Playwright. Running the script with one query is working.

Working:

  const { chromium } = require('playwright');

  (async () => {
  const browser = await chromium.launch({ headless: false, slowMo: 250 });
  const context = await browser.newContext()
  const page = await context.newPage();

  const keyWord = ('Arsenal');

  await page.goto('https://duckduckgo.com/');
  await page.fill('//input[@name="q"]',keyWord);
  await page.keyboard.press('Enter');

  const getOne =  ('  (//h2[@class="result__title"])[9]    ');
  await page.waitForSelector(getOne)
  const pushOne = await page.$(getOne);
  const One = await pushOne.evaluate(element => element.innerText);
  console.log(One);

  await page.goto('https://yandex.com/');
  await page.fill('//input[@aria-label="Request"]', keyWord);
  await page.keyboard.press('Enter');

  const getTwo =  ('  //li[@data-first-snippet] //div[@class="organic__url-text"]    ');
  await page.waitForSelector(getTwo)
  const pushTwo = await page.$(getTwo);
  const Two = await pushTwo.evaluate(element => element.innerText);
  console.log(Two);

  await browser.close()
  })()

But when I use an array with phrases (keyWordlist) I fail to get the script running. Have searched around for using Array with 'For' and 'Foreach' loops, but haven't been able to fix it. I want to run the different keywords through the different searchengines and list the results. For 3 keywords in two searchengines that would get 6 results.

  const { chromium } = require('playwright');

  (async () => {
  const browser = await chromium.launch({ headless: false, slowMo: 250 });
  const context = await browser.newContext()
  const page = await context.newPage();


  let kewWordlist = ['Arsenal', 'Liverpool', 'Ajax']
  
  for (var i=0; i<=kewWordlist.length; i++) {
        // for (const i in kewWordlist){
        async () => {
              
              const keyWord = kewWordlist[i];

              await page.goto('https://duckduckgo.com/');
              await page.fill('//input[@name="q"]',keyWord);
              // await page.fill('//input[@name="q"]',[i]);
              // await page.fill('//input[@name="q"]',`${keyWord}`);
              await page.keyboard.press('Enter');


              const getOne =  ('  (//h2[@class="result__title"])[9]    ');
              await page.waitForSelector(getOne)
              const pushOne = await page.$(getOne);
              const One = await pushOne.evaluate(element => element.innerText);
              console.log(One);


              // await page.goto('https://yandex.com/');
              // await page.fill('//input[@aria-label="Request"]', keyWord);
              // await page.keyboard.press('Enter');

              // const getTwo =  ('  //li[@data-first-snippet] //div[@class="organic__url-text"]    ');
              // await page.waitForSelector(getTwo)
              // const pushTwo = await page.$(getTwo);
              // const Two = await pushTwo.evaluate(element => element.innerText);
              // console.log(Two);

        }}
        await browser.close()
  })()

If anyone has some pointers on how to solve this, much obliged.

Answer 1

maybe the result selectors needs some tweaking but I think this is what you were looking for:

test.only('search search engines', async({page, context}) => {

    const search = [
        {
            name: 'yandex',
            url: 'https://yandex.com/',
            elementFill: '//input[@aria-label="Request"]',
            elementResult: '//li[@data-first-snippet] //div[@class="organic__url-text"]'
        },
        {
            name: 'google',
            url: 'https://www.google.nl',
            elementFill: '//input[@name="q"]',
            elementResult: '(//h2[@class="result__title"])[9]'
        },
        {
            name: '',
            url: 'https://duckduckgo.com/',
            elementFill: '//input[@name="q"]',
            elementResult: '(//h2[@class="result__title"])[9]'
        }
    ]
    const kewWordlist = ['Arsenal', 'Liverpool', 'Ajax']

    for (let i = 0; i < search.length; i++) {
        const searchName = search[i].name
        const searchResult = search[i].elementResult
        const searchFill = search[i].elementFill

        const searchPage = await context.newPage()
        await searchPage.waitForLoadState()
        await searchPage.goto(`${search[i].url}`)

        for (let i = 0; i < kewWordlist.length; i++) {
            await searchPage.fill(searchFill,kewWordlist[i])
            await searchPage.keyboard.press('Enter')
            await searchPage.waitForSelector(searchResult)
            const result = await page.$(searchResult)
            console.log(`${searchName}: ${result} `)
        }
    }
})

Answer 2

The reason your loop isn't working is that you have an async function inside of it that you never call. There are a few ways you could go about this:

You could take your first version, have it accept a word to search, and run that over each element of the array:

const searchOneKeyword = async (keyWord) => {
  const browser = await chromium.launch({ headless: false, slowMo: 250 });
  const context = await browser.newContext()
  const page = await context.newPage();

  // rest of code
}

const kewWordList = ['Arsenal', 'Liverpool', 'Ajax']

keyWordList.forEach((k) => {
  searchOneKeyword(k)
})

Or if you'd like to keep the same browser instance, you can do it in a loop in the function:

const search = async (words) => {
  const browser = await chromium.launch({ headless: false, slowMo: 250 });
  const context = await browser.newContext()
  const page = await context.newPage();

  for (const keyWord of words) {
    await page.goto('https://duckduckgo.com/');
    await page.fill('//input[@name="q"]',keyWord);
    await page.keyboard.press('Enter');

    const getOne =  ('  (//h2[@class="result__title"])[9]    ');
    await page.waitForSelector(getOne)
    const pushOne = await page.$(getOne);
    const One = await pushOne.evaluate(element => element.innerText);
    console.log(One);

    // etc.
  }

  await browser.close()
}

search(keyWordList)

In both of those cases, you're logging, but never returning anything, so if you need that data in another function afterwards, you'd have to change that. Example:

const search = async (words) => {
  const browser = await chromium.launch({ headless: false, slowMo: 250 });
  const context = await browser.newContext()
  const page = await context.newPage();

  const results = await Promise.all(words.map((keyWord) => {
    await page.goto('https://duckduckgo.com/');
    await page.fill('//input[@name="q"]',keyWord);
    await page.keyboard.press('Enter');

    // etc.
    
    return [ One, Two ]
  }))

  await browser.close()
  return results
}

search(keyWordList).then((results) => { console.log(results.flat()) })

Answer 3

I have spent a couple of hours trying to get the script working based on your suggestions. No result unfortunately. I get errors like 'await is only valid in async function' and 'Unreachable code detected'. Searched for other examples, for some inspiration, but none found. If you or someone else has a suggestion, please share: This is code I have now:

  const { chromium } = require('playwright');

  let keyWordList = ['Arsenal', 'Liverpool', 'Ajax']

  const search = async function words()  {
        const browser = await chromium.launch({ headless: false, slowMo: 250 });
        const context = await browser.newContext()
        const page = await context.newPage();
        }
        
        const results = await Promise.all(words.map(keyWord))

        //DUCKDUCKGO
        await page.goto('https://duckduckgo.com/');
        await page.fill('//input[@name="q"]',keyWord);
        await page.keyboard.press('Enter');
      
        const getOne =  ('  (//h2[@class="result__title"])[9]    ');
        await page.waitForSelector(getOne)
        const pushOne = await page.$(getOne);
        const One = await pushOne.evaluate(element => element.innerText);

        //YANDEX
        await page.goto('https://yandex.com/');
        await page.fill('//input[@aria-label="Request"]', keyWord);
        await page.keyboard.press('Enter');
  
        const getTwo =  ('  //li[@data-first-snippet] //div[@class="organic__url-text"]    ');
        await page.waitForSelector(getTwo)
        const pushTwo = await page.$(getTwo);
        const Two = await pushTwo.evaluate(element => element.innerText);
        console.log(Two);

        return [ One , Two ]
        return results
        
        search(keyWordList).then((results) => { console.log(results.flat())
              await browser.close();
  })