usr bin env python3 import logging import asyncio from multiprocessing

#!/usr/bin/env python3
import logging
import asyncio
from multiprocessing import cpu_count

from pyppeteer import errors
from pyppeteer import launch


async def hit_target(target_url, page_timeout):
    browser = None
    try:
        browser = await launch()
        try:
            page = await browser.newPage()
            await page.setUserAgent('asdf')
            res = await page.goto(target_url, timeout=page_timeout)
        except (errors.TimeoutError, errors.PageError) as ex:
            print('[FAIL] %s' % ex)
        else:
            print('[OK] %s' % target_url)
    finally:
        if browser:
            await browser.close()


async def thread_worker(taskq, wnum=None):
    while True:
        try:
            target_url = taskq.get_nowait()
        except asyncio.QueueEmpty:
            return
        else:
            if wnum == 0:
                raise Exception('Foo')
            page_timeout_ms = 20000
            # Wait for (page timeout + 50%)
            try:
                await asyncio.wait_for(
                    hit_target(target_url, page_timeout_ms),
                    timeout=((page_timeout_ms / 1000) * 1.5)
                )
            except asyncio.TimeoutError:
                print('Call to hit_target() timed out!')


async def main_async():
    taskq = asyncio.Queue()
    for _ in range(100):
        await taskq.put('https://yandex.ru/')


    pool = []
    for x in range(cpu_count() * 2):
        th = asyncio.ensure_future(thread_worker(taskq, x))
        pool.append(th)
    try:
        await asyncio.gather(*pool)
    except Exception as ex:
        logging.exception('Worker thread failed')
        for th in pool:
            print('Cancelling task %s' % th)
            th.cancel()


if __name__ == '__main__':
    asyncio.get_event_loop().set_debug(True)
    asyncio.get_event_loop().run_until_complete(main_async())