0

I'm crawling a site with high JS and websocket load. Using multythreading. As soon as I run more than 15 simultaneous copies, my server (Ryzen 7 3700 8-core with 64Gb RAM) become unstable. CPU is used for 100%

It seems i used all keys for speeding up chromium. What did i miss?

My goal is to run at least 50 at the same time.

def driver_activate(profile, url):
    caps = DesiredCapabilities().CHROME
    caps["pageLoadStrategy"] = "none"
    options = webdriver.ChromeOptions()
    options.add_argument("--window-size=1024,700")
    options.add_argument('--allow-profiles-outside-user-dir')
    options.add_argument('--enable-profile-shortcut-manager')
    options.add_argument(r'user-data-dir=/Session/{}'.format(profile))
    options.add_argument('--profile-directory=Profile 1')
    options.add_argument("--no-sandbox")
    options.add_argument("--remote-debugging-pipe")
    options.add_argument("--disable-proxy-certificate-handler")
    options.add_argument('--disable-blink-features=AutomationControlled')
    options.add_argument('--profiling-flush=5')
    options.add_argument('--enable-aggressive-domstorage-flushing')
    options.add_argument('--log-level=3')
    options.add_argument("--headless=new")
    options.add_experimental_option('excludeSwitches', ['enable-logging'])
    options.add_experimental_option('excludeSwitches', ['enable-automation'])
    options.add_experimental_option('useAutomationExtension', False)
    options.add_argument("--disable-extensions")
    options.add_argument("--enable-unsafe-swiftshader")
    options.add_argument("--disable-frame-rate-limit")
    options.add_argument("--disable-gpu-vsync")
    options.add_argument("--max-gum-fp=10")
    options.add_experimental_option(
        "prefs", {
        # block image loading
        "profile.managed_default_content_settings.images": 2,
        }
    )
    options.add_argument('--disable-dev-shm-usage')
    options.add_argument('--disable-renderer-backgrounding')
    options.add_argument('--disable-background-timer-throttling')
    options.add_argument('--disable-backgrounding-occluded-windows')
    options.add_argument('--disable-client-side-phishing-detection')
    options.add_argument('--disable-crash-reporter')
    options.add_argument('--disable-oopr-debug-crash-dump')
    options.add_argument('--no-crash-upload')
    options.add_argument('--disable-gpu')
    options.add_argument('--disable-low-res-tiling')
    options.page_load_strategy = 'normal'

    driver = webdriver.Chrome(options=options, seleniumwire_options={'proxy': {'http': f'http://{proxy}', 'https': f'https://{proxy}',}})
    driver.get(url)
    driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
      "source": """
        Object.defineProperty(navigator, 'webdriver', {
          get: () => undefined
        })
      """
    })

    ............
    driver.quit()

for cycle in range(iterations):
    with concurrent.futures.ThreadPoolExecutor(max_workers=15) as executor:
        futures = []
        for i in range(15):
            future = executor.submit(driver_activate, profile, url)
            futures.append(future)
        for future in concurrent.futures.as_completed(futures):
                    pass

tried to speed up, was looking for changing FPS but didnt find how

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.