I'm crawling a site with high JS and websocket load. Using multythreading. As soon as I run more than 15 simultaneous copies, my server (Ryzen 7 3700 8-core with 64Gb RAM) become unstable. CPU is used for 100%
It seems i used all keys for speeding up chromium. What did i miss?
My goal is to run at least 50 at the same time.
def driver_activate(profile, url):
caps = DesiredCapabilities().CHROME
caps["pageLoadStrategy"] = "none"
options = webdriver.ChromeOptions()
options.add_argument("--window-size=1024,700")
options.add_argument('--allow-profiles-outside-user-dir')
options.add_argument('--enable-profile-shortcut-manager')
options.add_argument(r'user-data-dir=/Session/{}'.format(profile))
options.add_argument('--profile-directory=Profile 1')
options.add_argument("--no-sandbox")
options.add_argument("--remote-debugging-pipe")
options.add_argument("--disable-proxy-certificate-handler")
options.add_argument('--disable-blink-features=AutomationControlled')
options.add_argument('--profiling-flush=5')
options.add_argument('--enable-aggressive-domstorage-flushing')
options.add_argument('--log-level=3')
options.add_argument("--headless=new")
options.add_experimental_option('excludeSwitches', ['enable-logging'])
options.add_experimental_option('excludeSwitches', ['enable-automation'])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument("--disable-extensions")
options.add_argument("--enable-unsafe-swiftshader")
options.add_argument("--disable-frame-rate-limit")
options.add_argument("--disable-gpu-vsync")
options.add_argument("--max-gum-fp=10")
options.add_experimental_option(
"prefs", {
# block image loading
"profile.managed_default_content_settings.images": 2,
}
)
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--disable-renderer-backgrounding')
options.add_argument('--disable-background-timer-throttling')
options.add_argument('--disable-backgrounding-occluded-windows')
options.add_argument('--disable-client-side-phishing-detection')
options.add_argument('--disable-crash-reporter')
options.add_argument('--disable-oopr-debug-crash-dump')
options.add_argument('--no-crash-upload')
options.add_argument('--disable-gpu')
options.add_argument('--disable-low-res-tiling')
options.page_load_strategy = 'normal'
driver = webdriver.Chrome(options=options, seleniumwire_options={'proxy': {'http': f'http://{proxy}', 'https': f'https://{proxy}',}})
driver.get(url)
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
"source": """
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
})
"""
})
............
driver.quit()
for cycle in range(iterations):
with concurrent.futures.ThreadPoolExecutor(max_workers=15) as executor:
futures = []
for i in range(15):
future = executor.submit(driver_activate, profile, url)
futures.append(future)
for future in concurrent.futures.as_completed(futures):
pass
tried to speed up, was looking for changing FPS but didnt find how