Skip to main content

Building a site scanner

· 4 min read

Ant colony galleries aboriginal art


In this post we will show you how we built the site scanner scan_domain feature in our SDK. We will go over collecting JS error logs and network calls with Chomedriver and Chrome Developer Protocol (CDP) in Python with Selenium.


We want to crawl a given domain with a given depth and collect JS errors from the console as well as HTTP requests with status code >= 400.

General design

We are going to traverse the site BFS. The link_manager will keep track of visited urls for us and handle the traversal logic.
from urllib.parse import urlparse
class SmartDriver:
def __init__(self, webdriver, api_key, options={}):

def scan_domain(self, url, max_depth=5):
self.domain = urlparse(url).netloc # extract the domain
self.link_manager.add_link(url, url, depth=0)
while self.link_manager.has_more_links():
referrer, link, depth = self.link_manager.get_link()
if depth <= max_depth:
self.process_link(link, referrer, depth)
else:'Skipping link {link} because it is too deep {depth}')
except Exception as e:
log.error(f"Error processing link {link}: {e}")

Initializing Chromedriver with CDP

To collect the right data we need console and performance logs from Chromedriver. The Chrome Developer Protocol (CDP) gives us access to this, we need to enable these additional features with a DesiredCapability goog:loggingPrefs, we also need to issue two CDP commands to enable these logs.
from selenium.webdriver import DesiredCapabilities
from import Service
from import ChromeDriverManager

# make chrome log requests
capabilities = DesiredCapabilities.CHROME
capabilities['goog:loggingPrefs'] = {'performance': 'ALL', 'browser': 'ALL'}
driver = Chrome(service=Service(ChromeDriverManager().install()), desired_capabilities=capabilities)
driver.execute_cdp_cmd("Network.enable", {})
driver.execute_cdp_cmd("Console.enable", {})

For each link, we will clear the logs, get the url then check the console error logs and the HTTP status codes.
    def process_link(self, link, referrer, depth):
_ = self.driver.get_log('browser') # clear logs
_ = self.driver.get_log('performance')
sleep(2.0)"Processing link {link}")
console_logs = self.driver.get_log("browser")
self.process_console_logs(console_logs, link)

perf_logs = self.driver.get_log("performance")
self.process_perf_logs(perf_logs, link)'Visited {link}')
local_referrer = link

links = self.driver.find_elements(By.TAG_NAME, 'a')
for link in links:
if urlparse(link.get_attribute('href')).netloc == self.domain:
self.link_manager.add_link(local_referrer, link.get_attribute('href'), depth + 1)

Processing the console logs

We simply look at the messages from the console, check for the a SEVERE log or for the word 'error'.
    def is_js_error(message):
#implement some logic here to filter out the errors you want
return 'error' in message.lower()

def process_console_logs(self, console_logs, link):
for l in console_logs:
if (l['level'] == 'SEVERE'):
log.debug(f"Bad JS: {l['message']}")
if is_js_error(l['message']):
log.debug(f"Bad JS: {l['message']}")

Processing the network logs

We get the log messages in JSON format, so we load them up in memory and filter for Network.responseReceived. After that we simply look at the status code to decide which requests are bad.

    def process_perf_logs(self, perf_logs, link):
perf_logs = [json.loads(lr["message"])["message"] for lr in perf_logs]
responses = [l for l in perf_logs if l["method"] == "Network.responseReceived"]
for r in responses:
status = r['params']['response']['status']
if status >= 400:
log.debug(f"Bad request: {status} {r['params']['response']['url']}")
self.save_bad_request(r['params']['response']['url'], status, link)

Everything together

When putting everything togethere, we have a simple crawler that registeres JS errors and bad HTTP requests. This whole feature is already implemented in our SDK, to use it simply make sure you set the desired capability googLoggingPrefs to 'performance' and 'browser'.

Here is a sample script to scan all the URLs in a text file.
from time import sleep

from selenium.webdriver import Chrome
from import By
from import Service
from import ChromeDriverManager
import logging

from devtools_ai.selenium import SmartDriver
from import Options
from selenium.webdriver import DesiredCapabilities

# import actionchains
from selenium.webdriver.common.action_chains import ActionChains
import os

def scan(url):
"""Main driver"""
chrome_options = Options()

# make chrome log requests
capabilities = DesiredCapabilities.CHROME
capabilities['goog:loggingPrefs'] = {'performance': 'ALL', 'browser': 'ALL'}
driver = Chrome(service=Service(ChromeDriverManager().install()), desired_capabilities=capabilities, options=chrome_options)
# Convert chrome_driver to smartDriver
driver = SmartDriver(driver, api_key="??API_KEY??") # get your API key at

# Navigate to
driver.scan_domain(url, max_depth=4)
except Exception as e:

if __name__ == "__main__":
with open('urls.txt') as f:
urls = f.readlines()
for url in urls: