8000 BBOT server improvements by TheTechromancer · Pull Request #2438 · blacklanternsecurity/bbot · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

BBOT server improvements #2438

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
May 9, 2025
Merged
72 changes: 72 additions & 0 deletions bbot/constants.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
SCAN_STATUS_QUEUED = 0
SCAN_STATUS_NOT_STARTED = 1
SCAN_STATUS_STARTING = 2
SCAN_STATUS_RUNNING = 3
SCAN_STATUS_FINISHING = 4
SCAN_STATUS_ABORTING = 5
SCAN_STATUS_FINISHED = 6
SCAN_STATUS_FAILED = 7
SCAN_STATUS_ABORTED = 8


SCAN_STATUSES = {
"QUEUED": SCAN_STATUS_QUEUED,
"NOT_STARTED": SCAN_STATUS_NOT_STARTED,
"STARTING": SCAN_STATUS_STARTING,
"RUNNING": SCAN_STATUS_RUNNING,
"FINISHING": SCAN_STATUS_FINISHING,
"ABORTING": SCAN_STATUS_ABORTING,
"FINISHED": SCAN_STATUS_FINISHED,
"FAILED": SCAN_STATUS_FAILED,
"ABORTED": SCAN_STATUS_ABORTED,
}

SCAN_STATUS_CODES = {v: k for k, v in SCAN_STATUSES.items()}


def is_valid_scan_status(status):
"""
Check if a status is a valid scan status
"""
return status in SCAN_STATUSES


def is_valid_scan_status_code(status):
"""
Check if a status is a valid scan status code
"""
return status in SCAN_STATUS_CODES


def get_scan_status_name(status):
"""
Convert a numeric scan status code to a string status name
"""
try:
if isinstance(status, str):
if not is_valid_scan_status(status):
raise ValueError(f"Invalid scan status: {status}")
return status
elif isinstance(status, int):
return SCAN_STATUS_CODES[status]
else:
raise ValueError(f"Invalid scan status: {status} (must be int or str)")
except KeyError:
raise ValueError(f"Invalid scan status: {status}")


def get_scan_status_code(status):
"""
Convert a scan status string to a numeric status code
"""
try:
if isinstance(status, int):
if not is_valid_scan_status_code(status):
raise ValueError(f"Invalid scan status code: {status}")
return status
elif isinstance(status, str):
return SCAN_STATUSES[status]
else:
raise ValueError(f"Invalid scan status: {status} (must be int or str)")
except KeyError:
raise ValueError(f"Invalid scan status: {status}")
6 changes: 5 additions & 1 deletion bbot/core/config/logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import sys
import atexit
import logging
import threading
from copy import copy
import multiprocessing
import logging.handlers
Expand Down Expand Up @@ -93,7 +94,10 @@ def cleanup_logging(self):

# Stop queue listener
with suppress(Exception):
self.listener.stop()
stop_thread = threading.Thread(target=self.listener.stop)
stop_thread.daemon = True
stop_thread.start()
stop_thread.join()

def setup_queue_handler(self, logging_queue=None, log_level=logging.DEBUG):
if logging_queue is None:
Expand Down
14 changes: 12 additions & 2 deletions bbot/core/event/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -403,14 +403,15 @@ def host_filterable(self):
@property
def port(self):
self.host
if self._port:
return self._port
if getattr(self, "parsed_url", None):
if self.parsed_url.port is not None:
return self.parsed_url.port
elif self.parsed_url.scheme == "https":
return 443
elif self.parsed_url.scheme == "http":
return 80
return self._port

@property
def netloc(self):
Expand Down Expand Up @@ -1086,9 +1087,10 @@ def __init__(self, *args, **kwargs):
parent_path = parent.data.get("path", None)
if parent_path is not None:
self.data["path"] = parent_path
# inherit closest host
# inherit closest host+port
if parent.host:
self.data["host"] = str(parent.host)
self._port = parent.port
# we do this to refresh the hash
self.data = self.data
break
Expand All @@ -1099,6 +1101,7 @@ def __init__(self, *args, **kwargs):

class DictPathEvent(DictEvent):
def sanitize_data(self, data):
data = super().sanitize_data(data)
new_data = dict(data)
new_data["path"] = str(new_data["path"])
file_blobs = getattr(self.scan, "_file_blobs", False)
Expand Down Expand Up @@ -1557,6 +1560,7 @@ class VULNERABILITY(ClosestHostEvent):
}

def sanitize_data(self, data):
data = super().sanitize_data(data)
self.add_tag(data["severity"].lower())
return data

Expand Down Expand Up @@ -1601,6 +1605,11 @@ class _data_validator(BaseModel):
_validate_url = field_validator("url")(validators.validate_url)
_validate_host = field_validator("host")(validators.validate_host)

def _sanitize_data(self, data):
data = super()._sanitize_data(data)
data["technology"] = data["technology"].lower()
return data

def _data_id(self):
# dedupe by host+port+tech
tech = self.data.get("technology", "")
Expand Down Expand Up @@ -1730,6 +1739,7 @@ class MOBILE_APP(DictEvent):
_always_emit = True

def _sanitize_data(self, data):
data = super()._sanitize_data(data)
if isinstance(data, str):
data = {"url": data}
if "url" not in data:
Expand Down
4 changes: 2 additions & 2 deletions bbot/modules/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -708,7 +708,7 @@ async def _worker(self):
self.trace(f"RuntimeError in module {self.name}: {e}")
except BaseException as e:
if self.helpers.in_exception_chain(e, (KeyboardInterrupt,)):
self.scan.stop()
await self.scan.async_stop()
else:
self.error(f"Critical failure in module {self.name}: {e}")
self.error(traceback.format_exc())
Expand Down Expand Up @@ -1730,7 +1730,7 @@ async def _worker(self):
self.trace(f"RuntimeError in intercept module {self.name}: {e}")
except BaseException as e:
if self.helpers.in_exception_chain(e, (KeyboardInterrupt,)):
self.scan.stop()
await self.scan.async_stop()
else:
self.critical(f"C 9E7A ritical failure in intercept module {self.name}: {e}")
self.critical(traceback.format_exc())
Expand Down
13 changes: 7 additions & 6 deletions bbot/scanner/dispatcher.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import logging
import traceback

log = logging.getLogger("bbot.scanner.dispatcher")
import contextlib


class Dispatcher:
Expand All @@ -11,6 +10,7 @@ class Dispatcher:

def set_scan(self, scan):
self.scan = scan
self.log = logging.getLogger("bbot.scanner.dispatcher")

async def on_start(self, scan):
return
Expand All @@ -24,9 +24,10 @@ async def on_status(self, status, scan_id):
"""
self.scan.debug(f"Setting scan status to {status}")

async def catch(self, callback, *args, **kwargs):
@contextlib.contextmanager
def catch(self):
try:
return await callback(*args, **kwargs)
yield
except Exception as e:
log.error(f"Error in {callback.__qualname__}(): {e}")
log.trace(traceback.format_exc())
self.log.error(f"Error in dispatcher: {e}")
self.log.trace(traceback.format_exc())
2 changes: 1 addition & 1 deletion bbot/scanner/preset/preset.py
Original file line number Diff line number Diff line change
Expand Up @@ -826,7 +826,7 @@ def to_dict(self, include_target=False, full_config=False, redact_secrets=False)
if self.scan_name:
preset_dict["scan_name"] = self.scan_name
if self.scan_name and self.output_dir is not None:
preset_dict["output_dir"] = self.output_dir
preset_dict["output_dir"] = str(self.output_dir)

# conditions
if self.conditions:
Expand Down
Loading
0