8000 perf: reduce import overhead on background jobs by ankush · Pull Request #25459 · frappe/frappe · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

perf: reduce import overhead on background jobs #25459

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 15, 2024
Merged

perf: reduce import overhead on background jobs #25459

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3226717
perf: avoid importing posthog if not required
ankush Mar 15, 2024
0d6ec13
perf: preload modules in worker pool
ankush Mar 15, 2024
b7a5884
perf: import sentinal locally
ankush Mar 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion frappe/hooks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -441,7 +441,7 @@
"frappe.recorder.dump",
"frappe.monitor.stop",
"frappe.utils.file_lock.release_document_locks",
"frappe.utils.telemetry.flush",
"frappe.utils.background_jobs.flush_telemetry",
]

extend_bootinfo = [
Expand Down
22 changes: 22 additions & 0 deletions frappe/utils/background_jobs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import time
from collections import defaultdict
from collections.abc import Callable
from contextlib import suppress
from functools import lru_cache
from typing import Any, NoReturn
from uuid import uuid4
Expand Down Expand Up @@ -323,6 +324,17 @@ def start_worker_pool(
"""

_start_sentry()

# If gc.freeze is done then importing modules before forking allows us to share the memory
import frappe.database.query # sqlparse and indirect imports
import frappe.query_builder # pypika
import frappe.utils.data # common utils
import frappe.utils.safe_exec
import frappe.utils.typing_validations # any whitelisted method uses this
import frappe.website.path_resolver # all the page types and resolver

# end: module pre-loading

_freeze_gc()

with frappe.init_site():
Expand Down Expand Up @@ -606,6 +618,16 @@ def truncate_failed_registry(job, connection, type, value, traceback):
job_obj and fail_registry.remove(job_obj, delete_job=True)


def flush_telemetry():
"""Forcefully flush pending events.

This is required in context of background jobs where process might die before posthog gets time
to push events."""
ph = getattr(frappe.local, "posthog", None)
with suppress(Exception):
ph and ph.flush()


def _start_sentry():
sentry_dsn = os.getenv("FRAPPE_SENTRY_DSN")
if not sentry_dsn:
Expand Down
3 changes: 2 additions & 1 deletion frappe/utils/redis_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

import redis
from redis.commands.search import Search
from redis.sentinel import Sentinel

import frappe
from frappe.utils import cstr
Expand Down Expand Up @@ -373,6 +372,8 @@ def get_sentinel_connection(
master_username=None,
master_password=None,
):
from redis.sentinel import Sentinel

sentinel_kwargs = {}
if sentinel_username:
sentinel_kwargs["username"] = sentinel_username
Expand Down
10 changes: 0 additions & 10 deletions frappe/utils/telemetry.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,16 +60,6 @@ def capture(event, app, **kwargs):
ph and ph.capture(distinct_id=frappe.local.site, event=f"{app}_{event}", **kwargs)


def flush():
"""Forcefully flush pending events.

This is required in context of background jobs where process might die before posthog gets time
to push events."""
ph: Posthog = getattr(frappe.local, "posthog", None)
with suppress(Exception):
ph and ph.flush()


def capture_doc(doc, action):
with suppress(Exception):
age = site_age()
Expand Down
0