From acba5980d09d8ad790df1c8d8f590e00aca4726a Mon Sep 17 00:00:00 2001 From: Dave Gaeddert Date: Fri, 20 Jun 2025 16:10:05 -0500 Subject: [PATCH 01/13] save --- demos/full/app/settings.py | 2 + demos/full/pyproject.toml | 1 + plain-admin/plain/admin/CHANGELOG.md | 1 - plain-api/plain/api/CHANGELOG.md | 1 - plain-auth/plain/auth/CHANGELOG.md | 1 - plain-cache/plain/cache/CHANGELOG.md | 1 - plain-code/plain/code/CHANGELOG.md | 1 - plain-dev/plain/dev/CHANGELOG.md | 1 - plain-elements/plain/elements/CHANGELOG.md | 1 - plain-email/plain/email/CHANGELOG.md | 1 - plain-esbuild/plain/esbuild/CHANGELOG.md | 1 - plain-flags/plain/flags/CHANGELOG.md | 1 - plain-htmx/plain/htmx/CHANGELOG.md | 1 - plain-loginlink/plain/loginlink/CHANGELOG.md | 1 - plain-models/plain/models/CHANGELOG.md | 1 - plain-models/plain/models/backends/utils.py | 33 ++-- plain-models/plain/models/observability.py | 143 ++++++++++++++++++ plain-models/pyproject.toml | 3 + plain-oauth/plain/oauth/CHANGELOG.md | 1 - plain-observe/LICENSE | 28 ++++ plain-observe/README.md | 1 + plain-observe/plain/observe/CHANGELOG.md | 1 + plain-observe/plain/observe/README.md | 3 + plain-observe/plain/observe/__init__.py | 0 plain-observe/plain/observe/admin.py | 66 ++++++++ plain-observe/plain/observe/config.py | 31 ++++ .../plain/observe/default_settings.py | 5 + .../plain/observe/migrations/0001_initial.py | 57 +++++++ ...id_span_parent_id_span_context_and_more.py | 70 +++++++++ ...n_attributes_alter_span_events_and_more.py | 36 +++++ ...quest_id_trace_session_id_trace_user_id.py | 28 ++++ ...05_alter_trace_options_trace_created_at.py | 26 ++++ ...006_alter_span_options_trace_start_time.py | 22 +++ .../0007_remove_trace_created_at.py | 16 ++ ...0008_alter_trace_options_trace_end_time.py | 22 +++ .../plain/observe/migrations/__init__.py | 0 plain-observe/plain/observe/models.py | 55 +++++++ plain-observe/plain/observe/otel.py | 141 +++++++++++++++++ .../templates/observability/spans.html | 104 +++++++++++++ .../templates/toolbar/observability.html | 28 ++++ plain-observe/plain/observe/urls.py | 10 ++ plain-observe/plain/observe/views.py | 62 ++++++++ plain-observe/pyproject.toml | 27 ++++ plain-observe/tests/app/settings.py | 24 +++ plain-observe/tests/app/urls.py | 24 +++ .../app/users/migrations/0001_initial.py | 21 +++ .../tests/app/users/migrations/__init__.py | 0 plain-observe/tests/app/users/models.py | 7 + plain-observe/tests/test_admin.py | 22 +++ plain-pages/plain/pages/CHANGELOG.md | 1 - plain-pageviews/plain/pageviews/CHANGELOG.md | 1 - plain-passwords/plain/passwords/CHANGELOG.md | 1 - plain-pytest/plain/pytest/CHANGELOG.md | 1 - .../plain/redirection/CHANGELOG.md | 1 - plain-sessions/plain/sessions/CHANGELOG.md | 1 - plain-support/plain/support/CHANGELOG.md | 1 - plain-tailwind/plain/tailwind/CHANGELOG.md | 1 - plain-tunnel/plain/tunnel/CHANGELOG.md | 1 - plain-vendor/plain/vendor/CHANGELOG.md | 1 - plain-worker/plain/worker/CHANGELOG.md | 1 - plain/plain/CHANGELOG.md | 1 - plain/plain/internal/handlers/base.py | 50 ++++-- plain/plain/views/base.py | 12 +- plain/pyproject.toml | 1 + pyproject.toml | 1 + uv.lock | 97 ++++++++++++ 66 files changed, 1256 insertions(+), 50 deletions(-) create mode 100644 plain-models/plain/models/observability.py create mode 100644 plain-observe/LICENSE create mode 120000 plain-observe/README.md create mode 100644 plain-observe/plain/observe/CHANGELOG.md create mode 100644 plain-observe/plain/observe/README.md create mode 100644 plain-observe/plain/observe/__init__.py create mode 100644 plain-observe/plain/observe/admin.py create mode 100644 plain-observe/plain/observe/config.py create mode 100644 plain-observe/plain/observe/default_settings.py create mode 100644 plain-observe/plain/observe/migrations/0001_initial.py create mode 100644 plain-observe/plain/observe/migrations/0002_rename_parent_span_id_span_parent_id_span_context_and_more.py create mode 100644 plain-observe/plain/observe/migrations/0003_alter_span_attributes_alter_span_events_and_more.py create mode 100644 plain-observe/plain/observe/migrations/0004_trace_request_id_trace_session_id_trace_user_id.py create mode 100644 plain-observe/plain/observe/migrations/0005_alter_trace_options_trace_created_at.py create mode 100644 plain-observe/plain/observe/migrations/0006_alter_span_options_trace_start_time.py create mode 100644 plain-observe/plain/observe/migrations/0007_remove_trace_created_at.py create mode 100644 plain-observe/plain/observe/migrations/0008_alter_trace_options_trace_end_time.py create mode 100644 plain-observe/plain/observe/migrations/__init__.py create mode 100644 plain-observe/plain/observe/models.py create mode 100644 plain-observe/plain/observe/otel.py create mode 100644 plain-observe/plain/observe/templates/observability/spans.html create mode 100644 plain-observe/plain/observe/templates/toolbar/observability.html create mode 100644 plain-observe/plain/observe/urls.py create mode 100644 plain-observe/plain/observe/views.py create mode 100644 plain-observe/pyproject.toml create mode 100644 plain-observe/tests/app/settings.py create mode 100644 plain-observe/tests/app/urls.py create mode 100644 plain-observe/tests/app/users/migrations/0001_initial.py create mode 100644 plain-observe/tests/app/users/migrations/__init__.py create mode 100644 plain-observe/tests/app/users/models.py create mode 100644 plain-observe/tests/test_admin.py diff --git a/demos/full/app/settings.py b/demos/full/app/settings.py index 8bf6dc5d40..9ee0ba4219 100644 --- a/demos/full/app/settings.py +++ b/demos/full/app/settings.py @@ -20,6 +20,7 @@ "plain.tailwind", "plain.worker", "plain.redirection", + "plain.observe", "app.users", ] @@ -38,6 +39,7 @@ MIDDLEWARE = [ "plain.sessions.middleware.SessionMiddleware", "plain.auth.middleware.AuthenticationMiddleware", + "plain.observe.middelware.ObserveMiddleware", "plain.admin.AdminMiddleware", ] diff --git a/demos/full/pyproject.toml b/demos/full/pyproject.toml index 03c6bda07e..8b765452ed 100644 --- a/demos/full/pyproject.toml +++ b/demos/full/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "plain-tunnel", "plain-vendor", "plain-worker", + "plain-observe", ] [tool.plain.tailwind] diff --git a/plain-admin/plain/admin/CHANGELOG.md b/plain-admin/plain/admin/CHANGELOG.md index 19c5f2324d..99f25eb1c1 100644 --- a/plain-admin/plain/admin/CHANGELOG.md +++ b/plain-admin/plain/admin/CHANGELOG.md @@ -1,2 +1 @@ # plain-admin changelog - diff --git a/plain-api/plain/api/CHANGELOG.md b/plain-api/plain/api/CHANGELOG.md index b3c4b7503f..47935e0be5 100644 --- a/plain-api/plain/api/CHANGELOG.md +++ b/plain-api/plain/api/CHANGELOG.md @@ -1,2 +1 @@ # plain-api changelog - diff --git a/plain-auth/plain/auth/CHANGELOG.md b/plain-auth/plain/auth/CHANGELOG.md index af7566a6ed..6c3e69eaa6 100644 --- a/plain-auth/plain/auth/CHANGELOG.md +++ b/plain-auth/plain/auth/CHANGELOG.md @@ -1,2 +1 @@ # plain-auth changelog - diff --git a/plain-cache/plain/cache/CHANGELOG.md b/plain-cache/plain/cache/CHANGELOG.md index c50837c545..d446a6b314 100644 --- a/plain-cache/plain/cache/CHANGELOG.md +++ b/plain-cache/plain/cache/CHANGELOG.md @@ -1,2 +1 @@ # plain-cache changelog - diff --git a/plain-code/plain/code/CHANGELOG.md b/plain-code/plain/code/CHANGELOG.md index b7b88e2b40..aa9148b617 100644 --- a/plain-code/plain/code/CHANGELOG.md +++ b/plain-code/plain/code/CHANGELOG.md @@ -1,2 +1 @@ # plain-code changelog - diff --git a/plain-dev/plain/dev/CHANGELOG.md b/plain-dev/plain/dev/CHANGELOG.md index c6d5373ee6..4d44d94532 100644 --- a/plain-dev/plain/dev/CHANGELOG.md +++ b/plain-dev/plain/dev/CHANGELOG.md @@ -1,2 +1 @@ # plain-dev changelog - diff --git a/plain-elements/plain/elements/CHANGELOG.md b/plain-elements/plain/elements/CHANGELOG.md index 79a3b47a01..41919bed48 100644 --- a/plain-elements/plain/elements/CHANGELOG.md +++ b/plain-elements/plain/elements/CHANGELOG.md @@ -1,2 +1 @@ # plain-elements changelog - diff --git a/plain-email/plain/email/CHANGELOG.md b/plain-email/plain/email/CHANGELOG.md index bf12161b5d..fce996a900 100644 --- a/plain-email/plain/email/CHANGELOG.md +++ b/plain-email/plain/email/CHANGELOG.md @@ -1,2 +1 @@ # plain-email changelog - diff --git a/plain-esbuild/plain/esbuild/CHANGELOG.md b/plain-esbuild/plain/esbuild/CHANGELOG.md index 05cb2057c0..7534a34d95 100644 --- a/plain-esbuild/plain/esbuild/CHANGELOG.md +++ b/plain-esbuild/plain/esbuild/CHANGELOG.md @@ -1,2 +1 @@ # plain-esbuild changelog - diff --git a/plain-flags/plain/flags/CHANGELOG.md b/plain-flags/plain/flags/CHANGELOG.md index 8e78b8e8bd..7405783b1f 100644 --- a/plain-flags/plain/flags/CHANGELOG.md +++ b/plain-flags/plain/flags/CHANGELOG.md @@ -1,2 +1 @@ # plain-flags changelog - diff --git a/plain-htmx/plain/htmx/CHANGELOG.md b/plain-htmx/plain/htmx/CHANGELOG.md index 315337d40a..11210d5efb 100644 --- a/plain-htmx/plain/htmx/CHANGELOG.md +++ b/plain-htmx/plain/htmx/CHANGELOG.md @@ -1,2 +1 @@ # plain-htmx changelog - diff --git a/plain-loginlink/plain/loginlink/CHANGELOG.md b/plain-loginlink/plain/loginlink/CHANGELOG.md index c815f6401c..4613b78a31 100644 --- a/plain-loginlink/plain/loginlink/CHANGELOG.md +++ b/plain-loginlink/plain/loginlink/CHANGELOG.md @@ -1,2 +1 @@ # plain-loginlink changelog - diff --git a/plain-models/plain/models/CHANGELOG.md b/plain-models/plain/models/CHANGELOG.md index fe4cd80b6f..90e71b5d58 100644 --- a/plain-models/plain/models/CHANGELOG.md +++ b/plain-models/plain/models/CHANGELOG.md @@ -1,2 +1 @@ # plain-models changelog - diff --git a/plain-models/plain/models/backends/utils.py b/plain-models/plain/models/backends/utils.py index 3b4da73ca7..021e6c6784 100644 --- a/plain-models/plain/models/backends/utils.py +++ b/plain-models/plain/models/backends/utils.py @@ -7,6 +7,9 @@ from hashlib import md5 from plain.models.db import NotSupportedError + +# OpenTelemetry observability helpers (centralised). +from plain.models.observability import db_span from plain.utils.dateparse import parse_time logger = logging.getLogger("plain.models.backends") @@ -80,18 +83,28 @@ def _execute_with_wrappers(self, sql, params, many, executor): return executor(sql, params, many, context) def _execute(self, sql, params, *ignored_wrapper_args): - self.db.validate_no_broken_transaction() - with self.db.wrap_database_errors: - if params is None: - # params default might be backend specific. - return self.cursor.execute(sql) - else: - return self.cursor.execute(sql, params) + # Wrap in an OpenTelemetry span with standard attributes. + with db_span(self.db, sql): + self.db.validate_no_broken_transaction() + with self.db.wrap_database_errors: + if params is None: + return self.cursor.execute(sql) + else: + return self.cursor.execute(sql, params) def _executemany(self, sql, param_list, *ignored_wrapper_args): - self.db.validate_no_broken_transaction() - with self.db.wrap_database_errors: - return self.cursor.executemany(sql, param_list) + # Determine batch size when param_list is sized; may be expensive for + # generators, so guard with try/except. + batch_size = None + try: + batch_size = len(param_list) + except TypeError: + pass + + with db_span(self.db, sql, many=True, batch_size=batch_size): + self.db.validate_no_broken_transaction() + with self.db.wrap_database_errors: + return self.cursor.executemany(sql, param_list) class CursorDebugWrapper(CursorWrapper): diff --git a/plain-models/plain/models/observability.py b/plain-models/plain/models/observability.py new file mode 100644 index 0000000000..2e967c8ae2 --- /dev/null +++ b/plain-models/plain/models/observability.py @@ -0,0 +1,143 @@ +from contextlib import contextmanager +from typing import Any + +from opentelemetry import context as otel_context +from opentelemetry import trace +from opentelemetry.semconv.attributes.db_attributes import ( + DB_NAMESPACE, + DB_OPERATION_BATCH_SIZE, + DB_OPERATION_NAME, + DB_QUERY_TEXT, + DB_SYSTEM_NAME, +) +from opentelemetry.semconv.attributes.network_attributes import ( + NETWORK_PEER_ADDRESS, + NETWORK_PEER_PORT, +) +from opentelemetry.semconv.trace import DbSystemValues +from opentelemetry.trace import SpanKind + +# Import the official suppression key used by OTel instrumentations when +# available (present once any opentelemetry-instrumentation- is +# installed). Fallback to a module-local object so our own helpers can still +# reference it. +try: + from opentelemetry.instrumentation.utils import ( + _SUPPRESS_INSTRUMENTATION_KEY as _SUPPRESS_KEY, + ) +except ImportError: # instrumentation extras not installed + _SUPPRESS_KEY = object() + + +tracer = trace.get_tracer("plain.models") + + +def db_system_for(vendor: str) -> str: # noqa: D401 – simple helper + """Return the canonical ``db.system.name`` value for a backend vendor.""" + + return { + "postgresql": DbSystemValues.POSTGRESQL.value, + "mysql": DbSystemValues.MYSQL.value, + "mariadb": DbSystemValues.MARIADB.value, + "sqlite": DbSystemValues.SQLITE.value, + }.get(vendor, vendor) + + +@contextmanager +def db_span(db, sql: Any, *, many: bool = False, batch_size: int | None = None): + """Open an OpenTelemetry CLIENT span for a database query. + + All common attributes (`db.*`, `network.*`, etc.) are set automatically. + """ + + # Fast-exit if instrumentation suppression flag set in context. + if otel_context.get_value(_SUPPRESS_KEY): + yield None + return + + # Derive operation keyword (SELECT, INSERT, …) if possible. + operation: str | None = None + if isinstance(sql, str): + stripped = sql.lstrip() + if stripped: + operation = stripped.split()[0].upper() + + # Span name per OTel SQL guidance. + if many: + span_name = (operation or "EXECUTEMANY") + " many" + else: + span_name = operation or "QUERY" + + # Build attribute set. + attrs: dict[str, Any] = { + DB_SYSTEM_NAME: db_system_for(db.vendor), + DB_NAMESPACE: db.settings_dict.get("NAME"), + DB_QUERY_TEXT: sql if isinstance(sql, str) else str(sql), + } + + if user := db.settings_dict.get("USER"): + attrs["db.user"] = user + + if host := db.settings_dict.get("HOST"): + attrs[NETWORK_PEER_ADDRESS] = host + + if port := db.settings_dict.get("PORT"): + try: + attrs[NETWORK_PEER_PORT] = int(port) + except (TypeError, ValueError): + pass + + # executemany: include batch size when >1 per semantic conventions. + if batch_size and batch_size > 1: + attrs[DB_OPERATION_BATCH_SIZE] = batch_size + + with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span: + for key, value in attrs.items(): + if value is not None: + span.set_attribute(key, value) + + if operation: + span.set_attribute(DB_OPERATION_NAME, operation) + + yield span + + +# --------------------------------------------------------------------------- +# Context manager to suppress *all* instrumentation inside the block +# --------------------------------------------------------------------------- + + +@contextmanager +def suppress_tracing(): + """Temporarily disable **all** OpenTelemetry instrumentation. + + This sets the standard suppression flag recognised by every official + instrumentation package, meaning *no spans* will be recorded for the + duration of the context – not just database spans. + """ + + token = otel_context.attach(otel_context.set_value(_SUPPRESS_KEY, True)) + try: + yield + finally: + otel_context.detach(token) + + +# --------------------------------------------------------------------------- +# Helper to disable DB spans temporarily +# --------------------------------------------------------------------------- + + +@contextmanager +def disable_db_spans(): + """Temporarily disable ``db_span`` within the current context. + + Useful when the application writes traces/spans back into the database to + avoid generating additional spans for those internal operations. + """ + + # Retained for backward compatibility; internally delegates to + # `suppress_tracing()` so that *all* instrumentation is suppressed, which + # is usually what callers expect. + with suppress_tracing(): + yield diff --git a/plain-models/pyproject.toml b/plain-models/pyproject.toml index 42b0992103..8aa371dbdb 100644 --- a/plain-models/pyproject.toml +++ b/plain-models/pyproject.toml @@ -8,6 +8,9 @@ requires-python = ">=3.11" dependencies = [ "plain<1.0.0", "sqlparse>=0.3.1", + # plain.models relies on OpenTelemetry semantic conventions for + # span/attribute constants used in observability helpers. + "opentelemetry-semantic-conventions>=0.55b1", ] [project.entry-points."plain.setup"] diff --git a/plain-oauth/plain/oauth/CHANGELOG.md b/plain-oauth/plain/oauth/CHANGELOG.md index bfa9b3c07f..8d90d717bf 100644 --- a/plain-oauth/plain/oauth/CHANGELOG.md +++ b/plain-oauth/plain/oauth/CHANGELOG.md @@ -1,2 +1 @@ # plain-oauth changelog - diff --git a/plain-observe/LICENSE b/plain-observe/LICENSE new file mode 100644 index 0000000000..e5391c216b --- /dev/null +++ b/plain-observe/LICENSE @@ -0,0 +1,28 @@ +BSD 3-Clause License + +Copyright (c) 2023, Dropseed, LLC + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/plain-observe/README.md b/plain-observe/README.md new file mode 120000 index 0000000000..f0a9e39854 --- /dev/null +++ b/plain-observe/README.md @@ -0,0 +1 @@ +plain/observe/README.md \ No newline at end of file diff --git a/plain-observe/plain/observe/CHANGELOG.md b/plain-observe/plain/observe/CHANGELOG.md new file mode 100644 index 0000000000..99f25eb1c1 --- /dev/null +++ b/plain-observe/plain/observe/CHANGELOG.md @@ -0,0 +1 @@ +# plain-admin changelog diff --git a/plain-observe/plain/observe/README.md b/plain-observe/plain/observe/README.md new file mode 100644 index 0000000000..0706b46557 --- /dev/null +++ b/plain-observe/plain/observe/README.md @@ -0,0 +1,3 @@ +# plain.observe + +**Monitor.** diff --git a/plain-observe/plain/observe/__init__.py b/plain-observe/plain/observe/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plain-observe/plain/observe/admin.py b/plain-observe/plain/observe/admin.py new file mode 100644 index 0000000000..3b78051e2a --- /dev/null +++ b/plain-observe/plain/observe/admin.py @@ -0,0 +1,66 @@ +from plain.admin.toolbar import ToolbarPanel, register_toolbar_panel +from plain.admin.views import ( + AdminModelDetailView, + AdminModelListView, + AdminViewset, + register_viewset, +) + +from .models import Span, Trace + + +@register_viewset +class TraceViewset(AdminViewset): + class ListView(AdminModelListView): + nav_section = "Observe" + model = Trace + fields = [ + "trace_id", + "request_id", + "session_id", + "user_id", + "start_time", + ] + allow_global_search = False + + class DetailView(AdminModelDetailView): + model = Trace + # title = "Cached item" + + +@register_viewset +class SpanViewset(AdminViewset): + class ListView(AdminModelListView): + nav_section = "Observe" + model = Span + fields = [ + "name", + "kind", + "span_id", + "parent_id", + "start_time", + ] + queryset_order = ["-pk"] + allow_global_search = False + + def get_objects(self): + return ( + super() + .get_objects() + .only( + "name", + "kind", + "span_id", + "parent_id", + "start_time", + ) + ) + + class DetailView(AdminModelDetailView): + model = Span + + +@register_toolbar_panel +class ObservabilityToolbarPanel(ToolbarPanel): + name = "Observability" + template_name = "toolbar/observability.html" diff --git a/plain-observe/plain/observe/config.py b/plain-observe/plain/observe/config.py new file mode 100644 index 0000000000..64c7b229b5 --- /dev/null +++ b/plain-observe/plain/observe/config.py @@ -0,0 +1,31 @@ +import re + +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider, sampling +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +from plain.packages import PackageConfig, register_config +from plain.runtime import settings + +from .otel import ObserveModelsExporter, PlainRequestSampler + + +@register_config +class Config(PackageConfig): + package_label = "plainobserve" + + def ready(self): + current_provider = trace.get_tracer_provider() + if current_provider and not isinstance( + current_provider, trace.ProxyTracerProvider + ): + return + + ignore_url_patterns = [re.compile(p) for p in settings.OBSERVE_IGNORE_URLS] + + sampler = PlainRequestSampler( + sampling.ParentBased(sampling.ALWAYS_ON), ignore_url_patterns + ) + provider = TracerProvider(sampler=sampler) + provider.add_span_processor(BatchSpanProcessor(ObserveModelsExporter())) + trace.set_tracer_provider(provider) diff --git a/plain-observe/plain/observe/default_settings.py b/plain-observe/plain/observe/default_settings.py new file mode 100644 index 0000000000..2a7a9a53b3 --- /dev/null +++ b/plain-observe/plain/observe/default_settings.py @@ -0,0 +1,5 @@ +OBSERVE_IGNORE_URLS: list[str] = [ + "/assets/.*", + "/admin/.*", + "/favicon.ico", +] diff --git a/plain-observe/plain/observe/migrations/0001_initial.py b/plain-observe/plain/observe/migrations/0001_initial.py new file mode 100644 index 0000000000..c24be4e27f --- /dev/null +++ b/plain-observe/plain/observe/migrations/0001_initial.py @@ -0,0 +1,57 @@ +# Generated by Plain 0.49.0 on 2025-06-20 16:56 + +import plain.models.deletion +from plain import models +from plain.models import migrations + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Span", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True)), + ("span_id", models.CharField(max_length=255)), + ( + "parent_span_id", + models.CharField(default="", max_length=255, required=False), + ), + ("name", models.CharField(max_length=255)), + ("start_time", models.DateTimeField()), + ("end_time", models.DateTimeField(allow_null=True, required=False)), + ("attributes", models.JSONField(default=dict)), + ], + ), + migrations.CreateModel( + name="Trace", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True)), + ("trace_id", models.CharField(max_length=255)), + ], + ), + migrations.AddConstraint( + model_name="trace", + constraint=models.UniqueConstraint( + fields=("trace_id",), name="observe_unique_trace_id" + ), + ), + migrations.AddField( + model_name="span", + name="trace", + field=models.ForeignKey( + on_delete=plain.models.deletion.CASCADE, + related_name="spans", + to="plainobserve.trace", + ), + ), + migrations.AddConstraint( + model_name="span", + constraint=models.UniqueConstraint( + fields=("trace", "span_id"), name="observe_unique_span_id" + ), + ), + ] diff --git a/plain-observe/plain/observe/migrations/0002_rename_parent_span_id_span_parent_id_span_context_and_more.py b/plain-observe/plain/observe/migrations/0002_rename_parent_span_id_span_parent_id_span_context_and_more.py new file mode 100644 index 0000000000..c15e6445da --- /dev/null +++ b/plain-observe/plain/observe/migrations/0002_rename_parent_span_id_span_parent_id_span_context_and_more.py @@ -0,0 +1,70 @@ +# Generated by Plain 0.49.0 on 2025-06-20 17:08 + +from plain import models +from plain.models import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("plainobserve", "0001_initial"), + ] + + operations = [ + migrations.RenameField( + model_name="span", + old_name="parent_span_id", + new_name="parent_id", + ), + migrations.AddField( + model_name="span", + name="context", + field=models.JSONField(default=dict), + ), + migrations.AddField( + model_name="span", + name="events", + field=models.JSONField(default=list), + ), + migrations.AddField( + model_name="span", + name="kind", + field=models.CharField(default="", max_length=50), + preserve_default=False, + ), + migrations.AddField( + model_name="span", + name="links", + field=models.JSONField(default=list), + ), + migrations.AddField( + model_name="span", + name="resource", + field=models.JSONField(default=dict), + ), + migrations.AddField( + model_name="span", + name="status", + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name="span", + name="start_time", + field=models.DateTimeField(allow_null=True, required=False), + ), + migrations.AddIndex( + model_name="span", + index=models.Index( + fields=["trace", "span_id"], name="plainobserv_trace_i_da191d_idx" + ), + ), + migrations.AddIndex( + model_name="span", + index=models.Index(fields=["trace"], name="plainobserv_trace_i_602183_idx"), + ), + migrations.AddIndex( + model_name="span", + index=models.Index( + fields=["start_time"], name="plainobserv_start_t_3c8738_idx" + ), + ), + ] diff --git a/plain-observe/plain/observe/migrations/0003_alter_span_attributes_alter_span_events_and_more.py b/plain-observe/plain/observe/migrations/0003_alter_span_attributes_alter_span_events_and_more.py new file mode 100644 index 0000000000..0b50bfb020 --- /dev/null +++ b/plain-observe/plain/observe/migrations/0003_alter_span_attributes_alter_span_events_and_more.py @@ -0,0 +1,36 @@ +# Generated by Plain 0.49.0 on 2025-06-20 17:11 + +from plain import models +from plain.models import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ( + "plainobserve", + "0002_rename_parent_span_id_span_parent_id_span_context_and_more", + ), + ] + + operations = [ + migrations.AlterField( + model_name="span", + name="attributes", + field=models.JSONField(default=dict, required=False), + ), + migrations.AlterField( + model_name="span", + name="events", + field=models.JSONField(default=list, required=False), + ), + migrations.AlterField( + model_name="span", + name="links", + field=models.JSONField(default=list, required=False), + ), + migrations.AlterField( + model_name="span", + name="resource", + field=models.JSONField(default=dict, required=False), + ), + ] diff --git a/plain-observe/plain/observe/migrations/0004_trace_request_id_trace_session_id_trace_user_id.py b/plain-observe/plain/observe/migrations/0004_trace_request_id_trace_session_id_trace_user_id.py new file mode 100644 index 0000000000..bfee329ce7 --- /dev/null +++ b/plain-observe/plain/observe/migrations/0004_trace_request_id_trace_session_id_trace_user_id.py @@ -0,0 +1,28 @@ +# Generated by Plain 0.49.0 on 2025-06-20 19:49 + +from plain import models +from plain.models import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("plainobserve", "0003_alter_span_attributes_alter_span_events_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="trace", + name="request_id", + field=models.CharField(default="", max_length=255, required=False), + ), + migrations.AddField( + model_name="trace", + name="session_id", + field=models.CharField(default="", max_length=255, required=False), + ), + migrations.AddField( + model_name="trace", + name="user_id", + field=models.CharField(default="", max_length=255, required=False), + ), + ] diff --git a/plain-observe/plain/observe/migrations/0005_alter_trace_options_trace_created_at.py b/plain-observe/plain/observe/migrations/0005_alter_trace_options_trace_created_at.py new file mode 100644 index 0000000000..4e1d9cae36 --- /dev/null +++ b/plain-observe/plain/observe/migrations/0005_alter_trace_options_trace_created_at.py @@ -0,0 +1,26 @@ +# Generated by Plain 0.49.0 on 2025-06-20 20:01 + +import plain.utils.timezone +from plain import models +from plain.models import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("plainobserve", "0004_trace_request_id_trace_session_id_trace_user_id"), + ] + + operations = [ + migrations.AlterModelOptions( + name="trace", + options={"ordering": ["-created_at"]}, + ), + migrations.AddField( + model_name="trace", + name="created_at", + field=models.DateTimeField( + auto_now_add=True, default=plain.utils.timezone.now + ), + preserve_default=False, + ), + ] diff --git a/plain-observe/plain/observe/migrations/0006_alter_span_options_trace_start_time.py b/plain-observe/plain/observe/migrations/0006_alter_span_options_trace_start_time.py new file mode 100644 index 0000000000..d79c599dda --- /dev/null +++ b/plain-observe/plain/observe/migrations/0006_alter_span_options_trace_start_time.py @@ -0,0 +1,22 @@ +# Generated by Plain 0.49.0 on 2025-06-20 20:13 + +from plain import models +from plain.models import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("plainobserve", "0005_alter_trace_options_trace_created_at"), + ] + + operations = [ + migrations.AlterModelOptions( + name="span", + options={"ordering": ["-start_time"]}, + ), + migrations.AddField( + model_name="trace", + name="start_time", + field=models.DateTimeField(allow_null=True, required=False), + ), + ] diff --git a/plain-observe/plain/observe/migrations/0007_remove_trace_created_at.py b/plain-observe/plain/observe/migrations/0007_remove_trace_created_at.py new file mode 100644 index 0000000000..4ce9b46b9e --- /dev/null +++ b/plain-observe/plain/observe/migrations/0007_remove_trace_created_at.py @@ -0,0 +1,16 @@ +# Generated by Plain 0.49.0 on 2025-06-20 20:14 + +from plain.models import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("plainobserve", "0006_alter_span_options_trace_start_time"), + ] + + operations = [ + migrations.RemoveField( + model_name="trace", + name="created_at", + ), + ] diff --git a/plain-observe/plain/observe/migrations/0008_alter_trace_options_trace_end_time.py b/plain-observe/plain/observe/migrations/0008_alter_trace_options_trace_end_time.py new file mode 100644 index 0000000000..15889523bd --- /dev/null +++ b/plain-observe/plain/observe/migrations/0008_alter_trace_options_trace_end_time.py @@ -0,0 +1,22 @@ +# Generated by Plain 0.49.0 on 2025-06-20 20:15 + +from plain import models +from plain.models import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("plainobserve", "0007_remove_trace_created_at"), + ] + + operations = [ + migrations.AlterModelOptions( + name="trace", + options={"ordering": ["-start_time"]}, + ), + migrations.AddField( + model_name="trace", + name="end_time", + field=models.DateTimeField(allow_null=True, required=False), + ), + ] diff --git a/plain-observe/plain/observe/migrations/__init__.py b/plain-observe/plain/observe/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plain-observe/plain/observe/models.py b/plain-observe/plain/observe/models.py new file mode 100644 index 0000000000..9197d5875f --- /dev/null +++ b/plain-observe/plain/observe/models.py @@ -0,0 +1,55 @@ +from plain import models + + +@models.register_model +class Trace(models.Model): + trace_id = models.CharField(max_length=255) + start_time = models.DateTimeField(allow_null=True, required=False) + end_time = models.DateTimeField(allow_null=True, required=False) + + # Plain fields + request_id = models.CharField(max_length=255, default="", required=False) + session_id = models.CharField(max_length=255, default="", required=False) + user_id = models.CharField(max_length=255, default="", required=False) + + class Meta: + ordering = ["-start_time"] + constraints = [ + models.UniqueConstraint( + fields=["trace_id"], + name="observe_unique_trace_id", + ) + ] + + +@models.register_model +class Span(models.Model): + trace = models.ForeignKey(Trace, on_delete=models.CASCADE, related_name="spans") + + span_id = models.CharField(max_length=255) + + name = models.CharField(max_length=255) + kind = models.CharField(max_length=50) + parent_id = models.CharField(max_length=255, default="", required=False) + start_time = models.DateTimeField(allow_null=True, required=False) + end_time = models.DateTimeField(allow_null=True, required=False) + status = models.JSONField(default=dict) + context = models.JSONField(default=dict) + attributes = models.JSONField(default=dict, required=False) + events = models.JSONField(default=list, required=False) + links = models.JSONField(default=list, required=False) + resource = models.JSONField(default=dict, required=False) + + class Meta: + ordering = ["-start_time"] + constraints = [ + models.UniqueConstraint( + fields=["trace", "span_id"], + name="observe_unique_span_id", + ) + ] + indexes = [ + models.Index(fields=["trace", "span_id"]), + models.Index(fields=["trace"]), + models.Index(fields=["start_time"]), + ] diff --git a/plain-observe/plain/observe/otel.py b/plain-observe/plain/observe/otel.py new file mode 100644 index 0000000000..0634035d7f --- /dev/null +++ b/plain-observe/plain/observe/otel.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import json +import re +from collections.abc import Sequence + +from opentelemetry.sdk.trace import sampling +from opentelemetry.sdk.trace.export import SpanExporter +from opentelemetry.semconv.attributes import url_attributes +from opentelemetry.trace import SpanKind + +from plain.models.observability import suppress_tracing + + +class PlainRequestSampler(sampling.Sampler): + """Drops traces based on request path or user role.""" + + def __init__( + self, delegate: sampling.Sampler, ignore_url_paths: Sequence[re.Pattern] + ): + self._delegate = delegate + self._ignore_url_paths = ignore_url_paths + + def should_sample( + self, + parent_context, + trace_id, + name, + kind: SpanKind | None = None, + attributes=None, + links=None, + trace_state=None, + **kwargs, + ): # type: ignore[override] + # Can we get the request path from the context or something instead of contextvar? + # not sure what to do with is_admin and stuff then... + + if attributes: + if url_path := attributes[url_attributes.URL_PATH]: + print("SHIT", url_path) + for pattern in self._ignore_url_paths: + if pattern.match(url_path): + return sampling.SamplingResult(sampling.Decision.DROP) + + # Example rule: drop staff/admin requests entirely. + # if getattr(request, "user", None) and getattr(request.user, "is_staff", False): + # return sampling.SamplingResult(sampling.Decision.DROP) + + # In dev we always sample? + + # Otherwise need an option to sample if session sampling enabled and is_admin + + # does empty parent context tell us we're at a root, and only check this there? + # maybe is_admin should be an attribute, and observe_enabled could be an attribute + + # Fallback to delegate sampler. + return self._delegate.should_sample( + parent_context, + trace_id, + name, + kind, + attributes, + links, + trace_state, + ) + + def get_description(self) -> str: + return "PlainRequestSampler" + + +class ObserveModelsExporter(SpanExporter): + """Exporter that writes spans into the observe models tables.""" + + def export(self, spans): # type: ignore[override] + """Persist spans in bulk for efficiency.""" + + from .models import Span, Trace + + with suppress_tracing(): + create_spans = [] + create_traces = {} + + for span in spans: + span_data = json.loads(span.to_json()) + trace_id = span_data["context"]["trace_id"] + + # There should be at least one span with this attribute + request_id = span_data["attributes"].get("plain.request_id", "") + + if trace := create_traces.get(trace_id): + if not trace.start_time: + trace.start_time = span_data["start_time"] + else: + trace.start_time = min( + trace.start_time, span_data["start_time"] + ) + + if not trace.end_time: + trace.end_time = span_data["end_time"] + else: + trace.end_time = max(trace.end_time, span_data["end_time"]) + + if not trace.request_id: + trace.request_id = request_id + else: + trace = Trace( + trace_id=trace_id, + start_time=span_data["start_time"], + end_time=span_data["end_time"], + request_id=request_id, + ) + create_traces[trace_id] = trace + + create_spans.append( + Span( + trace=trace, + span_id=span_data["context"]["span_id"], + name=span_data["name"], + kind=span_data["kind"], + parent_id=span_data["parent_id"] or "", + start_time=span_data["start_time"], + end_time=span_data["end_time"], + status=span_data["status"], + context=span_data["context"], + attributes=span_data["attributes"], + events=span_data["events"], + links=span_data["links"], + resource=span_data["resource"], + ) + ) + + # Trace.objects.bulk_create( + # create_traces.values() + # ) # , update_conflicts=True, update_fields=["start_time", "end_time", "request_id"]) + # Span.objects.bulk_create(create_spans) + + # TODO could delete old spans and stuff here instead of chore? or both? + # should be days based for sure (i.e. 30 days) + # could also be limit based as a fallback + + return True diff --git a/plain-observe/plain/observe/templates/observability/spans.html b/plain-observe/plain/observe/templates/observability/spans.html new file mode 100644 index 0000000000..f9f7c5b75d --- /dev/null +++ b/plain-observe/plain/observe/templates/observability/spans.html @@ -0,0 +1,104 @@ + + + + + + Querystats + {% tailwind_css %} + + + + {% if observability_enabled %} +
+ +
+
+
+ {{ csrf_input }} + +
+
+ {{ csrf_input }} + + +
+
+ {{ csrf_input }} + + +
+
+
+ {% endif %} + + {% if observability %} +
+ + +
+ {% for request_id, request_data in observability.get("requests", {}).items() %} + + {% endfor %} +
+
+ {% elif observability_enabled %} +
Querystats are enabled but nothing has been recorded yet.
+ {% else %} +
+
Querystats are disabled.
+
+ {{ csrf_input }} + + +
+
+ {% endif %} + + + + + diff --git a/plain-observe/plain/observe/templates/toolbar/observability.html b/plain-observe/plain/observe/templates/toolbar/observability.html new file mode 100644 index 0000000000..adc5c437f5 --- /dev/null +++ b/plain-observe/plain/observe/templates/toolbar/observability.html @@ -0,0 +1,28 @@ +
+
+

Loading spans...

+
+
+ diff --git a/plain-observe/plain/observe/urls.py b/plain-observe/plain/observe/urls.py new file mode 100644 index 0000000000..29809342cd --- /dev/null +++ b/plain-observe/plain/observe/urls.py @@ -0,0 +1,10 @@ +from plain.urls import Router, path + +from . import views + + +class ObserveRouter(Router): + namespace = "observe" + urls = [ + path("", views.ObservabilitySpansView, name="spans"), + ] diff --git a/plain-observe/plain/observe/views.py b/plain-observe/plain/observe/views.py new file mode 100644 index 0000000000..26e802d2c5 --- /dev/null +++ b/plain-observe/plain/observe/views.py @@ -0,0 +1,62 @@ +from plain.auth.views import AuthViewMixin +from plain.runtime import settings +from plain.views import TemplateView + +# from .middleware import OBSERVABILITY_SESSION_KEY + + +class ObservabilitySpansView(AuthViewMixin, TemplateView): + template_name = "observability/spans.html" + admin_required = True + + def check_auth(self): + # Allow the view if we're in DEBUG + if settings.DEBUG: + return + + super().check_auth() + + def get_response(self): + response = super().get_response() + # So we can load it in the toolbar + response.headers["X-Frame-Options"] = "SAMEORIGIN" + return response + + def get_template_context(self): + context = super().get_template_context() + + # observability = self.request.session.get(OBSERVABILITY_SESSION_KEY, {}) + + # for request_id in list(spans.keys()): + # try: + # spans[request_id] = json.loads(spans[request_id]) + # except (json.JSONDecodeError, TypeError): + # # If decoding fails, remove the entry from the dictionary + # del spans[request_id] + + # Order them by timestamp + # spans = dict( + # sorted( + # spans.items(), + # key=lambda item: item[1].get("timestamp", ""), + # reverse=True, + # ) + # ) + + # context["observability"] = observability + # context["observability_enabled"] = OBSERVABILITY_SESSION_KEY in self.request.session + + return context + + # def post(self): + # querystats_action = self.request.data["querystats_action"] + + # if querystats_action == "enable": + # self.request.session.setdefault(OBSERVABILITY_SESSION_KEY, {}) + # elif querystats_action == "clear": + # self.request.session[OBSERVABILITY_SESSION_KEY] = {} + # elif querystats_action == "disable" and OBSERVABILITY_SESSION_KEY in self.request.session: + # del self.request.session[OBSERVABILITY_SESSION_KEY] + + # # Redirect back to the page that submitted the form + # return ResponseRedirect(self.request.data.get("redirect_url", ".")) diff --git a/plain-observe/pyproject.toml b/plain-observe/pyproject.toml new file mode 100644 index 0000000000..4681d60114 --- /dev/null +++ b/plain-observe/pyproject.toml @@ -0,0 +1,27 @@ +[project] +name = "plain.observe" +version = "0.1.0" +description = "Basic observability and monitoring tools for Plain." +authors = [{name = "Dave Gaeddert", email = "dave.gaeddert@dropseed.dev"}] +license = "BSD-3-Clause" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "opentelemetry-sdk>=1.34.1", + "plain<1.0.0", + "plain.auth<1.0.0", + "plain.tailwind<1.0.0", + "sqlparse>=0.2.2", +] + +[tool.uv] +dev-dependencies = [ + "plain.pytest<1.0.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["plain"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/plain-observe/tests/app/settings.py b/plain-observe/tests/app/settings.py new file mode 100644 index 0000000000..a7ade6db0c --- /dev/null +++ b/plain-observe/tests/app/settings.py @@ -0,0 +1,24 @@ +SECRET_KEY = "test" +URLS_ROUTER = "app.urls.AppRouter" +INSTALLED_PACKAGES = [ + "plain.auth", + "plain.sessions", + "plain.models", + "plain.htmx", + "plain.tailwind", + "plain.admin", + "app.users", +] +DATABASES = { + "default": { + "ENGINE": "plain.models.backends.sqlite3", + "NAME": ":memory:", + } +} +MIDDLEWARE = [ + "plain.sessions.middleware.SessionMiddleware", + "plain.auth.middleware.AuthenticationMiddleware", + "plain.admin.AdminMiddleware", +] +AUTH_LOGIN_URL = "login" +AUTH_USER_MODEL = "users.User" diff --git a/plain-observe/tests/app/urls.py b/plain-observe/tests/app/urls.py new file mode 100644 index 0000000000..5af6f36911 --- /dev/null +++ b/plain-observe/tests/app/urls.py @@ -0,0 +1,24 @@ +from plain.admin.urls import AdminRouter +from plain.assets.urls import AssetsRouter +from plain.urls import Router, include, path +from plain.views import View + + +class LoginView(View): + def get(self): + return "Login!" + + +class LogoutView(View): + def get(self): + return "Logout!" + + +class AppRouter(Router): + namespace = "" + urls = [ + include("admin/", AdminRouter), + include("assets/", AssetsRouter), + path("login/", LoginView, name="login"), + path("logout/", LogoutView, name="logout"), + ] diff --git a/plain-observe/tests/app/users/migrations/0001_initial.py b/plain-observe/tests/app/users/migrations/0001_initial.py new file mode 100644 index 0000000000..c8e07079a0 --- /dev/null +++ b/plain-observe/tests/app/users/migrations/0001_initial.py @@ -0,0 +1,21 @@ +# Generated by Plain 0.21.5 on 2025-02-17 04:12 + +from plain import models +from plain.models import migrations + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="User", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True)), + ("username", models.CharField(max_length=255)), + ("is_admin", models.BooleanField(default=False)), + ], + ), + ] diff --git a/plain-observe/tests/app/users/migrations/__init__.py b/plain-observe/tests/app/users/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plain-observe/tests/app/users/models.py b/plain-observe/tests/app/users/models.py new file mode 100644 index 0000000000..e3ce5e3d4a --- /dev/null +++ b/plain-observe/tests/app/users/models.py @@ -0,0 +1,7 @@ +from plain import models + + +@models.register_model +class User(models.Model): + username = models.CharField(max_length=255) + is_admin = models.BooleanField(default=False) diff --git a/plain-observe/tests/test_admin.py b/plain-observe/tests/test_admin.py new file mode 100644 index 0000000000..d017f89c1c --- /dev/null +++ b/plain-observe/tests/test_admin.py @@ -0,0 +1,22 @@ +from app.users.models import User + +from plain.test import Client + + +def test_admin_login_required(db): + client = Client() + + # Login required + assert client.get("/admin/").status_code == 302 + + user = User.objects.create(username="test") + client.force_login(user) + + # Not admin yet + assert client.get("/admin/").status_code == 404 + + user.is_admin = True + user.save() + + # Now admin + assert client.get("/admin/").status_code == 200 diff --git a/plain-pages/plain/pages/CHANGELOG.md b/plain-pages/plain/pages/CHANGELOG.md index d2f169bb3a..53593a1564 100644 --- a/plain-pages/plain/pages/CHANGELOG.md +++ b/plain-pages/plain/pages/CHANGELOG.md @@ -1,2 +1 @@ # plain-pages changelog - diff --git a/plain-pageviews/plain/pageviews/CHANGELOG.md b/plain-pageviews/plain/pageviews/CHANGELOG.md index 6f59b329f0..e67fca2f11 100644 --- a/plain-pageviews/plain/pageviews/CHANGELOG.md +++ b/plain-pageviews/plain/pageviews/CHANGELOG.md @@ -1,2 +1 @@ # plain-pageviews changelog - diff --git a/plain-passwords/plain/passwords/CHANGELOG.md b/plain-passwords/plain/passwords/CHANGELOG.md index e5e776fd2c..a10bfbedbc 100644 --- a/plain-passwords/plain/passwords/CHANGELOG.md +++ b/plain-passwords/plain/passwords/CHANGELOG.md @@ -1,2 +1 @@ # plain-passwords changelog - diff --git a/plain-pytest/plain/pytest/CHANGELOG.md b/plain-pytest/plain/pytest/CHANGELOG.md index 100c25a1fd..a224c8c938 100644 --- a/plain-pytest/plain/pytest/CHANGELOG.md +++ b/plain-pytest/plain/pytest/CHANGELOG.md @@ -1,2 +1 @@ # plain-pytest changelog - diff --git a/plain-redirection/plain/redirection/CHANGELOG.md b/plain-redirection/plain/redirection/CHANGELOG.md index cba9a043a4..d287cf429b 100644 --- a/plain-redirection/plain/redirection/CHANGELOG.md +++ b/plain-redirection/plain/redirection/CHANGELOG.md @@ -1,2 +1 @@ # plain-redirection changelog - diff --git a/plain-sessions/plain/sessions/CHANGELOG.md b/plain-sessions/plain/sessions/CHANGELOG.md index 30bf46371d..872a152aef 100644 --- a/plain-sessions/plain/sessions/CHANGELOG.md +++ b/plain-sessions/plain/sessions/CHANGELOG.md @@ -1,2 +1 @@ # plain-sessions changelog - diff --git a/plain-support/plain/support/CHANGELOG.md b/plain-support/plain/support/CHANGELOG.md index d5a982fe1a..40517a85c0 100644 --- a/plain-support/plain/support/CHANGELOG.md +++ b/plain-support/plain/support/CHANGELOG.md @@ -1,2 +1 @@ # plain-support changelog - diff --git a/plain-tailwind/plain/tailwind/CHANGELOG.md b/plain-tailwind/plain/tailwind/CHANGELOG.md index d0b547d2e4..a8804e1859 100644 --- a/plain-tailwind/plain/tailwind/CHANGELOG.md +++ b/plain-tailwind/plain/tailwind/CHANGELOG.md @@ -1,2 +1 @@ # plain-tailwind changelog - diff --git a/plain-tunnel/plain/tunnel/CHANGELOG.md b/plain-tunnel/plain/tunnel/CHANGELOG.md index 6046cf15c3..32656ec7d2 100644 --- a/plain-tunnel/plain/tunnel/CHANGELOG.md +++ b/plain-tunnel/plain/tunnel/CHANGELOG.md @@ -1,2 +1 @@ # plain-tunnel changelog - diff --git a/plain-vendor/plain/vendor/CHANGELOG.md b/plain-vendor/plain/vendor/CHANGELOG.md index 04204221c1..a4288f9293 100644 --- a/plain-vendor/plain/vendor/CHANGELOG.md +++ b/plain-vendor/plain/vendor/CHANGELOG.md @@ -1,2 +1 @@ # plain-vendor changelog - diff --git a/plain-worker/plain/worker/CHANGELOG.md b/plain-worker/plain/worker/CHANGELOG.md index 4b7530e7cc..d0ce00660e 100644 --- a/plain-worker/plain/worker/CHANGELOG.md +++ b/plain-worker/plain/worker/CHANGELOG.md @@ -1,2 +1 @@ # plain-worker changelog - diff --git a/plain/plain/CHANGELOG.md b/plain/plain/CHANGELOG.md index 278dc5ccad..ebb87df35b 100644 --- a/plain/plain/CHANGELOG.md +++ b/plain/plain/CHANGELOG.md @@ -1,2 +1 @@ # plain changelog - diff --git a/plain/plain/internal/handlers/base.py b/plain/plain/internal/handlers/base.py index ccab6a68f8..02b443e065 100644 --- a/plain/plain/internal/handlers/base.py +++ b/plain/plain/internal/handlers/base.py @@ -1,6 +1,9 @@ import logging import types +from opentelemetry import trace +from opentelemetry.semconv.attributes import http_attributes, url_attributes + from plain.exceptions import ImproperlyConfigured from plain.logs import log_response from plain.runtime import settings @@ -26,6 +29,9 @@ ] +tracer = trace.get_tracer(__name__) + + class BaseHandler: _middleware_chain = None @@ -59,18 +65,32 @@ def load_middleware(self): def get_response(self, request): """Return a Response object for the given HttpRequest.""" - # Setup default url resolver for this thread - response = self._middleware_chain(request) - response._resource_closers.append(request.close) - if response.status_code >= 400: - log_response( - "%s: %s", - response.reason_phrase, - request.path, - response=response, - request=request, + + # Almost need to set request_for_tracing(request) here... + # maybe it isn't even a tracing thing -- just an available context var? + + # By moving this here instead of _get_response, we don't have our sampler configured yet + # for custom use... + with tracer.start_as_current_span("plain.get_response") as span: + span.set_attribute("plain.request_id", request.unique_id) + span.set_attribute(http_attributes.HTTP_REQUEST_METHOD, request.method) + + response = self._middleware_chain(request) + response._resource_closers.append(request.close) + + span.set_attribute( + http_attributes.HTTP_RESPONSE_STATUS_CODE, response.status_code ) - return response + + if response.status_code >= 400: + log_response( + "%s: %s", + response.reason_phrase, + request.path, + response=response, + request=request, + ) + return response def _get_response(self, request): """ @@ -94,9 +114,17 @@ def resolve_request(self, request): Retrieve/set the urlrouter for the request. Return the view resolved, with its args and kwargs. """ + + span = trace.get_current_span() + # TODO set the other url stuff + span.set_attribute(url_attributes.URL_PATH, request.path_info) + resolver = get_resolver() # Resolve the view, and assign the match object back to the request. resolver_match = resolver.resolve(request.path_info) + + span.set_attribute(http_attributes.HTTP_ROUTE, resolver_match.route) + request.resolver_match = resolver_match return resolver_match diff --git a/plain/plain/views/base.py b/plain/plain/views/base.py index 293198403b..c67f0aa4fc 100644 --- a/plain/plain/views/base.py +++ b/plain/plain/views/base.py @@ -1,6 +1,8 @@ import logging from http import HTTPMethod +from opentelemetry import trace + from plain.http import ( HttpRequest, JsonResponse, @@ -16,6 +18,9 @@ logger = logging.getLogger("plain.request") +tracer = trace.get_tracer(__name__) + + class View: request: HttpRequest url_args: tuple @@ -35,9 +40,10 @@ def setup(self, request: HttpRequest, *url_args, **url_kwargs) -> None: @classonlymethod def as_view(cls, *init_args, **init_kwargs): def view(request, *url_args, **url_kwargs): - v = cls(*init_args, **init_kwargs) - v.setup(request, *url_args, **url_kwargs) - return v.get_response() + with tracer.start_as_current_span("plain.view"): + v = cls(*init_args, **init_kwargs) + v.setup(request, *url_args, **url_kwargs) + return v.get_response() view.view_class = cls diff --git a/plain/pyproject.toml b/plain/pyproject.toml index af9b6be9c5..927f60477f 100644 --- a/plain/pyproject.toml +++ b/plain/pyproject.toml @@ -7,6 +7,7 @@ readme = "README.md" dependencies = [ "jinja2>=3.1.2", "click>=8.0.0", + "opentelemetry-api>=1.34.1", ] requires-python = ">=3.11" diff --git a/pyproject.toml b/pyproject.toml index b61d974bac..b1bc72e41f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ requires-python = ">=3.11" "plain-tunnel" = { workspace = true } "plain-vendor" = { workspace = true } "plain-worker" = { workspace = true } +"plain-observe" = { workspace = true } [tool.uv.workspace] members = [ diff --git a/uv.lock b/uv.lock index 85c76c92a5..b2dcf2e91b 100644 --- a/uv.lock +++ b/uv.lock @@ -20,6 +20,7 @@ members = [ "plain-loginlink", "plain-models", "plain-oauth", + "plain-observe", "plain-pages", "plain-pageviews", "plain-passwords", @@ -233,6 +234,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, ] +[[package]] +name = "importlib-metadata" +version = "8.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656, upload-time = "2025-04-27T15:29:00.214Z" }, +] + [[package]] name = "iniconfig" version = "2.1.0" @@ -350,6 +363,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl", hash = "sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac", size = 154731, upload-time = "2015-06-02T09:12:40.57Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.34.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/5e/94a8cb759e4e409022229418294e098ca7feca00eb3c467bb20cbd329bda/opentelemetry_api-1.34.1.tar.gz", hash = "sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3", size = 64987, upload-time = "2025-06-10T08:55:19.818Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/3a/2ba85557e8dc024c0842ad22c570418dc02c36cbd1ab4b832a93edf071b8/opentelemetry_api-1.34.1-py3-none-any.whl", hash = "sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c", size = 65767, upload-time = "2025-06-10T08:54:56.717Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.34.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/41/fe20f9036433da8e0fcef568984da4c1d1c771fa072ecd1a4d98779dccdd/opentelemetry_sdk-1.34.1.tar.gz", hash = "sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d", size = 159441, upload-time = "2025-06-10T08:55:33.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/1b/def4fe6aa73f483cabf4c748f4c25070d5f7604dcc8b52e962983491b29e/opentelemetry_sdk-1.34.1-py3-none-any.whl", hash = "sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e", size = 118477, upload-time = "2025-06-10T08:55:16.02Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.55b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/f0/f33458486da911f47c4aa6db9bda308bb80f3236c111bf848bd870c16b16/opentelemetry_semantic_conventions-0.55b1.tar.gz", hash = "sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3", size = 119829, upload-time = "2025-06-10T08:55:33.881Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/89/267b0af1b1d0ba828f0e60642b6a5116ac1fd917cde7fc02821627029bd1/opentelemetry_semantic_conventions-0.55b1-py3-none-any.whl", hash = "sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed", size = 196223, upload-time = "2025-06-10T08:55:17.638Z" }, +] + [[package]] name = "packaging" version = "24.2" @@ -366,6 +419,7 @@ source = { editable = "plain" } dependencies = [ { name = "click" }, { name = "jinja2" }, + { name = "opentelemetry-api" }, ] [package.dev-dependencies] @@ -377,6 +431,7 @@ dev = [ requires-dist = [ { name = "click", specifier = ">=8.0.0" }, { name = "jinja2", specifier = ">=3.1.2" }, + { name = "opentelemetry-api", specifier = ">=1.34.1" }, ] [package.metadata.requires-dev] @@ -497,6 +552,7 @@ dependencies = [ { name = "plain-loginlink" }, { name = "plain-models" }, { name = "plain-oauth" }, + { name = "plain-observe" }, { name = "plain-pages" }, { name = "plain-pageviews" }, { name = "plain-passwords" }, @@ -527,6 +583,7 @@ requires-dist = [ { name = "plain-loginlink", editable = "plain-loginlink" }, { name = "plain-models", editable = "plain-models" }, { name = "plain-oauth", editable = "plain-oauth" }, + { name = "plain-observe", editable = "plain-observe" }, { name = "plain-pages", editable = "plain-pages" }, { name = "plain-pageviews", editable = "plain-pageviews" }, { name = "plain-passwords", editable = "plain-passwords" }, @@ -674,6 +731,7 @@ name = "plain-models" version = "0.33.1" source = { editable = "plain-models" } dependencies = [ + { name = "opentelemetry-semantic-conventions" }, { name = "plain" }, { name = "sqlparse" }, ] @@ -685,6 +743,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "opentelemetry-semantic-conventions", specifier = ">=0.55b1" }, { name = "plain", editable = "plain" }, { name = "sqlparse", specifier = ">=0.3.1" }, ] @@ -719,6 +778,35 @@ requires-dist = [ [package.metadata.requires-dev] dev = [{ name = "plain-pytest", editable = "plain-pytest" }] +[[package]] +name = "plain-observe" +version = "0.1.0" +source = { editable = "plain-observe" } +dependencies = [ + { name = "opentelemetry-sdk" }, + { name = "plain" }, + { name = "plain-auth" }, + { name = "plain-tailwind" }, + { name = "sqlparse" }, +] + +[package.dev-dependencies] +dev = [ + { name = "plain-pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "opentelemetry-sdk", specifier = ">=1.34.1" }, + { name = "plain", editable = "plain" }, + { name = "plain-auth", editable = "plain-auth" }, + { name = "plain-tailwind", editable = "plain-tailwind" }, + { name = "sqlparse", specifier = ">=0.2.2" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "plain-pytest", editable = "plain-pytest" }] + [[package]] name = "plain-pages" version = "0.10.3" @@ -1270,3 +1358,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1b/6c/c65773d6cab416a64d191d6ee8a8b1c68a09970ea6909d16965d26bfed1e/websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561", size = 176837, upload-time = "2025-03-05T20:02:55.237Z" }, { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, ] + +[[package]] +name = "zipp" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, +] From 932c06795a7951f8c9e2678b4bfbed10834754fb Mon Sep 17 00:00:00 2001 From: Dave Gaeddert Date: Sat, 12 Jul 2025 22:09:18 -0500 Subject: [PATCH 02/13] save --- .gitignore | 1 + plain-admin/plain/admin/default_settings.py | 6 - plain-admin/plain/admin/middleware.py | 3 +- plain-admin/plain/admin/querystats/README.md | 146 --------------- .../plain/admin/querystats/__init__.py | 3 - plain-admin/plain/admin/querystats/core.py | 155 ---------------- .../plain/admin/querystats/middleware.py | 102 ----------- plain-admin/plain/admin/querystats/urls.py | 10 -- plain-admin/plain/admin/querystats/views.py | 74 -------- .../templates/querystats/querystats.html | 144 --------------- .../admin/templates/querystats/toolbar.html | 90 ---------- .../admin/templates/toolbar/querystats.html | 28 --- .../admin/templates/toolbar/toolbar.html | 2 - plain-admin/plain/admin/toolbar.py | 6 - plain-admin/plain/admin/urls.py | 2 - plain-auth/plain/auth/middleware.py | 4 + plain-models/plain/models/observability.py | 24 +-- plain-observe/plain/observe/admin.py | 13 ++ plain-observe/plain/observe/cli.py | 23 +++ plain-observe/plain/observe/config.py | 23 +-- .../plain/observe/default_settings.py | 4 +- .../migrations/0009_trace_description.py | 18 ++ plain-observe/plain/observe/models.py | 27 +++ plain-observe/plain/observe/otel.py | 131 ++++++++++---- .../templates/observability/spans.html | 104 ----------- .../templates/observability/traces.html | 169 ++++++++++++++++++ plain-observe/plain/observe/views.py | 53 ++---- plain-sessions/plain/sessions/middleware.py | 6 + plain/plain/cli/core.py | 20 +++ plain/plain/internal/handlers/base.py | 39 ++-- plain/plain/templates/core.py | 12 +- plain/plain/views/base.py | 9 +- 32 files changed, 460 insertions(+), 991 deletions(-) delete mode 100644 plain-admin/plain/admin/querystats/README.md delete mode 100644 plain-admin/plain/admin/querystats/__init__.py delete mode 100644 plain-admin/plain/admin/querystats/core.py delete mode 100644 plain-admin/plain/admin/querystats/middleware.py delete mode 100644 plain-admin/plain/admin/querystats/urls.py delete mode 100644 plain-admin/plain/admin/querystats/views.py delete mode 100644 plain-admin/plain/admin/templates/querystats/querystats.html delete mode 100644 plain-admin/plain/admin/templates/querystats/toolbar.html delete mode 100644 plain-admin/plain/admin/templates/toolbar/querystats.html create mode 100644 plain-observe/plain/observe/cli.py create mode 100644 plain-observe/plain/observe/migrations/0009_trace_description.py delete mode 100644 plain-observe/plain/observe/templates/observability/spans.html create mode 100644 plain-observe/plain/observe/templates/observability/traces.html diff --git a/.gitignore b/.gitignore index a5bdd4f207..ab6fcfebff 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ plain*/tests/.plain .plain coverage.xml +.vscode diff --git a/plain-admin/plain/admin/default_settings.py b/plain-admin/plain/admin/default_settings.py index 10b607d1a0..70bad53c3f 100644 --- a/plain-admin/plain/admin/default_settings.py +++ b/plain-admin/plain/admin/default_settings.py @@ -1,8 +1,2 @@ ADMIN_TOOLBAR_CLASS = "plain.admin.toolbar.Toolbar" ADMIN_TOOLBAR_VERSION: str = "dev" - -ADMIN_QUERYSTATS_IGNORE_URLS: list[str] = [ - "/assets/.*", - "/admin/querystats/.*", - "/favicon.ico", -] diff --git a/plain-admin/plain/admin/middleware.py b/plain-admin/plain/admin/middleware.py index f7e57cde45..a0d817c83f 100644 --- a/plain-admin/plain/admin/middleware.py +++ b/plain-admin/plain/admin/middleware.py @@ -1,5 +1,4 @@ from .impersonate.middleware import ImpersonateMiddleware -from .querystats.middleware import QueryStatsMiddleware class AdminMiddleware: @@ -9,4 +8,4 @@ def __init__(self, get_response): self.get_response = get_response def __call__(self, request): - return QueryStatsMiddleware(ImpersonateMiddleware(self.get_response))(request) + return ImpersonateMiddleware(self.get_response)(request) diff --git a/plain-admin/plain/admin/querystats/README.md b/plain-admin/plain/admin/querystats/README.md deleted file mode 100644 index 853dfa7162..0000000000 --- a/plain-admin/plain/admin/querystats/README.md +++ /dev/null @@ -1,146 +0,0 @@ -# plain.querystats - -On-page database query stats in development and production. - -On each page, the query stats will display how many database queries were performed and how long they took. - -[Watch on YouTube](https://www.youtube.com/watch?v=NX8VXxVJm08) - -Clicking the stats in the toolbar will show the full SQL query log with tracebacks and timings. -This is even designed to work in production, -making it much easier to discover and debug performance issues on production data! - -![Django query stats](https://user-images.githubusercontent.com/649496/213781593-54197bb6-36a8-4c9d-8294-5b43bd86a4c9.png) - -It will also point out duplicate queries, -which can typically be removed by using `select_related`, -`prefetch_related`, or otherwise refactoring your code. - -## Installation - -```python -# settings.py -INSTALLED_PACKAGES = [ - # ... - "plain.admin.querystats", -] - -MIDDLEWARE = [ - "plain.sessions.middleware.SessionMiddleware", - "plain.auth.middleware.AuthenticationMiddleware", - - "plain.admin.querystats.QueryStatsMiddleware", - # Put additional middleware below querystats - # ... -] -``` - -We strongly recommend using the plain-toolbar along with this, -but if you aren't, -you can add the querystats to your frontend templates with this include: - -```html -{% include "querystats/button.html" %} -``` - -_Note that you will likely want to surround this with an if `DEBUG` or `is_admin` check._ - -To view querystats you need to send a POST request to `?querystats=store` (i.e. via a `
`), -and the template include is the easiest way to do that. - -## Tailwind CSS - -This package is styled with [Tailwind CSS](https://tailwindcss.com/), -and pairs well with [`plain-tailwind`](https://github.com/plainpackages/plain-tailwind). - -If you are using your own Tailwind implementation, -you can modify the "content" in your Tailwind config to include any Plain packages: - -```js -// tailwind.config.js -module.exports = { - content: [ - // ... - ".venv/lib/python*/site-packages/plain*/**/*.{html,js}", - ], - // ... -} -``` - -If you aren't using Tailwind, and don't intend to, open an issue to discuss other options. - -# plain.toolbar - -The admin toolbar is enabled for every user who `is_admin`. - -![Plain admin toolbar](https://user-images.githubusercontent.com/649496/213781915-a2094f54-99b8-4a05-a36e-dee107405229.png) - -## Installation - -Add `plaintoolbar` to your `INSTALLED_PACKAGES`, -and the `{% toolbar %}` to your base template: - -```python -# settings.py -INSTALLED_PACKAGES += [ - "plaintoolbar", -] -``` - -```html - -{% load toolbar %} - - - - ... - - - {% toolbar %} - ... - -``` - -More specific settings can be found below. - -## Tailwind CSS - -This package is styled with [Tailwind CSS](https://tailwindcss.com/), -and pairs well with [`plain-tailwind`](https://github.com/plainpackages/plain-tailwind). - -If you are using your own Tailwind implementation, -you can modify the "content" in your Tailwind config to include any Plain packages: - -```js -// tailwind.config.js -module.exports = { - content: [ - // ... - ".venv/lib/python*/site-packages/plain*/**/*.{html,js}", - ], - // ... -} -``` - -If you aren't using Tailwind, and don't intend to, open an issue to discuss other options. - -## Tailwind CSS - -This package is styled with [Tailwind CSS](https://tailwindcss.com/), -and pairs well with [`plain-tailwind`](https://github.com/plainpackages/plain-tailwind). - -If you are using your own Tailwind implementation, -you can modify the "content" in your Tailwind config to include any Plain packages: - -```js -// tailwind.config.js -module.exports = { - content: [ - // ... - ".venv/lib/python*/site-packages/plain*/**/*.{html,js}", - ], - // ... -} -``` - -If you aren't using Tailwind, and don't intend to, open an issue to discuss other options. diff --git a/plain-admin/plain/admin/querystats/__init__.py b/plain-admin/plain/admin/querystats/__init__.py deleted file mode 100644 index 0d8edc7a76..0000000000 --- a/plain-admin/plain/admin/querystats/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .middleware import QueryStatsMiddleware - -__all__ = ["QueryStatsMiddleware"] diff --git a/plain-admin/plain/admin/querystats/core.py b/plain-admin/plain/admin/querystats/core.py deleted file mode 100644 index 90768b0b6b..0000000000 --- a/plain-admin/plain/admin/querystats/core.py +++ /dev/null @@ -1,155 +0,0 @@ -import datetime -import time -import traceback -from collections import Counter -from functools import cached_property - -import sqlparse - -IGNORE_STACK_FILES = [ - "threading", - "concurrent/futures", - "functools.py", - "socketserver", - "wsgiref", - "gunicorn", - "whitenoise", - "sentry_sdk", - "querystats/core", - "plain/template/base", - "plain/models", - "plain/internal", -] - - -def pretty_print_sql(sql): - return sqlparse.format(sql, reindent=True, keyword_case="upper") - - -def get_stack(): - return "".join(tidy_stack(traceback.format_stack())) - - -def tidy_stack(stack): - lines = [] - - skip_next = False - - for line in stack: - if skip_next: - skip_next = False - continue - - if line.startswith(' File "') and any( - ignore in line for ignore in IGNORE_STACK_FILES - ): - skip_next = True - continue - - lines.append(line) - - return lines - - -class QueryStats: - def __init__(self, include_tracebacks): - self.queries = [] - self.include_tracebacks = include_tracebacks - - def __str__(self): - s = f"{self.num_queries} queries in {self.total_time_display}" - if self.duplicate_queries: - s += f" ({self.num_duplicate_queries} duplicates)" - return s - - def __call__(self, execute, sql, params, many, context): - current_query = {"sql": sql, "params": params, "many": many} - start = time.monotonic() - - result = execute(sql, params, many, context) - - if self.include_tracebacks: - current_query["tb"] = get_stack() - - # if many, then X times is len(params) - - # current_query["result"] = result - - current_query["duration"] = time.monotonic() - start - - self.queries.append(current_query) - return result - - @cached_property - def total_time(self): - return sum(q["duration"] for q in self.queries) - - @staticmethod - def get_time_display(seconds): - if seconds < 0.01: - return f"{seconds * 1000:.0f} ms" - return f"{seconds:.2f} seconds" - - @cached_property - def total_time_display(self): - return self.get_time_display(self.total_time) - - @cached_property - def num_queries(self): - return len(self.queries) - - # @cached_property - # def models(self): - # # parse table names from self.queries sql - # table_names = [x for x in [q['sql'].split(' ')[2] for q in self.queries] if x] - # models = connection.introspection.installed_models(table_names) - # return models - - @cached_property - def duplicate_queries(self): - sqls = [q["sql"] for q in self.queries] - duplicates = {k: v for k, v in Counter(sqls).items() if v > 1} - return duplicates - - @cached_property - def num_duplicate_queries(self): - # Count the number of "excess" queries by getting how many there - # are minus the initial one (and potentially only one required) - return sum(self.duplicate_queries.values()) - len(self.duplicate_queries) - - def as_summary_dict(self): - return { - "summary": str(self), - "total_time": self.total_time, - "num_queries": self.num_queries, - "num_duplicate_queries": self.num_duplicate_queries, - } - - def as_context_dict(self, request): - # If we don't create a dict, the instance of this class - # is lost before we can use it in the template - for query in self.queries: - # Add some useful display info - query["duration_display"] = self.get_time_display(query["duration"]) - query["sql_display"] = pretty_print_sql(query["sql"]) - duplicates = self.duplicate_queries.get(query["sql"], 0) - if duplicates: - query["duplicate_count"] = duplicates - - return { - **self.as_summary_dict(), - "request": { - "path": request.path, - "method": request.method, - "unique_id": request.unique_id, - }, - "timestamp": datetime.datetime.now().isoformat(), - "total_time_display": self.total_time_display, - "queries": self.queries, - } - - def as_server_timing(self): - duration = self.total_time * 1000 # put in ms - duration = round(duration, 2) - description = str(self) - return f'querystats;dur={duration};desc="{description}"' diff --git a/plain-admin/plain/admin/querystats/middleware.py b/plain-admin/plain/admin/querystats/middleware.py deleted file mode 100644 index 0dfe8f9d85..0000000000 --- a/plain-admin/plain/admin/querystats/middleware.py +++ /dev/null @@ -1,102 +0,0 @@ -import json -import logging -import re - -from plain.json import PlainJSONEncoder -from plain.models import db_connection -from plain.runtime import settings - -from .core import QueryStats - -try: - import psycopg -except ImportError: - psycopg = None - -logger = logging.getLogger(__name__) - - -class QueryStatsJSONEncoder(PlainJSONEncoder): - def default(self, obj): - try: - return super().default(obj) - except TypeError: - if psycopg and isinstance(obj, psycopg.types.json.Json): - return obj.obj - elif psycopg and isinstance(obj, psycopg.types.json.Jsonb): - return obj.obj - else: - raise - - -class QueryStatsMiddleware: - def __init__(self, get_response): - self.get_response = get_response - self.ignore_url_patterns = [ - re.compile(url) for url in settings.ADMIN_QUERYSTATS_IGNORE_URLS - ] - - def should_ignore_request(self, request): - for url in self.ignore_url_patterns: - if url.match(request.path): - return True - - return False - - def __call__(self, request): - """ - Enables querystats for the current request. - - If DEBUG or an admin, then Server-Timing headers are always added to the response. - Full querystats are only stored in the session if they are manually enabled. - """ - - if self.should_ignore_request(request): - return self.get_response(request) - - def is_tracking(): - return "querystats" in request.session - - querystats = QueryStats(include_tracebacks=is_tracking()) - - with db_connection.execute_wrapper(querystats): - is_admin = self.is_admin_request(request) - - if settings.DEBUG or is_admin: - with db_connection.execute_wrapper(querystats): - response = self.get_response(request) - - if settings.DEBUG: - # TODO logging settings - logger.debug("Querystats: %s", querystats) - - # Make current querystats available on the current page - # by using the server timing API which can be parsed client-side - response.headers["Server-Timing"] = querystats.as_server_timing() - - if is_tracking() and querystats.num_queries > 0: - request.session["querystats"][request.unique_id] = json.dumps( - querystats.as_context_dict(request), cls=QueryStatsJSONEncoder - ) - - # Keep 30 requests max, in case it is left on by accident - if len(request.session["querystats"]) > 30: - del request.session["querystats"][ - list(request.session["querystats"])[0] - ] - - # Did a deeper modification to the session dict... - request.session.modified = True - - return response - - else: - return self.get_response(request) - - @staticmethod - def is_admin_request(request): - if getattr(request, "impersonator", None): - # Support for impersonation (still want the real admin user to see the querystats) - return request.impersonator and request.impersonator.is_admin - - return hasattr(request, "user") and request.user and request.user.is_admin diff --git a/plain-admin/plain/admin/querystats/urls.py b/plain-admin/plain/admin/querystats/urls.py deleted file mode 100644 index 4b5e359468..0000000000 --- a/plain-admin/plain/admin/querystats/urls.py +++ /dev/null @@ -1,10 +0,0 @@ -from plain.urls import Router, path - -from . import views - - -class QuerystatsRouter(Router): - namespace = "querystats" - urls = [ - path("", views.QuerystatsView, name="querystats"), - ] diff --git a/plain-admin/plain/admin/querystats/views.py b/plain-admin/plain/admin/querystats/views.py deleted file mode 100644 index c7715e2f49..0000000000 --- a/plain-admin/plain/admin/querystats/views.py +++ /dev/null @@ -1,74 +0,0 @@ -import json - -from plain.auth.views import AuthViewMixin -from plain.http import ResponseRedirect -from plain.runtime import settings -from plain.views import TemplateView - - -class QuerystatsView(AuthViewMixin, TemplateView): - template_name = "querystats/querystats.html" - admin_required = True - - def check_auth(self): - # Allow the view if we're in DEBUG - if settings.DEBUG: - return - - super().check_auth() - - def get_response(self): - response = super().get_response() - # So we can load it in the toolbar - response.headers["X-Frame-Options"] = "SAMEORIGIN" - return response - - def get(self): - # Give an easy out if things get messed up - if ( - "clear" in self.request.query_params - and "querystats" in self.request.session - ): - del self.request.session["querystats"] - self.request.session.modified = True - - return super().get() - - def get_template_context(self): - context = super().get_template_context() - - querystats = self.request.session.get("querystats", {}) - - for request_id in list(querystats.keys()): - try: - querystats[request_id] = json.loads(querystats[request_id]) - except (json.JSONDecodeError, TypeError): - # If decoding fails, remove the entry from the dictionary - del querystats[request_id] - - # Order them by timestamp - querystats = dict( - sorted( - querystats.items(), - key=lambda item: item[1].get("timestamp", ""), - reverse=True, - ) - ) - - context["querystats"] = querystats - context["querystats_enabled"] = "querystats" in self.request.session - - return context - - def post(self): - querystats_action = self.request.data["querystats_action"] - - if querystats_action == "enable": - self.request.session.setdefault("querystats", {}) - elif querystats_action == "clear": - self.request.session["querystats"] = {} - elif querystats_action == "disable" and "querystats" in self.request.session: - del self.request.session["querystats"] - - # Redirect back to the page that submitted the form - return ResponseRedirect(self.request.data.get("redirect_url", ".")) diff --git a/plain-admin/plain/admin/templates/querystats/querystats.html b/plain-admin/plain/admin/templates/querystats/querystats.html deleted file mode 100644 index 886b72c7c6..0000000000 --- a/plain-admin/plain/admin/templates/querystats/querystats.html +++ /dev/null @@ -1,144 +0,0 @@ - - - - - - Querystats - {% tailwind_css %} - - - - {% if querystats_enabled %} -
- -
-
- - {{ csrf_input }} - - -
- {{ csrf_input }} - - -
-
- {{ csrf_input }} - - -
-
-
- {% endif %} - - {% if querystats %} -
- - -
- {% for request_id, qs in querystats.items() %} - - {% endfor %} -
-
- {% elif querystats_enabled %} -
Querystats are enabled but nothing has been recorded yet.
- {% else %} -
-
Querystats are disabled.
-
- {{ csrf_input }} - - -
-
- {% endif %} - - - - - diff --git a/plain-admin/plain/admin/templates/querystats/toolbar.html b/plain-admin/plain/admin/templates/querystats/toolbar.html deleted file mode 100644 index 85ccf60cd1..0000000000 --- a/plain-admin/plain/admin/templates/querystats/toolbar.html +++ /dev/null @@ -1,90 +0,0 @@ - diff --git a/plain-admin/plain/admin/templates/toolbar/querystats.html b/plain-admin/plain/admin/templates/toolbar/querystats.html deleted file mode 100644 index bc8c10d40d..0000000000 --- a/plain-admin/plain/admin/templates/toolbar/querystats.html +++ /dev/null @@ -1,28 +0,0 @@ -
-
-

Loading querystats...

-
-
- diff --git a/plain-admin/plain/admin/templates/toolbar/toolbar.html b/plain-admin/plain/admin/templates/toolbar/toolbar.html index dbeb6803bf..b34faf5ed0 100644 --- a/plain-admin/plain/admin/templates/toolbar/toolbar.html +++ b/plain-admin/plain/admin/templates/toolbar/toolbar.html @@ -58,8 +58,6 @@
- {% include "querystats/toolbar.html" %} -
{% if toolbar.request_exception() %} diff --git a/plain-admin/plain/admin/toolbar.py b/plain-admin/plain/admin/toolbar.py index fba30b9a20..9ade066cb7 100644 --- a/plain-admin/plain/admin/toolbar.py +++ b/plain-admin/plain/admin/toolbar.py @@ -101,9 +101,3 @@ def get_template_context(self): class _RequestToolbarPanel(ToolbarPanel): name = "Request" template_name = "toolbar/request.html" - - -@register_toolbar_panel -class _QuerystatsToolbarPanel(ToolbarPanel): - name = "Queries" - template_name = "toolbar/querystats.html" diff --git a/plain-admin/plain/admin/urls.py b/plain-admin/plain/admin/urls.py index 07ccab5c98..81695a5ece 100644 --- a/plain-admin/plain/admin/urls.py +++ b/plain-admin/plain/admin/urls.py @@ -2,7 +2,6 @@ from plain.urls import Router, include, path from .impersonate.urls import ImpersonateRouter -from .querystats.urls import QuerystatsRouter from .views.base import AdminView from .views.registry import registry @@ -36,7 +35,6 @@ class AdminRouter(Router): urls = [ path("search/", AdminSearchView, name="search"), include("impersonate/", ImpersonateRouter), - include("querystats/", QuerystatsRouter), include("", registry.get_urls()), path("", AdminIndexView, name="index"), ] diff --git a/plain-auth/plain/auth/middleware.py b/plain-auth/plain/auth/middleware.py index 1f6e295075..d81e3215ca 100644 --- a/plain-auth/plain/auth/middleware.py +++ b/plain-auth/plain/auth/middleware.py @@ -1,3 +1,5 @@ +from opentelemetry import trace + from plain import auth from plain.exceptions import ImproperlyConfigured from plain.utils.functional import SimpleLazyObject @@ -6,6 +8,8 @@ def get_user(request): if not hasattr(request, "_cached_user"): request._cached_user = auth.get_user(request) + if request._cached_user: + trace.get_current_span().set_attribute("user.id", request._cached_user.id) return request._cached_user diff --git a/plain-models/plain/models/observability.py b/plain-models/plain/models/observability.py index 2e967c8ae2..6c87d449c8 100644 --- a/plain-models/plain/models/observability.py +++ b/plain-models/plain/models/observability.py @@ -7,6 +7,7 @@ DB_NAMESPACE, DB_OPERATION_BATCH_SIZE, DB_OPERATION_NAME, + DB_QUERY_SUMMARY, DB_QUERY_TEXT, DB_SYSTEM_NAME, ) @@ -55,24 +56,27 @@ def db_span(db, sql: Any, *, many: bool = False, batch_size: int | None = None): yield None return + sql = str(sql) # Ensure SQL is a string for span attributes. + # Derive operation keyword (SELECT, INSERT, …) if possible. - operation: str | None = None - if isinstance(sql, str): - stripped = sql.lstrip() - if stripped: - operation = stripped.split()[0].upper() + operation = sql.lstrip().split()[0] + if " FROM " in sql: + table_name = sql.split(" FROM ")[1].strip().split()[0] + summary = f"{operation} {table_name}" + else: + summary = operation - # Span name per OTel SQL guidance. if many: - span_name = (operation or "EXECUTEMANY") + " many" - else: - span_name = operation or "QUERY" + summary = f"{summary} many" + + span_name = summary[:255] # Build attribute set. attrs: dict[str, Any] = { DB_SYSTEM_NAME: db_system_for(db.vendor), DB_NAMESPACE: db.settings_dict.get("NAME"), - DB_QUERY_TEXT: sql if isinstance(sql, str) else str(sql), + DB_QUERY_TEXT: sql, + DB_QUERY_SUMMARY: summary, } if user := db.settings_dict.get("USER"): diff --git a/plain-observe/plain/observe/admin.py b/plain-observe/plain/observe/admin.py index 3b78051e2a..f9116f9d3f 100644 --- a/plain-observe/plain/observe/admin.py +++ b/plain-observe/plain/observe/admin.py @@ -22,6 +22,12 @@ class ListView(AdminModelListView): "start_time", ] allow_global_search = False + # Actually want a button to delete ALL! not possible yet + # actions = ["Delete"] + + # def perform_action(self, action: str, target_pks: list): + # if action == "Delete": + # Trace.objects.filter(id__in=target_pks).delete() class DetailView(AdminModelDetailView): model = Trace @@ -42,6 +48,7 @@ class ListView(AdminModelListView): ] queryset_order = ["-pk"] allow_global_search = False + displays = ["Parents only"] def get_objects(self): return ( @@ -56,6 +63,12 @@ def get_objects(self): ) ) + def get_initial_queryset(self): + queryset = super().get_initial_queryset() + if self.display == "Parents only": + queryset = queryset.filter(parent_id="") + return queryset + class DetailView(AdminModelDetailView): model = Span diff --git a/plain-observe/plain/observe/cli.py b/plain-observe/plain/observe/cli.py new file mode 100644 index 0000000000..aa50f2178b --- /dev/null +++ b/plain-observe/plain/observe/cli.py @@ -0,0 +1,23 @@ +import click + +from plain.cli import register_cli +from plain.observe.models import Trace + + +@register_cli("observe") +@click.group("observe") +def observe_cli(): + pass + + +@observe_cli.command() +@click.option("--force", is_flag=True, help="Force clear all observability data.") +def clear(force: bool): + """Clear all observability data.""" + if not force: + click.confirm( + "Are you sure you want to clear all observability data? This cannot be undone.", + abort=True, + ) + + print("Deleted", Trace.objects.all().delete()) diff --git a/plain-observe/plain/observe/config.py b/plain-observe/plain/observe/config.py index 64c7b229b5..3799eea89e 100644 --- a/plain-observe/plain/observe/config.py +++ b/plain-observe/plain/observe/config.py @@ -1,13 +1,6 @@ -import re - -from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider, sampling -from opentelemetry.sdk.trace.export import BatchSpanProcessor - from plain.packages import PackageConfig, register_config -from plain.runtime import settings -from .otel import ObserveModelsExporter, PlainRequestSampler +from .otel import has_existing_trace_provider, setup_debug_trace_provider @register_config @@ -15,17 +8,7 @@ class Config(PackageConfig): package_label = "plainobserve" def ready(self): - current_provider = trace.get_tracer_provider() - if current_provider and not isinstance( - current_provider, trace.ProxyTracerProvider - ): + if has_existing_trace_provider(): return - ignore_url_patterns = [re.compile(p) for p in settings.OBSERVE_IGNORE_URLS] - - sampler = PlainRequestSampler( - sampling.ParentBased(sampling.ALWAYS_ON), ignore_url_patterns - ) - provider = TracerProvider(sampler=sampler) - provider.add_span_processor(BatchSpanProcessor(ObserveModelsExporter())) - trace.set_tracer_provider(provider) + setup_debug_trace_provider() diff --git a/plain-observe/plain/observe/default_settings.py b/plain-observe/plain/observe/default_settings.py index 2a7a9a53b3..509177de8f 100644 --- a/plain-observe/plain/observe/default_settings.py +++ b/plain-observe/plain/observe/default_settings.py @@ -1,5 +1,7 @@ -OBSERVE_IGNORE_URLS: list[str] = [ +OBSERVE_IGNORE_URL_PATTERNS: list[str] = [ "/assets/.*", "/admin/.*", + "/observe/.*", "/favicon.ico", ] +OBSERVE_TRACE_LIMIT: int = 100 diff --git a/plain-observe/plain/observe/migrations/0009_trace_description.py b/plain-observe/plain/observe/migrations/0009_trace_description.py new file mode 100644 index 0000000000..3b264182d5 --- /dev/null +++ b/plain-observe/plain/observe/migrations/0009_trace_description.py @@ -0,0 +1,18 @@ +# Generated by Plain 0.52.2 on 2025-07-12 21:37 + +from plain import models +from plain.models import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("plainobserve", "0008_alter_trace_options_trace_end_time"), + ] + + operations = [ + migrations.AddField( + model_name="trace", + name="description", + field=models.TextField(default="", required=False), + ), + ] diff --git a/plain-observe/plain/observe/models.py b/plain-observe/plain/observe/models.py index 9197d5875f..901a179661 100644 --- a/plain-observe/plain/observe/models.py +++ b/plain-observe/plain/observe/models.py @@ -1,3 +1,5 @@ +from opentelemetry.semconv.attributes import db_attributes + from plain import models @@ -7,6 +9,8 @@ class Trace(models.Model): start_time = models.DateTimeField(allow_null=True, required=False) end_time = models.DateTimeField(allow_null=True, required=False) + description = models.TextField(default="", required=False) + # Plain fields request_id = models.CharField(max_length=255, default="", required=False) session_id = models.CharField(max_length=255, default="", required=False) @@ -21,6 +25,14 @@ class Meta: ) ] + def __str__(self): + return self.trace_id + + def duration_ms(self): + if self.start_time and self.end_time: + return (self.end_time - self.start_time).total_seconds() * 1000 + return None + @models.register_model class Span(models.Model): @@ -53,3 +65,18 @@ class Meta: models.Index(fields=["trace"]), models.Index(fields=["start_time"]), ] + + def __str__(self): + return self.span_id + + def duration_ms(self): + if self.start_time and self.end_time: + return (self.end_time - self.start_time).total_seconds() * 1000 + return None + + def description(self): + if summary := self.attributes.get(db_attributes.DB_QUERY_SUMMARY): + return summary + if query := self.attributes.get(db_attributes.DB_QUERY_TEXT): + return query + return self.name diff --git a/plain-observe/plain/observe/otel.py b/plain-observe/plain/observe/otel.py index 0634035d7f..752bf6ee97 100644 --- a/plain-observe/plain/observe/otel.py +++ b/plain-observe/plain/observe/otel.py @@ -1,25 +1,54 @@ from __future__ import annotations import json +import logging import re -from collections.abc import Sequence -from opentelemetry.sdk.trace import sampling -from opentelemetry.sdk.trace.export import SpanExporter +from opentelemetry import baggage, trace +from opentelemetry.sdk.trace import TracerProvider, sampling +from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter from opentelemetry.semconv.attributes import url_attributes from opentelemetry.trace import SpanKind from plain.models.observability import suppress_tracing +from plain.runtime import settings + +logger = logging.getLogger(__name__) + + +def has_existing_trace_provider() -> bool: + """Check if there is an existing trace provider.""" + current_provider = trace.get_tracer_provider() + return current_provider and not isinstance( + current_provider, trace.ProxyTracerProvider + ) + + +def setup_debug_trace_provider() -> None: + sampler = PlainRequestSampler() + provider = TracerProvider(sampler=sampler) + provider.add_span_processor(BatchSpanProcessor(ObserveModelsExporter())) + trace.set_tracer_provider(provider) + + +def is_debug_trace_provider() -> bool: + """Check if the current trace provider is the debug trace provider.""" + current_provider = trace.get_tracer_provider() + if current_provider and current_provider.sampler is not None: + return isinstance(current_provider.sampler, PlainRequestSampler) + return False class PlainRequestSampler(sampling.Sampler): """Drops traces based on request path or user role.""" - def __init__( - self, delegate: sampling.Sampler, ignore_url_paths: Sequence[re.Pattern] - ): - self._delegate = delegate - self._ignore_url_paths = ignore_url_paths + def __init__(self): + self._delegate = sampling.ParentBased(sampling.ALWAYS_ON) + + # TODO ignore url namespace instead? admin, observe, assets + self._ignore_url_paths = [ + re.compile(p) for p in settings.OBSERVE_IGNORE_URL_PATTERNS + ] def should_sample( self, @@ -31,27 +60,32 @@ def should_sample( links=None, trace_state=None, **kwargs, - ): # type: ignore[override] - # Can we get the request path from the context or something instead of contextvar? - # not sure what to do with is_admin and stuff then... - + ): + # First, drop if the URL should be ignored. if attributes: - if url_path := attributes[url_attributes.URL_PATH]: - print("SHIT", url_path) + if url_path := attributes.get(url_attributes.URL_PATH, ""): for pattern in self._ignore_url_paths: if pattern.match(url_path): - return sampling.SamplingResult(sampling.Decision.DROP) - - # Example rule: drop staff/admin requests entirely. - # if getattr(request, "user", None) and getattr(request.user, "is_staff", False): - # return sampling.SamplingResult(sampling.Decision.DROP) - - # In dev we always sample? - - # Otherwise need an option to sample if session sampling enabled and is_admin + return sampling.SamplingResult( + sampling.Decision.DROP, + attributes=attributes, + ) - # does empty parent context tell us we're at a root, and only check this there? - # maybe is_admin should be an attribute, and observe_enabled could be an attribute + # Look for the "observe" cookie in the request and + # sample if it is set to "true". + if parent_context: + if cookies := baggage.get_baggage("http.request.cookies", parent_context): + # Using a signed cookie would be better -- only set by authed route + if cookies.get("observe") == "true": + return sampling.SamplingResult( + sampling.Decision.RECORD_AND_SAMPLE, + attributes=attributes, + ) + else: + return sampling.SamplingResult( + sampling.Decision.DROP, + attributes=attributes, + ) # Fallback to delegate sampler. return self._delegate.should_sample( @@ -71,7 +105,7 @@ def get_description(self) -> str: class ObserveModelsExporter(SpanExporter): """Exporter that writes spans into the observe models tables.""" - def export(self, spans): # type: ignore[override] + def export(self, spans): """Persist spans in bulk for efficiency.""" from .models import Span, Trace @@ -85,7 +119,14 @@ def export(self, spans): # type: ignore[override] trace_id = span_data["context"]["trace_id"] # There should be at least one span with this attribute - request_id = span_data["attributes"].get("plain.request_id", "") + request_id = span_data["attributes"].get("plain.request.id", "") + user_id = span_data["attributes"].get("user.id", "") + session_id = span_data["attributes"].get("session.id", "") + + if not span_data["parent_id"]: + description = span_data["name"] + else: + description = "" if trace := create_traces.get(trace_id): if not trace.start_time: @@ -102,12 +143,24 @@ def export(self, spans): # type: ignore[override] if not trace.request_id: trace.request_id = request_id + + if not trace.user_id: + trace.user_id = user_id + + if not trace.session_id: + trace.session_id = session_id + + if not trace.description: + trace.description = description else: trace = Trace( trace_id=trace_id, start_time=span_data["start_time"], end_time=span_data["end_time"], request_id=request_id, + user_id=user_id, + session_id=session_id, + description=description, ) create_traces[trace_id] = trace @@ -129,13 +182,23 @@ def export(self, spans): # type: ignore[override] ) ) - # Trace.objects.bulk_create( - # create_traces.values() - # ) # , update_conflicts=True, update_fields=["start_time", "end_time", "request_id"]) - # Span.objects.bulk_create(create_spans) + try: + Trace.objects.bulk_create( + create_traces.values() + ) # , update_conflicts=True, update_fields=["start_time", "end_time", "request_id"]) + Span.objects.bulk_create(create_spans) + except Exception as e: + logger.error( + "Failed to export spans to database: %s", + e, + exc_info=True, + ) - # TODO could delete old spans and stuff here instead of chore? or both? - # should be days based for sure (i.e. 30 days) - # could also be limit based as a fallback + # Delete oldest traces if we exceed the limit + if Trace.objects.count() > settings.OBSERVE_TRACE_LIMIT: + delete_ids = Trace.objects.order_by("start_time")[ + : settings.OBSERVE_TRACE_LIMIT + ].values_list("id", flat=True) + Trace.objects.filter(id__in=delete_ids).delete() return True diff --git a/plain-observe/plain/observe/templates/observability/spans.html b/plain-observe/plain/observe/templates/observability/spans.html deleted file mode 100644 index f9f7c5b75d..0000000000 --- a/plain-observe/plain/observe/templates/observability/spans.html +++ /dev/null @@ -1,104 +0,0 @@ - - - - - - Querystats - {% tailwind_css %} - - - - {% if observability_enabled %} -
- -
-
-
- {{ csrf_input }} - -
-
- {{ csrf_input }} - - -
-
- {{ csrf_input }} - - -
-
-
- {% endif %} - - {% if observability %} -
- - -
- {% for request_id, request_data in observability.get("requests", {}).items() %} - - {% endfor %} -
-
- {% elif observability_enabled %} -
Querystats are enabled but nothing has been recorded yet.
- {% else %} -
-
Querystats are disabled.
-
- {{ csrf_input }} - - -
-
- {% endif %} - - - - - diff --git a/plain-observe/plain/observe/templates/observability/traces.html b/plain-observe/plain/observe/templates/observability/traces.html new file mode 100644 index 0000000000..1eeb91dfeb --- /dev/null +++ b/plain-observe/plain/observe/templates/observability/traces.html @@ -0,0 +1,169 @@ + + + + + + Querystats + {% tailwind_css %} + + + + +
+ +
+
+ {% if traces %} +
+ {{ csrf_input }} + + +
+ {% endif %} + {% if observability_enabled %} +
+ {{ csrf_input }} + + +
+ {% else %} +
+ {{ csrf_input }} + + +
+ {% endif %} +
+
+ + {% if traces %} +
+ + +
+ {% for trace in traces %} + + {% endfor %} +
+
+ {% elif observability_enabled %} +
Observing but nothing has been recorded yet.
+ {% else %} +
+
Observability is disabled.
+
+ {{ csrf_input }} + + +
+
+ {% endif %} + + + + + diff --git a/plain-observe/plain/observe/views.py b/plain-observe/plain/observe/views.py index 26e802d2c5..a673368dee 100644 --- a/plain-observe/plain/observe/views.py +++ b/plain-observe/plain/observe/views.py @@ -1,12 +1,13 @@ from plain.auth.views import AuthViewMixin +from plain.http import ResponseRedirect from plain.runtime import settings from plain.views import TemplateView -# from .middleware import OBSERVABILITY_SESSION_KEY +from .models import Trace class ObservabilitySpansView(AuthViewMixin, TemplateView): - template_name = "observability/spans.html" + template_name = "observability/traces.html" admin_required = True def check_auth(self): @@ -24,39 +25,23 @@ def get_response(self): def get_template_context(self): context = super().get_template_context() - - # observability = self.request.session.get(OBSERVABILITY_SESSION_KEY, {}) - - # for request_id in list(spans.keys()): - # try: - # spans[request_id] = json.loads(spans[request_id]) - # except (json.JSONDecodeError, TypeError): - # # If decoding fails, remove the entry from the dictionary - # del spans[request_id] - - # Order them by timestamp - # spans = dict( - # sorted( - # spans.items(), - # key=lambda item: item[1].get("timestamp", ""), - # reverse=True, - # ) - # ) - - # context["observability"] = observability - # context["observability_enabled"] = OBSERVABILITY_SESSION_KEY in self.request.session - + context["observability_enabled"] = self.request.cookies.get("observe") == "true" + context["traces"] = Trace.objects.all() return context - # def post(self): - # querystats_action = self.request.data["querystats_action"] + def post(self): + observe_action = self.request.data["observe_action"] + + response = ResponseRedirect(self.request.data.get("redirect_url", ".")) - # if querystats_action == "enable": - # self.request.session.setdefault(OBSERVABILITY_SESSION_KEY, {}) - # elif querystats_action == "clear": - # self.request.session[OBSERVABILITY_SESSION_KEY] = {} - # elif querystats_action == "disable" and OBSERVABILITY_SESSION_KEY in self.request.session: - # del self.request.session[OBSERVABILITY_SESSION_KEY] + if observe_action == "enable": + response.set_cookie("observe", "true", max_age=60 * 60 * 24) + # self.request.session.setdefault(OBSERVABILITY_SESSION_KEY, {}) + elif observe_action == "clear": + Trace.objects.all().delete() + # self.request.session[OBSERVABILITY_SESSION_KEY] = {} + elif observe_action == "disable" and "observe" in self.request.cookies: + response.delete_cookie("observe") - # # Redirect back to the page that submitted the form - # return ResponseRedirect(self.request.data.get("redirect_url", ".")) + # Redirect back to the page that submitted the form + return response diff --git a/plain-sessions/plain/sessions/middleware.py b/plain-sessions/plain/sessions/middleware.py index 800cc31ca7..5c537d30de 100644 --- a/plain-sessions/plain/sessions/middleware.py +++ b/plain-sessions/plain/sessions/middleware.py @@ -1,5 +1,7 @@ import time +from opentelemetry import trace + from plain.runtime import settings from plain.utils.cache import patch_vary_headers from plain.utils.http import http_date @@ -13,6 +15,10 @@ def __init__(self, get_response): def __call__(self, request): session_key = request.cookies.get(settings.SESSION_COOKIE_NAME) + + if session_key: + trace.get_current_span().set_attribute("session.id", session_key) + request.session = SessionStore(session_key) response = self.get_response(request) diff --git a/plain/plain/cli/core.py b/plain/plain/cli/core.py index 783dc0efec..a7f50c52b5 100644 --- a/plain/plain/cli/core.py +++ b/plain/plain/cli/core.py @@ -1,7 +1,9 @@ +import os import traceback import click from click.core import Command, Context +from opentelemetry import trace import plain.runtime from plain.exceptions import ImproperlyConfigured @@ -108,6 +110,24 @@ def __init__(self, *args, **kwargs): self.sources = sources + def main(self, *args, **kwargs): + """ + Wrap the CLI invocation in an OpenTelemetry span to mark the start of a trace. + """ + tracer = trace.get_tracer(__name__) + with tracer.start_as_current_span( + "plain", + kind=trace.SpanKind.INTERNAL, + attributes={ + "process.executable.name": "plain", + # "process.executable.path": sys.executable, + "process.pid": str(os.getpid()), + # "process.command_args": sys.argv[1:], # sensitive? + }, + ): + # process.exit.code + return super().main(*args, **kwargs) + def get_command(self, ctx: Context, cmd_name: str) -> Command | None: cmd = super().get_command(ctx, cmd_name) if cmd: diff --git a/plain/plain/internal/handlers/base.py b/plain/plain/internal/handlers/base.py index 02b443e065..02c708d018 100644 --- a/plain/plain/internal/handlers/base.py +++ b/plain/plain/internal/handlers/base.py @@ -1,7 +1,7 @@ import logging import types -from opentelemetry import trace +from opentelemetry import baggage, trace from opentelemetry.semconv.attributes import http_attributes, url_attributes from plain.exceptions import ImproperlyConfigured @@ -41,8 +41,7 @@ def load_middleware(self): Must be called after the environment is fixed (see __call__ in subclasses). """ - get_response = self._get_response - handler = convert_exception_to_response(get_response) + handler = convert_exception_to_response(self._get_response) middlewares = reversed( BUILTIN_BEFORE_MIDDLEWARE + settings.MIDDLEWARE + BUILTIN_AFTER_MIDDLEWARE @@ -66,21 +65,30 @@ def load_middleware(self): def get_response(self, request): """Return a Response object for the given HttpRequest.""" - # Almost need to set request_for_tracing(request) here... - # maybe it isn't even a tracing thing -- just an available context var? - - # By moving this here instead of _get_response, we don't have our sampler configured yet - # for custom use... - with tracer.start_as_current_span("plain.get_response") as span: - span.set_attribute("plain.request_id", request.unique_id) - span.set_attribute(http_attributes.HTTP_REQUEST_METHOD, request.method) - + span_attributes = { + "plain.request.id": request.unique_id, + http_attributes.HTTP_REQUEST_METHOD: request.method, + # TODO set the other url stuff? + url_attributes.URL_PATH: request.path_info, + # http_attributes: request.content_type, + } + span_context = baggage.set_baggage("http.request.cookies", request.cookies) + + with tracer.start_as_current_span( + f"{request.method} {request.path_info}", + context=span_context, + attributes=span_attributes, + kind=trace.SpanKind.SERVER, + ) as span: response = self._middleware_chain(request) response._resource_closers.append(request.close) span.set_attribute( http_attributes.HTTP_RESPONSE_STATUS_CODE, response.status_code ) + # span.set_attribute( + # http_attributes.HTTP_RESPONSE_REASON_PHRASE, response.reason_phrase + # ) if response.status_code >= 400: log_response( @@ -115,15 +123,14 @@ def resolve_request(self, request): with its args and kwargs. """ - span = trace.get_current_span() - # TODO set the other url stuff - span.set_attribute(url_attributes.URL_PATH, request.path_info) - resolver = get_resolver() # Resolve the view, and assign the match object back to the request. resolver_match = resolver.resolve(request.path_info) + span = trace.get_current_span() span.set_attribute(http_attributes.HTTP_ROUTE, resolver_match.route) + # Route makes a better name + span.update_name(f"{request.method} {resolver_match.route}") request.resolver_match = resolver_match return resolver_match diff --git a/plain/plain/templates/core.py b/plain/plain/templates/core.py index dd493cc780..1aacb29240 100644 --- a/plain/plain/templates/core.py +++ b/plain/plain/templates/core.py @@ -1,7 +1,10 @@ import jinja2 +from opentelemetry import trace from .jinja import environment +tracer = trace.get_tracer(__name__) + class TemplateFileMissing(Exception): def __str__(self) -> str: @@ -21,4 +24,11 @@ def __init__(self, filename: str) -> None: raise TemplateFileMissing(filename) def render(self, context: dict) -> str: - return self._jinja_template.render(context) + with tracer.start_as_current_span( + f"Template {self.filename}", + attributes={ + "template.filename": self.filename, + "template.context_keys": list(context.keys()), + }, + ): + return self._jinja_template.render(context) diff --git a/plain/plain/views/base.py b/plain/plain/views/base.py index c67f0aa4fc..ceacaedb36 100644 --- a/plain/plain/views/base.py +++ b/plain/plain/views/base.py @@ -40,7 +40,14 @@ def setup(self, request: HttpRequest, *url_args, **url_kwargs) -> None: @classonlymethod def as_view(cls, *init_args, **init_kwargs): def view(request, *url_args, **url_kwargs): - with tracer.start_as_current_span("plain.view"): + with tracer.start_as_current_span( + f"View {cls.__name__}", + attributes={ + "view.class": cls.__name__, + "view.url_args": url_args, + # "view.url_kwargs": list(url_kwargs), # can't be dict + }, + ): v = cls(*init_args, **init_kwargs) v.setup(request, *url_args, **url_kwargs) return v.get_response() From ac528ce7a9761e93e43981ad7f6299ee9c62c9b9 Mon Sep 17 00:00:00 2001 From: Dave Gaeddert Date: Sat, 12 Jul 2025 22:12:07 -0500 Subject: [PATCH 03/13] save --- plain-models/pyproject.toml | 3 --- uv.lock | 2 -- 2 files changed, 5 deletions(-) diff --git a/plain-models/pyproject.toml b/plain-models/pyproject.toml index cb3dfe203f..05958f4817 100644 --- a/plain-models/pyproject.toml +++ b/plain-models/pyproject.toml @@ -8,9 +8,6 @@ requires-python = ">=3.11" dependencies = [ "plain<1.0.0", "sqlparse>=0.3.1", - # plain.models relies on OpenTelemetry semantic conventions for - # span/attribute constants used in observability helpers. - "opentelemetry-semantic-conventions>=0.55b1", ] [project.entry-points."plain.setup"] diff --git a/uv.lock b/uv.lock index 61f52968ac..73e4410de3 100644 --- a/uv.lock +++ b/uv.lock @@ -753,7 +753,6 @@ name = "plain-models" version = "0.35.0" source = { editable = "plain-models" } dependencies = [ - { name = "opentelemetry-semantic-conventions" }, { name = "plain" }, { name = "sqlparse" }, ] @@ -765,7 +764,6 @@ dev = [ [package.metadata] requires-dist = [ - { name = "opentelemetry-semantic-conventions", specifier = ">=0.55b1" }, { name = "plain", editable = "plain" }, { name = "sqlparse", specifier = ">=0.3.1" }, ] From 4df9137ee1335ba235dac63fe2dff3c97b79d0fe Mon Sep 17 00:00:00 2001 From: Dave Gaeddert Date: Mon, 14 Jul 2025 13:51:43 -0500 Subject: [PATCH 04/13] save --- demos/full/app/settings.py | 1 - .../plain/admin/assets/toolbar/toolbar.js | 21 + .../admin/templates/toolbar/toolbar.html | 6 + plain-admin/plain/admin/toolbar.py | 9 + plain-models/plain/models/observability.py | 141 ++++--- plain-observe/plain/observe/admin.py | 24 +- plain-observe/plain/observe/core.py | 52 +++ .../plain/observe/default_settings.py | 1 + plain-observe/plain/observe/models.py | 75 ++++ plain-observe/plain/observe/otel.py | 394 ++++++++++++++---- .../templates/admin/observe/trace_detail.html | 11 + .../observability/_trace_detail.html | 279 +++++++++++++ .../templates/observability/traces.html | 364 +++++++++------- .../toolbar/observability_button.html | 49 +++ plain-observe/plain/observe/views.py | 76 +++- plain/plain/cli/core.py | 12 +- plain/plain/http/cookie.py | 44 ++ plain/plain/http/request.py | 15 + plain/plain/http/response.py | 17 +- plain/plain/internal/handlers/base.py | 19 +- plain/plain/templates/core.py | 18 +- plain/plain/templates/jinja/filters.py | 20 + plain/plain/views/base.py | 26 +- plain/pyproject.toml | 1 + uv.lock | 2 + 25 files changed, 1356 insertions(+), 321 deletions(-) create mode 100644 plain-observe/plain/observe/core.py create mode 100644 plain-observe/plain/observe/templates/admin/observe/trace_detail.html create mode 100644 plain-observe/plain/observe/templates/observability/_trace_detail.html create mode 100644 plain-observe/plain/observe/templates/toolbar/observability_button.html diff --git a/demos/full/app/settings.py b/demos/full/app/settings.py index 002cfbf6e9..520109ca9a 100644 --- a/demos/full/app/settings.py +++ b/demos/full/app/settings.py @@ -33,7 +33,6 @@ MIDDLEWARE = [ "plain.sessions.middleware.SessionMiddleware", "plain.auth.middleware.AuthenticationMiddleware", - "plain.observe.middelware.ObserveMiddleware", "plain.admin.AdminMiddleware", ] diff --git a/plain-admin/plain/admin/assets/toolbar/toolbar.js b/plain-admin/plain/admin/assets/toolbar/toolbar.js index 38c811840b..6b3c09edbb 100644 --- a/plain-admin/plain/admin/assets/toolbar/toolbar.js +++ b/plain-admin/plain/admin/assets/toolbar/toolbar.js @@ -78,6 +78,15 @@ const plainToolbar = { } localStorage.setItem("plaintoolbar.tab", tabName); }, + resetHeight: () => { + const content = document.querySelector( + "#plaintoolbar-details [data-resizer]", + )?.nextElementSibling; + if (content) { + content.style.height = ""; + localStorage.removeItem("plaintoolbar.height"); + } + }, }; // Render it hidden immediately if the user has hidden it before @@ -95,6 +104,16 @@ window.addEventListener("load", () => { if (lastTab) { plainToolbar.showTab(lastTab); } + // Restore custom height if it was set + const savedHeight = localStorage.getItem("plaintoolbar.height"); + if (savedHeight) { + const content = document.querySelector( + "#plaintoolbar-details [data-resizer]", + )?.nextElementSibling; + if (content) { + content.style.height = savedHeight; + } + } } else if (state === "0") { plainToolbar.collapse(); } @@ -172,6 +191,8 @@ window.addEventListener("load", () => { isDragging = false; handle.style.cursor = "grab"; document.body.style.userSelect = ""; + // Save the new height to localStorage + localStorage.setItem("plaintoolbar.height", content.style.height); } }); } diff --git a/plain-admin/plain/admin/templates/toolbar/toolbar.html b/plain-admin/plain/admin/templates/toolbar/toolbar.html index b34faf5ed0..95f5044d7a 100644 --- a/plain-admin/plain/admin/templates/toolbar/toolbar.html +++ b/plain-admin/plain/admin/templates/toolbar/toolbar.html @@ -68,6 +68,12 @@ {% endif %} + {% for panel in panels %} + {% if panel.button_template_name %} + {{ panel.render_button() }} + {% endif %} + {% endfor %} + Admin {% include "toolbar/links.html" ignore missing %} diff --git a/plain-admin/plain/admin/toolbar.py b/plain-admin/plain/admin/toolbar.py index 9ade066cb7..6dd6e69d92 100644 --- a/plain-admin/plain/admin/toolbar.py +++ b/plain-admin/plain/admin/toolbar.py @@ -49,6 +49,7 @@ def get_panels(self): class ToolbarPanel: name: str template_name: str + button_template_name: str = "" def __init__(self, request): self.request = request @@ -63,6 +64,14 @@ def render(self): context = self.get_template_context() return mark_safe(template.render(context)) + def render_button(self): + """Render the toolbar button for the minimized state.""" + if not self.button_template_name: + return "" + template = Template(self.button_template_name) + context = self.get_template_context() + return mark_safe(template.render(context)) + class _ToolbarPanelRegistry: def __init__(self): diff --git a/plain-models/plain/models/observability.py b/plain-models/plain/models/observability.py index 6c87d449c8..160aada49a 100644 --- a/plain-models/plain/models/observability.py +++ b/plain-models/plain/models/observability.py @@ -1,40 +1,32 @@ +import re from contextlib import contextmanager from typing import Any from opentelemetry import context as otel_context from opentelemetry import trace -from opentelemetry.semconv.attributes.db_attributes import ( +from opentelemetry.semconv._incubating.attributes.db_attributes import ( + DB_COLLECTION_NAME, DB_NAMESPACE, DB_OPERATION_BATCH_SIZE, DB_OPERATION_NAME, DB_QUERY_SUMMARY, DB_QUERY_TEXT, - DB_SYSTEM_NAME, + DB_SYSTEM, ) from opentelemetry.semconv.attributes.network_attributes import ( NETWORK_PEER_ADDRESS, NETWORK_PEER_PORT, ) from opentelemetry.semconv.trace import DbSystemValues -from opentelemetry.trace import SpanKind - -# Import the official suppression key used by OTel instrumentations when -# available (present once any opentelemetry-instrumentation- is -# installed). Fallback to a module-local object so our own helpers can still -# reference it. -try: - from opentelemetry.instrumentation.utils import ( - _SUPPRESS_INSTRUMENTATION_KEY as _SUPPRESS_KEY, - ) -except ImportError: # instrumentation extras not installed - _SUPPRESS_KEY = object() +from opentelemetry.trace import SpanKind, StatusCode +_SUPPRESS_KEY = object() tracer = trace.get_tracer("plain.models") def db_system_for(vendor: str) -> str: # noqa: D401 – simple helper - """Return the canonical ``db.system.name`` value for a backend vendor.""" + """Return the canonical ``db.system`` value for a backend vendor.""" return { "postgresql": DbSystemValues.POSTGRESQL.value, @@ -44,11 +36,61 @@ def db_system_for(vendor: str) -> str: # noqa: D401 – simple helper }.get(vendor, vendor) +def extract_operation_and_target(sql: str) -> tuple[str, str | None, str | None]: + """Extract operation, table name, and collection from SQL. + + Returns: (operation, summary, collection_name) + """ + sql_upper = sql.upper().strip() + operation = sql_upper.split()[0] if sql_upper else "UNKNOWN" + + # Pattern to match quoted and unquoted identifiers + # Matches: "quoted", `quoted`, [quoted], unquoted.name + identifier_pattern = r'("([^"]+)"|`([^`]+)`|\[([^\]]+)\]|([\w.]+))' + + # Extract table/collection name based on operation + collection_name = None + summary = operation + + if operation in ("SELECT", "DELETE"): + match = re.search(rf"FROM\s+{identifier_pattern}", sql, re.IGNORECASE) + if match: + collection_name = _clean_identifier(match.group(1)) + summary = f"{operation} {collection_name}" + + elif operation in ("INSERT", "REPLACE"): + match = re.search(rf"INTO\s+{identifier_pattern}", sql, re.IGNORECASE) + if match: + collection_name = _clean_identifier(match.group(1)) + summary = f"{operation} {collection_name}" + + elif operation == "UPDATE": + match = re.search(rf"UPDATE\s+{identifier_pattern}", sql, re.IGNORECASE) + if match: + collection_name = _clean_identifier(match.group(1)) + summary = f"{operation} {collection_name}" + + return operation, summary, collection_name + + +def _clean_identifier(identifier: str) -> str: + """Remove quotes from SQL identifiers.""" + # Remove different types of SQL quotes + if identifier.startswith('"') and identifier.endswith('"'): + return identifier[1:-1] + elif identifier.startswith("`") and identifier.endswith("`"): + return identifier[1:-1] + elif identifier.startswith("[") and identifier.endswith("]"): + return identifier[1:-1] + return identifier + + @contextmanager def db_span(db, sql: Any, *, many: bool = False, batch_size: int | None = None): """Open an OpenTelemetry CLIENT span for a database query. All common attributes (`db.*`, `network.*`, etc.) are set automatically. + Follows OpenTelemetry semantic conventions for database instrumentation. """ # Fast-exit if instrumentation suppression flag set in context. @@ -58,30 +100,36 @@ def db_span(db, sql: Any, *, many: bool = False, batch_size: int | None = None): sql = str(sql) # Ensure SQL is a string for span attributes. - # Derive operation keyword (SELECT, INSERT, …) if possible. - operation = sql.lstrip().split()[0] - if " FROM " in sql: - table_name = sql.split(" FROM ")[1].strip().split()[0] - summary = f"{operation} {table_name}" - else: - summary = operation + # Extract operation and target information + operation, summary, collection_name = extract_operation_and_target(sql) if many: summary = f"{summary} many" - span_name = summary[:255] + # Span name follows semantic conventions: {target} or {db.operation.name} {target} + if summary: + span_name = summary[:255] + else: + span_name = operation - # Build attribute set. + # Build attribute set following semantic conventions attrs: dict[str, Any] = { - DB_SYSTEM_NAME: db_system_for(db.vendor), + DB_SYSTEM: db_system_for(db.vendor), DB_NAMESPACE: db.settings_dict.get("NAME"), - DB_QUERY_TEXT: sql, + DB_QUERY_TEXT: sql, # Already parameterized from Django/Plain DB_QUERY_SUMMARY: summary, + DB_OPERATION_NAME: operation, } + # Add collection name if detected + if collection_name: + attrs[DB_COLLECTION_NAME] = collection_name + + # Add user attribute (following newer conventions) if user := db.settings_dict.get("USER"): attrs["db.user"] = user + # Network attributes if host := db.settings_dict.get("HOST"): attrs[NETWORK_PEER_ADDRESS] = host @@ -91,28 +139,27 @@ def db_span(db, sql: Any, *, many: bool = False, batch_size: int | None = None): except (TypeError, ValueError): pass - # executemany: include batch size when >1 per semantic conventions. + # Batch size for executemany operations if batch_size and batch_size > 1: attrs[DB_OPERATION_BATCH_SIZE] = batch_size with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span: + # Set all non-None attributes for key, value in attrs.items(): if value is not None: span.set_attribute(key, value) - if operation: - span.set_attribute(DB_OPERATION_NAME, operation) - - yield span - - -# --------------------------------------------------------------------------- -# Context manager to suppress *all* instrumentation inside the block -# --------------------------------------------------------------------------- + try: + yield span + except Exception as e: + # Record exception and set error status + span.record_exception(e) + span.set_status(StatusCode.ERROR, str(e)) + raise @contextmanager -def suppress_tracing(): +def suppress_db_tracing(): """Temporarily disable **all** OpenTelemetry instrumentation. This sets the standard suppression flag recognised by every official @@ -125,23 +172,3 @@ def suppress_tracing(): yield finally: otel_context.detach(token) - - -# --------------------------------------------------------------------------- -# Helper to disable DB spans temporarily -# --------------------------------------------------------------------------- - - -@contextmanager -def disable_db_spans(): - """Temporarily disable ``db_span`` within the current context. - - Useful when the application writes traces/spans back into the database to - avoid generating additional spans for those internal operations. - """ - - # Retained for backward compatibility; internally delegates to - # `suppress_tracing()` so that *all* instrumentation is suppressed, which - # is usually what callers expect. - with suppress_tracing(): - yield diff --git a/plain-observe/plain/observe/admin.py b/plain-observe/plain/observe/admin.py index f9116f9d3f..2683191558 100644 --- a/plain-observe/plain/observe/admin.py +++ b/plain-observe/plain/observe/admin.py @@ -1,3 +1,5 @@ +from functools import cached_property + from plain.admin.toolbar import ToolbarPanel, register_toolbar_panel from plain.admin.views import ( AdminModelDetailView, @@ -6,6 +8,7 @@ register_viewset, ) +from .core import Observer from .models import Span, Trace @@ -31,7 +34,15 @@ class ListView(AdminModelListView): class DetailView(AdminModelDetailView): model = Trace - # title = "Cached item" + template_name = "admin/observe/trace_detail.html" + + def get_template_context(self): + context = super().get_template_context() + trace_id = self.url_kwargs["pk"] + context["trace"] = Trace.objects.get(pk=trace_id) + context["observability_enabled"] = True # Always enabled in admin + context["show_delete_button"] = False + return context @register_viewset @@ -77,3 +88,14 @@ class DetailView(AdminModelDetailView): class ObservabilityToolbarPanel(ToolbarPanel): name = "Observability" template_name = "toolbar/observability.html" + button_template_name = "toolbar/observability_button.html" + + @cached_property + def observer(self): + """Get the Observer instance for this request.""" + return Observer(self.request) + + def get_template_context(self): + context = super().get_template_context() + context["observer"] = self.observer + return context diff --git a/plain-observe/plain/observe/core.py b/plain-observe/plain/observe/core.py new file mode 100644 index 0000000000..a59f0da601 --- /dev/null +++ b/plain-observe/plain/observe/core.py @@ -0,0 +1,52 @@ +""" +Core observability functionality and Observer class. +""" + + +class Observer: + """Central class for managing observability state and operations.""" + + def __init__(self, request): + self.request = request + + @property + def mode(self): + """Get the current observability mode from signed cookie.""" + return self.request.get_signed_cookie("observe", default=None) + + @property + def is_enabled(self): + """Check if observability is enabled (either record or sample mode).""" + return self.mode in ("record", "sample") + + @property + def is_sampling(self): + """Check if full sampling (with DB export) is enabled.""" + return self.mode == "sample" + + @property + def is_recording(self): + """Check if record-only mode is enabled.""" + return self.mode == "record" + + def enable_record_mode(self, response): + """Enable record-only mode (real-time monitoring, no DB export).""" + response.set_signed_cookie("observe", "record", max_age=60 * 60 * 24) + + def enable_sample_mode(self, response): + """Enable full sampling mode (real-time monitoring + DB export).""" + response.set_signed_cookie("observe", "sample", max_age=60 * 60 * 24) + + def disable(self, response): + """Disable observability by deleting the cookie.""" + response.delete_cookie("observe") + + def get_current_trace_summary(self): + """Get performance summary string for the currently active trace.""" + from .otel import get_span_collector + + span_collector = get_span_collector() + if not span_collector: + return None + + return span_collector.get_current_trace_summary() diff --git a/plain-observe/plain/observe/default_settings.py b/plain-observe/plain/observe/default_settings.py index 509177de8f..804ff17647 100644 --- a/plain-observe/plain/observe/default_settings.py +++ b/plain-observe/plain/observe/default_settings.py @@ -3,5 +3,6 @@ "/admin/.*", "/observe/.*", "/favicon.ico", + "/.well-known/.*", ] OBSERVE_TRACE_LIMIT: int = 100 diff --git a/plain-observe/plain/observe/models.py b/plain-observe/plain/observe/models.py index 901a179661..a71ce2fce8 100644 --- a/plain-observe/plain/observe/models.py +++ b/plain-observe/plain/observe/models.py @@ -1,3 +1,7 @@ +from datetime import UTC, datetime +from functools import cached_property + +import sqlparse from opentelemetry.semconv.attributes import db_attributes from plain import models @@ -33,6 +37,29 @@ def duration_ms(self): return (self.end_time - self.start_time).total_seconds() * 1000 return None + def get_summary(self): + """Get a concise summary string for toolbar display.""" + spans = self.spans.all() + + if not spans.exists(): + return "" + + total_spans = spans.count() + db_queries = spans.filter(attributes__has_key="db.system").count() + + # Build summary parts + parts = [f"{total_spans}sp"] + + if db_queries > 0: + parts.append(f"{db_queries}db") + + # Add duration if available + duration_ms = self.duration_ms() + if duration_ms is not None: + parts.append(f"{round(duration_ms, 1)}ms") + + return " ".join(parts) + @models.register_model class Span(models.Model): @@ -80,3 +107,51 @@ def description(self): if query := self.attributes.get(db_attributes.DB_QUERY_TEXT): return query return self.name + + @cached_property + def sql_query(self): + """Get the SQL query if this span contains one.""" + return self.attributes.get("db.query.text") + + def get_formatted_sql(self): + """Get the pretty-formatted SQL query if this span contains one.""" + sql = self.sql_query + if not sql: + return None + + return sqlparse.format( + sql, + reindent=True, + keyword_case="upper", + identifier_case="lower", + strip_comments=False, + strip_whitespace=True, + indent_width=2, + wrap_after=80, + comma_first=False, + ) + + def format_event_timestamp(self, timestamp): + """Convert event timestamp to a readable datetime.""" + if isinstance(timestamp, int | float): + try: + # Try as seconds first + if timestamp > 1e10: # Likely nanoseconds + timestamp = timestamp / 1e9 + elif timestamp > 1e7: # Likely milliseconds + timestamp = timestamp / 1e3 + + return datetime.fromtimestamp(timestamp, tz=UTC) + except (ValueError, OSError): + return str(timestamp) + return timestamp + + def get_exception_stacktrace(self): + """Get the exception stacktrace if this span has an exception event.""" + if not self.events: + return None + + for event in self.events: + if event.get("name") == "exception" and event.get("attributes"): + return event["attributes"].get("exception.stacktrace") + return None diff --git a/plain-observe/plain/observe/otel.py b/plain-observe/plain/observe/otel.py index 752bf6ee97..de03b9aad7 100644 --- a/plain-observe/plain/observe/otel.py +++ b/plain-observe/plain/observe/otel.py @@ -1,21 +1,158 @@ from __future__ import annotations -import json import logging import re +import threading +from collections import defaultdict +from datetime import UTC, datetime from opentelemetry import baggage, trace -from opentelemetry.sdk.trace import TracerProvider, sampling +from opentelemetry.sdk.trace import SpanProcessor, TracerProvider, sampling from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter from opentelemetry.semconv.attributes import url_attributes from opentelemetry.trace import SpanKind -from plain.models.observability import suppress_tracing +from plain.http.cookie import unsign_cookie_value +from plain.models.observability import suppress_db_tracing from plain.runtime import settings logger = logging.getLogger(__name__) +class RealTimeSpanCollector(SpanProcessor): + """Collects spans in real-time for current trace performance monitoring.""" + + def __init__(self): + self.active_spans_by_trace = defaultdict(dict) # trace_id -> {span_id: span} + self.completed_spans_by_trace = defaultdict(list) # trace_id -> [spans] + self.lock = threading.Lock() + + def on_start(self, span, parent_context=None): + """Called when a span starts.""" + with self.lock: + trace_id = format(span.get_span_context().trace_id, "032x") + span_id = format(span.get_span_context().span_id, "016x") + self.active_spans_by_trace[trace_id][span_id] = span + + def on_end(self, span): + """Called when a span ends.""" + with self.lock: + trace_id = format(span.get_span_context().trace_id, "032x") + span_id = format(span.get_span_context().span_id, "016x") + + # Move from active to completed + if trace_id in self.active_spans_by_trace: + span_obj = self.active_spans_by_trace[trace_id].pop(span_id, None) + if span_obj: + self.completed_spans_by_trace[trace_id].append(span_obj) + + # Clean up empty trace entries + if not self.active_spans_by_trace[trace_id]: + del self.active_spans_by_trace[trace_id] + + def get_current_trace_summary(self): + """Get performance summary for the currently active trace.""" + current_span = trace.get_current_span() + if not current_span: + # If no current span, check if we have any active traces at all + with self.lock: + if not self.active_spans_by_trace and not self.completed_spans_by_trace: + return None + + # Get the most recent trace if we can't find current span + all_trace_ids = list(self.active_spans_by_trace.keys()) + list( + self.completed_spans_by_trace.keys() + ) + if not all_trace_ids: + return None + trace_id = all_trace_ids[-1] # Use most recent + else: + # Use the current span's trace + trace_id = format(current_span.get_span_context().trace_id, "032x") + + with self.lock: + active_spans = list(self.active_spans_by_trace.get(trace_id, {}).values()) + completed_spans = self.completed_spans_by_trace.get(trace_id, []) + all_spans = active_spans + completed_spans + + if not all_spans: + return None + + # Calculate summary stats + db_queries = 0 + total_spans = len(all_spans) + earliest_start = None + latest_end = None + + for span in all_spans: + # Count DB queries + if span.attributes and span.attributes.get("db.system"): + db_queries += 1 + + # Calculate duration for completed spans + if span.end_time and span.start_time: + if earliest_start is None or span.start_time < earliest_start: + earliest_start = span.start_time + + if latest_end is None or span.end_time > latest_end: + latest_end = span.end_time + elif span.start_time: + # For active spans, track start time + if earliest_start is None or span.start_time < earliest_start: + earliest_start = span.start_time + + # Calculate overall duration (for the whole trace) + duration_ms = 0.0 + if earliest_start and latest_end: + duration_ms = (latest_end - earliest_start) / 1_000_000 # ns to ms + elif earliest_start: + # If trace is still active, calculate duration so far + import time + + current_time_ns = int(time.time() * 1_000_000_000) + duration_ms = (current_time_ns - earliest_start) / 1_000_000 + + # Build summary parts like the Trace model does + parts = [f"{total_spans}sp"] + + if db_queries > 0: + parts.append(f"{db_queries}db") + + if duration_ms > 0: + parts.append(f"{round(duration_ms, 1)}ms") + + return " ".join(parts) + + def shutdown(self): + """Cleanup when shutting down.""" + with self.lock: + self.active_spans_by_trace.clear() + self.completed_spans_by_trace.clear() + + def force_flush(self, timeout_millis=None): + """Required by SpanProcessor interface.""" + return True + + +def get_span_collector(): + """Get the span collector instance from the tracer provider.""" + current_provider = trace.get_tracer_provider() + if not current_provider or isinstance(current_provider, trace.ProxyTracerProvider): + return None + + # Look for RealTimeSpanCollector in the span processors + # Check if the provider has a _active_span_processor attribute + if hasattr(current_provider, "_active_span_processor"): + # It's a composite processor, check its _span_processors + composite_processor = current_provider._active_span_processor + if hasattr(composite_processor, "_span_processors"): + for processor in composite_processor._span_processors: + if isinstance(processor, RealTimeSpanCollector): + return processor + + return None + + def has_existing_trace_provider() -> bool: """Check if there is an existing trace provider.""" current_provider = trace.get_tracer_provider() @@ -27,7 +164,14 @@ def has_existing_trace_provider() -> bool: def setup_debug_trace_provider() -> None: sampler = PlainRequestSampler() provider = TracerProvider(sampler=sampler) + + # Add the real-time span collector first for immediate access + span_collector = RealTimeSpanCollector() + provider.add_span_processor(span_collector) + + # Add the database exporter provider.add_span_processor(BatchSpanProcessor(ObserveModelsExporter())) + trace.set_tracer_provider(provider) @@ -43,6 +187,7 @@ class PlainRequestSampler(sampling.Sampler): """Drops traces based on request path or user role.""" def __init__(self): + # Custom parent-based sampler that properly handles RECORD_ONLY inheritance self._delegate = sampling.ParentBased(sampling.ALWAYS_ON) # TODO ignore url namespace instead? admin, observe, assets @@ -50,6 +195,10 @@ def __init__(self): re.compile(p) for p in settings.OBSERVE_IGNORE_URL_PATTERNS ] + # Track sampling decisions by trace ID + self._trace_decisions = {} # trace_id -> Decision + self._lock = threading.Lock() + def should_sample( self, parent_context, @@ -59,7 +208,6 @@ def should_sample( attributes=None, links=None, trace_state=None, - **kwargs, ): # First, drop if the URL should be ignored. if attributes: @@ -71,31 +219,61 @@ def should_sample( attributes=attributes, ) - # Look for the "observe" cookie in the request and - # sample if it is set to "true". + # Check if we already have a decision for this trace + with self._lock: + if trace_id in self._trace_decisions: + decision = self._trace_decisions[trace_id] + return sampling.SamplingResult( + decision, + attributes=attributes, + ) + + # For new traces, check cookies in the context + decision = None if parent_context: + # Check cookies for root spans if cookies := baggage.get_baggage("http.request.cookies", parent_context): - # Using a signed cookie would be better -- only set by authed route - if cookies.get("observe") == "true": - return sampling.SamplingResult( - sampling.Decision.RECORD_AND_SAMPLE, - attributes=attributes, - ) - else: - return sampling.SamplingResult( - sampling.Decision.DROP, - attributes=attributes, + if observe_cookie := cookies.get("observe"): + unsigned_value = unsign_cookie_value( + "observe", observe_cookie, default=False ) - # Fallback to delegate sampler. - return self._delegate.should_sample( - parent_context, - trace_id, - name, - kind, - attributes, - links, - trace_state, + if unsigned_value == "sample": + decision = sampling.Decision.RECORD_AND_SAMPLE + elif unsigned_value == "record": + decision = sampling.Decision.RECORD_ONLY + else: + # Unknown value, drop the trace + decision = sampling.Decision.DROP + else: + # Cookies are there, but no observe cookie + decision = sampling.Decision.DROP + + # If no decision from cookies, use default + if decision is None: + result = self._delegate.should_sample( + parent_context, + trace_id, + name, + kind=kind, + attributes=attributes, + links=links, + trace_state=trace_state, + ) + decision = result.decision + + # Store the decision for this trace + with self._lock: + self._trace_decisions[trace_id] = decision + # Clean up old entries if too many (simple LRU) + if len(self._trace_decisions) > 1000: + # Remove oldest entries + for old_trace_id in list(self._trace_decisions.keys())[:100]: + del self._trace_decisions[old_trace_id] + + return sampling.SamplingResult( + decision, + attributes=attributes, ) def get_description(self) -> str: @@ -103,43 +281,65 @@ def get_description(self) -> str: class ObserveModelsExporter(SpanExporter): - """Exporter that writes spans into the observe models tables.""" + """Exporter that writes spans into the observe models tables. + + Note: This should only receive spans with RECORD_AND_SAMPLE sampling decision. + Spans with RECORD_ONLY should not reach this exporter. + """ def export(self, spans): """Persist spans in bulk for efficiency.""" from .models import Span, Trace - with suppress_tracing(): + with suppress_db_tracing(): create_spans = [] create_traces = {} for span in spans: - span_data = json.loads(span.to_json()) - trace_id = span_data["context"]["trace_id"] - - # There should be at least one span with this attribute - request_id = span_data["attributes"].get("plain.request.id", "") - user_id = span_data["attributes"].get("user.id", "") - session_id = span_data["attributes"].get("session.id", "") - - if not span_data["parent_id"]: - description = span_data["name"] - else: - description = "" + # Format IDs according to W3C Trace Context specification + # Trace ID: 128-bit -> 32 hex chars, Span ID: 64-bit -> 16 hex chars + try: + trace_id_hex = format(span.get_span_context().trace_id, "032x") + span_id_hex = format(span.get_span_context().span_id, "016x") + parent_id_hex = ( + format(span.parent.span_id, "016x") if span.parent else "" + ) + except (AttributeError, ValueError) as e: + logger.warning("Failed to format span IDs: %s", e) + continue + + # Extract attributes directly + attributes = dict(span.attributes) if span.attributes else {} + request_id = attributes.get("plain.request.id", "") + user_id = attributes.get("user.id", "") + session_id = attributes.get("session.id", "") + + # Set description for root spans + description = span.name if not parent_id_hex else "" + + # Convert timestamps from nanoseconds to datetime + start_time = ( + datetime.fromtimestamp(span.start_time / 1_000_000_000, tz=UTC) + if span.start_time + else None + ) + end_time = ( + datetime.fromtimestamp(span.end_time / 1_000_000_000, tz=UTC) + if span.end_time + else None + ) - if trace := create_traces.get(trace_id): + if trace := create_traces.get(trace_id_hex): if not trace.start_time: - trace.start_time = span_data["start_time"] - else: - trace.start_time = min( - trace.start_time, span_data["start_time"] - ) + trace.start_time = start_time + elif start_time: + trace.start_time = min(trace.start_time, start_time) if not trace.end_time: - trace.end_time = span_data["end_time"] - else: - trace.end_time = max(trace.end_time, span_data["end_time"]) + trace.end_time = end_time + elif end_time: + trace.end_time = max(trace.end_time, end_time) if not trace.request_id: trace.request_id = request_id @@ -154,31 +354,82 @@ def export(self, spans): trace.description = description else: trace = Trace( - trace_id=trace_id, - start_time=span_data["start_time"], - end_time=span_data["end_time"], + trace_id=trace_id_hex, + start_time=start_time, + end_time=end_time, request_id=request_id, user_id=user_id, session_id=session_id, description=description, ) - create_traces[trace_id] = trace + create_traces[trace_id_hex] = trace + + # Extract span kind directly from the span object + kind_str = span.kind.name if span.kind else "INTERNAL" + + # Convert events to JSON format + events_json = [] + if span.events: + for event in span.events: + events_json.append( + { + "name": event.name, + "timestamp": event.timestamp, + "attributes": ( + dict(event.attributes) if event.attributes else {} + ), + } + ) + + # Convert links to JSON format + links_json = [] + if span.links: + for link in span.links: + links_json.append( + { + "context": { + "trace_id": format(link.context.trace_id, "032x"), + "span_id": format(link.context.span_id, "016x"), + }, + "attributes": ( + dict(link.attributes) if link.attributes else {} + ), + } + ) create_spans.append( Span( trace=trace, - span_id=span_data["context"]["span_id"], - name=span_data["name"], - kind=span_data["kind"], - parent_id=span_data["parent_id"] or "", - start_time=span_data["start_time"], - end_time=span_data["end_time"], - status=span_data["status"], - context=span_data["context"], - attributes=span_data["attributes"], - events=span_data["events"], - links=span_data["links"], - resource=span_data["resource"], + span_id=span_id_hex, + name=span.name, + kind=kind_str, + parent_id=parent_id_hex, + start_time=start_time, + end_time=end_time, + status={ + "status_code": ( + span.status.status_code.name if span.status else "UNSET" + ), + "description": span.status.description + if span.status + else "", + }, + context={ + "trace_id": trace_id_hex, + "span_id": span_id_hex, + "trace_flags": span.get_span_context().trace_flags, + "trace_state": ( + dict(span.get_span_context().trace_state) + if span.get_span_context().trace_state + else {} + ), + }, + attributes=attributes, + events=events_json, + links=links_json, + resource=( + dict(span.resource.attributes) if span.resource else {} + ), ) ) @@ -187,18 +438,19 @@ def export(self, spans): create_traces.values() ) # , update_conflicts=True, update_fields=["start_time", "end_time", "request_id"]) Span.objects.bulk_create(create_spans) + + # Delete oldest traces if we exceed the limit + if Trace.objects.count() > settings.OBSERVE_TRACE_LIMIT: + delete_ids = Trace.objects.order_by("start_time")[ + : settings.OBSERVE_TRACE_LIMIT + ].values_list("id", flat=True) + Trace.objects.filter(id__in=delete_ids).delete() except Exception as e: logger.error( "Failed to export spans to database: %s", e, exc_info=True, ) - - # Delete oldest traces if we exceed the limit - if Trace.objects.count() > settings.OBSERVE_TRACE_LIMIT: - delete_ids = Trace.objects.order_by("start_time")[ - : settings.OBSERVE_TRACE_LIMIT - ].values_list("id", flat=True) - Trace.objects.filter(id__in=delete_ids).delete() + raise return True diff --git a/plain-observe/plain/observe/templates/admin/observe/trace_detail.html b/plain-observe/plain/observe/templates/admin/observe/trace_detail.html new file mode 100644 index 0000000000..a466d9ca78 --- /dev/null +++ b/plain-observe/plain/observe/templates/admin/observe/trace_detail.html @@ -0,0 +1,11 @@ +{% extends "admin/detail.html" %} + +{% block content %} + +{{ super() }} + +
+
+ {% include "observability/_trace_detail.html" %} +
+{% endblock %} diff --git a/plain-observe/plain/observe/templates/observability/_trace_detail.html b/plain-observe/plain/observe/templates/observability/_trace_detail.html new file mode 100644 index 0000000000..d3cb131db9 --- /dev/null +++ b/plain-observe/plain/observe/templates/observability/_trace_detail.html @@ -0,0 +1,279 @@ + +{% if trace %} +
+

Trace: {{ trace.description }}

+ {% if show_delete_button|default(true) %} + + {% endif %} +
+
+ Trace ID: {{ trace.trace_id }} + {{ trace.start_time|localtime|strftime("%b %-d, %-I:%M %p") }} +
+
+
+ Duration: {{ "%.2f"|format(trace.duration_ms()) }}ms +
+
+ Request ID: {{ trace.request_id }} +
+
+ Session: {{ trace.session_id or 'None' }} +
+
+ User: {{ trace.user_id or 'Anonymous' }} +
+
+ + +
+ {% set ns = namespace(indent=0) %} + {% for span in trace.spans.all().order_by("start_time") %} + + {% if not span.parent_id %} + {% set ns.indent = 0 %} + {% elif loop.changed(span.parent_id) %} + {% set ns.indent = ns.indent + 1 %} + {% endif %} + + + {% set span_start_offset = ((span.start_time - trace.start_time).total_seconds() * 1000) %} + {% set span_duration = span.duration_ms() %} + {% set start_percent = (span_start_offset / trace.duration_ms() * 100) if trace.duration_ms() > 0 else 0 %} + {% set width_percent = (span_duration / trace.duration_ms() * 100) if trace.duration_ms() > 0 else 0 %} + +
+
+ +
+ +
+ + + +
+ +
+
+ {{ span.start_time|localtime|strftime("%-I:%M:%S %p") }} +
+
{{ span.description() }}
+
+ + +
+
+ +
+
+ +
+ {{ "%.2f"|format(span_duration) }}ms +
+
+
+
+ + +
+ + {% if span.sql_query %} +
+

+ + + + Database Query +

+
+ 
{{ span.get_formatted_sql() }}
+
+
+ {% endif %} + + + {% if span.get_exception_stacktrace() %} +
+

+ + + + Exception Stacktrace +

+
+ 
{{ span.get_exception_stacktrace() }}
+
+
+ {% endif %} + +
+ +
+

Basic Information

+
+
+ Name: + {{ span.name }} +
+
+ Kind: + + {{ span.kind }} + +
+
+ Duration: + {{ "%.2f"|format(span.duration_ms()) }}ms +
+ {% if span.parent_id %} +
+ Parent: + {{ span.parent_id }} +
+ {% endif %} +
+
+ +
+

Timing

+
+
+ Started: + {{ span.start_time|localtime|strftime("%-I:%M:%S.%f %p") }} +
+
+ Ended: + {{ span.end_time|localtime|strftime("%-I:%M:%S.%f %p") }} +
+ {% if span.status and span.status.status_code != 'UNSET' %} +
+ Status: + + {{ span.status.status_code }} + +
+ {% endif %} +
+
+
+ + {% if span.attributes %} +
+

Attributes

+
+
+ {% for key, value in span.attributes.items() %} +
+ {{ key }}: + {{ value }} +
+ {% endfor %} +
+
+
+ {% endif %} + + {% if span.events %} +
+

Events ({{ span.events|length }})

+
+
+ {% for event in span.events %} +
+
+
{{ event.name }}
+
+ {% set formatted_time = span.format_event_timestamp(event.timestamp) %} + {% if formatted_time.__class__.__name__ == 'datetime' %} + {{ formatted_time|localtime|strftime("%-I:%M:%S.%f %p") }} + {% else %} + {{ formatted_time }} + {% endif %} +
+
+ {% if event.attributes %} +
+ {% for key, value in event.attributes.items() %} +
+ {{ key }}: + 
{{ value }}
+
+ {% endfor %} +
+ {% endif %} +
+ {% endfor %} +
+
+
+ {% endif %} + + {% if span.links %} +
+

Links ({{ span.links|length }})

+
+
+ {% for link in span.links %} +
+
{{ link.context.trace_id }}
+
{{ link.context.span_id }}
+
+ {% endfor %} +
+
+
+ {% endif %} +
+
+
+ {% else %} +
No spans...
+ {% endfor %} +
+{% else %} +
+

Select a trace from the sidebar to view details

+

Click on any trace in the list to see its spans and timeline

+
+{% endif %} + + diff --git a/plain-observe/plain/observe/templates/observability/traces.html b/plain-observe/plain/observe/templates/observability/traces.html index 1eeb91dfeb..c0af2b63af 100644 --- a/plain-observe/plain/observe/templates/observability/traces.html +++ b/plain-observe/plain/observe/templates/observability/traces.html @@ -5,165 +5,223 @@ Querystats {% tailwind_css %} + {% htmx_js %} - - - -
- -
-
- {% if traces %} -
- {{ csrf_input }} - - -
- {% endif %} - {% if observability_enabled %} -
- {{ csrf_input }} - - -
- {% else %} -
- {{ csrf_input }} - - -
- {% endif %} -
-
- - {% if traces %} -
- - -
- {% for trace in traces %} - + {% elif observer.is_enabled %} +
+
+

Observing but nothing has been recorded yet

+

Traces will appear here once requests are made

- {% endfor %} -
-
- {% elif observability_enabled %} -
Observing but nothing has been recorded yet.
- {% else %} -
-
Observability is disabled.
-
- {{ csrf_input }} - - -
+
+ + +
+
+ {% else %} +
+
+

Observability is disabled

+

Choose a mode to start monitoring

+
+
+
+ {{ csrf_input }} + + +
+
+ {{ csrf_input }} + + +
+
+
+

Record Only: Real-time performance monitoring in toolbar

+

Full Sampling: Real-time monitoring + persistent traces for viewing

+
+
+ {% endif %}
- {% endif %} - - diff --git a/plain-observe/plain/observe/templates/toolbar/observability_button.html b/plain-observe/plain/observe/templates/toolbar/observability_button.html new file mode 100644 index 0000000000..4f6f4eab7f --- /dev/null +++ b/plain-observe/plain/observe/templates/toolbar/observability_button.html @@ -0,0 +1,49 @@ +{% if observer.is_enabled %} +
+ + {% if observer.is_recording %} + {# Toggle from record to sample #} + + {% else %} + {# Toggle from sample to record #} + + {% endif %} + +
+{% else %} +
+ + + +
+{% endif %} diff --git a/plain-observe/plain/observe/views.py b/plain-observe/plain/observe/views.py index a673368dee..824b546af3 100644 --- a/plain-observe/plain/observe/views.py +++ b/plain-observe/plain/observe/views.py @@ -1,15 +1,24 @@ +from functools import cached_property + from plain.auth.views import AuthViewMixin -from plain.http import ResponseRedirect +from plain.htmx.views import HTMXViewMixin +from plain.http import Response, ResponseRedirect from plain.runtime import settings from plain.views import TemplateView +from .core import Observer from .models import Trace -class ObservabilitySpansView(AuthViewMixin, TemplateView): +class ObservabilitySpansView(AuthViewMixin, HTMXViewMixin, TemplateView): template_name = "observability/traces.html" admin_required = True + @cached_property + def observer(self): + """Get the Observer instance for this request.""" + return Observer(self.request) + def check_auth(self): # Allow the view if we're in DEBUG if settings.DEBUG: @@ -25,23 +34,70 @@ def get_response(self): def get_template_context(self): context = super().get_template_context() - context["observability_enabled"] = self.request.cookies.get("observe") == "true" + context["observer"] = self.observer context["traces"] = Trace.objects.all() + if trace_id := self.request.query_params.get("trace_id"): + context["trace"] = Trace.objects.filter(id=trace_id).first() + else: + context["trace"] = context["traces"].first() return context + def htmx_post_enable(self): + """Enable record-only mode via HTMX.""" + response = Response(self.get_template().render(self.get_template_context())) + self.observer.enable_record_mode(response) + return response + + def htmx_post_enable_sample(self): + """Enable full sampling mode via HTMX.""" + response = Response(self.get_template().render(self.get_template_context())) + self.observer.enable_sample_mode(response) + return response + + def htmx_post_disable(self): + """Disable observability via HTMX.""" + response = Response(self.get_template().render(self.get_template_context())) + self.observer.disable(response) + return response + + def htmx_delete_traces(self): + """Clear all traces via HTMX DELETE.""" + Trace.objects.all().delete() + + return Response(self.render_template()) + + def htmx_delete_trace(self): + """Delete a specific trace via HTMX DELETE.""" + trace_id = self.request.query_params.get("trace_id") + if trace_id: + try: + trace = Trace.objects.get(id=trace_id) + trace.delete() + except Trace.DoesNotExist: + pass + + # Check if there are any traces left after deletion + remaining_traces = Trace.objects.all() + if not remaining_traces.exists(): + # If no traces left, refresh the whole page to show empty state + response = Response(self.render_template()) + response.headers["HX-Refresh"] = "true" + return response + + return Response(self.render_template()) + def post(self): observe_action = self.request.data["observe_action"] response = ResponseRedirect(self.request.data.get("redirect_url", ".")) + observer = self.observer if observe_action == "enable": - response.set_cookie("observe", "true", max_age=60 * 60 * 24) - # self.request.session.setdefault(OBSERVABILITY_SESSION_KEY, {}) - elif observe_action == "clear": - Trace.objects.all().delete() - # self.request.session[OBSERVABILITY_SESSION_KEY] = {} - elif observe_action == "disable" and "observe" in self.request.cookies: - response.delete_cookie("observe") + observer.enable_record_mode(response) # Default to record mode + elif observe_action == "enable_sample": + observer.enable_sample_mode(response) + elif observe_action == "disable": + observer.disable(response) # Redirect back to the page that submitted the form return response diff --git a/plain/plain/cli/core.py b/plain/plain/cli/core.py index a7f50c52b5..0183920f3b 100644 --- a/plain/plain/cli/core.py +++ b/plain/plain/cli/core.py @@ -124,9 +124,15 @@ def main(self, *args, **kwargs): "process.pid": str(os.getpid()), # "process.command_args": sys.argv[1:], # sensitive? }, - ): - # process.exit.code - return super().main(*args, **kwargs) + ) as span: + try: + result = super().main(*args, **kwargs) + span.set_status(trace.StatusCode.OK) + return result + except Exception as e: + span.record_exception(e) + span.set_status(trace.StatusCode.ERROR, str(e)) + raise def get_command(self, ctx: Context, cmd_name: str) -> Command | None: cmd = super().get_command(ctx, cmd_name) diff --git a/plain/plain/http/cookie.py b/plain/plain/http/cookie.py index 52b2ef2779..fb5be86736 100644 --- a/plain/plain/http/cookie.py +++ b/plain/plain/http/cookie.py @@ -1,5 +1,9 @@ from http import cookies +from plain.runtime import settings +from plain.signing import BadSignature, TimestampSigner +from plain.utils.encoding import force_bytes + def parse_cookie(cookie): """ @@ -18,3 +22,43 @@ def parse_cookie(cookie): # unquote using Python's algorithm. cookiedict[key] = cookies._unquote(val) return cookiedict + + +def _cookie_key(key): + """ + Generate a key for cookie signing that matches the pattern used by + set_signed_cookie and get_signed_cookie. + """ + return b"plain.http.cookies" + force_bytes(key) + + +def get_signed_cookie_signer(key, salt=""): + """ + Create a TimestampSigner for signed cookies with the same configuration + used by both set_signed_cookie and get_signed_cookie. + """ + return TimestampSigner( + key=_cookie_key(settings.SECRET_KEY), + fallback_keys=map(_cookie_key, settings.SECRET_KEY_FALLBACKS), + salt=key + salt, + ) + + +def sign_cookie_value(key, value, salt=""): + """ + Sign a cookie value using the standard Plain cookie signing approach. + """ + signer = get_signed_cookie_signer(key, salt) + return signer.sign(value) + + +def unsign_cookie_value(key, signed_value, salt="", max_age=None, default=None): + """ + Unsign a cookie value using the standard Plain cookie signing approach. + Returns the default value if the signature is invalid or the cookie has expired. + """ + signer = get_signed_cookie_signer(key, salt) + try: + return signer.unsign(signed_value, max_age=max_age) + except BadSignature: + return default diff --git a/plain/plain/http/request.py b/plain/plain/http/request.py index e23369cdad..7040e3035b 100644 --- a/plain/plain/http/request.py +++ b/plain/plain/http/request.py @@ -13,6 +13,7 @@ RequestDataTooBig, TooManyFieldsSent, ) +from plain.http.cookie import unsign_cookie_value from plain.http.multipartparser import ( MultiPartParser, MultiPartParserError, @@ -427,6 +428,20 @@ def __iter__(self): def readlines(self): return list(self) + def get_signed_cookie(self, key, default=None, salt="", max_age=None): + """ + Retrieve a cookie value signed with the SECRET_KEY. + + Return default if the cookie doesn't exist or signature verification fails. + """ + + try: + cookie_value = self.cookies[key] + except KeyError: + return default + + return unsign_cookie_value(key, cookie_value, salt, max_age, default) + class HttpHeaders(CaseInsensitiveMapping): HTTP_PREFIX = "HTTP_" diff --git a/plain/plain/http/response.py b/plain/plain/http/response.py index bbbaee2847..c3a0ab6aed 100644 --- a/plain/plain/http/response.py +++ b/plain/plain/http/response.py @@ -12,13 +12,14 @@ from http.cookies import SimpleCookie from urllib.parse import urlparse -from plain import signals, signing +from plain import signals from plain.exceptions import DisallowedRedirect +from plain.http.cookie import sign_cookie_value from plain.json import PlainJSONEncoder from plain.runtime import settings from plain.utils import timezone from plain.utils.datastructures import CaseInsensitiveMapping -from plain.utils.encoding import force_bytes, iri_to_uri +from plain.utils.encoding import iri_to_uri from plain.utils.http import content_disposition_header, http_date from plain.utils.regex_helper import _lazy_re_compile @@ -260,16 +261,8 @@ def set_cookie( def set_signed_cookie(self, key, value, salt="", **kwargs): """Set a cookie signed with the SECRET_KEY.""" - def _cookie_key(k): - return b"plain.http.cookies" + force_bytes(k) - - signer = signing.TimestampSigner( - key=_cookie_key(settings.SECRET_KEY), - fallback_keys=map(_cookie_key, settings.SECRET_KEY_FALLBACKS), - salt=key + salt, - ) - value = signer.sign(value) - return self.set_cookie(key, value, **kwargs) + signed_value = sign_cookie_value(key, value, salt) + return self.set_cookie(key, signed_value, **kwargs) def delete_cookie(self, key, path="/", domain=None, samesite=None): # Browsers can ignore the Set-Cookie header if the cookie doesn't use diff --git a/plain/plain/internal/handlers/base.py b/plain/plain/internal/handlers/base.py index 02c708d018..376e7582df 100644 --- a/plain/plain/internal/handlers/base.py +++ b/plain/plain/internal/handlers/base.py @@ -74,6 +74,22 @@ def get_response(self, request): } span_context = baggage.set_baggage("http.request.cookies", request.cookies) + # Set sampling decision in baggage for child spans to inherit + if observe_cookie := request.cookies.get("observe"): + from plain.http.cookie import unsign_cookie_value + + unsigned_value = unsign_cookie_value( + "observe", observe_cookie, default=False + ) + if unsigned_value == "sample": + span_context = baggage.set_baggage( + "plain.sampling.decision", "RECORD_AND_SAMPLE", span_context + ) + elif unsigned_value == "record": + span_context = baggage.set_baggage( + "plain.sampling.decision", "RECORD_ONLY", span_context + ) + with tracer.start_as_current_span( f"{request.method} {request.path_info}", context=span_context, @@ -130,7 +146,8 @@ def resolve_request(self, request): span = trace.get_current_span() span.set_attribute(http_attributes.HTTP_ROUTE, resolver_match.route) # Route makes a better name - span.update_name(f"{request.method} {resolver_match.route}") + if resolver_match.route: + span.update_name(f"{request.method} {resolver_match.route}") request.resolver_match = resolver_match return resolver_match diff --git a/plain/plain/templates/core.py b/plain/plain/templates/core.py index 1aacb29240..f52c45d442 100644 --- a/plain/plain/templates/core.py +++ b/plain/plain/templates/core.py @@ -25,10 +25,20 @@ def __init__(self, filename: str) -> None: def render(self, context: dict) -> str: with tracer.start_as_current_span( - f"Template {self.filename}", + f"render {self.filename}", + kind=trace.SpanKind.INTERNAL, attributes={ + "code.function.name": "render", + "code.namespace": f"{self.__class__.__module__}.{self.__class__.__qualname__}", "template.filename": self.filename, - "template.context_keys": list(context.keys()), + "template.engine": "jinja2", }, - ): - return self._jinja_template.render(context) + ) as span: + try: + result = self._jinja_template.render(context) + span.set_status(trace.StatusCode.OK) + return result + except Exception as e: + span.record_exception(e) + span.set_status(trace.StatusCode.ERROR, str(e)) + raise diff --git a/plain/plain/templates/jinja/filters.py b/plain/plain/templates/jinja/filters.py index 311e16f71f..efdc2a6f11 100644 --- a/plain/plain/templates/jinja/filters.py +++ b/plain/plain/templates/jinja/filters.py @@ -15,6 +15,25 @@ def localtime_filter(value, timezone=None): return localtime(value, timezone) +def pluralize_filter(value, singular="", plural="s"): + """Returns plural suffix based on the value count. + + Usage: + {{ count }} item{{ count|pluralize }} + {{ count }} ox{{ count|pluralize("en") }} + {{ count }} cact{{ count|pluralize("us","i") }} + """ + try: + count = int(value) + except (ValueError, TypeError): + return singular + + if count == 1: + return singular + + return plural + + default_filters = { # The standard Python ones "strftime": datetime.datetime.strftime, @@ -27,4 +46,5 @@ def localtime_filter(value, timezone=None): "timesince": timesince, "json_script": json_script, "islice": islice, # slice for dict.items() + "pluralize": pluralize_filter, } diff --git a/plain/plain/views/base.py b/plain/plain/views/base.py index ceacaedb36..f064243883 100644 --- a/plain/plain/views/base.py +++ b/plain/plain/views/base.py @@ -41,16 +41,26 @@ def setup(self, request: HttpRequest, *url_args, **url_kwargs) -> None: def as_view(cls, *init_args, **init_kwargs): def view(request, *url_args, **url_kwargs): with tracer.start_as_current_span( - f"View {cls.__name__}", + f"{cls.__name__}", + kind=trace.SpanKind.INTERNAL, attributes={ - "view.class": cls.__name__, - "view.url_args": url_args, - # "view.url_kwargs": list(url_kwargs), # can't be dict + "code.function.name": "as_view", + "code.namespace": f"{cls.__module__}.{cls.__qualname__}", + "http.route": getattr( + getattr(request, "resolver_match", None), "route", None + ), }, - ): - v = cls(*init_args, **init_kwargs) - v.setup(request, *url_args, **url_kwargs) - return v.get_response() + ) as span: + try: + v = cls(*init_args, **init_kwargs) + v.setup(request, *url_args, **url_kwargs) + response = v.get_response() + span.set_status(trace.StatusCode.OK) + return response + except Exception as e: + span.record_exception(e) + span.set_status(trace.StatusCode.ERROR, str(e)) + raise view.view_class = cls diff --git a/plain/pyproject.toml b/plain/pyproject.toml index ff5f7cce7c..24c872aebe 100644 --- a/plain/pyproject.toml +++ b/plain/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "jinja2>=3.1.2", "click>=8.0.0", "opentelemetry-api>=1.34.1", + "opentelemetry-semantic-conventions>=0.55b1", ] requires-python = ">=3.11" diff --git a/uv.lock b/uv.lock index 73e4410de3..7dec6302ac 100644 --- a/uv.lock +++ b/uv.lock @@ -436,6 +436,7 @@ dependencies = [ { name = "click" }, { name = "jinja2" }, { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, ] [package.dev-dependencies] @@ -448,6 +449,7 @@ requires-dist = [ { name = "click", specifier = ">=8.0.0" }, { name = "jinja2", specifier = ">=3.1.2" }, { name = "opentelemetry-api", specifier = ">=1.34.1" }, + { name = "opentelemetry-semantic-conventions", specifier = ">=0.55b1" }, ] [package.metadata.requires-dev] From c51bd7825a5e62f29cc0051a502500a7dd283b0c Mon Sep 17 00:00:00 2001 From: Dave Gaeddert Date: Mon, 14 Jul 2025 13:58:04 -0500 Subject: [PATCH 05/13] Remove the makemigrations --merge option --- plain-models/plain/models/cli.py | 117 +----------------- .../plain/models/migrations/questioner.py | 14 --- 2 files changed, 4 insertions(+), 127 deletions(-) diff --git a/plain-models/plain/models/cli.py b/plain-models/plain/models/cli.py index 4b0f51e011..13895a41a0 100644 --- a/plain-models/plain/models/cli.py +++ b/plain-models/plain/models/cli.py @@ -2,7 +2,6 @@ import subprocess import sys import time -from itertools import takewhile import click @@ -22,12 +21,10 @@ from .migrations.optimizer import MigrationOptimizer from .migrations.questioner import ( InteractiveMigrationQuestioner, - MigrationQuestioner, NonInteractiveMigrationQuestioner, ) from .migrations.recorder import MigrationRecorder from .migrations.state import ModelState, ProjectState -from .migrations.utils import get_migration_name_timestamp from .migrations.writer import MigrationWriter from .registry import models_registry @@ -135,7 +132,6 @@ def list_models(package_labels, app_only): is_flag=True, help="Just show what migrations would be made; don't actually write them.", ) -@click.option("--merge", is_flag=True, help="Enable fixing of migration conflicts.") @click.option("--empty", is_flag=True, help="Create an empty migration.") @click.option( "--noinput", @@ -157,9 +153,7 @@ def list_models(package_labels, app_only): default=1, help="Verbosity level; 0=minimal output, 1=normal output, 2=verbose output, 3=very verbose output", ) -def makemigrations( - package_labels, dry_run, merge, empty, no_input, name, check, verbosity -): +def makemigrations(package_labels, dry_run, empty, no_input, name, check, verbosity): """Creates new migration(s) for packages.""" written_files = [] @@ -226,103 +220,6 @@ def write_migration_files(changes, update_previous_migration_paths=None): ) log(writer.as_string(), level=3) - def handle_merge(loader, conflicts): - """Handle merging conflicting migrations.""" - if interactive: - questioner = InteractiveMigrationQuestioner() - else: - questioner = MigrationQuestioner(defaults={"ask_merge": True}) - - for package_label, migration_names in conflicts.items(): - log(click.style(f"Merging {package_label}", fg="cyan", bold=True), level=1) - - merge_migrations = [] - for migration_name in migration_names: - migration = loader.get_migration(package_label, migration_name) - migration.ancestry = [ - mig - for mig in loader.graph.forwards_plan( - (package_label, migration_name) - ) - if mig[0] == migration.package_label - ] - merge_migrations.append(migration) - - def all_items_equal(seq): - return all(item == seq[0] for item in seq[1:]) - - merge_migrations_generations = zip(*(m.ancestry for m in merge_migrations)) - common_ancestor_count = sum( - 1 for _ in takewhile(all_items_equal, merge_migrations_generations) - ) - if not common_ancestor_count: - raise ValueError(f"Could not find common ancestor of {migration_names}") - - for migration in merge_migrations: - migration.branch = migration.ancestry[common_ancestor_count:] - migrations_ops = ( - loader.get_migration(node_package, node_name).operations - for node_package, node_name in migration.branch - ) - migration.merged_operations = sum(migrations_ops, []) - - for migration in merge_migrations: - log(click.style(f" Branch {migration.name}", fg="yellow"), level=1) - for operation in migration.merged_operations: - log(f" - {operation.describe()}", level=1) - - if questioner.ask_merge(package_label): - numbers = [ - MigrationAutodetector.parse_number(migration.name) - for migration in merge_migrations - ] - biggest_number = ( - max(x for x in numbers if x is not None) if numbers else 0 - ) - - subclass = type( - "Migration", - (Migration,), - { - "dependencies": [ - (package_label, migration.name) - for migration in merge_migrations - ], - }, - ) - - parts = [f"{biggest_number + 1:04d}"] - if migration_name: - parts.append(migration_name) - else: - parts.append("merge") - leaf_names = "_".join( - sorted(migration.name for migration in merge_migrations) - ) - if len(leaf_names) > 47: - parts.append(get_migration_name_timestamp()) - else: - parts.append(leaf_names) - - new_migration_name = "_".join(parts) - new_migration = subclass(new_migration_name, package_label) - writer = MigrationWriter(new_migration) - - if not dry_run: - with open(writer.path, "w", encoding="utf-8") as fh: - fh.write(writer.as_string()) - log(f"\nCreated new merge migration {writer.path}", level=1) - elif verbosity == 3: - log( - click.style( - f"Full merge migrations file '{writer.filename}':", - fg="cyan", - bold=True, - ), - level=3, - ) - log(writer.as_string(), level=3) - # Validate package labels package_labels = set(package_labels) has_bad_labels = False @@ -351,21 +248,16 @@ def all_items_equal(seq): if package_label in package_labels } - if conflicts and not merge: + if conflicts: name_str = "; ".join( "{} in {}".format(", ".join(names), package) for package, names in conflicts.items() ) raise click.ClickException( f"Conflicting migrations detected; multiple leaf nodes in the " - f"migration graph: ({name_str}).\nTo fix them run " - f"'python manage.py makemigrations --merge'" + f"migration graph: ({name_str})." ) - # Handle merge if requested - if merge and conflicts: - return handle_merge(loader, conflicts) - # Set up questioner if interactive: questioner = InteractiveMigrationQuestioner( @@ -533,8 +425,7 @@ def describe_operation(operation): ) raise click.ClickException( "Conflicting migrations detected; multiple leaf nodes in the " - f"migration graph: ({name_str}).\nTo fix them run " - "'python manage.py makemigrations --merge'" + f"migration graph: ({name_str})." ) # If they supplied command line arguments, work out what they mean. diff --git a/plain-models/plain/models/migrations/questioner.py b/plain-models/plain/models/migrations/questioner.py index e94f2fa4fc..af3633841d 100644 --- a/plain-models/plain/models/migrations/questioner.py +++ b/plain-models/plain/models/migrations/questioner.py @@ -74,10 +74,6 @@ def ask_rename_model(self, old_model_state, new_model_state): """Was this model really renamed?""" return self.defaults.get("ask_rename_model", False) - def ask_merge(self, package_label): - """Should these migrations really be merged?""" - return self.defaults.get("ask_merge", False) - def ask_auto_now_add_addition(self, field_name, model_name): """Adding an auto_now_add field to a model.""" # None means quit @@ -227,16 +223,6 @@ def ask_rename_model(self, old_model_state, new_model_state): default=False, ) - def ask_merge(self, package_label): - return self._boolean_input( - ( - "\nMerging will only work if the operations printed above do not conflict\n" - "with each other (working on different fields or models)\n" - "Should these migration branches be merged?" - ), - default=False, - ) - def ask_auto_now_add_addition(self, field_name, model_name): """Adding an auto_now_add field to a model.""" if not self.dry_run: From 9f7daaa89a60506e1a4fe4d80217d1cacdec8ee7 Mon Sep 17 00:00:00 2001 From: Dave Gaeddert Date: Mon, 14 Jul 2025 17:34:31 -0500 Subject: [PATCH 06/13] save --- plain-models/plain/models/test/pytest.py | 28 +- plain-models/plain/models/test/utils.py | 7 +- plain-observe/plain/observe/otel.py | 524 ++++++++++++----------- 3 files changed, 285 insertions(+), 274 deletions(-) diff --git a/plain-models/plain/models/test/pytest.py b/plain-models/plain/models/test/pytest.py index c717d73a2e..4038f45c00 100644 --- a/plain-models/plain/models/test/pytest.py +++ b/plain-models/plain/models/test/pytest.py @@ -2,6 +2,7 @@ import pytest +from plain.models.observability import suppress_db_tracing from plain.signals import request_finished, request_started from .. import transaction @@ -60,29 +61,32 @@ def setup_db(request): def db(setup_db, request): if "isolated_db" in request.fixturenames: pytest.fail("The 'db' and 'isolated_db' fixtures cannot be used together") + # Set .cursor() back to the original implementation to unblock it BaseDatabaseWrapper.cursor = BaseDatabaseWrapper._enabled_cursor if not db_connection.features.supports_transactions: pytest.fail("Database does not support transactions") - atomic = transaction.atomic() - atomic._from_testcase = True # TODO remove this somehow? - atomic.__enter__() + with suppress_db_tracing(): + atomic = transaction.atomic() + atomic._from_testcase = True # TODO remove this somehow? + atomic.__enter__() yield - if ( - db_connection.features.can_defer_constraint_checks - and not db_connection.needs_rollback - and db_connection.is_usable() - ): - db_connection.check_constraints() + with suppress_db_tracing(): + if ( + db_connection.features.can_defer_constraint_checks + and not db_connection.needs_rollback + and db_connection.is_usable() + ): + db_connection.check_constraints() - db_connection.set_rollback(True) - atomic.__exit__(None, None, None) + db_connection.set_rollback(True) + atomic.__exit__(None, None, None) - db_connection.close() + db_connection.close() @pytest.fixture diff --git a/plain-models/plain/models/test/utils.py b/plain-models/plain/models/test/utils.py index 5ea3ef3dcf..220b7a7cee 100644 --- a/plain-models/plain/models/test/utils.py +++ b/plain-models/plain/models/test/utils.py @@ -1,11 +1,14 @@ from plain.models import db_connection +from plain.models.observability import suppress_db_tracing def setup_database(*, verbosity, prefix=""): old_name = db_connection.settings_dict["NAME"] - db_connection.creation.create_test_db(verbosity=verbosity, prefix=prefix) + with suppress_db_tracing(): + db_connection.creation.create_test_db(verbosity=verbosity, prefix=prefix) return old_name def teardown_database(old_name, verbosity): - db_connection.creation.destroy_test_db(old_name, verbosity) + with suppress_db_tracing(): + db_connection.creation.destroy_test_db(old_name, verbosity) diff --git a/plain-observe/plain/observe/otel.py b/plain-observe/plain/observe/otel.py index de03b9aad7..0fdcbd9c1e 100644 --- a/plain-observe/plain/observe/otel.py +++ b/plain-observe/plain/observe/otel.py @@ -8,7 +8,10 @@ from opentelemetry import baggage, trace from opentelemetry.sdk.trace import SpanProcessor, TracerProvider, sampling -from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, + SpanExporter, +) from opentelemetry.semconv.attributes import url_attributes from opentelemetry.trace import SpanKind @@ -19,135 +22,20 @@ logger = logging.getLogger(__name__) -class RealTimeSpanCollector(SpanProcessor): - """Collects spans in real-time for current trace performance monitoring.""" - - def __init__(self): - self.active_spans_by_trace = defaultdict(dict) # trace_id -> {span_id: span} - self.completed_spans_by_trace = defaultdict(list) # trace_id -> [spans] - self.lock = threading.Lock() - - def on_start(self, span, parent_context=None): - """Called when a span starts.""" - with self.lock: - trace_id = format(span.get_span_context().trace_id, "032x") - span_id = format(span.get_span_context().span_id, "016x") - self.active_spans_by_trace[trace_id][span_id] = span - - def on_end(self, span): - """Called when a span ends.""" - with self.lock: - trace_id = format(span.get_span_context().trace_id, "032x") - span_id = format(span.get_span_context().span_id, "016x") - - # Move from active to completed - if trace_id in self.active_spans_by_trace: - span_obj = self.active_spans_by_trace[trace_id].pop(span_id, None) - if span_obj: - self.completed_spans_by_trace[trace_id].append(span_obj) - - # Clean up empty trace entries - if not self.active_spans_by_trace[trace_id]: - del self.active_spans_by_trace[trace_id] - - def get_current_trace_summary(self): - """Get performance summary for the currently active trace.""" - current_span = trace.get_current_span() - if not current_span: - # If no current span, check if we have any active traces at all - with self.lock: - if not self.active_spans_by_trace and not self.completed_spans_by_trace: - return None - - # Get the most recent trace if we can't find current span - all_trace_ids = list(self.active_spans_by_trace.keys()) + list( - self.completed_spans_by_trace.keys() - ) - if not all_trace_ids: - return None - trace_id = all_trace_ids[-1] # Use most recent - else: - # Use the current span's trace - trace_id = format(current_span.get_span_context().trace_id, "032x") - - with self.lock: - active_spans = list(self.active_spans_by_trace.get(trace_id, {}).values()) - completed_spans = self.completed_spans_by_trace.get(trace_id, []) - all_spans = active_spans + completed_spans - - if not all_spans: - return None - - # Calculate summary stats - db_queries = 0 - total_spans = len(all_spans) - earliest_start = None - latest_end = None - - for span in all_spans: - # Count DB queries - if span.attributes and span.attributes.get("db.system"): - db_queries += 1 - - # Calculate duration for completed spans - if span.end_time and span.start_time: - if earliest_start is None or span.start_time < earliest_start: - earliest_start = span.start_time - - if latest_end is None or span.end_time > latest_end: - latest_end = span.end_time - elif span.start_time: - # For active spans, track start time - if earliest_start is None or span.start_time < earliest_start: - earliest_start = span.start_time - - # Calculate overall duration (for the whole trace) - duration_ms = 0.0 - if earliest_start and latest_end: - duration_ms = (latest_end - earliest_start) / 1_000_000 # ns to ms - elif earliest_start: - # If trace is still active, calculate duration so far - import time - - current_time_ns = int(time.time() * 1_000_000_000) - duration_ms = (current_time_ns - earliest_start) / 1_000_000 - - # Build summary parts like the Trace model does - parts = [f"{total_spans}sp"] - - if db_queries > 0: - parts.append(f"{db_queries}db") - - if duration_ms > 0: - parts.append(f"{round(duration_ms, 1)}ms") - - return " ".join(parts) - - def shutdown(self): - """Cleanup when shutting down.""" - with self.lock: - self.active_spans_by_trace.clear() - self.completed_spans_by_trace.clear() - - def force_flush(self, timeout_millis=None): - """Required by SpanProcessor interface.""" - return True - - def get_span_collector(): """Get the span collector instance from the tracer provider.""" current_provider = trace.get_tracer_provider() if not current_provider or isinstance(current_provider, trace.ProxyTracerProvider): return None - # Look for RealTimeSpanCollector in the span processors + # Look for ObserveSpanProcessor in the span processors # Check if the provider has a _active_span_processor attribute if hasattr(current_provider, "_active_span_processor"): # It's a composite processor, check its _span_processors composite_processor = current_provider._active_span_processor if hasattr(composite_processor, "_span_processors"): for processor in composite_processor._span_processors: - if isinstance(processor, RealTimeSpanCollector): + if isinstance(processor, ObserveSpanProcessor): return processor return None @@ -165,12 +53,12 @@ def setup_debug_trace_provider() -> None: sampler = PlainRequestSampler() provider = TracerProvider(sampler=sampler) - # Add the real-time span collector first for immediate access - span_collector = RealTimeSpanCollector() + # Add the real-time span collector for immediate access + span_collector = ObserveSpanProcessor() provider.add_span_processor(span_collector) - # Add the database exporter - provider.add_span_processor(BatchSpanProcessor(ObserveModelsExporter())) + # Add the database exporter using SimpleSpanProcessor for immediate export + provider.add_span_processor(SimpleSpanProcessor(ObserveModelsExporter())) trace.set_tracer_provider(provider) @@ -242,11 +130,8 @@ def should_sample( decision = sampling.Decision.RECORD_AND_SAMPLE elif unsigned_value == "record": decision = sampling.Decision.RECORD_ONLY - else: - # Unknown value, drop the trace - decision = sampling.Decision.DROP - else: - # Cookies are there, but no observe cookie + + if decision is None: decision = sampling.Decision.DROP # If no decision from cookies, use default @@ -280,6 +165,121 @@ def get_description(self) -> str: return "PlainRequestSampler" +class ObserveSpanProcessor(SpanProcessor): + """Collects spans in real-time for current trace performance monitoring.""" + + def __init__(self): + self.active_spans_by_trace = defaultdict(dict) # trace_id -> {span_id: span} + self.completed_spans_by_trace = defaultdict(list) # trace_id -> [spans] + self.lock = threading.Lock() + + def on_start(self, span, parent_context=None): + """Called when a span starts.""" + with self.lock: + trace_id = format(span.get_span_context().trace_id, "032x") + span_id = format(span.get_span_context().span_id, "016x") + self.active_spans_by_trace[trace_id][span_id] = span + + def on_end(self, span): + """Called when a span ends.""" + with self.lock: + trace_id = format(span.get_span_context().trace_id, "032x") + span_id = format(span.get_span_context().span_id, "016x") + + # Move from active to completed + if trace_id in self.active_spans_by_trace: + span_obj = self.active_spans_by_trace[trace_id].pop(span_id, None) + if span_obj: + self.completed_spans_by_trace[trace_id].append(span_obj) + + # Clean up empty trace entries + if not self.active_spans_by_trace[trace_id]: + del self.active_spans_by_trace[trace_id] + + def get_current_trace_summary(self): + """Get performance summary for the currently active trace.""" + current_span = trace.get_current_span() + if not current_span: + # If no current span, check if we have any active traces at all + with self.lock: + if not self.active_spans_by_trace and not self.completed_spans_by_trace: + return None + + # Get the most recent trace if we can't find current span + all_trace_ids = list(self.active_spans_by_trace.keys()) + list( + self.completed_spans_by_trace.keys() + ) + if not all_trace_ids: + return None + trace_id = all_trace_ids[-1] # Use most recent + else: + # Use the current span's trace + trace_id = format(current_span.get_span_context().trace_id, "032x") + + with self.lock: + active_spans = list(self.active_spans_by_trace.get(trace_id, {}).values()) + completed_spans = self.completed_spans_by_trace.get(trace_id, []) + all_spans = active_spans + completed_spans + + if not all_spans: + return None + + # Calculate summary stats + db_queries = 0 + total_spans = len(all_spans) + earliest_start = None + latest_end = None + + for span in all_spans: + # Count DB queries + if span.attributes and span.attributes.get("db.system"): + db_queries += 1 + + # Calculate duration for completed spans + if span.end_time and span.start_time: + if earliest_start is None or span.start_time < earliest_start: + earliest_start = span.start_time + + if latest_end is None or span.end_time > latest_end: + latest_end = span.end_time + elif span.start_time: + # For active spans, track start time + if earliest_start is None or span.start_time < earliest_start: + earliest_start = span.start_time + + # Calculate overall duration (for the whole trace) + duration_ms = 0.0 + if earliest_start and latest_end: + duration_ms = (latest_end - earliest_start) / 1_000_000 # ns to ms + elif earliest_start: + # If trace is still active, calculate duration so far + import time + + current_time_ns = int(time.time() * 1_000_000_000) + duration_ms = (current_time_ns - earliest_start) / 1_000_000 + + # Build summary parts like the Trace model does + parts = [f"{total_spans}sp"] + + if db_queries > 0: + parts.append(f"{db_queries}db") + + if duration_ms > 0: + parts.append(f"{round(duration_ms, 1)}ms") + + return " ".join(parts) + + def shutdown(self): + """Cleanup when shutting down.""" + with self.lock: + self.active_spans_by_trace.clear() + self.completed_spans_by_trace.clear() + + def force_flush(self, timeout_millis=None): + """Required by SpanProcessor interface.""" + return True + + class ObserveModelsExporter(SpanExporter): """Exporter that writes spans into the observe models tables. @@ -288,169 +288,173 @@ class ObserveModelsExporter(SpanExporter): """ def export(self, spans): - """Persist spans in bulk for efficiency.""" + """Persist each span individually for immediate export.""" from .models import Span, Trace with suppress_db_tracing(): - create_spans = [] - create_traces = {} - for span in spans: - # Format IDs according to W3C Trace Context specification - # Trace ID: 128-bit -> 32 hex chars, Span ID: 64-bit -> 16 hex chars try: + # Format IDs according to W3C Trace Context specification trace_id_hex = format(span.get_span_context().trace_id, "032x") span_id_hex = format(span.get_span_context().span_id, "016x") parent_id_hex = ( format(span.parent.span_id, "016x") if span.parent else "" ) - except (AttributeError, ValueError) as e: - logger.warning("Failed to format span IDs: %s", e) - continue - - # Extract attributes directly - attributes = dict(span.attributes) if span.attributes else {} - request_id = attributes.get("plain.request.id", "") - user_id = attributes.get("user.id", "") - session_id = attributes.get("session.id", "") - - # Set description for root spans - description = span.name if not parent_id_hex else "" - - # Convert timestamps from nanoseconds to datetime - start_time = ( - datetime.fromtimestamp(span.start_time / 1_000_000_000, tz=UTC) - if span.start_time - else None - ) - end_time = ( - datetime.fromtimestamp(span.end_time / 1_000_000_000, tz=UTC) - if span.end_time - else None - ) - if trace := create_traces.get(trace_id_hex): - if not trace.start_time: - trace.start_time = start_time - elif start_time: - trace.start_time = min(trace.start_time, start_time) + # Extract attributes directly + attributes = dict(span.attributes) if span.attributes else {} + request_id = attributes.get("plain.request.id", "") + user_id = attributes.get("user.id", "") + session_id = attributes.get("session.id", "") - if not trace.end_time: - trace.end_time = end_time - elif end_time: - trace.end_time = max(trace.end_time, end_time) + # Set description for root spans + description = span.name if not parent_id_hex else "" - if not trace.request_id: - trace.request_id = request_id - - if not trace.user_id: - trace.user_id = user_id - - if not trace.session_id: - trace.session_id = session_id + # Convert timestamps from nanoseconds to datetime + start_time = ( + datetime.fromtimestamp(span.start_time / 1_000_000_000, tz=UTC) + if span.start_time + else None + ) + end_time = ( + datetime.fromtimestamp(span.end_time / 1_000_000_000, tz=UTC) + if span.end_time + else None + ) - if not trace.description: - trace.description = description - else: - trace = Trace( + # Get or create the trace + trace, created = Trace.objects.get_or_create( trace_id=trace_id_hex, - start_time=start_time, - end_time=end_time, - request_id=request_id, - user_id=user_id, - session_id=session_id, - description=description, + defaults={ + "start_time": start_time, + "end_time": end_time, + "request_id": request_id, + "user_id": user_id, + "session_id": session_id, + "description": description, + }, ) - create_traces[trace_id_hex] = trace - - # Extract span kind directly from the span object - kind_str = span.kind.name if span.kind else "INTERNAL" - - # Convert events to JSON format - events_json = [] - if span.events: - for event in span.events: - events_json.append( - { - "name": event.name, - "timestamp": event.timestamp, - "attributes": ( - dict(event.attributes) if event.attributes else {} - ), - } - ) - - # Convert links to JSON format - links_json = [] - if span.links: - for link in span.links: - links_json.append( - { - "context": { - "trace_id": format(link.context.trace_id, "032x"), - "span_id": format(link.context.span_id, "016x"), - }, - "attributes": ( - dict(link.attributes) if link.attributes else {} - ), - } - ) - create_spans.append( - Span( + # Update trace if we have better data + if not created: + updated = False + if start_time and ( + not trace.start_time or start_time < trace.start_time + ): + trace.start_time = start_time + updated = True + if end_time and ( + not trace.end_time or end_time > trace.end_time + ): + trace.end_time = end_time + updated = True + if request_id and not trace.request_id: + trace.request_id = request_id + updated = True + if user_id and not trace.user_id: + trace.user_id = user_id + updated = True + if session_id and not trace.session_id: + trace.session_id = session_id + updated = True + if description and not trace.description: + trace.description = description + updated = True + if updated: + trace.save() + + # Extract span kind directly from the span object + kind_str = span.kind.name if span.kind else "INTERNAL" + + # Convert events to JSON format + events_json = [] + if span.events: + for event in span.events: + events_json.append( + { + "name": event.name, + "timestamp": event.timestamp, + "attributes": ( + dict(event.attributes) + if event.attributes + else {} + ), + } + ) + + # Convert links to JSON format + links_json = [] + if span.links: + for link in span.links: + links_json.append( + { + "context": { + "trace_id": format( + link.context.trace_id, "032x" + ), + "span_id": format(link.context.span_id, "016x"), + }, + "attributes": ( + dict(link.attributes) if link.attributes else {} + ), + } + ) + + # Create the span + Span.objects.get_or_create( trace=trace, span_id=span_id_hex, - name=span.name, - kind=kind_str, - parent_id=parent_id_hex, - start_time=start_time, - end_time=end_time, - status={ - "status_code": ( - span.status.status_code.name if span.status else "UNSET" - ), - "description": span.status.description - if span.status - else "", - }, - context={ - "trace_id": trace_id_hex, - "span_id": span_id_hex, - "trace_flags": span.get_span_context().trace_flags, - "trace_state": ( - dict(span.get_span_context().trace_state) - if span.get_span_context().trace_state - else {} + defaults={ + "name": span.name, + "kind": kind_str, + "parent_id": parent_id_hex, + "start_time": start_time, + "end_time": end_time, + "status": { + "status_code": ( + span.status.status_code.name + if span.status + else "UNSET" + ), + "description": span.status.description + if span.status + else "", + }, + "context": { + "trace_id": trace_id_hex, + "span_id": span_id_hex, + "trace_flags": span.get_span_context().trace_flags, + "trace_state": ( + dict(span.get_span_context().trace_state) + if span.get_span_context().trace_state + else {} + ), + }, + "attributes": attributes, + "events": events_json, + "links": links_json, + "resource": ( + dict(span.resource.attributes) if span.resource else {} ), }, - attributes=attributes, - events=events_json, - links=links_json, - resource=( - dict(span.resource.attributes) if span.resource else {} - ), ) - ) - try: - Trace.objects.bulk_create( - create_traces.values() - ) # , update_conflicts=True, update_fields=["start_time", "end_time", "request_id"]) - Span.objects.bulk_create(create_spans) + except Exception as e: + logger.warning( + "Failed to export span to database: %s", + e, + exc_info=True, + ) - # Delete oldest traces if we exceed the limit + # Delete oldest traces if we exceed the limit + try: if Trace.objects.count() > settings.OBSERVE_TRACE_LIMIT: delete_ids = Trace.objects.order_by("start_time")[ : settings.OBSERVE_TRACE_LIMIT ].values_list("id", flat=True) Trace.objects.filter(id__in=delete_ids).delete() except Exception as e: - logger.error( - "Failed to export spans to database: %s", - e, - exc_info=True, - ) - raise + logger.warning("Failed to clean up old traces: %s", e) return True From 132a864ae5276d68a5ffb56cb2784e9e69497dd7 Mon Sep 17 00:00:00 2001 From: Dave Gaeddert Date: Mon, 14 Jul 2025 21:27:20 -0500 Subject: [PATCH 07/13] save --- demos/full/app/settings.py | 2 +- demos/full/pyproject.toml | 2 +- plain-observe/README.md | 1 - plain-observe/plain/observe/CHANGELOG.md | 1 - plain-observe/plain/observe/README.md | 3 - plain-observe/plain/observe/cli.py | 23 - plain-observe/plain/observe/config.py | 14 - .../plain/observe/default_settings.py | 8 - .../plain/observe/migrations/0001_initial.py | 57 --- ...id_span_parent_id_span_context_and_more.py | 70 --- ...n_attributes_alter_span_events_and_more.py | 36 -- ...quest_id_trace_session_id_trace_user_id.py | 28 -- ...05_alter_trace_options_trace_created_at.py | 26 - ...006_alter_span_options_trace_start_time.py | 22 - .../0007_remove_trace_created_at.py | 16 - ...0008_alter_trace_options_trace_end_time.py | 22 - .../migrations/0009_trace_description.py | 18 - plain-observe/plain/observe/otel.py | 460 ------------------ plain-observe/plain/observe/urls.py | 10 - {plain-observe => plain-observer}/LICENSE | 0 plain-observer/README.md | 1 + plain-observer/plain/observer/CHANGELOG.md | 1 + plain-observer/plain/observer/README.md | 3 + .../plain/observer}/__init__.py | 0 .../plain/observer}/admin.py | 14 +- plain-observer/plain/observer/cli.py | 23 + plain-observer/plain/observer/config.py | 42 ++ .../plain/observer}/core.py | 20 +- .../plain/observer/default_settings.py | 8 + plain-observer/plain/observer/exporter.py | 195 ++++++++ .../plain/observer/migrations/0001_initial.py | 100 ++++ .../plain/observer}/migrations/__init__.py | 0 .../plain/observer}/models.py | 4 +- plain-observer/plain/observer/processor.py | 139 ++++++ plain-observer/plain/observer/sampler.py | 104 ++++ .../admin/observer}/trace_detail.html | 2 +- .../templates/observer}/_trace_detail.html | 4 +- .../observer/templates/observer}/traces.html | 10 +- .../observer/templates/toolbar/observer.html | 6 +- .../templates/toolbar/observer_button.html | 8 +- plain-observer/plain/observer/urls.py | 10 + .../plain/observer}/views.py | 4 +- .../pyproject.toml | 2 +- .../tests/app/settings.py | 0 .../tests/app/urls.py | 0 .../app/users/migrations/0001_initial.py | 0 .../tests/app/users/migrations/__init__.py | 0 .../tests/app/users/models.py | 0 .../tests/test_admin.py | 0 plain/plain/internal/handlers/base.py | 17 +- pyproject.toml | 2 +- uv.lock | 10 +- 52 files changed, 676 insertions(+), 872 deletions(-) delete mode 120000 plain-observe/README.md delete mode 100644 plain-observe/plain/observe/CHANGELOG.md delete mode 100644 plain-observe/plain/observe/README.md delete mode 100644 plain-observe/plain/observe/cli.py delete mode 100644 plain-observe/plain/observe/config.py delete mode 100644 plain-observe/plain/observe/default_settings.py delete mode 100644 plain-observe/plain/observe/migrations/0001_initial.py delete mode 100644 plain-observe/plain/observe/migrations/0002_rename_parent_span_id_span_parent_id_span_context_and_more.py delete mode 100644 plain-observe/plain/observe/migrations/0003_alter_span_attributes_alter_span_events_and_more.py delete mode 100644 plain-observe/plain/observe/migrations/0004_trace_request_id_trace_session_id_trace_user_id.py delete mode 100644 plain-observe/plain/observe/migrations/0005_alter_trace_options_trace_created_at.py delete mode 100644 plain-observe/plain/observe/migrations/0006_alter_span_options_trace_start_time.py delete mode 100644 plain-observe/plain/observe/migrations/0007_remove_trace_created_at.py delete mode 100644 plain-observe/plain/observe/migrations/0008_alter_trace_options_trace_end_time.py delete mode 100644 plain-observe/plain/observe/migrations/0009_trace_description.py delete mode 100644 plain-observe/plain/observe/otel.py delete mode 100644 plain-observe/plain/observe/urls.py rename {plain-observe => plain-observer}/LICENSE (100%) create mode 120000 plain-observer/README.md create mode 100644 plain-observer/plain/observer/CHANGELOG.md create mode 100644 plain-observer/plain/observer/README.md rename {plain-observe/plain/observe => plain-observer/plain/observer}/__init__.py (100%) rename {plain-observe/plain/observe => plain-observer/plain/observer}/admin.py (89%) create mode 100644 plain-observer/plain/observer/cli.py create mode 100644 plain-observer/plain/observer/config.py rename {plain-observe/plain/observe => plain-observer/plain/observer}/core.py (71%) create mode 100644 plain-observer/plain/observer/default_settings.py create mode 100644 plain-observer/plain/observer/exporter.py create mode 100644 plain-observer/plain/observer/migrations/0001_initial.py rename {plain-observe/plain/observe => plain-observer/plain/observer}/migrations/__init__.py (100%) rename {plain-observe/plain/observe => plain-observer/plain/observer}/models.py (98%) create mode 100644 plain-observer/plain/observer/processor.py create mode 100644 plain-observer/plain/observer/sampler.py rename {plain-observe/plain/observe/templates/admin/observe => plain-observer/plain/observer/templates/admin/observer}/trace_detail.html (68%) rename {plain-observe/plain/observe/templates/observability => plain-observer/plain/observer/templates/observer}/_trace_detail.html (99%) rename {plain-observe/plain/observe/templates/observability => plain-observer/plain/observer/templates/observer}/traces.html (98%) rename plain-observe/plain/observe/templates/toolbar/observability.html => plain-observer/plain/observer/templates/toolbar/observer.html (82%) rename plain-observe/plain/observe/templates/toolbar/observability_button.html => plain-observer/plain/observer/templates/toolbar/observer_button.html (91%) create mode 100644 plain-observer/plain/observer/urls.py rename {plain-observe/plain/observe => plain-observer/plain/observer}/views.py (96%) rename {plain-observe => plain-observer}/pyproject.toml (96%) rename {plain-observe => plain-observer}/tests/app/settings.py (100%) rename {plain-observe => plain-observer}/tests/app/urls.py (100%) rename {plain-observe => plain-observer}/tests/app/users/migrations/0001_initial.py (100%) rename {plain-observe => plain-observer}/tests/app/users/migrations/__init__.py (100%) rename {plain-observe => plain-observer}/tests/app/users/models.py (100%) rename {plain-observe => plain-observer}/tests/test_admin.py (100%) diff --git a/demos/full/app/settings.py b/demos/full/app/settings.py index 520109ca9a..5747eb5ec4 100644 --- a/demos/full/app/settings.py +++ b/demos/full/app/settings.py @@ -21,7 +21,7 @@ "plain.tailwind", "plain.worker", "plain.redirection", - "plain.observe", + "plain.observer", "app.users", ] diff --git a/demos/full/pyproject.toml b/demos/full/pyproject.toml index 8c1475719c..6b0aa9962d 100644 --- a/demos/full/pyproject.toml +++ b/demos/full/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "plain-tunnel", "plain-vendor", "plain-worker", - "plain-observe", + "plain-observer", ] [tool.plain.tailwind] diff --git a/plain-observe/README.md b/plain-observe/README.md deleted file mode 120000 index f0a9e39854..0000000000 --- a/plain-observe/README.md +++ /dev/null @@ -1 +0,0 @@ -plain/observe/README.md \ No newline at end of file diff --git a/plain-observe/plain/observe/CHANGELOG.md b/plain-observe/plain/observe/CHANGELOG.md deleted file mode 100644 index 99f25eb1c1..0000000000 --- a/plain-observe/plain/observe/CHANGELOG.md +++ /dev/null @@ -1 +0,0 @@ -# plain-admin changelog diff --git a/plain-observe/plain/observe/README.md b/plain-observe/plain/observe/README.md deleted file mode 100644 index 0706b46557..0000000000 --- a/plain-observe/plain/observe/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# plain.observe - -**Monitor.** diff --git a/plain-observe/plain/observe/cli.py b/plain-observe/plain/observe/cli.py deleted file mode 100644 index aa50f2178b..0000000000 --- a/plain-observe/plain/observe/cli.py +++ /dev/null @@ -1,23 +0,0 @@ -import click - -from plain.cli import register_cli -from plain.observe.models import Trace - - -@register_cli("observe") -@click.group("observe") -def observe_cli(): - pass - - -@observe_cli.command() -@click.option("--force", is_flag=True, help="Force clear all observability data.") -def clear(force: bool): - """Clear all observability data.""" - if not force: - click.confirm( - "Are you sure you want to clear all observability data? This cannot be undone.", - abort=True, - ) - - print("Deleted", Trace.objects.all().delete()) diff --git a/plain-observe/plain/observe/config.py b/plain-observe/plain/observe/config.py deleted file mode 100644 index 3799eea89e..0000000000 --- a/plain-observe/plain/observe/config.py +++ /dev/null @@ -1,14 +0,0 @@ -from plain.packages import PackageConfig, register_config - -from .otel import has_existing_trace_provider, setup_debug_trace_provider - - -@register_config -class Config(PackageConfig): - package_label = "plainobserve" - - def ready(self): - if has_existing_trace_provider(): - return - - setup_debug_trace_provider() diff --git a/plain-observe/plain/observe/default_settings.py b/plain-observe/plain/observe/default_settings.py deleted file mode 100644 index 804ff17647..0000000000 --- a/plain-observe/plain/observe/default_settings.py +++ /dev/null @@ -1,8 +0,0 @@ -OBSERVE_IGNORE_URL_PATTERNS: list[str] = [ - "/assets/.*", - "/admin/.*", - "/observe/.*", - "/favicon.ico", - "/.well-known/.*", -] -OBSERVE_TRACE_LIMIT: int = 100 diff --git a/plain-observe/plain/observe/migrations/0001_initial.py b/plain-observe/plain/observe/migrations/0001_initial.py deleted file mode 100644 index c24be4e27f..0000000000 --- a/plain-observe/plain/observe/migrations/0001_initial.py +++ /dev/null @@ -1,57 +0,0 @@ -# Generated by Plain 0.49.0 on 2025-06-20 16:56 - -import plain.models.deletion -from plain import models -from plain.models import migrations - - -class Migration(migrations.Migration): - initial = True - - dependencies = [] - - operations = [ - migrations.CreateModel( - name="Span", - fields=[ - ("id", models.BigAutoField(auto_created=True, primary_key=True)), - ("span_id", models.CharField(max_length=255)), - ( - "parent_span_id", - models.CharField(default="", max_length=255, required=False), - ), - ("name", models.CharField(max_length=255)), - ("start_time", models.DateTimeField()), - ("end_time", models.DateTimeField(allow_null=True, required=False)), - ("attributes", models.JSONField(default=dict)), - ], - ), - migrations.CreateModel( - name="Trace", - fields=[ - ("id", models.BigAutoField(auto_created=True, primary_key=True)), - ("trace_id", models.CharField(max_length=255)), - ], - ), - migrations.AddConstraint( - model_name="trace", - constraint=models.UniqueConstraint( - fields=("trace_id",), name="observe_unique_trace_id" - ), - ), - migrations.AddField( - model_name="span", - name="trace", - field=models.ForeignKey( - on_delete=plain.models.deletion.CASCADE, - related_name="spans", - to="plainobserve.trace", - ), - ), - migrations.AddConstraint( - model_name="span", - constraint=models.UniqueConstraint( - fields=("trace", "span_id"), name="observe_unique_span_id" - ), - ), - ] diff --git a/plain-observe/plain/observe/migrations/0002_rename_parent_span_id_span_parent_id_span_context_and_more.py b/plain-observe/plain/observe/migrations/0002_rename_parent_span_id_span_parent_id_span_context_and_more.py deleted file mode 100644 index c15e6445da..0000000000 --- a/plain-observe/plain/observe/migrations/0002_rename_parent_span_id_span_parent_id_span_context_and_more.py +++ /dev/null @@ -1,70 +0,0 @@ -# Generated by Plain 0.49.0 on 2025-06-20 17:08 - -from plain import models -from plain.models import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("plainobserve", "0001_initial"), - ] - - operations = [ - migrations.RenameField( - model_name="span", - old_name="parent_span_id", - new_name="parent_id", - ), - migrations.AddField( - model_name="span", - name="context", - field=models.JSONField(default=dict), - ), - migrations.AddField( - model_name="span", - name="events", - field=models.JSONField(default=list), - ), - migrations.AddField( - model_name="span", - name="kind", - field=models.CharField(default="", max_length=50), - preserve_default=False, - ), - migrations.AddField( - model_name="span", - name="links", - field=models.JSONField(default=list), - ), - migrations.AddField( - model_name="span", - name="resource", - field=models.JSONField(default=dict), - ), - migrations.AddField( - model_name="span", - name="status", - field=models.JSONField(default=dict), - ), - migrations.AlterField( - model_name="span", - name="start_time", - field=models.DateTimeField(allow_null=True, required=False), - ), - migrations.AddIndex( - model_name="span", - index=models.Index( - fields=["trace", "span_id"], name="plainobserv_trace_i_da191d_idx" - ), - ), - migrations.AddIndex( - model_name="span", - index=models.Index(fields=["trace"], name="plainobserv_trace_i_602183_idx"), - ), - migrations.AddIndex( - model_name="span", - index=models.Index( - fields=["start_time"], name="plainobserv_start_t_3c8738_idx" - ), - ), - ] diff --git a/plain-observe/plain/observe/migrations/0003_alter_span_attributes_alter_span_events_and_more.py b/plain-observe/plain/observe/migrations/0003_alter_span_attributes_alter_span_events_and_more.py deleted file mode 100644 index 0b50bfb020..0000000000 --- a/plain-observe/plain/observe/migrations/0003_alter_span_attributes_alter_span_events_and_more.py +++ /dev/null @@ -1,36 +0,0 @@ -# Generated by Plain 0.49.0 on 2025-06-20 17:11 - -from plain import models -from plain.models import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ( - "plainobserve", - "0002_rename_parent_span_id_span_parent_id_span_context_and_more", - ), - ] - - operations = [ - migrations.AlterField( - model_name="span", - name="attributes", - field=models.JSONField(default=dict, required=False), - ), - migrations.AlterField( - model_name="span", - name="events", - field=models.JSONField(default=list, required=False), - ), - migrations.AlterField( - model_name="span", - name="links", - field=models.JSONField(default=list, required=False), - ), - migrations.AlterField( - model_name="span", - name="resource", - field=models.JSONField(default=dict, required=False), - ), - ] diff --git a/plain-observe/plain/observe/migrations/0004_trace_request_id_trace_session_id_trace_user_id.py b/plain-observe/plain/observe/migrations/0004_trace_request_id_trace_session_id_trace_user_id.py deleted file mode 100644 index bfee329ce7..0000000000 --- a/plain-observe/plain/observe/migrations/0004_trace_request_id_trace_session_id_trace_user_id.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by Plain 0.49.0 on 2025-06-20 19:49 - -from plain import models -from plain.models import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("plainobserve", "0003_alter_span_attributes_alter_span_events_and_more"), - ] - - operations = [ - migrations.AddField( - model_name="trace", - name="request_id", - field=models.CharField(default="", max_length=255, required=False), - ), - migrations.AddField( - model_name="trace", - name="session_id", - field=models.CharField(default="", max_length=255, required=False), - ), - migrations.AddField( - model_name="trace", - name="user_id", - field=models.CharField(default="", max_length=255, required=False), - ), - ] diff --git a/plain-observe/plain/observe/migrations/0005_alter_trace_options_trace_created_at.py b/plain-observe/plain/observe/migrations/0005_alter_trace_options_trace_created_at.py deleted file mode 100644 index 4e1d9cae36..0000000000 --- a/plain-observe/plain/observe/migrations/0005_alter_trace_options_trace_created_at.py +++ /dev/null @@ -1,26 +0,0 @@ -# Generated by Plain 0.49.0 on 2025-06-20 20:01 - -import plain.utils.timezone -from plain import models -from plain.models import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("plainobserve", "0004_trace_request_id_trace_session_id_trace_user_id"), - ] - - operations = [ - migrations.AlterModelOptions( - name="trace", - options={"ordering": ["-created_at"]}, - ), - migrations.AddField( - model_name="trace", - name="created_at", - field=models.DateTimeField( - auto_now_add=True, default=plain.utils.timezone.now - ), - preserve_default=False, - ), - ] diff --git a/plain-observe/plain/observe/migrations/0006_alter_span_options_trace_start_time.py b/plain-observe/plain/observe/migrations/0006_alter_span_options_trace_start_time.py deleted file mode 100644 index d79c599dda..0000000000 --- a/plain-observe/plain/observe/migrations/0006_alter_span_options_trace_start_time.py +++ /dev/null @@ -1,22 +0,0 @@ -# Generated by Plain 0.49.0 on 2025-06-20 20:13 - -from plain import models -from plain.models import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("plainobserve", "0005_alter_trace_options_trace_created_at"), - ] - - operations = [ - migrations.AlterModelOptions( - name="span", - options={"ordering": ["-start_time"]}, - ), - migrations.AddField( - model_name="trace", - name="start_time", - field=models.DateTimeField(allow_null=True, required=False), - ), - ] diff --git a/plain-observe/plain/observe/migrations/0007_remove_trace_created_at.py b/plain-observe/plain/observe/migrations/0007_remove_trace_created_at.py deleted file mode 100644 index 4ce9b46b9e..0000000000 --- a/plain-observe/plain/observe/migrations/0007_remove_trace_created_at.py +++ /dev/null @@ -1,16 +0,0 @@ -# Generated by Plain 0.49.0 on 2025-06-20 20:14 - -from plain.models import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("plainobserve", "0006_alter_span_options_trace_start_time"), - ] - - operations = [ - migrations.RemoveField( - model_name="trace", - name="created_at", - ), - ] diff --git a/plain-observe/plain/observe/migrations/0008_alter_trace_options_trace_end_time.py b/plain-observe/plain/observe/migrations/0008_alter_trace_options_trace_end_time.py deleted file mode 100644 index 15889523bd..0000000000 --- a/plain-observe/plain/observe/migrations/0008_alter_trace_options_trace_end_time.py +++ /dev/null @@ -1,22 +0,0 @@ -# Generated by Plain 0.49.0 on 2025-06-20 20:15 - -from plain import models -from plain.models import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("plainobserve", "0007_remove_trace_created_at"), - ] - - operations = [ - migrations.AlterModelOptions( - name="trace", - options={"ordering": ["-start_time"]}, - ), - migrations.AddField( - model_name="trace", - name="end_time", - field=models.DateTimeField(allow_null=True, required=False), - ), - ] diff --git a/plain-observe/plain/observe/migrations/0009_trace_description.py b/plain-observe/plain/observe/migrations/0009_trace_description.py deleted file mode 100644 index 3b264182d5..0000000000 --- a/plain-observe/plain/observe/migrations/0009_trace_description.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Plain 0.52.2 on 2025-07-12 21:37 - -from plain import models -from plain.models import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("plainobserve", "0008_alter_trace_options_trace_end_time"), - ] - - operations = [ - migrations.AddField( - model_name="trace", - name="description", - field=models.TextField(default="", required=False), - ), - ] diff --git a/plain-observe/plain/observe/otel.py b/plain-observe/plain/observe/otel.py deleted file mode 100644 index 0fdcbd9c1e..0000000000 --- a/plain-observe/plain/observe/otel.py +++ /dev/null @@ -1,460 +0,0 @@ -from __future__ import annotations - -import logging -import re -import threading -from collections import defaultdict -from datetime import UTC, datetime - -from opentelemetry import baggage, trace -from opentelemetry.sdk.trace import SpanProcessor, TracerProvider, sampling -from opentelemetry.sdk.trace.export import ( - SimpleSpanProcessor, - SpanExporter, -) -from opentelemetry.semconv.attributes import url_attributes -from opentelemetry.trace import SpanKind - -from plain.http.cookie import unsign_cookie_value -from plain.models.observability import suppress_db_tracing -from plain.runtime import settings - -logger = logging.getLogger(__name__) - - -def get_span_collector(): - """Get the span collector instance from the tracer provider.""" - current_provider = trace.get_tracer_provider() - if not current_provider or isinstance(current_provider, trace.ProxyTracerProvider): - return None - - # Look for ObserveSpanProcessor in the span processors - # Check if the provider has a _active_span_processor attribute - if hasattr(current_provider, "_active_span_processor"): - # It's a composite processor, check its _span_processors - composite_processor = current_provider._active_span_processor - if hasattr(composite_processor, "_span_processors"): - for processor in composite_processor._span_processors: - if isinstance(processor, ObserveSpanProcessor): - return processor - - return None - - -def has_existing_trace_provider() -> bool: - """Check if there is an existing trace provider.""" - current_provider = trace.get_tracer_provider() - return current_provider and not isinstance( - current_provider, trace.ProxyTracerProvider - ) - - -def setup_debug_trace_provider() -> None: - sampler = PlainRequestSampler() - provider = TracerProvider(sampler=sampler) - - # Add the real-time span collector for immediate access - span_collector = ObserveSpanProcessor() - provider.add_span_processor(span_collector) - - # Add the database exporter using SimpleSpanProcessor for immediate export - provider.add_span_processor(SimpleSpanProcessor(ObserveModelsExporter())) - - trace.set_tracer_provider(provider) - - -def is_debug_trace_provider() -> bool: - """Check if the current trace provider is the debug trace provider.""" - current_provider = trace.get_tracer_provider() - if current_provider and current_provider.sampler is not None: - return isinstance(current_provider.sampler, PlainRequestSampler) - return False - - -class PlainRequestSampler(sampling.Sampler): - """Drops traces based on request path or user role.""" - - def __init__(self): - # Custom parent-based sampler that properly handles RECORD_ONLY inheritance - self._delegate = sampling.ParentBased(sampling.ALWAYS_ON) - - # TODO ignore url namespace instead? admin, observe, assets - self._ignore_url_paths = [ - re.compile(p) for p in settings.OBSERVE_IGNORE_URL_PATTERNS - ] - - # Track sampling decisions by trace ID - self._trace_decisions = {} # trace_id -> Decision - self._lock = threading.Lock() - - def should_sample( - self, - parent_context, - trace_id, - name, - kind: SpanKind | None = None, - attributes=None, - links=None, - trace_state=None, - ): - # First, drop if the URL should be ignored. - if attributes: - if url_path := attributes.get(url_attributes.URL_PATH, ""): - for pattern in self._ignore_url_paths: - if pattern.match(url_path): - return sampling.SamplingResult( - sampling.Decision.DROP, - attributes=attributes, - ) - - # Check if we already have a decision for this trace - with self._lock: - if trace_id in self._trace_decisions: - decision = self._trace_decisions[trace_id] - return sampling.SamplingResult( - decision, - attributes=attributes, - ) - - # For new traces, check cookies in the context - decision = None - if parent_context: - # Check cookies for root spans - if cookies := baggage.get_baggage("http.request.cookies", parent_context): - if observe_cookie := cookies.get("observe"): - unsigned_value = unsign_cookie_value( - "observe", observe_cookie, default=False - ) - - if unsigned_value == "sample": - decision = sampling.Decision.RECORD_AND_SAMPLE - elif unsigned_value == "record": - decision = sampling.Decision.RECORD_ONLY - - if decision is None: - decision = sampling.Decision.DROP - - # If no decision from cookies, use default - if decision is None: - result = self._delegate.should_sample( - parent_context, - trace_id, - name, - kind=kind, - attributes=attributes, - links=links, - trace_state=trace_state, - ) - decision = result.decision - - # Store the decision for this trace - with self._lock: - self._trace_decisions[trace_id] = decision - # Clean up old entries if too many (simple LRU) - if len(self._trace_decisions) > 1000: - # Remove oldest entries - for old_trace_id in list(self._trace_decisions.keys())[:100]: - del self._trace_decisions[old_trace_id] - - return sampling.SamplingResult( - decision, - attributes=attributes, - ) - - def get_description(self) -> str: - return "PlainRequestSampler" - - -class ObserveSpanProcessor(SpanProcessor): - """Collects spans in real-time for current trace performance monitoring.""" - - def __init__(self): - self.active_spans_by_trace = defaultdict(dict) # trace_id -> {span_id: span} - self.completed_spans_by_trace = defaultdict(list) # trace_id -> [spans] - self.lock = threading.Lock() - - def on_start(self, span, parent_context=None): - """Called when a span starts.""" - with self.lock: - trace_id = format(span.get_span_context().trace_id, "032x") - span_id = format(span.get_span_context().span_id, "016x") - self.active_spans_by_trace[trace_id][span_id] = span - - def on_end(self, span): - """Called when a span ends.""" - with self.lock: - trace_id = format(span.get_span_context().trace_id, "032x") - span_id = format(span.get_span_context().span_id, "016x") - - # Move from active to completed - if trace_id in self.active_spans_by_trace: - span_obj = self.active_spans_by_trace[trace_id].pop(span_id, None) - if span_obj: - self.completed_spans_by_trace[trace_id].append(span_obj) - - # Clean up empty trace entries - if not self.active_spans_by_trace[trace_id]: - del self.active_spans_by_trace[trace_id] - - def get_current_trace_summary(self): - """Get performance summary for the currently active trace.""" - current_span = trace.get_current_span() - if not current_span: - # If no current span, check if we have any active traces at all - with self.lock: - if not self.active_spans_by_trace and not self.completed_spans_by_trace: - return None - - # Get the most recent trace if we can't find current span - all_trace_ids = list(self.active_spans_by_trace.keys()) + list( - self.completed_spans_by_trace.keys() - ) - if not all_trace_ids: - return None - trace_id = all_trace_ids[-1] # Use most recent - else: - # Use the current span's trace - trace_id = format(current_span.get_span_context().trace_id, "032x") - - with self.lock: - active_spans = list(self.active_spans_by_trace.get(trace_id, {}).values()) - completed_spans = self.completed_spans_by_trace.get(trace_id, []) - all_spans = active_spans + completed_spans - - if not all_spans: - return None - - # Calculate summary stats - db_queries = 0 - total_spans = len(all_spans) - earliest_start = None - latest_end = None - - for span in all_spans: - # Count DB queries - if span.attributes and span.attributes.get("db.system"): - db_queries += 1 - - # Calculate duration for completed spans - if span.end_time and span.start_time: - if earliest_start is None or span.start_time < earliest_start: - earliest_start = span.start_time - - if latest_end is None or span.end_time > latest_end: - latest_end = span.end_time - elif span.start_time: - # For active spans, track start time - if earliest_start is None or span.start_time < earliest_start: - earliest_start = span.start_time - - # Calculate overall duration (for the whole trace) - duration_ms = 0.0 - if earliest_start and latest_end: - duration_ms = (latest_end - earliest_start) / 1_000_000 # ns to ms - elif earliest_start: - # If trace is still active, calculate duration so far - import time - - current_time_ns = int(time.time() * 1_000_000_000) - duration_ms = (current_time_ns - earliest_start) / 1_000_000 - - # Build summary parts like the Trace model does - parts = [f"{total_spans}sp"] - - if db_queries > 0: - parts.append(f"{db_queries}db") - - if duration_ms > 0: - parts.append(f"{round(duration_ms, 1)}ms") - - return " ".join(parts) - - def shutdown(self): - """Cleanup when shutting down.""" - with self.lock: - self.active_spans_by_trace.clear() - self.completed_spans_by_trace.clear() - - def force_flush(self, timeout_millis=None): - """Required by SpanProcessor interface.""" - return True - - -class ObserveModelsExporter(SpanExporter): - """Exporter that writes spans into the observe models tables. - - Note: This should only receive spans with RECORD_AND_SAMPLE sampling decision. - Spans with RECORD_ONLY should not reach this exporter. - """ - - def export(self, spans): - """Persist each span individually for immediate export.""" - - from .models import Span, Trace - - with suppress_db_tracing(): - for span in spans: - try: - # Format IDs according to W3C Trace Context specification - trace_id_hex = format(span.get_span_context().trace_id, "032x") - span_id_hex = format(span.get_span_context().span_id, "016x") - parent_id_hex = ( - format(span.parent.span_id, "016x") if span.parent else "" - ) - - # Extract attributes directly - attributes = dict(span.attributes) if span.attributes else {} - request_id = attributes.get("plain.request.id", "") - user_id = attributes.get("user.id", "") - session_id = attributes.get("session.id", "") - - # Set description for root spans - description = span.name if not parent_id_hex else "" - - # Convert timestamps from nanoseconds to datetime - start_time = ( - datetime.fromtimestamp(span.start_time / 1_000_000_000, tz=UTC) - if span.start_time - else None - ) - end_time = ( - datetime.fromtimestamp(span.end_time / 1_000_000_000, tz=UTC) - if span.end_time - else None - ) - - # Get or create the trace - trace, created = Trace.objects.get_or_create( - trace_id=trace_id_hex, - defaults={ - "start_time": start_time, - "end_time": end_time, - "request_id": request_id, - "user_id": user_id, - "session_id": session_id, - "description": description, - }, - ) - - # Update trace if we have better data - if not created: - updated = False - if start_time and ( - not trace.start_time or start_time < trace.start_time - ): - trace.start_time = start_time - updated = True - if end_time and ( - not trace.end_time or end_time > trace.end_time - ): - trace.end_time = end_time - updated = True - if request_id and not trace.request_id: - trace.request_id = request_id - updated = True - if user_id and not trace.user_id: - trace.user_id = user_id - updated = True - if session_id and not trace.session_id: - trace.session_id = session_id - updated = True - if description and not trace.description: - trace.description = description - updated = True - if updated: - trace.save() - - # Extract span kind directly from the span object - kind_str = span.kind.name if span.kind else "INTERNAL" - - # Convert events to JSON format - events_json = [] - if span.events: - for event in span.events: - events_json.append( - { - "name": event.name, - "timestamp": event.timestamp, - "attributes": ( - dict(event.attributes) - if event.attributes - else {} - ), - } - ) - - # Convert links to JSON format - links_json = [] - if span.links: - for link in span.links: - links_json.append( - { - "context": { - "trace_id": format( - link.context.trace_id, "032x" - ), - "span_id": format(link.context.span_id, "016x"), - }, - "attributes": ( - dict(link.attributes) if link.attributes else {} - ), - } - ) - - # Create the span - Span.objects.get_or_create( - trace=trace, - span_id=span_id_hex, - defaults={ - "name": span.name, - "kind": kind_str, - "parent_id": parent_id_hex, - "start_time": start_time, - "end_time": end_time, - "status": { - "status_code": ( - span.status.status_code.name - if span.status - else "UNSET" - ), - "description": span.status.description - if span.status - else "", - }, - "context": { - "trace_id": trace_id_hex, - "span_id": span_id_hex, - "trace_flags": span.get_span_context().trace_flags, - "trace_state": ( - dict(span.get_span_context().trace_state) - if span.get_span_context().trace_state - else {} - ), - }, - "attributes": attributes, - "events": events_json, - "links": links_json, - "resource": ( - dict(span.resource.attributes) if span.resource else {} - ), - }, - ) - - except Exception as e: - logger.warning( - "Failed to export span to database: %s", - e, - exc_info=True, - ) - - # Delete oldest traces if we exceed the limit - try: - if Trace.objects.count() > settings.OBSERVE_TRACE_LIMIT: - delete_ids = Trace.objects.order_by("start_time")[ - : settings.OBSERVE_TRACE_LIMIT - ].values_list("id", flat=True) - Trace.objects.filter(id__in=delete_ids).delete() - except Exception as e: - logger.warning("Failed to clean up old traces: %s", e) - - return True diff --git a/plain-observe/plain/observe/urls.py b/plain-observe/plain/observe/urls.py deleted file mode 100644 index 29809342cd..0000000000 --- a/plain-observe/plain/observe/urls.py +++ /dev/null @@ -1,10 +0,0 @@ -from plain.urls import Router, path - -from . import views - - -class ObserveRouter(Router): - namespace = "observe" - urls = [ - path("", views.ObservabilitySpansView, name="spans"), - ] diff --git a/plain-observe/LICENSE b/plain-observer/LICENSE similarity index 100% rename from plain-observe/LICENSE rename to plain-observer/LICENSE diff --git a/plain-observer/README.md b/plain-observer/README.md new file mode 120000 index 0000000000..5f3ab6ccef --- /dev/null +++ b/plain-observer/README.md @@ -0,0 +1 @@ +plain/observer/README.md \ No newline at end of file diff --git a/plain-observer/plain/observer/CHANGELOG.md b/plain-observer/plain/observer/CHANGELOG.md new file mode 100644 index 0000000000..f291ea1c34 --- /dev/null +++ b/plain-observer/plain/observer/CHANGELOG.md @@ -0,0 +1 @@ +# plain-observer changelog diff --git a/plain-observer/plain/observer/README.md b/plain-observer/plain/observer/README.md new file mode 100644 index 0000000000..a1952e884a --- /dev/null +++ b/plain-observer/plain/observer/README.md @@ -0,0 +1,3 @@ +# plain.observer + +**Monitor.** diff --git a/plain-observe/plain/observe/__init__.py b/plain-observer/plain/observer/__init__.py similarity index 100% rename from plain-observe/plain/observe/__init__.py rename to plain-observer/plain/observer/__init__.py diff --git a/plain-observe/plain/observe/admin.py b/plain-observer/plain/observer/admin.py similarity index 89% rename from plain-observe/plain/observe/admin.py rename to plain-observer/plain/observer/admin.py index 2683191558..69e0030497 100644 --- a/plain-observe/plain/observe/admin.py +++ b/plain-observer/plain/observer/admin.py @@ -15,7 +15,7 @@ @register_viewset class TraceViewset(AdminViewset): class ListView(AdminModelListView): - nav_section = "Observe" + nav_section = "Observer" model = Trace fields = [ "trace_id", @@ -34,7 +34,7 @@ class ListView(AdminModelListView): class DetailView(AdminModelDetailView): model = Trace - template_name = "admin/observe/trace_detail.html" + template_name = "admin/observer/trace_detail.html" def get_template_context(self): context = super().get_template_context() @@ -48,7 +48,7 @@ def get_template_context(self): @register_viewset class SpanViewset(AdminViewset): class ListView(AdminModelListView): - nav_section = "Observe" + nav_section = "Observer" model = Span fields = [ "name", @@ -85,10 +85,10 @@ class DetailView(AdminModelDetailView): @register_toolbar_panel -class ObservabilityToolbarPanel(ToolbarPanel): - name = "Observability" - template_name = "toolbar/observability.html" - button_template_name = "toolbar/observability_button.html" +class ObserverToolbarPanel(ToolbarPanel): + name = "Observer" + template_name = "toolbar/observer.html" + button_template_name = "toolbar/observer_button.html" @cached_property def observer(self): diff --git a/plain-observer/plain/observer/cli.py b/plain-observer/plain/observer/cli.py new file mode 100644 index 0000000000..cefb6e201b --- /dev/null +++ b/plain-observer/plain/observer/cli.py @@ -0,0 +1,23 @@ +import click + +from plain.cli import register_cli +from plain.observer.models import Trace + + +@register_cli("observer") +@click.group("observer") +def observer_cli(): + pass + + +@observer_cli.command() +@click.option("--force", is_flag=True, help="Skip confirmation prompt.") +def clear(force: bool): + """Clear all observer data.""" + if not force: + click.confirm( + "Are you sure you want to clear all observer data? This cannot be undone.", + abort=True, + ) + + print("Deleted", Trace.objects.all().delete()) diff --git a/plain-observer/plain/observer/config.py b/plain-observer/plain/observer/config.py new file mode 100644 index 0000000000..4349834f5a --- /dev/null +++ b/plain-observer/plain/observer/config.py @@ -0,0 +1,42 @@ +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor + +from plain.packages import PackageConfig, register_config + +from .exporter import ObserverExporter +from .processor import ObserverSpanProcessor +from .sampler import ObserverSampler + + +@register_config +class Config(PackageConfig): + package_label = "plainobserver" + + def ready(self): + if self.has_existing_trace_provider(): + return + + self.setup_observer() + + @staticmethod + def has_existing_trace_provider() -> bool: + """Check if there is an existing trace provider.""" + current_provider = trace.get_tracer_provider() + return current_provider and not isinstance( + current_provider, trace.ProxyTracerProvider + ) + + @staticmethod + def setup_observer() -> None: + sampler = ObserverSampler() + provider = TracerProvider(sampler=sampler) + + # Add the real-time span collector for immediate access + span_collector = ObserverSpanProcessor() + provider.add_span_processor(span_collector) + + # Add the database exporter using SimpleSpanProcessor for immediate export + provider.add_span_processor(SimpleSpanProcessor(ObserverExporter())) + + trace.set_tracer_provider(provider) diff --git a/plain-observe/plain/observe/core.py b/plain-observer/plain/observer/core.py similarity index 71% rename from plain-observe/plain/observe/core.py rename to plain-observer/plain/observer/core.py index a59f0da601..e22a55cf66 100644 --- a/plain-observe/plain/observe/core.py +++ b/plain-observer/plain/observer/core.py @@ -2,17 +2,22 @@ Core observability functionality and Observer class. """ +from .processor import get_span_processor + class Observer: """Central class for managing observability state and operations.""" + COOKIE_NAME = "observer" + COOKIE_DURATION = 60 * 60 * 24 # 1 day in seconds + def __init__(self, request): self.request = request @property def mode(self): """Get the current observability mode from signed cookie.""" - return self.request.get_signed_cookie("observe", default=None) + return self.request.get_signed_cookie(self.COOKIE_NAME, default=None) @property def is_enabled(self): @@ -31,21 +36,24 @@ def is_recording(self): def enable_record_mode(self, response): """Enable record-only mode (real-time monitoring, no DB export).""" - response.set_signed_cookie("observe", "record", max_age=60 * 60 * 24) + response.set_signed_cookie( + self.COOKIE_NAME, "record", max_age=self.COOKIE_DURATION + ) def enable_sample_mode(self, response): """Enable full sampling mode (real-time monitoring + DB export).""" - response.set_signed_cookie("observe", "sample", max_age=60 * 60 * 24) + response.set_signed_cookie( + self.COOKIE_NAME, "sample", max_age=self.COOKIE_DURATION + ) def disable(self, response): """Disable observability by deleting the cookie.""" - response.delete_cookie("observe") + response.delete_cookie(self.COOKIE_NAME) def get_current_trace_summary(self): """Get performance summary string for the currently active trace.""" - from .otel import get_span_collector - span_collector = get_span_collector() + span_collector = get_span_processor() if not span_collector: return None diff --git a/plain-observer/plain/observer/default_settings.py b/plain-observer/plain/observer/default_settings.py new file mode 100644 index 0000000000..e56d7d0bde --- /dev/null +++ b/plain-observer/plain/observer/default_settings.py @@ -0,0 +1,8 @@ +OBSERVER_IGNORE_URL_PATTERNS: list[str] = [ + "/assets/.*", + "/admin/.*", + "/observer/.*", + "/favicon.ico", + "/.well-known/.*", +] +OBSERVER_TRACE_LIMIT: int = 100 diff --git a/plain-observer/plain/observer/exporter.py b/plain-observer/plain/observer/exporter.py new file mode 100644 index 0000000000..bc8ec4cf35 --- /dev/null +++ b/plain-observer/plain/observer/exporter.py @@ -0,0 +1,195 @@ +import logging +from datetime import UTC, datetime + +from opentelemetry.sdk.trace.export import SpanExporter + +from plain.models.observability import suppress_db_tracing +from plain.runtime import settings + +logger = logging.getLogger(__name__) + + +class ObserverExporter(SpanExporter): + """Exporter that writes spans into the observe models tables. + + Note: This should only receive spans with RECORD_AND_SAMPLE sampling decision. + Spans with RECORD_ONLY should not reach this exporter. + """ + + def export(self, spans): + """Persist each span individually for immediate export.""" + + from .models import Span, Trace + + with suppress_db_tracing(): + for span in spans: + try: + # Format IDs according to W3C Trace Context specification + trace_id_hex = format(span.get_span_context().trace_id, "032x") + span_id_hex = format(span.get_span_context().span_id, "016x") + parent_id_hex = ( + format(span.parent.span_id, "016x") if span.parent else "" + ) + + # Extract attributes directly + attributes = dict(span.attributes) if span.attributes else {} + request_id = attributes.get("plain.request.id", "") + user_id = attributes.get("user.id", "") + session_id = attributes.get("session.id", "") + + # Set description for root spans + description = span.name if not parent_id_hex else "" + + # Convert timestamps from nanoseconds to datetime + start_time = ( + datetime.fromtimestamp(span.start_time / 1_000_000_000, tz=UTC) + if span.start_time + else None + ) + end_time = ( + datetime.fromtimestamp(span.end_time / 1_000_000_000, tz=UTC) + if span.end_time + else None + ) + + # Get or create the trace + trace, created = Trace.objects.get_or_create( + trace_id=trace_id_hex, + defaults={ + "start_time": start_time, + "end_time": end_time, + "request_id": request_id, + "user_id": user_id, + "session_id": session_id, + "description": description, + }, + ) + + # Update trace if we have better data + if not created: + updated = False + if start_time and ( + not trace.start_time or start_time < trace.start_time + ): + trace.start_time = start_time + updated = True + if end_time and ( + not trace.end_time or end_time > trace.end_time + ): + trace.end_time = end_time + updated = True + if request_id and not trace.request_id: + trace.request_id = request_id + updated = True + if user_id and not trace.user_id: + trace.user_id = user_id + updated = True + if session_id and not trace.session_id: + trace.session_id = session_id + updated = True + if description and not trace.description: + trace.description = description + updated = True + if updated: + trace.save() + + # Extract span kind directly from the span object + kind_str = span.kind.name if span.kind else "INTERNAL" + + # Convert events to JSON format + events_json = [] + if span.events: + for event in span.events: + events_json.append( + { + "name": event.name, + "timestamp": event.timestamp, + "attributes": ( + dict(event.attributes) + if event.attributes + else {} + ), + } + ) + + # Convert links to JSON format + links_json = [] + if span.links: + for link in span.links: + links_json.append( + { + "context": { + "trace_id": format( + link.context.trace_id, "032x" + ), + "span_id": format(link.context.span_id, "016x"), + }, + "attributes": ( + dict(link.attributes) if link.attributes else {} + ), + } + ) + + # Create the span + Span.objects.get_or_create( + trace=trace, + span_id=span_id_hex, + defaults={ + "name": span.name, + "kind": kind_str, + "parent_id": parent_id_hex, + "start_time": start_time, + "end_time": end_time, + "status": { + "status_code": ( + span.status.status_code.name + if span.status + else "UNSET" + ), + "description": span.status.description + if span.status + else "", + }, + "context": { + "trace_id": trace_id_hex, + "span_id": span_id_hex, + "trace_flags": span.get_span_context().trace_flags, + "trace_state": ( + dict(span.get_span_context().trace_state) + if span.get_span_context().trace_state + else {} + ), + }, + "attributes": attributes, + "events": events_json, + "links": links_json, + "resource": ( + dict(span.resource.attributes) if span.resource else {} + ), + }, + ) + + except Exception as e: + logger.warning( + "Failed to export span to database: %s", + e, + exc_info=True, + ) + + # Delete oldest traces if we exceed the limit + try: + if Trace.objects.count() > settings.OBSERVER_TRACE_LIMIT: + delete_ids = Trace.objects.order_by("start_time")[ + : settings.OBSERVER_TRACE_LIMIT + ].values_list("id", flat=True) + Trace.objects.filter(id__in=delete_ids).delete() + except Exception as e: + logger.warning( + "Failed to update observer traces in db: %s", e, exc_info=True + ) + + return True + + def shutdown(self): + """Called when the SDK is shut down.""" + pass diff --git a/plain-observer/plain/observer/migrations/0001_initial.py b/plain-observer/plain/observer/migrations/0001_initial.py new file mode 100644 index 0000000000..c8be54137a --- /dev/null +++ b/plain-observer/plain/observer/migrations/0001_initial.py @@ -0,0 +1,100 @@ +# Generated by Plain 0.52.2 on 2025-07-15 02:14 + +import plain.models.deletion +from plain import models +from plain.models import migrations + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Span", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True)), + ("span_id", models.CharField(max_length=255)), + ("name", models.CharField(max_length=255)), + ("kind", models.CharField(max_length=50)), + ( + "parent_id", + models.CharField(default="", max_length=255, required=False), + ), + ("start_time", models.DateTimeField(allow_null=True, required=False)), + ("end_time", models.DateTimeField(allow_null=True, required=False)), + ("status", models.JSONField(default=dict)), + ("context", models.JSONField(default=dict)), + ("attributes", models.JSONField(default=dict, required=False)), + ("events", models.JSONField(default=list, required=False)), + ("links", models.JSONField(default=list, required=False)), + ("resource", models.JSONField(default=dict, required=False)), + ], + options={ + "ordering": ["-start_time"], + }, + ), + migrations.CreateModel( + name="Trace", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True)), + ("trace_id", models.CharField(max_length=255)), + ("start_time", models.DateTimeField(allow_null=True, required=False)), + ("end_time", models.DateTimeField(allow_null=True, required=False)), + ("description", models.TextField(default="", required=False)), + ( + "request_id", + models.CharField(default="", max_length=255, required=False), + ), + ( + "session_id", + models.CharField(default="", max_length=255, required=False), + ), + ( + "user_id", + models.CharField(default="", max_length=255, required=False), + ), + ], + options={ + "ordering": ["-start_time"], + }, + ), + migrations.AddConstraint( + model_name="trace", + constraint=models.UniqueConstraint( + fields=("trace_id",), name="observer_unique_trace_id" + ), + ), + migrations.AddField( + model_name="span", + name="trace", + field=models.ForeignKey( + on_delete=plain.models.deletion.CASCADE, + related_name="spans", + to="plainobserver.trace", + ), + ), + migrations.AddIndex( + model_name="span", + index=models.Index( + fields=["trace", "span_id"], name="plainobserv_trace_i_89a97c_idx" + ), + ), + migrations.AddIndex( + model_name="span", + index=models.Index(fields=["trace"], name="plainobserv_trace_i_84958a_idx"), + ), + migrations.AddIndex( + model_name="span", + index=models.Index( + fields=["start_time"], name="plainobserv_start_t_cb47a3_idx" + ), + ), + migrations.AddConstraint( + model_name="span", + constraint=models.UniqueConstraint( + fields=("trace", "span_id"), name="observer_unique_span_id" + ), + ), + ] diff --git a/plain-observe/plain/observe/migrations/__init__.py b/plain-observer/plain/observer/migrations/__init__.py similarity index 100% rename from plain-observe/plain/observe/migrations/__init__.py rename to plain-observer/plain/observer/migrations/__init__.py diff --git a/plain-observe/plain/observe/models.py b/plain-observer/plain/observer/models.py similarity index 98% rename from plain-observe/plain/observe/models.py rename to plain-observer/plain/observer/models.py index a71ce2fce8..361a5c2e55 100644 --- a/plain-observe/plain/observe/models.py +++ b/plain-observer/plain/observer/models.py @@ -25,7 +25,7 @@ class Meta: constraints = [ models.UniqueConstraint( fields=["trace_id"], - name="observe_unique_trace_id", + name="observer_unique_trace_id", ) ] @@ -84,7 +84,7 @@ class Meta: constraints = [ models.UniqueConstraint( fields=["trace", "span_id"], - name="observe_unique_span_id", + name="observer_unique_span_id", ) ] indexes = [ diff --git a/plain-observer/plain/observer/processor.py b/plain-observer/plain/observer/processor.py new file mode 100644 index 0000000000..a0f4990403 --- /dev/null +++ b/plain-observer/plain/observer/processor.py @@ -0,0 +1,139 @@ +import threading +from collections import defaultdict + +from opentelemetry import trace +from opentelemetry.sdk.trace import SpanProcessor + + +def get_span_processor(): + """Get the span collector instance from the tracer provider.""" + current_provider = trace.get_tracer_provider() + if not current_provider: + return None + + # Look for ObserverSpanProcessor in the span processors + # Check if the provider has a _active_span_processor attribute + if hasattr(current_provider, "_active_span_processor"): + # It's a composite processor, check its _span_processors + composite_processor = current_provider._active_span_processor + if hasattr(composite_processor, "_span_processors"): + for processor in composite_processor._span_processors: + if isinstance(processor, ObserverSpanProcessor): + return processor + + return None + + +class ObserverSpanProcessor(SpanProcessor): + """Collects spans in real-time for current trace performance monitoring.""" + + def __init__(self): + self.active_spans_by_trace = defaultdict(dict) # trace_id -> {span_id: span} + self.completed_spans_by_trace = defaultdict(list) # trace_id -> [spans] + self.lock = threading.Lock() + + def on_start(self, span, parent_context=None): + """Called when a span starts.""" + with self.lock: + trace_id = format(span.get_span_context().trace_id, "032x") + span_id = format(span.get_span_context().span_id, "016x") + self.active_spans_by_trace[trace_id][span_id] = span + + def on_end(self, span): + """Called when a span ends.""" + with self.lock: + trace_id = format(span.get_span_context().trace_id, "032x") + span_id = format(span.get_span_context().span_id, "016x") + + # Move from active to completed + if trace_id in self.active_spans_by_trace: + span_obj = self.active_spans_by_trace[trace_id].pop(span_id, None) + if span_obj: + self.completed_spans_by_trace[trace_id].append(span_obj) + + # Clean up empty trace entries + if not self.active_spans_by_trace[trace_id]: + del self.active_spans_by_trace[trace_id] + + def get_current_trace_summary(self): + """Get performance summary for the currently active trace.""" + current_span = trace.get_current_span() + if not current_span: + # If no current span, check if we have any active traces at all + with self.lock: + if not self.active_spans_by_trace and not self.completed_spans_by_trace: + return None + + # Get the most recent trace if we can't find current span + all_trace_ids = list(self.active_spans_by_trace.keys()) + list( + self.completed_spans_by_trace.keys() + ) + if not all_trace_ids: + return None + trace_id = all_trace_ids[-1] # Use most recent + else: + # Use the current span's trace + trace_id = format(current_span.get_span_context().trace_id, "032x") + + with self.lock: + active_spans = list(self.active_spans_by_trace.get(trace_id, {}).values()) + completed_spans = self.completed_spans_by_trace.get(trace_id, []) + all_spans = active_spans + completed_spans + + if not all_spans: + return None + + # Calculate summary stats + db_queries = 0 + total_spans = len(all_spans) + earliest_start = None + latest_end = None + + for span in all_spans: + # Count DB queries + if span.attributes and span.attributes.get("db.system"): + db_queries += 1 + + # Calculate duration for completed spans + if span.end_time and span.start_time: + if earliest_start is None or span.start_time < earliest_start: + earliest_start = span.start_time + + if latest_end is None or span.end_time > latest_end: + latest_end = span.end_time + elif span.start_time: + # For active spans, track start time + if earliest_start is None or span.start_time < earliest_start: + earliest_start = span.start_time + + # Calculate overall duration (for the whole trace) + duration_ms = 0.0 + if earliest_start and latest_end: + duration_ms = (latest_end - earliest_start) / 1_000_000 # ns to ms + elif earliest_start: + # If trace is still active, calculate duration so far + import time + + current_time_ns = int(time.time() * 1_000_000_000) + duration_ms = (current_time_ns - earliest_start) / 1_000_000 + + # Build summary parts like the Trace model does + parts = [f"{total_spans}sp"] + + if db_queries > 0: + parts.append(f"{db_queries}db") + + if duration_ms > 0: + parts.append(f"{round(duration_ms, 1)}ms") + + return " ".join(parts) + + def shutdown(self): + """Cleanup when shutting down.""" + with self.lock: + self.active_spans_by_trace.clear() + self.completed_spans_by_trace.clear() + + def force_flush(self, timeout_millis=None): + """Required by SpanProcessor interface.""" + return True diff --git a/plain-observer/plain/observer/sampler.py b/plain-observer/plain/observer/sampler.py new file mode 100644 index 0000000000..0721096ad9 --- /dev/null +++ b/plain-observer/plain/observer/sampler.py @@ -0,0 +1,104 @@ +import re +import threading + +from opentelemetry import baggage +from opentelemetry.sdk.trace import sampling +from opentelemetry.semconv.attributes import url_attributes +from opentelemetry.trace import SpanKind + +from plain.http.cookie import unsign_cookie_value +from plain.runtime import settings + + +class ObserverSampler(sampling.Sampler): + """Drops traces based on request path or user role.""" + + def __init__(self): + # Custom parent-based sampler that properly handles RECORD_ONLY inheritance + self._delegate = sampling.ParentBased(sampling.ALWAYS_OFF) + + # TODO ignore url namespace instead? admin, observer, assets + self._ignore_url_paths = [ + re.compile(p) for p in settings.OBSERVER_IGNORE_URL_PATTERNS + ] + + # Track sampling decisions by trace ID + self._trace_decisions = {} # trace_id -> Decision + self._lock = threading.Lock() + + def should_sample( + self, + parent_context, + trace_id, + name, + kind: SpanKind | None = None, + attributes=None, + links=None, + trace_state=None, + ): + # First, drop if the URL should be ignored. + if attributes: + if url_path := attributes.get(url_attributes.URL_PATH, ""): + for pattern in self._ignore_url_paths: + if pattern.match(url_path): + return sampling.SamplingResult( + sampling.Decision.DROP, + attributes=attributes, + ) + + # Check if we already have a decision for this trace + with self._lock: + if trace_id in self._trace_decisions: + decision = self._trace_decisions[trace_id] + return sampling.SamplingResult( + decision, + attributes=attributes, + ) + + # For new traces, check cookies in the context + decision = None + if parent_context: + # Check cookies for root spans + if cookies := baggage.get_baggage("http.request.cookies", parent_context): + if observer_cookie := cookies.get("observer"): + unsigned_value = unsign_cookie_value( + "observer", observer_cookie, default=False + ) + + if unsigned_value == "sample": + decision = sampling.Decision.RECORD_AND_SAMPLE + elif unsigned_value == "record": + decision = sampling.Decision.RECORD_ONLY + + if decision is None: + decision = sampling.Decision.DROP + + # If no decision from cookies, use default + if decision is None: + result = self._delegate.should_sample( + parent_context, + trace_id, + name, + kind=kind, + attributes=attributes, + links=links, + trace_state=trace_state, + ) + decision = result.decision + + # Store the decision for this trace + with self._lock: + self._trace_decisions[trace_id] = decision + # Clean up old entries if too many (simple LRU) + if len(self._trace_decisions) > 1000: + # Remove oldest entries + for old_trace_id in list(self._trace_decisions.keys())[:100]: + del self._trace_decisions[old_trace_id] + + return sampling.SamplingResult( + decision, + attributes=attributes, + ) + + def get_description(self) -> str: + return "ObserverSampler" diff --git a/plain-observe/plain/observe/templates/admin/observe/trace_detail.html b/plain-observer/plain/observer/templates/admin/observer/trace_detail.html similarity index 68% rename from plain-observe/plain/observe/templates/admin/observe/trace_detail.html rename to plain-observer/plain/observer/templates/admin/observer/trace_detail.html index a466d9ca78..2c7fb8be65 100644 --- a/plain-observe/plain/observe/templates/admin/observe/trace_detail.html +++ b/plain-observer/plain/observer/templates/admin/observer/trace_detail.html @@ -6,6 +6,6 @@
- {% include "observability/_trace_detail.html" %} + {% include "observer/_trace_detail.html" %}
{% endblock %} diff --git a/plain-observe/plain/observe/templates/observability/_trace_detail.html b/plain-observer/plain/observer/templates/observer/_trace_detail.html similarity index 99% rename from plain-observe/plain/observe/templates/observability/_trace_detail.html rename to plain-observer/plain/observer/templates/observer/_trace_detail.html index d3cb131db9..debd143b0e 100644 --- a/plain-observe/plain/observe/templates/observability/_trace_detail.html +++ b/plain-observer/plain/observer/templates/observer/_trace_detail.html @@ -55,7 +55,7 @@

Trace: {{ trace.description }}

{% set width_percent = (span_duration / trace.duration_ms() * 100) if trace.duration_ms() > 0 else 0 %}
-
+
@@ -73,7 +73,7 @@

Trace: {{ trace.description }}

-
+
- +
{% if traces %} @@ -144,10 +144,10 @@

Traces ({{ traces|length }

-
+
{% htmxfragment "trace" %} {% set show_delete_button = True %} - {% include "observability/_trace_detail.html" %} + {% include "observer/_trace_detail.html" %} {% endhtmxfragment %}
@@ -181,14 +181,14 @@

Traces ({{ traces|length } - Disable Observability + Disable Observer

{% else %}
-

Observability is disabled

+

Observer is disabled

Choose a mode to start monitoring

diff --git a/plain-observe/plain/observe/templates/toolbar/observability.html b/plain-observer/plain/observer/templates/toolbar/observer.html similarity index 82% rename from plain-observe/plain/observe/templates/toolbar/observability.html rename to plain-observer/plain/observer/templates/toolbar/observer.html index adc5c437f5..80bd327948 100644 --- a/plain-observe/plain/observe/templates/toolbar/observability.html +++ b/plain-observer/plain/observer/templates/toolbar/observer.html @@ -1,11 +1,11 @@ -
+

Loading spans...

diff --git a/plain-observer/plain/observer/templates/observer/traces.html b/plain-observer/plain/observer/templates/observer/traces.html index d1a50a9626..6733136b76 100644 --- a/plain-observer/plain/observer/templates/observer/traces.html +++ b/plain-observer/plain/observer/templates/observer/traces.html @@ -14,14 +14,14 @@
-{% endif %} + {% else %} +
No spans...
+ {% endfor %} +