-
Notifications
You must be signed in to change notification settings - Fork 310
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: improve / clean up logging (#7591)
* refactor: always use console log handler * refactor: json for k8s, plain otherwise * chore: remove syslog from wsgi.py * chore: remove debug.log() * chore: drop syslog from settings.py * refactor: use log.log() in person.utils * refactor: fetch_meeting_attendance->log.log() * chore: gunicorn logs as JSON (wip) * feat: better json log formatting * refactor: improve log config * feat: gunicorn access log fields * fix: remove type hints The gunicorn logger plays tricks with the LogRecord args parameter to let it have string keys instead of being a simple tuple. The mypy tests rightly flag this. Rather than fighting the typing, just remove the hints and leave a comment warning not to use the gunicorn-specific formatter with other loggers.
- Loading branch information
1 parent
704f996
commit 6a96a69
Showing
9 changed files
with
95 additions
and
52 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
# Copyright The IETF Trust 2024, All Rights Reserved | ||
|
||
# Log as JSON on stdout (to distinguish from Django's logs on stderr) | ||
# | ||
# This is applied as an update to gunicorn's glogging.CONFIG_DEFAULTS. | ||
logconfig_dict = { | ||
"version": 1, | ||
"disable_existing_loggers": False, | ||
"root": {"level": "INFO", "handlers": ["console"]}, | ||
"loggers": { | ||
"gunicorn.error": { | ||
"level": "INFO", | ||
"handlers": ["console"], | ||
"propagate": False, | ||
"qualname": "gunicorn.error" | ||
}, | ||
|
||
"gunicorn.access": { | ||
"level": "INFO", | ||
"handlers": ["access_console"], | ||
"propagate": False, | ||
"qualname": "gunicorn.access" | ||
} | ||
}, | ||
"handlers": { | ||
"console": { | ||
"class": "logging.StreamHandler", | ||
"formatter": "json", | ||
"stream": "ext://sys.stdout" | ||
}, | ||
"access_console": { | ||
"class": "logging.StreamHandler", | ||
"formatter": "access_json", | ||
"stream": "ext://sys.stdout" | ||
}, | ||
}, | ||
"formatters": { | ||
"json": { | ||
"class": "ietf.utils.jsonlogger.DatatrackerJsonFormatter", | ||
"style": "{", | ||
"format": "{asctime}{levelname}{message}{name}{process}", | ||
}, | ||
"access_json": { | ||
"class": "ietf.utils.jsonlogger.GunicornRequestJsonFormatter", | ||
"style": "{", | ||
"format": "{asctime}{levelname}{message}{name}{process}", | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
# Copyright The IETF Trust 2024, All Rights Reserved | ||
from pythonjsonlogger import jsonlogger | ||
import time | ||
|
||
|
||
class DatatrackerJsonFormatter(jsonlogger.JsonFormatter): | ||
converter = time.gmtime # use UTC | ||
default_msec_format = "%s.%03d" # '.' instead of ',' | ||
|
||
|
||
class GunicornRequestJsonFormatter(DatatrackerJsonFormatter): | ||
"""Only works with Gunicorn's logging""" | ||
def add_fields(self, log_record, record, message_dict): | ||
super().add_fields(log_record, record, message_dict) | ||
log_record.setdefault("method", record.args["m"]) | ||
log_record.setdefault("proto", record.args["H"]) | ||
log_record.setdefault("remote_ip", record.args["h"]) | ||
path = record.args["U"] # URL path | ||
if record.args["q"]: # URL query string | ||
path = "?".join([path, record.args["q"]]) | ||
log_record.setdefault("path", path) | ||
log_record.setdefault("status", record.args["s"]) | ||
log_record.setdefault("referer", record.args["f"]) | ||
log_record.setdefault("user_agent", record.args["a"]) | ||
log_record.setdefault("len_bytes", record.args["B"]) | ||
log_record.setdefault("duration_ms", record.args["M"]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,23 +1,17 @@ | ||
# Copyright The IETF Trust 2013-2021, All Rights Reserved | ||
# Copyright The IETF Trust 2013-2024, All Rights Reserved | ||
# -*- coding: utf-8 -*- | ||
|
||
|
||
|
||
import os | ||
import sys | ||
import syslog | ||
|
||
path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | ||
|
||
syslog.openlog(str("datatracker"), syslog.LOG_PID, syslog.LOG_USER) | ||
|
||
if not path in sys.path: | ||
sys.path.insert(0, path) | ||
|
||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings") | ||
|
||
syslog.syslog("Starting datatracker wsgi instance") | ||
|
||
from django.core.wsgi import get_wsgi_application | ||
application = get_wsgi_application() | ||
|
||
application = get_wsgi_application() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters