Global logger for fedbiomed
Written above origin Logger class provided by python.
Following features were added from to the original module:
- provides a logger instance of FedLogger, which is also a singleton, so it can be used "as is"
- provides a dedicated file handler
- provides a JSON/gRPC handler (this permit to send error messages from a node to a researcher)
- works on python scripts / ipython / notebook
- manages a dictionary of handlers. Default keys are 'CONSOLE', 'GRPC', 'FILE', but any key is allowed (only one handler by key)
- allow changing log level globally, or on a specific handler (using its key)
- log levels can be provided as string instead of logging.* levels (no need to import logging in caller's code) just as in the initial python logger
A typical usage is:
from fedbiomed.common.logger import logger
logger.info("information message")
All methods of the original python logger are provided. To name a few:
- logger.debug()
- logger.info()
- logger.warning()
- logger.error()
- logger.critical()
Contrary to other Fed-BioMed classes, the API of FedLogger is compliant with the coding conventions used for logger (lowerCameCase)
Dependency issue
Please pay attention to not create dependency loop then importing other fedbiomed package
Attributes
DEBUG_FORMAT module-attribute
DEBUG_FORMAT = f'%(asctime)s %(name)s{LOG_PREFIX} %(levelname)s [%(module)s.%(funcName)s:%(lineno)d] - %(message)s'
DEFAULT_FORMAT module-attribute
DEFAULT_FORMAT = f'%(asctime)s %(name)s{LOG_PREFIX} %(levelname)s - %(message)s'
DEFAULT_LOG_FILE module-attribute
DEFAULT_LOG_FILE = 'mylog.log'
DEFAULT_LOG_LEVEL module-attribute
DEFAULT_LOG_LEVEL = WARNING
DEFAULT_SECURITY_LOG_FILE module-attribute
DEFAULT_SECURITY_LOG_FILE = 'security_audit.log'
LOG_PREFIX module-attribute
LOG_PREFIX = '%(prefix)s'
SECURITY_CONTEXT module-attribute
SECURITY_CONTEXT = ContextVar('fedbiomed_security_context', default=None)
SYSLOG_FACILITY_MAP module-attribute
SYSLOG_FACILITY_MAP = {'kern': LOG_KERN, 'user': LOG_USER, 'mail': LOG_MAIL, 'daemon': LOG_DAEMON, 'auth': LOG_AUTH, 'syslog': LOG_SYSLOG, 'lpr': LOG_LPR, 'news': LOG_NEWS, 'uucp': LOG_UUCP, 'cron': LOG_CRON, 'authpriv': LOG_AUTHPRIV, 'ftp': LOG_FTP, 'local0': LOG_LOCAL0, 'local1': LOG_LOCAL1, 'local2': LOG_LOCAL2, 'local3': LOG_LOCAL3, 'local4': LOG_LOCAL4, 'local5': LOG_LOCAL5, 'local6': LOG_LOCAL6, 'local7': LOG_LOCAL7}
SYSLOG_IDENT module-attribute
SYSLOG_IDENT = 'fedbiomed: '
logger module-attribute
logger = FedLogger()
Classes
FedLogger
FedLogger(level=logging.getLevelName(DEFAULT_LOG_LEVEL))
Base class for the logger.
It uses python logging module by composition (only log() method is overwritten)
All methods from the logging module can be accessed through the _logger member of the class if necessary (instead of overloading all the methods) (ex: logger._logger.getEffectiveLevel() )
Should not be imported
An initial console logger is installed (so the logger has at minimum one handler)
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
level | str | initial loglevel. This loglevel will be the default for all handlers, if called without the default level | getLevelName(DEFAULT_LOG_LEVEL) |
Source code in fedbiomed/common/logger.py
def __init__(self, level: str = logging.getLevelName(DEFAULT_LOG_LEVEL)):
"""Constructor of base class
An initial console logger is installed (so the logger has at minimum one handler)
Args:
level: initial loglevel. This loglevel will be the default for all handlers, if called
without the default level
"""
# internal tables
# transform string to logging.level
self._nameToLevel = {
"DEBUG": logging.DEBUG,
"INFO": logging.INFO,
"WARNING": logging.WARNING,
"ERROR": logging.ERROR,
"CRITICAL": logging.CRITICAL,
}
# transform logging.level to string
self._levelToName = {
logging.DEBUG: "DEBUG",
logging.INFO: "INFO",
logging.WARNING: "WARNING",
logging.ERROR: "ERROR",
logging.CRITICAL: "CRITICAL",
}
# name this logger
self._logger = logging.getLogger("fedbiomed")
# Do not propagate (avoids log duplication when third party libraries uses logging module)
self._logger.propagate = False
self._default_level = DEFAULT_LOG_LEVEL # MANDATORY ! KEEP THIS PLEASE !!!
self._default_level = self._internal_level_translator(level)
self._logger.setLevel(self._default_level)
# Store base format used by handlers
self._original_format: Dict[str, Optional[str]] = {}
self._handler_prefix: Dict[str, str] = {}
# init the handlers list and add a console handler on startup
self._handlers: Dict[str, logging.Handler] = {}
self.add_console_handler()
# --- Security log defaults (filled by Node at startup) ---
self._security_defaults: Dict[str, Any] = {
"component_id": None,
"component_name": None,
"fedbiomed_version": None,
}
pass
Functions
add_console_handler
add_console_handler(format=DEFAULT_FORMAT, level=DEFAULT_LOG_LEVEL)
Adds a console handler
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
format | str | the format string of the logger | DEFAULT_FORMAT |
level | Any | initial level of the logger for this handler (optional) if not given, the default level is set | DEFAULT_LOG_LEVEL |
Source code in fedbiomed/common/logger.py
def add_console_handler(
self, format: str = DEFAULT_FORMAT, level: Any = DEFAULT_LOG_LEVEL
):
"""Adds a console handler
Args:
format: the format string of the logger
level: initial level of the logger for this handler (optional) if not given, the default level is set
"""
handler: logging.Handler
if is_ipython():
handler = _IpythonConsoleHandler()
else:
handler = logging.StreamHandler()
handler.setLevel(self._internal_level_translator(level))
# Prevent security audit logs from appearing in interactive console (e.g., IPython).
handler.addFilter(_ExcludeSecurityFilter())
self._internal_add_handler("CONSOLE", handler, format)
pass
add_file_handler
add_file_handler(filename=DEFAULT_LOG_FILE, format=DEFAULT_FORMAT, level=DEFAULT_LOG_LEVEL)
Adds a file handler
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename | str | File to log to | DEFAULT_LOG_FILE |
format | str | Log format | DEFAULT_FORMAT |
level | Any | Initial level of the logger | DEFAULT_LOG_LEVEL |
Source code in fedbiomed/common/logger.py
def add_file_handler(
self,
filename: str = DEFAULT_LOG_FILE,
format: str = DEFAULT_FORMAT,
level: Any = DEFAULT_LOG_LEVEL,
):
"""Adds a file handler
Args:
filename: File to log to
format: Log format
level: Initial level of the logger
"""
handler = logging.FileHandler(filename=filename, mode="a")
handler.setLevel(self._internal_level_translator(level))
self._internal_add_handler("FILE", handler, format)
add_grpc_handler
add_grpc_handler(on_log, node_id=None, level=logging.INFO)
Adds a gRPC handler, to publish error message on a topic
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
on_log | Callable | Provided by higher level GRPC implementation | required |
node_id | Optional[str] | id of the caller (necessary for msg formatting to the researcher) | None |
level | Any | level of this handler (non-mandatory) level must be lower than ERROR to ensure that the research get all ERROR/CRITICAL messages | INFO |
Source code in fedbiomed/common/logger.py
def add_grpc_handler(
self, on_log: Callable, node_id: Optional[str] = None, level: Any = logging.INFO
):
"""Adds a gRPC handler, to publish error message on a topic
Args:
on_log: Provided by higher level GRPC implementation
node_id: id of the caller (necessary for msg formatting to the researcher)
level: level of this handler (non-mandatory) level must be lower than ERROR to ensure that the
research get all ERROR/CRITICAL messages
"""
handler = _GrpcHandler(
on_log=on_log,
node_id=node_id,
)
# may be not necessary ?
handler.setLevel(self._internal_level_translator(level))
formatter = _GrpcFormatter(node_id)
handler.setFormatter(formatter)
# Never transmit security audit logs to the researcher via gRPC.
handler.addFilter(_ExcludeSecurityFilter())
self._internal_add_handler("GRPC", handler)
# as a side effect this will set the minimal level to ERROR
# FIXME: alitolga: This could cause problems in the logging level,
# I believe the level should be set when node and researcher get started.
# self.setLevel(level, "GRPC")
pass
add_security_file_handler
add_security_file_handler(filename=DEFAULT_SECURITY_LOG_FILE, level=logging.INFO)
Adds a dedicated SECURITY_FILE handler that writes JSONL only. Only records emitted with extra {"is_security": True} will be written. Automatically rotates daily at midnight. Old logs are never deleted.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename | str | Security log file path. Defaults to 'security_audit.log' | DEFAULT_SECURITY_LOG_FILE |
level | Any | Logging level for security events. Defaults to INFO | INFO |
Source code in fedbiomed/common/logger.py
def add_security_file_handler(
self,
filename: str = DEFAULT_SECURITY_LOG_FILE,
level: Any = logging.INFO,
) -> None:
"""
Adds a dedicated SECURITY_FILE handler that writes JSONL only.
Only records emitted with extra {"is_security": True} will be written.
Automatically rotates daily at midnight. Old logs are never deleted.
Args:
filename: Security log file path. Defaults to 'security_audit.log'
level: Logging level for security events. Defaults to INFO
"""
handler = TimedRotatingFileHandler(
filename=filename,
when="midnight",
interval=1,
backupCount=0, # Keep all old logs, never delete
)
handler.setLevel(self._internal_level_translator(level))
handler.setFormatter(_SecurityFormatter(self._security_defaults))
handler.addFilter(_SecurityOnlyFilter())
# Disable buffering to ensure immediate writes
handler.stream.reconfigure(line_buffering=True)
handler_name = "SECURITY_FILE"
# Register under its own key so it doesn't collide with FILE handler
if handler_name not in self._handlers:
self._logger.debug(" adding handler for: " + handler_name)
self._handlers[handler_name] = handler
self._logger.addHandler(handler)
self._original_format[handler_name] = None
self._handler_prefix[handler_name] = ""
add_syslog_handler
add_syslog_handler(host, port, protocol, facility, level)
Adds a remote syslog handler.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
host | remote syslog host | required | |
port | remote syslog port | required | |
protocol | 'udp' or 'tcp' | required | |
facility | syslog facility | required | |
level | fedbiomed log level | required |
Source code in fedbiomed/common/logger.py
def add_syslog_handler(
self,
host,
port,
protocol,
facility,
level,
):
"""Adds a remote syslog handler.
Args:
host: remote syslog host
port: remote syslog port
protocol: 'udp' or 'tcp'
facility: syslog facility
level: fedbiomed log level
"""
if protocol not in {"udp", "tcp"}:
raise FedbiomedError(f"Unsupported syslog protocol: {protocol}")
socktype = socket.SOCK_DGRAM if protocol == "udp" else socket.SOCK_STREAM
kwargs = {
"address": (host, port),
"facility": facility,
"socktype": socktype,
}
handler = SysLogHandler(**kwargs)
# Optional compatibility knobs from stdlib docs
handler.append_nul = False
handler.ident = SYSLOG_IDENT
handler_key = "SYSLOG"
handler.setLevel(self._internal_level_translator(level))
# Forward only security events
handler.addFilter(_SecurityOnlyFilter())
# Keep the same JSON structure as the security audit file
handler.setFormatter(_SecurityFormatter(self._security_defaults))
self._internal_add_handler(handler_key, handler)
configure_security
configure_security(*, component_id=None, component_name=None, fedbiomed_version=None)
Configure default fields that must appear in every security log entry.
Source code in fedbiomed/common/logger.py
def configure_security(
self,
*,
component_id: Optional[str] = None,
component_name: Optional[ComponentType] = None,
fedbiomed_version: Optional[str] = None,
) -> None:
"""Configure default fields that must appear in every security log entry."""
if component_id is not None:
self._security_defaults["component_id"] = component_id
if component_name is not None:
self._security_defaults["component_name"] = component_name
if fedbiomed_version is not None:
self._security_defaults["fedbiomed_version"] = fedbiomed_version
critical
critical(msg, *args, broadcast=False, researcher_id=None, **kwargs)
Same as info message
Source code in fedbiomed/common/logger.py
def critical(self, msg, *args, broadcast=False, researcher_id=None, **kwargs):
"""Same as info message"""
# Merge extra dicts properly to support is_security flag
extra = kwargs.pop("extra", {})
extra.update({"researcher_id": researcher_id, "broadcast": broadcast})
stacklevel = kwargs.pop("stacklevel", 2)
self._logger.critical(
msg,
*args,
**kwargs,
extra=extra,
stacklevel=stacklevel,
)
debug
debug(msg, *args, broadcast=False, researcher_id=None, **kwargs)
Same as info message
Source code in fedbiomed/common/logger.py
def debug(self, msg, *args, broadcast=False, researcher_id=None, **kwargs):
"""Same as info message"""
# Merge extra dicts properly to support is_security flag
extra = kwargs.pop("extra", {})
extra.update({"researcher_id": researcher_id, "broadcast": broadcast})
stacklevel = kwargs.pop("stacklevel", 2)
self._logger.debug(
msg,
*args,
**kwargs,
extra=extra,
stacklevel=stacklevel,
)
del_syslog_handler
del_syslog_handler(handler_key='SYSLOG')
Source code in fedbiomed/common/logger.py
def del_syslog_handler(self, handler_key: str = "SYSLOG"):
self._internal_add_handler(handler_key, None)
error
error(msg, *args, broadcast=False, researcher_id=None, **kwargs)
Same as info message
Source code in fedbiomed/common/logger.py
def error(self, msg, *args, broadcast=False, researcher_id=None, **kwargs):
"""Same as info message"""
# Merge extra dicts properly to support is_security flag
extra = kwargs.pop("extra", {})
extra.update({"researcher_id": researcher_id, "broadcast": broadcast})
stacklevel = kwargs.pop("stacklevel", 2)
self._logger.error(
msg,
*args,
**kwargs,
extra=extra,
stacklevel=stacklevel,
)
info
info(msg, *args, broadcast=False, researcher_id=None, **kwargs)
Extends arguments of info message.
Valid only GrpcHandler is existing
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
msg | Message to log | required | |
broadcast | Broadcast message to all available researchers | False | |
researcher_id | ID of the researcher that the message will be sent. If broadcast True researcher id will be ignored | None | |
**kwargs | Additional keyword arguments. Can include extra={"is_security": True} to also write to security log file | {} |
Source code in fedbiomed/common/logger.py
def info(self, msg, *args, broadcast=False, researcher_id=None, **kwargs):
"""Extends arguments of info message.
Valid only GrpcHandler is existing
Args:
msg: Message to log
broadcast: Broadcast message to all available researchers
researcher_id: ID of the researcher that the message will be sent.
If broadcast True researcher id will be ignored
**kwargs: Additional keyword arguments. Can include extra={"is_security": True}
to also write to security log file
"""
# Merge extra dicts properly to support is_security flag
extra = kwargs.pop("extra", {})
extra.update({"researcher_id": researcher_id, "broadcast": broadcast})
# Ensure caller attribution points to the real call site, not this wrapper.
# Users may override by explicitly passing stacklevel=...
stacklevel = kwargs.pop("stacklevel", 2)
self._logger.info(
msg,
*args,
**kwargs,
extra=extra,
stacklevel=stacklevel,
)
log
log(level, msg)
Overrides the logging.log() method to allow the use of string instead of a logging.* level
Source code in fedbiomed/common/logger.py
def log(self, level: Any, msg: str):
"""Overrides the logging.log() method to allow the use of string instead of a logging.* level"""
level = logger._internal_level_translator(level)
self._logger.log(level, msg)
security_context
security_context(**ctx)
Bind security context for downstream logging.
How reset works: - SECURITY_CONTEXT.set(new_dict) returns a token referencing the previous value. - In the finally block we call SECURITY_CONTEXT.reset(token) so the old context is restored even if exceptions occur (prevents context leaks).
Source code in fedbiomed/common/logger.py
@contextmanager
def security_context(self, **ctx: Any):
"""
Bind security context for downstream logging.
How reset works:
- SECURITY_CONTEXT.set(new_dict) returns a token referencing the previous value.
- In the `finally` block we call SECURITY_CONTEXT.reset(token) so the old context
is restored even if exceptions occur (prevents context leaks).
"""
current = dict(
SECURITY_CONTEXT.get() or {}
) # copy to avoid mutation across calls
current.update({k: v for k, v in ctx.items() if v is not None})
token = SECURITY_CONTEXT.set(current)
try:
yield
finally:
SECURITY_CONTEXT.reset(token)
security_event
security_event(*, operation=None, status=None, researcher_id=None, stacklevel=1, **fields)
Writes one JSON security/audit log line to security file only.
Always present
- node_id, researcher_id, timestamp, operation, status, fedbiomed_version
- caller_function, caller_module, caller_file, caller_line (automatically captured)
Values resolved as
- explicit args > bound SECURITY_CONTEXT > defaults (for node_id/version)
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
operation | Optional[str] | Name of the operation being logged (e.g., 'dataset_search', 'training_execute') | None |
status | Optional[str] | Status of the operation (e.g., 'success', 'failure', 'pending') | None |
researcher_id | Optional[str] | ID of the researcher performing the operation | None |
stacklevel | int | How many frames to skip when attributing the caller. Use higher values when calling | 1 |
**fields | Any | Additional fields to log (e.g., dataset_id, experiment_id, node_id) | {} |
Source code in fedbiomed/common/logger.py
def security_event(
self,
*,
operation: Optional[str] = None,
status: Optional[str] = None,
researcher_id: Optional[str] = None,
stacklevel: int = 1,
**fields: Any,
) -> None:
"""
Writes one JSON security/audit log line to security file only.
Always present:
- node_id, researcher_id, timestamp, operation, status, fedbiomed_version
- caller_function, caller_module, caller_file, caller_line (automatically captured)
Values resolved as:
- explicit args > bound SECURITY_CONTEXT > defaults (for node_id/version)
Args:
operation: Name of the operation being logged (e.g., 'dataset_search', 'training_execute')
status: Status of the operation (e.g., 'success', 'failure', 'pending')
researcher_id: ID of the researcher performing the operation
stacklevel: How many frames to skip when attributing the caller.
Use higher values when calling `security_event` from within wrappers.
**fields: Additional fields to log (e.g., dataset_id, experiment_id, node_id)
"""
ctx = dict(SECURITY_CONTEXT.get() or {})
op = operation or ctx.get("operation")
st = status or ctx.get("status")
rid = researcher_id or ctx.get("researcher_id")
# Capture caller information from the stack.
# `stacklevel` follows Python logging semantics: stacklevel=2 points to the
# caller of this wrapper method.
try:
frame = inspect.currentframe()
caller_frame = frame
steps = max(int(stacklevel), 0)
for _ in range(steps):
caller_frame = caller_frame.f_back if caller_frame else None
caller_info = {
"caller_function": caller_frame.f_code.co_name
if caller_frame
else "unknown",
"caller_module": os.path.basename(caller_frame.f_code.co_filename)
if caller_frame
else "unknown",
"caller_file": caller_frame.f_code.co_filename
if caller_frame
else "unknown",
"caller_line": caller_frame.f_lineno if caller_frame else 0,
}
except Exception as e:
FedbiomedError(
"Failed to capture caller information for security log entry. "
"Caller info will be set to 'unknown'. "
"Caller parser error: " + str(e)
)
caller_info = {
"caller_function": "unknown",
"caller_module": "unknown",
"caller_file": "unknown",
"caller_line": 0,
}
self._logger.warning(
json.dumps(
{
"timestamp": _utc_timestamp(),
"node_id": self._security_defaults.get("component_id"),
"caller_parsing": "failure",
"caller_parser_error": str(e),
},
default=str,
separators=(",", ":"),
),
extra={"is_security": True},
)
entry = {
"timestamp": _utc_timestamp(),
"component_id": self._security_defaults.get("component_id"),
"component_name": self._security_defaults.get("component_name"),
"researcher_id": rid,
"operation": op,
"status": st,
"fedbiomed_version": self._security_defaults.get("fedbiomed_version"),
**caller_info,
}
# Merge extra fields: context first, then explicit fields override
merged = {}
merged.update(ctx)
merged.update(fields)
# Avoid duplicating required keys in payload
for k in ("operation", "status", "researcher_id"):
merged.pop(k, None)
entry.update(merged)
# Write only to security file (is_security=True filters out other handlers)
self._logger.info(
json.dumps(entry, default=str, separators=(",", ":")),
extra={"is_security": True},
stacklevel=stacklevel,
)
setLevel
setLevel(level, htype=None)
Overrides the setLevel method, to deal with level given as a string and to change le level of one or all known handlers
This also change the default level for all future handlers.
Remark
Level should not be lower than CRITICAL (meaning CRITICAL errors are always displayed)
Example:
setLevel( logging.DEBUG, 'FILE')
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
level | level to modify, can be a string or a logging.* level (mandatory) | required | |
htype | if provided (non-mandatory), change the level of the given handler. if not provided (or None), change the level of all known handlers | required |
Source code in fedbiomed/common/logger.py
def setLevel(self, level: Any, htype: Any = None):
"""Overrides the setLevel method, to deal with level given as a string and to change le level of
one or all known handlers
This also change the default level for all future handlers.
!!! info "Remark"
Level should not be lower than CRITICAL (meaning CRITICAL errors are always displayed)
Example:
```python
setLevel( logging.DEBUG, 'FILE')
```
Args:
level : level to modify, can be a string or a logging.* level (mandatory)
htype : if provided (non-mandatory), change the level of the given handler. if not provided (or None),
change the level of all known handlers
"""
level = self._internal_level_translator(level)
if htype is None:
# store this level (for future handler adding)
self._logger.setLevel(level)
for h in self._handlers:
self._handlers[h].setLevel(level)
self._set_handler_formatter(h)
return
if htype in self._handlers:
self._handlers[htype].setLevel(level)
self._set_handler_formatter(htype)
return
# htype provided but no handler for this type exists
self._logger.warning(htype + " handler not initialized yet")
setPrefix
setPrefix(prefix='')
Sets a log prefix for all handlers.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
prefix | str | Prefix to add to all log messages | '' |
Source code in fedbiomed/common/logger.py
def setPrefix(self, prefix: str = "") -> None:
"""Sets a log prefix for all handlers.
Args:
prefix: Prefix to add to all log messages
"""
for h in self._handlers:
self._handler_prefix[h] = prefix
self._set_handler_formatter(h)
set_security_logs
set_security_logs(root_path)
Source code in fedbiomed/common/logger.py
def set_security_logs(self, root_path: str) -> None:
security_log_dir = os.path.join(root_path, "log")
os.makedirs(security_log_dir, exist_ok=True)
security_log_path = os.path.join(security_log_dir, "security_audit.log")
self.add_security_file_handler(filename=security_log_path)
warning
warning(msg, *args, broadcast=False, researcher_id=None, **kwargs)
Same as info message
Source code in fedbiomed/common/logger.py
def warning(self, msg, *args, broadcast=False, researcher_id=None, **kwargs):
"""Same as info message"""
# Merge extra dicts properly to support is_security flag
extra = kwargs.pop("extra", {})
extra.update({"researcher_id": researcher_id, "broadcast": broadcast})
stacklevel = kwargs.pop("stacklevel", 2)
self._logger.warning(
msg,
*args,
**kwargs,
extra=extra,
stacklevel=stacklevel,
)