Compare commits

..

3 Commits

3 changed files with 152 additions and 4 deletions
+3
View File
@@ -12,3 +12,6 @@ docs/resultats.json
.idea/ .idea/
.coverage .coverage
.env .env
.qwen
.ruff_cache
.pytest_cache
+40 -2
View File
@@ -48,24 +48,62 @@ def _match_n2_pattern(log_entry: LogEntry) -> str | None:
return None return None
def _error_identity(log_entry: LogEntry) -> str:
"""Return the error code of a log entry, or its raw message if it has none."""
return log_entry.error_name or log_entry.error_message
def _is_duplicate_error(previous: LogEntry, current: LogEntry) -> bool:
"""Return True when two consecutive entries describe the same error.
Two entries are the same error when they come from the same scan (same server,
MDC server, store and start time) and share the same error identity. Their error
times are ignored on purpose: a single scan writes the same error once per
occurrence, seconds apart.
"""
return (
previous.server_ip == current.server_ip
and previous.mdc_server_name == current.mdc_server_name
and previous.store_name == current.store_name
and previous.start_time == current.start_time
and _error_identity(previous) == _error_identity(current)
)
def classify_log_entries( def classify_log_entries(
log_entries: list[LogEntry], log_entries: list[LogEntry],
) -> tuple[list[LogEntry], list[LogEntry]]: ) -> tuple[list[LogEntry], list[LogEntry]]:
"""Separate logs that require N2 intervention from those that don't.""" """Separate logs that require N2 intervention from those that don't.
Consecutive entries describing the same error are merged into a single entry
(the first one) so the same error is not repeated in the reports.
"""
relevant_log_entries = [] relevant_log_entries = []
irrelevant_log_entries = [] irrelevant_log_entries = []
previous_log_entry: LogEntry | None = None
logger.info("\tclassification job started.") logger.info("\tclassification job started.")
for log_entry in log_entries: for log_entry in log_entries:
error_name = _match_n2_pattern(log_entry) error_name = _match_n2_pattern(log_entry)
if error_name: if error_name:
log_entry.error_name = error_name
if previous_log_entry is not None and _is_duplicate_error(
previous_log_entry, log_entry
):
logger.info(
f"\t\tDuplicated log entry ignored: '{log_entry.error_message}'"
)
continue
if log_entry.error_name:
logger.info( logger.info(
f"\t\tError '{error_name}' associated with log entry '{log_entry.error_message}'. This is a n2 log entry" f"\t\tError '{error_name}' associated with log entry '{log_entry.error_message}'. This is a n2 log entry"
) )
log_entry.error_name = error_name
relevant_log_entries.append(log_entry) relevant_log_entries.append(log_entry)
else: else:
irrelevant_log_entries.append(log_entry) irrelevant_log_entries.append(log_entry)
previous_log_entry = log_entry
logger.info( logger.info(
f"\t\tn2 logs: {len(relevant_log_entries)}, other logs: {len(irrelevant_log_entries)}" f"\t\tn2 logs: {len(relevant_log_entries)}, other logs: {len(irrelevant_log_entries)}"
+108 -1
View File
@@ -6,7 +6,7 @@ import pytest
from logwatcher.classifier import N2_PATTERNS, classify_log_entries from logwatcher.classifier import N2_PATTERNS, classify_log_entries
from logwatcher.config import FIXTURE_PATH from logwatcher.config import FIXTURE_PATH
from logwatcher.parser import parse_file from logwatcher.parser import parse_file, parse_lines
def _pattern_name_to_filename(name: str) -> str: def _pattern_name_to_filename(name: str) -> str:
@@ -118,3 +118,110 @@ def test_classify_log_entries_empty_log_entries(invalid_log_dir: Path):
relevant, irrelevant = classify_log_entries(log_entries) relevant, irrelevant = classify_log_entries(log_entries)
assert not relevant and not irrelevant assert not relevant and not irrelevant
N2_ERROR_MESSAGE = "Erreur : GetTvaId -> TVA non trouvée : 20,00"
OTHER_ERROR_MESSAGE = "Erreur FTP SDUpdatePrixISF : Requested action not taken"
def _build_log_line(
error_time: str,
error_message: str,
store_name: str = "GABRIEL",
start_time: str = "22/09/2026 09:28:58",
mdc_server_name: str = "MDC_220",
server_ip: str = "192.168.13.22",
) -> str:
"""Build a raw log line as written by a MDC scan."""
return (
f"\\\\{server_ip}\\e\\{mdc_server_name}\\Logs\\26\\09\\22\\20260922092847.txt "
f"[{start_time}] DOSSIER EN COURS : {store_name} "
f"[{error_time}] {error_message}"
)
def test_classify_log_entries_merges_consecutive_duplicates():
"""Consecutive identical errors become a single log entry."""
log_entries = parse_lines(
[
_build_log_line("22/09/2026 09:32:19", N2_ERROR_MESSAGE),
_build_log_line("22/09/2026 09:32:20", N2_ERROR_MESSAGE),
_build_log_line("22/09/2026 09:32:25", N2_ERROR_MESSAGE),
]
)
relevant, irrelevant = classify_log_entries(log_entries)
assert len(relevant) == 1
assert relevant[0].get_error_time() == "22/09/2026 09:32:19"
assert not irrelevant
def test_classify_log_entries_merges_same_error_name():
"""Same error code with a different amount is still a single log entry."""
log_entries = parse_lines(
[
_build_log_line(
"22/09/2026 09:32:19", "Erreur : GetTvaId -> TVA non trouvée : 8,50"
),
_build_log_line(
"22/09/2026 09:32:20", "Erreur : GetTvaId -> TVA non trouvée : 20,00"
),
]
)
relevant, _ = classify_log_entries(log_entries)
assert len(relevant) == 1
def test_classify_log_entries_keeps_duplicates_from_other_store_or_scan():
"""Same error from another scan or another store is not a duplicate."""
log_entries = parse_lines(
[
_build_log_line("22/09/2026 09:32:19", N2_ERROR_MESSAGE),
_build_log_line(
"22/09/2026 10:29:10",
N2_ERROR_MESSAGE,
start_time="22/09/2026 10:28:58",
),
_build_log_line(
"22/09/2026 09:32:19", N2_ERROR_MESSAGE, store_name="DUTOUR"
),
]
)
relevant, _ = classify_log_entries(log_entries)
assert len(relevant) == 3
def test_classify_log_entries_merges_consecutive_irrelevant_duplicates():
"""Deduplication also applies to errors that do not require N2 support."""
log_entries = parse_lines(
[
_build_log_line("22/09/2026 09:32:19", OTHER_ERROR_MESSAGE),
_build_log_line("22/09/2026 09:32:24", OTHER_ERROR_MESSAGE),
]
)
relevant, irrelevant = classify_log_entries(log_entries)
assert not relevant
assert len(irrelevant) == 1
def test_classify_log_entries_keeps_same_error_separated_by_another_error():
"""An interleaved error breaks the run, so both errors are kept."""
log_entries = parse_lines(
[
_build_log_line("22/09/2026 09:32:19", N2_ERROR_MESSAGE),
_build_log_line("22/09/2026 09:32:20", OTHER_ERROR_MESSAGE),
_build_log_line("22/09/2026 09:32:21", N2_ERROR_MESSAGE),
]
)
relevant, irrelevant = classify_log_entries(log_entries)
assert len(relevant) == 2
assert len(irrelevant) == 1