10 Commits

Author SHA1 Message Date
maurane f7495608fd feat(cli): ✨ add shorter options in cli
-o for --output-dir, -f for --input-files
2026-09-28 10:51:47 +02:00
maurane e50797bba6 test(cli): ✅ add tests for prune command
add tests for prune command and some code optimization and formatting.
2026-09-25 17:59:27 +02:00
maurane 48503cdd42 test(cleaner): ✅ add tests for cleaner module
remove also comments in `test_mail_reader.py` that didn't follow package conventions
2026-09-25 17:18:09 +02:00
maurane 02e85ddbfd feat(cleaner): ✨ add command prune
this new command will be called by cron-like process to clean purge old logs and old mails. All files were formatted and mail_utils was renamed in utils since it now defines utility function not only for mail functions.
2026-09-25 16:30:45 +02:00
maurane 63c4d1387e refactor(classifier): 🎨 readapt code to its intial structure
move the test on the error_name value as it was before the fix of deduplications
2026-09-23 16:17:36 +02:00
maurane 3ddefd24e0 fix(classifier): 🚑️ fix classifying criteria
two log entries that are consecutive and do have different error message are two different errors.
2026-09-23 16:07:09 +02:00
maurane 20ce3471b6 style(classifier): 🎨 improve if statements 2026-09-22 17:58:29 +02:00
maurane 48181c97b8 fix(classifier): 🩹 avoid having multiple times the same line 2026-09-22 17:46:57 +02:00
maurane 369e7f17c6 chore(project): 🙈 ignore cache and ai agent files 2026-09-21 16:13:22 +02:00
maurane f4ef4c11d4 chore(project): 🔖 update CHANGELOG for v0.1.0
close #72
2026-09-18 11:49:02 +02:00
26 changed files with 1009 additions and 207 deletions
+12
View File
@@ -8,3 +8,15 @@ EWS_URL=https://mail.client.fr/EWS/Exchange.asmx # Default mail server
## Report email
N2_REPORT_RECIPIENTS=n2-support@client.fr
N2_REPORT_SUBJECT=[Logwatcher] Rapport erreurs N2
## Log retention
### number of days before archiving logs
LOG_RETENTION_DAYS=30
### number of days before deleting archive logs
ARCHIVED_LOG_RETENTION_DAYS=60
## Mail retention
### number of days before deleting mails
MAIL_RETENTION_DAYS=30
+3
View File
@@ -12,3 +12,6 @@ docs/resultats.json
.idea/
.coverage
.env
.qwen
.ruff_cache
.pytest_cache
+22 -6
View File
@@ -1,10 +1,26 @@
# Changelog
## [0.0.1] - 2026-09-02
Toutes les modifications notables de ce projet sont documentées dans ce fichier.
Le format est basé sur [Keep a Changelog](https://keepachangelog.com/fr/1.1.0/),
et ce projet adhère au [Semantic Versioning](https://semver.org/lang/fr/).
## [0.1.0] - 2026-09-18
### Added
- Pipeline complet : parsing, classification N2, génération de rapports
- CLI avec options `--input-files`, `--input-dir`, `--output-dir`, `--version`
- Logging applicatif (jalons, compteurs)
- Tests unitaires et d'intégration (~95% de couverture)
- Documentation (README, usage, architecture)
- Commande CLI `from-files` : parsing, classification et génération des 3 rapports (`n2.log`, `other.log`, `all.log`) en encodage Windows-1252.
- Module `mail_reader` : récupération et filtrage des mails de logs via EWS (`exchangelib`).
- Module `notifier` : envoi du rapport N2 par mail avec gestion du déplacement vers le dossier `Logs/Sent`.
- Dictionnaire `N2_PATTERNS` : regex nommées pour la classification des erreurs (SizeRange, BL AUTO, Fedas, XML invalide, deadlock, TVA, etc.).
- Configuration via `.env` pour les variables sensibles (EWS, destinataires, sujets).
### Changed
- Mise à jour de la documentation utilisateur et technique (`usage.md`, `architecture.md`).
- Nettoyage des patterns : retrait de l'erreur `STORE_CODE_NOT_FOUND` et de ses fixtures/tests associés.
### Testing
- ~95% de couverture de tests.
- Mise en place de mocks `unittest.mock` sur les objets `Account` pour éviter les appels EWS réels pendant les tests.
### Security
- Centralisation prévue des accès aux variables d'environnement (dette technique identifiée pour refactor futur).
+41 -4
View File
@@ -48,27 +48,64 @@ def _match_n2_pattern(log_entry: LogEntry) -> str | None:
return None
def _error_identity(log_entry: LogEntry) -> str:
"""Return the error code of a log entry, or its raw message if it has none."""
return log_entry.error_name or log_entry.error_message
def _is_duplicate_error(previous: LogEntry, current: LogEntry) -> bool:
"""Return True when two consecutive entries describe the same error.
Two entries are the same error when they come from the same scan (same server,
MDC server, store and start time) and share the same error identity. Their error
times are ignored on purpose: a single scan writes the same error once per
occurrence, seconds apart.
"""
return (
previous.server_ip == current.server_ip
and previous.mdc_server_name == current.mdc_server_name
and previous.store_name == current.store_name
and previous.start_time == current.start_time
and previous.error_message == current.error_message
)
def classify_log_entries(
log_entries: list[LogEntry],
) -> tuple[list[LogEntry], list[LogEntry]]:
"""Separate logs that require N2 intervention from those that don't."""
"""Separate logs that require N2 intervention from those that don't.
Consecutive entries describing the same error are merged into a single entry
(the first one) so the same error is not repeated in the reports.
"""
relevant_log_entries = []
irrelevant_log_entries = []
previous_log_entry: LogEntry | None = None
logger.info("\tclassification job started.")
logger.info("classification job started.")
for log_entry in log_entries:
error_name = _match_n2_pattern(log_entry)
if previous_log_entry is not None and _is_duplicate_error(
previous_log_entry, log_entry
):
logger.info(
f"\t\tDuplicated log entry ignored: '{log_entry.error_message}'"
)
continue
if error_name:
log_entry.error_name = error_name
logger.info(
f"\t\tError '{error_name}' associated with log entry '{log_entry.error_message}'. This is a n2 log entry"
)
log_entry.error_name = error_name
relevant_log_entries.append(log_entry)
else:
irrelevant_log_entries.append(log_entry)
previous_log_entry = log_entry
logger.info(
f"\t\tn2 logs: {len(relevant_log_entries)}, other logs: {len(irrelevant_log_entries)}"
)
logger.info("\tclassification job completed.")
logger.info("classification job completed.")
return relevant_log_entries, irrelevant_log_entries
+153
View File
@@ -0,0 +1,153 @@
import datetime
import logging
import os
from pathlib import Path
from zipfile import ZipFile
from exchangelib import Account
from logwatcher.config import (
ARCHIVE_DIR,
DATE_FILE_FORMAT,
FRENCH_TIMEZONE,
LOGGING_PATH,
)
from logwatcher.utils import ANALYZED_FOLDER, LOG_FOLDER, SENT_FOLDER, is_log_file
logger = logging.getLogger(__name__)
def _get_retention_days(key: str) -> int:
"""Get specific environment variable defined by a number of days.
Args:
key: name of the environment variable
Raises:
KeyError
ValueError
Returns:
number of days found in environment variable `key`.
"""
try:
value = int(os.environ[key])
except KeyError:
logger.error(f"'{key}' is not defined in environment variables.")
raise
except ValueError:
logger.error(f"{os.environ[key]} must be an integer.")
raise
return value
def get_logs_to_archive(today: datetime.datetime) -> list[Path]:
"""Get all logs that are older than `LOG_RETENTION_DAYS`.
Args:
today: date and time of the current day
Returns:
A list of log file to archive.
"""
nb_days = _get_retention_days("LOG_RETENTION_DAYS")
logs_to_archive = []
cutoff_date = today - datetime.timedelta(days=nb_days)
for log_file in LOGGING_PATH.iterdir():
if is_log_file(log_file):
log_file_date = datetime.datetime.fromtimestamp(
log_file.stat().st_mtime, tz=FRENCH_TIMEZONE
)
if log_file_date < cutoff_date:
logs_to_archive.append(log_file)
if not logs_to_archive:
logger.warning("No log files old enough to be archived.")
return logs_to_archive
def archive_logs(today: datetime.datetime, log_files: list[Path]) -> None:
"""Archive old logs.
Due to its file name, the newly created archive could overwrite the old one
if the function is called at the same seconds.
Do nothing if `log_files` is empty.
Args:
today: date and time of the current day
log_files: log file to archive.
"""
if not log_files:
logger.info("No file to archive. Skip.")
return
ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
archive_file_name = f"archive_{today.strftime(DATE_FILE_FORMAT)}.zip"
with ZipFile(ARCHIVE_DIR / archive_file_name, "w") as archive_file:
for log_file in log_files:
logger.debug(f"Add '{log_file}' to archive.")
archive_file.write(log_file, arcname=log_file.name)
logger.debug(f"Delete '{log_file}'")
log_file.unlink()
def prune_logs(today: datetime.datetime) -> None:
"""Detect and delete old log archives.
Args:
today: date and time of the current day
"""
nb_days = _get_retention_days("ARCHIVED_LOG_RETENTION_DAYS")
cutoff_date = today - datetime.timedelta(days=nb_days)
ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
archive_files = list(ARCHIVE_DIR.iterdir())
if not archive_files:
logger.warning("No archive found.")
return
for archive_file in archive_files:
archive_date = datetime.datetime.fromtimestamp(
archive_file.stat().st_mtime, tz=FRENCH_TIMEZONE
)
if archive_date < cutoff_date:
archive_file.unlink()
def prune_mails(today: datetime.datetime, account: Account) -> None:
"""Get and delete all old mails from `ANALYZED_FOLDER` and `SENT_FOLDER`.
Args:
today: date and time of the current day
account: EWS account that owns and manage log mails
"""
analyzed_folder = account.msg_folder_root / LOG_FOLDER / ANALYZED_FOLDER
sent_folder = account.msg_folder_root / LOG_FOLDER / SENT_FOLDER
nb_days = _get_retention_days("MAIL_RETENTION_DAYS")
cutoff_date = today - datetime.timedelta(days=nb_days)
analyzed_mails = analyzed_folder.filter(datetime_sent__lt=cutoff_date)
sent_mails = sent_folder.filter(datetime_sent__lt=cutoff_date)
mails = list(analyzed_mails) + list(sent_mails)
if not mails:
logger.warning(f"No mail older than '{nb_days}' days found.")
return
ids = [(m.id, m.changekey) for m in mails]
account.bulk_delete(ids=ids)
+95 -37
View File
@@ -7,7 +7,20 @@ import typer
from logwatcher import __version__
from logwatcher.classifier import classify_log_entries
from logwatcher.config import DATETIME_FORMAT, FRENCH_TIMEZONE, SourceType
from logwatcher.cleaner import (
get_logs_to_archive,
archive_logs,
prune_logs,
prune_mails,
)
from logwatcher.config import (
DATE_FILE_FORMAT,
DATETIME_FORMAT,
FRENCH_TIMEZONE,
LOGGING_PATH,
RESULT_PATH,
SourceType,
)
from logwatcher.logging_config import setup_logging
from logwatcher.mail_reader import (
connect_to_mailbox,
@@ -19,6 +32,7 @@ from logwatcher.models import LogEntry
from logwatcher.notifier import send_n2_report
from logwatcher.parser import parse_file, parse_lines
from logwatcher.reporter import build_mail_summary, get_period, write_log_report
from logwatcher.utils import is_log_file
app = typer.Typer(help="Logwatcher - Analysis and sorting of LAME MDC logs.")
@@ -26,41 +40,46 @@ app = typer.Typer(help="Logwatcher - Analysis and sorting of LAME MDC logs.")
logger = logging.getLogger("logwatcher")
def _is_log_file(path: Path) -> bool:
"""Return True if file is a log file (.txt, .log, or extensionless).
def _get_files_from_dir(input_dir: Path) -> list[Path]:
"""Retrieve all log files from the `input_dir` directory.
Args:
path: Path of a file
input_dir: directory that contains log files
Returns:
True if file has .txt or .log extension or extensionless
False otherwise.
list of log files
"""
if not path.is_file():
return False
return path.suffix in {"", ".txt", ".log"}
def _get_files_from_dir(input_dir: Path) -> list[Path]:
if not input_dir.exists():
logger.error(f"Unrecognized input directory '{input_dir}'.")
raise FileNotFoundError(f"No such directory at '{input_dir}'")
return [file for file in input_dir.iterdir() if _is_log_file(file)]
return [
current_file
for current_file in input_dir.iterdir()
if is_log_file(current_file)
]
def _files_to_log_entries(input_files: list[Path]) -> list[LogEntry]:
"""Parse each input file, raising if any path is not a file."""
"""Parse each input file, raising if any path is not a file.
Args:
input_files: list of log files
Returns:
list of log entries
"""
log_entries = []
for file in input_files:
if not file.is_file():
logger.error(f"Unrecognized input file '{file}'.")
raise ValueError(f"'{file}' is not a file")
log_entries.extend(parse_file(file))
for current_file in input_files:
if not current_file.is_file():
logger.error(f"Unrecognized input file '{current_file}'.")
raise ValueError(f"'{current_file}' is not a file")
log_entries.extend(parse_file(current_file))
return log_entries
def version_callback(value: bool):
def version_callback(value: bool) -> None:
"""Print the version of package.
Stop the CLI after printing version.
@@ -81,15 +100,15 @@ def from_mails(
typer.Option(
"--output-dir", help="Output directories where reports will be stored."
),
] = Path("output"),
):
] = RESULT_PATH,
) -> None:
"""Convert mail containing logs into reports for N2 technicians.
Connect to the EWS server with service account, filter most recent mail logs,
generate reports based on these mails and send mails to the technicians.
Args:
output_dir: Path to the generated reports folder
output_dir: Output directories where reports will be stored.
"""
logger.info("============ logwatcher mail started ============")
@@ -100,7 +119,7 @@ def from_mails(
nb_mails = len(messages)
relevant, irrelevant = classify_log_entries(log_entries)
today = datetime.now(FRENCH_TIMEZONE).strftime(DATETIME_FORMAT)
today = datetime.now(tz=FRENCH_TIMEZONE).strftime(DATETIME_FORMAT)
start_date, end_date = get_period(relevant + irrelevant)
write_log_report(
@@ -134,17 +153,17 @@ def from_files(
output_dir: Annotated[
Path,
typer.Option(
"--output-dir", help="Output directories where reports will be stored."
"--output-dir", "-o", help="Output directories where reports will be stored."
),
] = Path("output"),
] = RESULT_PATH,
input_files: Annotated[
list[Path] | None, typer.Option("--input-files", help="Log files to scan.")
list[Path] | None, typer.Option("--input-files", "-f", help="Log files to scan.")
] = None,
input_dir: Annotated[
Path | None,
typer.Option("--input-dir", help="Directories containing log files."),
typer.Option("--input-dir", "-d", help="Directories containing log files."),
] = None,
):
) -> None:
"""Convert log files into two reports, one of n2 related logs, one without any n2 related logs.
These reports are sent by mail to n2 support technicians.
@@ -153,11 +172,7 @@ def from_files(
Args:
input_files: Log files to scan. Either input_files or input_dir must be provided.
input_dir: Directories containing log files. Either input_files or input_dir must be provided.
output_dir: Required. Output directories where reports will be stored.
version: Option that returns the version of package
Returns:
None
output_dir: Output directories where reports will be stored.
"""
logger.info("============ logwatcher file started ============")
@@ -176,11 +191,11 @@ def from_files(
logger.error("Neither input-files nor input-dir were provided.")
raise typer.BadParameter("Either input_files or input_dir must be provided")
logger.info(f"reports will be located at '{output_dir.cwd()}'")
logger.info(f"reports will be located at '{output_dir.resolve()}'")
log_entries = _files_to_log_entries(log_files)
relevant, irrelevant = classify_log_entries(log_entries)
today = datetime.now(FRENCH_TIMEZONE).strftime(DATETIME_FORMAT)
today = datetime.now(tz=FRENCH_TIMEZONE).strftime(DATETIME_FORMAT)
start_date, end_date = get_period(relevant + irrelevant)
write_log_report(
source_type=SourceType.FILE,
@@ -195,6 +210,47 @@ def from_files(
logger.info("============ logwatcher file ended ============")
@app.command()
def prune(
mails: Annotated[
bool,
typer.Option("-m", "--mails", help="Prune old mails."),
] = False,
logs: Annotated[
bool,
typer.Option("-l", "--logs", help="Prune old logs."),
] = False,
) -> None:
"""Purge mails and logs based on the value of environment variables."""
if not mails and not logs:
raise typer.BadParameter("At least one of --mails or --logs must be provided.")
logger.info("============ logwatcher prune started ============")
today = datetime.now(tz=FRENCH_TIMEZONE)
if mails:
logger.debug("Mails selected to be pruned")
account = connect_to_mailbox()
logger.info("Start pruning mails.")
prune_mails(today, account)
logger.info("Pruning mails done.")
else:
logger.debug("No mails were purged.")
if logs:
logger.debug("Logs selected to be pruned")
logger.info("Start pruning logs.")
log_files = get_logs_to_archive(today)
if log_files:
archive_logs(today, log_files)
prune_logs(today)
logger.info("Pruning logs done.")
else:
logger.debug("No logs were purged.")
logger.info("============ logwatcher prune ended ============")
@app.callback()
def cli(
version: Annotated[
@@ -222,7 +278,9 @@ def cli(
version: if provided, script returns the version of package
"""
setup_logging(verbose=verbose)
log_file_name = datetime.now(FRENCH_TIMEZONE).strftime(DATE_FILE_FORMAT) + ".log"
log_file = LOGGING_PATH / log_file_name # date and time of execution
setup_logging(verbose=verbose, file_path=log_file)
if __name__ == "__main__":
+2
View File
@@ -7,12 +7,14 @@ from pathlib import Path
OUTPUT_PATH = Path("output")
RESULT_PATH = OUTPUT_PATH / "results"
LOGGING_PATH = OUTPUT_PATH / "logs"
ARCHIVE_DIR = LOGGING_PATH / "archives"
# test paths
TEST_PATH = Path("tests")
FIXTURE_PATH = TEST_PATH / "fixtures"
# date and time
DATE_FILE_FORMAT = "%d%m%Y_%H%M%S"
DATETIME_FORMAT = "%d/%m/%Y %H:%M:%S"
FRENCH_TIMEZONE = ZoneInfo("Europe/Paris")
+5 -3
View File
@@ -7,13 +7,13 @@ from logwatcher.config import DATETIME_FORMAT
def _setup_formatter(
format: str = "[%(asctime)s] - %(levelname)s: %(message)s",
datefmt: str = DATETIME_FORMAT,
) -> logging.Formatter:
) -> logging.Formatter: # TODO: Add documentation
return logging.Formatter(fmt=format, datefmt=datefmt)
def _setup_handler(
formatter: logging.Formatter, level: int, filename: Path | None = None
) -> logging.Handler:
) -> logging.Handler: # TODO: Add documentation
handler: logging.Handler
if filename:
handler = logging.FileHandler(filename=filename, encoding="utf-8")
@@ -32,8 +32,10 @@ def setup_logging(
"""Create a new instance of Logger named 'logger' customized for the logwatcher package.
The format display in order the time, the level of log and the log message
The logs are only redirect in the standard output.
The logs are only redirect in the standard output. # TODO Complete docstring
"""
file_path.parent.mkdir(parents=True, exist_ok=True)
_formatter = formatter if formatter else _setup_formatter()
if verbose:
+9 -8
View File
@@ -14,7 +14,12 @@ from exchangelib import (
)
from exchangelib.errors import UnauthorizedError
from logwatcher.mail_utils import ANALYZED_FOLDER, LOG_IN_ATTACHMENT_PATTERN, get_or_create_folder
from logwatcher.utils import (
ANALYZED_FOLDER,
LOG_FOLDER,
LOG_IN_ATTACHMENT_PATTERN,
get_or_create_folder,
)
logger = logging.getLogger(__name__)
@@ -135,7 +140,7 @@ def fetch_log_messages(account: Account) -> list[Message]:
queryset of log-related mails
"""
logs_folder = account.msg_folder_root / "Logs"
logs_folder = account.msg_folder_root / LOG_FOLDER
return list(logs_folder.all())
@@ -192,15 +197,11 @@ def move_analyzed_mails(account: Account, messages: list[Message]) -> None:
logger.info("No mails to move.")
return
logger.info(
f"Moving {len(messages)} mails to '{ANALYZED_FOLDER}'"
)
logger.info(f"Moving {len(messages)} mails to '{ANALYZED_FOLDER}'")
analyzed_folder = get_or_create_folder(account, ANALYZED_FOLDER)
ids = [(m.id, m.changekey) for m in messages]
account.bulk_move(ids=ids, to_folder=analyzed_folder)
logger.info(
f"Mails were successfully moved to '{ANALYZED_FOLDER}'."
)
logger.info(f"Mails were successfully moved to '{ANALYZED_FOLDER}'.")
+6 -2
View File
@@ -24,11 +24,15 @@ class LogEntry:
"""Return raw line of the log entry."""
return self.raw_line
def get_start_time(self) -> str: # TODO: remove or adapt mentions of start_time in code base
def get_start_time(
self,
) -> str: # TODO: remove or adapt mentions of start_time in code base
"""Return formatted start time."""
return self.start_time.strftime(DATETIME_FORMAT)
def get_error_time(self) -> str: # TODO: remove or adapt mentions of error_time in code base
def get_error_time(
self,
) -> str: # TODO: remove or adapt mentions of error_time in code base
"""Return formatted error time."""
return self.error_time.strftime(DATETIME_FORMAT)
+11 -5
View File
@@ -4,7 +4,7 @@ from pathlib import Path
from exchangelib import Account, FileAttachment, Message
from logwatcher.mail_utils import get_or_create_folder
from logwatcher.utils import SENT_FOLDER, get_or_create_folder
logger = logging.getLogger(__name__)
@@ -27,17 +27,23 @@ def send_n2_report(account: Account, summary: str, n2_log_file: Path) -> None:
# build file attachment
logger.info(f"Create attachment based on '{n2_log_file}' file.")
n2_log_attachment = FileAttachment(name=n2_log_file.name, content=n2_log_file.read_bytes(), is_inline=False)
n2_log_attachment = FileAttachment(
name=n2_log_file.name, content=n2_log_file.read_bytes(), is_inline=False
)
logger.info(f"Attachment '{n2_log_attachment.name}' was successfully created.")
# get sent folder
sent_folder = get_or_create_folder(account, "Sent")
sent_folder = get_or_create_folder(account, SENT_FOLDER)
# create mail
logger.info(f"Create mail with account='{account.fullname}', recipients='{to}', subject='{subject}'.")
logger.info(
f"Create mail with account='{account.fullname}', recipients='{to}', subject='{subject}'."
)
mail = Message(account=account, to_recipients=to, subject=subject, body=summary)
mail.attach(n2_log_attachment)
logger.info(f"Mail '{mail.subject}' was successfully created with attachment '{n2_log_attachment.name}'.")
logger.info(
f"Mail '{mail.subject}' was successfully created with attachment '{n2_log_attachment.name}'."
)
# send and keep a copy to the Logs/Sent folder
logger.info(f"Attempt to send mail '{mail.subject}'...")
+1 -1
View File
@@ -99,5 +99,5 @@ def parse_file(log_file_path: Path) -> list[LogEntry]:
logger.info(f"\tparsing log file '{log_file_path.name}' started.")
log_entries = parse_lines(log_file)
logger.info(f"\t\t{len(log_entries)} lines were parsed.")
logger.info("\tparsing completed.")
logger.info("parsing completed.")
return log_entries
+19 -20
View File
@@ -48,7 +48,7 @@ ERROR_TEMPLATE = Template("""
MAIL_TEMPLATE = Template("""
Rapport analyse de log - $generation_date
Nb de mail(s) analysé(s)\t : $nb_mail
Nb de mail(s) analysé(s)\t : $nb_mails
Erreurs N2/Erreurs totales\t : $n2_errors/$total_errors
Période : $start_date -> $end_date
@@ -59,7 +59,7 @@ La liste des erreurs se trouve en pièce jointe `n2.log`.
def get_period(entries: list[LogEntry]) -> tuple[str, str]:
"""Get start date and end date among dates of log entries.
Check all log entries dates and find the timestamp of earliest and latest generated logs
Check all log entries dates and find the timestamp of earliest and latest generated logs.
Args:
entries: list of log entries
@@ -84,36 +84,36 @@ def write_log_report(
end_date: str,
output_dir: Path = RESULT_PATH,
) -> None:
"""Write each relevant, irrelevant and general reports in their respective output file.
"""Write each relevant, irrelevant and all error reports in their respective output file.
Use the range date of relevant and irrelevant lists to get the period of time
the logs were generated.
Args:
end_date: # TODO
start_date: # TODO
today: # TODO
source_type: # TODO
relevant: List of n2 log entry
irrelevant: List of not n2 log entry
nb_files: Number of files scanned
today: # TODO
start_date: # TODO
end_date: # TODO
output_dir: Location where all reports will be written
"""
logger.info("\twriting reports job started.")
logger.info("writing reports job started.")
reports_dict = build_reports(
source_type, relevant, irrelevant, today, start_date, end_date, nb_files
)
output_dir.mkdir(parents=True, exist_ok=True)
for target in reports_dict:
logger.info(f"\t\tgenerating report '{target}'...")
output_dir.mkdir(parents=True, exist_ok=True)
logger.debug(f"\t\t\t{output_dir} exists.")
target_file = output_dir / str(target + ".log")
logger.info(f"Generating report '{target}'...")
logger.debug(f"'{output_dir}' exists.")
target_file = output_dir / f"{target}.log"
target_file.write_text(reports_dict[target], encoding="windows-1252")
logger.debug(f"\t\t\treport was written in {target_file.name}")
logger.info(f"\t\treport '{target}' was generated.")
logger.info("\twriting reports job completed.")
logger.debug(f"Report was written in {target_file.name}")
logger.info(f"Report '{target}' was generated.")
logger.info("Writing reports job completed.")
def build_reports(
@@ -143,7 +143,7 @@ def build_reports(
Dictionary of reports in string format
"""
logger.info("\t\tbuilding reports job started.")
logger.info("building reports job started.")
relevant_report = _render_target_report(
log_entries=relevant, target_template=N2_SUPPORT_TEMPLATE
)
@@ -151,8 +151,7 @@ def build_reports(
irrelevant_report = _render_target_report(
log_entries=irrelevant, target_template=OTHER_TEMPLATE
)
logger.info("\t\t\tn2 and other reports were generated.")
logger.info("\t\tbuilding reports job completed.")
logger.info("building reports job completed.")
source_value = source_type.value.capitalize()
@@ -199,7 +198,7 @@ def build_mail_summary(
"""Build a short plain-text summary of a logwatcher run.
Args:
nb_mails: Number of log files parsed.
nb_mails: Number of log mails parsed.
nb_errors_total: Total number of errors across all reports.
nb_errors_n2: Number of N2-relevant errors.
today: # TODO
@@ -213,7 +212,7 @@ def build_mail_summary(
logger.info("generating mail body...")
return MAIL_TEMPLATE.substitute(
generation_date=today,
nb_mail=nb_mails,
nb_mails=nb_mails,
n2_errors=nb_errors_n2,
total_errors=nb_errors_total,
start_date=start_date,
@@ -231,7 +230,7 @@ def _render_target_report(
Args:
log_entries: List of log entries
target_template: Template to use for N2_SUPPORT_TEMPLATE
target_template: Template to use for N2_SUPPORT_TEMPLATE and OTHER_TEMPLATE
Returns:
content of report in string format
@@ -1,11 +1,14 @@
import logging
from pathlib import Path
from exchangelib import Account, Folder
logger = logging.getLogger(__name__)
ANALYZED_FOLDER = "Analyzed"
LOG_FOLDER = "Logs"
LOG_IN_ATTACHMENT_PATTERN = "Le compte-rendu contient plus de 100 lignes."
SENT_FOLDER = "Sent"
def get_or_create_folder(account: Account, folder_name: str) -> Folder:
@@ -19,7 +22,7 @@ def get_or_create_folder(account: Account, folder_name: str) -> Folder:
The existing or newly created 'Analyzed' folder.
"""
log_folder = account.msg_folder_root / "Logs"
log_folder = account.msg_folder_root / LOG_FOLDER
matches = list(log_folder.glob(folder_name))
if matches:
@@ -30,3 +33,19 @@ def get_or_create_folder(account: Account, folder_name: str) -> Folder:
folder = Folder(parent=log_folder, name=folder_name)
folder.save()
return folder
def is_log_file(path: Path) -> bool:
"""Return True if file is a log file (.txt, .log, or extensionless).
Args:
path: Path of a file
Returns:
True if file has .txt or .log extension or extensionless
False otherwise.
"""
if not path.is_file():
return False
return path.suffix in {"", ".txt", ".log"}
+1
View File
@@ -0,0 +1 @@
"""Entry point to all package tests."""
+18 -2
View File
@@ -6,7 +6,7 @@ from unittest.mock import MagicMock
import pytest
from exchangelib import Account, Message
from logwatcher.config import FIXTURE_PATH
from logwatcher.config import FIXTURE_PATH, OUTPUT_PATH
from logwatcher.logging_config import setup_logging
# original log directory
@@ -19,18 +19,32 @@ VALID_LOG_DIR = FIXTURE_PATH / "valid_logs"
INVALID_LOG_DIR = FIXTURE_PATH / "invalid_logs"
# define temp directory only for testing
os.environ["PYTEST_DEBUG_TEMPROOT"] = str(Path("output/test").absolute())
os.environ["PYTEST_DEBUG_TEMPROOT"] = str(OUTPUT_PATH)
@pytest.fixture
def tmp_log_file(tmp_path):
"""Return path of logs logwatcher related."""
return tmp_path / "logwatcher_test.log"
@pytest.fixture(autouse=True)
def tmp_setup_logging(tmp_log_file):
"""Redirige les logs vers un dossier temporaire pendant les tests."""
setup_logging(file_path=tmp_log_file)
@pytest.fixture(autouse=True)
def tmp_cli_logging(monkeypatch, tmp_path):
"""Keep the CLI callback from writing run logs into the project `output/logs`.
The callback builds `LOGGING_PATH / <timestamp>.log` on every command call,
so `runner.invoke()` would otherwise create a file in the working tree and
add two handlers to the root logger for each test.
"""
monkeypatch.setattr("logwatcher.cli.LOGGING_PATH", tmp_path / "logs")
@pytest.fixture(name="original_log_dir")
def original_log_dir_fixture() -> Path:
"""Return log directory containing real logs.
@@ -41,6 +55,7 @@ def original_log_dir_fixture() -> Path:
"""
return FIXTURE_PATH / "original_logs"
@pytest.fixture(name="valid_log_dir")
def valid_log_dir_fixture() -> Path:
"""Return log directory containing only valid logs.
@@ -51,6 +66,7 @@ def valid_log_dir_fixture() -> Path:
"""
return FIXTURE_PATH / "valid_logs"
@pytest.fixture(name="invalid_log_dir")
def invalid_log_dir_fixture() -> Path:
"""Return log directory containing invalid logs.
+108 -1
View File
@@ -6,7 +6,7 @@ import pytest
from logwatcher.classifier import N2_PATTERNS, classify_log_entries
from logwatcher.config import FIXTURE_PATH
from logwatcher.parser import parse_file
from logwatcher.parser import parse_file, parse_lines
def _pattern_name_to_filename(name: str) -> str:
@@ -118,3 +118,110 @@ def test_classify_log_entries_empty_log_entries(invalid_log_dir: Path):
relevant, irrelevant = classify_log_entries(log_entries)
assert not relevant and not irrelevant
N2_ERROR_MESSAGE = "Erreur : GetTvaId -> TVA non trouvée : 20,00"
OTHER_ERROR_MESSAGE = "Erreur FTP SDUpdatePrixISF : Requested action not taken"
def _build_log_line(
error_time: str,
error_message: str,
store_name: str = "GABRIEL",
start_time: str = "22/09/2026 09:28:58",
mdc_server_name: str = "MDC_220",
server_ip: str = "192.168.13.22",
) -> str:
"""Build a raw log line as written by a MDC scan."""
return (
f"\\\\{server_ip}\\e\\{mdc_server_name}\\Logs\\26\\09\\22\\20260922092847.txt "
f"[{start_time}] DOSSIER EN COURS : {store_name} "
f"[{error_time}] {error_message}"
)
def test_classify_log_entries_merges_consecutive_duplicates():
"""Consecutive identical errors become a single log entry."""
log_entries = parse_lines(
[
_build_log_line("22/09/2026 09:32:19", N2_ERROR_MESSAGE),
_build_log_line("22/09/2026 09:32:20", N2_ERROR_MESSAGE),
_build_log_line("22/09/2026 09:32:25", N2_ERROR_MESSAGE),
]
)
relevant, irrelevant = classify_log_entries(log_entries)
assert len(relevant) == 1
assert relevant[0].get_error_time() == "22/09/2026 09:32:19"
assert not irrelevant
def test_classify_log_entries_merges_same_error_name():
"""Same error code with a different amount is two different entries."""
log_entries = parse_lines(
[
_build_log_line(
"22/09/2026 09:32:19", "Erreur : GetTvaId -> TVA non trouvée : 8,50"
),
_build_log_line(
"22/09/2026 09:32:20", "Erreur : GetTvaId -> TVA non trouvée : 20,00"
),
]
)
relevant, _ = classify_log_entries(log_entries)
assert len(relevant) == 2
def test_classify_log_entries_keeps_duplicates_from_other_store_or_scan():
"""Same error from another scan or another store is not a duplicate."""
log_entries = parse_lines(
[
_build_log_line("22/09/2026 09:32:19", N2_ERROR_MESSAGE),
_build_log_line(
"22/09/2026 10:29:10",
N2_ERROR_MESSAGE,
start_time="22/09/2026 10:28:58",
),
_build_log_line(
"22/09/2026 09:32:19", N2_ERROR_MESSAGE, store_name="DUTOUR"
),
]
)
relevant, _ = classify_log_entries(log_entries)
assert len(relevant) == 3
def test_classify_log_entries_merges_consecutive_irrelevant_duplicates():
"""Deduplication also applies to errors that do not require N2 support."""
log_entries = parse_lines(
[
_build_log_line("22/09/2026 09:32:19", OTHER_ERROR_MESSAGE),
_build_log_line("22/09/2026 09:32:24", OTHER_ERROR_MESSAGE),
]
)
relevant, irrelevant = classify_log_entries(log_entries)
assert not relevant
assert len(irrelevant) == 1
def test_classify_log_entries_keeps_same_error_separated_by_another_error():
"""An interleaved error breaks the run, so both errors are kept."""
log_entries = parse_lines(
[
_build_log_line("22/09/2026 09:32:19", N2_ERROR_MESSAGE),
_build_log_line("22/09/2026 09:32:20", OTHER_ERROR_MESSAGE),
_build_log_line("22/09/2026 09:32:21", N2_ERROR_MESSAGE),
]
)
relevant, irrelevant = classify_log_entries(log_entries)
assert len(relevant) == 2
assert len(irrelevant) == 1
+272
View File
@@ -0,0 +1,272 @@
import datetime
import logging
import os
from pathlib import Path
from unittest.mock import MagicMock
from zipfile import ZipFile
import pytest
from exchangelib import Account, Message
from logwatcher.cleaner import (
_get_retention_days,
archive_logs,
get_logs_to_archive,
prune_logs,
prune_mails,
)
from logwatcher.config import DATE_FILE_FORMAT, FRENCH_TIMEZONE
from logwatcher.utils import ANALYZED_FOLDER, SENT_FOLDER
# fixed date so retention cut-offs do not depend on the machine clock
TODAY = datetime.datetime(2026, 9, 25, 12, 0, 0, tzinfo=FRENCH_TIMEZONE)
@pytest.fixture(name="log_dir")
def log_dir_fixture(tmp_path: Path, monkeypatch) -> Path:
"""Redirect `LOGGING_PATH` to a temporary log directory.
Args:
tmp_path: Temporary root path for testing results
monkeypatch: MonkeyPatch used to fake paths
Returns:
the temporary log directory, created on disk
"""
logs_dir = tmp_path / "logs"
logs_dir.mkdir()
monkeypatch.setattr("logwatcher.cleaner.LOGGING_PATH", logs_dir)
return logs_dir
@pytest.fixture(name="archive_dir")
def archive_dir_fixture(tmp_path: Path, monkeypatch) -> Path:
"""Redirect `ARCHIVE_DIR` to a temporary directory.
The directory is left uncreated so tests can also check the missing directory case.
Args:
tmp_path: Temporary root path for testing results
monkeypatch: MonkeyPatch used to fake paths
Returns:
the temporary archive directory path, not created on disk
"""
archives_dir = tmp_path / "archives"
monkeypatch.setattr("logwatcher.cleaner.ARCHIVE_DIR", archives_dir)
return archives_dir
def _create_file(path: Path, age_in_days: int | None = None) -> Path:
"""Create a file and optionally backdate its `st_mtime`.
Args:
path: path of the file to create
age_in_days: number of days between `TODAY` and the file modification time.
When None, the modification time is left untouched.
Returns:
path of the created file
"""
path.write_text("log content", encoding="utf-8")
if age_in_days is not None:
timestamp = (TODAY - datetime.timedelta(days=age_in_days)).timestamp()
os.utime(path, (timestamp, timestamp))
return path
def test_get_retention_days_returns_integer(monkeypatch):
"""Return the number of days found in the environment variable."""
monkeypatch.setenv("RETENTION_DAYS_TEST", "42")
assert _get_retention_days("RETENTION_DAYS_TEST") == 42
def test_get_retention_days_missing_variable(monkeypatch, caplog):
"""Raise KeyError and log an error when the variable is not defined."""
monkeypatch.delenv("RETENTION_DAYS_TEST", raising=False)
with caplog.at_level(logging.ERROR, logger="logwatcher"):
with pytest.raises(KeyError):
_get_retention_days("RETENTION_DAYS_TEST")
assert (
"'RETENTION_DAYS_TEST' is not defined in environment variables."
in caplog.messages
)
def test_get_retention_days_not_an_integer(monkeypatch, caplog):
"""Raise ValueError and log an error when the variable is not a number."""
monkeypatch.setenv("RETENTION_DAYS_TEST", "thirty")
with caplog.at_level(logging.ERROR, logger="logwatcher"):
with pytest.raises(ValueError):
_get_retention_days("RETENTION_DAYS_TEST")
assert "thirty must be an integer." in caplog.messages
def test_get_logs_to_archive_selects_old_logs_only(log_dir: Path, monkeypatch):
"""Return only the log files older than `LOG_RETENTION_DAYS`."""
monkeypatch.setenv("LOG_RETENTION_DAYS", "30")
old_log = _create_file(log_dir / "old.log", age_in_days=40)
_create_file(log_dir / "recent.log", age_in_days=1)
assert get_logs_to_archive(TODAY) == [old_log]
def test_get_logs_to_archive_ignores_everything_but_logs(log_dir: Path, monkeypatch):
"""Ignore archives, foreign extensions and directories."""
monkeypatch.setenv("LOG_RETENTION_DAYS", "30")
(log_dir / "archives").mkdir()
_create_file(log_dir / "old.zip", age_in_days=40)
_create_file(log_dir / "old.csv", age_in_days=40)
expected_log = _create_file(log_dir / "old.txt", age_in_days=40)
assert get_logs_to_archive(TODAY) == [expected_log]
def test_get_logs_to_archive_warns_when_nothing_is_old_enough(
log_dir: Path, monkeypatch, caplog
):
"""Warn and return an empty list when no log reached the retention duration."""
monkeypatch.setenv("LOG_RETENTION_DAYS", "30")
_create_file(log_dir / "recent.log", age_in_days=1)
with caplog.at_level(logging.WARNING, logger="logwatcher"):
logs = get_logs_to_archive(TODAY)
assert logs == []
assert "No log files old enough to be archived." in caplog.messages
def test_archive_logs_creates_zip_and_deletes_originals(
log_dir: Path, archive_dir: Path
):
"""Zip the given logs in `ARCHIVE_DIR` then delete them from the log directory."""
first_log = _create_file(log_dir / "first.log")
second_log = _create_file(log_dir / "second.log")
archive_logs(TODAY, [first_log, second_log])
archived_files = list(archive_dir.iterdir())
assert len(archived_files) == 1
assert archived_files[0].name == f"archive_{TODAY.strftime(DATE_FILE_FORMAT)}.zip"
with ZipFile(archived_files[0]) as archive_file:
assert sorted(archive_file.namelist()) == ["first.log", "second.log"]
assert not first_log.exists()
assert not second_log.exists()
def test_archive_logs_does_nothing_without_file(
log_dir: Path, archive_dir: Path, caplog
):
"""Do not create the archive directory when there is no log to archive."""
with caplog.at_level(logging.INFO, logger="logwatcher"):
archive_logs(TODAY, [])
assert not archive_dir.exists()
assert "No file to archive. Skip." in caplog.messages
def test_prune_logs_deletes_old_archives_only(archive_dir: Path, monkeypatch):
"""Delete archives older than `ARCHIVED_LOG_RETENTION_DAYS` and keep recent ones."""
monkeypatch.setenv("ARCHIVED_LOG_RETENTION_DAYS", "60")
archive_dir.mkdir()
old_archive = _create_file(archive_dir / "archive_old.zip", age_in_days=90)
recent_archive = _create_file(archive_dir / "archive_recent.zip", age_in_days=1)
prune_logs(TODAY)
assert not old_archive.exists()
assert recent_archive.exists()
def test_prune_logs_creates_missing_directory(archive_dir: Path, monkeypatch, caplog):
"""Create the archive directory then warn when it holds no archive."""
monkeypatch.setenv("ARCHIVED_LOG_RETENTION_DAYS", "60")
with caplog.at_level(logging.WARNING, logger="logwatcher"):
prune_logs(TODAY)
assert archive_dir.is_dir()
assert "No archive found." in caplog.messages
def test_prune_logs_warns_on_empty_directory(archive_dir: Path, monkeypatch, caplog):
"""Warn when the archive directory exists but holds no archive."""
monkeypatch.setenv("ARCHIVED_LOG_RETENTION_DAYS", "60")
archive_dir.mkdir()
with caplog.at_level(logging.WARNING, logger="logwatcher"):
prune_logs(TODAY)
assert "No archive found." in caplog.messages
@pytest.fixture(name="mails_account")
def mails_account_fixture() -> tuple[MagicMock, MagicMock, MagicMock]:
"""Return a fake account exposing distinct 'Analyzed' and 'Sent' folders.
`account.msg_folder_root / LOG_FOLDER / <folder>` resolves to one mock per
folder, so each folder can be asserted independently.
Returns:
the fake account, the 'Analyzed' folder mock and the 'Sent' folder mock
"""
account = MagicMock(spec=Account)
folders = {ANALYZED_FOLDER: MagicMock(), SENT_FOLDER: MagicMock()}
logs_folder = account.msg_folder_root.__truediv__.return_value
logs_folder.__truediv__.side_effect = lambda folder_name: folders[folder_name]
return account, folders[ANALYZED_FOLDER], folders[SENT_FOLDER]
def test_prune_mails_filters_with_retention_cutoff(mails_account, monkeypatch):
"""Filter both mail folders with `MAIL_RETENTION_DAYS` applied to the current day."""
account, analyzed_folder, sent_folder = mails_account
monkeypatch.setenv("MAIL_RETENTION_DAYS", "30")
analyzed_folder.filter.return_value = []
sent_folder.filter.return_value = []
expected_cutoff = TODAY - datetime.timedelta(days=30)
prune_mails(TODAY, account)
analyzed_folder.filter.assert_called_once_with(datetime_sent__lt=expected_cutoff)
sent_folder.filter.assert_called_once_with(datetime_sent__lt=expected_cutoff)
def test_prune_mails_deletes_old_mails(mails_account, monkeypatch):
"""Delete every old mail of both folders with a single hard delete."""
account, analyzed_folder, sent_folder = mails_account
monkeypatch.setenv("MAIL_RETENTION_DAYS", "30")
analyzed_mail = Message(subject="analyzed", text_body="analyzed")
analyzed_mail.id, analyzed_mail.changekey = "id1", "ck1"
sent_mail = Message(subject="sent", text_body="sent")
sent_mail.id, sent_mail.changekey = "id2", "ck2"
analyzed_folder.filter.return_value = [analyzed_mail]
sent_folder.filter.return_value = [sent_mail]
prune_mails(TODAY, account)
account.bulk_delete.assert_called_once_with(ids=[("id1", "ck1"), ("id2", "ck2")])
# `bulk_delete` must stay a hard delete: no `delete_type` nor `to_folder` argument
assert set(account.bulk_delete.call_args.kwargs) == {"ids"}
def test_prune_mails_warns_when_no_old_mail(mails_account, monkeypatch, caplog):
"""Warn and delete nothing when both folders only hold recent mails."""
account, analyzed_folder, sent_folder = mails_account
monkeypatch.setenv("MAIL_RETENTION_DAYS", "30")
analyzed_folder.filter.return_value = []
sent_folder.filter.return_value = []
with caplog.at_level(logging.WARNING, logger="logwatcher"):
prune_mails(TODAY, account)
assert "No mail older than '30' days found." in caplog.messages
account.bulk_delete.assert_not_called()
+97 -40
View File
@@ -8,7 +8,9 @@ from unittest.mock import MagicMock
from logwatcher import __version__
from logwatcher.cli import _get_files_from_dir, _is_log_file, app
from logwatcher.cli import _get_files_from_dir, app
from logwatcher.config import RESULT_PATH
from logwatcher.utils import is_log_file
runner = CliRunner()
@@ -35,9 +37,7 @@ def get_argument_from_files(log_dir: Path, output_path: Path) -> list[str]:
List of arguments in string format
"""
files = [
str(file) for file in log_dir.iterdir() if file.suffix in ("", ".txt", ".log")
]
files = [str(file) for file in log_dir.iterdir() if is_log_file(file)]
arguments = ["from-files", "--output-dir", str(output_path)]
for file in files:
@@ -62,30 +62,6 @@ def test_version_cli():
assert f"logwatcher version: {__version__}" in result.output
def test_is_log_file(tmp_path: Path):
"""Assert only .log, .txt or file without extensions are valid.
Args:
tmp_path: Temporary root path for testing results
"""
# set up files
Path(tmp_path / "a.log").touch()
Path(tmp_path / "a.txt").touch()
Path(tmp_path / "a.csv").touch()
Path(tmp_path / "a.py").touch()
Path(tmp_path / "a").touch()
Path(tmp_path / "a.pdf").touch()
# test
assert _is_log_file(Path(tmp_path / "a.log"))
assert _is_log_file(Path(tmp_path / "a.txt"))
assert _is_log_file(Path(tmp_path / "a")) # without extensions
assert not _is_log_file(Path(tmp_path / "a.csv"))
assert not _is_log_file(Path(tmp_path / "a.py"))
assert not _is_log_file(Path(tmp_path / "a.pdf"))
def test_get_files_from_dir_empty_dir(tmp_path: Path):
"""Assert that a non-existing directory (or wrong path) cannot be given as a working input directory.
@@ -163,7 +139,9 @@ def test_cli_from_files_neither_input_files_nor_input_dir():
assert "Either input_files or input_dir must be provided" in results.output
def test_cli_from_files_bad_input_file(tmp_path: Path, output_path: Path, invalid_log_dir: Path):
def test_cli_from_files_bad_input_file(
tmp_path: Path, output_path: Path, invalid_log_dir: Path
):
"""Test calling cli with bad input file (directory instead of file).
Must return an exit code at 1 and the ValueError exception raised
@@ -406,24 +384,28 @@ def test_cli_from_files_valid_logs(output_path: Path, valid_log_dir: Path):
def test_cli_from_mails_no_output_dir(monkeypatch, make_mock_account):
"""Assert everything works when no output_path is provided.
Must use "output/" directory, located in project root.
Must use `RESULT_PATH` as the default output directory.
"""
messages = [
Message(subject="subject test 1", text_body="body test 1"),
Message(subject="subject test 2", text_body="body test 2"),
Message(subject="subject test 3", text_body="body test 3")
Message(subject="subject test 3", text_body="body test 3"),
]
write_log_report_mock = MagicMock()
with monkeypatch.context() as m:
m.setattr("logwatcher.cli.connect_to_mailbox", lambda: make_mock_account(messages=messages, analyzed_folder="Analyzed"))
m.setattr("logwatcher.notifier.Message.send", MagicMock(return_value=None))
m.setattr(
"logwatcher.cli.connect_to_mailbox",
lambda: make_mock_account(messages=messages, analyzed_folder="Analyzed"),
)
m.setattr("logwatcher.cli.write_log_report", write_log_report_mock)
m.setattr("logwatcher.cli.send_n2_report", MagicMock())
arguments = ["from-mails"]
results = runner.invoke(app, arguments)
assert results.exit_code == 0
assert results.exit_code == 0
assert write_log_report_mock.call_args.kwargs["output_dir"] == RESULT_PATH
def test_cli_from_mails_invalid_output_dir(tmp_path, monkeypatch):
@@ -442,7 +424,10 @@ def test_cli_from_mails_invalid_output_dir(tmp_path, monkeypatch):
]
with monkeypatch.context() as m:
m.setattr("logwatcher.cli.connect_to_mailbox", lambda: make_mock_account(messages=messages, analyzed_folder="Analyzed"))
m.setattr(
"logwatcher.cli.connect_to_mailbox",
lambda: make_mock_account(messages=messages, analyzed_folder="Analyzed"),
)
m.setattr("logwatcher.notifier.Message.send", MagicMock(return_value=None))
arguments = ["from-mails", "--output-dir", bad_output_dir]
@@ -464,11 +449,14 @@ def test_cli_from_mails_valid_output_dir(tmp_path, monkeypatch, make_mock_accoun
messages = [
Message(subject="subject test 1", text_body="body test 1"),
Message(subject="subject test 2", text_body="body test 2"),
Message(subject="subject test 3", text_body="body test 3")
Message(subject="subject test 3", text_body="body test 3"),
]
with monkeypatch.context() as m:
m.setattr("logwatcher.cli.connect_to_mailbox", lambda: make_mock_account(messages=messages, analyzed_folder="Analyzed"))
m.setattr(
"logwatcher.cli.connect_to_mailbox",
lambda: make_mock_account(messages=messages, analyzed_folder="Analyzed"),
)
m.setattr("logwatcher.notifier.Message.send", MagicMock(return_value=None))
arguments = ["from-mails", "--output-dir", output_path]
@@ -481,3 +469,72 @@ def test_cli_from_mails_valid_output_dir(tmp_path, monkeypatch, make_mock_accoun
assert len(output_children) == len(expected)
for name in expected:
assert name in file_names
def test_prune_help_lists_targets():
"""Assert the prune command lists both of its targets."""
results = runner.invoke(app, ["prune", "--help"])
assert results.exit_code == 0
assert "--mails" in results.output
assert "--logs" in results.output
def test_prune_without_target_raises():
"""Assert that prune without any target is rejected."""
results = runner.invoke(app, ["prune"])
assert results.exit_code == 2
assert "At least one of --mails or --logs must be provided." in results.output
def test_prune_logs_only(monkeypatch):
"""Assert `--logs` prunes logs only, without connecting to the mailbox."""
connect_mock = MagicMock()
prune_mails_mock = MagicMock()
get_logs_mock = MagicMock(return_value=[MagicMock()])
archive_logs_mock = MagicMock()
prune_logs_mock = MagicMock()
with monkeypatch.context() as m:
m.setattr("logwatcher.cli.connect_to_mailbox", connect_mock)
m.setattr("logwatcher.cli.prune_mails", prune_mails_mock)
m.setattr("logwatcher.cli.get_logs_to_archive", get_logs_mock)
m.setattr("logwatcher.cli.archive_logs", archive_logs_mock)
m.setattr("logwatcher.cli.prune_logs", prune_logs_mock)
results = runner.invoke(app, ["prune", "--logs"])
assert results.exit_code == 0
get_logs_mock.assert_called_once()
archive_logs_mock.assert_called_once()
prune_logs_mock.assert_called_once()
connect_mock.assert_not_called()
prune_mails_mock.assert_not_called()
def test_prune_mails_only(monkeypatch):
"""Assert `--mails` prunes mails only, without touching log archives."""
fake_account = MagicMock()
connect_mock = MagicMock(return_value=fake_account)
prune_mails_mock = MagicMock()
get_logs_mock = MagicMock()
archive_logs_mock = MagicMock()
prune_logs_mock = MagicMock()
with monkeypatch.context() as m:
m.setattr("logwatcher.cli.connect_to_mailbox", connect_mock)
m.setattr("logwatcher.cli.prune_mails", prune_mails_mock)
m.setattr("logwatcher.cli.get_logs_to_archive", get_logs_mock)
m.setattr("logwatcher.cli.archive_logs", archive_logs_mock)
m.setattr("logwatcher.cli.prune_logs", prune_logs_mock)
results = runner.invoke(app, ["prune", "--mails"])
assert results.exit_code == 0
connect_mock.assert_called_once()
prune_mails_mock.assert_called_once()
assert prune_mails_mock.call_args.args[1] is fake_account
get_logs_mock.assert_not_called()
archive_logs_mock.assert_not_called()
prune_logs_mock.assert_not_called()
-24
View File
@@ -114,9 +114,6 @@ class OkAccount:
self.protocol = OkProtocol()
#########################################
# _find_log_attachment #
#########################################
def test_find_log_attachment_returns_log_file(
empty_mail: Message,
):
@@ -221,9 +218,6 @@ def test_find_log_attachment_rejects_non_file_attachment(empty_mail: Message):
assert attachment is None
############################
# _get_attachment_content #
############################
def test_get_attachment_content_decodes_windows_1252(original_log_dir: Path):
"""Decodes content with accents correctly."""
file_with_accents = original_log_dir / "CR_20260727110008.txt"
@@ -249,9 +243,6 @@ def test_get_attachment_content_empty_bytes():
assert content == ""
########################
# _get_mail_body(mail) #
########################
def test_get_mail_body_from_marker_to_end(empty_mail: Message):
"""Returns body starting at 'Répertoire scanné'."""
empty_mail.text_body += "Répertoire scanné : \\192.168.60.40\\e$\\MDC_1110\\Logs"
@@ -291,9 +282,6 @@ def test_get_mail_body_multiple_markers(empty_mail: Message):
assert "Répertoire scanné\nRépertoire scanné\nRépertoire scanné" == content
################################
# _has_attachment_marker(body) #
################################
def test_has_attachment_marker_true():
"""Returns True when the exact pattern is present."""
body_with_pattern = "Compte-rendu du suivi des imports NOSYMAG du 02/09/2026 à 15:00:10\n\nLe compte-rendu contient plus de 100 lignes.\nSe reporter à la pièce jointe."
@@ -318,9 +306,6 @@ def test_has_attachment_marker_empty_string():
assert not _has_attachment_marker(body_empty)
########################
# connect_to_mailbox() #
########################
def test_connect_to_mailbox_success(monkeypatch: MonkeyPatch, caplog, set_env):
"""Returns an Account when credentials are valid."""
with monkeypatch.context() as m, caplog.at_level(logging.INFO, "logwatcher"):
@@ -355,9 +340,6 @@ def test_connect_to_mailbox_invalid_credentials(
assert "Connection established successfully." not in caplog.messages
###############################
# fetch_log_messages(account) #
###############################
def test_fetch_log_messages_returns_messages(make_mock_account):
"""Returns the .all() of the Logs folder."""
mail_inline = Message(subject="mail1", text_body="text1")
@@ -376,9 +358,6 @@ def test_fetch_log_messages_folder_path(make_mock_account):
account.msg_folder_root.__truediv__.assert_called_once_with("Logs")
##############################
# extract_log_lines(content) #
##############################
def test_extract_log_lines_split_and_clean(original_log_dir: Path):
r"""Split content, strips '\r\n', removes empty lines."""
file = original_log_dir / "CR_20260727110008.txt"
@@ -425,9 +404,6 @@ def test_extract_log_lines_crlf():
assert "\r" not in line
#############################
# extract_logs_from_mails() #
#############################
def test_extract_logs_from_mails_with_attachment(original_log_dir: Path):
"""Accumulates log lines from attachment mails."""
# create mail with attachments
+30 -12
View File
@@ -11,46 +11,64 @@ def test_valid_log_entry():
LogEntry(
server_ip="192.168.13.27",
mdc_server_name="MDC_720",
start_time=datetime.strptime("18/07/2026 11:34:41", DATETIME_FORMAT).astimezone(FRENCH_TIMEZONE),
start_time=datetime.strptime("18/07/2026 11:34:41", DATETIME_FORMAT).astimezone(
FRENCH_TIMEZONE
),
store_name="PECCOUD",
error_time=datetime.strptime("18/07/2026 11:35:58", DATETIME_FORMAT).astimezone(FRENCH_TIMEZONE),
error_time=datetime.strptime("18/07/2026 11:35:58", DATETIME_FORMAT).astimezone(
FRENCH_TIMEZONE
),
error_message="Erreur FTP SDUpdatePrixISF : Requested action not taken",
raw_line="\\\\192.168.13.27\\e\\MDC_720\\Logs\\26\\07\\18\\20260718113428.txt [18/07/2026 11:34:41] DOSSIER EN COURS : PECCOUD [18/07/2026 11:35:58] Erreur FTP SDUpdatePrixISF : Requested action not taken"
raw_line="\\\\192.168.13.27\\e\\MDC_720\\Logs\\26\\07\\18\\20260718113428.txt [18/07/2026 11:34:41] DOSSIER EN COURS : PECCOUD [18/07/2026 11:35:58] Erreur FTP SDUpdatePrixISF : Requested action not taken",
)
def test_log_entry_invalid_dates():
"""Assert a log entry cannot be created if start time is not later than error time."""
with pytest.raises(ValueError, match="Error in date-times"):
LogEntry(
server_ip="192.168.13.27",
mdc_server_name="MDC_720",
start_time=datetime.strptime("18/07/2026 11:35:58", DATETIME_FORMAT).astimezone(FRENCH_TIMEZONE),
start_time=datetime.strptime(
"18/07/2026 11:35:58", DATETIME_FORMAT
).astimezone(FRENCH_TIMEZONE),
store_name="PECCOUD",
error_time=datetime.strptime("18/07/2026 11:34:41", DATETIME_FORMAT).astimezone(FRENCH_TIMEZONE),
error_time=datetime.strptime(
"18/07/2026 11:34:41", DATETIME_FORMAT
).astimezone(FRENCH_TIMEZONE),
error_message="Erreur FTP SDUpdatePrixISF : Requested action not taken",
raw_line="\\\\192.168.13.27\\e\\MDC_720\\Logs\\26\\07\\18\\20260718113428.txt [18/07/2026 11:34:41] DOSSIER EN COURS : PECCOUD [18/07/2026 11:35:58] Erreur FTP SDUpdatePrixISF : Requested action not taken"
raw_line="\\\\192.168.13.27\\e\\MDC_720\\Logs\\26\\07\\18\\20260718113428.txt [18/07/2026 11:34:41] DOSSIER EN COURS : PECCOUD [18/07/2026 11:35:58] Erreur FTP SDUpdatePrixISF : Requested action not taken",
)
def test_equals_models():
"""Assert two models are equals if they have the same value as attributes."""
log_entry_1 = LogEntry(
server_ip="192.168.13.27",
mdc_server_name="MDC_720",
start_time=datetime.strptime("18/07/2026 11:34:41", DATETIME_FORMAT).astimezone(FRENCH_TIMEZONE),
start_time=datetime.strptime("18/07/2026 11:34:41", DATETIME_FORMAT).astimezone(
FRENCH_TIMEZONE
),
store_name="PECCOUD",
error_time=datetime.strptime("18/07/2026 11:35:58", DATETIME_FORMAT).astimezone(FRENCH_TIMEZONE),
error_time=datetime.strptime("18/07/2026 11:35:58", DATETIME_FORMAT).astimezone(
FRENCH_TIMEZONE
),
error_message="Erreur FTP SDUpdatePrixISF : Requested action not taken",
raw_line="\\\\192.168.13.27\\e\\MDC_720\\Logs\\26\\07\\18\\20260718113428.txt [18/07/2026 11:34:41] DOSSIER EN COURS : PECCOUD [18/07/2026 11:35:58] Erreur FTP SDUpdatePrixISF : Requested action not taken"
raw_line="\\\\192.168.13.27\\e\\MDC_720\\Logs\\26\\07\\18\\20260718113428.txt [18/07/2026 11:34:41] DOSSIER EN COURS : PECCOUD [18/07/2026 11:35:58] Erreur FTP SDUpdatePrixISF : Requested action not taken",
)
log_entry_2 = LogEntry(
server_ip="192.168.13.27",
mdc_server_name="MDC_720",
start_time=datetime.strptime("18/07/2026 11:34:41", DATETIME_FORMAT).astimezone(FRENCH_TIMEZONE),
start_time=datetime.strptime("18/07/2026 11:34:41", DATETIME_FORMAT).astimezone(
FRENCH_TIMEZONE
),
store_name="PECCOUD",
error_time=datetime.strptime("18/07/2026 11:35:58", DATETIME_FORMAT).astimezone(FRENCH_TIMEZONE),
error_time=datetime.strptime("18/07/2026 11:35:58", DATETIME_FORMAT).astimezone(
FRENCH_TIMEZONE
),
error_message="Erreur FTP SDUpdatePrixISF : Requested action not taken",
raw_line="\\\\192.168.13.27\\e\\MDC_720\\Logs\\26\\07\\18\\20260718113428.txt [18/07/2026 11:34:41] DOSSIER EN COURS : PECCOUD [18/07/2026 11:35:58] Erreur FTP SDUpdatePrixISF : Requested action not taken"
raw_line="\\\\192.168.13.27\\e\\MDC_720\\Logs\\26\\07\\18\\20260718113428.txt [18/07/2026 11:34:41] DOSSIER EN COURS : PECCOUD [18/07/2026 11:35:58] Erreur FTP SDUpdatePrixISF : Requested action not taken",
)
assert log_entry_1 == log_entry_2
+23 -5
View File
@@ -11,8 +11,14 @@ def mock_get_or_create_folder(account, folder_name): # noqa: D103
return folder_name
@pytest.mark.parametrize("vars", [("N2_REPORT_SUBJECT", "N2_REPORT_RECIPIENTS"), ("N2_REPORT_RECIPIENTS", "N2_REPORT_SUBJECT")])
def test_send_n2_report_missing_recipients_env(vars, monkeypatch, ):
@pytest.mark.parametrize(
"vars",
[
("N2_REPORT_SUBJECT", "N2_REPORT_RECIPIENTS"),
("N2_REPORT_RECIPIENTS", "N2_REPORT_SUBJECT"),
],
)
def test_send_n2_report_missing_recipients_env(vars, monkeypatch):
"""Assert that missing values in environmnent file raises KeyError."""
monkeypatch.delenv("N2_REPORT_RECIPIENTS")
monkeypatch.delenv("N2_REPORT_SUBJECT")
@@ -24,7 +30,15 @@ def test_send_n2_report_missing_recipients_env(vars, monkeypatch, ):
assert vars[1] in err.value.args
@pytest.mark.parametrize("recipients", ["mail@client.fr", "mail@ client.fr ", "mail1@client.fr,mail2@client.fr,mail3@client.fr", "mail1@client.fr , mail2@client.fr , mail3@client.fr"])
@pytest.mark.parametrize(
"recipients",
[
"mail@client.fr",
"mail@ client.fr ",
"mail1@client.fr,mail2@client.fr,mail3@client.fr",
"mail1@client.fr , mail2@client.fr , mail3@client.fr",
],
)
def test_send_n2_report_multiple_recipients(recipients, make_mock_account, monkeypatch):
"""Assert gathering multiple account as recipients from env file is valid.
@@ -42,5 +56,9 @@ def test_send_n2_report_multiple_recipients(recipients, make_mock_account, monke
with monkeypatch.context() as m:
m.setattr("logwatcher.notifier.get_or_create_folder", mock_get_or_create_folder)
m.setattr("logwatcher.notifier.Message.send", MagicMock(id="1", return_value=None))
send_n2_report(account=make_mock_account(), summary="summary", n2_log_file=MagicMock())
m.setattr(
"logwatcher.notifier.Message.send", MagicMock(id="1", return_value=None)
)
send_n2_report(
account=make_mock_account(), summary="summary", n2_log_file=MagicMock()
)
+1 -1
View File
@@ -235,7 +235,7 @@ def _test_write_log_report(
SourceType.FILE,
relevant,
irrelevant,
datetime.now(FRENCH_TIMEZONE).strftime(DATETIME_FORMAT),
today,
start_date,
end_date,
nb_files,
@@ -1,6 +1,31 @@
from pathlib import Path
from unittest.mock import MagicMock
from logwatcher.mail_utils import get_or_create_folder
from logwatcher.utils import get_or_create_folder, is_log_file
def test_is_log_file(tmp_path: Path):
"""Assert only .log, .txt or file without extensions are valid.
Args:
tmp_path: Temporary root path for testing results
"""
# set up files
Path(tmp_path / "a.log").touch()
Path(tmp_path / "a.txt").touch()
Path(tmp_path / "a.csv").touch()
Path(tmp_path / "a.py").touch()
Path(tmp_path / "a").touch()
Path(tmp_path / "a.pdf").touch()
# test
assert is_log_file(Path(tmp_path / "a.log"))
assert is_log_file(Path(tmp_path / "a.txt"))
assert is_log_file(Path(tmp_path / "a")) # without extensions
assert not is_log_file(Path(tmp_path / "a.csv"))
assert not is_log_file(Path(tmp_path / "a.py"))
assert not is_log_file(Path(tmp_path / "a.pdf"))
def test_get_or_create_analyzed_folder_exists(make_mock_account, monkeypatch):
@@ -16,7 +41,7 @@ def test_get_or_create_analyzed_folder_exists(make_mock_account, monkeypatch):
fake_folder_cls = MagicMock()
with monkeypatch.context() as m:
m.setattr("logwatcher.mail_utils.Folder", fake_folder_cls)
m.setattr("logwatcher.utils.Folder", fake_folder_cls)
result = get_or_create_folder(account, "Analyzed")
assert result is existing_folder
@@ -30,7 +55,7 @@ def test_get_or_create_analyzed_folder_creates(make_mock_account, monkeypatch):
fake_folder_instance = MagicMock()
fake_folder_cls = MagicMock(return_value=fake_folder_instance)
with monkeypatch.context() as m:
m.setattr("logwatcher.mail_utils.Folder", fake_folder_cls)
m.setattr("logwatcher.utils.Folder", fake_folder_cls)
result = get_or_create_folder(account, "Analyzed")
assert result is fake_folder_instance