diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ea23b0..61f5cc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ Format : [Keep a Changelog](https://keepachangelog.com/fr/1.0.0/) — versionnag --- +## [1.6.32] - 2026-05-19 + +### Modifié +- Admin stats / Agents détectés : UA affiché en entier (plus de troncature à 55 car.) dans le drill-down IP et la liste agents +- Admin stats / Agents détectés : bouton « + bot » sur chaque agent non classé — ajoute le UA aux patterns via AJAX sans recharger la page, déplace la ligne vers "Bots connus" +- Admin stats / Agents détectés : section alimentée par `FOLIO_ALL_UAS` (tous UAs publics, bots inclus) plutôt que par agrégation depuis `ip_data` +- AccessLogParser : filtrage des bots dans les compteurs pages/livres/IPs — les requêtes détectées comme bot n'alimentent plus les stats de fréquentation ; `all_uas` expose tous les UAs (bots inclus) pour la section Agents +- `index.php` : chargement de `bots.json` avant la création du parser pour passer les patterns au constructeur ; `admin_add_bot` vide les caches stats après ajout ; `admin_save_bots` vide également les caches +- Template : `FOLIO_ALL_UAS` et `FOLIO_CSRF` ajoutés aux variables JS de la page stats + +--- + ## [1.6.31] - 2026-05-19 ### Ajouté diff --git a/public/assets/js/admin-stats.js b/public/assets/js/admin-stats.js index 7d7e3ec..46f80f9 100644 --- a/public/assets/js/admin-stats.js +++ b/public/assets/js/admin-stats.js @@ -103,11 +103,11 @@ function botBadge(ua) { var ips = ipsByAsn[asnKey] || []; var ipRows = ips.slice(0, 20).map(function (ipInfo) { - // Agents sous l'IP avec badge bot + // Agents sous l'IP avec badge bot (UA en entier) var agentsHtml = ''; (ipInfo.agents || []).forEach(function (ua) { agentsHtml += '
' - + botBadge(ua) + esc(trunc(ua, 55)) + '
'; + + botBadge(ua) + esc(ua) + ''; }); // Chemins triés : /post/ et /book/ avec ts, reste sans ts @@ -211,23 +211,15 @@ function botBadge(ua) { // ── Liste consolidée de tous les agents ────────────────────────────────────── (function () { - var el = document.getElementById('stats-agents-container'); - var badge = document.getElementById('agents-count'); - var ipData = (typeof FOLIO_IP_DATA !== 'undefined') ? FOLIO_IP_DATA : {}; + var el = document.getElementById('stats-agents-container'); + var badge = document.getElementById('agents-count'); + var allUas = (typeof FOLIO_ALL_UAS !== 'undefined') ? FOLIO_ALL_UAS : {}; + var csrf = (typeof FOLIO_CSRF !== 'undefined') ? FOLIO_CSRF : ''; if (!el) { return; } - // Agréger toutes les UAs depuis FOLIO_IP_DATA - var uaCounts = {}; - Object.keys(ipData).forEach(function (ip) { - (ipData[ip].agents || []).forEach(function (ua) { - uaCounts[ua] = (uaCounts[ua] || 0) + (ipData[ip].hits || 0); - }); - }); - - var agents = Object.keys(uaCounts).map(function (ua) { - return { ua: ua, hits: uaCounts[ua], bot: isBot(ua) }; + var agents = Object.keys(allUas).map(function (ua) { + return { ua: ua, hits: allUas[ua], bot: isBot(ua) }; }).sort(function (a, b) { - // Bots d'abord, puis par hits desc if (a.bot !== b.bot) { return a.bot ? -1 : 1; } return b.hits - a.hits; }); @@ -239,14 +231,38 @@ function botBadge(ua) { var bots = agents.filter(function (a) { return a.bot; }); var unknown = agents.filter(function (a) { return !a.bot; }); - if (badge) { badge.textContent = '— ' + bots.length + ' bot(s) détecté(s) sur ' + agents.length; } + if (badge) { badge.textContent = '— ' + bots.length + ' bot(s) sur ' + agents.length; } + + function addBot(ua, btn) { + btn.disabled = true; + var fd = new FormData(); + fd.append('_csrf', csrf); + fd.append('pattern', ua); + fetch('/?action=admin_add_bot', { method: 'POST', body: fd }) + .then(function (r) { return r.json(); }) + .then(function (d) { + if (d.ok) { + _botPatterns.push(ua); + btn.closest('tr').querySelector('td:first-child').innerHTML = '🤖'; + btn.remove(); + } else { + btn.disabled = false; + } + }) + .catch(function () { btn.disabled = false; }); + } function agentRow(a) { + var addBtn = (!a.bot) + ? '' + : ''; return '' + '' + (a.bot ? '🤖' : '?') + '' + '' - + '' + esc(a.ua) + '' + + '' + esc(a.ua) + '' + + addBtn + '' + '' + a.hits.toLocaleString('fr-FR') + '' + ''; @@ -277,6 +293,15 @@ function botBadge(ua) { html += ''; el.innerHTML = html; + + // Délégation : boutons "+ bot" + el.addEventListener('click', function (e) { + var btn = e.target.closest('.add-bot-btn'); + if (!btn) { return; } + var row = btn.closest('tr'); + var code = row ? row.querySelector('code') : null; + if (code) { addBot(code.textContent, btn); } + }); }()); // ── Pages les plus visitées (RSS XML + sparklines) ─────────────────────────── diff --git a/public/index.php b/public/index.php index d469912..6e8d878 100644 --- a/public/index.php +++ b/public/index.php @@ -2725,6 +2725,38 @@ switch ($action) { require_once BASE_PATH . '/src/AccessLogParser.php'; require_once BASE_PATH . '/src/AsnLookup.php'; + // Patterns de bots — initialisation si absent + $botsFile = DATA_PATH . '/bots.json'; + if (!file_exists($botsFile)) { + $defaultBots = [ + 'Googlebot', 'Googlebot-Image', 'Google-InspectionTool', 'Google-Extended', + 'bingbot', 'BingPreview', 'msnbot', + 'DuckDuckBot', 'DuckDuckGo-Favicons-Bot', + 'Baiduspider', 'YandexBot', 'YandexImages', 'YandexMetrika', + 'Applebot', + 'facebookexternalhit', 'facebot', + 'Twitterbot', 'LinkedInBot', 'Slackbot', 'TelegramBot', 'WhatsApp', 'Discordbot', + 'PetalBot', 'Bytespider', 'SogouSpider', 'SeznamBot', 'Exabot', + 'AhrefsBot', 'SemrushBot', 'MJ12bot', 'DotBot', 'rogerbot', 'BLEXBot', 'DataForSeoBot', + 'Screaming Frog SEO Spider', + 'ClaudeBot', 'GPTBot', 'PerplexityBot', 'cohere-ai', 'anthropic-ai', + 'meta-externalagent', 'OAI-SearchBot', 'Amazonbot', + 'CCBot', 'ia_archiver', 'archive.org_bot', + 'NetcraftSurveyAgent', + 'python-requests', 'python-urllib', 'Python/', + 'curl/', 'wget/', 'Wget/', + 'Go-http-client/1', 'Java/', 'Apache-HttpClient', 'okhttp/', + 'Scrapy', 'HeadlessChrome', 'PhantomJS', 'Puppeteer', 'Playwright', 'Selenium', + 'UptimeRobot', 'Pingdom', 'StatusCake', 'Site24x7', 'GTmetrix', + 'Chrome-Lighthouse', 'PageSpeed', 'Zabbix', 'check_http', + 'libwww-perl', 'GuzzleHttp', 'masscan', 'zgrab', 'nuclei', + ]; + @mkdir(dirname($botsFile), 0755, true); + @file_put_contents($botsFile, json_encode($defaultBots, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT)); + } + $botPatterns = json_decode((string) file_get_contents($botsFile), true) ?: []; + $adminData['bot_patterns'] = $botPatterns; + $statsCacheFile = DATA_PATH . '/.stats_cache.json'; $statsRaw = null; if (file_exists($statsCacheFile) && (time() - filemtime($statsCacheFile)) < 60) { @@ -2733,7 +2765,7 @@ switch ($action) { if ($statsRaw === null) { $cutoff14 = strtotime('-14 days midnight') ?: (time() - 14 * 86400); $tParser = new TrendingParser('/var/log/apache2', apacheAccessLog()); - $accessParser = new AccessLogParser('/var/log/apache2', apacheAccessLog()); + $accessParser = new AccessLogParser('/var/log/apache2', apacheAccessLog(), '', 600, 14, $botPatterns); $accessStats = $accessParser->stats(); $topIps = array_slice($accessStats['ips'], 0, 200, true); $asnMap = (new AsnLookup())->batchLookup(array_keys($topIps)); @@ -2758,6 +2790,7 @@ switch ($action) { 'as' => AsnLookup::aggregateByAs($topIps, $asnMap), 'pages_by_day' => $accessStats['pages_by_day'] ?? [], 'ip_data' => $ipData, + 'all_uas' => $accessStats['all_uas'] ?? [], ]; @file_put_contents($statsCacheFile, json_encode($statsRaw)); } @@ -2768,37 +2801,7 @@ switch ($action) { $adminData['as_groups'] = asGroups(); $adminData['stats_pages_by_day'] = $statsRaw['pages_by_day'] ?? []; $adminData['stats_ip_data'] = $statsRaw['ip_data'] ?? []; - - // Patterns de bots — initialisation si absent - $botsFile = DATA_PATH . '/bots.json'; - if (!file_exists($botsFile)) { - $defaultBots = [ - 'Googlebot','Googlebot-Image','Google-InspectionTool','Google-Extended', - 'bingbot','BingPreview','msnbot', - 'DuckDuckBot','DuckDuckGo-Favicons-Bot', - 'Baiduspider','YandexBot','YandexImages','YandexMetrika', - 'Applebot', - 'facebookexternalhit','facebot', - 'Twitterbot','LinkedInBot','Slackbot','TelegramBot','WhatsApp','Discordbot', - 'PetalBot','Bytespider','SogouSpider','SeznamBot','Exabot', - 'AhrefsBot','SemrushBot','MJ12bot','DotBot','rogerbot','BLEXBot','DataForSeoBot', - 'Screaming Frog SEO Spider', - 'ClaudeBot','GPTBot','Google-Extended','PerplexityBot','cohere-ai','anthropic-ai', - 'meta-externalagent','OAI-SearchBot','Amazonbot', - 'CCBot','ia_archiver','archive.org_bot', - 'NetcraftSurveyAgent', - 'python-requests','python-urllib','Python/', - 'curl/','wget/','Wget/', - 'Go-http-client/1','Java/','Apache-HttpClient','okhttp/', - 'Scrapy','HeadlessChrome','PhantomJS','Puppeteer','Playwright','Selenium', - 'UptimeRobot','Pingdom','StatusCake','Site24x7','GTmetrix', - 'Chrome-Lighthouse','PageSpeed','Zabbix','check_http', - 'libwww-perl','GuzzleHttp','masscan','zgrab','nuclei', - ]; - @mkdir(dirname($botsFile), 0755, true); - @file_put_contents($botsFile, json_encode($defaultBots, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT)); - } - $adminData['bot_patterns'] = json_decode((string) file_get_contents($botsFile), true) ?: []; + $adminData['stats_all_uas'] = $statsRaw['all_uas'] ?? []; } if ($tab === 'categories') { @@ -3248,9 +3251,45 @@ switch ($action) { array_map('trim', explode("\n", (string) ($_POST['bot_patterns'] ?? ''))) ))); $ok = @file_put_contents($botsFile, json_encode($patterns, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT)) !== false; + if ($ok) { + @unlink(DATA_PATH . '/.stats_cache.json'); + @unlink(BASE_PATH . '/_cache/access_stats.json'); + } header('Location: /admin/stats?' . ($ok ? 'saved=1' : 'error=write')); exit; + case 'admin_add_bot': + requireAuth(); + if (!isAdmin() || $_SERVER['REQUEST_METHOD'] !== 'POST') { + http_response_code(403); + exit; + } + $csrf = $_POST['_csrf'] ?? ''; + if ($csrf !== ($_session['csrf'] ?? '')) { + http_response_code(403); + header('Content-Type: application/json'); + echo json_encode(['ok' => false, 'error' => 'csrf']); + exit; + } + $addPattern = trim((string) ($_POST['pattern'] ?? '')); + if ($addPattern === '') { + http_response_code(400); + header('Content-Type: application/json'); + echo json_encode(['ok' => false, 'error' => 'empty']); + exit; + } + $botsFile = DATA_PATH . '/bots.json'; + $botPatterns = is_file($botsFile) ? (json_decode((string) file_get_contents($botsFile), true) ?: []) : []; + if (!in_array($addPattern, $botPatterns, true)) { + $botPatterns[] = $addPattern; + @file_put_contents($botsFile, json_encode($botPatterns, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT)); + } + @unlink(DATA_PATH . '/.stats_cache.json'); + @unlink(BASE_PATH . '/_cache/access_stats.json'); + header('Content-Type: application/json'); + echo json_encode(['ok' => true, 'pattern' => $addPattern]); + exit; + case 'admin_create_role': requireAuth(); if (!isAdmin() || $_SERVER['REQUEST_METHOD'] !== 'POST') { diff --git a/public/version.txt b/public/version.txt index 599e1a1..e038092 100644 --- a/public/version.txt +++ b/public/version.txt @@ -1 +1 @@ -1.6.31 +1.6.32 diff --git a/src/AccessLogParser.php b/src/AccessLogParser.php index c140236..4a5fc88 100644 --- a/src/AccessLogParser.php +++ b/src/AccessLogParser.php @@ -9,24 +9,31 @@ class AccessLogParser private string $cacheFile; private int $cacheTtl; private int $days; + /** @var list */ + private array $botPatterns; private static ?array $memo = null; // Apache COMBINED : IP - - [timestamp] "METHOD /path HTTP/x" STATUS bytes "ref" "ua" private const RE = '/^(\S+) \S+ \S+ \[(\d{2}\/\w+\/\d{4}:\d{2}:\d{2}:\d{2} [+-]\d{4})\] "[A-Z-]+ ([^\s"?]+)[^"]*" (\d{3}) \S+ "[^"]*" "([^"]*)"/u'; + /** + * @param list $botPatterns + */ public function __construct( - string $logDir = '/var/log/apache2', - string $pattern = '*-access.log', - string $cacheFile = '', - int $cacheTtl = 600, - int $days = 14 + string $logDir = '/var/log/apache2', + string $pattern = '*-access.log', + string $cacheFile = '', + int $cacheTtl = 600, + int $days = 14, + array $botPatterns = [] ) { - $this->logDir = rtrim($logDir, '/'); - $this->pattern = $pattern; - $this->cacheFile = $cacheFile !== '' ? $cacheFile : dirname(__DIR__) . '/_cache/access_stats.json'; - $this->cacheTtl = $cacheTtl; - $this->days = $days; + $this->logDir = rtrim($logDir, '/'); + $this->pattern = $pattern; + $this->cacheFile = $cacheFile !== '' ? $cacheFile : dirname(__DIR__) . '/_cache/access_stats.json'; + $this->cacheTtl = $cacheTtl; + $this->days = $days; + $this->botPatterns = array_map('strtolower', $botPatterns); } /** @@ -37,7 +44,8 @@ class AccessLogParser * pages_by_day:array>, * ips_by_day:array>, * ip_top_paths:array>, - * ip_agents:array> + * ip_agents:array>, + * all_uas:array * } */ public function stats(): array @@ -52,24 +60,26 @@ class AccessLogParser } } - $cutoff = strtotime("-{$this->days} days midnight") ?: (time() - $this->days * 86400); - $pages = []; - $books = []; - $ips = []; // toutes requêtes publiques (tous chemins, tous statuts) - $dayPages = []; - $ipPaths = []; // chemins /post/ et /book/ avec statut 200 (pour les ts) - $ipPathTs = []; - $ipAllPaths = []; // tous chemins, tous statuts - $ipAllDays = []; // tous jours, tous statuts - $ipAgents = []; // tous user-agents par IP + $cutoff = strtotime("-{$this->days} days midnight") ?: (time() - $this->days * 86400); + $pages = []; + $books = []; + $ips = []; // requêtes publiques non-bot (tous chemins, tous statuts) + $dayPages = []; + $ipPaths = []; // chemins /post/ et /book/ avec statut 200 (pour les ts) + $ipPathTs = []; + $ipAllPaths = []; // tous chemins, tous statuts, non-bots + $ipAllDays = []; // tous jours, tous statuts, non-bots + $ipAgents = []; // user-agents non-bot par IP + $allUas = []; // tous UAs publics (bots inclus) pour "Agents détectés" foreach ($this->logFiles() as $file) { - $this->parseFile($file, $cutoff, $pages, $books, $ips, $dayPages, $ipPaths, $ipPathTs, $ipAllPaths, $ipAllDays, $ipAgents); + $this->parseFile($file, $cutoff, $pages, $books, $ips, $dayPages, $ipPaths, $ipPathTs, $ipAllPaths, $ipAllDays, $ipAgents, $allUas); } arsort($pages); arsort($books); arsort($ips); + arsort($allUas); $pagesByDay = []; foreach ($dayPages as $path => $byOffset) { @@ -82,7 +92,7 @@ class AccessLogParser $pagesByDay[$path] = $arr; } - // Top 200 IPs par volume total de requêtes + // Top 200 IPs non-bot par volume total de requêtes $topIpKeys = array_keys(array_slice($ips, 0, 200, true)); $ipsByDay = []; $ipTopPaths = []; @@ -119,9 +129,11 @@ class AccessLogParser 'ips_by_day' => $ipsByDay, 'ip_top_paths' => $ipTopPaths, 'ip_agents' => $ipTopAgents, + 'all_uas' => array_slice($allUas, 0, 300, true), ]; @mkdir(dirname($this->cacheFile), 0755, true); @file_put_contents($this->cacheFile, json_encode($result, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)); + return self::$memo = $result; } @@ -136,6 +148,21 @@ class AccessLogParser && (time() - filemtime($this->cacheFile)) < $this->cacheTtl; } + private function matchesBot(string $ua): bool + { + if ($ua === '' || $this->botPatterns === []) { + return false; + } + $lo = strtolower($ua); + foreach ($this->botPatterns as $p) { + if ($p !== '' && str_contains($lo, $p)) { + return true; + } + } + + return false; + } + /** @return list */ private function logFiles(): array { @@ -172,6 +199,7 @@ class AccessLogParser if (!preg_match('/(\d{2})\/(\w{3})\/(\d{4}):(\d{2}:\d{2}:\d{2}) ([+-]\d{4})/', $raw, $m)) { return 0; } + return (int) strtotime("{$m[1]} {$m[2]} {$m[3]} {$m[4]} {$m[5]}"); } @@ -186,7 +214,8 @@ class AccessLogParser array &$ipPathTs, array &$ipAllPaths, array &$ipAllDays, - array &$ipAgents + array &$ipAgents, + array &$allUas ): void { if (!preg_match(self::RE, $line, $m)) { return; @@ -200,19 +229,25 @@ class AccessLogParser $publicIp = filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE) !== false; $dayOffset = (int) floor(($tsVal - $cutoff) / 86400); + $isBot = $this->matchesBot($ua); - // Toutes les requêtes publiques : comptage global, chemins, jours, agents - if ($publicIp) { + // Tous les UAs publics pour la section "Agents détectés" (bots inclus) + if ($publicIp && $ua !== '') { + $allUas[$ua] = ($allUas[$ua] ?? 0) + 1; + } + + // Requêtes publiques non-bot : comptage visiteurs, chemins, jours, agents + if ($publicIp && !$isBot) { $ips[$ip] = ($ips[$ip] ?? 0) + 1; $ipAllPaths[$ip][$path] = ($ipAllPaths[$ip][$path] ?? 0) + 1; $ipAllDays[$ip][$dayOffset] = ($ipAllDays[$ip][$dayOffset] ?? 0) + 1; if ($ua !== '') { - $ipAgents[$ip][$ua] = ($ipAgents[$ip][$ua] ?? 0) + 1; + $ipAgents[$ip][$ua] = ($ipAgents[$ip][$ua] ?? 0) + 1; } } - // Comptage spécifique aux pages de contenu (statut 200, /post/ ou /book/) - if ($status !== '200') { + // Comptage spécifique aux pages de contenu (statut 200, non-bot) + if ($status !== '200' || $isBot) { return; } @@ -247,7 +282,8 @@ class AccessLogParser array &$ipPathTs, array &$ipAllPaths, array &$ipAllDays, - array &$ipAgents + array &$ipAgents, + array &$allUas ): void { if ($file['type'] === 'tgz') { try { @@ -258,7 +294,7 @@ class AccessLogParser continue; } foreach (explode("\n", $content) as $line) { - $this->parseLine($line, $cutoff, $pages, $books, $ips, $dayPages, $ipPaths, $ipPathTs, $ipAllPaths, $ipAllDays, $ipAgents); + $this->parseLine($line, $cutoff, $pages, $books, $ips, $dayPages, $ipPaths, $ipPathTs, $ipAllPaths, $ipAllDays, $ipAgents, $allUas); } } } catch (\Exception $e) { @@ -271,7 +307,7 @@ class AccessLogParser while (!gzeof($h)) { $line = gzgets($h, 8192); if ($line !== false) { - $this->parseLine($line, $cutoff, $pages, $books, $ips, $dayPages, $ipPaths, $ipPathTs, $ipAllPaths, $ipAllDays, $ipAgents); + $this->parseLine($line, $cutoff, $pages, $books, $ips, $dayPages, $ipPaths, $ipPathTs, $ipAllPaths, $ipAllDays, $ipAgents, $allUas); } } gzclose($h); @@ -281,7 +317,7 @@ class AccessLogParser return; } while (($line = fgets($h)) !== false) { - $this->parseLine($line, $cutoff, $pages, $books, $ips, $dayPages, $ipPaths, $ipPathTs, $ipAllPaths, $ipAllDays, $ipAgents); + $this->parseLine($line, $cutoff, $pages, $books, $ips, $dayPages, $ipPaths, $ipPathTs, $ipAllPaths, $ipAllDays, $ipAgents, $allUas); } fclose($h); } diff --git a/templates/admin_stats.php b/templates/admin_stats.php index 36e33f9..81340f8 100644 --- a/templates/admin_stats.php +++ b/templates/admin_stats.php @@ -7,6 +7,7 @@ $_asList = $adminData['stats_as'] ?? []; $_pagesByDay = $adminData['stats_pages_by_day'] ?? []; $_ipData = $adminData['stats_ip_data'] ?? []; $_botPatterns = $adminData['bot_patterns'] ?? []; +$_allUas = $adminData['stats_all_uas'] ?? []; ?> @@ -29,6 +30,8 @@ var FOLIO_PAGES_BY_DAY = ; var FOLIO_IP_DATA = ; var FOLIO_BOT_PATTERNS = ; +var FOLIO_ALL_UAS = ; +var FOLIO_CSRF = ;