From 1023edd66b395ec62b6cc4315093fafe3bb9f9e5 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com.> Date: Sat, 14 Jun 2025 00:14:47 +0100 Subject: [PATCH] updates --- .../proprietary/audit/AuditEventType.java | 6 +- .../audit/ControllerAuditAspect.java | 15 ++--- .../model/security/PersistentAuditEvent.java | 12 +++- .../PersistentAuditEventRepository.java | 6 +- .../service/AuditCleanupService.java | 67 +++++++++++++++++-- .../proprietary/util/SecretMasker.java | 2 +- .../resources/templates/audit/dashboard.html | 66 ++++++++++++++---- 7 files changed, 142 insertions(+), 32 deletions(-) diff --git a/proprietary/src/main/java/stirling/software/proprietary/audit/AuditEventType.java b/proprietary/src/main/java/stirling/software/proprietary/audit/AuditEventType.java index 85e7fd245..421f41ce5 100644 --- a/proprietary/src/main/java/stirling/software/proprietary/audit/AuditEventType.java +++ b/proprietary/src/main/java/stirling/software/proprietary/audit/AuditEventType.java @@ -2,12 +2,11 @@ package stirling.software.proprietary.audit; /** * Standardized audit event types for the application. - * Using an enum ensures consistency in event type naming and categorization. */ public enum AuditEventType { // Authentication events - BASIC level USER_LOGIN("User login"), - USER_LOGOUT("User logout"), + USER_LOGOUT("User logout"), USER_FAILED_LOGIN("Failed login attempt"), // User/admin events - BASIC level @@ -17,8 +16,7 @@ public enum AuditEventType { SETTINGS_CHANGED("System or admin settings operation"), // File operations - STANDARD level - FILE_UPLOAD("File uploaded"), - FILE_DOWNLOAD("File downloaded"), + FILE_OPERATION("File operation"), // PDF operations - STANDARD level PDF_PROCESS("PDF processing operation"), diff --git a/proprietary/src/main/java/stirling/software/proprietary/audit/ControllerAuditAspect.java b/proprietary/src/main/java/stirling/software/proprietary/audit/ControllerAuditAspect.java index 8b0b175ed..8ec7402d7 100644 --- a/proprietary/src/main/java/stirling/software/proprietary/audit/ControllerAuditAspect.java +++ b/proprietary/src/main/java/stirling/software/proprietary/audit/ControllerAuditAspect.java @@ -176,15 +176,12 @@ public class ControllerAuditAspect { return AuditEventType.SETTINGS_CHANGED; } - // File operations - else if (className.contains("file") || path.contains("file")) { - if (path.contains("upload") || path.contains("add")) { - return AuditEventType.FILE_UPLOAD; - } else if (path.contains("download")) { - return AuditEventType.FILE_DOWNLOAD; - } else { - return AuditEventType.FILE_UPLOAD; - } + // File operations - using path prefixes to avoid false matches + else if (className.contains("file") || + path.startsWith("/file") || + path.startsWith("/files/") || + path.matches("(?i).*/(upload|download)/.*")) { + return AuditEventType.FILE_OPERATION; } // Default to PDF operations for most controllers diff --git a/proprietary/src/main/java/stirling/software/proprietary/model/security/PersistentAuditEvent.java b/proprietary/src/main/java/stirling/software/proprietary/model/security/PersistentAuditEvent.java index 1ea2482e1..8fb520156 100644 --- a/proprietary/src/main/java/stirling/software/proprietary/model/security/PersistentAuditEvent.java +++ b/proprietary/src/main/java/stirling/software/proprietary/model/security/PersistentAuditEvent.java @@ -2,11 +2,21 @@ package stirling.software.proprietary.model.security; import jakarta.persistence.*; import lombok.*; +import org.hibernate.annotations.Index; import java.time.Instant; @Entity -@Table(name = "audit_events") +@Table( + name = "audit_events", + indexes = { + @jakarta.persistence.Index(name = "idx_audit_timestamp", columnList = "timestamp"), + @jakarta.persistence.Index(name = "idx_audit_principal", columnList = "principal"), + @jakarta.persistence.Index(name = "idx_audit_type", columnList = "type"), + @jakarta.persistence.Index(name = "idx_audit_principal_type", columnList = "principal,type"), + @jakarta.persistence.Index(name = "idx_audit_type_timestamp", columnList = "type,timestamp") + } +) @Data @Builder @NoArgsConstructor @AllArgsConstructor public class PersistentAuditEvent { diff --git a/proprietary/src/main/java/stirling/software/proprietary/repository/PersistentAuditEventRepository.java b/proprietary/src/main/java/stirling/software/proprietary/repository/PersistentAuditEventRepository.java index ef603b52a..9c4d98ef2 100644 --- a/proprietary/src/main/java/stirling/software/proprietary/repository/PersistentAuditEventRepository.java +++ b/proprietary/src/main/java/stirling/software/proprietary/repository/PersistentAuditEventRepository.java @@ -38,7 +38,11 @@ public interface PersistentAuditEventRepository @Query("DELETE FROM PersistentAuditEvent e WHERE e.timestamp < ?1") @org.springframework.data.jpa.repository.Modifying @org.springframework.transaction.annotation.Transactional - void deleteByTimestampBefore(Instant cutoffDate); + int deleteByTimestampBefore(Instant cutoffDate); + + // Find IDs for batch deletion - using JPQL with setMaxResults instead of native query + @Query("SELECT e.id FROM PersistentAuditEvent e WHERE e.timestamp < ?1 ORDER BY e.id") + List findIdsForBatchDeletion(Instant cutoffDate, Pageable pageable); // Stats queries @Query("SELECT e.type, COUNT(e) FROM PersistentAuditEvent e GROUP BY e.type") diff --git a/proprietary/src/main/java/stirling/software/proprietary/service/AuditCleanupService.java b/proprietary/src/main/java/stirling/software/proprietary/service/AuditCleanupService.java index fafcb52e4..442deecb1 100644 --- a/proprietary/src/main/java/stirling/software/proprietary/service/AuditCleanupService.java +++ b/proprietary/src/main/java/stirling/software/proprietary/service/AuditCleanupService.java @@ -2,10 +2,14 @@ package stirling.software.proprietary.service; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.List; import java.util.concurrent.TimeUnit; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -23,6 +27,9 @@ public class AuditCleanupService { private final PersistentAuditEventRepository auditRepository; private final AuditConfigurationProperties auditConfig; + // Default batch size for deletions + private static final int BATCH_SIZE = 10000; + /** * Scheduled task that runs daily to clean up old audit events. * The retention period is configurable in settings.yml. @@ -30,13 +37,11 @@ public class AuditCleanupService { @Scheduled(fixedDelay = 1, initialDelay = 1, timeUnit = TimeUnit.DAYS) public void cleanupOldAuditEvents() { if (!auditConfig.isEnabled()) { - log.debug("Audit system is disabled, skipping cleanup"); return; } int retentionDays = auditConfig.getRetentionDays(); if (retentionDays <= 0) { - log.info("Audit retention is set to {} days, no cleanup needed", retentionDays); return; } @@ -44,10 +49,64 @@ public class AuditCleanupService { try { Instant cutoffDate = Instant.now().minus(retentionDays, ChronoUnit.DAYS); - auditRepository.deleteByTimestampBefore(cutoffDate); - log.info("Successfully cleaned up audit events older than {}", cutoffDate); + int totalDeleted = batchDeleteEvents(cutoffDate); + log.info("Successfully cleaned up {} audit events older than {}", totalDeleted, cutoffDate); } catch (Exception e) { log.error("Error cleaning up old audit events", e); } } + + /** + * Performs batch deletion of events to prevent long-running transactions + * and potential database locks. + */ + private int batchDeleteEvents(Instant cutoffDate) { + int totalDeleted = 0; + boolean hasMore = true; + + while (hasMore) { + // Start a new transaction for each batch + List batchIds = findBatchOfIdsToDelete(cutoffDate); + + if (batchIds.isEmpty()) { + hasMore = false; + } else { + int deleted = deleteBatch(batchIds); + totalDeleted += deleted; + + // If we got fewer records than the batch size, we're done + if (batchIds.size() < BATCH_SIZE) { + hasMore = false; + } + } + } + + return totalDeleted; + } + + /** + * Finds a batch of IDs to delete. + */ + @Transactional(readOnly = true) + private List findBatchOfIdsToDelete(Instant cutoffDate) { + PageRequest pageRequest = PageRequest.of(0, BATCH_SIZE, Sort.by("id")); + return auditRepository.findIdsForBatchDeletion(cutoffDate, pageRequest); + } + + /** + * Deletes a batch of events by ID. + * Each batch is in its own transaction. + */ + @Transactional + private int deleteBatch(List batchIds) { + if (batchIds.isEmpty()) { + return 0; + } + + int batchSize = batchIds.size(); + auditRepository.deleteAllByIdInBatch(batchIds); + log.debug("Deleted batch of {} audit events", batchSize); + + return batchSize; + } } \ No newline at end of file diff --git a/proprietary/src/main/java/stirling/software/proprietary/util/SecretMasker.java b/proprietary/src/main/java/stirling/software/proprietary/util/SecretMasker.java index b0d1fb283..00079fd02 100644 --- a/proprietary/src/main/java/stirling/software/proprietary/util/SecretMasker.java +++ b/proprietary/src/main/java/stirling/software/proprietary/util/SecretMasker.java @@ -9,7 +9,7 @@ import java.util.stream.Collectors; public final class SecretMasker { private static final Pattern SENSITIVE = - Pattern.compile("(?i)(password|token|secret|api[_-]?key|authorization|auth)"); + Pattern.compile("(?i)(password|token|secret|api[_-]?key|authorization|auth|jwt|cred|cert)"); private SecretMasker() {} diff --git a/proprietary/src/main/resources/templates/audit/dashboard.html b/proprietary/src/main/resources/templates/audit/dashboard.html index 40bb85079..85c5a59f8 100644 --- a/proprietary/src/main/resources/templates/audit/dashboard.html +++ b/proprietary/src/main/resources/templates/audit/dashboard.html @@ -502,8 +502,7 @@
  • HTTP_REQUEST - GET requests for viewing
  • PDF_PROCESS - PDF processing operations
    • -
  • FILE_UPLOAD - File uploads
    • -
  • FILE_DOWNLOAD - File downloads
    • +
  • FILE_OPERATION - File-related operations
  • SETTINGS_CHANGED - System or admin settings operations
@@ -835,8 +834,8 @@ datasets: [{ label: 'Events by Type', data: typeValues, - backgroundColor: 'rgba(54, 162, 235, 0.6)', - borderColor: 'rgba(54, 162, 235, 1)', + backgroundColor: getChartColors(typeLabels.length), + borderColor: getChartColors(typeLabels.length, 1), // Full opacity for borders borderWidth: 1 }] }, @@ -864,14 +863,7 @@ datasets: [{ label: 'Events by User', data: userValues, - backgroundColor: [ - 'rgba(54, 162, 235, 0.6)', - 'rgba(255, 99, 132, 0.6)', - 'rgba(255, 206, 86, 0.6)', - 'rgba(75, 192, 192, 0.6)', - 'rgba(153, 102, 255, 0.6)', - 'rgba(255, 159, 64, 0.6)' - ], + backgroundColor: getChartColors(userLabels.length), borderWidth: 1 }] }, @@ -938,6 +930,56 @@ const loading = document.getElementById(id); if (loading) loading.style.display = 'none'; } + + // Function to generate a palette of colors for charts + function getChartColors(count, opacity = 0.6) { + // Base colors - a larger palette than the default + const colors = [ + [54, 162, 235], // blue + [255, 99, 132], // red + [75, 192, 192], // teal + [255, 206, 86], // yellow + [153, 102, 255], // purple + [255, 159, 64], // orange + [46, 204, 113], // green + [231, 76, 60], // dark red + [52, 152, 219], // light blue + [155, 89, 182], // violet + [241, 196, 15], // dark yellow + [26, 188, 156], // turquoise + [230, 126, 34], // dark orange + [149, 165, 166], // light gray + [243, 156, 18], // amber + [39, 174, 96], // emerald + [211, 84, 0], // dark orange red + [22, 160, 133], // green sea + [41, 128, 185], // belize hole + [142, 68, 173] // wisteria + ]; + + const result = []; + + // Always use the same format regardless of color source + if (count > colors.length) { + // Generate colors algorithmically for large sets + for (let i = 0; i < count; i++) { + // Generate a color based on position in the hue circle (0-360) + const hue = (i * 360 / count) % 360; + const sat = 70 + Math.random() * 10; // 70-80% + const light = 50 + Math.random() * 10; // 50-60% + + result.push(`hsla(${hue}, ${sat}%, ${light}%, ${opacity})`); + } + } else { + // Use colors from our palette but also return in hsla format for consistency + for (let i = 0; i < count; i++) { + const color = colors[i % colors.length]; + result.push(`rgba(${color[0]}, ${color[1]}, ${color[2]}, ${opacity})`); + } + } + + return result; + }