修改log收集逻辑

This commit is contained in:
Alex Yang
2025-12-05 00:30:05 +08:00
parent baeece18ef
commit 8944fceeed
5 changed files with 362 additions and 7203 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -8,6 +8,7 @@ import (
stdnet "net" stdnet "net"
"net/http" "net/http"
"os" "os"
"path/filepath"
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
@@ -738,25 +739,49 @@ func collectDiskDetails() ([]DiskDetailMetrics, error) {
// 采集系统日志 // 采集系统日志
func collectLogs() ([]LogEntry, error) { func collectLogs() ([]LogEntry, error) {
// 日志文件路径 // 日志目录
logFile := "/var/log/messages" logDir := "/var/log"
log.Printf("Attempting to collect logs from %s", logFile) log.Printf("Attempting to collect logs from %s", logDir)
// 收集所有.log文件
logFiles, err := filepath.Glob(filepath.Join(logDir, "*.log"))
if err != nil {
log.Printf("Failed to match log files: %v", err)
return nil, err
}
// 过滤掉*.log.*文件
filteredFiles := make([]string, 0, len(logFiles))
for _, file := range logFiles {
base := filepath.Base(file)
// 只保留真正的*.log文件排除*.log.*文件
if ext := filepath.Ext(strings.TrimSuffix(base, ".log")); ext == "" {
filteredFiles = append(filteredFiles, file)
}
}
log.Printf("Found %d log files to process", len(filteredFiles))
// 创建日志条目切片
logs := make([]LogEntry, 0, 50) // 最多保存50条日志
// 处理每个日志文件
for _, logFile := range filteredFiles {
log.Printf("Processing log file: %s", logFile)
// 检查文件是否存在和权限 // 检查文件是否存在和权限
fileInfo, err := os.Stat(logFile) fileInfo, err := os.Stat(logFile)
if err != nil { if err != nil {
log.Printf("Failed to stat log file %s: %v", logFile, err) log.Printf("Failed to stat log file %s: %v", logFile, err)
return nil, fmt.Errorf("failed to stat log file %s: %w", logFile, err) continue
} }
log.Printf("Log file %s exists, size: %d bytes", logFile, fileInfo.Size())
// 打开日志文件 // 打开日志文件
file, err := os.Open(logFile) file, err := os.Open(logFile)
if err != nil { if err != nil {
log.Printf("Failed to open log file %s: %v", logFile, err) log.Printf("Failed to open log file %s: %v", logFile, err)
return nil, fmt.Errorf("failed to open log file %s: %w", logFile, err) continue
} }
defer file.Close()
// 读取文件末尾内容 // 读取文件末尾内容
const maxReadSize = 1024 * 1024 // 最多读取1MB const maxReadSize = 1024 * 1024 // 最多读取1MB
@@ -764,22 +789,18 @@ func collectLogs() ([]LogEntry, error) {
if fileInfo.Size() < int64(maxReadSize) { if fileInfo.Size() < int64(maxReadSize) {
readSize = int(fileInfo.Size()) readSize = int(fileInfo.Size())
} }
log.Printf("Reading %d bytes from log file", readSize)
buf := make([]byte, readSize) buf := make([]byte, readSize)
bytesRead, err := file.ReadAt(buf, fileInfo.Size()-int64(readSize)) bytesRead, err := file.ReadAt(buf, fileInfo.Size()-int64(readSize))
file.Close() // 关闭文件
if err != nil { if err != nil {
log.Printf("Failed to read log file: %v", err) log.Printf("Failed to read log file %s: %v", logFile, err)
return nil, fmt.Errorf("failed to read log file: %w", err) continue
} }
log.Printf("Successfully read %d bytes from log file", bytesRead)
// 分割日志行 // 分割日志行
lines := bytes.Split(buf[:bytesRead], []byte("\n")) lines := bytes.Split(buf[:bytesRead], []byte("\n"))
log.Printf("Found %d lines in log file", len(lines))
// 创建日志条目切片
logs := make([]LogEntry, 0, 50) // 最多保存50条日志
// 从后往前解析日志行 // 从后往前解析日志行
for i := len(lines) - 1; i >= 0 && len(logs) < 50; i-- { for i := len(lines) - 1; i >= 0 && len(logs) < 50; i-- {
@@ -788,30 +809,23 @@ func collectLogs() ([]LogEntry, error) {
continue continue
} }
// 打印前5行日志行用于调试
if i >= len(lines)-5 {
log.Printf("Processing log line %d: %s", i, string(line))
}
// 使用字符串处理,更方便处理空格 // 使用字符串处理,更方便处理空格
lineStr := string(line) lineStr := string(line)
// 使用strings.Fields分割日志行自动处理连续空格 // 使用strings.Fields分割日志行自动处理连续空格
fields := strings.Fields(lineStr) fields := strings.Fields(lineStr)
log.Printf("Line %d fields: %v, length: %d", i, fields, len(fields))
if len(fields) < 6 { if len(fields) < 6 {
log.Printf("Skipping line %d: not enough fields (%d) after splitting", i, len(fields)) // 尝试其他时间格式或跳过
continue continue
} }
// 构建时间字符串:月份 日期 时间 // 构建时间字符串:月份 日期 时间
timeStr := fmt.Sprintf("%s %s %s", fields[0], fields[1], fields[2]) timeStr := fmt.Sprintf("%s %s %s", fields[0], fields[1], fields[2])
log.Printf("Line %d timeStr: '%s'", i, timeStr)
// 解析时间 // 解析时间
t, err := time.Parse("Jan 2 15:04:05", timeStr) t, err := time.Parse("Jan 2 15:04:05", timeStr)
if err != nil { if err != nil {
log.Printf("Skipping line %d: failed to parse time '%s': %v", i, timeStr, err) // 尝试其他时间格式
continue continue
} }
@@ -822,12 +836,11 @@ func collectLogs() ([]LogEntry, error) {
// 寻找第一个冒号用于分割source和message // 寻找第一个冒号用于分割source和message
colonIndex := strings.Index(lineStr, ": ") colonIndex := strings.Index(lineStr, ": ")
if colonIndex == -1 { if colonIndex == -1 {
log.Printf("Skipping line %d: no colon found to split source and message", i)
continue continue
} }
// 解析source和message // 解析source和message添加文件名作为source的一部分
source := lineStr[:colonIndex] source := fmt.Sprintf("%s %s", filepath.Base(logFile), lineStr[:colonIndex])
message := lineStr[colonIndex+2:] message := lineStr[colonIndex+2:]
// 创建日志条目 // 创建日志条目
@@ -842,6 +855,11 @@ func collectLogs() ([]LogEntry, error) {
logs = append([]LogEntry{logEntry}, logs...) logs = append([]LogEntry{logEntry}, logs...)
} }
if len(logs) >= 50 {
break // 达到最大日志数量,停止处理
}
}
log.Printf("Successfully collected %d logs", len(logs)) log.Printf("Successfully collected %d logs", len(logs))
return logs, nil return logs, nil
} }

Binary file not shown.

10
test_agent.json Normal file
View File

@@ -0,0 +1,10 @@
{
"server_url": "http://localhost:8080/api",
"id": "test-agent",
"name": "Test Agent",
"device_id": "test-device",
"token": "bb30bfaee01bf7b541bbefe422f72645",
"interval": "5s",
"debug": true,
"api_port": 8081
}

11
test_metrics.sh Executable file
View File

@@ -0,0 +1,11 @@
#!/bin/bash
# 测试发送单个指标对象
echo "测试发送单个指标对象:"
curl -v -X POST -H "Content-Type: application/json" -H "X-Device-ID: test-device" -H "X-Agent-Name: test-agent" -H "X-Device-Token: test-token" -d '{"cpu": 50.5, "memory": 30.2, "disk": {":/": 45.6}, "network": {"eth0": {"bytes_sent": 1000, "bytes_received": 2000}}}' http://localhost:8080/api/metrics/
echo "\n---\n"
# 测试发送指标数组
echo "测试发送指标数组:"
curl -v -X POST -H "Content-Type: application/json" -H "X-Device-ID: test-device" -H "X-Agent-Name: test-agent" -H "X-Device-Token: test-token" -d '[{"cpu": 50.5, "memory": 30.2, "disk": {":/": 45.6}, "network": {"eth0": {"bytes_sent": 1000, "bytes_received": 2000}}}, {"cpu": 60.5, "memory": 40.2, "disk": {":/": 55.6}, "network": {"eth0": {"bytes_sent": 2000, "bytes_received": 3000}}}]' http://localhost:8080/api/metrics/