initial commit

internal/parser/access.go

@@ -0,0 +1,70 @@
+package parser
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"time"
+)
+
+// AccessEntry holds a parsed nginx combined-format access log entry.
+type AccessEntry struct {
+	RemoteAddr string
+	RemoteUser string
+	Time       time.Time
+	Method     string
+	Path       string
+	Protocol   string
+	Status     int
+	BytesSent  int64
+	Referer    string
+	UserAgent  string
+}
+
+// accessLogRe matches nginx combined log format.
+// Groups: remoteAddr, remoteUser, time, method, path, protocol, status, bytes, referer, userAgent
+var accessLogRe = regexp.MustCompile(
+	`^(\S+) - (\S+) \[([^\]]+)\] "(\S+) (\S+) (\S+)" (\d+) (\d+|-) "([^"]*)" "([^"]*)"$`,
+)
+
+const accessTimeLayout = "02/Jan/2006:15:04:05 -0700"
+
+// ParseAccessLine parses a single nginx access log line.
+// Returns an error for malformed lines.
+func ParseAccessLine(line string) (AccessEntry, error) {
+	m := accessLogRe.FindStringSubmatch(line)
+	if m == nil {
+		return AccessEntry{}, fmt.Errorf("access: no match: %q", line)
+	}
+
+	t, err := time.Parse(accessTimeLayout, m[3])
+	if err != nil {
+		return AccessEntry{}, fmt.Errorf("access: parse time %q: %w", m[3], err)
+	}
+
+	status, err := strconv.Atoi(m[7])
+	if err != nil {
+		return AccessEntry{}, fmt.Errorf("access: parse status %q: %w", m[7], err)
+	}
+
+	var bytes int64
+	if m[8] != "-" {
+		bytes, err = strconv.ParseInt(m[8], 10, 64)
+		if err != nil {
+			return AccessEntry{}, fmt.Errorf("access: parse bytes %q: %w", m[8], err)
+		}
+	}
+
+	return AccessEntry{
+		RemoteAddr: m[1],
+		RemoteUser: m[2],
+		Time:       t,
+		Method:     m[4],
+		Path:       m[5],
+		Protocol:   m[6],
+		Status:     status,
+		BytesSent:  bytes,
+		Referer:    m[9],
+		UserAgent:  m[10],
+	}, nil
+}

internal/parser/access_test.go

@@ -0,0 +1,110 @@
+package parser
+
+import (
+	"testing"
+	"time"
+)
+
+func TestParseAccessLine_Valid(t *testing.T) {
+	line := `192.168.1.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"`
+	entry, err := ParseAccessLine(line)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if entry.RemoteAddr != "192.168.1.1" {
+		t.Errorf("RemoteAddr = %q, want %q", entry.RemoteAddr, "192.168.1.1")
+	}
+	if entry.RemoteUser != "frank" {
+		t.Errorf("RemoteUser = %q, want %q", entry.RemoteUser, "frank")
+	}
+	want := time.Date(2000, 10, 10, 13, 55, 36, 0, time.FixedZone("", -7*3600))
+	if !entry.Time.Equal(want) {
+		t.Errorf("Time = %v, want %v", entry.Time, want)
+	}
+	if entry.Method != "GET" {
+		t.Errorf("Method = %q, want %q", entry.Method, "GET")
+	}
+	if entry.Path != "/apache_pb.gif" {
+		t.Errorf("Path = %q, want %q", entry.Path, "/apache_pb.gif")
+	}
+	if entry.Protocol != "HTTP/1.0" {
+		t.Errorf("Protocol = %q, want %q", entry.Protocol, "HTTP/1.0")
+	}
+	if entry.Status != 200 {
+		t.Errorf("Status = %d, want %d", entry.Status, 200)
+	}
+	if entry.BytesSent != 2326 {
+		t.Errorf("BytesSent = %d, want %d", entry.BytesSent, 2326)
+	}
+	if entry.Referer != "http://www.example.com/start.html" {
+		t.Errorf("Referer = %q, want %q", entry.Referer, "http://www.example.com/start.html")
+	}
+}
+
+func TestParseAccessLine_DashFields(t *testing.T) {
+	line := `10.0.0.1 - - [15/Jan/2024:08:23:41 +0000] "POST /api/users HTTP/1.1" 201 512 "-" "curl/7.68.0"`
+	entry, err := ParseAccessLine(line)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if entry.RemoteUser != "-" {
+		t.Errorf("RemoteUser = %q, want %q", entry.RemoteUser, "-")
+	}
+	if entry.Referer != "-" {
+		t.Errorf("Referer = %q, want %q", entry.Referer, "-")
+	}
+	if entry.BytesSent != 512 {
+		t.Errorf("BytesSent = %d, want %d", entry.BytesSent, 512)
+	}
+}
+
+func TestParseAccessLine_BytesDash(t *testing.T) {
+	// bytes sent as "-" should parse as 0
+	line := `127.0.0.1 - - [22/Feb/2024:10:00:00 +0000] "GET /health HTTP/1.1" 200 - "-" "kube-probe/1.27"`
+	entry, err := ParseAccessLine(line)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if entry.BytesSent != 0 {
+		t.Errorf("BytesSent = %d, want 0", entry.BytesSent)
+	}
+}
+
+func TestParseAccessLine_IPv6(t *testing.T) {
+	line := `2001:db8::1 - - [20/Feb/2024:16:45:00 +0000] "GET /index.html HTTP/2.0" 200 4096 "-" "Googlebot/2.1"`
+	entry, err := ParseAccessLine(line)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if entry.RemoteAddr != "2001:db8::1" {
+		t.Errorf("RemoteAddr = %q, want %q", entry.RemoteAddr, "2001:db8::1")
+	}
+}
+
+func TestParseAccessLine_Malformed(t *testing.T) {
+	cases := []string{
+		"this is a malformed line",
+		"",
+		"incomplete line without proper format",
+	}
+	for _, c := range cases {
+		_, err := ParseAccessLine(c)
+		if err == nil {
+			t.Errorf("expected error for malformed line %q, got nil", c)
+		}
+	}
+}
+
+func TestParseAccessLine_Status(t *testing.T) {
+	line := `172.16.0.50 - admin [01/Mar/2024:12:00:00 +0100] "DELETE /resource/42 HTTP/1.1" 404 0 "https://example.org/page" "Mozilla/5.0"`
+	entry, err := ParseAccessLine(line)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if entry.Status != 404 {
+		t.Errorf("Status = %d, want 404", entry.Status)
+	}
+	if entry.Method != "DELETE" {
+		t.Errorf("Method = %q, want DELETE", entry.Method)
+	}
+}

internal/parser/error.go

@@ -0,0 +1,67 @@
+package parser
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"time"
+)
+
+// ErrorEntry holds a parsed nginx error log entry.
+type ErrorEntry struct {
+	Time    time.Time
+	Level   string // debug/info/notice/warn/error/crit/alert/emerg
+	PID     int
+	TID     int
+	ConnID  int // 0 if not present
+	Message string
+}
+
+// errorLogRe matches nginx error log format.
+// Groups: time, level, pid, tid, connID (optional), message
+var errorLogRe = regexp.MustCompile(
+	`^(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(\w+)\] (\d+)#(\d+): (?:\*(\d+) )?(.+)$`,
+)
+
+const errorTimeLayout = "2006/01/02 15:04:05"
+
+// ParseErrorLine parses a single nginx error log line.
+// Returns an error for malformed lines.
+func ParseErrorLine(line string) (ErrorEntry, error) {
+	m := errorLogRe.FindStringSubmatch(line)
+	if m == nil {
+		return ErrorEntry{}, fmt.Errorf("error: no match: %q", line)
+	}
+
+	t, err := time.ParseInLocation(errorTimeLayout, m[1], time.UTC)
+	if err != nil {
+		return ErrorEntry{}, fmt.Errorf("error: parse time %q: %w", m[1], err)
+	}
+
+	pid, err := strconv.Atoi(m[3])
+	if err != nil {
+		return ErrorEntry{}, fmt.Errorf("error: parse pid %q: %w", m[3], err)
+	}
+
+	tid, err := strconv.Atoi(m[4])
+	if err != nil {
+		return ErrorEntry{}, fmt.Errorf("error: parse tid %q: %w", m[4], err)
+	}
+
+	var connID int
+	if m[5] != "" {
+		connID, err = strconv.Atoi(m[5])
+		if err != nil {
+			return ErrorEntry{}, fmt.Errorf("error: parse connid %q: %w", m[5], err)
+		}
+	}
+
+	return ErrorEntry{
+		Time:    t,
+		Level:   m[2],
+		PID:     pid,
+		TID:     tid,
+		ConnID:  connID,
+		Message: m[6],
+	}, nil
+}

internal/parser/error_test.go

@@ -0,0 +1,93 @@
+package parser
+
+import (
+	"testing"
+	"time"
+)
+
+func TestParseErrorLine_WithConnID(t *testing.T) {
+	line := `2024/02/20 16:45:01 [error] 1234#5678: *99 connect() failed (111: Connection refused) while connecting to upstream`
+	entry, err := ParseErrorLine(line)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	want := time.Date(2024, 2, 20, 16, 45, 1, 0, time.UTC)
+	if !entry.Time.Equal(want) {
+		t.Errorf("Time = %v, want %v", entry.Time, want)
+	}
+	if entry.Level != "error" {
+		t.Errorf("Level = %q, want %q", entry.Level, "error")
+	}
+	if entry.PID != 1234 {
+		t.Errorf("PID = %d, want 1234", entry.PID)
+	}
+	if entry.TID != 5678 {
+		t.Errorf("TID = %d, want 5678", entry.TID)
+	}
+	if entry.ConnID != 99 {
+		t.Errorf("ConnID = %d, want 99", entry.ConnID)
+	}
+	if entry.Message != "connect() failed (111: Connection refused) while connecting to upstream" {
+		t.Errorf("Message = %q", entry.Message)
+	}
+}
+
+func TestParseErrorLine_WithoutConnID(t *testing.T) {
+	line := `2024/02/20 17:00:00 [warn] 1234#5678: worker process 9999 exited on signal 15`
+	entry, err := ParseErrorLine(line)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if entry.Level != "warn" {
+		t.Errorf("Level = %q, want %q", entry.Level, "warn")
+	}
+	if entry.ConnID != 0 {
+		t.Errorf("ConnID = %d, want 0", entry.ConnID)
+	}
+	if entry.Message != "worker process 9999 exited on signal 15" {
+		t.Errorf("Message = %q", entry.Message)
+	}
+}
+
+func TestParseErrorLine_AllLevels(t *testing.T) {
+	levels := []string{"debug", "info", "notice", "warn", "error", "crit", "alert", "emerg"}
+	for _, level := range levels {
+		line := "2024/01/01 00:00:00 [" + level + "] 100#200: test message"
+		entry, err := ParseErrorLine(line)
+		if err != nil {
+			t.Errorf("level %q: unexpected error: %v", level, err)
+			continue
+		}
+		if entry.Level != level {
+			t.Errorf("level %q: got Level = %q", level, entry.Level)
+		}
+	}
+}
+
+func TestParseErrorLine_Info(t *testing.T) {
+	line := `2024/02/21 08:30:00 [info] 1234#5678: start worker process 10001`
+	entry, err := ParseErrorLine(line)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if entry.Level != "info" {
+		t.Errorf("Level = %q, want info", entry.Level)
+	}
+	if entry.PID != 1234 {
+		t.Errorf("PID = %d, want 1234", entry.PID)
+	}
+}
+
+func TestParseErrorLine_Malformed(t *testing.T) {
+	cases := []string{
+		"this is a malformed line",
+		"",
+		"2024/02/20 not-a-proper-format",
+	}
+	for _, c := range cases {
+		_, err := ParseErrorLine(c)
+		if err == nil {
+			t.Errorf("expected error for malformed line %q, got nil", c)
+		}
+	}
+}

internal/parser/scanner.go

@@ -0,0 +1,167 @@
+package parser
+
+import (
+	"bufio"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// ScanDirectory scans dir for nginx access and error log files.
+// It matches filenames like access*.log* and error*.log*.
+// If includeGzip is false, .gz files are skipped.
+// Returns (accessFiles, errorFiles, error) sorted newest-first.
+func ScanDirectory(dir string, includeGzip bool) ([]string, []string, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, nil, fmt.Errorf("scandir %q: %w", dir, err)
+	}
+
+	var accessFiles, errorFiles []string
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		name := e.Name()
+		if !includeGzip && strings.HasSuffix(name, ".gz") {
+			continue
+		}
+		if matchesLogPattern(name, "access") {
+			accessFiles = append(accessFiles, filepath.Join(dir, name))
+		} else if matchesLogPattern(name, "error") {
+			errorFiles = append(errorFiles, filepath.Join(dir, name))
+		}
+	}
+
+	sortLogFiles(accessFiles)
+	sortLogFiles(errorFiles)
+	return accessFiles, errorFiles, nil
+}
+
+// matchesLogPattern returns true if filename starts with prefix and contains ".log".
+func matchesLogPattern(name, prefix string) bool {
+	if !strings.HasPrefix(name, prefix) {
+		return false
+	}
+	return strings.Contains(name, ".log")
+}
+
+// logSortKey returns a sort key such that:
+//   - "access.log"      → (0, 0)   — most recent
+//   - "access.log.1"    → (1, 0)
+//   - "access.log.1.gz" → (1, 1)
+//   - "access.log.2.gz" → (2, 1)
+func logSortKey(path string) (int, int) {
+	name := filepath.Base(path)
+	// strip prefix up to and including ".log"
+	idx := strings.Index(name, ".log")
+	if idx < 0 {
+		return 999, 0
+	}
+	suffix := name[idx+4:] // e.g. "", ".1", ".1.gz", ".2.gz"
+
+	isGz := 0
+	if strings.HasSuffix(suffix, ".gz") {
+		isGz = 1
+		suffix = strings.TrimSuffix(suffix, ".gz")
+	}
+	suffix = strings.TrimPrefix(suffix, ".")
+
+	num := 0
+	if suffix != "" {
+		fmt.Sscanf(suffix, "%d", &num)
+	}
+	return num, isGz
+}
+
+func sortLogFiles(files []string) {
+	sort.SliceStable(files, func(i, j int) bool {
+		ni, gi := logSortKey(files[i])
+		nj, gj := logSortKey(files[j])
+		if ni != nj {
+			return ni < nj
+		}
+		return gi < gj
+	})
+}
+
+// openLogFile opens a log file for reading.
+// If the path ends in ".gz", it transparently decompresses.
+func openLogFile(path string) (io.ReadCloser, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("open %q: %w", path, err)
+	}
+	if strings.HasSuffix(path, ".gz") {
+		gz, err := gzip.NewReader(f)
+		if err != nil {
+			f.Close()
+			return nil, fmt.Errorf("gzip %q: %w", path, err)
+		}
+		return &gzipReadCloser{gz: gz, f: f}, nil
+	}
+	return f, nil
+}
+
+// gzipReadCloser closes both the gzip reader and underlying file.
+type gzipReadCloser struct {
+	gz *gzip.Reader
+	f  *os.File
+}
+
+func (g *gzipReadCloser) Read(p []byte) (int, error) { return g.gz.Read(p) }
+func (g *gzipReadCloser) Close() error {
+	err := g.gz.Close()
+	g.f.Close()
+	return err
+}
+
+// ReadAccessEntries reads all access log entries from the given files.
+// Malformed lines are silently skipped.
+func ReadAccessEntries(files []string) ([]AccessEntry, error) {
+	var entries []AccessEntry
+	for _, path := range files {
+		rc, err := openLogFile(path)
+		if err != nil {
+			return nil, err
+		}
+		scanner := bufio.NewScanner(rc)
+		for scanner.Scan() {
+			if e, err := ParseAccessLine(scanner.Text()); err == nil {
+				entries = append(entries, e)
+			}
+		}
+		rc.Close()
+		if err := scanner.Err(); err != nil {
+			return nil, fmt.Errorf("scan %q: %w", path, err)
+		}
+	}
+	return entries, nil
+}
+
+// ReadErrorEntries reads all error log entries from the given files.
+// Malformed lines are silently skipped.
+func ReadErrorEntries(files []string) ([]ErrorEntry, error) {
+	var entries []ErrorEntry
+	for _, path := range files {
+		rc, err := openLogFile(path)
+		if err != nil {
+			return nil, err
+		}
+		scanner := bufio.NewScanner(rc)
+		for scanner.Scan() {
+			if e, err := ParseErrorLine(scanner.Text()); err == nil {
+				entries = append(entries, e)
+			}
+		}
+		rc.Close()
+		if err := scanner.Err(); err != nil {
+			return nil, fmt.Errorf("scan %q: %w", path, err)
+		}
+	}
+	return entries, nil
+}

internal/parser/scanner_test.go

@@ -0,0 +1,155 @@
+package parser
+
+import (
+	"bufio"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestScanDirectory_FindsAccessAndErrorLogs(t *testing.T) {
+	dir := t.TempDir()
+
+	// Create test log files
+	files := []string{
+		"access.log",
+		"access.log.1",
+		"error.log",
+		"error.log.1",
+		"other.txt", // should be ignored
+	}
+	for _, f := range files {
+		if err := os.WriteFile(filepath.Join(dir, f), []byte(""), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	accessFiles, errorFiles, err := ScanDirectory(dir, false)
+	if err != nil {
+		t.Fatalf("ScanDirectory error: %v", err)
+	}
+	if len(accessFiles) != 2 {
+		t.Errorf("expected 2 access files, got %d: %v", len(accessFiles), accessFiles)
+	}
+	if len(errorFiles) != 2 {
+		t.Errorf("expected 2 error files, got %d: %v", len(errorFiles), errorFiles)
+	}
+}
+
+func TestScanDirectory_GzipIncluded(t *testing.T) {
+	dir := t.TempDir()
+	files := []string{
+		"access.log",
+		"access.log.1.gz",
+		"error.log",
+	}
+	for _, f := range files {
+		if err := os.WriteFile(filepath.Join(dir, f), []byte(""), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	accessFiles, _, err := ScanDirectory(dir, true)
+	if err != nil {
+		t.Fatalf("ScanDirectory error: %v", err)
+	}
+	if len(accessFiles) != 2 {
+		t.Errorf("expected 2 access files (including gz), got %d", len(accessFiles))
+	}
+}
+
+func TestScanDirectory_GzipExcluded(t *testing.T) {
+	dir := t.TempDir()
+	files := []string{
+		"access.log",
+		"access.log.1.gz",
+	}
+	for _, f := range files {
+		if err := os.WriteFile(filepath.Join(dir, f), []byte(""), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	accessFiles, _, err := ScanDirectory(dir, false)
+	if err != nil {
+		t.Fatalf("ScanDirectory error: %v", err)
+	}
+	if len(accessFiles) != 1 {
+		t.Errorf("expected 1 access file (gz excluded), got %d", len(accessFiles))
+	}
+}
+
+func TestScanDirectory_NonexistentDir(t *testing.T) {
+	_, _, err := ScanDirectory("/nonexistent/path/xyz", true)
+	if err == nil {
+		t.Error("expected error for nonexistent directory, got nil")
+	}
+}
+
+func TestOpenLogFile_PlainText(t *testing.T) {
+	dir := t.TempDir()
+	content := "test line\n"
+	path := filepath.Join(dir, "test.log")
+	if err := os.WriteFile(path, []byte(content), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	rc, err := openLogFile(path)
+	if err != nil {
+		t.Fatalf("openLogFile error: %v", err)
+	}
+	defer rc.Close()
+
+	scanner := bufio.NewScanner(rc)
+	if !scanner.Scan() {
+		t.Fatal("expected to read a line")
+	}
+	if scanner.Text() != "test line" {
+		t.Errorf("got %q, want %q", scanner.Text(), "test line")
+	}
+}
+
+func TestOpenLogFile_Gzip(t *testing.T) {
+	// Use the pre-made fixture
+	rc, err := openLogFile("testdata/access.log.1.gz")
+	if err != nil {
+		t.Fatalf("openLogFile gz error: %v", err)
+	}
+	defer rc.Close()
+
+	scanner := bufio.NewScanner(rc)
+	if !scanner.Scan() {
+		t.Fatal("expected to read a line from gz file")
+	}
+	line := scanner.Text()
+	if line == "" {
+		t.Error("expected non-empty line from gz file")
+	}
+}
+
+func TestScanDirectory_SortOrder(t *testing.T) {
+	dir := t.TempDir()
+	files := []string{
+		"access.log.2.gz",
+		"access.log",
+		"access.log.1",
+		"access.log.1.gz",
+	}
+	for _, f := range files {
+		if err := os.WriteFile(filepath.Join(dir, f), []byte(""), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	accessFiles, _, err := ScanDirectory(dir, true)
+	if err != nil {
+		t.Fatalf("ScanDirectory error: %v", err)
+	}
+	if len(accessFiles) != 4 {
+		t.Fatalf("expected 4 files, got %d", len(accessFiles))
+	}
+	// access.log should come first (most recent)
+	if filepath.Base(accessFiles[0]) != "access.log" {
+		t.Errorf("expected access.log first, got %q", filepath.Base(accessFiles[0]))
+	}
+}

internal/parser/testdata/access.log

@@ -0,0 +1,6 @@
+192.168.1.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
+10.0.0.1 - - [15/Jan/2024:08:23:41 +0000] "POST /api/users HTTP/1.1" 201 512 "-" "curl/7.68.0"
+172.16.0.50 - admin [01/Mar/2024:12:00:00 +0100] "DELETE /resource/42 HTTP/1.1" 404 0 "https://example.org/page" "Mozilla/5.0"
+2001:db8::1 - - [20/Feb/2024:16:45:00 +0000] "GET /index.html HTTP/2.0" 200 4096 "-" "Googlebot/2.1"
+this is a malformed line that should be skipped
+127.0.0.1 - - [22/Feb/2024:10:00:00 +0000] "GET /health HTTP/1.1" 200 - "-" "kube-probe/1.27"

internal/parser/testdata/error.log

@@ -0,0 +1,6 @@
+2024/02/20 16:45:01 [error] 1234#5678: *99 connect() failed (111: Connection refused) while connecting to upstream
+2024/02/20 17:00:00 [warn] 1234#5678: worker process 9999 exited on signal 15
+2024/02/21 08:30:00 [info] 1234#5678: start worker process 10001
+2024/02/21 09:00:00 [crit] 9999#0: *1 SSL_do_handshake() failed (SSL: error:14094412) while SSL handshaking
+this is a malformed error line
+2024/02/22 10:00:00 [notice] 1234#5678: signal process started