commit 885e7d6e0247ec61c6328984a5a4ef0e9af59658 Author: mr0xb <47467008+mr0xb@users.noreply.github.com> Date: Mon Aug 24 14:44:54 2026 -0400 initial commit diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8bbb7a3 --- /dev/null +++ b/Makefile @@ -0,0 +1,62 @@ +BINARY := nxstats +MODULE := github.com/mr0xb/nxstats +VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") +LDFLAGS := -ldflags "-s -w -X main.version=$(VERSION)" +OUTDIR := dist + +TARGETS := \ + darwin/amd64 \ + darwin/arm64 \ + linux/amd64 \ + linux/arm64 \ + linux/arm + +.PHONY: all clean test test-verbose test-cover build $(TARGETS) + +all: test build + +## build — compile for all target platforms +build: $(TARGETS) + +darwin/amd64: + GOOS=darwin GOARCH=amd64 go build $(LDFLAGS) -o $(OUTDIR)/$(BINARY)-darwin-amd64 . + +darwin/arm64: + GOOS=darwin GOARCH=arm64 go build $(LDFLAGS) -o $(OUTDIR)/$(BINARY)-darwin-arm64 . + +linux/amd64: + GOOS=linux GOARCH=amd64 go build $(LDFLAGS) -o $(OUTDIR)/$(BINARY)-linux-amd64 . + +linux/arm64: + GOOS=linux GOARCH=arm64 go build $(LDFLAGS) -o $(OUTDIR)/$(BINARY)-linux-arm64 . + +linux/arm: + GOOS=linux GOARCH=arm GOARM=7 go build $(LDFLAGS) -o $(OUTDIR)/$(BINARY)-linux-armv7 . + +## test — run all tests +test: + go test ./... + +## test-verbose — run all tests with per-test output +test-verbose: + go test -v ./... + +## test-cover — run all tests and open an HTML coverage report +test-cover: + go test -coverprofile=coverage.out ./... + go tool cover -html=coverage.out -o coverage.html + @echo "Coverage report: coverage.html" + +## clean — remove build artefacts and coverage files +clean: + rm -rf $(OUTDIR) coverage.out coverage.html + +$(OUTDIR): + mkdir -p $(OUTDIR) + +# Ensure output directory exists before any build target runs +darwin/amd64 darwin/arm64 linux/amd64 linux/arm64 linux/arm: | $(OUTDIR) + +## help — list available targets +help: + @grep -E '^## ' Makefile | sed 's/## / /' diff --git a/README.md b/README.md new file mode 100644 index 0000000..b4f9ae5 --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +> **AI-Generated Content Warning:** This project was developed with the assistance of generative AI (Claude by Anthropic). Code, documentation, and design decisions may reflect AI-generated output and should be reviewed accordingly before use in production environments. + +--- + +# // NXSTATS // + +A fast nginx log parser that generates cyberpunk-themed HTML reports with charts, searchable tables, and optional GeoIP2 hit maps. + +![Dashboard overview](screenshot1.png) + +![Access log table](screenshot2.png) + +## Features + +- Parses nginx **access** and **error** logs, including gzip-compressed rotated logs +- Generates a self-contained **single-file HTML report** — no server required +- **Status code distribution** bar chart +- **Top paths** and **requests-per-hour** charts (Chart.js) +- Searchable, filterable **access log table** with status badges +- Optional **GeoIP2 world map** (Leaflet.js) when a MaxMind `.mmdb` is supplied +- **Split-by-day** mode writes one report per calendar day plus a summary index page +- Zero runtime dependencies — the binary is statically compiled Go + +## Installation + +### Build from source + +Requires Go 1.21+. + +```bash +git clone https://github.com/mr0xb/nxstats.git +cd nxstats +go build -o nxstats . +``` + +### Pre-built binary + +Download the latest release binary from the [releases page](../../releases) and place it somewhere on your `$PATH`. + +## Usage + +``` +nxstats [flags] +``` + +| Flag | Short | Default | Description | +|---|---|---|---| +| `--dir` | `-d` | `/var/log/nginx` | Directory to scan for nginx logs | +| `--output` | `-o` | `report.html` | Output HTML file path | +| `--access-log` | `-a` | | Specific access log file (skips dir scan) | +| `--error-log` | `-e` | | Specific error log file (skips dir scan) | +| `--geoip` | | | Path to MaxMind `GeoLite2-City.mmdb` (enables map) | +| `--no-gzip` | | `false` | Skip `.gz` compressed rotated logs | +| `--split-by-day` | | `false` | Write one HTML report per calendar day plus index | + +### Examples + +**Quick report from the default nginx log directory:** +```bash +nxstats -o report.html +``` + +**Specific log files:** +```bash +nxstats -a /var/log/nginx/access.log -e /var/log/nginx/error.log -o report.html +``` + +**With GeoIP2 map enabled:** +```bash +nxstats --dir /var/log/nginx --geoip ./GeoLite2-City.mmdb -o report.html +``` + +**Split into per-day reports:** +```bash +nxstats --dir /var/log/nginx --split-by-day -o index.html +# Writes index.html + 2024-01-15.html, 2024-01-16.html, etc. +``` + +## GeoIP2 Setup + +The geographic distribution map requires a free MaxMind GeoLite2-City database. + +1. Sign up at [maxmind.com](https://www.maxmind.com/en/geolite2/signup) +2. Download `GeoLite2-City.mmdb` +3. Pass the path via `--geoip ./GeoLite2-City.mmdb` + +## Log Format + +nxstats expects the standard nginx combined log format for access logs: + +``` +$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" +``` + +Error logs use the standard nginx error log format. + +## License + +MIT diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..98b275f --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,291 @@ +package cmd + +import ( + "fmt" + "net" + "os" + "path/filepath" + "sort" + "time" + + "github.com/mr0xb/nxstats/internal/geo" + "github.com/mr0xb/nxstats/internal/parser" + "github.com/mr0xb/nxstats/internal/report" + "github.com/spf13/cobra" +) + +var ( + flagDir string + flagOutput string + flagAccessLog string + flagErrorLog string + flagNoGzip bool + flagGeoIP string + flagSplitByDay bool +) + +var rootCmd = &cobra.Command{ + Use: "nxstats", + Short: "Nginx log parser and HTML report generator", + Long: `nxstats parses nginx access and error logs (including gzip-compressed +rotated logs) and generates a rich cyberpunk-themed HTML report with +charts, searchable tables, and optional GeoIP2 hit maps.`, + RunE: runE, +} + +func init() { + rootCmd.Flags().StringVarP(&flagDir, "dir", "d", "/var/log/nginx", "Directory to scan for nginx logs") + rootCmd.Flags().StringVarP(&flagOutput, "output", "o", "report.html", "Output HTML file path") + rootCmd.Flags().StringVarP(&flagAccessLog, "access-log", "a", "", "Specific access log file (skips dir scan for access logs)") + rootCmd.Flags().StringVarP(&flagErrorLog, "error-log", "e", "", "Specific error log file (skips dir scan for error logs)") + rootCmd.Flags().BoolVar(&flagNoGzip, "no-gzip", false, "Skip .gz compressed rotated logs") + rootCmd.Flags().StringVar(&flagGeoIP, "geoip", "", "Path to MaxMind GeoLite2-City.mmdb (enables map)") + rootCmd.Flags().BoolVar(&flagSplitByDay, "split-by-day", false, "Write one HTML report per calendar day plus an index page") +} + +// Execute runs the root command. +func Execute() { + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} + +func runE(cmd *cobra.Command, args []string) error { + accessFiles, errorFiles, err := resolveLogFiles() + if err != nil { + return err + } + + fmt.Fprintf(os.Stderr, "nxstats: reading %d access file(s), %d error file(s)\n", + len(accessFiles), len(errorFiles)) + + accessEntries, err := parser.ReadAccessEntries(accessFiles) + if err != nil { + return fmt.Errorf("read access logs: %w", err) + } + errorEntries, err := parser.ReadErrorEntries(errorFiles) + if err != nil { + return fmt.Errorf("read error logs: %w", err) + } + + fmt.Fprintf(os.Stderr, "nxstats: %d access entries, %d error entries\n", + len(accessEntries), len(errorEntries)) + + gl, err := geo.NewGeoLookup(flagGeoIP) + if err != nil { + return fmt.Errorf("geoip: %w", err) + } + defer gl.Close() + + // Build global IP→GeoPoint map (one lookup per unique IP, reused per day). + geoPoints := buildGeoPoints(gl, accessEntries) + + if flagSplitByDay { + return runSplitByDay(accessEntries, errorEntries, geoPoints) + } + + // ── Single-report path ───────────────────────────────────────────────── + data := report.ReportData{ + GeneratedAt: time.Now(), + AccessEntries: accessEntries, + ErrorEntries: errorEntries, + } + report.ComputeStats(&data) + + if gl.IsEnabled() { + pts := geoPointsSlice(geoPoints) + data.GeoLocations = pts + fmt.Fprintf(os.Stderr, "nxstats: resolved %d geo locations\n", len(pts)) + } + + html, err := report.GenerateHTML(data) + if err != nil { + return fmt.Errorf("generate html: %w", err) + } + if err := os.WriteFile(flagOutput, []byte(html), 0644); err != nil { + return fmt.Errorf("write output %q: %w", flagOutput, err) + } + fmt.Fprintf(os.Stderr, "nxstats: report written to %s\n", flagOutput) + return nil +} + +// resolveLogFiles returns the access and error file lists, honouring --access-log, +// --error-log, and --dir flags. +func resolveLogFiles() (accessFiles, errorFiles []string, err error) { + includeGzip := !flagNoGzip + + if flagAccessLog != "" { + accessFiles = []string{flagAccessLog} + } else { + accessFiles, _, err = parser.ScanDirectory(flagDir, includeGzip) + if err != nil { + return nil, nil, fmt.Errorf("scan dir %q: %w", flagDir, err) + } + } + + if flagErrorLog != "" { + errorFiles = []string{flagErrorLog} + } else { + _, errorFiles, err = parser.ScanDirectory(flagDir, includeGzip) + if err != nil { + return nil, nil, fmt.Errorf("scan dir %q: %w", flagDir, err) + } + } + return accessFiles, errorFiles, nil +} + +// buildGeoPoints runs GeoIP lookups for all unique IPs in accessEntries and +// returns a map of ip string → report.GeoPoint. Returns an empty map when +// GeoIP is disabled. +func buildGeoPoints(gl *geo.GeoLookup, accessEntries []parser.AccessEntry) map[string]report.GeoPoint { + result := make(map[string]report.GeoPoint) + if !gl.IsEnabled() { + return result + } + for _, e := range accessEntries { + if _, seen := result[e.RemoteAddr]; seen { + continue + } + ip := net.ParseIP(e.RemoteAddr) + if ip == nil { + continue + } + pt := gl.Lookup(ip) + if pt == nil { + continue + } + result[e.RemoteAddr] = report.GeoPoint{ + IP: pt.IP, + Lat: pt.Lat, + Lon: pt.Lon, + Country: pt.Country, + CountryCode: pt.CountryCode, + } + } + return result +} + +// geoPointsSlice converts the geo map to a slice for a single-report run, +// tallying the request count for each IP from a pre-built count map. +func geoPointsSlice(geoPoints map[string]report.GeoPoint) []report.GeoPoint { + pts := make([]report.GeoPoint, 0, len(geoPoints)) + for _, p := range geoPoints { + pts = append(pts, p) + } + return pts +} + +// runSplitByDay groups entries by calendar day, writes one HTML file per day, +// then writes the index page to flagOutput. +func runSplitByDay( + accessEntries []parser.AccessEntry, + errorEntries []parser.ErrorEntry, + geoPoints map[string]report.GeoPoint, +) error { + indexBase := filepath.Base(flagOutput) + + days := report.GroupByDay(flagOutput, accessEntries, errorEntries, geoPoints) + if len(days) == 0 { + fmt.Fprintln(os.Stderr, "nxstats: no entries found, writing empty index") + } + + // Write per-day reports (oldest → newest) + for i := range days { + days[i].Data.IndexFile = indexBase + days[i].Data.DayTitle = days[i].Date.Format("2006-01-02") + + // Per-day geo: set Count from this day's access entries + if len(geoPoints) > 0 { + countByIP := make(map[string]int) + for _, e := range days[i].Data.AccessEntries { + countByIP[e.RemoteAddr]++ + } + updated := make([]report.GeoPoint, len(days[i].Data.GeoLocations)) + for j, g := range days[i].Data.GeoLocations { + g.Count = countByIP[g.IP] + updated[j] = g + } + days[i].Data.GeoLocations = updated + } + + html, err := report.GenerateHTML(days[i].Data) + if err != nil { + return fmt.Errorf("generate html for %s: %w", days[i].Filename, err) + } + dailyPath := filepath.Join(filepath.Dir(flagOutput), days[i].Filename) + if err := os.WriteFile(dailyPath, []byte(html), 0644); err != nil { + return fmt.Errorf("write %q: %w", dailyPath, err) + } + fmt.Fprintf(os.Stderr, "nxstats: wrote %s\n", dailyPath) + } + + // Build IndexData (days newest first in the table) + idx := buildIndexData(accessEntries, days) + + indexHTML, err := report.GenerateIndexHTML(idx) + if err != nil { + return fmt.Errorf("generate index html: %w", err) + } + if err := os.WriteFile(flagOutput, []byte(indexHTML), 0644); err != nil { + return fmt.Errorf("write index %q: %w", flagOutput, err) + } + fmt.Fprintf(os.Stderr, "nxstats: index written to %s (%d daily report(s))\n", + flagOutput, len(days)) + return nil +} + +// buildIndexData assembles the IndexData from all daily reports. Days are +// returned newest-first in the table. Unique IPs are counted across all +// access entries (not summed per day) to avoid double-counting. +func buildIndexData(allAccess []parser.AccessEntry, days []report.DayReport) report.IndexData { + // Reverse a copy for newest-first display + summaries := make([]report.DaySummary, len(days)) + for i, d := range days { + summaries[len(days)-1-i] = report.DaySummary{ + Date: d.Date, + Filename: d.Filename, + Requests: d.Data.TotalRequests, + UniqueIPs: d.Data.UniqueIPs, + TotalBytes: d.Data.TotalBytes, + ErrorRate: d.Data.ErrorRate, + } + } + + // Sort summaries newest first (they were built from oldest-first days) + sort.Slice(summaries, func(i, j int) bool { + return summaries[i].Date.After(summaries[j].Date) + }) + + var totalReqs int + var totalBytes int64 + var totalErrors int + for _, d := range days { + totalReqs += d.Data.TotalRequests + totalBytes += d.Data.TotalBytes + for status, count := range d.Data.StatusCounts { + if status >= 500 { + totalErrors += count + } + } + } + + // Unique IPs across all days (union, not sum) + ipSet := make(map[string]struct{}, len(allAccess)) + for _, e := range allAccess { + ipSet[e.RemoteAddr] = struct{}{} + } + + var errorRate float64 + if totalReqs > 0 { + errorRate = float64(totalErrors) / float64(totalReqs) * 100.0 + } + + return report.IndexData{ + GeneratedAt: time.Now(), + Days: summaries, + TotalRequests: totalReqs, + TotalUniqueIPs: len(ipSet), + TotalBytes: totalBytes, + OverallErrorRate: errorRate, + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6c9a3c3 --- /dev/null +++ b/go.mod @@ -0,0 +1,15 @@ +module github.com/mr0xb/nxstats + +go 1.25.0 + +require ( + github.com/oschwald/geoip2-golang v1.13.0 + github.com/spf13/cobra v1.10.2 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/oschwald/maxminddb-golang v1.13.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect + golang.org/x/sys v0.20.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3d4067b --- /dev/null +++ b/go.sum @@ -0,0 +1,24 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNsSFzQATJI= +github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= +github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= +github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/geo/geo.go b/internal/geo/geo.go new file mode 100644 index 0000000..3531593 --- /dev/null +++ b/internal/geo/geo.go @@ -0,0 +1,68 @@ +package geo + +import ( + "fmt" + "net" + + "github.com/oschwald/geoip2-golang" +) + +// GeoPoint holds geographic information for an IP address. +type GeoPoint struct { + IP string + Lat float64 + Lon float64 + Country string + CountryCode string + Count int +} + +// GeoLookup wraps the geoip2 database reader. +// If no database path is configured, all lookups return nil (no-op). +type GeoLookup struct { + db *geoip2.Reader +} + +// NewGeoLookup creates a GeoLookup. If path is empty, returns a disabled (no-op) lookup. +// Returns an error if path is non-empty but the file cannot be opened. +func NewGeoLookup(path string) (*GeoLookup, error) { + if path == "" { + return &GeoLookup{}, nil + } + db, err := geoip2.Open(path) + if err != nil { + return nil, fmt.Errorf("geoip2 open %q: %w", path, err) + } + return &GeoLookup{db: db}, nil +} + +// IsEnabled returns true if a GeoIP2 database is loaded. +func (g *GeoLookup) IsEnabled() bool { + return g.db != nil +} + +// Close releases the database if open. +func (g *GeoLookup) Close() { + if g.db != nil { + g.db.Close() + } +} + +// Lookup returns a GeoPoint for the given IP, or nil if the database is not loaded +// or the IP is nil/not found. +func (g *GeoLookup) Lookup(ip net.IP) *GeoPoint { + if g.db == nil || ip == nil { + return nil + } + record, err := g.db.City(ip) + if err != nil { + return nil + } + return &GeoPoint{ + IP: ip.String(), + Lat: record.Location.Latitude, + Lon: record.Location.Longitude, + Country: record.Country.Names["en"], + CountryCode: record.Country.IsoCode, + } +} diff --git a/internal/geo/geo_test.go b/internal/geo/geo_test.go new file mode 100644 index 0000000..b80da1a --- /dev/null +++ b/internal/geo/geo_test.go @@ -0,0 +1,59 @@ +package geo + +import ( + "net" + "testing" +) + +func TestNewGeoLookup_NilPath(t *testing.T) { + gl, err := NewGeoLookup("") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if gl == nil { + t.Fatal("expected non-nil GeoLookup") + } + if gl.IsEnabled() { + t.Error("expected IsEnabled() = false for empty path") + } +} + +func TestGeoLookup_LookupWithNoDb(t *testing.T) { + gl, _ := NewGeoLookup("") + result := gl.Lookup(net.ParseIP("8.8.8.8")) + if result != nil { + t.Errorf("expected nil result when no db, got %+v", result) + } +} + +func TestGeoLookup_LookupNilIP(t *testing.T) { + gl, _ := NewGeoLookup("") + result := gl.Lookup(nil) + if result != nil { + t.Errorf("expected nil result for nil IP, got %+v", result) + } +} + +func TestNewGeoLookup_InvalidPath(t *testing.T) { + _, err := NewGeoLookup("/nonexistent/path/GeoLite2-City.mmdb") + if err == nil { + t.Error("expected error for nonexistent db file, got nil") + } +} + +func TestGeoPoint_Fields(t *testing.T) { + gp := GeoPoint{ + IP: "1.2.3.4", + Lat: 37.751, + Lon: -97.822, + Country: "United States", + CountryCode: "US", + Count: 42, + } + if gp.IP != "1.2.3.4" { + t.Errorf("IP = %q", gp.IP) + } + if gp.Count != 42 { + t.Errorf("Count = %d", gp.Count) + } +} diff --git a/internal/parser/access.go b/internal/parser/access.go new file mode 100644 index 0000000..b68fc30 --- /dev/null +++ b/internal/parser/access.go @@ -0,0 +1,70 @@ +package parser + +import ( + "fmt" + "regexp" + "strconv" + "time" +) + +// AccessEntry holds a parsed nginx combined-format access log entry. +type AccessEntry struct { + RemoteAddr string + RemoteUser string + Time time.Time + Method string + Path string + Protocol string + Status int + BytesSent int64 + Referer string + UserAgent string +} + +// accessLogRe matches nginx combined log format. +// Groups: remoteAddr, remoteUser, time, method, path, protocol, status, bytes, referer, userAgent +var accessLogRe = regexp.MustCompile( + `^(\S+) - (\S+) \[([^\]]+)\] "(\S+) (\S+) (\S+)" (\d+) (\d+|-) "([^"]*)" "([^"]*)"$`, +) + +const accessTimeLayout = "02/Jan/2006:15:04:05 -0700" + +// ParseAccessLine parses a single nginx access log line. +// Returns an error for malformed lines. +func ParseAccessLine(line string) (AccessEntry, error) { + m := accessLogRe.FindStringSubmatch(line) + if m == nil { + return AccessEntry{}, fmt.Errorf("access: no match: %q", line) + } + + t, err := time.Parse(accessTimeLayout, m[3]) + if err != nil { + return AccessEntry{}, fmt.Errorf("access: parse time %q: %w", m[3], err) + } + + status, err := strconv.Atoi(m[7]) + if err != nil { + return AccessEntry{}, fmt.Errorf("access: parse status %q: %w", m[7], err) + } + + var bytes int64 + if m[8] != "-" { + bytes, err = strconv.ParseInt(m[8], 10, 64) + if err != nil { + return AccessEntry{}, fmt.Errorf("access: parse bytes %q: %w", m[8], err) + } + } + + return AccessEntry{ + RemoteAddr: m[1], + RemoteUser: m[2], + Time: t, + Method: m[4], + Path: m[5], + Protocol: m[6], + Status: status, + BytesSent: bytes, + Referer: m[9], + UserAgent: m[10], + }, nil +} diff --git a/internal/parser/access_test.go b/internal/parser/access_test.go new file mode 100644 index 0000000..769e374 --- /dev/null +++ b/internal/parser/access_test.go @@ -0,0 +1,110 @@ +package parser + +import ( + "testing" + "time" +) + +func TestParseAccessLine_Valid(t *testing.T) { + line := `192.168.1.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"` + entry, err := ParseAccessLine(line) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if entry.RemoteAddr != "192.168.1.1" { + t.Errorf("RemoteAddr = %q, want %q", entry.RemoteAddr, "192.168.1.1") + } + if entry.RemoteUser != "frank" { + t.Errorf("RemoteUser = %q, want %q", entry.RemoteUser, "frank") + } + want := time.Date(2000, 10, 10, 13, 55, 36, 0, time.FixedZone("", -7*3600)) + if !entry.Time.Equal(want) { + t.Errorf("Time = %v, want %v", entry.Time, want) + } + if entry.Method != "GET" { + t.Errorf("Method = %q, want %q", entry.Method, "GET") + } + if entry.Path != "/apache_pb.gif" { + t.Errorf("Path = %q, want %q", entry.Path, "/apache_pb.gif") + } + if entry.Protocol != "HTTP/1.0" { + t.Errorf("Protocol = %q, want %q", entry.Protocol, "HTTP/1.0") + } + if entry.Status != 200 { + t.Errorf("Status = %d, want %d", entry.Status, 200) + } + if entry.BytesSent != 2326 { + t.Errorf("BytesSent = %d, want %d", entry.BytesSent, 2326) + } + if entry.Referer != "http://www.example.com/start.html" { + t.Errorf("Referer = %q, want %q", entry.Referer, "http://www.example.com/start.html") + } +} + +func TestParseAccessLine_DashFields(t *testing.T) { + line := `10.0.0.1 - - [15/Jan/2024:08:23:41 +0000] "POST /api/users HTTP/1.1" 201 512 "-" "curl/7.68.0"` + entry, err := ParseAccessLine(line) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if entry.RemoteUser != "-" { + t.Errorf("RemoteUser = %q, want %q", entry.RemoteUser, "-") + } + if entry.Referer != "-" { + t.Errorf("Referer = %q, want %q", entry.Referer, "-") + } + if entry.BytesSent != 512 { + t.Errorf("BytesSent = %d, want %d", entry.BytesSent, 512) + } +} + +func TestParseAccessLine_BytesDash(t *testing.T) { + // bytes sent as "-" should parse as 0 + line := `127.0.0.1 - - [22/Feb/2024:10:00:00 +0000] "GET /health HTTP/1.1" 200 - "-" "kube-probe/1.27"` + entry, err := ParseAccessLine(line) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if entry.BytesSent != 0 { + t.Errorf("BytesSent = %d, want 0", entry.BytesSent) + } +} + +func TestParseAccessLine_IPv6(t *testing.T) { + line := `2001:db8::1 - - [20/Feb/2024:16:45:00 +0000] "GET /index.html HTTP/2.0" 200 4096 "-" "Googlebot/2.1"` + entry, err := ParseAccessLine(line) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if entry.RemoteAddr != "2001:db8::1" { + t.Errorf("RemoteAddr = %q, want %q", entry.RemoteAddr, "2001:db8::1") + } +} + +func TestParseAccessLine_Malformed(t *testing.T) { + cases := []string{ + "this is a malformed line", + "", + "incomplete line without proper format", + } + for _, c := range cases { + _, err := ParseAccessLine(c) + if err == nil { + t.Errorf("expected error for malformed line %q, got nil", c) + } + } +} + +func TestParseAccessLine_Status(t *testing.T) { + line := `172.16.0.50 - admin [01/Mar/2024:12:00:00 +0100] "DELETE /resource/42 HTTP/1.1" 404 0 "https://example.org/page" "Mozilla/5.0"` + entry, err := ParseAccessLine(line) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if entry.Status != 404 { + t.Errorf("Status = %d, want 404", entry.Status) + } + if entry.Method != "DELETE" { + t.Errorf("Method = %q, want DELETE", entry.Method) + } +} diff --git a/internal/parser/error.go b/internal/parser/error.go new file mode 100644 index 0000000..7b6c3cc --- /dev/null +++ b/internal/parser/error.go @@ -0,0 +1,67 @@ +package parser + +import ( + "fmt" + "regexp" + "strconv" + "time" +) + +// ErrorEntry holds a parsed nginx error log entry. +type ErrorEntry struct { + Time time.Time + Level string // debug/info/notice/warn/error/crit/alert/emerg + PID int + TID int + ConnID int // 0 if not present + Message string +} + +// errorLogRe matches nginx error log format. +// Groups: time, level, pid, tid, connID (optional), message +var errorLogRe = regexp.MustCompile( + `^(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(\w+)\] (\d+)#(\d+): (?:\*(\d+) )?(.+)$`, +) + +const errorTimeLayout = "2006/01/02 15:04:05" + +// ParseErrorLine parses a single nginx error log line. +// Returns an error for malformed lines. +func ParseErrorLine(line string) (ErrorEntry, error) { + m := errorLogRe.FindStringSubmatch(line) + if m == nil { + return ErrorEntry{}, fmt.Errorf("error: no match: %q", line) + } + + t, err := time.ParseInLocation(errorTimeLayout, m[1], time.UTC) + if err != nil { + return ErrorEntry{}, fmt.Errorf("error: parse time %q: %w", m[1], err) + } + + pid, err := strconv.Atoi(m[3]) + if err != nil { + return ErrorEntry{}, fmt.Errorf("error: parse pid %q: %w", m[3], err) + } + + tid, err := strconv.Atoi(m[4]) + if err != nil { + return ErrorEntry{}, fmt.Errorf("error: parse tid %q: %w", m[4], err) + } + + var connID int + if m[5] != "" { + connID, err = strconv.Atoi(m[5]) + if err != nil { + return ErrorEntry{}, fmt.Errorf("error: parse connid %q: %w", m[5], err) + } + } + + return ErrorEntry{ + Time: t, + Level: m[2], + PID: pid, + TID: tid, + ConnID: connID, + Message: m[6], + }, nil +} diff --git a/internal/parser/error_test.go b/internal/parser/error_test.go new file mode 100644 index 0000000..d847f4a --- /dev/null +++ b/internal/parser/error_test.go @@ -0,0 +1,93 @@ +package parser + +import ( + "testing" + "time" +) + +func TestParseErrorLine_WithConnID(t *testing.T) { + line := `2024/02/20 16:45:01 [error] 1234#5678: *99 connect() failed (111: Connection refused) while connecting to upstream` + entry, err := ParseErrorLine(line) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + want := time.Date(2024, 2, 20, 16, 45, 1, 0, time.UTC) + if !entry.Time.Equal(want) { + t.Errorf("Time = %v, want %v", entry.Time, want) + } + if entry.Level != "error" { + t.Errorf("Level = %q, want %q", entry.Level, "error") + } + if entry.PID != 1234 { + t.Errorf("PID = %d, want 1234", entry.PID) + } + if entry.TID != 5678 { + t.Errorf("TID = %d, want 5678", entry.TID) + } + if entry.ConnID != 99 { + t.Errorf("ConnID = %d, want 99", entry.ConnID) + } + if entry.Message != "connect() failed (111: Connection refused) while connecting to upstream" { + t.Errorf("Message = %q", entry.Message) + } +} + +func TestParseErrorLine_WithoutConnID(t *testing.T) { + line := `2024/02/20 17:00:00 [warn] 1234#5678: worker process 9999 exited on signal 15` + entry, err := ParseErrorLine(line) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if entry.Level != "warn" { + t.Errorf("Level = %q, want %q", entry.Level, "warn") + } + if entry.ConnID != 0 { + t.Errorf("ConnID = %d, want 0", entry.ConnID) + } + if entry.Message != "worker process 9999 exited on signal 15" { + t.Errorf("Message = %q", entry.Message) + } +} + +func TestParseErrorLine_AllLevels(t *testing.T) { + levels := []string{"debug", "info", "notice", "warn", "error", "crit", "alert", "emerg"} + for _, level := range levels { + line := "2024/01/01 00:00:00 [" + level + "] 100#200: test message" + entry, err := ParseErrorLine(line) + if err != nil { + t.Errorf("level %q: unexpected error: %v", level, err) + continue + } + if entry.Level != level { + t.Errorf("level %q: got Level = %q", level, entry.Level) + } + } +} + +func TestParseErrorLine_Info(t *testing.T) { + line := `2024/02/21 08:30:00 [info] 1234#5678: start worker process 10001` + entry, err := ParseErrorLine(line) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if entry.Level != "info" { + t.Errorf("Level = %q, want info", entry.Level) + } + if entry.PID != 1234 { + t.Errorf("PID = %d, want 1234", entry.PID) + } +} + +func TestParseErrorLine_Malformed(t *testing.T) { + cases := []string{ + "this is a malformed line", + "", + "2024/02/20 not-a-proper-format", + } + for _, c := range cases { + _, err := ParseErrorLine(c) + if err == nil { + t.Errorf("expected error for malformed line %q, got nil", c) + } + } +} diff --git a/internal/parser/scanner.go b/internal/parser/scanner.go new file mode 100644 index 0000000..2b14fae --- /dev/null +++ b/internal/parser/scanner.go @@ -0,0 +1,167 @@ +package parser + +import ( + "bufio" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" +) + +// ScanDirectory scans dir for nginx access and error log files. +// It matches filenames like access*.log* and error*.log*. +// If includeGzip is false, .gz files are skipped. +// Returns (accessFiles, errorFiles, error) sorted newest-first. +func ScanDirectory(dir string, includeGzip bool) ([]string, []string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, nil, fmt.Errorf("scandir %q: %w", dir, err) + } + + var accessFiles, errorFiles []string + for _, e := range entries { + if e.IsDir() { + continue + } + name := e.Name() + if !includeGzip && strings.HasSuffix(name, ".gz") { + continue + } + if matchesLogPattern(name, "access") { + accessFiles = append(accessFiles, filepath.Join(dir, name)) + } else if matchesLogPattern(name, "error") { + errorFiles = append(errorFiles, filepath.Join(dir, name)) + } + } + + sortLogFiles(accessFiles) + sortLogFiles(errorFiles) + return accessFiles, errorFiles, nil +} + +// matchesLogPattern returns true if filename starts with prefix and contains ".log". +func matchesLogPattern(name, prefix string) bool { + if !strings.HasPrefix(name, prefix) { + return false + } + return strings.Contains(name, ".log") +} + +// logSortKey returns a sort key such that: +// - "access.log" → (0, 0) — most recent +// - "access.log.1" → (1, 0) +// - "access.log.1.gz" → (1, 1) +// - "access.log.2.gz" → (2, 1) +func logSortKey(path string) (int, int) { + name := filepath.Base(path) + // strip prefix up to and including ".log" + idx := strings.Index(name, ".log") + if idx < 0 { + return 999, 0 + } + suffix := name[idx+4:] // e.g. "", ".1", ".1.gz", ".2.gz" + + isGz := 0 + if strings.HasSuffix(suffix, ".gz") { + isGz = 1 + suffix = strings.TrimSuffix(suffix, ".gz") + } + suffix = strings.TrimPrefix(suffix, ".") + + num := 0 + if suffix != "" { + fmt.Sscanf(suffix, "%d", &num) + } + return num, isGz +} + +func sortLogFiles(files []string) { + sort.SliceStable(files, func(i, j int) bool { + ni, gi := logSortKey(files[i]) + nj, gj := logSortKey(files[j]) + if ni != nj { + return ni < nj + } + return gi < gj + }) +} + +// openLogFile opens a log file for reading. +// If the path ends in ".gz", it transparently decompresses. +func openLogFile(path string) (io.ReadCloser, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open %q: %w", path, err) + } + if strings.HasSuffix(path, ".gz") { + gz, err := gzip.NewReader(f) + if err != nil { + f.Close() + return nil, fmt.Errorf("gzip %q: %w", path, err) + } + return &gzipReadCloser{gz: gz, f: f}, nil + } + return f, nil +} + +// gzipReadCloser closes both the gzip reader and underlying file. +type gzipReadCloser struct { + gz *gzip.Reader + f *os.File +} + +func (g *gzipReadCloser) Read(p []byte) (int, error) { return g.gz.Read(p) } +func (g *gzipReadCloser) Close() error { + err := g.gz.Close() + g.f.Close() + return err +} + +// ReadAccessEntries reads all access log entries from the given files. +// Malformed lines are silently skipped. +func ReadAccessEntries(files []string) ([]AccessEntry, error) { + var entries []AccessEntry + for _, path := range files { + rc, err := openLogFile(path) + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(rc) + for scanner.Scan() { + if e, err := ParseAccessLine(scanner.Text()); err == nil { + entries = append(entries, e) + } + } + rc.Close() + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("scan %q: %w", path, err) + } + } + return entries, nil +} + +// ReadErrorEntries reads all error log entries from the given files. +// Malformed lines are silently skipped. +func ReadErrorEntries(files []string) ([]ErrorEntry, error) { + var entries []ErrorEntry + for _, path := range files { + rc, err := openLogFile(path) + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(rc) + for scanner.Scan() { + if e, err := ParseErrorLine(scanner.Text()); err == nil { + entries = append(entries, e) + } + } + rc.Close() + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("scan %q: %w", path, err) + } + } + return entries, nil +} diff --git a/internal/parser/scanner_test.go b/internal/parser/scanner_test.go new file mode 100644 index 0000000..2508b40 --- /dev/null +++ b/internal/parser/scanner_test.go @@ -0,0 +1,155 @@ +package parser + +import ( + "bufio" + "os" + "path/filepath" + "testing" +) + +func TestScanDirectory_FindsAccessAndErrorLogs(t *testing.T) { + dir := t.TempDir() + + // Create test log files + files := []string{ + "access.log", + "access.log.1", + "error.log", + "error.log.1", + "other.txt", // should be ignored + } + for _, f := range files { + if err := os.WriteFile(filepath.Join(dir, f), []byte(""), 0644); err != nil { + t.Fatal(err) + } + } + + accessFiles, errorFiles, err := ScanDirectory(dir, false) + if err != nil { + t.Fatalf("ScanDirectory error: %v", err) + } + if len(accessFiles) != 2 { + t.Errorf("expected 2 access files, got %d: %v", len(accessFiles), accessFiles) + } + if len(errorFiles) != 2 { + t.Errorf("expected 2 error files, got %d: %v", len(errorFiles), errorFiles) + } +} + +func TestScanDirectory_GzipIncluded(t *testing.T) { + dir := t.TempDir() + files := []string{ + "access.log", + "access.log.1.gz", + "error.log", + } + for _, f := range files { + if err := os.WriteFile(filepath.Join(dir, f), []byte(""), 0644); err != nil { + t.Fatal(err) + } + } + + accessFiles, _, err := ScanDirectory(dir, true) + if err != nil { + t.Fatalf("ScanDirectory error: %v", err) + } + if len(accessFiles) != 2 { + t.Errorf("expected 2 access files (including gz), got %d", len(accessFiles)) + } +} + +func TestScanDirectory_GzipExcluded(t *testing.T) { + dir := t.TempDir() + files := []string{ + "access.log", + "access.log.1.gz", + } + for _, f := range files { + if err := os.WriteFile(filepath.Join(dir, f), []byte(""), 0644); err != nil { + t.Fatal(err) + } + } + + accessFiles, _, err := ScanDirectory(dir, false) + if err != nil { + t.Fatalf("ScanDirectory error: %v", err) + } + if len(accessFiles) != 1 { + t.Errorf("expected 1 access file (gz excluded), got %d", len(accessFiles)) + } +} + +func TestScanDirectory_NonexistentDir(t *testing.T) { + _, _, err := ScanDirectory("/nonexistent/path/xyz", true) + if err == nil { + t.Error("expected error for nonexistent directory, got nil") + } +} + +func TestOpenLogFile_PlainText(t *testing.T) { + dir := t.TempDir() + content := "test line\n" + path := filepath.Join(dir, "test.log") + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + + rc, err := openLogFile(path) + if err != nil { + t.Fatalf("openLogFile error: %v", err) + } + defer rc.Close() + + scanner := bufio.NewScanner(rc) + if !scanner.Scan() { + t.Fatal("expected to read a line") + } + if scanner.Text() != "test line" { + t.Errorf("got %q, want %q", scanner.Text(), "test line") + } +} + +func TestOpenLogFile_Gzip(t *testing.T) { + // Use the pre-made fixture + rc, err := openLogFile("testdata/access.log.1.gz") + if err != nil { + t.Fatalf("openLogFile gz error: %v", err) + } + defer rc.Close() + + scanner := bufio.NewScanner(rc) + if !scanner.Scan() { + t.Fatal("expected to read a line from gz file") + } + line := scanner.Text() + if line == "" { + t.Error("expected non-empty line from gz file") + } +} + +func TestScanDirectory_SortOrder(t *testing.T) { + dir := t.TempDir() + files := []string{ + "access.log.2.gz", + "access.log", + "access.log.1", + "access.log.1.gz", + } + for _, f := range files { + if err := os.WriteFile(filepath.Join(dir, f), []byte(""), 0644); err != nil { + t.Fatal(err) + } + } + + accessFiles, _, err := ScanDirectory(dir, true) + if err != nil { + t.Fatalf("ScanDirectory error: %v", err) + } + if len(accessFiles) != 4 { + t.Fatalf("expected 4 files, got %d", len(accessFiles)) + } + // access.log should come first (most recent) + if filepath.Base(accessFiles[0]) != "access.log" { + t.Errorf("expected access.log first, got %q", filepath.Base(accessFiles[0])) + } +} diff --git a/internal/parser/testdata/access.log b/internal/parser/testdata/access.log new file mode 100644 index 0000000..023d211 --- /dev/null +++ b/internal/parser/testdata/access.log @@ -0,0 +1,6 @@ +192.168.1.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)" +10.0.0.1 - - [15/Jan/2024:08:23:41 +0000] "POST /api/users HTTP/1.1" 201 512 "-" "curl/7.68.0" +172.16.0.50 - admin [01/Mar/2024:12:00:00 +0100] "DELETE /resource/42 HTTP/1.1" 404 0 "https://example.org/page" "Mozilla/5.0" +2001:db8::1 - - [20/Feb/2024:16:45:00 +0000] "GET /index.html HTTP/2.0" 200 4096 "-" "Googlebot/2.1" +this is a malformed line that should be skipped +127.0.0.1 - - [22/Feb/2024:10:00:00 +0000] "GET /health HTTP/1.1" 200 - "-" "kube-probe/1.27" diff --git a/internal/parser/testdata/access.log.1.gz b/internal/parser/testdata/access.log.1.gz new file mode 100644 index 0000000..8a207fb Binary files /dev/null and b/internal/parser/testdata/access.log.1.gz differ diff --git a/internal/parser/testdata/error.log b/internal/parser/testdata/error.log new file mode 100644 index 0000000..45415de --- /dev/null +++ b/internal/parser/testdata/error.log @@ -0,0 +1,6 @@ +2024/02/20 16:45:01 [error] 1234#5678: *99 connect() failed (111: Connection refused) while connecting to upstream +2024/02/20 17:00:00 [warn] 1234#5678: worker process 9999 exited on signal 15 +2024/02/21 08:30:00 [info] 1234#5678: start worker process 10001 +2024/02/21 09:00:00 [crit] 9999#0: *1 SSL_do_handshake() failed (SSL: error:14094412) while SSL handshaking +this is a malformed error line +2024/02/22 10:00:00 [notice] 1234#5678: signal process started diff --git a/internal/report/report.go b/internal/report/report.go new file mode 100644 index 0000000..70daade --- /dev/null +++ b/internal/report/report.go @@ -0,0 +1,435 @@ +package report + +import ( + "bytes" + "encoding/json" + "fmt" + "html/template" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/mr0xb/nxstats/internal/parser" +) + +// IPCount holds a remote address and its request count. +type IPCount struct { + IP string + Count int +} + +// PathCount holds a path, its request count, and the most frequently +// returned HTTP status code for that path. +type PathCount struct { + Path string + Count int + TopStatus int +} + +// TimePoint holds a time bucket and request count. +type TimePoint struct { + Hour time.Time + Count int +} + +// GeoPoint holds geographic information for an IP. +type GeoPoint struct { + IP string + Lat float64 + Lon float64 + Country string + CountryCode string + Count int +} + +// ReportData aggregates all data needed to render the HTML report. +type ReportData struct { + GeneratedAt time.Time + AccessEntries []parser.AccessEntry + ErrorEntries []parser.ErrorEntry + + TotalRequests int + TotalBytes int64 + UniqueIPs int + ErrorRate float64 + StatusCounts map[int]int + TopIPs []IPCount + TopPaths []PathCount + TimeSeriesData []TimePoint + GeoLocations []GeoPoint + + // Set when this is a per-day report in split-by-day mode. + IndexFile string // non-empty → render "← Back to Index" nav link + DayTitle string // e.g. "2024-02-20", shown in the header +} + +// DaySummary is one row in the index page's daily breakdown table. +type DaySummary struct { + Date time.Time + Filename string // bare filename, e.g. "report-2024-02-20.html" + Requests int + UniqueIPs int + TotalBytes int64 + ErrorRate float64 +} + +// IndexData is the template data for the --split-by-day index page. +type IndexData struct { + GeneratedAt time.Time + Days []DaySummary // newest first + TotalRequests int + TotalUniqueIPs int + TotalBytes int64 + OverallErrorRate float64 +} + +// DayReport bundles one calendar day's fully-computed ReportData with +// its output filename, ready to be written to disk. +type DayReport struct { + Date time.Time // UTC midnight for this day + Filename string // bare filename, e.g. "report-2024-02-20.html" + Data ReportData // ComputeStats already called +} + +// DeriveDailyFilename builds a per-day output path from the index output path. +// E.g. ("report.html", "2024-02-20") → "report-2024-02-20.html". +func DeriveDailyFilename(outputPath, dateKey string) string { + ext := filepath.Ext(outputPath) + base := strings.TrimSuffix(filepath.Base(outputPath), ext) + dir := filepath.Dir(outputPath) + return filepath.Join(dir, base+"-"+dateKey+ext) +} + +// GroupByDay partitions access and error entries by calendar day (UTC), +// runs ComputeStats on each day's ReportData, and filters geoPoints to only +// IPs present in that day's access entries. Returns DayReports sorted +// oldest-first. geoPoints may be nil (GeoIP disabled). +func GroupByDay( + outputPath string, + accessEntries []parser.AccessEntry, + errorEntries []parser.ErrorEntry, + geoPoints map[string]GeoPoint, +) []DayReport { + accessByDay := make(map[string][]parser.AccessEntry) + for _, e := range accessEntries { + key := e.Time.UTC().Format("2006-01-02") + accessByDay[key] = append(accessByDay[key], e) + } + errorByDay := make(map[string][]parser.ErrorEntry) + for _, e := range errorEntries { + key := e.Time.UTC().Format("2006-01-02") + errorByDay[key] = append(errorByDay[key], e) + } + + // Union of all day keys + seen := make(map[string]struct{}) + for k := range accessByDay { + seen[k] = struct{}{} + } + for k := range errorByDay { + seen[k] = struct{}{} + } + keys := make([]string, 0, len(seen)) + for k := range seen { + keys = append(keys, k) + } + sort.Strings(keys) // oldest first (YYYY-MM-DD sorts lexicographically) + + days := make([]DayReport, 0, len(keys)) + for _, key := range keys { + dayAccess := accessByDay[key] + dayErrors := errorByDay[key] + + rd := ReportData{ + GeneratedAt: time.Now(), + AccessEntries: dayAccess, + ErrorEntries: dayErrors, + } + ComputeStats(&rd) + + // Filter geo locations to IPs seen on this day + if len(geoPoints) > 0 { + dayIPSet := make(map[string]struct{}, len(dayAccess)) + for _, e := range dayAccess { + dayIPSet[e.RemoteAddr] = struct{}{} + } + var dayGeo []GeoPoint + for ip, pt := range geoPoints { + if _, ok := dayIPSet[ip]; ok { + dayGeo = append(dayGeo, pt) + } + } + rd.GeoLocations = dayGeo + } + + t, _ := time.Parse("2006-01-02", key) + days = append(days, DayReport{ + Date: t.UTC(), + Filename: filepath.Base(DeriveDailyFilename(outputPath, key)), + Data: rd, + }) + } + return days +} + +// GenerateIndexHTML renders the index page from IndexData. +func GenerateIndexHTML(idx IndexData) (string, error) { + funcMap := template.FuncMap{ + "formatBytes": func(b int64) string { + switch { + case b >= 1<<30: + return fmt.Sprintf("%.2f GB", float64(b)/(1<<30)) + case b >= 1<<20: + return fmt.Sprintf("%.2f MB", float64(b)/(1<<20)) + case b >= 1<<10: + return fmt.Sprintf("%.2f KB", float64(b)/(1<<10)) + default: + return fmt.Sprintf("%d B", b) + } + }, + "formatFloat": func(f float64) string { return fmt.Sprintf("%.2f", f) }, + "errorClass": func(rate float64) string { + switch { + case rate >= 10: + return "err-high" + case rate >= 2: + return "err-med" + default: + return "err-low" + } + }, + "fmtDate": func(t time.Time) string { return t.UTC().Format("2006-01-02") }, + } + tmpl, err := template.New("index").Funcs(funcMap).Parse(indexTemplate) + if err != nil { + return "", err + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, idx); err != nil { + return "", err + } + return buf.String(), nil +} + +// ComputeStats fills the aggregate fields of data from AccessEntries and ErrorEntries. +func ComputeStats(data *ReportData) { + data.TotalRequests = len(data.AccessEntries) + + ipMap := make(map[string]int) + pathMap := make(map[string]map[int]int) // path → {status → count} + hourMap := make(map[time.Time]int) + statusMap := make(map[int]int) + var totalBytes int64 + errorCount := 0 + + for _, e := range data.AccessEntries { + totalBytes += e.BytesSent + ipMap[e.RemoteAddr]++ + if pathMap[e.Path] == nil { + pathMap[e.Path] = make(map[int]int) + } + pathMap[e.Path][e.Status]++ + statusMap[e.Status]++ + if e.Status >= 500 { + errorCount++ + } + hour := e.Time.Truncate(time.Hour) + hourMap[hour]++ + } + + data.TotalBytes = totalBytes + data.UniqueIPs = len(ipMap) + data.StatusCounts = statusMap + + if data.TotalRequests > 0 { + data.ErrorRate = float64(errorCount) / float64(data.TotalRequests) * 100.0 + } + + // Top IPs (top 10) + ips := make([]IPCount, 0, len(ipMap)) + for ip, cnt := range ipMap { + ips = append(ips, IPCount{IP: ip, Count: cnt}) + } + sort.Slice(ips, func(i, j int) bool { return ips[i].Count > ips[j].Count }) + if len(ips) > 10 { + ips = ips[:10] + } + data.TopIPs = ips + + // Top Paths (top 10) + paths := make([]PathCount, 0, len(pathMap)) + for path, statusCounts := range pathMap { + total := 0 + topStatus, topCount := 0, 0 + for status, count := range statusCounts { + total += count + // Pick the most frequent; break ties by lower status code + if count > topCount || (count == topCount && status < topStatus) { + topStatus = status + topCount = count + } + } + paths = append(paths, PathCount{ + Path: path, + Count: total, + TopStatus: topStatus, + }) + } + sort.Slice(paths, func(i, j int) bool { return paths[i].Count > paths[j].Count }) + if len(paths) > 10 { + paths = paths[:10] + } + data.TopPaths = paths + + // Time series + hours := make([]time.Time, 0, len(hourMap)) + for h := range hourMap { + hours = append(hours, h) + } + sort.Slice(hours, func(i, j int) bool { return hours[i].Before(hours[j]) }) + ts := make([]TimePoint, 0, len(hours)) + for _, h := range hours { + ts = append(ts, TimePoint{Hour: h, Count: hourMap[h]}) + } + data.TimeSeriesData = ts +} + +// GenerateHTML renders the full HTML report from data. +func GenerateHTML(data ReportData) (string, error) { + funcMap := template.FuncMap{ + "add": func(a, b int) int { return a + b }, + "formatBytes": func(b int64) string { + switch { + case b >= 1<<30: + return fmt.Sprintf("%.2f GB", float64(b)/(1<<30)) + case b >= 1<<20: + return fmt.Sprintf("%.2f MB", float64(b)/(1<<20)) + case b >= 1<<10: + return fmt.Sprintf("%.2f KB", float64(b)/(1<<10)) + default: + return fmt.Sprintf("%d B", b) + } + }, + "formatFloat": func(f float64) string { return fmt.Sprintf("%.2f", f) }, + "statusBadge": func(status int) template.HTML { + cls := "s2xx" + switch { + case status >= 500: + cls = "s5xx" + case status >= 400: + cls = "s4xx" + case status >= 300: + cls = "s3xx" + } + return template.HTML(fmt.Sprintf(`%d`, cls, status)) + }, + "levelBadge": func(level string) template.HTML { + cls := "l-" + level + return template.HTML(fmt.Sprintf(`%s`, cls, level)) + }, + "statusJSON": func(m map[int]int) template.JS { + if m == nil { + return template.JS("{}") + } + b, _ := json.Marshal(m) + return template.JS(b) + }, + "topPathsJSON": func(paths []PathCount) template.JS { + type item struct { + Path string `json:"path"` + Count int `json:"count"` + } + items := make([]item, len(paths)) + for i, p := range paths { + items[i] = item{Path: p.Path, Count: p.Count} + } + b, _ := json.Marshal(items) + return template.JS(b) + }, + "timeSeriesJSON": func(ts []TimePoint) template.JS { + type item struct { + Hour string `json:"hour"` + Count int `json:"count"` + } + items := make([]item, len(ts)) + for i, t := range ts { + items[i] = item{Hour: t.Hour.Format("2006-01-02 15:04"), Count: t.Count} + } + b, _ := json.Marshal(items) + return template.JS(b) + }, + "geoJSON": func(pts []GeoPoint) template.JS { + type item struct { + IP string `json:"ip"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + Country string `json:"country"` + CC string `json:"cc"` + Count int `json:"count"` + } + items := make([]item, len(pts)) + for i, p := range pts { + items[i] = item{IP: p.IP, Lat: p.Lat, Lon: p.Lon, Country: p.Country, CC: p.CountryCode, Count: p.Count} + } + b, _ := json.Marshal(items) + return template.JS(b) + }, + "accessEntriesJSON": func(entries []parser.AccessEntry) template.JS { + type row struct { + Time string `json:"t"` + IP string `json:"ip"` + Method string `json:"m"` + Path string `json:"p"` + Status int `json:"s"` + Bytes int64 `json:"b"` + UA string `json:"ua"` + } + rows := make([]row, len(entries)) + for i, e := range entries { + rows[i] = row{ + Time: e.Time.Format("2006-01-02 15:04:05"), + IP: e.RemoteAddr, + Method: e.Method, + Path: e.Path, + Status: e.Status, + Bytes: e.BytesSent, + UA: e.UserAgent, + } + } + b, _ := json.Marshal(rows) + return template.JS(b) + }, + "errorEntriesJSON": func(entries []parser.ErrorEntry) template.JS { + type row struct { + Time string `json:"t"` + Level string `json:"l"` + PID int `json:"pid"` + ConnID int `json:"cid"` + Msg string `json:"msg"` + } + rows := make([]row, len(entries)) + for i, e := range entries { + rows[i] = row{ + Time: e.Time.Format("2006-01-02 15:04:05"), + Level: e.Level, + PID: e.PID, + ConnID: e.ConnID, + Msg: e.Message, + } + } + b, _ := json.Marshal(rows) + return template.JS(b) + }, + } + + tmpl, err := template.New("report").Funcs(funcMap).Parse(reportTemplate) + if err != nil { + return "", err + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return "", err + } + return buf.String(), nil +} diff --git a/internal/report/report_test.go b/internal/report/report_test.go new file mode 100644 index 0000000..cb5c432 --- /dev/null +++ b/internal/report/report_test.go @@ -0,0 +1,520 @@ +package report + +import ( + "strings" + "testing" + "time" + + "github.com/mr0xb/nxstats/internal/parser" +) + +func makeTestData() ReportData { + t1 := time.Date(2024, 2, 20, 10, 0, 0, 0, time.UTC) + t2 := time.Date(2024, 2, 20, 11, 0, 0, 0, time.UTC) + t3 := time.Date(2024, 2, 20, 11, 30, 0, 0, time.UTC) + + return ReportData{ + GeneratedAt: time.Now(), + AccessEntries: []parser.AccessEntry{ + {RemoteAddr: "1.1.1.1", Method: "GET", Path: "/", Status: 200, BytesSent: 1024, Time: t1, UserAgent: "TestAgent/1.0"}, + {RemoteAddr: "2.2.2.2", Method: "POST", Path: "/api", Status: 201, BytesSent: 512, Time: t2, UserAgent: "curl/7.0"}, + {RemoteAddr: "1.1.1.1", Method: "GET", Path: "/about", Status: 200, BytesSent: 2048, Time: t3, UserAgent: "TestAgent/1.0"}, + {RemoteAddr: "3.3.3.3", Method: "GET", Path: "/", Status: 500, BytesSent: 128, Time: t3, UserAgent: "BadBot/1.0"}, + }, + ErrorEntries: []parser.ErrorEntry{ + {Time: t1, Level: "error", PID: 100, TID: 200, Message: "connection refused"}, + {Time: t2, Level: "warn", PID: 100, TID: 200, Message: "slow query"}, + }, + } +} + +func TestGenerateHTML_ReturnsHTML(t *testing.T) { + data := makeTestData() + ComputeStats(&data) + + html, err := GenerateHTML(data) + if err != nil { + t.Fatalf("GenerateHTML error: %v", err) + } + if !strings.HasPrefix(strings.TrimSpace(html), "") { + t.Error("expected output to start with ") + } +} + +func TestGenerateHTML_ContainsStatsCards(t *testing.T) { + data := makeTestData() + ComputeStats(&data) + + html, err := GenerateHTML(data) + if err != nil { + t.Fatalf("GenerateHTML error: %v", err) + } + checks := []string{ + "Total Requests", + "Unique IPs", + "Total Bytes", + "Error Rate", + } + for _, c := range checks { + if !strings.Contains(html, c) { + t.Errorf("expected %q in HTML output", c) + } + } +} + +func TestGenerateHTML_ContainsSections(t *testing.T) { + data := makeTestData() + ComputeStats(&data) + + html, err := GenerateHTML(data) + if err != nil { + t.Fatalf("GenerateHTML error: %v", err) + } + sections := []string{ + "Access Log", + "Error Log", + "Top IPs", + "Top Paths", + "chart", + } + for _, s := range sections { + if !strings.Contains(html, s) { + t.Errorf("expected section %q in HTML output", s) + } + } +} + +func TestGenerateHTML_ContainsIPData(t *testing.T) { + data := makeTestData() + ComputeStats(&data) + + html, err := GenerateHTML(data) + if err != nil { + t.Fatalf("GenerateHTML error: %v", err) + } + if !strings.Contains(html, "1.1.1.1") { + t.Error("expected top IP 1.1.1.1 in output") + } +} + +func TestComputeStats_Counts(t *testing.T) { + data := makeTestData() + ComputeStats(&data) + + if data.TotalRequests != 4 { + t.Errorf("TotalRequests = %d, want 4", data.TotalRequests) + } + if data.UniqueIPs != 3 { + t.Errorf("UniqueIPs = %d, want 3", data.UniqueIPs) + } + if data.TotalBytes != 3712 { + t.Errorf("TotalBytes = %d, want 3712", data.TotalBytes) + } +} + +func TestComputeStats_ErrorRate(t *testing.T) { + data := makeTestData() + ComputeStats(&data) + + // 1 out of 4 requests is 5xx → 25% + if data.ErrorRate < 24.9 || data.ErrorRate > 25.1 { + t.Errorf("ErrorRate = %f, want ~25.0", data.ErrorRate) + } +} + +func TestComputeStats_TopIPs(t *testing.T) { + data := makeTestData() + ComputeStats(&data) + + if len(data.TopIPs) == 0 { + t.Fatal("expected TopIPs to be non-empty") + } + if data.TopIPs[0].IP != "1.1.1.1" { + t.Errorf("top IP = %q, want 1.1.1.1", data.TopIPs[0].IP) + } + if data.TopIPs[0].Count != 2 { + t.Errorf("top IP count = %d, want 2", data.TopIPs[0].Count) + } +} + +func TestComputeStats_TopPaths(t *testing.T) { + data := makeTestData() + ComputeStats(&data) + + if len(data.TopPaths) == 0 { + t.Fatal("expected TopPaths to be non-empty") + } + if data.TopPaths[0].Path != "/" { + t.Errorf("top path = %q, want /", data.TopPaths[0].Path) + } +} + +func TestComputeStats_TopPathsTopStatus(t *testing.T) { + // "/" gets two requests: one 200 and one 500. + // TopStatus should be 200 (the most frequent), not an average (350). + data := makeTestData() + ComputeStats(&data) + + var slashEntry *PathCount + for i := range data.TopPaths { + if data.TopPaths[i].Path == "/" { + slashEntry = &data.TopPaths[i] + break + } + } + if slashEntry == nil { + t.Fatal("expected / in TopPaths") + } + if slashEntry.TopStatus != 200 { + t.Errorf("TopStatus for / = %d, want 200 (most frequent, not an average)", slashEntry.TopStatus) + } +} + +func TestComputeStats_StatusCounts(t *testing.T) { + data := makeTestData() + ComputeStats(&data) + + if data.StatusCounts[200] != 2 { + t.Errorf("StatusCounts[200] = %d, want 2", data.StatusCounts[200]) + } + if data.StatusCounts[500] != 1 { + t.Errorf("StatusCounts[500] = %d, want 1", data.StatusCounts[500]) + } +} + +func TestGenerateHTML_SearchInputs(t *testing.T) { + data := makeTestData() + ComputeStats(&data) + + html, err := GenerateHTML(data) + if err != nil { + t.Fatalf("GenerateHTML error: %v", err) + } + // One search input per paginated table + if strings.Count(html, `type="text"`) < 2 { + t.Error("expected at least 2 search text inputs") + } + if !strings.Contains(html, "onkeyup") { + t.Error("expected onkeyup handler on search inputs") + } +} + +func TestGenerateHTML_PaginationControls(t *testing.T) { + data := makeTestData() + ComputeStats(&data) + + html, err := GenerateHTML(data) + if err != nil { + t.Fatalf("GenerateHTML error: %v", err) + } + // Prev/Next buttons for both tables + if strings.Count(html, "Prev") < 2 { + t.Error("expected Prev button for each paginated table") + } + if strings.Count(html, "Next") < 2 { + t.Error("expected Next button for each paginated table") + } + // Page-size selector + if strings.Count(html, ` + +
+ + +
+ +
+ + +
+ +
+ + + + + +
TimeIPMethodPathStatusBytesUser Agent
+
+ + + +
+

// Error Log

+
+
+ + +
+
+ + +
+ +
+ + +
+
+
+ + + + + +
TimeLevelPIDConnMessage
+
+
+ + + + + + + + +` + +// indexTemplate is the cyberpunk-themed HTML template for the index page +// generated by --split-by-day. It lists all daily reports with aggregate stats. +const indexTemplate = ` + + + + +nxstats — Daily Index + + + + +
+

// nxstats //

+
daily index — generated {{.GeneratedAt.Format "2006-01-02 15:04:05 UTC"}}
+
+ +
+ + +
+
+
Total Requests
+
{{.TotalRequests}}
+
+
+
Unique IPs
+
{{.TotalUniqueIPs}}
+
+
+
Total Bytes
+
{{formatBytes .TotalBytes}}
+
+
+
Overall Error Rate
+
{{formatFloat .OverallErrorRate}}%
+
+
+
Days Covered
+
{{len .Days}}
+
+
+ + +
+

// Daily Breakdown

+
+ + + + + + + + + + {{range .Days}} + + + + + + + + {{end}} + +
DateRequestsUnique IPsBytesError Rate
{{fmtDate .Date}}{{.Requests}}{{.UniqueIPs}}{{formatBytes .TotalBytes}}{{formatFloat .ErrorRate}}%
+
+
+ +
+ + + + + +` diff --git a/main.go b/main.go new file mode 100644 index 0000000..47da21a --- /dev/null +++ b/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/mr0xb/nxstats/cmd" + +func main() { + cmd.Execute() +} diff --git a/screenshot1.png b/screenshot1.png new file mode 100644 index 0000000..8661118 Binary files /dev/null and b/screenshot1.png differ diff --git a/screenshot2.png b/screenshot2.png new file mode 100644 index 0000000..a2bbef2 Binary files /dev/null and b/screenshot2.png differ