Skip to content

Commit

Permalink
Merge pull request #11 from bitfield/column
Browse files Browse the repository at this point in the history 
Add Column() filter and Visitors example
  • Loading branch information
@bitfield
bitfield authored Jul 3, 2019
2 parents f8d212a + 77c46ad commit aa865a3
Show file tree
Hide file tree
Showing 12 changed files with 360 additions and 64 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,6 @@ examples/cat/cat
examples/grep/grep
examples/cat2/cat2
examples/echo/echo
examples/head/head
examples/visitors/visitors
.vscode/settings.json
268 changes: 219 additions & 49 deletions README.md
View file

Large diffs are not rendered by default.

25 changes: 25 additions & 0 deletions examples/visitors/access.log
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 2028 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 162544 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 9419 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 2058 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 343743 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
212.205.21.11 - - [30/Jun/2019:17:06:16 +0000] "GET / HTTP/1.1" 200 1150 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
212.205.21.11 - - [30/Jun/2019:17:06:16 +0000] "GET / HTTP/1.1" 200 2946 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
176.182.2.191 - - [30/Jun/2019:17:06:17 +0000] "GET / HTTP/1.1" 200 13278 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 29474 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 29349 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 48271 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 1380 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 2028 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 91819 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:19 +0000] "GET / HTTP/1.1" 200 305667 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 13194 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 12935 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 14598 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 22458 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 200 15737 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:20 +0000] "GET / HTTP/1.1" 404 17679 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
176.182.2.191 - - [30/Jun/2019:17:06:23 +0000] "GET / HTTP/1.1" 200 5995 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
190.253.121.1 - - [30/Jun/2019:17:06:23 +0000] "GET / HTTP/1.1" 200 8809 "-" "Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-J415FN Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.2 Chrome/67.0.3396.87 Mobile Safari/537.36"
176.182.2.191 - - [30/Jun/2019:17:06:24 +0000] "GET / HTTP/1.1" 200 162544 "https://example.com/ "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
90.53.111.17 - - [30/Jun/2019:17:06:24 +0000] "GET / HTTP/1.1" 200 - "https://example.com/ "Mozilla/5.0 (Linux; Android 9; SAMSUNG SM-J415FN Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.2 Chrome/67.0.3396.87 Mobile Safari/537.36"
5 changes: 5 additions & 0 deletions examples/visitors/go.mod
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
module visitors

go 1.12

require github.com/bitfield/script v0.9.0
24 changes: 24 additions & 0 deletions examples/visitors/main.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
This program reads an Apache logfile in Common Log Format, like this:
212.205.21.11 - - [30/Jun/2019:17:06:15 +0000] "GET / HTTP/1.1" 200 2028 "https://example.com/ "Mozilla/5.0 (Linux; Android 8.0.0; FIG-LX1 Build/HUAWEIFIG-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.156 Mobile Safari/537.36"
It extracts the first column of each line (the visitor IP address), counts the
frequency of each unique IP address in the log, and outputs the 10 most frequent
visitors in the log. Example output:
16 176.182.2.191
7 212.205.21.11
1 190.253.121.1
1 90.53.111.17
*/
package main

import (
"github.com/bitfield/script"
)

func main() {
script.Stdin().Column(1).Freq().First(10).Stdout()
}
32 changes: 27 additions & 5 deletions filters.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"os/exec"
"regexp"
"sort"
"strconv"
"strings"
)

Expand All @@ -19,7 +20,7 @@ func (p *Pipe) Match(s string) *Pipe {
return p.EachLine(func(line string, out *strings.Builder) {
if strings.Contains(line, s) {
out.WriteString(line)
out.WriteByte('\n')
out.WriteRune('\n')
}
})
}
Expand All @@ -31,7 +32,7 @@ func (p *Pipe) MatchRegexp(re *regexp.Regexp) *Pipe {
return p.EachLine(func(line string, out *strings.Builder) {
if re.MatchString(line) {
out.WriteString(line)
out.WriteByte('\n')
out.WriteRune('\n')
}
})
}
Expand All @@ -43,7 +44,7 @@ func (p *Pipe) Reject(s string) *Pipe {
return p.EachLine(func(line string, out *strings.Builder) {
if !strings.Contains(line, s) {
out.WriteString(line)
out.WriteByte('\n')
out.WriteRune('\n')
}
})
}
Expand All @@ -55,7 +56,7 @@ func (p *Pipe) RejectRegexp(re *regexp.Regexp) *Pipe {
return p.EachLine(func(line string, out *strings.Builder) {
if !re.MatchString(line) {
out.WriteString(line)
out.WriteByte('\n')
out.WriteRune('\n')
}
})
}
Expand Down Expand Up @@ -153,6 +154,7 @@ func (p *Pipe) First(lines int) *Pipe {
output.WriteString(scanner.Text())
output.WriteRune('\n')
}
p.Close()
err := scanner.Err()
if err != nil {
p.SetError(err)
Expand All @@ -178,19 +180,39 @@ func (p *Pipe) Freq() *Pipe {
count int
}
var freqs = make([]frequency, 0, len(freq))
var maxCount int
for line, count := range freq {
freqs = append(freqs, frequency{line, count})
if count > maxCount {
maxCount = count
}
}
sort.Slice(freqs, func(i, j int) bool {
if freqs[i].count == freqs[j].count {
return freqs[i].line < freqs[j].line
}
return freqs[i].count > freqs[j].count
})
fieldWidth := len(strconv.Itoa(maxCount))
var output strings.Builder
for _, item := range freqs {
output.WriteString(fmt.Sprintf("%d %s", item.count, item.line))
output.WriteString(fmt.Sprintf("%*d %s", fieldWidth, item.count, item.line))
output.WriteRune('\n')
}
return Echo(output.String())
}

// Column reads from the pipe, and returns a new pipe containing only the Nth
// column of each line in the input, where '1' means the first column, and
// columns are delimited by whitespace. Specifically, whatever Unicode defines
// as whitespace ('WSpace=yes'). If there is an error reading the pipe, the
// pipe's error status is also set.
func (p *Pipe) Column(col int) *Pipe {
return p.EachLine(func(line string, out *strings.Builder) {
columns := strings.Fields(line)
if col <= len(columns) {
out.WriteString(columns[col-1])
out.WriteRune('\n')
}
})
}
32 changes: 27 additions & 5 deletions filters_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,23 +226,30 @@ func TestFirst(t *testing.T) {
if err != nil {
t.Fatal(err)
}
got, err := File("testdata/first.input.txt").First(10).Bytes()
input := File("testdata/first.input.txt")
got, err := input.First(10).Bytes()
if err != nil {
t.Error(err)
}
if !bytes.Equal(got, want) {
t.Errorf("want %q, got %q", want, got)
}
// First(0) should return zero lines
zero := File("testdata/first.input.txt").First(0)
gotZero, err := zero.CountLines()
_, err = ioutil.ReadAll(input.Reader)
if err == nil {
t.Error("input not closed after reading")
}
input = File("testdata/first.input.txt")
gotZero, err := input.First(0).CountLines()
if err != nil {
t.Fatal(err)
}
if gotZero != 0 {
t.Errorf("want 0 lines, got %d lines", gotZero)
}
// First(N) where the input has less than N lines, should just return the input.
_, err = ioutil.ReadAll(input.Reader)
if err == nil {
t.Error("input not closed after reading")
}
want, err = File("testdata/first.input.txt").Bytes()
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -270,3 +277,18 @@ func TestFreq(t *testing.T) {
t.Errorf("want %q, got %q", want, got)
}
}

func TestColumn(t *testing.T) {
t.Parallel()
want, err := ioutil.ReadFile("testdata/column.golden.txt")
if err != nil {
t.Fatal(err)
}
got, err := File("testdata/column.input.txt").Column(3).Bytes()
if err != nil {
t.Error(err)
}
if !bytes.Equal(got, want) {
t.Errorf("want %q, got %q", want, got)
}
}
2 changes: 2 additions & 0 deletions pipes_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ func doMethodsOnPipe(t *testing.T, p *Pipe, kind string) {
p.First(1)
action = "Freq()"
p.Freq()
action = "Column()"
p.Column(2)
}

func TestNilPipes(t *testing.T) {
Expand Down
8 changes: 8 additions & 0 deletions testdata/column.golden.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
Ss+
R+
Ss
Ss+
Ss+
Ss+
Ss+
Ss
9 changes: 9 additions & 0 deletions testdata/column.input.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
60916 s003 Ss+ 0:00.51 /bin/bash -l
6653 s004 R+ 0:00.01 ps ax
bogus line
80159 s004 Ss 0:00.56 /bin/bash -l
60942 s006 Ss+ 0:00.53 /bin/bash -l
60943 s007 Ss+ 0:00.51 /bin/bash -l
60977 s009 Ss+ 0:00.52 /bin/bash -l
60978 s010 Ss+ 0:00.53 /bin/bash -l
61356 s011 Ss 0:00.54 /bin/bash -l
8 changes: 4 additions & 4 deletions testdata/freq.golden.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
4 apple
4 banana
2 orange
1 kumquat
10 apple
4 banana
2 orange
1 kumquat
8 changes: 7 additions & 1 deletion testdata/freq.input.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,10 @@ kumquat
apple
apple
banana
banana
banana
apple
apple
apple
apple
apple
apple

0 comments on commit aa865a3

Please sign in to comment.