From a3e30a5a3c776c97401ed06b824c3cbf87385d64 Mon Sep 17 00:00:00 2001 From: Ray Miller Date: Sat, 10 Apr 2021 14:01:23 +0100 Subject: [PATCH 01/10] Add gpx-anomalies command --- cmd/gpx-anomalies/main.go | 131 ++++++++++++++++++++++++++++++++++++++ go.mod | 10 +-- go.sum | 36 ++++++++--- 3 files changed, 165 insertions(+), 12 deletions(-) create mode 100644 cmd/gpx-anomalies/main.go diff --git a/cmd/gpx-anomalies/main.go b/cmd/gpx-anomalies/main.go new file mode 100644 index 0000000..d162a31 --- /dev/null +++ b/cmd/gpx-anomalies/main.go @@ -0,0 +1,131 @@ +package main + +import ( + "fmt" + "log" + "math" + "os" + + "github.com/fofanov/go-osgb" + "github.com/twpayne/go-gpx" + "github.com/urfave/cli/v2" +) + +func main() { + log.SetFlags(0) + app := &cli.App{ + Name: "gpx-anomalies", + Usage: "Find repeated points in a GPX track", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "gpx-file", + Aliases: []string{"g"}, + Usage: "Name of GPX file to process", + Required: true, + }, + &cli.Float64Flag{ + Name: "fuzz", + Aliases: []string{"f"}, + Usage: "Consider two points coincident if they are within FUZZ kilometres of each other", + Value: 0.005, + }, + &cli.Float64Flag{ + Name: "min-distance", + Aliases: []string{"min"}, + Usage: "Only show repeats that appear at least MIN kilometers apart", + Value: 0.1, + }, + &cli.Float64Flag{ + Name: "max-distance", + Aliases: []string{"max"}, + Usage: "Do not show repeats that appear more than MAX kilometers apart", + Value: 5.0, + }, + }, + Action: func(c *cli.Context) error { + points, err := readGPXTrack(c.String("gpx-file")) + if err != nil { + log.Fatal(err) + } + findDuplicates( + points, + c.Float64("fuzz")*1000.0, + c.Float64("min-distance")*1000.0, + c.Float64("max-distance")*1000.0, + ) + return nil + }, + } + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } + +} + +func findDuplicates(points []RoutePoint, fuzz, minDist, maxDist float64) { + for i := range points { + p := points[i] + for j := i + 1; j < len(points); j++ { + q := points[j] + if p.Distance == q.Distance { + continue + } + d := euclideanDistance(p.Coordinate, q.Coordinate) + D := q.Distance - p.Distance + if d < fuzz && D > minDist && D < maxDist { + fmt.Printf("Point (%0.f, %0.f) revisited at %0.2f km and %0.2f km\n", + p.Coordinate.Easting, p.Coordinate.Northing, p.Distance/1000.0, q.Distance/1000.0) + } + } + } +} + +func euclideanDistance(p, q *osgb.OSGB36Coordinate) float64 { + x := p.Easting - q.Easting + y := p.Northing - q.Northing + return math.Sqrt(x*x + y*y) +} + +func readGPXTrack(filename string) ([]RoutePoint, error) { + r, err := os.Open(filename) + if err != nil { + return nil, fmt.Errorf("error opening %s for reading: %v", filename, err) + } + defer r.Close() + g, err := gpx.Read(r) + if err != nil { + return nil, fmt.Errorf("error reading GPS track %s: %v", filename, err) + } + trans, err := osgb.NewOSTN15Transformer() + if err != nil { + return nil, fmt.Errorf("error constructing coordinate transformer: %v", err) + } + distance := 0.0 + var prevPoint *osgb.OSGB36Coordinate + var points []RoutePoint + for _, trk := range g.Trk { + for _, seg := range trk.TrkSeg { + for _, trkPt := range seg.TrkPt { + gpsCoord := osgb.NewETRS89Coord(trkPt.Lon, trkPt.Lat, trkPt.Ele) + p, err := trans.ToNationalGrid(gpsCoord) + if err != nil { + return nil, fmt.Errorf("error converting coordinates to National Grid: %v", err) + } + if prevPoint != nil { + distance += euclideanDistance(prevPoint, p) + } + points = append(points, RoutePoint{ + Coordinate: p, + Distance: distance, + }) + prevPoint = p + } + } + } + return points, nil +} + +type RoutePoint struct { + Coordinate *osgb.OSGB36Coordinate + Distance float64 +} diff --git a/go.mod b/go.mod index 6c63ab4..2a46e8e 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,14 @@ module github.com/ray1729/gpx-utils go 1.13 require ( - github.com/d4l3k/messagediff v1.2.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/dhconnelly/rtreego v1.0.0 github.com/fofanov/go-osgb v0.0.0-20170711141822-6893d1f95cd9 + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/twpayne/go-geom v1.3.6 // indirect github.com/twpayne/go-gpx v1.2.0 - github.com/wlbr/mule v0.0.0-20200329114911-0724e1639b62 // indirect - golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7 // indirect - golang.org/x/text v0.3.4 // indirect + github.com/urfave/cli/v2 v2.3.0 + github.com/wlbr/mule v0.0.0-20200517121540-6f9faa2e2d0b // indirect + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect + golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1 // indirect ) diff --git a/go.sum b/go.sum index d3ee44a..8926029 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,5 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.2/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= @@ -7,8 +8,12 @@ github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jB github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dhconnelly/rtreego v1.0.0 h1:1+V1STGw+zwx7jpvH/fwbeC5w5gZfn+XinARU45oRek= github.com/dhconnelly/rtreego v1.0.0/go.mod h1:SDozu0Fjy17XH1svEXJgdYq8Tah6Zjfa/4Q33Z80+KM= @@ -32,7 +37,14 @@ github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zM github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/ory/dockertest/v3 v3.6.0/go.mod h1:4ZOpj8qBUmh8fcBSVzkH2bws2s91JdGvHUqan4GHEuQ= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -41,31 +53,32 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/twpayne/go-geom v1.0.0 h1:ARrRnN4+rBX3LZFZQy9NFeXXlgQqVL4OOvAcnLdgy2g= github.com/twpayne/go-geom v1.0.0/go.mod h1:RWsl+e3XSahOul/KH2BHCfF0QxSL4RMnMlFw/TNmET0= github.com/twpayne/go-geom v1.3.6 h1:O27mIXZnMYiZi0ZD8ewjs/IT/ZOFVbZHBzPjA9skdmg= github.com/twpayne/go-geom v1.3.6/go.mod h1:XTyWHR6+l9TUYONbbK4ImUTYbWDCu2ySSPrZmmiA0Pg= -github.com/twpayne/go-gpx v1.1.1 h1:vbg0lRc/ZKSu8ev84/hJWZtplKJdBbucNmks4TNzSqQ= -github.com/twpayne/go-gpx v1.1.1/go.mod h1:fQ+EsiFNgDuErUYyI0ZOgZPB+ACxW58L16oormty798= github.com/twpayne/go-gpx v1.2.0 h1:Jjq0NKXgHmEXXhmQue4KWtAVG5gxkAYY+FvsM1AliLQ= github.com/twpayne/go-gpx v1.2.0/go.mod h1:70xTQn0dGph3dgKIPxfl0K3XMVNpulC70/e383iHouA= github.com/twpayne/go-kml v1.0.0/go.mod h1:LlvLIQSfMqYk2O7Nx8vYAbSLv4K9rjMvLlEdUKWdjq0= github.com/twpayne/go-kml v1.5.1/go.mod h1:kz8jAiIz6FIdU2Zjce9qGlVtgFYES9vt7BTPBHf5jl4= github.com/twpayne/go-polyline v1.0.0/go.mod h1:ICh24bcLYBX8CknfvNPKqoTbe+eg+MX1NPyJmSBo7pU= github.com/twpayne/go-waypoint v0.0.0-20200706203930-b263a7f6e4e8/go.mod h1:qj5pHncxKhu9gxtZEYWypA/z097sxhFlbTyOyt9gcnU= -github.com/wlbr/mule v0.0.0-20200329114911-0724e1639b62 h1:vHDdpwOGHzfFKbMLEnnM0s1jnGLjsQ9EPWtCFWMJs8o= -github.com/wlbr/mule v0.0.0-20200329114911-0724e1639b62/go.mod h1:uDXgZTfL0uJWiY/MQKcqI5VPQV8PCooNsWXozHf7CJ8= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/wlbr/mule v0.0.0-20200517121540-6f9faa2e2d0b h1:wZDyxL+jeSaBLmUFM+k/P97BbS8z3QYKcfbvWPnKq9Q= +github.com/wlbr/mule v0.0.0-20200517121540-6f9faa2e2d0b/go.mod h1:uDXgZTfL0uJWiY/MQKcqI5VPQV8PCooNsWXozHf7CJ8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/net v0.0.0-20180824152047-4bcd98cce591 h1:4S2XUgvg3hUNTvxI307qkFPb9zKHG3Nf9TXFzX/DZZI= golang.org/x/net v0.0.0-20180824152047-4bcd98cce591/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7 h1:3uJsdck53FDIpWwLeAXlia9p4C8j0BO2xZrqzKpL0D8= golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1 h1:4qWs8cYYH6PoEFy4dfhDFgoMGkwAcETd+MmPdCPMzUc= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -74,16 +87,23 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200121082415-34d275377bf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= From 08f37e8592f3473fd1ee8a37fdaf886d418547f7 Mon Sep 17 00:00:00 2001 From: Ray Miller Date: Sat, 10 Apr 2021 14:15:03 +0100 Subject: [PATCH 02/10] Add CI/CD config --- .gitlab-ci.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..5bbcad4 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,25 @@ +stages: + - build + - deploy + +build-gpx-anomalies: + stage: build + image: golang:1.15-buster + before_script: + - apt-get -qq update && apt-get --yes install zip + script: + - env GOOS=windows go build -o gpx-anomalies ./cmd/gpx-anomalies + - zip gpx-anomalies.zip gpx-anomalies + artifacts: + paths: + - gpx-anomalies.zip + +publish-gpx-anomalies: + stage: deploy + image: curlimages/curl:latest + variables: + GIT_STRATEGY: none + script: + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file gpx-anomalies.zip "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/gpx-anomalies/latest/gpx-anomalies.zip"' + rules: + - if: '$CI_COMMIT_BRANCH == "master"' From bc6affbce19c4a0dff9b4e0983cb3eb5320c6006 Mon Sep 17 00:00:00 2001 From: Ray Miller Date: Sat, 10 Apr 2021 14:22:22 +0100 Subject: [PATCH 03/10] Call the executable exe --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5bbcad4..5665847 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,8 +8,8 @@ build-gpx-anomalies: before_script: - apt-get -qq update && apt-get --yes install zip script: - - env GOOS=windows go build -o gpx-anomalies ./cmd/gpx-anomalies - - zip gpx-anomalies.zip gpx-anomalies + - env GOOS=windows go build -o gpx-anomalies.exe ./cmd/gpx-anomalies + - zip gpx-anomalies.zip gpx-anomalies.exe artifacts: paths: - gpx-anomalies.zip From 34bdaeab71a4cf1fdad882dfebc796e055b51657 Mon Sep 17 00:00:00 2001 From: Ray Miller Date: Sun, 11 Apr 2021 18:39:17 +0100 Subject: [PATCH 04/10] Suppress duplicates in the output --- cmd/gpx-anomalies/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/gpx-anomalies/main.go b/cmd/gpx-anomalies/main.go index d162a31..22c43ab 100644 --- a/cmd/gpx-anomalies/main.go +++ b/cmd/gpx-anomalies/main.go @@ -63,6 +63,7 @@ func main() { } func findDuplicates(points []RoutePoint, fuzz, minDist, maxDist float64) { + var lastError *RoutePoint for i := range points { p := points[i] for j := i + 1; j < len(points); j++ { @@ -72,7 +73,8 @@ func findDuplicates(points []RoutePoint, fuzz, minDist, maxDist float64) { } d := euclideanDistance(p.Coordinate, q.Coordinate) D := q.Distance - p.Distance - if d < fuzz && D > minDist && D < maxDist { + if d < fuzz && D > minDist && D < maxDist && (lastError == nil || p.Distance-lastError.Distance > 500) { + lastError = &p fmt.Printf("Point (%0.f, %0.f) revisited at %0.2f km and %0.2f km\n", p.Coordinate.Easting, p.Coordinate.Northing, p.Distance/1000.0, q.Distance/1000.0) } From f740686d2f2c0be11482afae1f245a0b5499f791 Mon Sep 17 00:00:00 2001 From: Ray Miller Date: Mon, 12 Apr 2021 09:15:20 +0100 Subject: [PATCH 05/10] Improved duplicate suppression --- cmd/gpx-anomalies/main.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/gpx-anomalies/main.go b/cmd/gpx-anomalies/main.go index 22c43ab..28856c4 100644 --- a/cmd/gpx-anomalies/main.go +++ b/cmd/gpx-anomalies/main.go @@ -73,10 +73,12 @@ func findDuplicates(points []RoutePoint, fuzz, minDist, maxDist float64) { } d := euclideanDistance(p.Coordinate, q.Coordinate) D := q.Distance - p.Distance - if d < fuzz && D > minDist && D < maxDist && (lastError == nil || p.Distance-lastError.Distance > 500) { + if d < fuzz && D > minDist && D < maxDist { + if lastError == nil || p.Distance-lastError.Distance > 500 { + fmt.Printf("Point (%0.f, %0.f) revisited at %0.2f km and %0.2f km\n", + p.Coordinate.Easting, p.Coordinate.Northing, p.Distance/1000.0, q.Distance/1000.0) + } lastError = &p - fmt.Printf("Point (%0.f, %0.f) revisited at %0.2f km and %0.2f km\n", - p.Coordinate.Easting, p.Coordinate.Northing, p.Distance/1000.0, q.Distance/1000.0) } } } From 9ec3b2e9802289f710d515021be97d6e3a3d36df Mon Sep 17 00:00:00 2001 From: Ray Miller Date: Fri, 18 Jun 2021 11:56:16 +0100 Subject: [PATCH 06/10] Increase the search radius for cafe stops --- pkg/cafes/common.go | 5 ++++- scripts/summarizeRoutes.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 scripts/summarizeRoutes.py diff --git a/pkg/cafes/common.go b/pkg/cafes/common.go index e2f21bc..d059b09 100644 --- a/pkg/cafes/common.go +++ b/pkg/cafes/common.go @@ -9,6 +9,9 @@ import ( "github.com/dhconnelly/rtreego" ) +// Size (in metres) of the bounding box around a stop +const stopRectangleSize = 500 + type RefreshmentStop struct { Name string Url string @@ -18,7 +21,7 @@ type RefreshmentStop struct { func (s *RefreshmentStop) Bounds() *rtreego.Rect { p := rtreego.Point{s.Easting, s.Northing} - return p.ToRect(100) + return p.ToRect(stopRectangleSize) } func (s *RefreshmentStop) Contains(p rtreego.Point) bool { diff --git a/scripts/summarizeRoutes.py b/scripts/summarizeRoutes.py new file mode 100644 index 0000000..c18bd4f --- /dev/null +++ b/scripts/summarizeRoutes.py @@ -0,0 +1,21 @@ +#!/usr/bin/python3 + +import csv +import sys + +path = sys.argv[1] + +with open(path) as f: + r = csv.reader(f) + skip = True + for x in r: + if skip: + skip = False + continue + coffee = x[7].strip() + lunch = x[8].strip() + tea = x[9].strip() + if coffee and tea: + print(coffee + ", " + lunch + ", " + tea) + + From 4de001c8677c61df38bfa0a8e0ec57ffd31e1a68 Mon Sep 17 00:00:00 2001 From: Ray Miller Date: Thu, 19 Jan 2023 16:20:22 +0000 Subject: [PATCH 07/10] Fix bug in logging: need Printf as string contains format directives. --- pkg/rwgps/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rwgps/handler.go b/pkg/rwgps/handler.go index f2fe661..4ac61d4 100644 --- a/pkg/rwgps/handler.go +++ b/pkg/rwgps/handler.go @@ -41,7 +41,7 @@ func (h *RWGPSHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } routeId, err := strconv.Atoi(rawRouteId) if err != nil { - log.Println("Error parsing route id '%s': %v", rawRouteId, err) + log.Printf("Error parsing route id '%s': %v", rawRouteId, err) http.Error(w, fmt.Sprintf("Invalid routeId: %s", rawRouteId), http.StatusBadRequest) return } From bd7eb246d5d0fc0b1663d70d9544cf8c46126527 Mon Sep 17 00:00:00 2001 From: Ray Miller Date: Thu, 19 Jan 2023 16:21:32 +0000 Subject: [PATCH 08/10] Improvements to duplicate suppression, cafe stop search, and configurability. When suppressing duplicate cafes and place names, look back a certain distance along the route rather than just the previous point of interest. When searching for cafes, use SearchIntersect() to return all entries in the bounding rectangle, not just the nearest. Remove (most) hard-coded constants and allow these to be overriden by options to the NewGPXSummarizer() constructor. --- cmd/analyze-gpx/main.go | 17 +++-- pkg/cafes/common.go | 2 +- pkg/placenames/summarize.go | 135 ++++++++++++++++++++++++++++-------- 3 files changed, 118 insertions(+), 36 deletions(-) diff --git a/cmd/analyze-gpx/main.go b/cmd/analyze-gpx/main.go index e88cdb8..fa1c0bc 100644 --- a/cmd/analyze-gpx/main.go +++ b/cmd/analyze-gpx/main.go @@ -19,8 +19,11 @@ import ( func main() { log.SetFlags(0) stopNames := flag.String("stops", "", "Source for refreshment stops") - minDist := flag.Float64("min-dist", 0.2, "Minimum distance (km) between points of interest") - minSettlement := flag.String("min-settlement", "Other Settlement", "Exclude populated places smaller than this (City, Town, Village, Hamlet, Other Settlement)") + stopRect := flag.Float64("sr", placenames.DefaultGPXSummarizerConfig.CoffeeStopSearchRectangleSize, "Size (m) of the rectangle we search for coffee stops near the route") + stopDupDist := flag.Float64("sdd", placenames.DefaultGPXSummarizerConfig.CoffeeStopDuplicateDistance, "Suppress recurrences of coffee stops within this distance (km)") + dupDist := flag.Float64("dd", placenames.DefaultGPXSummarizerConfig.PointOfInterestDuplicateDistance, "Suppress recurrences of points of interest within this distance (km)") + minDist := flag.Float64("md", placenames.DefaultGPXSummarizerConfig.PointOfInterestMinimumDistance, "Minimum distance (km) between points of interest") + minSettlement := flag.String("ms", "Other Settlement", "Exclude populated places smaller than this (City, Town, Village, Hamlet, Other Settlement)") flag.Parse() if flag.NArg() != 1 { log.Fatal("Usage: %s [--stops=ctccambridge|cyclingmaps] [--min-dist X] [--min-settlement S] GPX_FILE_OR_DIRECTORY") @@ -38,12 +41,16 @@ func main() { log.Fatal(err) } } - gs, err := placenames.NewGPXSummarizer() + gs, err := placenames.NewGPXSummarizer( + placenames.WithMinimumSettlement(*minSettlement), + placenames.WithPointOfInterestMinimumDistance(*minDist), + placenames.WithPointOfInterestDuplicateDistance(*dupDist), + placenames.WithCoffeeStopSearchRectangleSize(*stopRect), + placenames.WithCoffeeStopDuplicateDistance(*stopDupDist), + ) if err != nil { log.Fatal(err) } - gs.SetMinDistance(*minDist) - gs.SetMinSettlement(*minSettlement) if info.IsDir() { err = summarizeDirectory(gs, stops, inFile) } else { diff --git a/pkg/cafes/common.go b/pkg/cafes/common.go index d059b09..b216e7e 100644 --- a/pkg/cafes/common.go +++ b/pkg/cafes/common.go @@ -10,7 +10,7 @@ import ( ) // Size (in metres) of the bounding box around a stop -const stopRectangleSize = 500 +const stopRectangleSize = 50 type RefreshmentStop struct { Name string diff --git a/pkg/placenames/summarize.go b/pkg/placenames/summarize.go index b436237..cfb770d 100644 --- a/pkg/placenames/summarize.go +++ b/pkg/placenames/summarize.go @@ -9,9 +9,8 @@ import ( "github.com/dhconnelly/rtreego" "github.com/fofanov/go-osgb" - "github.com/twpayne/go-gpx" - "github.com/ray1729/gpx-utils/pkg/cafes" + "github.com/twpayne/go-gpx" ) var populatedPlaceRank = map[string]int{ @@ -22,14 +21,80 @@ var populatedPlaceRank = map[string]int{ "Other Settlement": 1, } -type GPXSummarizer struct { - poi *rtreego.Rtree - trans osgb.CoordinateTransformer - minDist float64 - minSettlementRank int +// GPXSummarizerConfig allows override of defaults used by the search algorithm. +type GPXSummarizerConfig struct { + CoffeeStopSearchRectangleSize float64 + CoffeeStopDuplicateDistance float64 + PointOfInterestDuplicateDistance float64 + PointOfInterestMinimumDistance float64 + MinimumSettlementRank int } -func NewGPXSummarizer() (*GPXSummarizer, error) { +var DefaultGPXSummarizerConfig = GPXSummarizerConfig{ + CoffeeStopSearchRectangleSize: 500.0, // m + CoffeeStopDuplicateDistance: 2.0, // km + PointOfInterestDuplicateDistance: 1.0, // km + PointOfInterestMinimumDistance: 0.2, // km + MinimumSettlementRank: 1, // "Other Settlement" +} + +type Option func(*GPXSummarizerConfig) + +// WithCoffeeStopSearchRectangleSize overrides the size (in metres) of the rectangle searched +// for coffee stops near the route. Default 500m. +func WithCoffeeStopSearchRectangleSize(d float64) Option { + return func(c *GPXSummarizerConfig) { + c.CoffeeStopSearchRectangleSize = d + } +} + +// WithCoffeeStopDuplicateDistance overrides the distance (in kilometers) we look back along the +// route when suppressing duplicate coffee stop entries. This should be at least twice the +// CoffeeStopSearchRectangleSize. Default 2km. +func WithCoffeeStopDuplicateDistance(d float64) Option { + return func(c *GPXSummarizerConfig) { + c.CoffeeStopDuplicateDistance = d + } +} + +// WithPointOfInterestDuplicateDistance overrides the distance (in km) we look back along +// the route when suppressing duplicate points of interest. +func WithPointOfInterestDuplicateDistance(d float64) Option { + return func(c *GPXSummarizerConfig) { + c.PointOfInterestDuplicateDistance = d + } +} + +// WithPointOfInterestMinimumDistance overrides the minimum distance (in km) between points +// of interest (if two POI appear within this distance, the second one is suppressed). Default +// 0km (no suppression). +func WithPointOfInterestMinimumDistance(d float64) Option { + return func(c *GPXSummarizerConfig) { + c.PointOfInterestMinimumDistance = d + } +} + +func WithMinimumSettlement(s string) Option { + rank, ok := populatedPlaceRank[s] + if !ok { + panic(fmt.Sprintf("invalid settlement type: %s", s)) + } + return func(c *GPXSummarizerConfig) { + c.MinimumSettlementRank = rank + } +} + +type GPXSummarizer struct { + poi *rtreego.Rtree + trans osgb.CoordinateTransformer + conf GPXSummarizerConfig +} + +func NewGPXSummarizer(opts ...Option) (*GPXSummarizer, error) { + conf := DefaultGPXSummarizerConfig + for _, f := range opts { + f(&conf) + } trans, err := osgb.NewOSTN15Transformer() if err != nil { return nil, err @@ -38,15 +103,7 @@ func NewGPXSummarizer() (*GPXSummarizer, error) { if err != nil { return nil, err } - return &GPXSummarizer{poi: rt, trans: trans, minDist: 0.2, minSettlementRank: 1}, nil -} - -func (gs *GPXSummarizer) SetMinSettlement(t string) { - gs.minSettlementRank = populatedPlaceRank[t] -} - -func (gs *GPXSummarizer) SetMinDistance(d float64) { - gs.minDist = d + return &GPXSummarizer{poi: rt, trans: trans, conf: conf}, nil } func distance(p1, p2 rtreego.Point) float64 { @@ -108,7 +165,6 @@ func (gs *GPXSummarizer) SummarizeTrack(r io.Reader, stops *rtreego.Rtree) (*Tra var prevPlace string var prevPlacePoint rtreego.Point var prevPoint rtreego.Point - var prevStop *cafes.RefreshmentStop var start rtreego.Point var dN, dE float64 @@ -141,25 +197,44 @@ func (gs *GPXSummarizer) SummarizeTrack(r io.Reader, stops *rtreego.Rtree) (*Tra s.Distance += distance(thisPoint, prevPoint) dE += thisPoint[0] - start[0] dN += thisPoint[1] - start[1] - if nn.Contains(thisPoint) { + if nn.Contains(thisPoint) && populatedPlaceRank[nn.Type] >= gs.conf.MinimumSettlementRank { s.Counties[nn.County]++ - if nn.Name != prevPlace && - distance(thisPoint, prevPlacePoint) > gs.minDist && - populatedPlaceRank[nn.Type] >= gs.minSettlementRank { + seenRecently := false + for i := len(s.PointsOfInterest) - 1; i >= 0; i-- { + if i < len(s.PointsOfInterest)-1 && s.Distance-s.PointsOfInterest[i].Distance > gs.conf.PointOfInterestDuplicateDistance { + break + } + if nn.Name == s.PointsOfInterest[i].Name { + seenRecently = true + break + } + } + if !seenRecently && distance(thisPoint, prevPlacePoint) > gs.conf.PointOfInterestMinimumDistance { s.PointsOfInterest = append(s.PointsOfInterest, POI{Name: nn.Name, Type: nn.Type, Distance: s.Distance}) prevPlace = nn.Name prevPlacePoint = thisPoint } } if stops != nil { - stop, ok := stops.NearestNeighbor(thisPoint).(*cafes.RefreshmentStop) - if ok && stop.Contains(thisPoint) && (prevStop == nil || stop.Name != prevStop.Name) { - s.RefreshmentStops = append(s.RefreshmentStops, RefreshmentStop{ - Name: stop.Name, - Url: stop.Url, - Distance: s.Distance, - }) - prevStop = stop + for _, nearbyStop := range stops.SearchIntersect(thisPoint.ToRect(gs.conf.CoffeeStopSearchRectangleSize)) { + stop := nearbyStop.(*cafes.RefreshmentStop) + seenRecently := false + for i := len(s.RefreshmentStops) - 1; i >= 0; i-- { + if i < len(s.RefreshmentStops)-1 && s.Distance-s.RefreshmentStops[i].Distance > gs.conf.CoffeeStopDuplicateDistance { + break + } + if s.RefreshmentStops[i].Name == stop.Name { + seenRecently = true + break + } + } + if !seenRecently { + s.RefreshmentStops = append(s.RefreshmentStops, RefreshmentStop{ + Name: stop.Name, + Url: stop.Url, + Distance: s.Distance, + }) + } } } prevPoint = thisPoint From 4ef3eca5a4195cad555d8c63a1e44f7b6db2aefd Mon Sep 17 00:00:00 2001 From: Ray Miller Date: Thu, 19 Jan 2023 16:43:09 +0000 Subject: [PATCH 09/10] Fix default to match docs --- pkg/placenames/summarize.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/placenames/summarize.go b/pkg/placenames/summarize.go index cfb770d..9eaa445 100644 --- a/pkg/placenames/summarize.go +++ b/pkg/placenames/summarize.go @@ -34,7 +34,7 @@ var DefaultGPXSummarizerConfig = GPXSummarizerConfig{ CoffeeStopSearchRectangleSize: 500.0, // m CoffeeStopDuplicateDistance: 2.0, // km PointOfInterestDuplicateDistance: 1.0, // km - PointOfInterestMinimumDistance: 0.2, // km + PointOfInterestMinimumDistance: 0.0, // km MinimumSettlementRank: 1, // "Other Settlement" } From f09fd9a418132e4e5a4741e6c4fd70dfa7e1df94 Mon Sep 17 00:00:00 2001 From: Ray Miller Date: Wed, 17 Apr 2024 17:19:42 +0100 Subject: [PATCH 10/10] Set a custom user-agent header when fetching CTC Cambridge stops data This is required because their hosting provider is returning a 503 "service unavailable" when the user agent is Go-http-client. --- pkg/cafes/ctccam.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/cafes/ctccam.go b/pkg/cafes/ctccam.go index a95c147..0f396ba 100644 --- a/pkg/cafes/ctccam.go +++ b/pkg/cafes/ctccam.go @@ -54,7 +54,12 @@ func BuildCtcCamIndex(r io.Reader) (*rtreego.Rtree, error) { func FetchCtcCamIndex() (*rtreego.Rtree, error) { log.Printf("Fetching %s", ctcCamWaypointsUrl) - res, err := http.Get(ctcCamWaypointsUrl) + req, err := http.NewRequest(http.MethodGet, ctcCamWaypointsUrl, nil) + if err != nil { + return nil, fmt.Errorf("error constructing waypoints request: %v", err) + } + req.Header.Set("User-Agent", "gpx-utils") + res, err := http.DefaultClient.Do(req) if err != nil { return nil, fmt.Errorf("error getting %s: %v", ctcCamWaypointsUrl, err) }