Skip to content
This repository was archived by the owner on Feb 2, 2023. It is now read-only.
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: client9/ipcat
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: master
Choose a base ref
...
head repository: oftn-oswg/ipcat
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: master
Choose a head ref
  • 11 commits
  • 13 files changed
  • 1 contributor

Commits on May 4, 2018

  1. Copy the full SHA
    5f35256 View commit details
  2. Copy the full SHA
    775fc89 View commit details

Commits on May 5, 2018

  1. Copy the full SHA
    14fdd04 View commit details
  2. Copy the full SHA
    bb5c6c3 View commit details
  3. Copy the full SHA
    47a98fd View commit details
  4. Use Go builtin IP to string

    dsamarin committed May 5, 2018
    Copy the full SHA
    0d29dae View commit details

Commits on May 6, 2018

  1. Copy the full SHA
    c9e767d View commit details
  2. Copy the full SHA
    09871cb View commit details
  3. Copy the full SHA
    37afc05 View commit details
  4. Support Tor exit nodes

    dsamarin committed May 6, 2018
    Copy the full SHA
    a512522 View commit details

Commits on May 17, 2018

  1. Copy the full SHA
    a8ecd7e View commit details
Showing with 775 additions and 182 deletions.
  1. +3 −0 Makefile
  2. +19 −10 appengine.go
  3. +25 −9 aws.go
  4. +31 −13 aws_test.go
  5. +0 −2 azure.go
  6. +25 −9 cloudflare.go
  7. +10 −1 cmd/ipcat/main.go
  8. +3 −3 datacenters-stats.csv
  9. +327 −23 datacenters.csv
  10. +141 −97 ipset.go
  11. +26 −15 ipset_test.go
  12. +100 −0 tor.csv
  13. +65 −0 tor.go
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -16,6 +16,9 @@ appengine:
cloudflare:
go run ./cmd/ipcat/main.go -cloudflare

tor:
go run ./cmd/ipcat/main.go -tor -csvfile tor.csv -statsfile=""

test:
find . -name '*.go' | xargs gofmt -w -s
find . -name '*.go' | xargs goimports -w
29 changes: 19 additions & 10 deletions appengine.go
Original file line number Diff line number Diff line change
@@ -30,22 +30,31 @@ func lookupSPFRecord(name string, f func(dir string) error) error {
// DownloadAppEngine downloads and returns raw bytes of the Google App Engine ip
// range list
func DownloadAppEngine() ([]string, error) {
var ranges []string
if err := lookupSPFRecord("_cloud-netblocks.googleusercontent.com", func(dir string) error {
inc := strings.TrimPrefix(dir, "include:")
if dir == inc {
return nil
}
ranges := []string{}
domainList := []string{"_cloud-netblocks.googleusercontent.com"}

return lookupSPFRecord(inc, func(dir string) error {
for len(domainList) > 0 {
// Dequeue a domain from this list
var domain string
domain, domainList = domainList[0], domainList[1:]
err := lookupSPFRecord(domain, func(dir string) error {
// Enqueue domain from this record
if inc := strings.TrimPrefix(dir, "include:"); dir != inc {
domainList = append(domainList, inc)
}
// Add IPv4 range
if ip4 := strings.TrimPrefix(dir, "ip4:"); dir != ip4 {
ranges = append(ranges, ip4)
}

// Add IPv6 range
if ip6 := strings.TrimPrefix(dir, "ip6:"); dir != ip6 {
ranges = append(ranges, ip6)
}
return nil
})
}); err != nil {
return nil, err
if err != nil {
return nil, err
}
}

return ranges, nil
34 changes: 25 additions & 9 deletions aws.go
Original file line number Diff line number Diff line change
@@ -13,16 +13,18 @@ var (

// AWSPrefix is AWS prefix in their IP ranges file
type AWSPrefix struct {
IPPrefix string `json:"ip_prefix"`
Region string `json:"region"`
Service string `json:"service"`
IPPrefix string `json:"ip_prefix"`
IPv6Prefix string `json:"ipv6_prefix"`
Region string `json:"region"`
Service string `json:"service"`
}

// AWS is main record for AWS IP info
type AWS struct {
SyncToken string `json:"syncToken"`
CreateDate string `json:"createDate"`
Prefixes []AWSPrefix `json:"prefixes"`
SyncToken string `json:"syncToken"`
CreateDate string `json:"createDate"`
Prefixes []AWSPrefix `json:"prefixes"`
IPv6Prefixes []AWSPrefix `json:"ipv6_prefixes"`
}

// DownloadAWS downloads the latest AWS IP ranges list
@@ -60,9 +62,23 @@ func UpdateAWS(ipmap *IntervalSet, body []byte) error {
ipmap.DeleteByName(awsName)

// and add back
for _, rec := range aws.Prefixes {
if rec.Service == "EC2" {
err := ipmap.AddCIDR(rec.IPPrefix, awsName, awsURL)
for _, prefixList := range []*[]AWSPrefix{&aws.Prefixes, &aws.IPv6Prefixes} {
for _, rec := range *prefixList {
if rec.Service != "AMAZON" {
// Service is the subset of IP address ranges. Specify AMAZON to get
// all IP address ranges (for example, the ranges in the EC2 subset
// are also in the AMAZON subset). Note that some IP address ranges
// are only in the AMAZON subset.
// <https://docs.aws.amazon.com/general/latest/gr/aws-ip-ranges.html>
continue
}

prefix := rec.IPPrefix
if prefix == "" {
prefix = rec.IPv6Prefix
}

err := ipmap.AddCIDR(prefix, awsName, awsURL)
if err != nil {
return err
}
44 changes: 31 additions & 13 deletions aws_test.go
Original file line number Diff line number Diff line change
@@ -23,6 +23,16 @@ func TestAWS(t *testing.T) {
"ip_prefix": "13.54.0.0/15",
"region": "ap-southeast-2",
"service": "EC2"
},
{
"ipv6_prefix": "2a05:d016::/36",
"region": "eu-north-1",
"service": "AMAZON"
},
{
"ipv6_prefix": "2a05:d018::/36",
"region": "eu-west-1",
"service": "EC2"
}
]
}`)
@@ -39,19 +49,27 @@ func TestAWS(t *testing.T) {
if err != nil {
t.Fatalf("UpdateAWS error: %v", err)
}
// UpdateAWS should only add EC2 IP ranges
rec, err := ipset.Contains("216.182.224.0")
if err != nil {
t.Fatalf("ipset.Contains(%q) error: %v", "216.182.224.0", err)
}
if rec != nil {
t.Errorf("ipset.Contains(%q) rec = %v, want nil", "216.182.224.0", rec)
}
rec, err = ipset.Contains("13.54.0.1")
if err != nil {
t.Fatalf("ipset.Contains(%q) error: %v", "13.54.0.0", err)

tests := []struct {
IP string
Contains bool
}{
{IP: "216.182.224.0", Contains: true},
{IP: "2a05:d016::20", Contains: true},

// Only AMAZON services will be contained (superset of all)
{IP: "13.54.0.1", Contains: false},
{IP: "2a05:d018::20", Contains: false},
}
if rec == nil {
t.Errorf("ipset.Contains(%q) rec = nil, want exists", "13.54.0.0")

for _, test := range tests {
record, err := ipset.Contains(test.IP)
if err != nil {
t.Fatalf("ipset.Contains(%q) error: %v", test.IP, err)
}
if (record != nil) != test.Contains {
t.Errorf("ipset.Contains(%q) rec = %v, want rec != nil to be %v",
test.IP, record, test.Contains)
}
}
}
2 changes: 0 additions & 2 deletions azure.go
Original file line number Diff line number Diff line change
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"io/ioutil"
"log"
"net/http"
"regexp"
)
@@ -58,7 +57,6 @@ func DownloadAzure() ([]byte, error) {
return nil, fmt.Errorf("failed to find public IPs url during retry: %s", err)
}

log.Printf("Attempting ip range download with url %s...", url)
resp, err := http.Get(url)
if err != nil {
return nil, err
34 changes: 25 additions & 9 deletions cloudflare.go
Original file line number Diff line number Diff line change
@@ -3,28 +3,44 @@ package ipcat
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
)

var (
cloudflareDownload = "https://www.cloudflare.com/ips-v4"
cloudflareDownload = []string{
"https://www.cloudflare.com/ips-v4",
"https://www.cloudflare.com/ips-v6",
}
)

// DownloadCloudflare downloads the latest Cloudflare IP ranges list
func DownloadCloudflare() ([]byte, error) {
resp, err := http.Get(cloudflareDownload)
if err != nil {
return nil, err
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("Failed to download Cloudflare ranges: status code %s", resp.Status)
readers := make([]io.Reader, 0, len(cloudflareDownload))

defer func() {
for _, reader := range readers {
reader.(io.ReadCloser).Close()
}
}()

for _, uri := range cloudflareDownload {
resp, err := http.Get(uri)
if err != nil {
return nil, err
}
readers = append(readers, resp.Body)

if resp.StatusCode != 200 {
return nil, fmt.Errorf("Failed to download Cloudflare ranges: status code %s", resp.Status)
}
}
body, err := ioutil.ReadAll(resp.Body)

body, err := ioutil.ReadAll(io.MultiReader(readers...))
if err != nil {
return nil, err
}
resp.Body.Close()

return bytes.TrimSpace(body), nil
}
11 changes: 10 additions & 1 deletion cmd/ipcat/main.go
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@ import (
"flag"
"fmt"
"log"
"net"
"os"
"strings"

@@ -16,6 +17,7 @@ func main() {
updateAzure := flag.Bool("azure", false, "update Azure records")
updateAppEngine := flag.Bool("appengine", false, "update AppEngine (Google App Engine) records")
updateCloudflare := flag.Bool("cloudflare", false, "update Cloudflare records")
updateTor := flag.Bool("tor", false, "update Tor records")
datafile := flag.String("csvfile", "datacenters.csv", "read/write from this file")
statsfile := flag.String("statsfile", "datacenters-stats.csv", "write statistics to this file")
addCIDR := flag.String("addcidr", "", "add this CIDR range to the data file [CIDR,name,url]")
@@ -41,7 +43,7 @@ func main() {
if rec == nil {
log.Fatalf("Not found: %s", *lookup)
}
fmt.Printf("[%s:%s] %s %s\n", rec.LeftDots, rec.RightDots, rec.Name, rec.URL)
fmt.Printf("[%s:%s] %s %s\n", net.IP(rec.Left[:]), net.IP(rec.Right[:]), rec.Name, rec.URL)
return
}

@@ -89,6 +91,13 @@ func main() {
}
}

if *updateTor {
err = ipcat.UpdateTor(&set)
if err != nil {
log.Fatalf("Unable to update Tor IP ranges: %s", err)
}
}

if *addCIDR != "" {
t := strings.Split(*addCIDR, ",")
if len(t) != 3 {
6 changes: 3 additions & 3 deletions datacenters-stats.csv
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
Datacenter Name, Total IPs
Amazon AWS,29459456
Amazon AWS,1542202570030687795110271325976
Cloudflare Inc,1109194275199700726309617091584
Google App Engine,9903520314283042199196141312
Microsoft Azure,13024127
Akamai,8145728
Google App Engine,2431744
SoftLayer,1903104
Cloudflare Inc,1786880
"ThePlanet.com Internet Services, Inc.",1638656
PEER 1,1462288
Ubiquity Server Solutions,1441792
Loading