From 473f93f08bacc295d672e2eeff3b467f94cd1a1d Mon Sep 17 00:00:00 2001
From: Robert Butts <robert.o.butts@gmail.com>
Date: Wed, 1 Mar 2017 11:22:07 -0700
Subject: [PATCH 1/3] Add TM2 offline validator utility pkg and service

---
 .../traffic_monitor/tmcheck/tmcheck.go        | 172 ++++++++++++++++++
 .../traffic_monitor/tools/validate-offline.go | 116 ++++++++++++
 2 files changed, 288 insertions(+)
 create mode 100644 traffic_monitor_golang/traffic_monitor/tmcheck/tmcheck.go
 create mode 100644 traffic_monitor_golang/traffic_monitor/tools/validate-offline.go

diff --git a/traffic_monitor_golang/traffic_monitor/tmcheck/tmcheck.go b/traffic_monitor_golang/traffic_monitor/tmcheck/tmcheck.go
new file mode 100644
index 0000000000..d6a964ec58
--- /dev/null
+++ b/traffic_monitor_golang/traffic_monitor/tmcheck/tmcheck.go
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// package tmcheck contains utility functions for validating a Traffic Monitor is acting correctly.
+package tmcheck
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/crconfig"
+	"github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/enum"
+	"github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/peer"
+	to "github.com/apache/incubator-trafficcontrol/traffic_ops/client"
+)
+
+const RequestTimeout = time.Second * time.Duration(30)
+
+const TrafficMonitorCRStatesPath = "/publish/CrStates"
+const TrafficMonitorConfigDocPath = "/publish/ConfigDoc"
+
+func getClient() *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
+		Timeout:   RequestTimeout,
+	}
+}
+
+// TrafficMonitorConfigDoc represents the JSON returned by Traffic Monitor's ConfigDoc endpoint. This currently only contains the CDN member, as needed by this library.
+type TrafficMonitorConfigDoc struct {
+	CDN string `json:"cdnName"`
+}
+
+// GetCDN gets the CDN of the given Traffic Monitor.
+func GetCDN(uri string) (string, error) {
+	resp, err := getClient().Get(uri + TrafficMonitorConfigDocPath)
+	if err != nil {
+		return "", fmt.Errorf("reading reply from %v: %v\n", uri, err)
+	}
+	respBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("reading reply from %v: %v\n", uri, err)
+	}
+
+	configDoc := TrafficMonitorConfigDoc{}
+	if err := json.Unmarshal(respBytes, &configDoc); err != nil {
+		return "", fmt.Errorf("unmarshalling: %v", err)
+	}
+	return configDoc.CDN, nil
+}
+
+// GetCRStates gets the CRStates from the given Traffic Monitor.
+func GetCRStates(uri string) (*peer.Crstates, error) {
+	resp, err := getClient().Get(uri)
+	if err != nil {
+		return nil, fmt.Errorf("reading reply from %v: %v\n", uri, err)
+	}
+	respBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("reading reply from %v: %v\n", uri, err)
+	}
+
+	states := peer.Crstates{}
+	if err := json.Unmarshal(respBytes, &states); err != nil {
+		return nil, fmt.Errorf("unmarshalling: %v", err)
+	}
+	return &states, nil
+}
+
+// ValidateOfflineStates validates that no OFFLINE or ADMIN_DOWN caches in the given Traffic Ops' CRConfig are marked Available in the given Traffic Monitor's CRStates.
+func ValidateOfflineStates(tmURI string, toClient *to.Session) error {
+	cdn, err := GetCDN(tmURI)
+	if err != nil {
+		return fmt.Errorf("getting CDN from Traffic Monitor: %v", err)
+	}
+
+	crConfigBytes, err := toClient.CRConfigRaw(cdn)
+	if err != nil {
+		return fmt.Errorf("getting CRConfig: %v", err)
+	}
+
+	crConfig := crconfig.CRConfig{}
+	if err := json.Unmarshal(crConfigBytes, &crConfig); err != nil {
+		return fmt.Errorf("unmarshalling CRConfig JSON: %v", err)
+	}
+
+	crStates, err := GetCRStates(tmURI + TrafficMonitorCRStatesPath)
+	if err != nil {
+		return fmt.Errorf("getting CRStates: %v", err)
+	}
+
+	return ValidateCRStates(crStates, &crConfig)
+}
+
+// ValidateCRStates validates that no OFFLINE or ADMIN_DOWN caches in the given CRConfig are marked Available in the given CRStates.
+func ValidateCRStates(crstates *peer.Crstates, crconfig *crconfig.CRConfig) error {
+	for cacheName, cacheInfo := range crconfig.ContentServers {
+		status := enum.CacheStatusFromString(string(*cacheInfo.Status))
+		if status != enum.CacheStatusOffline || status != enum.CacheStatusOffline {
+			continue
+		}
+
+		available, ok := crstates.Caches[enum.CacheName(cacheName)]
+		if !ok {
+			return fmt.Errorf("Cache %v in CRConfig but not CRStates", cacheName)
+		}
+
+		if available.IsAvailable {
+			return fmt.Errorf("Cache %v is %v in CRConfig, but available in CRStates", cacheName, status)
+		}
+
+	}
+	return nil
+}
+
+// Validator is designed to be run as a goroutine, and does not return. It continously validates every `interval`, and calls `onErr` on failure, `onResumeSuccess` when a failure ceases, and `onCheck` on every poll.
+func Validator(
+	tmURI string,
+	toClient *to.Session,
+	interval time.Duration,
+	grace time.Duration,
+	onErr func(error),
+	onResumeSuccess func(),
+	onCheck func(error),
+) {
+	invalid := false
+	invalidStart := time.Time{}
+	for {
+		err := ValidateOfflineStates(tmURI, toClient)
+
+		if err != nil && !invalid {
+			invalid = true
+			invalidStart = time.Now()
+		}
+
+		if err != nil {
+			invalidSpan := time.Now().Sub(invalidStart)
+			if invalidSpan > grace {
+				onErr(fmt.Errorf("invalid state for %v: %v\n", invalidSpan, err))
+			}
+		}
+
+		if err == nil && invalid {
+			onResumeSuccess()
+			invalid = false
+		}
+
+		onCheck(err)
+
+		time.Sleep(interval)
+	}
+}
diff --git a/traffic_monitor_golang/traffic_monitor/tools/validate-offline.go b/traffic_monitor_golang/traffic_monitor/tools/validate-offline.go
new file mode 100644
index 0000000000..30574a0c90
--- /dev/null
+++ b/traffic_monitor_golang/traffic_monitor/tools/validate-offline.go
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// validate-offline is a utility HTTP service which polls the given Traffic Monitor and validates that no OFFLINE or ADMIN_DOWN caches in the Traffic Ops CRConfig are marked Available in Traffic Monitor's CRstates endpoint.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/tmcheck"
+	to "github.com/apache/incubator-trafficcontrol/traffic_ops/client"
+	"net/http"
+	"sync"
+	"time"
+)
+
+const UserAgent = "tm-offline-validator/0.1"
+
+type Log struct {
+	l *[]string
+	m *sync.RWMutex
+}
+
+func (l *Log) Add(msg string) {
+	l.m.Lock()
+	defer l.m.Unlock()
+	*l.l = append(*l.l, msg)
+}
+
+func (l *Log) Get() []string {
+	l.m.RLock()
+	defer l.m.RUnlock()
+	return *l.l
+}
+
+func NewLog() Log {
+	s := make([]string, 0)
+	return Log{l: &s, m: &sync.RWMutex{}}
+}
+
+func main() {
+	toURI := flag.String("to", "", "The Traffic Ops URI, whose CRConfig to validate")
+	toUser := flag.String("touser", "", "The Traffic Ops user")
+	toPass := flag.String("topass", "", "The Traffic Ops password")
+	tmURI := flag.String("tm", "", "The Traffic Monitor URI whose CRStates to validate")
+	interval := flag.Duration("interval", time.Second*time.Duration(5), "The interval to validate")
+	grace := flag.Duration("grace", time.Second*time.Duration(30), "The grace period before invalid states are reported")
+	help := flag.Bool("help", false, "Usage info")
+	helpBrief := flag.Bool("h", false, "Usage info")
+	flag.Parse()
+	if *help || *helpBrief {
+		fmt.Printf("Usage: go run validate-offline -to https://traffic-ops.example.net -touser bill -topass thelizard -tm http://traffic-monitor.example.net -interval 5s -grace 30s\n")
+		return
+	}
+
+	toClient, err := to.LoginWithAgent(*toURI, *toUser, *toPass, true, UserAgent, false, tmcheck.RequestTimeout)
+	if err != nil {
+		fmt.Printf("Error logging in to Traffic Ops: %v\n", err)
+		return
+	}
+
+	log := NewLog()
+
+	onErr := func(err error) {
+		log.Add(fmt.Sprintf("%v ERROR %v\n", time.Now(), err))
+	}
+
+	onResumeSuccess := func() {
+		log.Add(fmt.Sprintf("%v INFO State Valid\n", time.Now()))
+	}
+
+	onCheck := func(err error) {
+		if err != nil {
+			log.Add(fmt.Sprintf("%v DEBUG invalid: %v\n", time.Now(), err))
+		} else {
+			log.Add(fmt.Sprintf("%v DEBUG valid\n", time.Now()))
+		}
+	}
+
+	go tmcheck.Validator(*tmURI, toClient, *interval, *grace, onErr, onResumeSuccess, onCheck)
+
+	if err := serve(log); err != nil {
+		fmt.Printf("Serve error: %v\n", err)
+	}
+}
+
+func serve(log Log) error {
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprintf(w, `<html><head><meta http-equiv="refresh" content="5"></head><body><pre>`)
+		logCopy := log.Get()
+		for i := len(logCopy) - 1; i >= 0; i-- {
+			fmt.Fprintf(w, "%s\n", logCopy[i])
+		}
+		fmt.Fprintf(w, `</pre></body></html>`)
+	})
+	return http.ListenAndServe(":80", nil)
+}

From cc29fab471f03cac7cf9afce50891f7cd7e00097 Mon Sep 17 00:00:00 2001
From: Robert Butts <robert.o.butts@gmail.com>
Date: Wed, 1 Mar 2017 16:42:58 -0700
Subject: [PATCH 2/3] Add TM2 offline-validator log limit, html format

---
 .../traffic_monitor/tools/validate-offline.go | 59 +++++++++++++++----
 1 file changed, 49 insertions(+), 10 deletions(-)

diff --git a/traffic_monitor_golang/traffic_monitor/tools/validate-offline.go b/traffic_monitor_golang/traffic_monitor/tools/validate-offline.go
index 30574a0c90..92b17c7814 100644
--- a/traffic_monitor_golang/traffic_monitor/tools/validate-offline.go
+++ b/traffic_monitor_golang/traffic_monitor/tools/validate-offline.go
@@ -33,26 +33,47 @@ import (
 
 const UserAgent = "tm-offline-validator/0.1"
 
+const LogLimit = 100000
+
 type Log struct {
-	l *[]string
-	m *sync.RWMutex
+	log     *[]string
+	limit   int
+	errored *bool
+	m       *sync.RWMutex
 }
 
 func (l *Log) Add(msg string) {
 	l.m.Lock()
 	defer l.m.Unlock()
-	*l.l = append(*l.l, msg)
+	*l.log = append([]string{msg}, *l.log...)
+	if len(*l.log) > l.limit {
+		*l.log = (*l.log)[:l.limit]
+	}
 }
 
 func (l *Log) Get() []string {
 	l.m.RLock()
 	defer l.m.RUnlock()
-	return *l.l
+	return *l.log
+}
+
+func (l *Log) GetErrored() bool {
+	l.m.RLock()
+	defer l.m.RUnlock()
+	return *l.errored
+}
+
+func (l *Log) SetErrored(e bool) {
+	l.m.Lock()
+	defer l.m.Unlock()
+	*l.errored = e
 }
 
 func NewLog() Log {
-	s := make([]string, 0)
-	return Log{l: &s, m: &sync.RWMutex{}}
+	log := make([]string, 0, LogLimit+1)
+	errored := false
+	limit := LogLimit
+	return Log{log: &log, errored: &errored, m: &sync.RWMutex{}, limit: limit}
 }
 
 func main() {
@@ -80,10 +101,12 @@ func main() {
 
 	onErr := func(err error) {
 		log.Add(fmt.Sprintf("%v ERROR %v\n", time.Now(), err))
+		log.SetErrored(true)
 	}
 
 	onResumeSuccess := func() {
 		log.Add(fmt.Sprintf("%v INFO State Valid\n", time.Now()))
+		log.SetErrored(false)
 	}
 
 	onCheck := func(err error) {
@@ -96,21 +119,37 @@ func main() {
 
 	go tmcheck.Validator(*tmURI, toClient, *interval, *grace, onErr, onResumeSuccess, onCheck)
 
-	if err := serve(log); err != nil {
+	if err := serve(log, *toURI, *tmURI); err != nil {
 		fmt.Printf("Serve error: %v\n", err)
 	}
 }
 
-func serve(log Log) error {
+func serve(log Log, toURI string, tmURI string) error {
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Access-Control-Allow-Origin", "*")
 		w.Header().Set("Content-Type", "text/html")
-		fmt.Fprintf(w, `<html><head><meta http-equiv="refresh" content="5"></head><body><pre>`)
+		fmt.Fprintf(w, `<!DOCTYPE html>
+<meta http-equiv="refresh" content="5">
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Traffic Monitor Offline Validator</title>
+<style type="text/css">body{margin:40px auto;max-width:650px;line-height:1.6;font-size:18px;color:#444;padding:0 10px}h1,h2,h3{line-height:1.2}</style>`)
+
+		fmt.Fprintf(w, `<p>%s`, toURI)
+		fmt.Fprintf(w, `<p>%s`, tmURI)
+		if log.GetErrored() {
+			fmt.Fprintf(w, `<h1 style="color:red">Invalid</h1>`)
+		} else {
+			fmt.Fprintf(w, `<h1 style="color:limegreen">Valid</h1>`)
+		}
+
+		fmt.Fprintf(w, `<pre>`)
 		logCopy := log.Get()
 		for i := len(logCopy) - 1; i >= 0; i-- {
 			fmt.Fprintf(w, "%s\n", logCopy[i])
 		}
-		fmt.Fprintf(w, `</pre></body></html>`)
+		fmt.Fprintf(w, `</pre>`)
+
 	})
 	return http.ListenAndServe(":80", nil)
 }

From ec6a74faa287af76c9d873f82fe22477abf2d4d3 Mon Sep 17 00:00:00 2001
From: Robert Butts <robert.o.butts@gmail.com>
Date: Wed, 1 Mar 2017 21:02:47 -0700
Subject: [PATCH 3/3] Fix TM2 datareq missing license headers

---
 .../traffic_monitor/datareq/bandwidth.go      | 19 +++++++++++++++++++
 .../datareq/bandwidthcapacity.go              | 19 +++++++++++++++++++
 .../datareq/cacheavailablecount.go            | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/cachecount.go     | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/cachedowncount.go | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/cachestat.go      | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/cachestate.go     | 19 +++++++++++++++++++
 .../datareq/cachestatfilter.go                | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/configdoc.go      | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/crconfig.go       | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/crstate.go        | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/datareq.go        |  4 ++--
 .../traffic_monitor/datareq/dsstat.go         | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/dsstatfilter.go   | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/eventlog.go       | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/monitorconfig.go  | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/peerstate.go      | 19 +++++++++++++++++++
 .../datareq/peerstatefilter.go                | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/stat.go           | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/statsummary.go    | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/trafficopsuri.go  | 19 +++++++++++++++++++
 .../traffic_monitor/datareq/version.go        | 19 +++++++++++++++++++
 22 files changed, 401 insertions(+), 2 deletions(-)

diff --git a/traffic_monitor_golang/traffic_monitor/datareq/bandwidth.go b/traffic_monitor_golang/traffic_monitor/datareq/bandwidth.go
index 3306004d70..7c638892e1 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/bandwidth.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/bandwidth.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/bandwidthcapacity.go b/traffic_monitor_golang/traffic_monitor/datareq/bandwidthcapacity.go
index c044b29b0b..a5dbec737d 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/bandwidthcapacity.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/bandwidthcapacity.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/cacheavailablecount.go b/traffic_monitor_golang/traffic_monitor/datareq/cacheavailablecount.go
index be838b57bf..46e27cb1e3 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/cacheavailablecount.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/cacheavailablecount.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/cachecount.go b/traffic_monitor_golang/traffic_monitor/datareq/cachecount.go
index c25b791f76..bbe7607577 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/cachecount.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/cachecount.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/cachedowncount.go b/traffic_monitor_golang/traffic_monitor/datareq/cachedowncount.go
index 321f7249ea..211c755fa2 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/cachedowncount.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/cachedowncount.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/cachestat.go b/traffic_monitor_golang/traffic_monitor/datareq/cachestat.go
index fbcd36993d..ca03f12646 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/cachestat.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/cachestat.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/cachestate.go b/traffic_monitor_golang/traffic_monitor/datareq/cachestate.go
index 871be49799..750305fb34 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/cachestate.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/cachestate.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/cachestatfilter.go b/traffic_monitor_golang/traffic_monitor/datareq/cachestatfilter.go
index 6e9a76409b..95f4770782 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/cachestatfilter.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/cachestatfilter.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/configdoc.go b/traffic_monitor_golang/traffic_monitor/datareq/configdoc.go
index 0501b9fd00..56988ada82 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/configdoc.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/configdoc.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/crconfig.go b/traffic_monitor_golang/traffic_monitor/datareq/crconfig.go
index e8cbaac2a7..89adb308c9 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/crconfig.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/crconfig.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/crstate.go b/traffic_monitor_golang/traffic_monitor/datareq/crstate.go
index f672207eba..d87d31fbbb 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/crstate.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/crstate.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/datareq.go b/traffic_monitor_golang/traffic_monitor/datareq/datareq.go
index d2919bb3ff..9f8e33bced 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/datareq.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/datareq.go
@@ -1,5 +1,3 @@
-package datareq
-
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -19,6 +17,8 @@ package datareq
  * under the License.
  */
 
+package datareq
+
 import (
 	"fmt"
 	"net/http"
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/dsstat.go b/traffic_monitor_golang/traffic_monitor/datareq/dsstat.go
index e2a0bd4974..7c285cddff 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/dsstat.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/dsstat.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/dsstatfilter.go b/traffic_monitor_golang/traffic_monitor/datareq/dsstatfilter.go
index a29ea3a0fb..13c08715b8 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/dsstatfilter.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/dsstatfilter.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/eventlog.go b/traffic_monitor_golang/traffic_monitor/datareq/eventlog.go
index db50690824..400bf1c0fe 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/eventlog.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/eventlog.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/monitorconfig.go b/traffic_monitor_golang/traffic_monitor/datareq/monitorconfig.go
index 15000ce0ce..53281d1ccd 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/monitorconfig.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/monitorconfig.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/peerstate.go b/traffic_monitor_golang/traffic_monitor/datareq/peerstate.go
index b1040a85c7..45ba5afd45 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/peerstate.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/peerstate.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/peerstatefilter.go b/traffic_monitor_golang/traffic_monitor/datareq/peerstatefilter.go
index 65a114d694..f256a4d618 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/peerstatefilter.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/peerstatefilter.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/stat.go b/traffic_monitor_golang/traffic_monitor/datareq/stat.go
index 16f93aebb7..67f6cc821f 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/stat.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/stat.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/statsummary.go b/traffic_monitor_golang/traffic_monitor/datareq/statsummary.go
index 85f2cbddcd..ea55a16ed0 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/statsummary.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/statsummary.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/trafficopsuri.go b/traffic_monitor_golang/traffic_monitor/datareq/trafficopsuri.go
index 6c2abeeceb..5f46661691 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/trafficopsuri.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/trafficopsuri.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/version.go b/traffic_monitor_golang/traffic_monitor/datareq/version.go
index 2e7840bf6d..b55cbb289d 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/version.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/version.go
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package datareq
 
 import (