/
main.go
77 lines (63 loc) · 3.19 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
package main
import (
"flag"
"fmt"
"os"
"path/filepath"
"github.com/ooni/probe-engine/pkg/minipipeline"
"github.com/ooni/probe-engine/pkg/must"
"github.com/ooni/probe-engine/pkg/runtimex"
)
var (
// destdirFlag is the -destdir flag
destdirFlag = flag.String("destdir", ".", "destination directory to use")
// helpFlag is the -help flag
helpFlag = flag.Bool("help", false, "Show the help message")
// measurementFlag is the -measurement flag
measurementFlag = flag.String("measurement", "", "measurement file to analyze")
// mustWriteFileLn allows overwriting must.WriteFile in tests
mustWriteFileFn = must.WriteFile
// prefixFlag is the -prefix flag
prefixFlag = flag.String("prefix", "", "prefix to add to generated files")
// osExitFn allows overwriting os.Exit in tests
osExitFn = os.Exit
)
func main() {
flag.Parse()
if *helpFlag || *measurementFlag == "" {
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "usage: %s -measurement <file> [-destdir <dir>] [-prefix <prefix>]\n", filepath.Base(os.Args[0]))
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "Mini measurement processing pipeline to reprocess recent probe measurements\n")
fmt.Fprintf(os.Stderr, "and align results calculation with ooni/data.\n")
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "Analyzes the <file> provided using -measurement <file> and writes the\n")
fmt.Fprintf(os.Stderr, "observations.json and analysis.json files in the -destdir <dir> directory,\n")
fmt.Fprintf(os.Stderr, "which must already exist. Additionally, we also perform a \"classic\"\n")
fmt.Fprintf(os.Stderr, "analysis like the one in Web Connectivity v0.4 and generate accordingly the\n")
fmt.Fprintf(os.Stderr, "observations_classic.json and analysis_classic.json files.\n")
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "Use -prefix <prefix> to add <prefix> in front of the generated files names.\n")
fmt.Fprintf(os.Stderr, "\n")
osExitFn(1)
}
// parse the measurement file
var parsed minipipeline.WebMeasurement
must.UnmarshalJSON(must.ReadFile(*measurementFlag), &parsed)
// generate and write observations
observationsPath := filepath.Join(*destdirFlag, *prefixFlag+"observations.json")
container := runtimex.Try1(minipipeline.IngestWebMeasurement(&parsed))
mustWriteFileFn(observationsPath, must.MarshalAndIndentJSON(container, "", " "), 0600)
// generate and write classic observations
classicObservationsPath := filepath.Join(*destdirFlag, *prefixFlag+"observations_classic.json")
containerClassic := minipipeline.ClassicFilter(container)
mustWriteFileFn(classicObservationsPath, must.MarshalAndIndentJSON(containerClassic, "", " "), 0600)
// generate and write observations analysis
analysisPath := filepath.Join(*destdirFlag, *prefixFlag+"analysis.json")
analysis := minipipeline.AnalyzeWebObservations(container)
mustWriteFileFn(analysisPath, must.MarshalAndIndentJSON(analysis, "", " "), 0600)
// generate and write the classic analysis
classicAnalysisPath := filepath.Join(*destdirFlag, *prefixFlag+"analysis_classic.json")
analysisClassic := minipipeline.AnalyzeWebObservations(containerClassic)
mustWriteFileFn(classicAnalysisPath, must.MarshalAndIndentJSON(analysisClassic, "", " "), 0600)
}