-
Notifications
You must be signed in to change notification settings - Fork 10
/
sp-rest-api-tutorial.txt
3685 lines (3328 loc) · 157 KB
/
sp-rest-api-tutorial.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#+OPTIONS: date:t H:3 ^:{} author:t email:nil num:t title:t makeindex:t
#+OPTIONS: toc:t
#+MACRO: NEWLINE @@latex:\\@@ @@html:<br>@@
#+MACRO: IMAGE @@latex:\\ \vspace{8cm} \hfill \usebox{\titleimage}@@
#+TITLE: Sightline REST API Cookbook {{{NEWLINE}}} Sightline v9.0 {{{NEWLINE}}} API v5
#+DATE: <2018-10-10 Wed>
#+AUTHOR: NETSCOUT|Arbor Networks
#+LATEX_CLASS: tufte-book
# #+LATEX_CLASS: memoir
#+LaTeX_CLASS_OPTIONS: [koma,utopia,12pt,symmetric,notoc]
#+LATEX_HEADER: \usepackage{xcolor}
#+LATEX_HEADER: \usepackage{minted}
#+LATEX_HEADER: \usepackage{makeidx}
#+LATEX_HEADER: \usepackage{grffile}
#+LATEX_HEADER: \setcounter{tocdepth}{1}
#+LATEX_HEADER: \makeindex
#+LATEX_HEADER: \newsavebox{\titleimage}
#+LATEX_HEADER: \savebox{\titleimage}{\includegraphics[height=7\baselineskip]{images/ArborLogo.png}}
#+BEGIN_COMMENT
To use the minted package, you should also add these lines to your
.emacs file:
-------
(setq org-latex-minted-options '(
("frame" "lines")
("fontsize" "\\scriptsize")
("xleftmargin" "\\parindent")
("linenos" "")
("breaklines" "")
("breakafter" "+")
))
-------
if that is untenable to you, comment out the \usepackage{minted} line
above and uncomment the following LATEX_HEADER lines to switch to the
listings package.
#+END_COMMENT
# #+LATEX_HEADER: \usepackage{listings}
# #+LATEX_HEADER: \lstset{
# #+LATEX_HEADER: frame=single,
# #+LATEX_HEADER: breaklines=true,
# #+LATEX_HEADER: showstringspaces=false,
# #+LATEX_HEADER: postbreak=\raisebox{0ex}[0ex][0ex]{\ensuremath{\color{red}\hookrightarrow\space}}
# #+LATEX_HEADER: }
#+LATEX: \renewcommand \plaintitle{Sightline REST API Cookbook, Sightline v9.0, API v5} % keep the image out of the headers
#+LATEX: \renewcommand{\ref}{\nameref}
* Introduction
NETSCOUT Sightline (formerly Arbor Networks SP) is a very
sophisticated piece of software that gives visibility into the
network traffic on networks ranging up to those that are massively
large; Sightline supports hundreds of configuration parameters for
defining what exactly visibility means and the reported data from
Sightline is very rich and very broad. Almost no network operators
who use Sightline make use of the entire range of its capabilities.
The configuration of Sightline and the management of its data is
done in two ways: the Sightline web-based UI and the web-based API
interfaces to Sightline. While Sightline (and SP before it) has had
APIs for many versions, the versions starting with SP 8.1 have had a
REST API as one of the options for configuration and data retrieval.
This book describes how to programmatically configure Sightline and
get data about your network from Sightline using a json:api-based
REST API to Sightline.
#+INDEX: REST
#+INDEX: SOAP API
REST is a common, HTTP-based protocol that defines web-based
services that allow clients (sometimes called "requesting systems")
to access and manipulate textual representations (in the case of
Sightline, JSON representations) of Sightline configuration and
reporting data (generically called "Web resources") using a uniform
and predefined set of stateless, from the client's perspective,
operations. All of the state, configuration and data, is maintained
within Sightline, not by the API client. The API described here is
the Sightline REST API. Sightline has other APIs (SOAP and
WebServices) that are documented in other Sightline documentation.
The intent of the Sightline REST API is to, over time, include all
of the functionality of the SOAP and WebServices API, at which point
those will be deprecated.
#+INDEX: jsonapi
The Sightline REST API uses a particular API-focused JavaScript
Object Notation (JSON) format to represent its data. JSON is a
textual representation of data in collections (Python calls these
dictionaries, C calls them structures, Go calls them maps, Ruby and
Perl call them hashes) and lists (Python and Perl also calls them
lists, Go calls them slices, C and Ruby call them arrays). The
elements in collections and lists are other collections, other
lists, Boolean values, numeric values, or string values as described
by the JSON specification (http://www.json.org/). The particular
JSON representation that the Sightline REST API adheres to is called
json:api (http://jsonapi.org/); the rules and guidelines described
in Version 1.0 of the json:api specification are implemented for the
Sightline REST API.
The Sightline REST API is under development. Each release of Arbor
Networks SP or Sightline after SP version 8.0 includes a REST API
that has more features than the prior release. Arbor Networks
expects that the REST API will not be complete for several years,
but incremental steps will be included in each version of Sightline.
This book describes Version 5 of the API that is included in
Sightline 9.0. (The versioning mechanism is described in more
detail in the section [[Guiding Principles of the Sightline REST API]].)
This book is intended for the Sightline administrator or advanced
user who has some programming or shell scripting experience and who
wants to automate network management or reporting tasks, or combine
Sightline data with other sources of data for reporting,
configuration, or decision making.
** Conventions Used in this Book
Things in a fixed-width font are either input or output, for the
most part.
See the section [[Reporting Errors to Arbor]] for advice on how to
provide feedback.
This is a collaborative document, feedback, additions, edits,
changes, modifications, etc. are all greatly appreciated. Please
see the section [[Contributing and other technical details]] for advice
on ways to contribute.
The examples that were written for a version of Sightline and its
REST API will report their version in both their =meta= object and
in the URLs; those examples are not generally reproduced for the
latest version, but also should continue to work even on the latest
version (as well as on the prior version of the API in newer
versions of Sightline. Put another way, if you see an example that
uses a URL with a specific =/v[VersionNumber]/= in it or example
output that includes a =meta= object like
#+BEGIN_EXAMPLE
"meta": {
"api": "SP",
"api_version": "3",
"sp_build_id": "HGRD",
"sp_version": "8.3"
}
#+END_EXAMPLE
you can, in probably all cases, use the latest version of Sightline
and the REST API safely. Alternatively, you can use the latest
version of Sightline and the specified version of the REST
API.[fn:8]
** Sightline APIs
#+INDEX: SOAP API
NETSCOUT Sightline has three officially supported types of APIs:
the SOAP API, the WebServices API, and the REST API.
The SOAP and WebServices API are documented in the online Sightline
documentation and in the API software development kit available
from the Sightline web UI at the Administration > Download Arbor
API SDK menu item.
Arbor Networks has decided to focus on the REST API (the topic of
this book) for its future development with the goal of encompassing
the functionality of the other APIs, the Sightline command-line
interface, and all of the functionality provided by the web-based
UI that is the most familiar to Sightline users.
As you develop clients for Sightline's APIs, keep in mind that the
REST API is the officially preferred API for Sightline. However,
in this version (and, likely, several upcoming versions), the REST
API does not provide all of the functionality of the SOAP and
WebServices APIs. Arbor recommends using the REST API where you
can in your clients, and augmenting it with API requests to the
other APIs, replacing those requests as the REST API gains
functionality.
** Guiding Principles of the Sightline REST API
As Arbor Networks develops the REST API, we abide by some standards
and principles that you can rely on as you you create API clients.
*** JSON API
#+INDEX: jsonapi
The first of these guidelines is that the structure of the JSON
that is used for input and output from the API follows the JSON
API specification ([[http://jsonapi.org/]]). While you don't need to
understand the specification to use the Sightline REST API, it may
help answer some questions about why things are the way they are
in the API, and how Arbor is likely to handle future additions to
the API.
*** Useful Errors for Humans and Computers
#+INDEX: errors
The second guideline is that the Sightline REST API should provide
you with useful error messages that can be interpreted by both a
human and a computer program. Arbor makes use of a combination of
HTTP response codes as described in Section 10 of RFC 2616
(https://www.ietf.org/rfc/rfc2616.txt) and error objects that
contain some or all of an error title string, an error detail
string, an HTTP status code string, and a pointer to the error in
the input. An example error object that resulted from the REST
client requesting a nonexistent endpoint is:
#+NAME: errorExample
#+CAPTION: Error Example
#+BEGIN_SRC json
{
"detail": "Resource https://leader.example.com/api/sp/asdf could not be found.",
"meta": {
"api": "SP",
"api_version": "2",
"sp_build_id": "HGRD",
"sp_version": "8.3"
},
"status": "404",
"title": "Missing resource error."
}
#+END_SRC
This error object contains the error detail, status code, and
title along with the standard ="meta"= section which gives
information about the API itself. Error objects are further
described in the JSON API specification in the /Errors/ section
(http://jsonapi.org/format/#errors).
*** Backward compatibility
Another guideline is that the REST API will maintain
backward-compatibility as it increases in version. The version
for the API is a combination of the Sightline version and the API
version number; in the example above the API version would be
reported as "SP v8.3, API v2". It is possible that there will be
an API v2 for SP v8.2.3 that is different from the API v2 for SP
v8.3. NETSCOUT|Arbor will retain at least one prior version of
the API in each version of SP so if you have written clients that
use v3 of the API in SP v8.4 and upgrade to Sightline v9.0 that
defaults API v4, you can still access API v3 in Sightline v9.0
with your clients while you work to migrate them to the latest
version. API version increases will happen when Arbor Networks
introduces a change in the API that could potentially break
existing clients; this includes things like changing the name of
an API key, changing the type of an API value, or removing an API
endpoint. Adding keys and endpoints is not considered breaking,
as well-written API clients should ignore extra data returned to
them.
*** Discoverability
#+INDEX: HATEOAS
#+INDEX: Fielding, Roy
Arbor Networks makes every effort to ensure the Sightline REST API
is fully discoverable from the root URL, =/api/sp/=. Starting
there, every response you receive may contain an object called
=relationships= that will lead you to the related objects. There
are no detached nodes of the API tree. This is called Hypermedia
as the Engine of Application State (HATEOAS) and is defined[fn:1]
by Roy T. Fielding (the inventor of REST) as:
#+BEGIN_QUOTE
A REST API should be entered with no prior knowledge beyond the
initial URI (bookmark) and set of standardized media types that
are appropriate for the intended audience (i.e., expected to be
understood by any client that might use the API). From that point
on, all application state transitions must be driven by client
selection of server-provided choices that are present in the
received representations or implied by the user’s manipulation of
those representations. The transitions may be determined (or
limited by) the client’s knowledge of media types and resource
communication mechanisms, both of which may be improved on-the-fly
(e.g., code-on-demand).
#+END_QUOTE
In the case of the Sightline REST API, the "initial URI" is
https://sightline-leader.example.com/api/sp/ and following that
the elements in the =links= objects provide connectivity to the
rest of the API.
** Online Reference Manual
The Sightline REST API includes a reference manual for each
endpoint with a description of each key in that endpoint; there are
very few examples of using the API in the reference manual (hence
this book), but all of the endpoints and their properties are
listed for all of the versions of the API supported on the
installed version of Sightline.
In addition, changes between versions are in the Deprecation Policy
section of the reference manual.
This book will often make reference to the online reference manual;
having access to that manual will be helpful as you develop
clients. The online reference manual is available from the Arbor
Sightline UI by selecting the "REST API Documentation" from the
"Administration" menu. A PDF copy of this is available from your
local engineer or from the Arbor Technical Assistance Center.
* Set-up
** Permissions, Access Tokens, and SSL Certificates
*** Access Tokens
#+INDEX: token
Sightline REST API tokens are created from the Sightline command
line interface with the command:
#+BEGIN_SRC sh
/ services aaa local apitoken generate admin "comment"
#+END_SRC
where =admin= is the username of the administrative user and
="comment"= is a useful comment for you. Arbor recommends the
local name of the user and the date the token is created, for
example:
#+BEGIN_SRC sh
/ services aaa local apitoken generate admin "fred@example.com 2017-07-01"
#+END_SRC
You can list the tokens that have been created with the command:
#+BEGIN_SRC sh :exports code
/ services aaa local apitoken show
#+END_SRC
which will result in a list of API tokens and their associated
comments, something like this:
#+LATEX: \scriptsize
#+BEGIN_EXAMPLE
admin:
wz8SmDeJRSkz_0kNbdSajSQ_Jk82EVOqRU6CPU_O susy@example.com - 2017-06-01
YK3va5ATpXdmTWwfgzZPBckrj7zue205CzjLBtK5 fred@example.com - 2017-07-01
#+END_EXAMPLE
#+LATEX: \normalsize
To remove the token =YK3va5ATpXdmTWwfgzZPBckrj7zue205CzjLBtK5= type
the command:
#+BEGIN_SRC sh :exports code
/ services aaa local apitoken remove YK3va5ATpXdmTWwfgzZPBckrj7zue205CzjLBtK5
#+END_SRC
*** SSL Certificates
#+INDEX: ssl
All Sightline REST API requests should use the secure HTTP
transport (=https=). This will ensure that your API key and the
contents of requests and responses are encrypted. In addition,
Sightline will attempt to redirect unencrypted HTTP requests to
the encrypted port, but that can affect the use of the POST and
PATCH options.
Sightline comes with its own SSL certificate, and your
organization may also add their own; to verify that your
communications are secure, you will need a copy of that
certificate. The person who maintains your Sightline deployment
should be able to provide that to you; if you have shell access to
the Sightline leader, the file containing the certificates is at
=/etc/ca-bundle.crt=. Copying that file to your API client's
environment will allow you to refer to it. In the examples in
this book that file is referred to at =certfile= in most places.
** Useful Tools
When working with REST APIs there are several common actions that
you will take: getting data from the API, processing retrieved
data, and sending data to the API. There are many free programs
that can help you do this in an interactive way. This can be
useful during development of more complicated programs, for quick
API queries that don't require a whole program, for use in a shell
script, or for interactively working the Sightline REST API.
Examples later in this book will make use of at least some of these
tools.
*** Any web browser
The Sightline API is available via any web browser by simply going
to a URL of the form
=https//sightline-leader.example.com/api/sp/=; this will present a
nicely formatted web page showing the request and the results.
It's not as convenient as other tools (cURL or HTTPie, see the
following section) for actions other than GET, but as a quick test
of an API request, it can be very handy. An example of how it
looks is in Figure [[fig:api-results-web-browser]].
#+CAPTION: The Sightline REST API can be accessed using a web browser
#+CAPTION: and the query and results are displayed nicely.
#+NAME: fig:api-results-web-browser
[[./images/api-results-web-browser.png]]
*** cURL
#+INDEX: curl!command line
cURL is a classic program (dating back to 1997) for accessing HTTP
services from the command line. It is open source and available
for free from the main cURL website http://curl.haxx.se or via
package managers for your platform. cURL is available for 34
different operating systems, so should work in almost all
scenarios. There are more modern and feature-rich tools for
accessing REST APIs (or other HTTP services), but cURL is nearly
ubiquitous.
**** Getting data with cURL
An example of using cURL to get data from Sightline via the
Sightline REST API is:
#+BEGIN_SRC sh :exports code
curl --cacert certfile --ssl -X GET -H "Content-Type:application/vnd.api+json" -H "X-Arbux-APIToken:wz8SmD3JRSKz_0kNbdSejSQ_Jk92EVOqRU6CPU_O" https://leader.example.com/api/sp/
#+END_SRC
In order of appearance, the options used in the =curl= command
above are:
- =--cacert certfile= tells =curl= to use the SSL certificate
called =certfile= (more on this later)
- =--ssl= tells =curl= to use Secure Sockets Layer (SSL) for its
connection
- =-X GET= tells =curl= to issue an HTTP =GET= command (more on
this later, too)
- =-H "Content-Type:application/vnd.api+json"= sends the HTTP
header =Content-Type= set to the value of
=application/vnd.api+json=. Other than the endpoints at
=/insight/=, all of the Sightline API uses this content type
- =-H
"X-Arbux-APIToken:wz8SmD3JRSKz_0kNbdSejSQ_Jk92EVOqRU6CPU_O"=
sends the HTTP header =X-Arbux-APIToken= set to the value of
the API key that was set via the Sightline command line
- =https://leader.example.com/api/sp/= which is the URL
(hostname and endpoint) that is requested. In this case, it is
the index endpoint for the API.
The output of this particular command is the index of the
Sightline REST API represented as a JSON object (defined by the
outer-most curly braces) with two JSON objects in it, =meta= and
=links=.
Successful requests for data from the Sightline REST API will
return the requested data along with the HTTP status code =200
OK=.
#+BEGIN_COMMENT
#+BEGIN_QUOTE
A brief aside:
The =meta= object tells you a little about the version of
Sightline and the API on the Sightline leader you are connecting
to. This is most useful when writing clients that need to work
with more than one version of Sightline and when submitting
questions to the Arbor Technical Assistance Center (ATAC). The
=links= object contains a map between the endpoint type (for
example, =alert=) and the URL where that data is accessible in
this version (in the case of =alert=, that is
=https://leader.example.com/api/sp/v3/alerts/=).
#+INDEX: /alerts/ endpoint
You should try to write your clients using this mapping; the key
names will not change, but the URLs may and if your program uses
the key names, this won't affect you as you upgrade Sightline.
#+END_QUOTE
#+END_COMMENT
**** Sending data with cURL
An example of using cURL to send data to Sightline via the
Sightline REST API is:
#+INDEX: /alerts/ endpoint
#+INDEX: /alerts/ endpoint!annotations
#+BEGIN_SRC sh :results output :exports code
curl --cacert certfile --ssl -X POST -H "Content-Type:application/vnd.api+json" -H "X-Arbux-APIToken:wz8SmD3JRSKz_0kNbdSejSQ_Jk92EVOqRU6CPU_O" -d @input.json https://leader.example.com/api/sp/alerts/12519/annotations/
#+END_SRC
In order of appearance, the options used in the =curl= command
above are:
- =--cacert certfile= tells =curl= to use the SSL certificate
called =certfile= (more on this later)
- =--ssl= tells =curl= to use Secure Sockets Layer (SSL) for its
connection
- =-X POST= tells =curl= to issue an HTTP =POST= command (more
on this later, too)
- =-H "Content-Type:application/vnd.api+json"= sends the HTTP
header =Content-Type= set to the value of
=application/vnd.api+json=. Other than the endpoints at
=/insight/=, all of the Sightline API uses this content type
- =-H
"X-Arbux-APIToken:wz8SmD3JRSKz_0kNbdSejSQ_Jk92EVOqRU6CPU_O"=
sends the HTTP header =X-Arbux-APIToken= set to the value of
the API key that was set via the Sightline command line
- =-d @input.json= tells cURL that the data to be sent can be
found in the file named =input.json=
- =https://leader.example.com/api/sp/alerts/12519/annotations/=
#+INDEX: /alerts/ endpoint!annotations
which is the URL (hostname and endpoint) to which the data
will be sent. In this case, it is an alert annotation for the
alert with id 12519.
Although this will be discussed in more depth later in this book,
the format of the data for an alert annotation (in this example,
the contents of the file =input.json=) is:
#+BEGIN_SRC json
{
"data": {
"attributes": {
"author": "The name of the author",
"text": "The text of the annotation"
}
}
}
#+END_SRC
Using the HTTP verb =PATCH= to change data is very similar to how
the =POST= verb is used.
Successful API =POST= and =PATCH= requests will return all of the
data for the object that was created or modified along with
either a =200 OK= or =201 Created= status code.
**** Deleting data with cURL
An example of using cURL to delete data from Sightline via the
Sightline REST API is:
#+INDEX: /routers/ endpoint
#+BEGIN_SRC sh :results output :exports code
curl --cacert certfile --ssl -X DELETE -H "Content-Type:application/vnd.api+json" -H "X-Arbux-APIToken:wz8SmD3JRSKz_0kNbdSejSQ_Jk92EVOqRU6CPU_O" https://leader.example.com/api/sp/routers/132
#+END_SRC
In order of appearance, the options used in the =curl= command
above are:
- =--cacert certfile= tells =curl= to use the SSL certificate
called =certfile= (more on this later)
- =--ssl= tells =curl= to use Secure Sockets Layer (SSL) for its
connection
- =-X DELETE= tells =curl= to issue an HTTP =DELETE= command
(more on this later, too)
- =-H "Content-Type:application/vnd.api+json"= sends the HTTP
header =Content-Type= set to the value of
=application/vnd.api+json=. Other than the endpoints at
=/insight/=, all of the Sightline API uses this content type
- =-H
"X-Arbux-APIToken:wz8SmD3JRSKz_0kNbdSejSQ_Jk92EVOqRU6CPU_O"=
sends the HTTP header =X-Arbux-APIToken= set to the value of
the API key that was set via the Sightline command line
- =https://leader.example.com/api/sp/routers/132= which is the
#+INDEX: /routers/ endpoint
URL (hostname and endpoint) that identifies what resource in
Sightline will be deleted. In this case, it is the router
with id 132.
API delete requests do not return any data, since the Sightline
object you were acting on is now gone. The HTTP status code that
will be returned for successfully deletions is =204 No Content=.
*** HTTPie
#+INDEX: httpie
HTTPie is a Python program that is much newer than cURL (although
has still seen a lot of development; the initial commit to Github
was on 25 February 2012) and has some conveniences that cURL
doesn't. The biggest convenience, in my opinion, is the ability
to save configuration data for a host, and select which host
configuration to use when the the command is run.
HTTPie is open source and available for free from
https://httpie.org/ or via package managers for your platform.
HTTPie is based on Python, so is available for any platform that
supports Python. There are installation instructions for MacOS,
Linux, and Windows at https://httpie.org/doc#installation.
Among the Arbor Networks engineers, HTTPie is widely used.
**** Configuring HTTPie
There is much more information on the configuration of HTTPie in
its online documentation, but the two things that might be useful
are setting the defaults and creating some sessions.
HTTPie's configuration is kept in a simple JSON-formatted file
called =config.json= (see
https://httpie.org/doc#config-file-location for more); a good
start for configuration is:
#+BEGIN_SRC json
{
"__meta__": {
"about": "HTTPie configuration file",
"help": "https://github.com/jkbrzt/httpie#config",
"httpie": "0.9.4"
},
"default_options": [
"--verify=/path/to/a/ssl/certfile",
"--session=sightline-leader.my.example.com",
"--timeout=60",
"--follow"
]
}
#+END_SRC
The ="__meta__"= section comes with HTTPie, and the default
options set:
- the path to a SSL certificate file bundle
- the default connection information (what HTTPie calls a
session)
- a useful timeout value of 60 seconds
- the option to follow HTTP redirects
Once you have created =config.json= you can add some information
to the saved session by typing all of the information once on the
command line. The information the Sightline REST API needs is
the =Content-Type= and =X-Arbux-APIToken= headers; this is
entered by typing:
#+BEGIN_SRC sh :results output :exports code
http https://sightline-leader.example.com/api/sp X-Arbux-APIToken:wz8SmD3JRSKz_0kNbdSejSQ_Jk92EVOqRU6CPU_O Content-Type:application/vnd.api+json
#+END_SRC
(replacing, of course, the hostname and API token in that example
with your own hostname and API token). After you do that once,
you can then type:
#+BEGIN_SRC sh :results output :exports code
http https://sightline-leader.my.comany.com/api/sp
#+END_SRC
and the configuration will read the data from the saved file.
Save session information is in the =.httpie/sessions/= directory
and are JSON files in directories named after the host.
**** Getting data with HTTPie
Replicating the cURL example from earlier, getting the Sightline
REST API index using HTTPie, after following the configuration
steps, is done using the command:
#+BEGIN_SRC sh :results output :exports code
http GET https://sightline-leader.example.com/api/sp/
#+END_SRC
When HTTPie detects that is is writing to the terminal, it will
include additional information that might look about like:
#+BEGIN_VERSE
HTTP/1.1 200 OK
Connection: Keep-Alive
Content-Length: 1593
Content-Security-Policy: default-src 'self' 'unsafe-inline' 'unsafe-eval'
Content-Type: application/vnd.api+json
Date: Tue, 01 Aug 2017 00:55:31 GMT
Keep-Alive: timeout=15, max=10000
P3P: policyref="/w3c/p3p.xml", CP="NOI DSP COR TAIa OUR NOR UNI"
Server: Apache
Strict-Transport-Security: max-age=1261440000
Via: 1.1 127.0.0.1
X-Frame-Options: SAMEORIGIN
#+END_VERSE
when HTTPie thinks it is directing its output somewhere else, it
will not print this information; if you want to override HTTPie's
decisions about this, there are command-line options to do so.
**** Sending data with HTTPie
An example of using HTTPie to send the same data to Sightline we
sent via the Sightline REST API is using cURL earlier is (see
=input.json= in the cURL example of sending data to Sightline):
#+BEGIN_SRC sh :results output :exports code
cat input.json | http POST http https://sightline-leader.example.com/api/sp/alerts/12519/annotations/
#+END_SRC
The two differences between getting and sending data to Sightline
via the REST API using HTTPie are:
- using the =POST= verb instead of the =GET= verb before the URL
- the data you want to send to Sightline via the REST API is in
the same format as in the cURL example, but it is piped into
the =http= command
To make changes to Sightline via the REST API, you can use the
=PATCH= verb to HTTPie instead of =POST=. There will be more
examples of this later.
**** Deleting data with HTTPie
Repeating the example of deleting data from the cURL section, the
syntax for using HTTPie to delete data from Sightline via the
Sightline REST API is:
#+BEGIN_SRC sh :results output :exports code
http DELETE https://sightline-leader.example.com/api/sp/routers/132
#+END_SRC
The Sightline REST API will return the HTTP status code =204 No Content=
and HTTPie will print this if you aren't piping or redirecting
the output somewhere else.
*** Postman
#+INDEX: postman
Postman is an application that provides a graphical user interface
for working with REST APIs. It is free to use the basic version,
with the versions more suitable for development enterprises
costing money. Postman is available from
https://www.getpostman.com/ for MacOS, Linux, and Windows. It is
also available in the Chrome App Store as a plug-in for the Google
Chrome web browser[fn:2].
Among the Arbor Networks engineers, Postman is widely used.
**** Configuring Postman
If you are installing the Postman client there are three
configuration settings you need to make before using it with the
Sightline REST API; adding the certificate bundle and adding the
=Content-Type= and =X-Arbux-APIToken= headers. If you are using
the Postman Chrome app, you don't need to add the certificates;
Postman will use those in your browser, but you may still have to
add them to your browser.
To add the Sightline certificate bundle to the Postman
application, select the wrench icon in the upper right part of
the Postman window and choose Settings from the menu. In the
resulting window, choose the Certificates option from the top row
of options, then click Add Certificate. In the Host text entry
field, type =*.mydomain.com= (replacing =mydomain.com= with the
domain where your Sightline leader is), and in the CRT File text
entry field type the path to the certificate bundle for your
Sightline environment. It should look something like what is
shown in Figure [[fig:postman-cert-config]].
#+CAPTION: The Postman certificate configuration window should
#+CAPTION: look similar to this, with your information filled in;
#+CAPTION: if you are using the Chrome app, you won't see the
#+CAPTION: Certificates option in the top row; Postman will use
#+CAPTION: your browser's certificates.
#+NAME: fig:postman-cert-config
[[./images/postman-cert-config.png]]
For both the application and Chrome app you will need to set the
=Content-Type= and =X-Arbux-APIToken= headers. This is done in
the Headers section that is immediately below the URL entry field
(after making a request, there will be another Headers section
for the response). You simply type in each key and the values
for them. For all of the Sightline REST API endpoints /except/
those under =/insight/= you must set =Content-Type= to
=application/vnd+api.json=; for the endpoints under =/insight/=
you must set =Content-Type= to =application/json=.
**** Getting data with Postman
Following the cURL and HTTPie examples, to get the Sightline REST
API index using Postman you make sure that =GET= is selected to
the left of the text entry box that says "Enter request URL",
then enter the URL for the index endpoint
(=https://sightline-leader.example.com/api/sp/=) and press the Send
button. The results (the body, cookies, and headers) will appear
in the bottom pane. It will look something like what is shown in
Figure [[fig:postman-GET-index]].
#+CAPTION: Retrieving data from the Sightline REST API using Postman
#+CAPTION: is done by setting the two headers shown, entering
#+CAPTION: a URL, and pressing the Send button. The data
#+CAPTION: retrieved is displayed in the lower pane.
#+NAME: fig:postman-GET-index
[[./images/postman-GET-index.png]]
**** Sending data with Postman
Once again using the same =input.json= file from earlier, we will
set an annotation on an alert, this time using Postman.
The steps in Postman to get ready to send date are:
- change the HTTP verb to the left of the URL from =GET= to
=POST=
- enter the URL to which you are sending the data in the text
entry box that says "Enter request URL"
- select Body from the items below the URL
- select raw from the items above the text entry box
- select JSON from the dropdown to the right of type selections
- enter the JSON body you are sending in the text box.
Having done all of those steps, you should have something that
looks like Figure [[fig:postman-POST-annotation]].
#+CAPTION: Sending data to Sightline using the REST API and Postman
#+CAPTION: is done by setting the HTTP verb to POST, filling
#+CAPTION: in the URL, selecting the Body option, then the raw
#+CAPTION: option and the JSON type, and filling in the JSON
#+CAPTION: body. Once that is complete, press the Send button.
#+NAME: fig:postman-POST-annotation
[[./images/postman-POST-annotation.png]]
After sending the body with the correct settings, the results
will appear in the bottom text area in Postman.
**** Deleting data with Postman
Repeating the example of deleting data from the cURL and HTTPie
sections, the method for using Postman to delete data from
Sightline via the Sightline REST API is to enter the URL for the
object you want to delete, change the HTTP verb to Delete, and
press the Send button.
The Sightline REST API will return the HTTP status code =204 No
Content= and no body content.
After a successful DELETE operation you should have something
that looks like Figure [[fig:postman-DELETE-router]].
#+CAPTION: Deleting data from Sightline using the REST API and Postman
#+CAPTION: is done by selecting the =DELETE= HTTP verb from the
#+CAPTION: menu, entering the URL of the item to be deleted and
#+CAPTION: pressing Send. There will be no content in returned
#+CAPTION: but on the right side just above the (empty) returned
#+CAPTION: data box the status will read =204 NO CONTENT=.
#+NAME: fig:postman-DELETE-router
[[./images/postman-DELETE-router.png]]
*** =jq=
#+INDEX: jq
=jq= is a JSON parsing and manipulation tool in the spirit of
what =sed=[fn:3] and =awk=[fn:4] are for text.
=jq= is free and open-source; it is available at
https://stedolan.github.io/jq/ for MacOS, Linux, and Windows or
via package managers for your platform. The C source code and
build instructions are available at https://github.com/stedolan/jq
for other platforms.
=jq= is very powerful, but complicated. It can be used to extract
subsets of data from JSON input, search for keys or values in JSON
data and only print the objects that match the search criteria,
select and recast data from JSON input into different JSON
output. The complete manual is available at the =jq= website, but
we will provide a few examples of using =jq= with the JSON from
the Sightline REST API.
**** Extracting only names, descriptions, and ids of routers into a new JSON format
The HTTPie command to get the router information from Sightline
using the REST API is:
#+BEGIN_SRC sh :results output :exports code
http https://leader.example.com/api/sp/routers/
#+END_SRC
but on a system with 12 routers this results in 503 lines of JSON
output. If all you need is the id, name, and description of each
router, that is a lot of information to deal with.
Using =jq= we can extract the 36 pieces of data that we need.
Looking at the original output from the =/routers/= endpoint, we
need to make note of two things: the information we want is in a
list of objects in the ="data"= object; the fields we want are at
the top level of each data object (="id"=) and in the
="attributes"= sub-object for each data object.
The first thing we can do is have =jq= print each object in the
="data"= list by giving it the filter =.data[]= which means "from
the top level (=.=) print every element =[]= in the ="data"=
array". In practice this looks like:
#+BEGIN_SRC sh :results output :exports code
http https://leader.example.com/api/sp/routers/ | jq '.data[]'
#+END_SRC
The =jq= filter is in single-quotes to prevent the square
brackets from being interpreted by the shell.
=jq= uses periods to denote the JSON hierarchy (that's why
=.data= starts with a period, it is the root of the JSON tree),
the pipe (=|=) character to chain together its filters, and
commas to group elements together for selection.
So the next thing we can do is extract one element from the JSON
output of the Sightline REST API by adding it to the =.data[]=
filter. To get the names of the routers we can add
=.attributes.name= (remember, ="name"= is in the ="attributes"=
subobject of data. On the command line this looks like:
#+BEGIN_SRC sh :results output :exports code
http https://leader.example.com/api/sp/routers/ | jq '.data[].attributes.name'
#+END_SRC
This will print a list of the names of all of the routers
configured in Sightline. This is part of our original goal, but
we still also want the id and description of the routers.
To select more than one element, =jq= supports sending each list
item over which it is iterating to a group of elements for
selection. To get the router id and description along with the
name, we can type:
#+BEGIN_SRC sh :results output :exports code
http https://leader.example.com/api/sp/routers/ | jq '.data[] | .id,.attributes.name,.attributes.description'
#+END_SRC
This will then print a list that is something like:
#+LATEX: \scriptsize
#+BEGIN_EXAMPLE
"121"
"rtr1.nyc"
"a router in New York"
"122"
"rtr2.chi"
"a router in Chicago"
"123"
"rtr3.lax"
"a router in Los Angeles"
"124"
"pigeon.net"
"See RFC 1149"
#+END_EXAMPLE
#+LATEX: \normalsize
where the order of the elements is the same order as they were
requested in the filter. But this isn't very easy to read, and
is not very useful as input to computer programs. =jq= can do
better.
To make JSON-compliant output that is easy to read and easy for
another computer program to ingest, =jq= needs to know how things
should be grouped.
#+BEGIN_SRC sh :results output :exports code
http https://leader.example.com/api/sp/routers/ | jq '{"routers": [.data[] | {id:.id,name:.attributes.name,description:.attributes.description}]}'
#+END_SRC
will print a list that looks something like:
#+BEGIN_SRC json
{
"routers": [
{
"id": "121",
"name": "rtr1.nyc",
"description": "a router in New York"
},
{
"id": "122",
"name": "rtr2.chi",
"description": "a router in Chicago"
},
{
"id": "123",
"name": "rtr3.lax",
"description": "a router in Los Angeles"
},
{
"id": "124",
"name": "pigeon.net",
"description": "See RFC 1149"
}
]
}
#+END_SRC
While this example is for four routers, with the original example
of 12 routers we have gone from 503 lines of JSON output to 64
lines, removing the 87% of the data we didn't need using one
filter in =jq=.
**** Extracting alerts that have a severity percentage above 200
=jq= can also look for certain properties of objects and print
only the objects that have those properties.
For example, a typical alert from the Sightline REST API looks
like:
#+BEGIN_SRC json
{
"data": {
"attributes": {
"alert_class": "dos",
"alert_type": "dos_host_detection",
"classification": "Possible Attack",
"importance": 2,
"ongoing": true,
"start_time": "2017-08-01T19:44:45+00:00",
"subobject": {
"fast_detected": false,
"host_address": "192.168.12.203",
"impact_boundary": "rtr1.nyc",
"impact_bps": 8251040,
"impact_pps": 3010,
"ip_version": 4,
"misuse_types": [
"icmp",
"total",
"dns",
"udp"
],
"severity_percent": 153,
"severity_threshold": 1000,
"severity_unit": "bps",
"summary_url": "/page?id=profile_summary&gid=154"
}
},
"id": "799318",
"links": {
"self": "https://leader.example.com/api/sp/v3/alerts/799318"
},
"relationships": {
"annotations": {
"data": [
{
"id": "1492098",
"type": "alert_annotation"
},
{
"id": "1492097",
"type": "alert_annotation"
},
{
"id": "1492096",
"type": "alert_annotation"
},
{
"id": "1492095",
"type": "alert_annotation"
},
{
"id": "1492094",
"type": "alert_annotation"
},
{
"id": "1492093",
"type": "alert_annotation"
}
],
"links": {
"related": "https://leader.example.com/api/sp/v3/alerts/799318/annotations/"
}
},
"device": {
"data": {
"id": "115",
"type": "device"
},
"links": {
"related": "https://leader.example.com/api/sp/v3/devices/115"
}
},
"managed_object": {
"data": {
"id": "154",
"type": "managed_object"
},
"links": {
"related": "https://leader.example.com/api/sp/v3/managed_objects/154"
}
},
"packet_size_distribution": {
"data": {
"id": "packet-size-distribution-799318",
"type": "alert_packet_size_distribution"
},
"links": {
"related": "https://leader.example.com/api/sp/v3/alerts/799318/packet_size_distribution"
}