/
if_igb.c
6441 lines (5678 loc) · 177 KB
/
if_igb.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/******************************************************************************
Copyright (c) 2001-2015, Intel Corporation
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the Intel Corporation nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
/*$FreeBSD$*/
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_rss.h"
#ifdef HAVE_KERNEL_OPTION_HEADERS
#include "opt_device_polling.h"
#include "opt_altq.h"
#endif
#include "if_igb.h"
/*********************************************************************
* Driver version:
*********************************************************************/
char igb_driver_version[] = "2.5.3-k";
/*********************************************************************
* PCI Device ID Table
*
* Used by probe to select devices to load on
* Last field stores an index into e1000_strings
* Last entry must be all 0s
*
* { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
*********************************************************************/
static igb_vendor_info_t igb_vendor_info_array[] =
{
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 0, 0, 0},
{IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII, 0, 0, 0},
/* required last entry */
{0, 0, 0, 0, 0}
};
/*********************************************************************
* Table of branding strings for all supported NICs.
*********************************************************************/
static char *igb_strings[] = {
"Intel(R) PRO/1000 Network Connection"
};
/*********************************************************************
* Function prototypes
*********************************************************************/
static int igb_probe(device_t);
static int igb_attach(device_t);
static int igb_detach(device_t);
static int igb_shutdown(device_t);
static int igb_suspend(device_t);
static int igb_resume(device_t);
#ifndef IGB_LEGACY_TX
static int igb_mq_start(struct ifnet *, struct mbuf *);
static int igb_mq_start_locked(struct ifnet *, struct tx_ring *);
static void igb_qflush(struct ifnet *);
static void igb_deferred_mq_start(void *, int);
#else
static void igb_start(struct ifnet *);
static void igb_start_locked(struct tx_ring *, struct ifnet *ifp);
#endif
static int igb_ioctl(struct ifnet *, u_long, caddr_t);
static uint64_t igb_get_counter(if_t, ift_counter);
static void igb_init(void *);
static void igb_init_locked(struct adapter *);
static void igb_stop(void *);
static void igb_media_status(struct ifnet *, struct ifmediareq *);
static int igb_media_change(struct ifnet *);
static void igb_identify_hardware(struct adapter *);
static int igb_allocate_pci_resources(struct adapter *);
static int igb_allocate_msix(struct adapter *);
static int igb_allocate_legacy(struct adapter *);
static int igb_setup_msix(struct adapter *);
static void igb_free_pci_resources(struct adapter *);
static void igb_local_timer(void *);
static void igb_reset(struct adapter *);
static int igb_setup_interface(device_t, struct adapter *);
static int igb_allocate_queues(struct adapter *);
static void igb_configure_queues(struct adapter *);
static int igb_allocate_transmit_buffers(struct tx_ring *);
static void igb_setup_transmit_structures(struct adapter *);
static void igb_setup_transmit_ring(struct tx_ring *);
static void igb_initialize_transmit_units(struct adapter *);
static void igb_free_transmit_structures(struct adapter *);
static void igb_free_transmit_buffers(struct tx_ring *);
static int igb_allocate_receive_buffers(struct rx_ring *);
static int igb_setup_receive_structures(struct adapter *);
static int igb_setup_receive_ring(struct rx_ring *);
static void igb_initialize_receive_units(struct adapter *);
static void igb_free_receive_structures(struct adapter *);
static void igb_free_receive_buffers(struct rx_ring *);
static void igb_free_receive_ring(struct rx_ring *);
static void igb_enable_intr(struct adapter *);
static void igb_disable_intr(struct adapter *);
static void igb_update_stats_counters(struct adapter *);
static bool igb_txeof(struct tx_ring *);
static __inline void igb_rx_discard(struct rx_ring *, int);
static __inline void igb_rx_input(struct rx_ring *,
struct ifnet *, struct mbuf *, u32);
static bool igb_rxeof(struct igb_queue *, int, int *);
static void igb_rx_checksum(u32, struct mbuf *, u32);
static int igb_tx_ctx_setup(struct tx_ring *,
struct mbuf *, u32 *, u32 *);
static int igb_tso_setup(struct tx_ring *,
struct mbuf *, u32 *, u32 *);
static void igb_set_promisc(struct adapter *);
static void igb_disable_promisc(struct adapter *);
static void igb_set_multi(struct adapter *);
static void igb_update_link_status(struct adapter *);
static void igb_refresh_mbufs(struct rx_ring *, int);
static void igb_register_vlan(void *, struct ifnet *, u16);
static void igb_unregister_vlan(void *, struct ifnet *, u16);
static void igb_setup_vlan_hw_support(struct adapter *);
static int igb_xmit(struct tx_ring *, struct mbuf **);
static int igb_dma_malloc(struct adapter *, bus_size_t,
struct igb_dma_alloc *, int);
static void igb_dma_free(struct adapter *, struct igb_dma_alloc *);
static int igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
static void igb_print_nvm_info(struct adapter *);
static int igb_is_valid_ether_addr(u8 *);
static void igb_add_hw_stats(struct adapter *);
static void igb_vf_init_stats(struct adapter *);
static void igb_update_vf_stats_counters(struct adapter *);
/* Management and WOL Support */
static void igb_init_manageability(struct adapter *);
static void igb_release_manageability(struct adapter *);
static void igb_get_hw_control(struct adapter *);
static void igb_release_hw_control(struct adapter *);
static void igb_enable_wakeup(device_t);
static void igb_led_func(void *, int);
static int igb_irq_fast(void *);
static void igb_msix_que(void *);
static void igb_msix_link(void *);
static void igb_handle_que(void *context, int pending);
static void igb_handle_link(void *context, int pending);
static void igb_handle_link_locked(struct adapter *);
static void igb_set_sysctl_value(struct adapter *, const char *,
const char *, int *, int);
static int igb_set_flowcntl(SYSCTL_HANDLER_ARGS);
static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS);
static int igb_sysctl_eee(SYSCTL_HANDLER_ARGS);
#ifdef DEVICE_POLLING
static poll_handler_t igb_poll;
#endif /* POLLING */
/*********************************************************************
* FreeBSD Device Interface Entry Points
*********************************************************************/
static device_method_t igb_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, igb_probe),
DEVMETHOD(device_attach, igb_attach),
DEVMETHOD(device_detach, igb_detach),
DEVMETHOD(device_shutdown, igb_shutdown),
DEVMETHOD(device_suspend, igb_suspend),
DEVMETHOD(device_resume, igb_resume),
DEVMETHOD_END
};
static driver_t igb_driver = {
"igb", igb_methods, sizeof(struct adapter),
};
static devclass_t igb_devclass;
DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
MODULE_DEPEND(igb, pci, 1, 1, 1);
MODULE_DEPEND(igb, ether, 1, 1, 1);
#ifdef DEV_NETMAP
MODULE_DEPEND(igb, netmap, 1, 1, 1);
#endif /* DEV_NETMAP */
/*********************************************************************
* Tunable default values.
*********************************************************************/
static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters");
/* Descriptor defaults */
static int igb_rxd = IGB_DEFAULT_RXD;
static int igb_txd = IGB_DEFAULT_TXD;
SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0,
"Number of receive descriptors per queue");
SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0,
"Number of transmit descriptors per queue");
/*
** AIM: Adaptive Interrupt Moderation
** which means that the interrupt rate
** is varied over time based on the
** traffic for that interrupt vector
*/
static int igb_enable_aim = TRUE;
SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &igb_enable_aim, 0,
"Enable adaptive interrupt moderation");
/*
* MSIX should be the default for best performance,
* but this allows it to be forced off for testing.
*/
static int igb_enable_msix = 1;
SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0,
"Enable MSI-X interrupts");
/*
** Tuneable Interrupt rate
*/
static int igb_max_interrupt_rate = 8000;
SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
&igb_max_interrupt_rate, 0, "Maximum interrupts per second");
#ifndef IGB_LEGACY_TX
/*
** Tuneable number of buffers in the buf-ring (drbr_xxx)
*/
static int igb_buf_ring_size = IGB_BR_SIZE;
SYSCTL_INT(_hw_igb, OID_AUTO, buf_ring_size, CTLFLAG_RDTUN,
&igb_buf_ring_size, 0, "Size of the bufring");
#endif
/*
** Header split causes the packet header to
** be dma'd to a separate mbuf from the payload.
** this can have memory alignment benefits. But
** another plus is that small packets often fit
** into the header and thus use no cluster. Its
** a very workload dependent type feature.
*/
static int igb_header_split = FALSE;
SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0,
"Enable receive mbuf header split");
/*
** This will autoconfigure based on the
** number of CPUs and max supported
** MSIX messages if left at 0.
*/
static int igb_num_queues = 0;
SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0,
"Number of queues to configure, 0 indicates autoconfigure");
/*
** Global variable to store last used CPU when binding queues
** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a
** queue is bound to a cpu.
*/
static int igb_last_bind_cpu = -1;
/* How many packets rxeof tries to clean at a time */
static int igb_rx_process_limit = 100;
SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
&igb_rx_process_limit, 0,
"Maximum number of received packets to process at a time, -1 means unlimited");
/* How many packets txeof tries to clean at a time */
static int igb_tx_process_limit = -1;
SYSCTL_INT(_hw_igb, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
&igb_tx_process_limit, 0,
"Maximum number of sent packets to process at a time, -1 means unlimited");
#ifdef DEV_NETMAP /* see ixgbe.c for details */
#include <dev/netmap/if_igb_netmap.h>
#endif /* DEV_NETMAP */
/*********************************************************************
* Device identification routine
*
* igb_probe determines if the driver should be loaded on
* adapter based on PCI vendor/device id of the adapter.
*
* return BUS_PROBE_DEFAULT on success, positive on failure
*********************************************************************/
static int
igb_probe(device_t dev)
{
char adapter_name[256];
uint16_t pci_vendor_id = 0;
uint16_t pci_device_id = 0;
uint16_t pci_subvendor_id = 0;
uint16_t pci_subdevice_id = 0;
igb_vendor_info_t *ent;
INIT_DEBUGOUT("igb_probe: begin");
pci_vendor_id = pci_get_vendor(dev);
if (pci_vendor_id != IGB_INTEL_VENDOR_ID)
return (ENXIO);
pci_device_id = pci_get_device(dev);
pci_subvendor_id = pci_get_subvendor(dev);
pci_subdevice_id = pci_get_subdevice(dev);
ent = igb_vendor_info_array;
while (ent->vendor_id != 0) {
if ((pci_vendor_id == ent->vendor_id) &&
(pci_device_id == ent->device_id) &&
((pci_subvendor_id == ent->subvendor_id) ||
(ent->subvendor_id == 0)) &&
((pci_subdevice_id == ent->subdevice_id) ||
(ent->subdevice_id == 0))) {
sprintf(adapter_name, "%s, Version - %s",
igb_strings[ent->index],
igb_driver_version);
device_set_desc_copy(dev, adapter_name);
return (BUS_PROBE_DEFAULT);
}
ent++;
}
return (ENXIO);
}
/*********************************************************************
* Device initialization routine
*
* The attach entry point is called when the driver is being loaded.
* This routine identifies the type of hardware, allocates all resources
* and initializes the hardware.
*
* return 0 on success, positive on failure
*********************************************************************/
static int
igb_attach(device_t dev)
{
struct adapter *adapter;
int error = 0;
u16 eeprom_data;
INIT_DEBUGOUT("igb_attach: begin");
if (resource_disabled("igb", device_get_unit(dev))) {
device_printf(dev, "Disabled by device hint\n");
return (ENXIO);
}
adapter = device_get_softc(dev);
adapter->dev = adapter->osdep.dev = dev;
IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
/* SYSCTLs */
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
igb_sysctl_nvm_info, "I", "NVM Information");
igb_set_sysctl_value(adapter, "enable_aim",
"Interrupt Moderation", &adapter->enable_aim,
igb_enable_aim);
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW,
adapter, 0, igb_set_flowcntl, "I", "Flow Control");
callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
/* Determine hardware and mac info */
igb_identify_hardware(adapter);
/* Setup PCI resources */
if (igb_allocate_pci_resources(adapter)) {
device_printf(dev, "Allocation of PCI resources failed\n");
error = ENXIO;
goto err_pci;
}
/* Do Shared Code initialization */
if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
device_printf(dev, "Setup of Shared code failed\n");
error = ENXIO;
goto err_pci;
}
e1000_get_bus_info(&adapter->hw);
/* Sysctls for limiting the amount of work done in the taskqueues */
igb_set_sysctl_value(adapter, "rx_processing_limit",
"max number of rx packets to process",
&adapter->rx_process_limit, igb_rx_process_limit);
igb_set_sysctl_value(adapter, "tx_processing_limit",
"max number of tx packets to process",
&adapter->tx_process_limit, igb_tx_process_limit);
/*
* Validate number of transmit and receive descriptors. It
* must not exceed hardware maximum, and must be multiple
* of E1000_DBA_ALIGN.
*/
if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
(igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
device_printf(dev, "Using %d TX descriptors instead of %d!\n",
IGB_DEFAULT_TXD, igb_txd);
adapter->num_tx_desc = IGB_DEFAULT_TXD;
} else
adapter->num_tx_desc = igb_txd;
if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
(igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
device_printf(dev, "Using %d RX descriptors instead of %d!\n",
IGB_DEFAULT_RXD, igb_rxd);
adapter->num_rx_desc = IGB_DEFAULT_RXD;
} else
adapter->num_rx_desc = igb_rxd;
adapter->hw.mac.autoneg = DO_AUTO_NEG;
adapter->hw.phy.autoneg_wait_to_complete = FALSE;
adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
/* Copper options */
if (adapter->hw.phy.media_type == e1000_media_type_copper) {
adapter->hw.phy.mdix = AUTO_ALL_MODES;
adapter->hw.phy.disable_polarity_correction = FALSE;
adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
}
/*
* Set the frame limits assuming
* standard ethernet sized frames.
*/
adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
/*
** Allocate and Setup Queues
*/
if (igb_allocate_queues(adapter)) {
error = ENOMEM;
goto err_pci;
}
/* Allocate the appropriate stats memory */
if (adapter->vf_ifp) {
adapter->stats =
(struct e1000_vf_stats *)malloc(sizeof \
(struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
igb_vf_init_stats(adapter);
} else
adapter->stats =
(struct e1000_hw_stats *)malloc(sizeof \
(struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
if (adapter->stats == NULL) {
device_printf(dev, "Can not allocate stats memory\n");
error = ENOMEM;
goto err_late;
}
/* Allocate multicast array memory. */
adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
if (adapter->mta == NULL) {
device_printf(dev, "Can not allocate multicast setup array\n");
error = ENOMEM;
goto err_late;
}
/* Some adapter-specific advanced features */
if (adapter->hw.mac.type >= e1000_i350) {
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW,
adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "eee_disabled", CTLTYPE_INT|CTLFLAG_RW,
adapter, 0, igb_sysctl_eee, "I",
"Disable Energy Efficient Ethernet");
if (adapter->hw.phy.media_type == e1000_media_type_copper) {
if (adapter->hw.mac.type == e1000_i354)
e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
else
e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
}
}
/*
** Start from a known state, this is
** important in reading the nvm and
** mac from that.
*/
e1000_reset_hw(&adapter->hw);
/* Make sure we have a good EEPROM before we read from it */
if (((adapter->hw.mac.type != e1000_i210) &&
(adapter->hw.mac.type != e1000_i211)) &&
(e1000_validate_nvm_checksum(&adapter->hw) < 0)) {
/*
** Some PCI-E parts fail the first check due to
** the link being in sleep state, call it again,
** if it fails a second time its a real issue.
*/
if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
device_printf(dev,
"The EEPROM Checksum Is Not Valid\n");
error = EIO;
goto err_late;
}
}
/*
** Copy the permanent MAC address out of the EEPROM
*/
if (e1000_read_mac_addr(&adapter->hw) < 0) {
device_printf(dev, "EEPROM read error while reading MAC"
" address\n");
error = EIO;
goto err_late;
}
/* Check its sanity */
if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
device_printf(dev, "Invalid MAC address\n");
error = EIO;
goto err_late;
}
/* Setup OS specific network interface */
if (igb_setup_interface(dev, adapter) != 0)
goto err_late;
/* Now get a good starting state */
igb_reset(adapter);
/* Initialize statistics */
igb_update_stats_counters(adapter);
adapter->hw.mac.get_link_status = 1;
igb_update_link_status(adapter);
/* Indicate SOL/IDER usage */
if (e1000_check_reset_block(&adapter->hw))
device_printf(dev,
"PHY reset is blocked due to SOL/IDER session.\n");
/* Determine if we have to control management hardware */
adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
/*
* Setup Wake-on-Lan
*/
/* APME bit in EEPROM is mapped to WUC.APME */
eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
if (eeprom_data)
adapter->wol = E1000_WUFC_MAG;
/* Register for VLAN events */
adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
igb_add_hw_stats(adapter);
/* Tell the stack that the interface is not active */
adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
adapter->led_dev = led_create(igb_led_func, adapter,
device_get_nameunit(dev));
/*
** Configure Interrupts
*/
if ((adapter->msix > 1) && (igb_enable_msix))
error = igb_allocate_msix(adapter);
else /* MSI or Legacy */
error = igb_allocate_legacy(adapter);
if (error)
goto err_late;
#ifdef DEV_NETMAP
igb_netmap_attach(adapter);
#endif /* DEV_NETMAP */
INIT_DEBUGOUT("igb_attach: end");
return (0);
err_late:
if (igb_detach(dev) == 0) /* igb_detach() already did the cleanup */
return(error);
igb_free_transmit_structures(adapter);
igb_free_receive_structures(adapter);
igb_release_hw_control(adapter);
err_pci:
igb_free_pci_resources(adapter);
if (adapter->ifp != NULL)
if_free(adapter->ifp);
free(adapter->mta, M_DEVBUF);
IGB_CORE_LOCK_DESTROY(adapter);
return (error);
}
/*********************************************************************
* Device removal routine
*
* The detach entry point is called when the driver is being removed.
* This routine stops the adapter and deallocates all the resources
* that were allocated for driver operation.
*
* return 0 on success, positive on failure
*********************************************************************/
static int
igb_detach(device_t dev)
{
struct adapter *adapter = device_get_softc(dev);
struct ifnet *ifp = adapter->ifp;
INIT_DEBUGOUT("igb_detach: begin");
/* Make sure VLANS are not using driver */
if (adapter->ifp->if_vlantrunk != NULL) {
device_printf(dev,"Vlan in use, detach first\n");
return (EBUSY);
}
ether_ifdetach(adapter->ifp);
if (adapter->led_dev != NULL)
led_destroy(adapter->led_dev);
#ifdef DEVICE_POLLING
if (ifp->if_capenable & IFCAP_POLLING)
ether_poll_deregister(ifp);
#endif
IGB_CORE_LOCK(adapter);
adapter->in_detach = 1;
igb_stop(adapter);
IGB_CORE_UNLOCK(adapter);
e1000_phy_hw_reset(&adapter->hw);
/* Give control back to firmware */
igb_release_manageability(adapter);
igb_release_hw_control(adapter);
if (adapter->wol) {
E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
igb_enable_wakeup(dev);
}
/* Unregister VLAN events */
if (adapter->vlan_attach != NULL)
EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
if (adapter->vlan_detach != NULL)
EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
callout_drain(&adapter->timer);
#ifdef DEV_NETMAP
netmap_detach(adapter->ifp);
#endif /* DEV_NETMAP */
igb_free_pci_resources(adapter);
bus_generic_detach(dev);
if_free(ifp);
igb_free_transmit_structures(adapter);
igb_free_receive_structures(adapter);
if (adapter->mta != NULL)
free(adapter->mta, M_DEVBUF);
IGB_CORE_LOCK_DESTROY(adapter);
return (0);
}
/*********************************************************************
*
* Shutdown entry point
*
**********************************************************************/
static int
igb_shutdown(device_t dev)
{
return igb_suspend(dev);
}
/*
* Suspend/resume device methods.
*/
static int
igb_suspend(device_t dev)
{
struct adapter *adapter = device_get_softc(dev);
IGB_CORE_LOCK(adapter);
igb_stop(adapter);
igb_release_manageability(adapter);
igb_release_hw_control(adapter);
if (adapter->wol) {
E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
igb_enable_wakeup(dev);
}
IGB_CORE_UNLOCK(adapter);
return bus_generic_suspend(dev);
}
static int
igb_resume(device_t dev)
{
struct adapter *adapter = device_get_softc(dev);
struct tx_ring *txr = adapter->tx_rings;
struct ifnet *ifp = adapter->ifp;
IGB_CORE_LOCK(adapter);
igb_init_locked(adapter);
igb_init_manageability(adapter);
if ((ifp->if_flags & IFF_UP) &&
(ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
for (int i = 0; i < adapter->num_queues; i++, txr++) {
IGB_TX_LOCK(txr);
#ifndef IGB_LEGACY_TX
/* Process the stack queue only if not depleted */
if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
!drbr_empty(ifp, txr->br))
igb_mq_start_locked(ifp, txr);
#else
if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
igb_start_locked(txr, ifp);
#endif
IGB_TX_UNLOCK(txr);
}
}
IGB_CORE_UNLOCK(adapter);
return bus_generic_resume(dev);
}
#ifdef IGB_LEGACY_TX
/*********************************************************************
* Transmit entry point
*
* igb_start is called by the stack to initiate a transmit.
* The driver will remain in this routine as long as there are
* packets to transmit and transmit resources are available.
* In case resources are not available stack is notified and
* the packet is requeued.
**********************************************************************/
static void
igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
{
struct adapter *adapter = ifp->if_softc;
struct mbuf *m_head;
IGB_TX_LOCK_ASSERT(txr);
if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
IFF_DRV_RUNNING)
return;
if (!adapter->link_active)
return;
/* Call cleanup if number of TX descriptors low */
if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
igb_txeof(txr);
while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
if (txr->tx_avail <= IGB_MAX_SCATTER) {
txr->queue_status |= IGB_QUEUE_DEPLETED;
break;
}
IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
if (m_head == NULL)
break;
/*
* Encapsulation can modify our pointer, and or make it
* NULL on failure. In that event, we can't requeue.
*/
if (igb_xmit(txr, &m_head)) {
if (m_head != NULL)
IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
if (txr->tx_avail <= IGB_MAX_SCATTER)
txr->queue_status |= IGB_QUEUE_DEPLETED;
break;
}
/* Send a copy of the frame to the BPF listener */
ETHER_BPF_MTAP(ifp, m_head);
/* Set watchdog on */
txr->watchdog_time = ticks;
txr->queue_status |= IGB_QUEUE_WORKING;
}
}
/*
* Legacy TX driver routine, called from the
* stack, always uses tx[0], and spins for it.
* Should not be used with multiqueue tx
*/
static void
igb_start(struct ifnet *ifp)
{
struct adapter *adapter = ifp->if_softc;
struct tx_ring *txr = adapter->tx_rings;
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
IGB_TX_LOCK(txr);
igb_start_locked(txr, ifp);
IGB_TX_UNLOCK(txr);
}
return;
}
#else /* ~IGB_LEGACY_TX */
/*
** Multiqueue Transmit Entry:
** quick turnaround to the stack
**
*/
static int
igb_mq_start(struct ifnet *ifp, struct mbuf *m)
{
struct adapter *adapter = ifp->if_softc;
struct igb_queue *que;
struct tx_ring *txr;
int i, err = 0;
#ifdef RSS
uint32_t bucket_id;
#endif
/* Which queue to use */
/*
* When doing RSS, map it to the same outbound queue
* as the incoming flow would be mapped to.
*
* If everything is setup correctly, it should be the
* same bucket that the current CPU we're on is.
*/
if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
#ifdef RSS
if (rss_hash2bucket(m->m_pkthdr.flowid,
M_HASHTYPE_GET(m), &bucket_id) == 0) {
/* XXX TODO: spit out something if bucket_id > num_queues? */
i = bucket_id % adapter->num_queues;
} else {
#endif
i = m->m_pkthdr.flowid % adapter->num_queues;
#ifdef RSS
}
#endif
} else {
i = curcpu % adapter->num_queues;
}
txr = &adapter->tx_rings[i];
que = &adapter->queues[i];
err = drbr_enqueue(ifp, txr->br, m);
if (err)
return (err);
if (IGB_TX_TRYLOCK(txr)) {
igb_mq_start_locked(ifp, txr);
IGB_TX_UNLOCK(txr);
} else
taskqueue_enqueue(que->tq, &txr->txq_task);
return (0);
}
static int
igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
{
struct adapter *adapter = txr->adapter;
struct mbuf *next;
int err = 0, enq = 0;
IGB_TX_LOCK_ASSERT(txr);
if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
adapter->link_active == 0)
return (ENETDOWN);
/* Process the queue */
while ((next = drbr_peek(ifp, txr->br)) != NULL) {
if ((err = igb_xmit(txr, &next)) != 0) {
if (next == NULL) {
/* It was freed, move forward */
drbr_advance(ifp, txr->br);
} else {
/*
* Still have one left, it may not be
* the same since the transmit function
* may have changed it.
*/
drbr_putback(ifp, txr->br, next);
}
break;
}
drbr_advance(ifp, txr->br);
enq++;
if (next->m_flags & M_MCAST && adapter->vf_ifp)
if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
ETHER_BPF_MTAP(ifp, next);
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
break;
}
if (enq > 0) {
/* Set the watchdog */
txr->queue_status |= IGB_QUEUE_WORKING;
txr->watchdog_time = ticks;
}
if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)