-
Notifications
You must be signed in to change notification settings - Fork 32
/
system.go
236 lines (195 loc) · 5.37 KB
/
system.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (c) 2022-2023 Intel Corporation.
* Copyright (c) 2022 Canopus Networks.
*/
package cne
/*
#include <cne.h>
#include <cne_mmap.h>
#include <pktmbuf.h>
#include <pktdev.h>
#include <uds.h>
#include <uds_connect.h>
*/
import "C"
import (
"errors"
"fmt"
"os"
"runtime"
"sync"
"unsafe"
"github.com/tidwall/jsonc"
"golang.org/x/sys/unix"
)
const (
DefaultLCoreGroup = "default" // the lcore group used if a thread if not assigned to a group
MainThreadName = "main" // the thread name used to register the main CNDP thread
)
// System structure to hold internal information for binding layer to CNDP
type System struct {
jcfg *Config
mu sync.Mutex
tidMap map[string]*int // map of thread name to cne uid
groupSet map[string]*unix.CPUSet // map of lcore group name to cpu set to use when registering
oldSet map[string]*unix.CPUSet // map of thread name to cpu set before the thread was registered
}
// Create and setup the UMEM and LPort structures
func (sys *System) initializeSystem() error {
if err := sys.setupUMEMs(); err != nil {
return err
}
if err := sys.setupLPorts(); err != nil {
sys.cleanupUMEM()
return err
}
return nil
}
// InitCNE initializes the CNE system and returns error on failure.
//
// The cne_init() is also called from OpenWithConfig(), which means
// this routine does not need to be called if OpenWith*() routines
// are used. It is OK to call this routine multiple times as it is
// protected to only initialize once.
func InitCNE() error {
if C.cne_init() < 0 {
return fmt.Errorf("error initializing CNE subsystem")
}
return nil
}
// OpenWithConfig by passing in a initialized cne.Config structure
func OpenWithConfig(cfg *Config) (*System, error) {
if err := cfg.validateConfig(); err != nil {
return nil, fmt.Errorf("failed to validate configuration: %w", err)
}
sys := &System{jcfg: cfg}
sys.tidMap = make(map[string]*int)
sys.groupSet = make(map[string]*unix.CPUSet)
sys.oldSet = make(map[string]*unix.CPUSet)
if err := InitCNE(); err != nil {
return nil, err
}
// create cpu sets for lcore groups
for group, cores := range cfg.LCoreGroupData {
var cs unix.CPUSet
for _, n := range cores {
cs.Set(n)
}
sys.groupSet[group] = &cs
}
err := sys.RegisterThread(MainThreadName)
if err != nil {
return nil, err
}
err = sys.initializeSystem()
if err != nil {
sys.UnregisterThread(MainThreadName)
return nil, err
}
return sys, nil
}
func openWithBytes(b []byte) (*System, error) {
jcfg, err := processConfig(jsonc.ToJSON(b))
if err != nil {
return nil, fmt.Errorf("failed to process configuration: %w", err)
}
return OpenWithConfig(jcfg)
}
// OpenWithString by passing in a string containing JSON-C or JSON text
func OpenWithString(s string) (*System, error) {
return openWithBytes([]byte(s))
}
// OpenWithFile by passing in a filename or path to a JSON-C or JSON configuration
func OpenWithFile(path string) (*System, error) {
b, err := os.ReadFile(path)
if err != nil {
return nil, err
}
return openWithBytes(b)
}
// Close the binding layer and cleanup UMEM and LPorts
func (sys *System) Close() error {
sys.mu.Lock()
defer sys.mu.Unlock()
if sys.tidMap[MainThreadName] == nil {
return errors.New("system not initialized")
}
defer sys.unregister(MainThreadName)
if err := sys.cleanupUMEM(); err != nil {
return fmt.Errorf("error cleaning up mempool %w", err)
}
if err := sys.cleanupLPorts(); err != nil {
return fmt.Errorf("error cleaning up lports %w", err)
}
return nil
}
// JsonCfg returns the JSON configuration structure pointer
func (sys *System) JsonCfg() *Config {
return sys.jcfg
}
// RegisterThread locks a thread and registers it to CNDP to get the CNE UID value
func (sys *System) RegisterThread(name string) error {
sys.mu.Lock()
defer sys.mu.Unlock()
if sys.tidMap[name] != nil {
return fmt.Errorf("thread %s already registered", name)
}
cName := C.CString(name)
defer C.free(unsafe.Pointer(cName))
runtime.LockOSThread()
tid := int(C.cne_register(cName))
if tid < 0 {
runtime.UnlockOSThread()
return fmt.Errorf("failed to register %s thread", name)
}
sys.tidMap[name] = &tid
// get cpu set from thread lcore group
var newSet *unix.CPUSet
if threadInfo := sys.jcfg.ThreadInfoMap[name]; threadInfo != nil {
newSet = sys.groupSet[threadInfo.Group]
}
if newSet == nil {
newSet = sys.groupSet[DefaultLCoreGroup]
if newSet == nil {
return nil
}
}
// save current cpu affinity settings
var oldSet unix.CPUSet
err := unix.SchedGetaffinity(0, &oldSet)
if err != nil {
sys.unregister(name)
return err
}
sys.oldSet[name] = &oldSet
// set new cpu affinity settings
err = unix.SchedSetaffinity(0, newSet)
if err != nil {
sys.unregister(name)
return err
}
return nil
}
func (sys *System) unregister(name string) error {
tid := sys.tidMap[name]
if tid == nil {
return fmt.Errorf("thread %s not registered", name)
}
if ret := int(C.cne_unregister(C.int(*tid))); ret < 0 {
return fmt.Errorf("cne_unregister(%v) failed", tid)
}
delete(sys.tidMap, name)
runtime.UnlockOSThread()
oldSet := sys.oldSet[name]
if oldSet != nil {
delete(sys.oldSet, name)
return unix.SchedSetaffinity(0, oldSet)
}
return nil
}
// UnregisterThread unregister the thread from CNDP
func (sys *System) UnregisterThread(name string) error {
sys.mu.Lock()
defer sys.mu.Unlock()
return sys.unregister(name)
}