New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Registry log #9829
Registry log #9829
Changes from 5 commits
e97a90b
a9a4061
245e6ae
3b2e5e2
dd67ed6
e2bbe45
559b9a7
1a021e6
4742293
cf5ba3d
42be1a0
2092bc4
f5fcacf
2e576ef
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,110 @@ | ||
{ | ||
"Model": {}, | ||
"Registry": {}, | ||
"Component": { | ||
"Filter": { | ||
"Attempt": 1, | ||
"Error": { | ||
"Code": "11098", | ||
"Severity": 2, | ||
"ShortDescription": [ | ||
"Empty schema for the component" | ||
], | ||
"LongDescription": null, | ||
"ProbableCause": [ | ||
"The schema is empty for the component." | ||
], | ||
"SuggestedRemediation": [ | ||
"For the particular component the schema is empty. Use the docs or discussion forum for more details " | ||
] | ||
} | ||
}, | ||
"Filter Policy": { | ||
"Attempt": 1, | ||
"Error": { | ||
"Code": "11098", | ||
"Severity": 2, | ||
"ShortDescription": [ | ||
"Empty schema for the component" | ||
], | ||
"LongDescription": null, | ||
"ProbableCause": [ | ||
"The schema is empty for the component." | ||
], | ||
"SuggestedRemediation": [ | ||
"For the particular component the schema is empty. Use the docs or discussion forum for more details " | ||
] | ||
} | ||
}, | ||
"Project": { | ||
"Attempt": 1, | ||
"Error": { | ||
"Code": "11098", | ||
"Severity": 2, | ||
"ShortDescription": [ | ||
"Empty schema for the component" | ||
], | ||
"LongDescription": null, | ||
"ProbableCause": [ | ||
"The schema is empty for the component." | ||
], | ||
"SuggestedRemediation": [ | ||
"For the particular component the schema is empty. Use the docs or discussion forum for more details " | ||
] | ||
} | ||
}, | ||
"Project Controller": { | ||
"Attempt": 1, | ||
"Error": { | ||
"Code": "11098", | ||
"Severity": 2, | ||
"ShortDescription": [ | ||
"Empty schema for the component" | ||
], | ||
"LongDescription": null, | ||
"ProbableCause": [ | ||
"The schema is empty for the component." | ||
], | ||
"SuggestedRemediation": [ | ||
"For the particular component the schema is empty. Use the docs or discussion forum for more details " | ||
] | ||
} | ||
}, | ||
"Project Revision": { | ||
"Attempt": 1, | ||
"Error": { | ||
"Code": "11098", | ||
"Severity": 2, | ||
"ShortDescription": [ | ||
"Empty schema for the component" | ||
], | ||
"LongDescription": null, | ||
"ProbableCause": [ | ||
"The schema is empty for the component." | ||
], | ||
"SuggestedRemediation": [ | ||
"For the particular component the schema is empty. Use the docs or discussion forum for more details " | ||
] | ||
} | ||
}, | ||
"Rate Limit": { | ||
"Attempt": 1, | ||
"Error": { | ||
"Code": "11098", | ||
"Severity": 2, | ||
"ShortDescription": [ | ||
"Empty schema for the component" | ||
], | ||
"LongDescription": null, | ||
"ProbableCause": [ | ||
"The schema is empty for the component." | ||
], | ||
"SuggestedRemediation": [ | ||
"For the particular component the schema is empty. Use the docs or discussion forum for more details " | ||
] | ||
} | ||
} | ||
}, | ||
"Relationship": {}, | ||
"Policy": {} | ||
} |
Original file line number | Diff line number | Diff line change | ||
---|---|---|---|---|
|
@@ -2,13 +2,16 @@ | |||
|
||||
import ( | ||||
"encoding/json" | ||||
"fmt" | ||||
"net/http" | ||||
"strconv" | ||||
|
||||
"github.com/gofrs/uuid" | ||||
"github.com/gorilla/mux" | ||||
"github.com/layer5io/meshery/server/helpers/utils" | ||||
"github.com/layer5io/meshery/server/models" | ||||
"github.com/layer5io/meshery/server/models/pattern/core" | ||||
"github.com/layer5io/meshkit/models/events" | ||||
"github.com/layer5io/meshkit/models/meshmodel/core/types" | ||||
"github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1" | ||||
"github.com/layer5io/meshkit/models/meshmodel/registry" | ||||
|
@@ -1275,6 +1278,85 @@ | |||
go h.config.MeshModelSummaryChannel.Publish() | ||||
} | ||||
|
||||
// swagger:route POST /api/meshmodel/nonRegisterEntity | ||||
// Handle POST request for registering meshmodel components. | ||||
// | ||||
// Validate the given value with the given schema | ||||
// responses: | ||||
// 200:MeshModelHostsWithEntitySummary | ||||
|
||||
// request body should be json | ||||
// request body should be of Host format | ||||
|
||||
func (handler *Handler) NonRegisterEntity(responseWriter http.ResponseWriter, request *http.Request, _ *models.Preference, user *models.User, provider models.Provider) { | ||||
responseWriter.Header().Set("Content-Type", "application/json") | ||||
defer request.Body.Close() | ||||
|
||||
var receivedHost registry.Host | ||||
if err := json.NewDecoder(request.Body).Decode(&receivedHost); err != nil { | ||||
http.Error(responseWriter, err.Error(), http.StatusBadRequest) | ||||
return | ||||
} | ||||
|
||||
filter := &v1alpha1.HostFilter{ | ||||
DisplayName: receivedHost.Hostname, | ||||
} | ||||
|
||||
hosts, _, _ := handler.registryManager.GetRegistrants(filter) | ||||
|
||||
successMessage := "" | ||||
for _, host := range hosts { | ||||
successMessage = fmt.Sprintf("For registrant %s successfully imported", receivedHost.Hostname) | ||||
|
||||
appendIfNonZero := func(value int64, label string) { | ||||
if value != 0 { | ||||
successMessage += fmt.Sprintf(" %d %s", value, label) | ||||
} | ||||
} | ||||
|
||||
appendIfNonZero(host.Summary.Models, "models") | ||||
appendIfNonZero(host.Summary.Components, "components") | ||||
appendIfNonZero(host.Summary.Relationships, "relationships") | ||||
appendIfNonZero(host.Summary.Policies, "policies") | ||||
|
||||
} | ||||
|
||||
// Event creation | ||||
userID := uuid.FromStringOrNil(user.ID) | ||||
eventBuilder := events.NewEvent().FromUser(userID).FromSystem(*handler.SystemID).WithCategory("entity").WithAction("get_summary") | ||||
|
||||
// Adding metadata for the event | ||||
eventBuilder.WithMetadata(map[string]interface{}{ | ||||
"Hostname": receivedHost.Hostname, | ||||
}) | ||||
|
||||
// Success event | ||||
eventBuilder.WithSeverity(events.Success).WithDescription(successMessage) | ||||
successEvent := eventBuilder.Build() | ||||
|
||||
// Build and publish the events | ||||
_ = provider.PersistEvent(successEvent) | ||||
|
||||
go handler.config.EventBroadcaster.Publish(userID, successEvent) | ||||
|
||||
failedMessage, _ := registry.FailedMsgCompute("", receivedHost.Hostname) | ||||
Check failure on line 1342 in server/handlers/component_handler.go GitHub Actions / golangci-lint (1.21, ubuntu-22.04)
Check failure on line 1342 in server/handlers/component_handler.go GitHub Actions / golangci-lint (1.21, ubuntu-22.04)
Check failure on line 1342 in server/handlers/component_handler.go GitHub Actions / golangci-lint (1.21, ubuntu-22.04)
|
||||
if failedMessage != "" { | ||||
failedMessage = fmt.Sprintf("For registrant %s %s", receivedHost.Hostname, failedMessage) | ||||
// Error event | ||||
errorEventBuilder := events.NewEvent().FromUser(userID).FromSystem(*handler.SystemID).WithCategory("entity").WithAction("get_summary") | ||||
errorEventBuilder.WithSeverity(events.Error).WithDescription(failedMessage) | ||||
errorEvent := errorEventBuilder.Build() | ||||
errorEventBuilder.WithMetadata(map[string]interface{}{ | ||||
"Hostname": receivedHost.Hostname, | ||||
"Details": fmt.Sprintf("The import process for a registrant %s encountered difficulties,due to which %s. Specific issues during the import process resulted in certain entities not being successfully registered in the table.", receivedHost.Hostname, failedMessage), | ||||
"Suggested-remediation": "Check /server/cmd/registery_attempts.json for futher details", | ||||
}) | ||||
_ = provider.PersistEvent(errorEvent) | ||||
go handler.config.EventBroadcaster.Publish(userID, errorEvent) | ||||
} | ||||
|
||||
} | ||||
|
||||
// swagger:route GET /api/meshmodels/registrants GetMeshmodelRegistrants | ||||
// Handle GET request for getting all meshmodel registrants | ||||
// | ||||
|
@@ -1296,7 +1378,7 @@ | |||
func (h *Handler) GetMeshmodelRegistrants(rw http.ResponseWriter, r *http.Request) { | ||||
rw.Header().Add("Content-Type", "application/json") | ||||
enc := json.NewEncoder(rw) | ||||
|
||||
registry.Mutex.Lock() | ||||
Check failure on line 1381 in server/handlers/component_handler.go GitHub Actions / golangci-lint (1.21, ubuntu-22.04)
Check failure on line 1381 in server/handlers/component_handler.go GitHub Actions / golangci-lint (1.21, ubuntu-22.04)
|
||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @MUzairS15 should i use this to sync the registering and fetching of the data i Don't think it is necessary just a fail safe if someone opens the server if the entites are still being registered can you confirm this meshery/server/meshmodel/helper.go Line 167 in dd67ed6
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No not required |
||||
limitstr := r.URL.Query().Get("pagesize") | ||||
pagestr := r.URL.Query().Get("page") | ||||
|
||||
|
@@ -1330,7 +1412,7 @@ | |||
http.Error(rw, ErrGetMeshModels(err).Error(), http.StatusInternalServerError) | ||||
return | ||||
} | ||||
|
||||
registry.Mutex.Unlock() | ||||
Check failure on line 1415 in server/handlers/component_handler.go GitHub Actions / golangci-lint (1.21, ubuntu-22.04)
Check failure on line 1415 in server/handlers/component_handler.go GitHub Actions / golangci-lint (1.21, ubuntu-22.04)
|
||||
var pgSize int64 | ||||
|
||||
if limitstr == "all" { | ||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -14,6 +14,8 @@ | |
"github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1" | ||
meshmodel "github.com/layer5io/meshkit/models/meshmodel/registry" | ||
"github.com/pkg/errors" | ||
"github.com/sirupsen/logrus" | ||
"github.com/spf13/viper" | ||
) | ||
|
||
var ArtifactHubComponentsHandler = meshmodel.ArtifactHub{} //The components generated in output directory will be handled by kubernetes | ||
|
@@ -162,6 +164,7 @@ | |
// If an error occurs, it logs the error | ||
func (erh *EntityRegistrationHelper) watchComponents(ctx context.Context) { | ||
var err error | ||
meshmodel.Mutex.Lock() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These mutex is for shared variables right used in registryLog function? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @MUzairS15 |
||
for { | ||
select { | ||
case comp := <-erh.componentChan: | ||
|
@@ -180,6 +183,8 @@ | |
} | ||
|
||
case <-ctx.Done(): | ||
registryLog(erh.regManager) | ||
meshmodel.Mutex.Unlock() | ||
return | ||
} | ||
|
||
|
@@ -188,3 +193,72 @@ | |
} | ||
} | ||
} | ||
func writeToFile(filePath string, data []byte) error { | ||
file, err := os.Create(filePath) | ||
if err != nil { | ||
return err | ||
} | ||
defer file.Close() | ||
|
||
_, err = file.Write(data) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
return nil | ||
} | ||
func registryLog(regManager *meshmodel.RegistryManager) { | ||
logLevel := viper.GetInt("LOG_LEVEL") | ||
if viper.GetBool("DEBUG") { | ||
logLevel = int(logrus.DebugLevel) | ||
} | ||
// Initialize Logger instance | ||
log, err := logger.New("meshery", logger.Options{ | ||
Format: logger.SyslogLogFormat, | ||
LogLevel: logLevel, | ||
}) | ||
|
||
if err != nil { | ||
logrus.Error(err) | ||
os.Exit(1) | ||
} | ||
|
||
hosts, _, err := regManager.GetRegistrants(&v1alpha1.HostFilter{}) | ||
if err != nil { | ||
log.Error(err) | ||
} | ||
for _, host := range hosts { | ||
summary := host.Summary | ||
addNonZero := func(a, b int64) int64 { | ||
if b != 0 { | ||
return a + b | ||
} | ||
return a | ||
} | ||
totalModels := addNonZero(0, summary.Models) | ||
totalComponents := addNonZero(0, summary.Components) | ||
totalRelationships := addNonZero(0, summary.Relationships) | ||
totalPolicies := addNonZero(0, summary.Policies) | ||
|
||
log.Info(fmt.Sprintf("For registrant %s successfully imported %d models %d components %d relationships %d policy", | ||
host.Hostname, totalModels, totalComponents, totalRelationships, totalPolicies)) | ||
|
||
failedMsg, _ := meshmodel.FailedMsgCompute("", host.Hostname) | ||
if failedMsg != "" { | ||
log.Error(meshmodel.ErrRegisteringEntity(failedMsg, host.Hostname)) | ||
} | ||
} | ||
|
||
filePath := "register_attempts.json" | ||
jsonData, err := json.MarshalIndent(meshmodel.RegisterAttempts, "", " ") | ||
if err != nil { | ||
log.Error(meshmodel.ErrMarshalingRegisteryAttempts(err)) | ||
return | ||
} | ||
|
||
err = writeToFile(filePath, jsonData) | ||
if err != nil { | ||
log.Error(meshmodel.ErrWritingRegisteryAttempts(err)) | ||
return | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
import React, { useEffect } from 'react'; | ||
import { useDispatch } from 'react-redux'; | ||
import { loadEvents } from '../../store/slices/events'; | ||
|
||
const NotifyRegistrant = () => { | ||
const dispatch = useDispatch(); | ||
|
||
const fetchData = async () => { | ||
try { | ||
const response = await fetch('api/meshmodels/registrants'); | ||
const data = await response.json(); | ||
console.log(response); | ||
if (data.total_count > 0) { | ||
for (const registrant of data.registrants) { | ||
const { hostname } = registrant; | ||
await fetch('api/meshmodels/nonRegisterEntity', { | ||
method: 'POST', | ||
headers: { | ||
'Content-Type': 'application/json', | ||
}, | ||
body: JSON.stringify({ Hostname: hostname }), | ||
}); | ||
} | ||
} | ||
} catch (error) { | ||
console.error('Error fetching data:', error); | ||
} | ||
}; | ||
|
||
useEffect(() => { | ||
const isInitialVisit = localStorage.getItem('isInitialVisit') === null; | ||
|
||
if (isInitialVisit) { | ||
dispatch(loadEvents(fetchData, 1, {})); | ||
localStorage.setItem('isInitialVisit', 'true'); | ||
} else { | ||
fetchData(); | ||
} | ||
}, [dispatch]); | ||
|
||
return <div></div>; | ||
}; | ||
|
||
export default NotifyRegistrant; |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Still not according to API design.
@Jougan-0 The functionality that you are defining in the PR is crucial, while the output that we get from the current implementation is nice, the approach is not ideal.
How do you ensure that the registration might have been completed by the time UI makes a request to this endpoint?
My question still stands, if the server is doing the registration why you cannot consolidate all errors and send it all at once?
See the errorChan channel it listens to all the errors encountered, can't you send in more details on this channel
model, component and error
or theevent
object itself, store it in an array, and call thecancel
function of the passed in context (when registration completes), when you will call the cancel func the ctx.Done case will be called, there you can iterate over that array and add in more details (i.e. summary, x model, y comps failed to register..... )There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Screencast from 2024-01-09 06-22-53.webm
@MUzairS15 the reason for not using the errorChan is the same I am displaying in the video when the return type of schema error in meshkit is nil the code works fine and these components are not registered but if we return an error such as schema empty the registration breaks I don't know the reason for it but this doesn't make sense i searched but i don't know why we exit quietly when the schema is empty and if we return an error the code breaks so i opt for using the meshkit to store the error and store their in meshkit itself when the entites try to register .
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
API design is still not accurate, invoking the event from UI is also inaccurate, find a way to send events from the server.