diff --git a/.github/workflows/agent.yml b/.github/workflows/agent.yml index e0fc52958..863cb8a2a 100644 --- a/.github/workflows/agent.yml +++ b/.github/workflows/agent.yml @@ -15,6 +15,16 @@ jobs: - name: Checkout code uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.21 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.12' + - name: Build and sign agent services id: set-env run: | diff --git a/agent/agent/main.go b/agent/agent/main.go index 6c691bd16..26544d129 100644 --- a/agent/agent/main.go +++ b/agent/agent/main.go @@ -29,7 +29,7 @@ func main() { h.FatalError("Failed to get current path: %v", err) } - // Configuring log saving + // Configuring log saving var logger = utils.CreateLogger(filepath.Join(path, "logs", configuration.SERV_LOG)) defer logger.Close() log.SetOutput(logger) @@ -83,7 +83,7 @@ func main() { h.FatalError("error configuring syslog server: %v", err) } - // Install Beats + // Install Beats if err = beats.InstallBeats(*cnf, h); err != nil { fmt.Printf("error installing beats: %v", err) h.FatalError("error installing beats: %v", err) @@ -96,7 +96,7 @@ func main() { msg := os.Args[2] logp := logservice.GetLogProcessor() - // Read config + // Read config cnf, err := configuration.GetCurrentConfig() if err != nil { os.Exit(0) diff --git a/backend/src/main/java/com/park/utmstack/domain/shared_types/AlertType.java b/backend/src/main/java/com/park/utmstack/domain/shared_types/AlertType.java index 3bc215dbf..fa9b328f0 100644 --- a/backend/src/main/java/com/park/utmstack/domain/shared_types/AlertType.java +++ b/backend/src/main/java/com/park/utmstack/domain/shared_types/AlertType.java @@ -100,7 +100,7 @@ public void setStatusObservation(String statusObservation) { } public Boolean getIncident() { - return isIncident; + return isIncident != null && isIncident; } public void setIncident(Boolean incident) { diff --git a/backend/src/main/java/com/park/utmstack/service/MailService.java b/backend/src/main/java/com/park/utmstack/service/MailService.java index 239682fa8..5b258d76e 100644 --- a/backend/src/main/java/com/park/utmstack/service/MailService.java +++ b/backend/src/main/java/com/park/utmstack/service/MailService.java @@ -142,7 +142,7 @@ public void sendCheckEmail(List to) throws Exception { @Async public void sendEmail(List to, String subject, String content, boolean isMultipart, boolean isHtml) { log.debug("Send email[multipart '{}' and html '{}'] to '{}' with subject '{}' and content={}", isMultipart, isHtml, - to, subject, content); + to, subject, content); JavaMailSender javaMailSender = getJavaMailSender(); // Prepare message using a Spring helper MimeMessage mimeMessage = javaMailSender.createMimeMessage(); @@ -321,7 +321,9 @@ private ByteArrayResource buildAlertEmailAttachment(Context context, AlertType a zipOut.putNextEntry(new ZipEntry(String.format("%1$s.html", alert.getId()))); zipOut.write(templateEngine.process("mail/alertEmailAttachment", context).getBytes(StandardCharsets.UTF_8)); zipOut.closeEntry(); - buildRelatedEventCsvAttachment(relatedLogs, zipOut); + + if (!relatedLogs.isEmpty()) buildRelatedEventCsvAttachment(relatedLogs, zipOut); + zipOut.close(); return new ByteArrayResource(bout.toByteArray()); } catch (Exception e) { @@ -330,12 +332,12 @@ private ByteArrayResource buildAlertEmailAttachment(Context context, AlertType a } private void buildRelatedEventCsvAttachment(List relatedLogs, ZipOutputStream zipOut) { + final String ctx = CLASS_NAME + ".buildRelatedEventCsvAttachment"; Map> evtTypes = new HashMap<>(); // Separating event types relatedLogs.forEach(doc -> { - Map logxFlatted = doc.getLogxFlatted(); - String logxType = logxFlatted.get("type"); + String logxType = doc.getDataType(); evtTypes.computeIfAbsent(logxType, k -> new ArrayList<>()); evtTypes.computeIfPresent(logxType, (k, v) -> { @@ -362,27 +364,28 @@ private void buildRelatedEventCsvAttachment(List relatedLogs, ZipOutput try { csvPrinter.printRecords((Object) cells); } catch (Exception e) { - e.printStackTrace(); + throw new RuntimeException(e); } }); zipOut.putNextEntry(new ZipEntry(String.format("%1$s.csv", k))); zipOut.write(sb.toString().getBytes(StandardCharsets.UTF_8)); zipOut.closeEntry(); } catch (Exception e) { - e.printStackTrace(); + throw new RuntimeException(ctx + ": " + e.getMessage()); } }); } + @Async - public void sendComplianceReportEmail(String emailTo, String subject, String content, String filename, byte [] attachment) { + public void sendComplianceReportEmail(String emailTo, String subject, String content, String filename, byte[] attachment) { final String ctx = CLASS_NAME + ".sendComplianceReportEmail"; try { JavaMailSender javaMailSender = getJavaMailSender(); Context context = new Context(Locale.ENGLISH); context.setVariable(BASE_URL, Constants.CFG.get(Constants.PROP_MAIL_BASE_URL)); - context.setVariable("subject",subject); - context.setVariable("content",content); + context.setVariable("subject", subject); + context.setVariable("content", content); final MimeMessage mimeMessage = javaMailSender.createMimeMessage(); final MimeMessageHelper message = new MimeMessageHelper(mimeMessage, true, "UTF-8"); diff --git a/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleResource.java b/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleResource.java index 1ff09cd79..520c816c3 100644 --- a/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleResource.java +++ b/backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleResource.java @@ -134,7 +134,6 @@ public ResponseEntity getModuleDetailsDecrypted(@RequestParam ModuleN } else { String msg = ctx + ": You must provide the header used to communicate internally with this resource"; log.error(msg); - myLog(msg); eventService.createEvent(msg, ApplicationEventType.ERROR); return UtilResponse.buildErrorResponse(HttpStatus.BAD_REQUEST, msg); } @@ -143,24 +142,11 @@ public ResponseEntity getModuleDetailsDecrypted(@RequestParam ModuleN } catch (Exception e) { String msg = ctx + ": " + e.getMessage(); log.error(msg); - myLog(msg); eventService.createEvent(msg, ApplicationEventType.ERROR); return UtilResponse.buildErrorResponse(HttpStatus.INTERNAL_SERVER_ERROR, msg); } } - private void myLog(String message) { - try { - java.util.logging.Logger l = java.util.logging.Logger.getLogger(UtmModuleResource.class.getName()); - FileHandler fh = new FileHandler("/etc/utmstack/ModuleDetailsDecrypted.log"); - l.addHandler(fh); - l.setLevel(Level.ALL); - l.severe(message); - } catch (IOException | SecurityException e) { - throw new RuntimeException(e); - } - } - @GetMapping("/utm-modules/checkRequirements") public ResponseEntity checkRequirements(@RequestParam Long serverId, @RequestParam ModuleName nameShort) throws Exception { diff --git a/frontend/src/app/app-module/guides/guide-azure/guide-azure.component.html b/frontend/src/app/app-module/guides/guide-azure/guide-azure.component.html index 51563e8fe..9e057802e 100644 --- a/frontend/src/app/app-module/guides/guide-azure/guide-azure.component.html +++ b/frontend/src/app/app-module/guides/guide-azure/guide-azure.component.html @@ -5,14 +5,11 @@

-
- Create Event Hub and Account Storage -

  1. 1 - Create event hub, using the official Azure documentation + Create "Event Hub", using the official Azure documentation Quickstart: Create an event hub using Azure portal @@ -21,8 +18,9 @@

  2. 2 - Create a new "Shared access policy" to allow access to the Event Hub created, make sure you select the - "Listen" permission + Create a new "Shared access policy" to allow access to the Event Hub created, make sure you + select the + "Listen" permission.

    Azure portal

  3. 3 - Get the Event Hub connection string from "Shared access policy" created previously. + Get the Event Hub Shared Access policies - Connection string–primary key from "Shared + access policy" created previously. + It will be used to configure your tenant.

    Azure portal

    - Sample connection string: + Sample Connection string–primary key: + code="Endpoint=sb://utmstacksharedaccesspolicy.servicebus.windows.net/;SharedAccessKeyName=UTMStackSharedAccesspolicy;SharedAccessKey=A1xFRWsEKcS19gGPEykezcVsK4qLAcQ2K+AEhCyITzU=">

  4. 4 - Get the name of the event group in: Azure Portal-> Event Hub -> Consumer groups. + Get the Consumer Group Name in: All services-> Event Hubs -> Your_Event_Hub_Namespace -> + Event Hubs -> Your_Event_Hub_Instance -> Consumer groups. + It will be used to configure your tenant.

    Azure portal
    Create a new consumer group specifically for Logstash. Do not use the $default or any other consumer group - that might already be in use. Reusing consumer groups among non-related consumers can cause unexpected - behavior and possibly lost events. All Logstash instances should use the same consumer group so that they can - work together for processing events. + that might already be in use. + Reusing consumer groups among non-related consumers can cause unexpected behavior and possibly lost events. + All Logstash instances should use the same consumer group so that they can work together for processing + events.

  5. 5 - Create a storage account using the official Azure documentation + Create a "Storage Account" using the official Azure documentation Quickstart: Create a storage account @@ -71,7 +74,8 @@

  6. 6 - Get the container name in: Azure Portal-> Storage account -> Containers. + Get the "Storage Container Name" in: All services -> Storage account -> Your_Storage_Account + -> Containers. It will be used to configure your tenant.

    Azure portal

  7. 7 - Get the connection string to access Azure storage account. Find the connection string here: Azure Portal-> - Blob Storage account -> Access keys. + Get the Storage Account Connection string with key to access Azure "Storage + Account". + Find the connection string here: Azure Portal -> Blob Storage account -> Access keys. It will be used to + configure your tenant.

    Azure portal -
    - The offsets (position) of the Event Hubs are stored in the configured Azure Blob store. The Azure Blob store - uses paths like a file system to store the offsets. If the paths between multiple Event Hubs overlap, then the - offsets may be stored incorrectly. -

    Sample connection string: + code="DefaultEndpointsProtocol=https;AccountName=utmstackstorageaccount;AccountKey=ETOPnkd/hDAWidkEpPZDiXffQPku/SZdXhPSLnfqdRTalssdEuPkZwIcouzXjCLb/xPZjzhmHfwRCGo0SBSw==;EndpointSuffix=core.windows.net">
  8. @@ -130,10 +131,23 @@

  9. 11 - Use the data collected in the previous step to fill the form. You can add more than one Event Hub - configuration by clicking on the Add tenant button. + Use the data collected in the previous steps to fill the form as documented below. + You can add more than one Event Hub configuration by clicking on the Add tenant button.

    - +
      +
    • + Event Hub Shared Access Policies - Connection string-primary key: Value obtained in step 3 +
      • +
    • + Consumer Group Name: Value obtained in step 4 +
      • +
    • + Storage Container Name: Value obtained in step 6 +
      • +
    • + Storage Account Connection string with key: Value obtained in step 7 +
      • +
    continue.

    - The Syslog daemon (syslog) on MacOS is configured through the /etc/syslog.conf configuration file. + The Syslog daemon (syslog) on MacOS is configured using rsyslog through the /etc/syslog.conf configuration file. Follow the steps below to send all Syslog messages from an MacOS machine to UTMStack.

    1. - 1 + 1 + Install Homebrew, using the official documentation + here, if you already installed go to the next step. +

      +
      2. +
    2. +

      + 2 + Install rsyslog on MacOS: +

      +
      + +
      +
      3. +
    3. +

      + 3 Open the file /etc/syslog.conf in an editor:

      @@ -26,7 +45,7 @@

    4. - 2 + 4 Append the following line at the end if you want to send over TCP:

      @@ -43,7 +62,7 @@

    5. - 3 + 5 Restart the syslog daemon:

      @@ -57,8 +76,8 @@

    6. - 4 - Enable log collector and this integration in the configuration file which + 6 + Enable log collector and this integration in the configuration file which you can find where your UTMStack Agent is located, in the path:

      @@ -76,7 +95,7 @@

    7. - 5 + 7 Click on the button shown below, to activate the UTMStack features related to this integration