Updating azure guide on dev

.github/workflows/agent.yml

@@ -15,6 +15,16 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
 
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.21
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.12'
+
       - name: Build and sign agent services
         id: set-env
         run: |

agent/agent/main.go

@@ -29,7 +29,7 @@ func main() {
 		h.FatalError("Failed to get current path: %v", err)
 	}
 
-	// Configuring log saving
+	// Configuring  log saving
 	var logger = utils.CreateLogger(filepath.Join(path, "logs", configuration.SERV_LOG))
 	defer logger.Close()
 	log.SetOutput(logger)
@@ -83,7 +83,7 @@ func main() {
 				h.FatalError("error configuring syslog server: %v", err)
 			}
 
-			// Install Beats
+			// Install  Beats
 			if err = beats.InstallBeats(*cnf, h); err != nil {
 				fmt.Printf("error installing beats: %v", err)
 				h.FatalError("error installing beats: %v", err)
@@ -96,7 +96,7 @@ func main() {
 			msg := os.Args[2]
 			logp := logservice.GetLogProcessor()
 
-			// Read config
+			// Read  config
 			cnf, err := configuration.GetCurrentConfig()
 			if err != nil {
 				os.Exit(0)

backend/src/main/java/com/park/utmstack/domain/shared_types/AlertType.java

@@ -100,7 +100,7 @@ public void setStatusObservation(String statusObservation) {
     }
 
     public Boolean getIncident() {
-        return isIncident;
+        return isIncident != null && isIncident;
     }
 
     public void setIncident(Boolean incident) {

backend/src/main/java/com/park/utmstack/service/MailService.java

@@ -142,7 +142,7 @@ public void sendCheckEmail(List<String> to) throws Exception {
     @Async
     public void sendEmail(List<String> to, String subject, String content, boolean isMultipart, boolean isHtml) {
         log.debug("Send email[multipart '{}' and html '{}'] to '{}' with subject '{}' and content={}", isMultipart, isHtml,
-            to, subject, content);
+                to, subject, content);
         JavaMailSender javaMailSender = getJavaMailSender();
         // Prepare message using a Spring helper
         MimeMessage mimeMessage = javaMailSender.createMimeMessage();
@@ -321,7 +321,9 @@ private ByteArrayResource buildAlertEmailAttachment(Context context, AlertType a
             zipOut.putNextEntry(new ZipEntry(String.format("%1$s.html", alert.getId())));
             zipOut.write(templateEngine.process("mail/alertEmailAttachment", context).getBytes(StandardCharsets.UTF_8));
             zipOut.closeEntry();
-            buildRelatedEventCsvAttachment(relatedLogs, zipOut);
+
+            if (!relatedLogs.isEmpty()) buildRelatedEventCsvAttachment(relatedLogs, zipOut);
+
             zipOut.close();
             return new ByteArrayResource(bout.toByteArray());
         } catch (Exception e) {
@@ -330,12 +332,12 @@ private ByteArrayResource buildAlertEmailAttachment(Context context, AlertType a
     }
 
     private void buildRelatedEventCsvAttachment(List<LogType> relatedLogs, ZipOutputStream zipOut) {
+        final String ctx = CLASS_NAME + ".buildRelatedEventCsvAttachment";
         Map<String, List<LogType>> evtTypes = new HashMap<>();
 
         // Separating event types
         relatedLogs.forEach(doc -> {
-            Map<String, String> logxFlatted = doc.getLogxFlatted();
-            String logxType = logxFlatted.get("type");
+            String logxType = doc.getDataType();
 
             evtTypes.computeIfAbsent(logxType, k -> new ArrayList<>());
             evtTypes.computeIfPresent(logxType, (k, v) -> {
@@ -362,27 +364,28 @@ private void buildRelatedEventCsvAttachment(List<LogType> relatedLogs, ZipOutput
                     try {
                         csvPrinter.printRecords((Object) cells);
                     } catch (Exception e) {
-                        e.printStackTrace();
+                        throw new RuntimeException(e);
                     }
                 });
                 zipOut.putNextEntry(new ZipEntry(String.format("%1$s.csv", k)));
                 zipOut.write(sb.toString().getBytes(StandardCharsets.UTF_8));
                 zipOut.closeEntry();
             } catch (Exception e) {
-                e.printStackTrace();
+                throw new RuntimeException(ctx + ": " + e.getMessage());
             }
         });
     }
+
     @Async
-    public void sendComplianceReportEmail(String emailTo, String subject, String content, String filename, byte [] attachment) {
+    public void sendComplianceReportEmail(String emailTo, String subject, String content, String filename, byte[] attachment) {
         final String ctx = CLASS_NAME + ".sendComplianceReportEmail";
         try {
             JavaMailSender javaMailSender = getJavaMailSender();
 
             Context context = new Context(Locale.ENGLISH);
             context.setVariable(BASE_URL, Constants.CFG.get(Constants.PROP_MAIL_BASE_URL));
-            context.setVariable("subject",subject);
-            context.setVariable("content",content);
+            context.setVariable("subject", subject);
+            context.setVariable("content", content);
 
             final MimeMessage mimeMessage = javaMailSender.createMimeMessage();
             final MimeMessageHelper message = new MimeMessageHelper(mimeMessage, true, "UTF-8");

backend/src/main/java/com/park/utmstack/web/rest/application_modules/UtmModuleResource.java

@@ -134,7 +134,6 @@ public ResponseEntity<UtmModule> getModuleDetailsDecrypted(@RequestParam ModuleN
             } else {
                 String msg = ctx + ": You must provide the header used to communicate internally with this resource";
                 log.error(msg);
-                myLog(msg);
                 eventService.createEvent(msg, ApplicationEventType.ERROR);
                 return UtilResponse.buildErrorResponse(HttpStatus.BAD_REQUEST, msg);
             }
@@ -143,24 +142,11 @@ public ResponseEntity<UtmModule> getModuleDetailsDecrypted(@RequestParam ModuleN
         } catch (Exception e) {
             String msg = ctx + ": " + e.getMessage();
             log.error(msg);
-            myLog(msg);
             eventService.createEvent(msg, ApplicationEventType.ERROR);
             return UtilResponse.buildErrorResponse(HttpStatus.INTERNAL_SERVER_ERROR, msg);
         }
     }
 
-    private void myLog(String message) {
-        try {
-            java.util.logging.Logger l = java.util.logging.Logger.getLogger(UtmModuleResource.class.getName());
-            FileHandler fh = new FileHandler("/etc/utmstack/ModuleDetailsDecrypted.log");
-            l.addHandler(fh);
-            l.setLevel(Level.ALL);
-            l.severe(message);
-        } catch (IOException | SecurityException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
     @GetMapping("/utm-modules/checkRequirements")
     public ResponseEntity<CheckRequirementsResponse> checkRequirements(@RequestParam Long serverId,
                                                                        @RequestParam ModuleName nameShort) throws Exception {

frontend/src/app/app-module/guides/guide-azure/guide-azure.component.html

@@ -5,14 +5,11 @@ <h4 class="card-title mb-0 text-primary">
     </h4>
   </div>
   <div class="card-body">
-    <h6 class="font-weight-semibold mb-3">
-      Create Event Hub and Account Storage
-    </h6>
     <ol class="setup_list">
       <li>
         <p class="step-guide">
           <span class="step_number">1</span>
-          Create event hub, using the official Azure documentation
+          Create <strong>"Event Hub"</strong>, using the official Azure documentation
           <a class="text-primary font-weight-semibold"
              href="https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-create"
              target="_blank">Quickstart: Create an event hub using Azure portal</a>
@@ -21,8 +18,9 @@ <h6 class="font-weight-semibold mb-3">
       <li>
         <p class="step-guide">
           <span class="step_number">2</span>
-          Create a new "Shared access policy" to allow access to the Event Hub created, make sure you select the
-          "Listen" permission
+          Create a new <strong>"Shared access policy"</strong> to allow access to the Event Hub created, make sure you
+          select the
+          <strong>"Listen"</strong> permission.
         </p>
         <img alt="Azure portal" class="step-img"
              height="auto" src="../../../../assets/img/guides/azure/evenhubaccesspolicy.png"
@@ -31,38 +29,43 @@ <h6 class="font-weight-semibold mb-3">
       <li>
         <p class="step-guide">
           <span class="step_number">3</span>
-          Get the Event Hub connection string from "Shared access policy" created previously.
+          Get the <strong>Event Hub Shared Access policies - Connection string–primary key</strong> from <strong>"Shared
+          access policy"</strong> created previously.
+          It will be used to configure your tenant.
         </p>
         <img alt="Azure portal" class="step-img"
              height="auto" src="../../../../assets/img/guides/azure/aacess_shared_policy.png"
              width="800">
         <br>
         <br>
-        Sample connection string:
+        Sample <strong>Connection string–primary key</strong>:
         <app-utm-code-view
           [allowCopy]="false"
-          code="Endpoint=sb://utmstack.servicebus.windows.net/;SharedAccessKeyName=activity-log-read-only;SharedAccessKey=mm6AbDcEfj8lk7sjsbzoTJ10qAkiSaG663YykEAG2eg=;EntityPath=insights-operational-logs">
+          code="Endpoint=sb://utmstacksharedaccesspolicy.servicebus.windows.net/;SharedAccessKeyName=UTMStackSharedAccesspolicy;SharedAccessKey=A1xFRWsEKcS19gGPEykezcVsK4qLAcQ2K+AEhCyITzU=">
         </app-utm-code-view>
       </li>
       <li>
         <p class="step-guide">
           <span class="step_number">4</span>
-          Get the name of the event group in: Azure Portal-> Event Hub -> Consumer groups.
+          Get the <strong>Consumer Group Name</strong> in: All services-> Event Hubs -> Your_Event_Hub_Namespace ->
+          Event Hubs -> Your_Event_Hub_Instance -> Consumer groups.
+          It will be used to configure your tenant.
         </p>
         <img alt="Azure portal" class="step-img"
              height="auto" src="../../../../assets/img/guides/azure/consumer_group.png"
              width="600">
         <div class="alert alert-info alert-styled-right mt-3">
           Create a new consumer group specifically for Logstash. Do not use the $default or any other consumer group
-          that might already be in use. Reusing consumer groups among non-related consumers can cause unexpected
-          behavior and possibly lost events. All Logstash instances should use the same consumer group so that they can
-          work together for processing events.
+          that might already be in use.
+          Reusing consumer groups among non-related consumers can cause unexpected behavior and possibly lost events.
+          All Logstash instances should use the same consumer group so that they can work together for processing
+          events.
         </div>
       </li>
       <li>
         <p class="step-guide">
           <span class="step_number">5</span>
-          Create a storage account using the official Azure documentation
+          Create a <strong>"Storage Account"</strong> using the official Azure documentation
           <a class="text-primary font-weight-semibold"
              href="https://docs.microsoft.com/en-us/azure/storage/common/storage-account-create?tabs=azure-portal"
              target="_blank">Quickstart: Create a storage account</a>
@@ -71,7 +74,8 @@ <h6 class="font-weight-semibold mb-3">
       <li>
         <p class="step-guide">
           <span class="step_number">6</span>
-          Get the container name in: Azure Portal-> Storage account -> Containers.
+          Get the <strong>"Storage Container Name"</strong> in: All services -> Storage account -> Your_Storage_Account
+          -> Containers. It will be used to configure your tenant.
         </p>
         <img alt="Azure portal" class="step-img"
              height="auto" src="../../../../assets/img/guides/azure/container_name.png"
@@ -80,23 +84,20 @@ <h6 class="font-weight-semibold mb-3">
       <li>
         <p class="step-guide">
           <span class="step_number">7</span>
-          Get the connection string to access Azure storage account. Find the connection string here: Azure Portal->
-          Blob Storage account -> Access keys.
+          Get the <strong>Storage Account Connection string with key</strong> to access Azure <strong>"Storage
+          Account"</strong>.
+          Find the connection string here: Azure Portal -> Blob Storage account -> Access keys. It will be used to
+          configure your tenant.
         </p>
 
         <img alt="Azure portal" class="step-img"
              height="auto" src="../../../../assets/img/guides/azure/account_storage.png"
              width="800">
-        <div class="alert alert-info alert-styled-right mt-3">
-          The offsets (position) of the Event Hubs are stored in the configured Azure Blob store. The Azure Blob store
-          uses paths like a file system to store the offsets. If the paths between multiple Event Hubs overlap, then the
-          offsets may be stored incorrectly.
-        </div>
         <br>
         Sample connection string:
         <app-utm-code-view
           [allowCopy]="false"
-          code="DefaultEndpointsProtocol=https;AccountName=logstash;AccountKey=ETOPnkd/hDAWidkEpPZDiXffQPku/SZdXhPSLnfqdRTalssdEuPkZwIcouzXjCLb/xPZjzhmHfwRCGo0SBSw==;EndpointSuffix=core.windows.net">
+          code="DefaultEndpointsProtocol=https;AccountName=utmstackstorageaccount;AccountKey=ETOPnkd/hDAWidkEpPZDiXffQPku/SZdXhPSLnfqdRTalssdEuPkZwIcouzXjCLb/xPZjzhmHfwRCGo0SBSw==;EndpointSuffix=core.windows.net">
         </app-utm-code-view>
       </li>
       <li>
@@ -130,10 +131,23 @@ <h6 class="font-weight-semibold mb-3">
       <li>
         <p class="step-guide">
           <span class="step_number">11</span>
-          Use the data collected in the previous step to fill the form. You can add more than one Event Hub
-          configuration by clicking on the Add tenant button.
+          Use the data collected in the previous steps to fill the form as documented below.
+          You can add more than one Event Hub configuration by clicking on the Add tenant button.
         </p>
-
+        <ul class="mt-3 pl-3" style="list-style-type: circle !important;">
+          <li>
+            <strong>Event Hub Shared Access Policies - Connection string-primary key: </strong> <u>Value obtained in step 3</u>
+          </li>
+          <li>
+            <strong>Consumer Group Name: </strong> <u>Value obtained in step 4</u>
+          </li>
+          <li>
+            <strong>Storage Container Name: </strong> <u>Value obtained in step 6</u>
+          </li>
+          <li>
+            <strong>Storage Account Connection string with key: </strong> <u>Value obtained in step 7</u>
+          </li>
+        </ul>
         <div class="row mt-3">
           <div class="col-lg-12 col-md-12 col-sm-12">
             <app-int-generic-group-config [moduleId]="integrationId"

frontend/src/app/app-module/guides/guide-macos-agent/guide-macos-agent.component.html

@@ -10,13 +10,32 @@ <h4 class="card-title mb-0 text-primary">
       continue.
     </div>
     <p class="font-size-base text-justify mb-3">
-      The Syslog daemon (syslog) on MacOS is configured through the /etc/syslog.conf configuration file.
+      The Syslog daemon (syslog) on MacOS is configured using <strong>rsyslog</strong> through the /etc/syslog.conf configuration file.
       Follow the steps below to send all Syslog messages from an MacOS machine to UTMStack.
     </p>
     <ol class="setup_list">
       <li>
         <p class="step-guide" >
-            <span class="step_number">1</span>
+          <span class="step_number">1</span>
+          Install Homebrew, using the official documentation
+          <a class="text-primary font-weight-semibold"
+             href="https://docs.brew.sh/Installation"
+             target="_blank">here</a>, if you already installed go to the next step.
+        </p>
+      </li>
+      <li>
+        <p class="step-guide" >
+          <span class="step_number">2</span>
+          Install rsyslog on MacOS:
+        </p>
+        <div class="w-75 mt-3">
+          <app-utm-code-view
+            [code]="'brew install rsyslog'"></app-utm-code-view>
+        </div>
+      </li>
+      <li>
+        <p class="step-guide" >
+            <span class="step_number">3</span>
           Open the file /etc/syslog.conf in an editor:
           </p>
         <div class="w-75 mt-3">
@@ -26,7 +45,7 @@ <h4 class="card-title mb-0 text-primary">
       </li>
       <li>
         <p class="step-guide" >
-          <span class="step_number">2</span>
+          <span class="step_number">4</span>
           Append the following line at the end if you want to send over TCP:
         </p>
         <div class="w-75 mt-3">
@@ -43,7 +62,7 @@ <h4 class="card-title mb-0 text-primary">
       </li>
       <li>
         <p class="step-guide" >
-          <span class="step_number">3</span>
+          <span class="step_number">5</span>
           Restart the syslog daemon:
         </p>
         <div class="w-75 mt-3">
@@ -57,8 +76,8 @@ <h4 class="card-title mb-0 text-primary">
       </li>
       <li>
         <p class="step-guide">
-          <span class="step_number">4</span>
-          Enable log collector and this integration in the configuration file which 
+          <span class="step_number">6</span>
+          Enable log collector and this integration in the configuration file which
           you can find where your UTMStack Agent is located, in the path:
         </p>
         <div class="w-75 mt-3">
@@ -76,7 +95,7 @@ <h4 class="card-title mb-0 text-primary">
       </li>
       <li>
         <p class="step-guide mb-3">
-          <span class="step_number">5</span>
+          <span class="step_number">7</span>
           Click on the button shown below, to activate the UTMStack features related to this integration
         </p>
         <app-app-module-activate-button [module]="filebeatModule" [type]="'integration'"

version.yml

@@ -1 +1 @@
-version: 10.2.3
+version: 10.3.0