updates

diskover-web/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Diskover-web v2 Community Edition Change Log
 
+# [2.0.7] - 2023-01-08
+### fixed
+- Cross-Site Scripting (XSS) vulnerability in nav.php
+- php warning when reloading indices and there is a corrupt index
+- delete index not working on indices page
+### added
+- show multi-fields on help page fields section and filter fields, e.g. field.subfield
+- user alert when trying to sort on a field not found in index or trying to sort on a text field
+- index name to delete prompt when deleting index on indices page
+
+
 # [2.0.6] - 2022-11-06
 ### fixed
 - issue searching for full paths to hidden dot files/folders and files with double extensions (e.g. tar.gz)

diskover-web/public/js/diskover.js

@@ -757,18 +757,17 @@ function clipboardNotice() {
 
 // delete index check
 function checkIndexDel() {
-    var indices = document.getElementsByClassName('indexcheck');
-    var checked = 0;
+    var indices = document.getElementsByName('delindices_arr[]');
+    var checked = document.getElementsByName('delindices_arr[]').length;
+    var indices_names = [];
     Array.from(indices).forEach(function(item, index){
-        if($(item).prop("checked") == true){
-            checked += 1;
-        }
+        indices_names.push(item.value);
     });
     if (checked == 0) {
         alert("select at least one index")
         return false;
     }
-    if (confirm('Are you sure you want to remove the selected indices?')) {
+    if (confirm('Are you sure you want to remove the selected ' + checked + ' indices? (' + indices_names.join(", ") + ')')) {
         // submit form
         $('#form-deleteindex').submit();
     } else {
@@ -785,36 +784,6 @@ function checkForceIndexDel(i) {
     }
 }
 
-// index alias check
-function checkIndexAlias() {
-    var indices = document.getElementsByClassName('indexcheck');
-    var checked = 0;
-    Array.from(indices).forEach(function(item, index){
-        if($(item).prop("checked") == true){
-            checked += 1;
-        }
-    });
-    if (checked == 0) {
-        alert("select at least one index")
-        return false;
-    }
-    if (!$('#aliasname').val()) {
-        alert("no alias name")
-        return false;
-    }
-    if ($('#aliasname').val().substring(0, 9) !== "diskover-" || $('#aliasname').val().length < 10) {
-        console.log($('#aliasname').val().substring(0, 9))
-        alert("alias name needs to start with diskover-")
-        return false;
-    }
-    if (confirm('Are you sure you want to change the selected indices alias?')) {
-        // submit form
-        return true;
-    } else {
-        return false;
-    }
-}
-
 // select index check
 function checkSelectedIndex() {
     var indices = document.getElementsByClassName('indexcheck');
@@ -833,26 +802,16 @@ function checkSelectedIndex() {
 }
 
 function addHidden() {
-    var item = document.getElementsByClassName('indexcheck');
-    if($(item).prop("checked") == true){
-        var id = 'hidden_index_del_' + item.value;
-        $('#form-deleteindex').append('<input type="hidden" name="delindices_arr[]" value="' + item.value + '" id="' + id + '" />');
-    } else {
-        var id = 'hidden_index_del_' + item.value;
-        $('#' + id).remove();
-    }
-}
-
-function toggleHiddenInput(item) {
-    if($(item).prop("checked") == true){
-        var id = 'hidden_index_del_' + item.value;
-        if (!$('#' + id).length) {
+    var indices = document.getElementsByClassName('indexcheck');
+    Array.from(indices).forEach(function(item, index){
+        if($(item).prop("checked") == true){
+            var id = 'hidden_index_del_' + item.value;
             $('#form-deleteindex').append('<input type="hidden" name="delindices_arr[]" value="' + item.value + '" id="' + id + '" />');
+        } else {
+            var id = 'hidden_index_del_' + item.value;
+            $('#' + id).remove();
         }
-    } else {
-        var id = 'hidden_index_del_' + item.value;
-        $('#' + id).remove();
-    }
+    });
 }
 
 function checkSelected() {

diskover-web/public/nav.php

@@ -322,7 +322,7 @@
                 <input type="hidden" name="submitted" value="true" />
                 <input type="hidden" name="p" value="1" />
                 <?php if (isset($_REQUEST['resultsize'])) {
-                    $resultSize = $_REQUEST['resultsize'];
+                    $resultSize = htmlspecialchars($_REQUEST['resultsize']);
                 } elseif (getCookie("resultsize") != "") {
                     $resultSize = getCookie("resultsize");
                 } else {

diskover-web/public/search.php

@@ -54,7 +54,7 @@
     $request = predict_search($searchquery);
 
     // check for path in search query and update paths in session and cookies
-    if (!isset($_GET['path']) && strpos($request, 'parent_path:\/') !== false) {
+    if (empty($_GET['path']) && strpos($request, 'parent_path:\/') !== false && substr_count($request, 'parent_path:\/') === 1) {
         // parse out actual path from es query string
         $pp = explode('parent_path:', $request)[1];
         $pp = preg_replace('/ (AND|OR) .*/i', '', $pp);
@@ -65,7 +65,6 @@
         if (!is_null($rootpath)) {
             $_SESSION['rootpath'] = $rootpath;
             createCookie('rootpath', $rootpath);
-            // set path cookie to update tree
             createCookie('path', $path);
             createCookie('parentpath', getParentDir($path));
             $_GET['path'] = $path;
@@ -144,9 +143,18 @@
         // Send search query to Elasticsearch and get scroll id and first page of results
         $queryResponse = $client->search($searchParams);
     } catch (Missing404Exception $e) {
-        handleError("Selected indices are no longer available.");
+        handleError("Selected indices are no longer available. Please select a different index.");
     } catch (Exception $e) {
-        handleError('ES error: ' . $e->getMessage(), false);
+        // reset sort order and reload page if error contains reason that we can not sort
+        $error_arr = json_decode($e->getMessage(), true);
+        $error_reason = $error_arr['error']['root_cause'][0]['reason'];
+        if (strpos($error_reason, "No mapping found for") !== false) {
+            resetSort('nomapping');
+        } elseif (strpos($error_reason, "Text fields are not optimised") !== false) {
+            resetSort('textfield');
+        } else {
+            handleError('ES error: ' . $e->getMessage(), false);
+        }
     }
 
     // set total hits
@@ -188,6 +196,11 @@
                 }
             }
             arsort($ext_onpage);
+            // clear current scroll window
+            if (!empty($scroll_id)) {
+                $client->clearScroll(array('scroll_id' => $scroll_id));
+                $scroll_id = null;
+            }
             // end loop
             break;
         }
@@ -197,7 +210,7 @@
         $queryResponse = $client->scroll([
             "body" => [
                 "scroll_id" => $scroll_id,
-                "scroll" => "1m"
+                "scroll" => "30s"
             ]
         ]);
 

diskover-web/public/selectindices.php

@@ -401,7 +401,7 @@ function gtag(){dataLayer.push(arguments);}
                                         echo "<tr>
                                         <td>";
                                         if (!$disabled) {
-                                            echo "<input onclick=\"toggleHiddenInput(this)\" type=\"radio\" name=\"index\" id=\"index_" . $val . "\" class=\"indexcheck\" value=\"" . $val . "\" $checked></td>";
+                                            echo "<input onclick=\"addHidden()\" type=\"radio\" name=\"index\" id=\"index_" . $val . "\" class=\"indexcheck\" value=\"" . $val . "\" $checked></td>";
                                         } else {
                                             echo "<a href=\"#\" title=\"force delete\" onclick=\"checkForceIndexDel('" . $val . "')\" class=\"btn btn-xs btn-primary\"><i class=\"far fa-trash-alt\"></i>";
                                         }

diskover-web/src/diskover/Diskover.php

@@ -94,7 +94,9 @@ function getIndicesInfoCurl()
             'total',
             'used',
             'free',
+            'free_percent',
             'available',
+            'available_percent',
             'file_size',
             'file_size_du',
             'file_count',
@@ -114,6 +116,15 @@ function getIndicesInfoCurl()
                 if (!in_array($fieldname, $field_exclusions)) {
                     $fields[] = $fieldname;
                 }
+                // check for multi-field add add additional sub-fields fieldname.subfield
+                if (array_key_exists('properties', $val['mappings']['properties'][$fieldname])) {
+                    foreach ($val['mappings']['properties'][$fieldname]['properties'] as $k => $v) {
+                        $field = $fieldname . '.' . $k;
+                        if (!in_array($field, $field_exclusions)) {
+                            $fields[] = $field;
+                        }
+                    }
+                }
             }
             $indices_curl_info_data[$key] = [
                 'uuid' => $val['settings']['index']['uuid'],

diskover-web/src/diskover/version.php

@@ -17,4 +17,4 @@
 */
 
 // diskover-web version
-$VERSION = '2.0.6 community edition (ce)';
+$VERSION = '2.0.7 community edition (ce)';

diskover/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Diskover v2 Community Edition Change Log
 
+# [2.0.7] - 2023-01-08
+### fixed
+- python error when indexing spaceinfo doc and total disk space > max size for ES long field mapping (AWS storage gateway)
+- exception handling for Elasticsearch exception TransportError during bulk uploads
+- exception handling for close function call for plugins and alt scanners
+### added
+- free_percent and available_percent to spaceinfo doc and to es index mappings
+
+
 # [2.0.6] - 2022-11-06
 ### changed
 - better handling of errors when importing alternate scanner modules

diskover/diskover.py

@@ -31,6 +31,7 @@
 from queue import Queue
 from random import choice
 from elasticsearch.helpers.errors import BulkIndexError
+from elasticsearch.exceptions import TransportError
 
 from diskover_elasticsearch import elasticsearch_connection, \
     check_index_exists, create_index, bulk_upload, tune_index
@@ -40,7 +41,7 @@
     get_file_name, load_plugins, list_plugins, get_plugins_info, set_times, \
     get_mem_usage, get_win_path, rem_win_path
 
-version = '2.0.6 community edition (ce)'
+version = '2.0.7 community edition (ce)'
 __version__ = version
 
 # Windows check
@@ -245,13 +246,22 @@ def close_app():
     # close any plugins
     if plugins_enabled and plugins:
         for plugin in plugins:
-            plugin.close(globals())
+            try:
+                plugin.close(globals())
+            except AttributeError:
+                pass
+            except Exception as e:
+                logger.exception(e, exc_info=1)
+                if logtofile: logger_warn.exception(e, exc_info=1)
     # alt scanner close
     if alt_scanner:
         try:
             alt_scanner.close(globals())
         except AttributeError:
             pass
+        except Exception as e:
+            logger.exception(e, exc_info=1)
+            if logtofile: logger_warn.exception(e, exc_info=1)
     # if any warnings, exit with custom exit code 64 to indicate index finished but with warnings
     if warnings > 0:
         sys.exit(64)
@@ -263,13 +273,25 @@ def close_app_critical_error():
     # close any plugins
     if plugins_enabled and plugins:
         for plugin in plugins:
-            plugin.close(globals())
+            try:
+                plugin.close(globals())
+            except AttributeError:
+                pass
+            except Exception as e:
+                logger.exception(e, exc_info=1)
+                if logtofile: logger_warn.exception(e, exc_info=1)
     # alt scanner close
     if alt_scanner:
         try:
             alt_scanner.close(globals())
         except AttributeError:
             pass
+        except Exception as e:
+            logger.exception(e, exc_info=1)
+            if logtofile: logger_warn.exception(e, exc_info=1)
+    logmsg = 'CRITICAL ERROR EXITING'
+    logger.critical(logmsg)
+    if logtofile: logger_warn.critical(logmsg)
     os._exit(1)
 
 
@@ -285,8 +307,8 @@ def start_bulk_upload(thread, root, docs):
     es_upload_start = time.time()
     try:
         bulk_upload(es, options.index, docs)
-    except BulkIndexError as e:
-        logmsg = '[{0}] FATAL ERROR: Elasticsearch bulk index error! ({1})'.format(thread, e)
+    except (BulkIndexError, TransportError) as e:
+        logmsg = '[{0}] FATAL ERROR: Elasticsearch bulk index/transport error! ({1})'.format(thread, e)
         logger.critical(logmsg, exc_info=1)
         if logtofile: logger_warn.critical(logmsg, exc_info=1)
         close_app_critical_error()

diskover/diskover_elasticsearch.py

@@ -372,9 +372,15 @@ def create_index(indexname, es):
                 'free': {
                     'type': 'long'
                 },
+                'free_percent': {
+                    'type': 'float'
+                },
                 'available': {
                     'type': 'long'
                 },
+                'available_percent': {
+                    'type': 'float'
+                },
                 'file_size': {
                     'type': 'long'
                 },

diskover/diskover_helpers.py

@@ -293,12 +293,22 @@ def index_info_crawlstart(es, index, path, start, ver, altscanner):
         mount_path = path
         if replacepaths:
             mount_path = replace_path(mount_path)
+        # Check if too large for long field mapping used by total
+        maxlongint = 18446744073709551615
+        if total > maxlongint:
+            total = maxlongint
+        if free > maxlongint:
+            free = maxlongint
+        if available > maxlongint:
+            available = maxlongint
         data = {
             'path': mount_path,
             'total': total,
             'used': total - free,
             'free': free,
+            'free_percent': round((total-(total-free))/total*100, 6),
             'available': available,
+            'available_percent': round((total-(total-available))/total*100, 6),
             'type': 'spaceinfo'
         }
         es.index(index=index, body=data)
@@ -336,12 +346,22 @@ def index_info_crawlstart(es, index, path, start, ver, altscanner):
                 available = available_bytes.value
             if replacepaths:
                 mount_path = replace_path(mount_path)
+            # Check if too large for long field mapping used by total
+            maxlongint = 18446744073709551615
+            if total > maxlongint:
+                total = maxlongint
+            if free > maxlongint:
+                free = maxlongint
+            if available > maxlongint:
+                available = maxlongint
             data = {
                 'path': mount_path,
                 'total': total,
                 'used': total - free,
                 'free': free,
+                'free_percent': round((total-(total-free))/total*100, 6),
                 'available': available,
+                'available_percent': round((total-(total-available))/total*100, 6),
                 'type': 'spaceinfo'
             }
             es.index(index=index, body=data)