Refactor node timeout (#2053)

refactor: set a cached down time for dead nodes
UPC · May 2, 2024 · 5db1514 · 5db1514
1 parent df781a0
commit 5db1514
Show file tree

Hide file tree

Showing 5 changed files with 36 additions and 12 deletions.
diff --git a/.github/workflows/github-action-test.yml b/.github/workflows/github-action-test.yml
@@ -7,14 +7,14 @@ jobs:
       - uses: actions/checkout@v2
       - name: tmpfs run user
         run: sudo mkdir -p /run/user/$(id -u) ; sudo chown $(id -u) /run/user/$(id -u)
-      - uses: mirromutth/mysql-action@v1.1
+      - uses: getong/mariadb-action@v1.1
         with:
           character set server: 'utf8' # Optional, default value is 'utf8mb4'. The '--character-set-server' option for mysqld
           collation server: 'utf8_general_ci' # Optional, default value is 'utf8mb4_general_ci'. The '--collation-server' option for mysqld
+          mariadb version: '10.4.10' # Optional, default value is "latest". The version of the MariaDB
           mysql database: 'ravada' # Optional, default value is "test". The specified database which will be create
-          mysql root password: root
-          mysql user: 'rvd_user' # Required if "mysql root password" is empty, default is empty. The superuser for the specified database. Can use secrets, too
-          mysql password: Pword12345*
+          mysql user: 'rvd_user'
+          mysql password: 'Pword12345*'
       - name: Update packages
         run: sudo apt update
       - name: Install auth packages
@@ -34,10 +34,6 @@ jobs:
         run: cat /etc/ravada.conf
       - name: Install mysql perl
         run: sudo apt install libdbd-mysql-perl libhtml-lint-perl
-      - name: create mysql user
-        run: sudo mysql -u root --password=root --protocol=tcp -h localhost -e "create user 'rvd_user'@'%' IDENTIFIED WITH mysql_native_password BY 'Pword12345*'"
-      - name: allow mysql user
-        run: sudo mysql -u root --password=root --protocol=tcp -h localhost -e "grant all on ravada.* to 'rvd_user'@'%'"
       - name: Test mojo grants
         run: prove -l t/mojo/30_grants.t
       - name: Test critic

diff --git a/lib/Ravada.pm b/lib/Ravada.pm
@@ -2880,6 +2880,7 @@ sub _upgrade_tables {
     $self->_upgrade_table('vms', 'clone_storage','varchar(64) DEFAULT NULL');
     $self->_upgrade_table('vms','dir_backup','varchar(128) DEFAULT NULL');
     $self->_upgrade_table('vms','version','varchar(64) DEFAULT NULL');
+    $self->_upgrade_table('vms','cached_down','int DEFAULT 0');
 
     $self->_upgrade_table('requests','at_time','int(11) DEFAULT NULL');
     $self->_upgrade_table('requests','pid','int(11) DEFAULT NULL');
@@ -3039,7 +3040,7 @@ sub _connect_dbh($self=undef) {
 
         return $con if $con && !$err;
         sleep 1;
-        warn "Try $try $@\n";
+        warn "Try $try $data_source $@\n";
     }
     die ($@ or "Can't connect to $driver $db at $host");
 }

diff --git a/lib/Ravada/VM.pm b/lib/Ravada/VM.pm
@@ -46,6 +46,8 @@ our $MIN_DISK_MB = 1024 * 1024;
 our $CACHE_TIMEOUT = 60;
 our $FIELD_TIMEOUT = '_data_timeout';
 
+our $TIMEOUT_DOWN_CACHE = 120;
+
 our %VM; # cache Virtual Manager Connection
 our %SSH;
 
@@ -343,7 +345,21 @@ sub _connect {
     return $result;
 }
 
+=head2 timeout_down_cache
+
+  Returns time in seconds for nodes to be kept cached down.
+
+=cut
+
+sub timeout_down_cache($self) {
+    return $TIMEOUT_DOWN_CACHE;
+}
+
 sub _around_connect($orig, $self) {
+
+    if ($self->_data('cached_down') && time-$self->_data('cached_down')< $self->timeout_down_cache()) {
+            return;
+    }
     my $result = $self->$orig();
     if ($result) {
         $self->is_active(1);

diff --git a/lib/Ravada/VM/KVM.pm b/lib/Ravada/VM/KVM.pm
@@ -109,6 +109,9 @@ sub _connect {
     } else {
         confess "Error: You can't connect to remote VMs in readonly mode"
             if $self->readonly;
+        if ($self->_data('cached_down') && time-$self->_data('cached_down')<$self->timeout_down_cache()) {
+            return;
+        }
         my $transport = 'ssh';
         my $address = $con_type."+".$transport
                                             ."://".'root@'.$self->host
@@ -123,7 +126,16 @@ sub _connect {
                               ]
                           );
          };
-         confess $@ if $@;
+        my $error = $@;
+        my $is_alive;
+        eval { $is_alive = $vm->is_alive if $vm };
+        warn $@ if $@;
+        if ( !$vm || !$is_alive ) {
+            $self->_data('cached_down' => time);
+            confess $error if $error;
+            return;
+        }
+        $self->_data('cached_down' => 0);
     }
     return $vm;
 }

diff --git a/t/lib/Test/Ravada.pm b/t/lib/Test/Ravada.pm
@@ -1726,8 +1726,7 @@ sub remove_old_storage_pools() {
     remove_old_storage_pools_void();
 }
 
-sub clean($ldap=undef) {
-    my $file_remote_config = shift;
+sub clean($ldap=undef, $file_remote_config=undef) {
     remove_old_domains();
     remove_old_disks();
     remove_old_pools();