Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Increase SQLite busy timeout and reduce the number of SQLite connections #2559

Merged
merged 1 commit into from
Nov 29, 2019
Merged

Increase SQLite busy timeout and reduce the number of SQLite connections #2559

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions lib/OpenQA/CacheService.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ use OpenQA::CacheService::Model::Cache;

use constant DEFAULT_MINION_WORKERS => 5;

has cache => sub { OpenQA::CacheService::Model::Cache->from_worker(log => shift->log) };

sub startup {
my $self = shift;

Expand All @@ -35,8 +37,17 @@ sub startup {
my $log = $self->log;
$log->unsubscribe('message') if $ENV{OPENQA_CACHE_SERVICE_QUIET};

my $db_file = OpenQA::CacheService::Model::Cache->from_worker(log => $log)->db_file;
$self->plugin(Minion => {SQLite => "sqlite:$db_file"});
# Increase busy timeout to 5 minutes
my $cache = $self->cache;
my $sqlite = $cache->sqlite;
$sqlite->on(
connection => sub {
my ($sqlite, $dbh) = @_;
$dbh->sqlite_busy_timeout(360000);
});
$cache->init;

$self->plugin(Minion => {SQLite => $sqlite});
$self->plugin('Minion::Admin');
$self->plugin('OpenQA::CacheService::Task::Asset');
$self->plugin('OpenQA::CacheService::Task::Sync');
Expand Down
38 changes: 13 additions & 25 deletions lib/OpenQA/CacheService/Model/Cache.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,13 @@ use Mojo::File 'path';
use Mojo::Log;
use POSIX;

has [qw(host cache location db_file dsn)];
has [qw(cache location dsn)];
has limit => 50 * (1024**3);
has log => sub { Mojo::Log->new };
has sleep_time => 5;
has sqlite => sub { Mojo::SQLite->new(shift->dsn) };
has sqlite => sub { Mojo::SQLite->new };
has ua => sub { Mojo::UserAgent->new(max_redirects => 2, max_response_size => 0) };

sub new { shift->SUPER::new(@_)->init }

sub from_worker {
my $class = shift;

Expand Down Expand Up @@ -63,8 +61,7 @@ sub init {
my $log = $self->log;

my $db_file = path($location, 'cache.sqlite');
$self->db_file($db_file);
$self->dsn("sqlite:$db_file");
$self->sqlite->from_string("sqlite:$db_file");
$self->_deploy_cache unless -e $db_file;
eval { $self->sqlite->migrations->name('cache_service')->from_data->migrate };
if (my $err = $@) {
Expand All @@ -80,12 +77,12 @@ sub init {
}

sub _download_asset {
my ($self, $id, $type, $asset, $etag) = @_;
my ($self, $host, $id, $type, $asset, $etag) = @_;

my $log = $self->log;

my $ua = $self->ua;
my $url = sprintf '%s/tests/%d/asset/%s/%s', $self->host, $id, $type, basename($asset);
my $url = sprintf '%s/tests/%d/asset/%s/%s', $host, $id, $type, basename($asset);
$log->info('Downloading "' . basename($asset) . qq{" from "$url"});

# Keep temporary files on the same partition as the cache
Expand Down Expand Up @@ -147,11 +144,11 @@ sub _download_asset {
}

sub get_asset {
my ($self, $job, $asset_type, $asset) = @_;
my ($self, $host, $job, $asset_type, $asset) = @_;

my $log = $self->log;

my $location = path($self->location, base_host($self->host));
my $location = path($self->location, base_host($host));
$location->make_path unless -d $location;
$asset = $location->child(path($asset)->basename);

Expand All @@ -161,7 +158,7 @@ sub get_asset {
my $result = $self->asset($asset);

my $ret;
eval { $ret = $self->_download_asset($job->{id}, lc($asset_type), $asset, $result->{etag}) };
eval { $ret = $self->_download_asset($host, $job->{id}, lc($asset_type), $asset, $result->{etag}) };
last unless $ret;

if ($ret =~ /^5[0-9]{2}$/ && --$n) {
Expand Down Expand Up @@ -189,11 +186,8 @@ sub track_asset {
my ($self, $asset) = @_;

eval {
my $db = $self->sqlite->db;
my $tx = $db->begin('exclusive');
my $sql = "INSERT OR IGNORE INTO assets (filename, size, last_use) VALUES (?, 0, strftime('%s','now'))";
$db->query($sql, $asset)->arrays;
$tx->commit;
$self->sqlite->db->query($sql, $asset)->arrays;
};
if (my $err = $@) { $self->log->error("Tracking asset failed: $err") }
}
Expand All @@ -202,11 +196,8 @@ sub _update_asset_last_use {
my ($self, $asset) = @_;

eval {
my $db = $self->sqlite->db;
my $tx = $db->begin('exclusive');
my $sql = "UPDATE assets set last_use = strftime('%s','now') where filename = ?";
$db->query($sql, $asset);
$tx->commit;
$self->sqlite->db->query($sql, $asset);
};
if (my $err = $@) {
$self->log->error("Updating last use failed: $err");
Expand Down Expand Up @@ -243,12 +234,9 @@ sub purge_asset {

my $log = $self->log;
eval {
my $db = $self->sqlite->db;
my $tx = $db->begin();
$db->delete('assets', {filename => $asset});
$tx->commit;
if (-e $asset) { $log->error(qq{Unlinking "$asset" failed: $!}) unless unlink $asset }
else { $log->debug(qq{Purging "$asset" failed because the asset did not exist}) }
$self->sqlite->db->delete('assets', {filename => $asset});
if (-e $asset) { $log->error(qq{Unlinking "$asset" failed: $!}) unless unlink $asset }
else { $log->debug(qq{Purging "$asset" failed because the asset did not exist}) }
};
if (my $err = $@) {
$log->error(qq{Purging "$asset" failed: $err});
Expand Down
6 changes: 2 additions & 4 deletions lib/OpenQA/CacheService/Task/Asset.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@
package OpenQA::CacheService::Task::Asset;
use Mojo::Base 'Mojolicious::Plugin';

use OpenQA::CacheService::Model::Cache;

sub register {
my ($self, $app) = @_;

Expand Down Expand Up @@ -51,8 +49,8 @@ sub _cache_asset {
$output .= "[$level] " . join "\n", @lines, '';
});

my $cache = OpenQA::CacheService::Model::Cache->from_worker(log => $ctx);
$cache->host($host)->get_asset({id => $id}, $type, $asset_name);
my $cache = $app->cache->log($ctx)->init;
$cache->get_asset($host, {id => $id}, $type, $asset_name);
$job->note(output => $output);
$ctx->info('Finished download');
}
Expand Down
6 changes: 6 additions & 0 deletions t/25-cache-service.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,12 @@ subtest 'Asset exists' => sub {

};

subtest 'Increased SQLite busy timeout' => sub {
my $cache = OpenQA::CacheService->new;
is $cache->cache->sqlite->db->dbh->sqlite_busy_timeout, 360000, '5 minute busy timeout';
is $cache->minion->backend->sqlite->db->dbh->sqlite_busy_timeout, 360000, '5 minute busy timeout';
};

subtest 'Job progress (guard against parallel downloads of the same file)' => sub {
my $app = OpenQA::CacheService->new;
ok !$app->progress->is_downloading('foo'), 'Queue works';
Expand Down
29 changes: 11 additions & 18 deletions t/25-cache.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,13 +74,7 @@ sub stop_server {
$server_instance->stop();
}

my $cache = OpenQA::CacheService::Model::Cache->new(host => $host, location => $cachedir->to_string, log => $log);

subtest 'base_host' => sub {
my $cache_test
= OpenQA::CacheService::Model::Cache->new(host => $host, location => $cachedir->to_string, log => $log);
is base_host($cache_test->host), 'localhost';
};
my $cache = OpenQA::CacheService::Model::Cache->new(location => $cachedir->to_string, log => $log);

is $cache->init, $cache;
is $cache->sqlite->migrations->latest, 1, 'version 1 is the latest version';
Expand Down Expand Up @@ -119,7 +113,7 @@ like $cache_log, qr/Purging ".*1.qcow2" because we need space for new assets, re
ok(!-e '1.qcow2', 'Oldest asset (1.qcow2) was sucessfully removed');
$cache_log = '';

$cache->get_asset({id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-textmode@64bit.qcow2');
$cache->get_asset($host, {id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-textmode@64bit.qcow2');
my $from = "$host/tests/922756/asset/hdd/sle-12-SP3-x86_64-0368-textmode@64bit.qcow2";
like $cache_log, qr/Downloading "sle-12-SP3-x86_64-0368-textmode\@64bit.qcow2" from "$from"/, 'Asset download attempt';
like $cache_log, qr/failed: 521/, 'Asset download fails with: 521 - Connection refused';
Expand All @@ -129,21 +123,20 @@ $port = Mojo::IOLoop::Server->generate_port;
$host = "http://127.0.0.1:$port";
start_server;

$cache->host($host);
$cache->limit(1024);
$cache->init;

$cache->get_asset({id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-404@64bit.qcow2');
$cache->get_asset($host, {id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-404@64bit.qcow2');
like $cache_log, qr/Downloading "sle-12-SP3-x86_64-0368-404\@64bit.qcow2" from/, 'Asset download attempt';
like $cache_log, qr/failed: 404/, 'Asset download fails with: 404 - Not Found';
$cache_log = '';

$cache->get_asset({id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-400@64bit.qcow2');
$cache->get_asset($host, {id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-400@64bit.qcow2');
like $cache_log, qr/Downloading "sle-12-SP3-x86_64-0368-400\@64bit.qcow2" from/, 'Asset download attempt';
like $cache_log, qr/failed: 400/, 'Asset download fails with 400 - Bad Request';
$cache_log = '';

$cache->get_asset({id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-589@64bit.qcow2');
$cache->get_asset($host, {id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-589@64bit.qcow2');
like $cache_log, qr/Downloading "sle-12-SP3-x86_64-0368-589\@64bit.qcow2" from/, 'Asset download attempt';
like $cache_log, qr/Size of .+ differs, expected 10 but downloaded 6/, 'Incomplete download logged';
like $cache_log, qr/Download error 598, waiting 1 seconds for next try \(4 remaining\)/, '4 tries remaining';
Expand All @@ -153,32 +146,32 @@ like $cache_log, qr/Download error 598, waiting 1 seconds for next try \(1 remai
like $cache_log, qr/Too many download errors, aborting/, 'Bailing out after too many retries';
$cache_log = '';

$cache->get_asset({id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-503@64bit.qcow2');
$cache->get_asset($host, {id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-503@64bit.qcow2');
like $cache_log, qr/Downloading "sle-12-SP3-x86_64-0368-503\@64bit.qcow2" from/, 'Asset download attempt';
like $cache_log, qr/Downloading ".*0368-503\@64bit.qcow2" failed with server error 503/,
'Asset download fails with 503 - Server not available';
like $cache_log, qr/Download error 503, waiting 1 seconds for next try \(4 remaining\)/, '4 tries remaining';
like $cache_log, qr/Too many download errors, aborting/, 'Bailing out after too many retries';
$cache_log = '';

$cache->get_asset({id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-200@64bit.qcow2');
$cache->get_asset($host, {id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-200@64bit.qcow2');
like $cache_log, qr/Downloading "sle-12-SP3-x86_64-0368-200\@64bit.qcow2" from/, 'Asset download attempt';
like $cache_log, qr/Download of ".*sle-12-SP3-x86_64-0368-200.*" successful, new cache size is 1024/,
'Full download logged';
like $cache_log, qr/Size of .* is 1024, with ETag "andi \$a3, \$t1, 41399"/, 'Etag and size are logged';
$cache_log = '';

$cache->get_asset({id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-200@64bit.qcow2');
$cache->get_asset($host, {id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-200@64bit.qcow2');
like $cache_log, qr/Downloading "sle-12-SP3-x86_64-0368-200\@64bit.qcow2" from/, 'Asset download attempt';
like $cache_log, qr/Content of ".*0368-200@64bit.qcow2" has not changed, updating last use/, 'Content has not changed';
$cache_log = '';

$cache->get_asset({id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-200@64bit.qcow2');
$cache->get_asset($host, {id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-200@64bit.qcow2');
like $cache_log, qr/Downloading "sle-12-SP3-x86_64-0368-200\@64bit.qcow2" from/, 'Asset download attempt';
like $cache_log, qr/Content of ".*-0368-200@64bit.qcow2" has not changed, updating last use/, 'Content has not changed';
$cache_log = '';

$cache->get_asset({id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-200_256@64bit.qcow2');
$cache->get_asset($host, {id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-200_256@64bit.qcow2');
like $cache_log, qr/Downloading "sle-12-SP3-x86_64-0368-200_256\@64bit.qcow2" from/, 'Asset download attempt';
like $cache_log, qr/Download of ".*sle-12-SP3-x86_64-0368-200_256.*" successful, new cache size is 256/,
'Full download logged';
Expand Down Expand Up @@ -241,7 +234,7 @@ subtest 'cache tmp directory is used for downloads' => sub {
});
});
local $ENV{MOJO_MAX_MEMORY_SIZE} = 1;
$cache->get_asset({id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-200@64bit.qcow2');
$cache->get_asset($host, {id => 922756}, 'hdd', 'sle-12-SP3-x86_64-0368-200@64bit.qcow2');
is path($tmpfile)->dirname, path($cache->location, 'tmp'), 'cache tmp directory was used';
};

Expand Down