From 289c41ccaea06196c0830b86f0d23395b7e2cb62 Mon Sep 17 00:00:00 2001 From: Jeremy Mitchell Date: Tue, 7 Mar 2017 12:37:53 -0700 Subject: [PATCH 1/3] simplifies config file / snapshot compare script --- test/traffic_ops_cfg/Readme.md | 21 ++-- test/traffic_ops_cfg/cfg_test.pl | 160 +------------------------------ test/traffic_ops_cfg/test.config | 19 +--- 3 files changed, 18 insertions(+), 182 deletions(-) diff --git a/test/traffic_ops_cfg/Readme.md b/test/traffic_ops_cfg/Readme.md index c10c0f8135..cf83b5c6a0 100644 --- a/test/traffic_ops_cfg/Readme.md +++ b/test/traffic_ops_cfg/Readme.md @@ -1,15 +1,18 @@ -# Traffic Ops Config Test +# Traffic Ops Config File / Snapshot Compare -This test allows you to check all generated config files from Traffic Ops. +This test allows you to compare all generated config files and CDN snapshots (CRConfig.json) from 2 instances of Traffic Ops. For example, you could compare config files / snapshots of a MySQL vs Postgres Traffic Ops. You could even compare across releases (1.7.0 vs 1.8.0). -*How to Test* +*Prerequisites* -1. Make sure no changes are made in the reference system, and all changes are "snapped" and "queued". -2. Get a copy of the reference DB (using tools->DB Dump), and save it. -3. Get the files from the reference system generates by running `./cfg_test.pl getref test.config` - make sure `test.config` is right. It will prompt you for the user's passwd, and get all files for all profiles into `/tmp/files/ref`. -4. Load the database into your new system. run migrations (includeing mysql -> postgres) , and move the riak data to your test system from the ref system. -5. `./cfg_test.pl getnew test.config` your new files go into `/tmp/files/new` -6. `./cfg_test.pl compare test.config` - all `not ok` lines should be explained. +1. Make sure the data in your databases are synced to avoid getting false positives. +2. Queue updates for all servers in both instances. +3. Modify test.config with proper settings. Set perform_snapshot=1 if you want to force a snapshot in both instances. + +*Running the Test* + +1. `./cfg_test.pl getref test.config` your ref files go into `/tmp/files/ref` +2. `./cfg_test.pl getnew test.config` your new files go into `/tmp/files/new` +3. `./cfg_test.pl compare test.config` - all `not ok` lines should be explained. It will compare all files for all profiles, _including_ the CRConfig.json. diff --git a/test/traffic_ops_cfg/cfg_test.pl b/test/traffic_ops_cfg/cfg_test.pl index 3eac8e7eb0..852aa97316 100755 --- a/test/traffic_ops_cfg/cfg_test.pl +++ b/test/traffic_ops_cfg/cfg_test.pl @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - -# Some misc subs to help with testing. # use warnings; use strict; @@ -31,16 +29,15 @@ use List::Compare; my $config; -my $ref_db_file = "/tmp/to_ref.mysql"; my $tmp_dir_base = "/tmp/files"; my $tmp_dir = $tmp_dir_base . "/ref"; my $CURL_OPTS; my $cookie; -my $gen_crconfig = 1; # TODO parameter - &configure( $ARGV[1] ); +my $perform_snapshot = $config->{perform_snapshot}; + if ( $ARGV[0] eq "getref" ) { &get_ref(); } @@ -54,47 +51,6 @@ print "Help\n"; } -if (0) { - &build_rpms(); - &build_runner_containers(); - &load_mysql_database(); - &pg_migrate(); - - # get a cookie from the reference system; cookie and CURL_OPTs are globals - # if ( !defined( $config->{ref_to_passwd} ) ) { - # $config->{ref_to_passwd} = &get_to_passwd( $config->{ref_to_user} ); - # } - # my $to_login = $config->{ref_to_user} . ":" . $config->{ref_to_passwd}; - # $cookie = &get_cookie( $config->{ref_to_url}, $to_login ); - # $CURL_OPTS = "-H 'Cookie: $cookie' -w %{response_code} -k -L -s -S --connect-timeout 5 --retry 5 --retry-delay 5 --basic"; - # - # &get_files( $config->{ref_to_url} ); - # &get_crconfigs( $config->{ref_to_url} ); - # -## start riak; it needs to be available at 127.0.0.1:8088 -## update server set status=1 where type=41; -## insert into server (host_name, domain_name, type, ip_address, ip_gateway, ip_netmask, profile, tcp_port,interface_name, phys_location, cachegroup,status, cdn_id) values ('riak-local', 'cdnlab.comcast.net', 41, '127.0.0.1', '127.0.0.2', '255.255.255.0', 51, 8088,'eth0', 1, 1, 2, 2); -## /etc/hosts: 127.0.0.1 localhost riak-local.cdnlab.comcast.net - # - # # start morbo like export `MOJO_MODE=test ./bin/start.pl` - # # get a cookie from the system we're testing; cookie and CURL_OPTs are globals - # - # if ( !defined( $config->{to_passwd} ) ) { - # $config->{to_passwd} = &get_to_passwd( $config->{to_user} ); - # } - # my $to_login = $config->{to_user} . ":" . $config->{to_passwd}; - # $cookie = &get_cookie( $config->{ref_to_url}, $to_login ); - # $CURL_OPTS = "-H 'Cookie: $cookie' -w %{response_code} -k -L -s -S --connect-timeout 5 --retry 5 --retry-delay 5 --basic"; - # - # $tmp_dir = $tmp_dir_base . "/new"; - # &get_files( $config->{to_url} ); - # &get_crconfigs( $config->{to_url} ); - # - # # - # &compare_all_files(); - # -} - sub get_ref { # get a cookie from the reference system; cookie and CURL_OPTs are globals @@ -130,8 +86,6 @@ sub get_new { sub do_the_compare { &compare_all_files(); } -# -#&compare_files( "/tmp/files/ref/odol-atsec-uta-14/parent.config", "/tmp/files/new/odol-atsec-uta-14/parent.config" ); done_testing(); exit(0); @@ -263,10 +217,6 @@ sub compare_files { } } -sub copy_riak_config { - -} - sub get_crconfigs { my $to_url = shift; @@ -279,7 +229,7 @@ sub get_crconfigs { next unless $cdn->{name} ne "ALL"; my $dir = $tmp_dir . '/cdn-' . $cdn->{name}; system( 'mkdir -p ' . $dir ); - if ($gen_crconfig) { + if ($perform_snapshot) { print "Generating CRConfig for " . $cdn->{name}; my $start = [gettimeofday]; &curl_me( $to_url . "/tools/write_crconfig/" . $cdn->{name} ); @@ -390,52 +340,6 @@ sub get_files_new { } } -sub load_mysql_database { - my $cmd = "mysql "; - my $args = "-h " . $config->{mysql_db_host} . " -u " . $config->{mysql_dbadmin_user} . " -p" . $config->{mysql_dbadmin_passwd}; - $cmd .= $args; - my $bash_cmd = "echo drop database " . $config->{mysql_db_name} . " | " . $cmd; - print $bash_cmd . "\n"; - system($bash_cmd ); - $bash_cmd = "echo create database " . $config->{mysql_db_name} . " | " . $cmd; - print $bash_cmd . "\n"; - system($bash_cmd ); - $bash_cmd = $cmd . " " . $config->{mysql_db_name} . " < " . $ref_db_file; - print $bash_cmd . "\n"; - system($bash_cmd ); -} - -# perform the migration from mysql -> postgres -sub pg_migrate { - - my $drop_cmd = "echo drop database " . $config->{pg_db_name} . " | psql postgres"; - print $drop_cmd . "\n"; - system($drop_cmd); - - my $cr_cmd = "echo create database " . $config->{pg_db_name} . " | psql postgres"; - print $cr_cmd . "\n"; - system($cr_cmd); - - my $cmd = "pgloader --cast 'type tinyint to smallint drop typemod' --cast 'type varchar to text drop typemod'"; - $cmd .= " --cast 'type double to numeric drop typemod'"; - my $args = - " mysql://" - . $config->{mysql_dbadmin_user} . ":" - . $config->{mysql_dbadmin_passwd} . "@" - . $config->{mysql_db_host} . ":" - . $config->{mysql_db_port}; - $args .= "/" . $config->{mysql_db_name}; - $args .= " postgresql://" . "/" . $config->{pg_db_name}; # TODO add username / passwd if you have it. - $cmd .= $args; - print $cmd . "\n"; - system($cmd); - chdir( $config->{working_dir} . "/traffic_ops/app" ) || die "can't chdir to " . $config->{working_dir} . "/traffic_ops/app"; - $cmd = "psql " . $config->{pg_db_name} . "< db/convert_bools.sql"; - system($cmd); - $cmd = "goose -env=" . $config->{goose_env} . " up"; - system($cmd); -} - sub get_to_passwd { my $user = shift; @@ -447,59 +351,6 @@ sub get_to_passwd { return $passwd; } -# TODO JvD: finish -sub get_reference_database { - - #"https://tm.comcast.net/dbdump?filename=to-backup-ipcdn-tools-03.cdnlab.comcast.net-20170114222140.mysql - -} - -# TODO - need to get postgres container -sub start_runner_containers { - - #docker run --name my-traffic-vault --hostname my-traffic-vault --net cdnet --env ADMIN_PASS=riakadminsecret --env USER_PASS=marginallylesssecret - # --env CERT_COUNTRY=US --env CERT_STATE=Colorado --env CERT_CITY=Denver --env CERT_COMPANY=NotComcast --env TRAFFIC_OPS_URI=http://my-traffic-ops:3000 - # --env TRAFFIC_OPS_USER=superroot --env TRAFFIC_OPS_PASS=supersecreterpassward --env DOMAIN=cdnet --detach traffic_vault:1.6.0 - # at some point you'll have to have done `docker network create cdnet` - my $tv_args = " --name traffic-vault --hostname traffic-vault --net cdnet --env ADMIN_PASS=riakadminsecret"; - $tv_args .= " --env USER_PASS=marginallylesssecret --env CERT_COUNTRY=US --env CERT_STATE=Colorado --env CERT_CITY=Denver"; - $tv_args .= " --env CERT_COMPANY=NotComcast --env TRAFFIC_OPS_URI=http://my-traffic-ops:3000 --env TRAFFIC_OPS_USER=superroot"; - $tv_args .= " --env TRAFFIC_OPS_PASS=supersecreterpassward --env DOMAIN=cdnet --detach traffic_vault:" . $config->{git_branch}; - - my $to_args = - " --name traffic-ops --hostname my-traffic-ops --net cdnet --publish 443:443 --env MYSQL_IP=my-traffic-ops-mysql --env MYSQL_PORT=3306 --env MYSQL_ROOT_PASS=secretrootpass --env MYSQL_TRAFFIC_OPS_PASS=supersecretpassword --env ADMIN_USER=superroot --env ADMIN_PASS=supersecreterpassward --env CERT_COUNTRY=US --env CERT_STATE=Colorado --env CERT_CITY=Denver --env CERT_COMPANY=NotComcast --env TRAFFIC_VAULT_PASS=marginallylesssecret --env DOMAIN=cdnet --detach traffic_ops:1.5.1" - -} - -sub build_rpms { - my $dir = $config->{working_dir}; - chdir($dir) || die( "Can't chdir to " . $dir ); - - $ENV{'BRANCH'} = $config->{git_branch}; - $ENV{'GITREPO'} = $config->{git_repo}; - - foreach my $builder (qw/traffic_monitor_build traffic_ops_build traffic_portal_build traffic_router_build traffic_stats_build/) { - system( "docker-compose -f infrastructure/docker/build/docker-compose.yml up " . $builder ); - } -} - -sub build_runner_containers { - foreach my $runner (qw/traffic_monitor traffic_ops traffic_router traffic_stats/) { - my $dir = $config->{working_dir} . "/infrastructure/docker/" . $runner; - chdir($dir) || die( "Can't chdir to " . $dir ); - my $rpm_filename = $runner . "*" . "el7.x86_64.rpm"; - my $rpm = $config->{working_dir} . "/infrastructure/docker/build/artifacts/" . $rpm_filename; - my $cp_cmd = "cp " . $rpm . " ."; - print $cp_cmd . " \n"; - system($cp_cmd); - my $branch = $config->{git_branch}; - my $args = " --rm --build-arg RPM=" . $rpm_filename . " --tag " . $runner . ":" . $branch . " ."; - my $dbuild_cmd = "docker build " . $args; - print $dbuild_cmd. "\n"; - system($dbuild_cmd ); - } -} - # read the config json. sub configure { my $filename = shift; @@ -512,12 +363,7 @@ sub configure { }; my $json = JSON->new; - - #print Dumper($config); $config = $json->decode($json_text); - - #print Dumper($config); - } ## rest is from other scripts, should probably be replaced by something better. diff --git a/test/traffic_ops_cfg/test.config b/test/traffic_ops_cfg/test.config index cec7d9066b..6ceb72eaf8 100644 --- a/test/traffic_ops_cfg/test.config +++ b/test/traffic_ops_cfg/test.config @@ -1,20 +1,7 @@ { - "ref_to_user": "jvando001", + "ref_to_user": "my-username", "ref_to_url": "https://to.kabletown.net", - "to_user": "jvd", + "to_user": "my-username", "to_url": "http://localhost:3000", - "working_dir": "/Users/jvd/work/gh/knutsel/incubator-trafficcontrol", - "mysql_dbadmin_user": "root", - "mysql_dbadmin_passwd": "secretrootpass", - "mysql_db_host": "127.0.0.1", - "mysql_db_port": "3306", - "mysql_db_name": "to_test", - "pg_db_name": "to_test", - "pg_dbadmin_user": "", - "pg_dbadmin_passwd": "", - "pg_db_host": "127.0.0.1", - "pg_db_port": "5432", - "goose_env": "test", - "git_branch": "cdn-table-1", - "git_repo": "https://github.com/knutsel/incubator-trafficcontrol" + "perform_snapshot": 0 } From 871b96c2e83ca29b8049c242afe62afd83e8d79b Mon Sep 17 00:00:00 2001 From: Jeremy Mitchell Date: Thu, 9 Mar 2017 15:56:19 -0700 Subject: [PATCH 2/3] fixes bug. wrong config param being used. --- test/traffic_ops_cfg/cfg_test.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/traffic_ops_cfg/cfg_test.pl b/test/traffic_ops_cfg/cfg_test.pl index 852aa97316..52482609f9 100755 --- a/test/traffic_ops_cfg/cfg_test.pl +++ b/test/traffic_ops_cfg/cfg_test.pl @@ -72,7 +72,7 @@ sub get_new { $config->{to_passwd} = &get_to_passwd( $config->{to_user} ); } my $to_login = $config->{to_user} . ":" . $config->{to_passwd}; - $cookie = &get_cookie( $config->{ref_to_url}, $to_login ); + $cookie = &get_cookie( $config->{to_url}, $to_login ); $CURL_OPTS = "-H 'Cookie: $cookie' -w %{response_code} -k -L -s -S --connect-timeout 5 --retry 5 --retry-delay 5 --basic"; $tmp_dir = $tmp_dir_base . "/new"; From 99359de92d4b9b73e92890809989c580f26094fa Mon Sep 17 00:00:00 2001 From: Jeremy Mitchell Date: Fri, 10 Mar 2017 09:37:53 -0700 Subject: [PATCH 3/3] updates readme --- test/traffic_ops_cfg/Readme.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/traffic_ops_cfg/Readme.md b/test/traffic_ops_cfg/Readme.md index cf83b5c6a0..8b74e6fec1 100644 --- a/test/traffic_ops_cfg/Readme.md +++ b/test/traffic_ops_cfg/Readme.md @@ -4,9 +4,8 @@ This test allows you to compare all generated config files and CDN snapshots (CR *Prerequisites* -1. Make sure the data in your databases are synced to avoid getting false positives. -2. Queue updates for all servers in both instances. -3. Modify test.config with proper settings. Set perform_snapshot=1 if you want to force a snapshot in both instances. +1. Make sure the data in your 2 Traffic Ops databases are synced to avoid getting false positives. +2. Modify test.config with proper settings. Set perform_snapshot=1 if you want to force a snapshot in both Traffic Ops instances. *Running the Test*