From 97da71cb613143c6d2cd5af6c4b6cbd94cddbb74 Mon Sep 17 00:00:00 2001 From: Derek Gelinas Date: Thu, 6 Apr 2017 16:35:27 +0000 Subject: [PATCH 1/5] changes scope for facts, records.config, and header rewrite --- .../lib/API/Configs/ApacheTrafficServer.pm | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm b/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm index b879088e2a..1a0956e27c 100644 --- a/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm +++ b/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm @@ -67,9 +67,12 @@ sub get_config_metadata { my $tm_cache_url = $self->db->resultset('Parameter')->search( { -and => [ name => 'tm_cache.url', config_file => 'global' ] } )->get_column('value')->first(); my $cdn_name = $server_obj->cdn->name; my $server = $rs_server->next; + my $config_file_obj; + my @config_files; if ($server) { $data_obj->{'info'}->{'server_name'} = $server_obj->host_name; + $data_obj->{'info'}->{'server_ipv4'} = $server_obj->ip_address; $data_obj->{'info'}->{'server_id'} = $server_obj->id; $data_obj->{'info'}->{'profile_name'} = $server->profile->name; $data_obj->{'info'}->{'profile_id'} = $server->profile->id; @@ -91,29 +94,36 @@ sub get_config_metadata { my $rs_param = $self->db->resultset('Parameter')->search( \%condition, { join => 'profile_parameters' } ); while ( my $param = $rs_param->next ) { if ( $param->name eq 'location' ) { - $data_obj->{'config_files'}->{ $param->config_file }->{'location'} = $param->value; + $config_file_obj->{ $param->config_file }->{'name'} = $param->config_file; + $config_file_obj->{ $param->config_file }->{'location'} = $param->value; + } } } - foreach my $config_file ( keys %{ $data_obj->{'config_files'} } ) { - $data_obj->{'config_files'}->{$config_file}->{'scope'} = $self->get_scope($config_file); - my $scope = $data_obj->{'config_files'}->{$config_file}->{'scope'}; + foreach my $config_file ( keys %{ $config_file_obj } ) { + my $scope = $self->get_scope($config_file); my $scope_id; if ( $scope eq 'cdn' ) { - $scope_id = $server->cdn->id; + $scope_id = $server->cdn->name; } elsif ( $scope eq 'profile' ) { - $scope_id = $server->profile->id; + $scope_id = $server->profile->name; } else { - $scope_id = $server_obj->id; + $scope_id = $server_obj->host_name; } - $data_obj->{'config_files'}->{$config_file}->{'API_URI'} = "/api/1.2/" . $scope . "/" . $scope_id . "/configfiles/ats/" . $config_file; + $config_file_obj->{$config_file}->{'API_URI'} = "/api/1.2/" . $scope . "/" . $scope_id . "/configfiles/ats/" . $config_file; + $config_file_obj->{$config_file}->{'scope'} = $scope; + } + + foreach my $config_file ( keys %{ $config_file_obj } ) { + push ( @config_files, $config_file_obj->{$config_file} ); } + $data_obj->{'config_files'} = \@config_files; my $file_contents = encode_json($data_obj); return $self->render( text => $file_contents, format => 'txt' ); @@ -144,12 +154,9 @@ sub get_server_config { #generate the config file using the appropriate function my $file_contents; - if ( $filename eq "12M_facts" ) { $file_contents = $self->facts( $server_obj, $filename ); } - elsif ( $filename =~ /to_ext_.*\.config/ ) { $file_contents = $self->to_ext_dot_config( $server_obj, $filename ); } - elsif ( $filename =~ /hdr_rw_.*\.config/ ) { $file_contents = $self->header_rewrite_dot_config( $server_obj, $filename ); } + if ( $filename =~ /to_ext_.*\.config/ ) { $file_contents = $self->to_ext_dot_config( $server_obj, $filename ); } elsif ( $filename eq "ip_allow.config" ) { $file_contents = $self->ip_allow_dot_config( $server_obj, $filename ); } elsif ( $filename eq "parent.config" ) { $file_contents = $self->parent_dot_config( $server_obj, $filename ); } - elsif ( $filename eq "records.config" ) { $file_contents = $self->generic_server_config( $server_obj, $filename ); } elsif ( $filename eq "remap.config" ) { $file_contents = $self->remap_dot_config( $server_obj, $filename ); } elsif ( $filename eq "hosting.config" ) { $file_contents = $self->hosting_dot_config( $server_obj, $filename ); } elsif ( $filename eq "cache.config" ) { $file_contents = $self->cache_dot_config( $server_obj, $filename ); } @@ -205,6 +212,7 @@ sub get_cdn_config { my $file_contents; if ( $filename eq "bg_fetch.config" ) { $file_contents = $self->bg_fetch_dot_config( $cdn_obj, $filename ); } elsif ( $filename =~ /cacheurl.*\.config/ ) { $file_contents = $self->cacheurl_dot_config( $cdn_obj, $filename ); } + elsif ( $filename =~ /hdr_rw_.*\.config/ ) { $file_contents = $self->header_rewrite_dot_config( $cdn_obj, $filename ); } elsif ( $filename =~ /regex_remap_.*\.config/ ) { $file_contents = $self->regex_remap_dot_config( $cdn_obj, $filename ); } elsif ( $filename eq "regex_revalidate.config" ) { $file_contents = $self->regex_revalidate_dot_config( $cdn_obj, $filename ); } elsif ( $filename =~ /set_dscp_.*\.config/ ) { $file_contents = $self->set_dscp_dot_config( $cdn_obj, $filename ); } @@ -244,10 +252,12 @@ sub get_profile_config { #generate the config file using the appropriate function my $file_contents; if ( $filename eq "50-ats.rules" ) { $file_contents = $self->ats_dot_rules( $profile_obj, $filename ); } + elsif ( $filename eq "12M_facts" ) { $file_contents = $self->facts( $profile_obj, $filename ); } elsif ( $filename eq "astats.config" ) { $file_contents = $self->generic_profile_config( $profile_obj, $filename ); } elsif ( $filename eq "drop_qstring.config" ) { $file_contents = $self->drop_qstring_dot_config( $profile_obj, $filename ); } elsif ( $filename eq "logs_xml.config" ) { $file_contents = $self->logs_xml_dot_config( $profile_obj, $filename ); } elsif ( $filename eq "plugin.config" ) { $file_contents = $self->generic_profile_config( $profile_obj, $filename ); } + elsif ( $filename eq "records.config" ) { $file_contents = $self->generic_profile_config( $profile_obj, $filename ); } elsif ( $filename eq "storage.config" ) { $file_contents = $self->storage_dot_config( $profile_obj, $filename ); } elsif ( $filename eq "sysctl.conf" ) { $file_contents = $self->generic_profile_config( $profile_obj, $filename ); } elsif ( $filename =~ /url_sig_.*\.config/ ) { $file_contents = $self->url_sig_dot_config( $profile_obj, $filename ); } @@ -282,17 +292,16 @@ sub get_scope { my $fname = shift; my $scope; - if ( $fname eq "12M_facts" ) { $scope = 'server' } - elsif ( $fname eq "ip_allow.config" ) { $scope = 'server' } + if ( $fname eq "ip_allow.config" ) { $scope = 'server' } elsif ( $fname eq "parent.config" ) { $scope = 'server' } - elsif ( $fname eq "records.config" ) { $scope = 'server' } + elsif ( $fname eq "records.config" ) { $scope = 'profile' } elsif ( $fname eq "remap.config" ) { $scope = 'server' } elsif ( $fname =~ /to_ext_.*\.config/ ) { $scope = 'server' } - elsif ( $fname =~ /hdr_rw_.*\.config/ ) { $scope = 'server' } elsif ( $fname eq "hosting.config" ) { $scope = 'server' } elsif ( $fname eq "cache.config" ) { $scope = 'server' } elsif ( $fname eq "packages" ) { $scope = 'server' } elsif ( $fname eq "chkconfig" ) { $scope = 'server' } + elsif ( $fname eq "12M_facts" ) { $scope = 'profile' } elsif ( $fname eq "50-ats.rules" ) { $scope = 'profile' } elsif ( $fname eq "astats.config" ) { $scope = 'profile' } elsif ( $fname eq "drop_qstring.config" ) { $scope = 'profile' } @@ -304,6 +313,7 @@ sub get_scope { elsif ( $fname eq "volume.config" ) { $scope = 'profile' } elsif ( $fname eq "bg_fetch.config" ) { $scope = 'cdn' } elsif ( $fname =~ /cacheurl.*\.config/ ) { $scope = 'cdn' } + elsif ( $fname =~ /hdr_rw_.*\.config/ ) { $scope = 'cdn' } elsif ( $fname =~ /regex_remap_.*\.config/ ) { $scope = 'cdn' } elsif ( $fname eq "regex_revalidate.config" ) { $scope = 'cdn' } elsif ( $fname =~ /set_dscp_.*\.config/ ) { $scope = 'cdn' } @@ -707,10 +717,10 @@ sub ds_data { #generates the 12m_facts file sub facts { my $self = shift; - my $server_obj = shift; + my $profile_obj = shift; my $filename = shift; - my $text = $self->header_comment( $server_obj->host_name ); - $text .= "profile:" . $server_obj->profile->name . "\n"; + my $text = $self->header_comment( $profile_obj->name ); + $text .= "profile:" . $profile_obj->name . "\n"; return $text; } @@ -995,10 +1005,10 @@ sub bg_fetch_dot_config { sub header_rewrite_dot_config { my $self = shift; - my $server_obj = shift; + my $cdn_obj = shift; my $filename = shift; - my $text = $self->header_comment( $server_obj->host_name ); + my $text = $self->header_comment( $cdn_obj->name ); my $ds_xml_id = undef; if ( $filename =~ /^hdr_rw_mid_(.*)\.config$/ ) { $ds_xml_id = $1; @@ -1014,8 +1024,10 @@ sub header_rewrite_dot_config { } $text =~ s/\s*__RETURN__\s*/\n/g; - my $ipv4 = $server_obj->ip_address; - $text =~ s/__CACHE_IPV4__/$ipv4/g; + + #do the following text replacement at the ORT level for cacheability + # my $ipv4 = $server_obj->ip_address; + # $text =~ s/__CACHE_IPV4__/$ipv4/g; return $text; } From 740a00531c9d7a22c1b90987a385c315b567f583 Mon Sep 17 00:00:00 2001 From: Derek Gelinas Date: Thu, 6 Apr 2017 16:58:08 +0000 Subject: [PATCH 2/5] changes scope for remap.config --- .../lib/API/Configs/ApacheTrafficServer.pm | 322 ++++++++++++++++-- 1 file changed, 289 insertions(+), 33 deletions(-) diff --git a/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm b/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm index 1a0956e27c..ed36d0c838 100644 --- a/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm +++ b/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm @@ -64,23 +64,24 @@ sub get_config_metadata { my %condition = ( 'me.host_name' => $host_name ); my $rs_server = $self->db->resultset('Server')->search( \%condition, { prefetch => [ 'cdn', 'profile' ] } ); my $tm_url = $self->db->resultset('Parameter')->search( { -and => [ name => 'tm.url', config_file => 'global' ] } )->get_column('value')->first(); - my $tm_cache_url = $self->db->resultset('Parameter')->search( { -and => [ name => 'tm_cache.url', config_file => 'global' ] } )->get_column('value')->first(); + my $tm_rev_proxy_url = $self->db->resultset('Parameter')->search( { -and => [ name => 'tm_rev_proxy.url', config_file => 'global' ] } )->get_column('value')->first(); my $cdn_name = $server_obj->cdn->name; my $server = $rs_server->next; my $config_file_obj; my @config_files; if ($server) { - $data_obj->{'info'}->{'server_name'} = $server_obj->host_name; - $data_obj->{'info'}->{'server_ipv4'} = $server_obj->ip_address; - $data_obj->{'info'}->{'server_id'} = $server_obj->id; - $data_obj->{'info'}->{'profile_name'} = $server->profile->name; - $data_obj->{'info'}->{'profile_id'} = $server->profile->id; - $data_obj->{'info'}->{'cdn_name'} = $cdn_name; - $data_obj->{'info'}->{'cdn_id'} = $server->cdn->id; - $data_obj->{'info'}->{'to_url'} = $tm_url; - if ( $tm_cache_url ) { - $data_obj->{'info'}->{'to_cache_url'} = $tm_cache_url; + $data_obj->{'info'}->{'serverName'} = $server_obj->host_name; + $data_obj->{'info'}->{'serverIpv4'} = $server_obj->ip_address; + $data_obj->{'info'}->{'serverId'} = $server_obj->id; + $data_obj->{'info'}->{'profileName'} = $server->profile->name; + $data_obj->{'info'}->{'profileId'} = $server->profile->id; + $data_obj->{'info'}->{'cdnName'} = $cdn_name; + $data_obj->{'info'}->{'cdnId'} = $server->cdn->id; + $data_obj->{'info'}->{'toUrl'} = $tm_url; + $data_obj->{'info'}->{'serverTcpPort'} = $server_obj->tcp_port; + if ( $tm_rev_proxy_url ) { + $data_obj->{'info'}->{'toRevProxyUrl'} = $tm_rev_proxy_url; } #$data_obj->{'profile'}->{'name'} = $server->profile->name; @@ -115,7 +116,7 @@ sub get_config_metadata { else { $scope_id = $server_obj->host_name; } - $config_file_obj->{$config_file}->{'API_URI'} = "/api/1.2/" . $scope . "/" . $scope_id . "/configfiles/ats/" . $config_file; + $config_file_obj->{$config_file}->{'apiUri'} = "/api/1.2/" . $scope . "/" . $scope_id . "/configfiles/ats/" . $config_file; $config_file_obj->{$config_file}->{'scope'} = $scope; } @@ -123,7 +124,7 @@ sub get_config_metadata { push ( @config_files, $config_file_obj->{$config_file} ); } - $data_obj->{'config_files'} = \@config_files; + $data_obj->{'configFiles'} = \@config_files; my $file_contents = encode_json($data_obj); return $self->render( text => $file_contents, format => 'txt' ); @@ -157,7 +158,6 @@ sub get_server_config { if ( $filename =~ /to_ext_.*\.config/ ) { $file_contents = $self->to_ext_dot_config( $server_obj, $filename ); } elsif ( $filename eq "ip_allow.config" ) { $file_contents = $self->ip_allow_dot_config( $server_obj, $filename ); } elsif ( $filename eq "parent.config" ) { $file_contents = $self->parent_dot_config( $server_obj, $filename ); } - elsif ( $filename eq "remap.config" ) { $file_contents = $self->remap_dot_config( $server_obj, $filename ); } elsif ( $filename eq "hosting.config" ) { $file_contents = $self->hosting_dot_config( $server_obj, $filename ); } elsif ( $filename eq "cache.config" ) { $file_contents = $self->cache_dot_config( $server_obj, $filename ); } elsif ( $filename eq "packages" ) { @@ -258,6 +258,7 @@ sub get_profile_config { elsif ( $filename eq "logs_xml.config" ) { $file_contents = $self->logs_xml_dot_config( $profile_obj, $filename ); } elsif ( $filename eq "plugin.config" ) { $file_contents = $self->generic_profile_config( $profile_obj, $filename ); } elsif ( $filename eq "records.config" ) { $file_contents = $self->generic_profile_config( $profile_obj, $filename ); } + elsif ( $filename eq "remap.config" ) { $file_contents = $self->remap_dot_config( $profile_obj, $filename ); } elsif ( $filename eq "storage.config" ) { $file_contents = $self->storage_dot_config( $profile_obj, $filename ); } elsif ( $filename eq "sysctl.conf" ) { $file_contents = $self->generic_profile_config( $profile_obj, $filename ); } elsif ( $filename =~ /url_sig_.*\.config/ ) { $file_contents = $self->url_sig_dot_config( $profile_obj, $filename ); } @@ -294,8 +295,6 @@ sub get_scope { if ( $fname eq "ip_allow.config" ) { $scope = 'server' } elsif ( $fname eq "parent.config" ) { $scope = 'server' } - elsif ( $fname eq "records.config" ) { $scope = 'profile' } - elsif ( $fname eq "remap.config" ) { $scope = 'server' } elsif ( $fname =~ /to_ext_.*\.config/ ) { $scope = 'server' } elsif ( $fname eq "hosting.config" ) { $scope = 'server' } elsif ( $fname eq "cache.config" ) { $scope = 'server' } @@ -307,6 +306,8 @@ sub get_scope { elsif ( $fname eq "drop_qstring.config" ) { $scope = 'profile' } elsif ( $fname eq "logs_xml.config" ) { $scope = 'profile' } elsif ( $fname eq "plugin.config" ) { $scope = 'profile' } + elsif ( $fname eq "records.config" ) { $scope = 'profile' } + elsif ( $fname eq "remap.config" ) { $scope = 'profile' } elsif ( $fname eq "storage.config" ) { $scope = 'profile' } elsif ( $fname eq "sysctl.conf" ) { $scope = 'profile' } elsif ( $fname =~ /url_sig_.*\.config/ ) { $scope = 'profile' } @@ -459,6 +460,261 @@ sub profile_param_value { return ( defined $param ? $param->parameter->value : $default ); } +sub delivery_service_data_by_profile { + my $self = shift; + my $profile_id = shift; + my @data; + + my $dbh = $self->db->storage->dbh; + my $qry = 'select + deliveryservice.id, + deliveryservice.xml_id, + deliveryservice.dscp, + deliveryservice.signed, + deliveryservice.qstring_ignore, + deliveryservice.org_server_fqdn, + deliveryservice.origin_shield, + regex.pattern AS pattern, + retype.name AS re_type, + dstype.name AS ds_type, + cdn.domain_name AS domain_name, + deliveryservice.profile, + deliveryservice.protocol, + deliveryservice.ssl_key_version, + deliveryservice.range_request_handling, + deliveryservice.edge_header_rewrite, + deliveryservice.mid_header_rewrite, + deliveryservice.regex_remap, + deliveryservice.cacheurl, + deliveryservice.remap_text, + deliveryservice.multi_site_origin, + deliveryservice.multi_site_origin_algorithm + from deliveryservice + JOIN deliveryservice_regex ON deliveryservice_regex.deliveryservice = deliveryservice.id + JOIN regex ON deliveryservice_regex.regex = regex.id + JOIN type as retype ON regex.type = retype.id + JOIN type as dstype ON deliveryservice.type = dstype.id + JOIN cdn ON cdn.id = deliveryservice.cdn_id + where deliveryservice.id = ANY ( + select DISTINCT + deliveryservice + from deliveryservice_server + where server = ANY ( + select + id + from server + where profile = '.$profile_id.' + ) + ); + '; + + my $stmt = $dbh->prepare($qry); + $stmt->execute(); + + my $deliveryservice_id; + my $deliveryservice_xml_id; + my $deliveryservice_dscp; + my $deliveryservice_signed; + my $deliveryservice_qstring_ignore; + my $deliveryservice_org_server_fqdn; + my $deliveryservice_origin_shield; + my $pattern; + my $re_type; + my $ds_type; + my $domain_name; + my $deliveryservice_profile; + my $deliveryservice_protocol; + my $deliveryservice_ssl_key_version; + my $deliveryservice_range_request_handling; + my $deliveryservice_edge_header_rewrite; + my $deliveryservice_mid_header_rewrite; + my $deliveryservice_regex_remap; + my $deliveryservice_cacheurl; + my $deliveryservice_remap_text; + my $deliveryservice_multi_site_origin; + my $deliveryservice_multi_site_origin_algorithm; + + $stmt->bind_columns( + \$deliveryservice_id, + \$deliveryservice_xml_id, + \$deliveryservice_dscp, + \$deliveryservice_signed, + \$deliveryservice_qstring_ignore, + \$deliveryservice_org_server_fqdn, + \$deliveryservice_origin_shield, + \$pattern, + \$re_type, + \$ds_type, + \$domain_name, + \$deliveryservice_profile, + \$deliveryservice_protocol, + \$deliveryservice_ssl_key_version, + \$deliveryservice_range_request_handling, + \$deliveryservice_edge_header_rewrite, + \$deliveryservice_mid_header_rewrite, + \$deliveryservice_regex_remap, + \$deliveryservice_cacheurl, + \$deliveryservice_remap_text, + \$deliveryservice_multi_site_origin, + \$deliveryservice_multi_site_origin_algorithm + ); + + while ( my $row = $stmt->fetch() ) { + push( + @data, { + "id" => $deliveryservice_id, + "xml_id" => $deliveryservice_xml_id, + "dscp" => $deliveryservice_dscp, + "signed" => $deliveryservice_signed, + "qstring_ignore" => $deliveryservice_qstring_ignore, + "org_server_fqdn" => $deliveryservice_org_server_fqdn, + "origin_shield" => $deliveryservice_origin_shield, + "pattern" => $pattern, + "re_type" => $re_type, + "ds_type" => $ds_type, + "domain_name" => $domain_name, + "profile" => $deliveryservice_profile, + "protocol" => $deliveryservice_protocol, + "ssl_key_version" => $deliveryservice_ssl_key_version, + "range_request_handling" => $deliveryservice_range_request_handling, + "edge_header_rewrite" => $deliveryservice_edge_header_rewrite, + "mid_header_rewrite" => $deliveryservice_mid_header_rewrite, + "regex_remap" => $deliveryservice_regex_remap, + "cacheurl" => $deliveryservice_cacheurl, + "remap_text" => $deliveryservice_remap_text, + "multi_site_origin" => $deliveryservice_multi_site_origin, + "multi_site_origin_algorithm" => $deliveryservice_multi_site_origin_algorithm + } + ); + } + + return @data; +} + +#gets the delivery service data for all servers assigned to a profile. +sub profile_ds_data { + my $self = shift; + my $profile_obj = shift; + my $dsinfo; + + my @rs = $self->delivery_service_data_by_profile( $profile_obj->id ); + + my $j = 0; + while ( scalar(@rs) !=0 ) { + my $row = shift(@rs); + + my $org_server = $row->{'org_server_fqdn'}; + my $dscp = $row->{'dscp'}; + my $re_type = $row->{'re_type'}; + my $ds_type = $row->{'ds_type'}; + my $signed = $row->{'signed'}; + my $qstring_ignore = $row->{'qstring_ignore'}; + my $ds_xml_id = $row->{'xml_id'}; + my $ds_domain = $row->{'domain_name'}; + my $edge_header_rewrite = $row->{'edge_header_rewrite'}; + my $mid_header_rewrite = $row->{'mid_header_rewrite'}; + my $regex_remap = $row->{'regex_remap'}; + my $protocol = $row->{'protocol'}; + my $range_request_handling = $row->{'range_request_handling'}; + my $origin_shield = $row->{'origin_shield'}; + my $cacheurl = $row->{'cacheurl'}; + my $remap_text = $row->{'remap_text'}; + my $multi_site_origin = $row->{'multi_site_origin'}; + + if ( $re_type eq 'HOST_REGEXP' ) { + my $host_re = $row->{'pattern'}; + #my $map_to = $org_server . "/"; + my $map_to; + if ( defined($org_server) ) { + $map_to = $org_server . "/"; + } + if ( $host_re =~ /\.\*$/ ) { + my $re = $host_re; + $re =~ s/\\//g; + $re =~ s/\.\*//g; + my $hname = $ds_type =~ /^DNS/ ? "edge" : "ccr"; + my $portstr = ":" . "__SERVER_TCP_PORT__"; + my $map_from = "http://" . $hname . $re . $ds_domain . $portstr . "/"; + if ( $protocol == HTTP ) { + $dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to; + } + elsif ( $protocol == HTTPS || $protocol == HTTP_TO_HTTPS ) { + $map_from = "https://" . $hname . $re . $ds_domain . "/"; + $dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to; + } + elsif ( $protocol == HTTP_AND_HTTPS ) { + + #add the first one with http + $dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to; + + #add the second one for https + my $map_from2 = "https://" . $hname . $re . $ds_domain . "/"; + $dsinfo->{dslist}->[$j]->{"remap_line2"}->{$map_from2} = $map_to; + } + } + else { + my $map_from = "http://" . $host_re . "/"; + if ( $protocol == HTTP ) { + $dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to; + } + elsif ( $protocol == HTTPS || $protocol == HTTP_TO_HTTPS ) { + $map_from = "https://" . $host_re . "/"; + $dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to; + } + elsif ( $protocol == HTTP_AND_HTTPS ) { + + #add the first with http + $dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to; + + #add the second with https + my $map_from2 = "https://" . $host_re . "/"; + $dsinfo->{dslist}->[$j]->{"remap_line2"}->{$map_from2} = $map_to; + } + } + } + + $dsinfo->{dslist}->[$j]->{"dscp"} = $dscp; + $dsinfo->{dslist}->[$j]->{"org"} = $org_server; + $dsinfo->{dslist}->[$j]->{"type"} = $ds_type; + $dsinfo->{dslist}->[$j]->{"domain"} = $ds_domain; + $dsinfo->{dslist}->[$j]->{"signed"} = $signed; + $dsinfo->{dslist}->[$j]->{"qstring_ignore"} = $qstring_ignore; + $dsinfo->{dslist}->[$j]->{"ds_xml_id"} = $ds_xml_id; + $dsinfo->{dslist}->[$j]->{"edge_header_rewrite"} = $edge_header_rewrite; + $dsinfo->{dslist}->[$j]->{"mid_header_rewrite"} = $mid_header_rewrite; + $dsinfo->{dslist}->[$j]->{"regex_remap"} = $regex_remap; + $dsinfo->{dslist}->[$j]->{"range_request_handling"} = $range_request_handling; + $dsinfo->{dslist}->[$j]->{"origin_shield"} = $origin_shield; + $dsinfo->{dslist}->[$j]->{"cacheurl"} = $cacheurl; + $dsinfo->{dslist}->[$j]->{"remap_text"} = $remap_text; + $dsinfo->{dslist}->[$j]->{"multi_site_origin"} = $multi_site_origin; + + if ( defined($edge_header_rewrite) ) { + my $fname = "hdr_rw_" . $ds_xml_id . ".config"; + $dsinfo->{dslist}->[$j]->{"hdr_rw_file"} = $fname; + } + if ( defined($mid_header_rewrite) ) { + my $fname = "hdr_rw_mid_" . $ds_xml_id . ".config"; + $dsinfo->{dslist}->[$j]->{"mid_hdr_rw_file"} = $fname; + } + if ( defined($cacheurl) ) { + my $fname = "cacheurl_" . $ds_xml_id . ".config"; + $dsinfo->{dslist}->[$j]->{"cacheurl_file"} = $fname; + } + if ( defined( $row->{'profile'} ) ) { + my $dsparamrs = $self->db->resultset('ProfileParameter')->search( { profile => $row->{'profile'} }, { prefetch => [ 'profile', 'parameter' ] } ); + while ( my $prow = $dsparamrs->next ) { + $dsinfo->{dslist}->[$j]->{'param'}->{ $prow->parameter->config_file }->{ $prow->parameter->name } = $prow->parameter->value; + } + } + + $j++; + } + + # $self->app->session->{dsinfo} = $dsinfo; + return $dsinfo; +} + #gets the delivery service data for an entire CDN. sub cdn_ds_data { my $self = shift; @@ -1809,16 +2065,16 @@ sub parent_dot_config { sub remap_dot_config { my $self = shift; - my $server_obj = shift; + my $profile_obj = shift; my $data; - my $pdata = $self->param_data( $server_obj, 'package' ); - my $text = $self->header_comment( $server_obj->host_name ); + my $pdata = $self->profile_param_data( $profile_obj->id, 'package' ); + my $text = $self->header_comment( $profile_obj->name ); if ( !defined($data) ) { - $data = $self->ds_data($server_obj); + $data = $self->profile_ds_data($profile_obj); } - if ( $server_obj->type->name =~ m/^MID/ ) { + if ( $profile_obj->name =~ m/^MID/ ) { my %mid_remap; foreach my $ds ( @{ $data->{dslist} } ) { if ( $ds->{type} =~ /LIVE/ && $ds->{type} !~ /NATNL/ ) { @@ -1852,25 +2108,25 @@ sub remap_dot_config { foreach my $ds ( @{ $data->{dslist} } ) { foreach my $map_from ( keys %{ $ds->{remap_line} } ) { my $map_to = $ds->{remap_line}->{$map_from}; - $text = $self->build_remap_line( $server_obj, $pdata, $text, $data, $ds, $map_from, $map_to ); + $text = $self->build_remap_line( $profile_obj, $pdata, $text, $data, $ds, $map_from, $map_to ); } foreach my $map_from ( keys %{ $ds->{remap_line2} } ) { my $map_to = $ds->{remap_line2}->{$map_from}; - $text = $self->build_remap_line( $server_obj, $pdata, $text, $data, $ds, $map_from, $map_to ); + $text = $self->build_remap_line( $profile_obj, $pdata, $text, $data, $ds, $map_from, $map_to ); } } return $text; } sub build_remap_line { - my $self = shift; - my $server_obj = shift; - my $pdata = shift; - my $text = shift; - my $data = shift; - my $remap = shift; - my $map_from = shift; - my $map_to = shift; + my $self = shift; + my $profile_obj = shift; + my $pdata = shift; + my $text = shift; + my $data = shift; + my $remap = shift; + my $map_from = shift; + my $map_to = shift; if ( $remap->{type} eq 'ANY_MAP' ) { $text .= $remap->{remap_text} . "\n"; @@ -1880,7 +2136,7 @@ sub build_remap_line { my $host_name = $data->{host_name}; my $dscp = $remap->{dscp}; - $map_from =~ s/ccr/$host_name/; + $map_from =~ s/ccr/__HOSTNAME__/; if ( defined( $pdata->{'dscp_remap'} ) ) { $text .= "map " . $map_from . " " . $map_to . " \@plugin=dscp_remap.so \@pparam=" . $dscp; @@ -1899,7 +2155,7 @@ sub build_remap_line { $text .= " \@plugin=regex_remap.so \@pparam=" . $dqs_file; } elsif ( $remap->{qstring_ignore} == 1 ) { - my $global_exists = $self->profile_param_value( $server_obj->profile->id, 'cacheurl.config', 'location', undef ); + my $global_exists = $self->profile_param_value( $profile_obj->id, 'cacheurl.config', 'location', undef ); if ($global_exists) { $self->app->log->debug( "qstring_ignore == 1, but global cacheurl.config param exists, so skipping remap rename config_file=cacheurl.config parameter if you want to change" From 37476e17a9b8043e7f9fe46230cea176c5ff58e7 Mon Sep 17 00:00:00 2001 From: Derek Gelinas Date: Thu, 6 Apr 2017 17:26:56 +0000 Subject: [PATCH 3/5] modifies name to fname on disk and fixes rev proxy url lookup --- traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm b/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm index ed36d0c838..63855897ce 100644 --- a/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm +++ b/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm @@ -64,7 +64,7 @@ sub get_config_metadata { my %condition = ( 'me.host_name' => $host_name ); my $rs_server = $self->db->resultset('Server')->search( \%condition, { prefetch => [ 'cdn', 'profile' ] } ); my $tm_url = $self->db->resultset('Parameter')->search( { -and => [ name => 'tm.url', config_file => 'global' ] } )->get_column('value')->first(); - my $tm_rev_proxy_url = $self->db->resultset('Parameter')->search( { -and => [ name => 'tm_rev_proxy.url', config_file => 'global' ] } )->get_column('value')->first(); + my $tm_rev_proxy_url = $self->db->resultset('Parameter')->search( { -and => [ name => 'tm.rev_proxy.url', config_file => 'global' ] } )->get_column('value')->first(); my $cdn_name = $server_obj->cdn->name; my $server = $rs_server->next; my $config_file_obj; @@ -95,7 +95,7 @@ sub get_config_metadata { my $rs_param = $self->db->resultset('Parameter')->search( \%condition, { join => 'profile_parameters' } ); while ( my $param = $rs_param->next ) { if ( $param->name eq 'location' ) { - $config_file_obj->{ $param->config_file }->{'name'} = $param->config_file; + $config_file_obj->{ $param->config_file }->{'fnameOnDisk'} = $param->config_file; $config_file_obj->{ $param->config_file }->{'location'} = $param->value; } From eb1f97dda6f3cf8c717c65506a082cedeb0df9ff Mon Sep 17 00:00:00 2001 From: Derek Gelinas Date: Thu, 6 Apr 2017 18:54:19 +0000 Subject: [PATCH 4/5] fix hostname reference in profile parameter lookups --- traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm b/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm index 63855897ce..385a72a87b 100644 --- a/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm +++ b/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm @@ -438,6 +438,9 @@ sub profile_param_data { if ( defined( $data->{$key} ) ) { $key .= "__" . $row->parameter->id; } + if ( $value =~ /^STRING __HOSTNAME__$/ ) { + $value = "STRING " . "__FULL_HOSTNAME__"; + } $data->{$key} = $value; } return $data; From 43f11426bb01c7dc15c8cf18bb4ba338c7f3d900 Mon Sep 17 00:00:00 2001 From: Derek Gelinas Date: Thu, 6 Apr 2017 19:51:02 +0000 Subject: [PATCH 5/5] move cache.config scope to profile --- .../lib/API/Configs/ApacheTrafficServer.pm | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm b/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm index 385a72a87b..25d86af319 100644 --- a/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm +++ b/traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm @@ -159,7 +159,6 @@ sub get_server_config { elsif ( $filename eq "ip_allow.config" ) { $file_contents = $self->ip_allow_dot_config( $server_obj, $filename ); } elsif ( $filename eq "parent.config" ) { $file_contents = $self->parent_dot_config( $server_obj, $filename ); } elsif ( $filename eq "hosting.config" ) { $file_contents = $self->hosting_dot_config( $server_obj, $filename ); } - elsif ( $filename eq "cache.config" ) { $file_contents = $self->cache_dot_config( $server_obj, $filename ); } elsif ( $filename eq "packages" ) { $file_contents = $self->get_package_versions( $server_obj, $filename ); $file_contents = encode_json($file_contents); @@ -254,6 +253,7 @@ sub get_profile_config { if ( $filename eq "50-ats.rules" ) { $file_contents = $self->ats_dot_rules( $profile_obj, $filename ); } elsif ( $filename eq "12M_facts" ) { $file_contents = $self->facts( $profile_obj, $filename ); } elsif ( $filename eq "astats.config" ) { $file_contents = $self->generic_profile_config( $profile_obj, $filename ); } + elsif ( $filename eq "cache.config" ) { $file_contents = $self->cache_dot_config( $profile_obj, $filename ); } elsif ( $filename eq "drop_qstring.config" ) { $file_contents = $self->drop_qstring_dot_config( $profile_obj, $filename ); } elsif ( $filename eq "logs_xml.config" ) { $file_contents = $self->logs_xml_dot_config( $profile_obj, $filename ); } elsif ( $filename eq "plugin.config" ) { $file_contents = $self->generic_profile_config( $profile_obj, $filename ); } @@ -297,12 +297,12 @@ sub get_scope { elsif ( $fname eq "parent.config" ) { $scope = 'server' } elsif ( $fname =~ /to_ext_.*\.config/ ) { $scope = 'server' } elsif ( $fname eq "hosting.config" ) { $scope = 'server' } - elsif ( $fname eq "cache.config" ) { $scope = 'server' } elsif ( $fname eq "packages" ) { $scope = 'server' } elsif ( $fname eq "chkconfig" ) { $scope = 'server' } elsif ( $fname eq "12M_facts" ) { $scope = 'profile' } elsif ( $fname eq "50-ats.rules" ) { $scope = 'profile' } elsif ( $fname eq "astats.config" ) { $scope = 'profile' } + elsif ( $fname eq "cache.config" ) { $scope = 'profile' } elsif ( $fname eq "drop_qstring.config" ) { $scope = 'profile' } elsif ( $fname eq "logs_xml.config" ) { $scope = 'profile' } elsif ( $fname eq "plugin.config" ) { $scope = 'profile' } @@ -1507,11 +1507,12 @@ sub url_sig_dot_config { sub cache_dot_config { my $self = shift; - my $server_obj = shift; + my $profile_obj = shift; my $filename = shift; + - my $text = $self->header_comment( $server_obj->host_name ); - my $data = $self->ds_data($server_obj); + my $text = $self->header_comment( $profile_obj->name ); + my $data = $self->profile_ds_data($profile_obj); foreach my $ds ( @{ $data->{dslist} } ) { if ( $ds->{type} eq "HTTP_NO_CACHE" ) { @@ -1521,6 +1522,14 @@ sub cache_dot_config { } } + #remove duplicate lines, since we're looking up by profile + my @lines = split(/\n/,$text); + + my %seen; + @lines = grep { !$seen{$_}++ } @lines; + + $text = join("\n",@lines); + return $text; }