Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 279 lines (253 sloc) 10.686 kb
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
1 #!/usr/bin/env perl
2 # -*- Mode: CPerl -*-
3 #
4 # Copyright (C) 2004 Nicolas Niclausse
5 #
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
19
9ae525de » nniclausse
2004-10-19 cleanup
20 # Auteur: Nicolas Niclausse (nicolas@niclux.org)
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
21 # Version: $Id$
22
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
23 # purpose: create a config file for tsung from a Combined Log file
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
24
25 use strict;
26 use Getopt::Long;
27 use Time::Local;
28
29 use vars qw ($help *verbose $version $thinktime_threshold $visit_timeout
30 $session_threshold $max_pages $max_duration);
31 my %Months=('Jan','0', 'Feb','1', 'Mar','2', 'Apr','3', 'May','4', 'Jun','5',
32 'Jul','6', 'Aug','7', 'Sep','8', 'Oct','9', 'Nov','10', 'Dec','11');
33
13dfdfdc » nniclausse
2005-04-29 update configure & Makefile
34 my $tagvsn = '@PACKAGE_VERSION@';
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
35
36 GetOptions( "help",\$help,
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
37 "verbose",\$verbose,
38 "tt=i",\$thinktime_threshold,
39 "st=i",\$session_threshold,
40 "visit_timeout=i",\$visit_timeout,
41 "max_pages=i",\$max_pages,
42 "max_duration=i",\$max_duration,
43 "version",\$version
44 );
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
45
13dfdfdc » nniclausse
2005-04-29 update configure & Makefile
46 my $prefix ="@prefix@";
47 my $dtd ="@datadir@/@DTD@";
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
48
49 # remove thinktime less than 1 sec
50 $thinktime_threshold ="1" unless $thinktime_threshold;
51 # remove session with less than 2 requests
52 $session_threshold ="2"unless $session_threshold;
53
54 my $ims = "Fri, 14 Nov 2003 02:43:31 GMT"; # if modified since ... for 304
55
56 # if thinktime is more than $visit_timeout, it's a new session
57 $visit_timeout=600 unless $visit_timeout;
58
59 $max_pages = 100 unless $max_pages ; # 100 pages max per session
60 $max_duration = 3600 unless $max_duration; # 1hour max session duration
61
62 my %hit;
63 my %http;
64 my $visite;
65 my ($time,$sec,$min,$hour,$mday,$mon,$year);
66 my $total;
67 my $bad = 0;
68 my $user;
69 my $id;
70 my $visit_tot=0;
71
72 &usage if $help or $Getopt::Long::error;
73 &version if $version;
74
75 while (<>) {
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
76 if (m@^([\w\.]+) \S+ \S+ \[(\w+/\w+/\w+:\d+:\d+:\d+)([^\]]+)\] \"(\w+) ([^\"]+)\" (\d+) (\S+) \"([^\"]*)\" \"([^\"]*)\"$@) {
77 my $ip = $1;
78 my $date = $2;
79 my $code = $6;
80 my $referer = $8;
81 my $method = $4;
82 my $user_agent = $9;
83 my $req = $5;
84 my ($url, $protocole) = split(/\s+/,$req);
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
85 $url = &replace_entities($url);
86 my $version;
87 if ($protocole =~ /HTTP\/(\d\.\d)/) {
88 $version=$1;
89 } else {
90 $version="1.0";
91 }
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
92 $date =~ m'(\d+)/(\w+)/(\d+):(\d+):(\d+):(\d+)';
93 $mday = $1;
94 $mon = $Months{$2};
95 $year = $3 - 1900;
96 $hour = $4;
97 $min = $5;
98 $sec = $6;
99 $time = timelocal($sec,$min,$hour,$mday,$mon,$year);
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
100 $user = "$ip-$user_agent";
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
101 if ($visite->{$user}) {
102 if ($time - $visite->{$user}->{'last_visit'} > $visit_timeout) {
103 # new visit
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
104 $visit_tot ++;
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
105 $visite->{$user}->{'id'}++;
106 $id = $visite->{$user}->{'id'};
107 $visite->{$user}->{'last_visit'}=$time;
108 $visite->{$user}->{'last_referer'}=$referer;
109 $visite->{$user}->{$id}->{'started'}=$time;
110 $visite->{$user}->{$id}->{'last_request'}=$time;
111 $visite->{$user}->{$id}->{'page'}=1;
112 $visite->{$user}->{$id}->{'hit'}=1;
113 $visite->{$user}->{$id}->{'duration'}=0;
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
114 $visite->{$user}->{$id}->{'tsung'} = '<session name="'.$ip."-".$id.'" type="ts_http">'."\n";
115 $visite->{$user}->{$id}->{'tsung'} .= "\t".'<request><http url="'.$url.'" version="'.$version.'" method="'.$method.'"';
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
116 if ($code == 304) {
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
117 $visite->{$user}->{$id}->{'tsung'} .= ' if_modified_since="'.$ims.'">';
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
118 } else {
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
119 $visite->{$user}->{$id}->{'tsung'} .= '>';
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
120 }
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
121 $visite->{$user}->{$id}->{'tsung'} .= "</http></request>\n";
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
122 } else {
123 # same visit
124 $id = $visite->{$user}->{'id'};
125 $visite->{$user}->{$id}->{'hit'}++;
126 my $thinktime = $time - $visite->{$user}->{$id}->{'last_request'};
127 $visite->{$user}->{'last_visit'}=$time;
128 $visite->{$user}->{$id}->{'last_request'}=$time;
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
129 $visite->{$user}->{$id}->{'tsung'} .= "\t".'<thinktime value="'.$thinktime.'"/>'."\n\n" if $thinktime > $thinktime_threshold;
130 $visite->{$user}->{$id}->{'tsung'} .= "\t".'<request><http url="'.$url.'" version="'.$version.'" method="'.$method.'"></http></request>'."\n";
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
131 # update duration
132 $visite->{$user}->{$id}->{'duration'} = $time - $visite->{$user}->{$id}->{'started'} ;
133 if ($visite->{$user}->{'last_referer'} eq $referer) {
134 # same page/frame
135 } else {
136 # new frame/page
137 $visite->{$user}->{$id}->{'page'}++;
138 $visite->{$user}->{'last_referer'}=$referer;
139 }
140 }
141 } else {# new visitor
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
142 $visit_tot ++;
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
143 $visite->{$user}->{'id'}=1;
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
144 $id = 1;
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
145 $visite->{$user}->{'last_visit'}=$time;
146 $visite->{$user}->{'last_referer'}=$referer;
147 $visite->{$user}->{$id}->{'started'}=$time;
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
148 $visite->{$user}->{$id}->{'last_request'}=$time;
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
149 $visite->{$user}->{$id}->{'hit'}=1;
150 $visite->{$user}->{$id}->{'page'}=1;
151 $visite->{$user}->{$id}->{'duration'}=0;
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
152 $visite->{$user}->{$id}->{'tsung'} = '<session name="'.$ip."-".$id.'" type="ts_http">'."\n";
153 $visite->{$user}->{$id}->{'tsung'} .= "\t".'<request><http url="'.$url.'" version="'.$version.'" method="'.$method.'"></http></request>'."\n";
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
154 }
155 $total ++;
156 } else {
157 # print STDERR "$_\n";
158 $bad ++;
159 }
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
160 }
161 my $users_tot=scalar %{$visite};
162 my $page_tot=0;
163 my $hit_tot=0;
164 my $bad_visit =0;
165 my $bad_pages =0;
166 print STDERR "number of unique users is $users_tot\n" if $verbose;
167 print '<?xml version="1.0"?>
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
168 <!DOCTYPE tsung SYSTEM "'.$dtd.'" [] >
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
169 ';
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
170 print '<tsung loglevel="notice" dumptraffic="false" version="1.0">
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
171 <clients>
172 <client host="myhostname" weight="2" maxusers="950">
173 <ip value="192.168.0.2"></ip>
174 </client>
175 </clients>
da56d5ea » nniclausse
2006-05-20 fix xml
176 <servers>
177 <server host="myservername" port="80" type="tcp"></server>
178 </servers>
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
179
180 <arrivalphase phase="1" duration="10" unit="minute">
181 <users interarrival="0.1" unit="second"></users>
182 </arrivalphase>
183
da56d5ea » nniclausse
2006-05-20 fix xml
184 <sessions>
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
185 ';
186 my $real_visit = 0;
187 foreach my $key (keys %$visite) {
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
188 foreach my $id (1..$visite->{$key}->{'id'}) {
189 my $page = $visite->{$key}->{$id}->{'page'};
190 my $hit = $visite->{$key}->{$id}->{'hit'};
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
191 $real_visit ++ if $hit > $session_threshold;
192 }
193 }
194 foreach my $key (sort {$visite->{$a}->{'id'} cmp $visite->{$b}->{'id'}} keys %$visite) {
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
195 my $tot_id = $visite->{$key}->{'id'};
196 print STDERR "number of visit for $key is $tot_id\n" if $verbose;
197 foreach my $id (1..$tot_id) {
198 my $page = $visite->{$key}->{$id}->{'page'};
199 my $hit = $visite->{$key}->{$id}->{'hit'};
200 my $duration = $visite->{$key}->{$id}->{'duration'};
201 if ($page < $max_pages and $duration < $max_duration) {
202 $page_tot += $page;
203 $hit_tot += $hit;
204 print STDERR " page=$page hit=$hit duration=$duration\n" if $verbose;
205 } else {
206 $bad_visit++;
207 $bad_pages +=$page;
208
209 print STDERR "# page=$page hit=$hit duration=$duration\n" if $verbose;
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
210
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
211 }
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
212 next unless $hit > $session_threshold;
213 my $pop=sprintf "%.3f",100/$real_visit;
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
214 my $tsung = $visite->{$key}->{$id}->{'tsung'};
215 $tsung =~ s/\<session/<session popularity=\"$pop\"/;
216 print "$tsung</session>\n";
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
217 }
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
218 }
da56d5ea » nniclausse
2006-05-20 fix xml
219 print '</sessions></tsung>';
9ae525de » nniclausse
2004-10-19 cleanup
220 if ($verbose) {
221 select STDERR;
222 print "real_visit = $real_visit\n";
223 print "total_visit = $visit_tot , bad visit = $bad_visit ";
224 printf "page/visit = %.2f\n",($page_tot/($visit_tot-$bad_visit));
225 print "good_pages = $page_tot , bad pages = $bad_pages ";
226 printf "hit/page = %.2f\n",($hit_tot/$page_tot);
227 print "bad = $bad\n";
228 }
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
229
230 sub replace_entities {
231 my $str = shift;
232 $str =~ s/\&/\&amp;/g;
233 $str =~ s/\'/\&apos;/g;
234 $str =~ s/\"/\&quot;/g;
235 $str =~ s/>/\&gt;/g;
236 $str =~ s/</\&lt;/g;
237 return $str;
238 }
239 sub usage {
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
240 print "log2tsung.pl: create a config file for tsung from a Combined Log file\n\n";
241 print "This script is part of tsung version $tagvsn,
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
242 Copyright (C) 2004 Nicolas Niclausse\n\n";
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
243 print "tsung comes with ABSOLUTELY NO WARRANTY; This is free software, and
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
244 ou are welcome to redistribute it under certain conditions
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
245 type `log2tsung.pl --version` for details.\n\n";
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
246
247 print "Usage: $0 [<options>] <log file>\n","Available options:\n\t",
248 "[--help] (this help text)\n\t",
249 "[--version] (print version)\n\t",
250 "[--tt <integer>] (thinktime threshold: min thinktime (def=2))\n\t",
251 "[--st <integer>] (session threshold : min number of requests (def=2))\n\t",
252 "[--max_duration <integer>] (maximum session duration in sec. (3600))\n\t",
253 "[--max_pages <integer>] (maximum number of pages winthin a session. (100))\n\t";
a085fb8d » nniclausse
2004-10-19 cleanup: tabs to space
254 exit;
255 }
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
256
257 sub version {
5345a3b5 » nniclausse
2005-11-27 rename idx-tsunami to tsung step 3
258 print "this script is part of tsung version $tagvsn
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
259
260 Written by Nicolas Niclausse
261
da56d5ea » nniclausse
2006-05-20 fix xml
262 Copyright (C) 2004-2006 Nicolas Niclausse
75eb47d5 » nniclausse
2004-10-19 add a script to convert apache combined log files to tsunami XML conf…
263
264 This program is free software; you can redistribute it and/or
265 modify it under the terms of the GNU General Public License
266 as published by the Free Software Foundation; either version 2
267 of the License, or (at your option) any later version.
268
269 This program is distributed in the hope that it will be useful,
270 but WITHOUT ANY WARRANTY; without even the implied warranty of
271 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
272 GNU General Public License for more details.
273
274 You should have received a copy of the GNU General Public License
275 along with this program (see COPYING); if not, write to the
276 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
277 Boston, MA 02111-1307, USA.";
278 exit;
279 }
Something went wrong with that request. Please try again.