Permalink
Switch branches/tags
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
105 lines (100 sloc) 14.7 KB
# Aliases to extract individual fields of an apache access log in extended format
# Order follows the individual fields
# Some values are transformed
alias alip='cut -d\ -f1'
alias alcountry='cut -d\ -f2'
alias aluser='cut -d\ -f3'
alias altimestamp='cut -d\ -f4,5 | tr -d "[]"'
alias alrequestline='cut -d\" -f2'
alias almethod='cut -d\" -f2 | cut -d\ -f1 | sed "s/^-$/**NONE**/"'
alias aluri='cut -d\" -f2 | cut -d\ -f2 | sed "s/^-$/**NONE**/"'
alias alprotocol='cut -d\" -f2 | cut -d\ -f3 | sed "s/^-$/**NONE**/"'
alias alstatus='cut -d\" -f3 | cut -d\ -f2'
alias alresponsebodysize='cut -d\" -f3 | cut -d\ -f3'
alias alreferer='cut -d\" -f4 | sed "s/^-$/**NONE**/"'
alias alreferrer='cut -d\" -f4 | sed "s/^-$/**NONE**/"'
alias aluseragent='cut -d\" -f6 | sed "s/^-$/**NONE**/"'
alias alservername='cut -d\" -f7 | cut -d\ -f2'
alias alservername='cut -d\" -f7 | cut -d\ -f2'
alias allocalip='cut -d\" -f7 | cut -d\ -f3'
alias alcanonicalport='cut -d\" -f7 | cut -d\ -f4'
alias alport='cut -d\" -f7 | cut -d\ -f4'
alias alhandler='cut -d\" -f7 | cut -d\ -f5'
alias albalroute='cut -d\" -f7 | cut -d\ -f6'
alias alconnstatus='cut -d\" -f7 | cut -d\ -f7'
alias altrkcookie='cut -d\" -f8'
alias alreqid='cut -d\" -f9 | cut -d\ -f2'
alias aluniqueid='cut -d\" -f9 | cut -d\ -f2'
alias alsslprotocol='cut -d\" -f9 | cut -d\ -f3'
alias alsslcipher='cut -d\" -f9 | cut -d\ -f4'
alias alioin='cut -d\" -f9 | cut -d\ -f5'
alias alioout='cut -d\" -f9 | cut -d\ -f6'
alias aldeflateratio='cut -d\" -f9 | cut -d\ -f7 | tr -d %'
alias alduration='cut -d\" -f9 | cut -d\ -f8'
alias aldurationin='cut -d\" -f9 | cut -d\ -f9'
alias aldurationapp='cut -d\" -f9 | cut -d\ -f10'
alias aldurationout='cut -d\" -f9 | cut -d\ -f11'
alias alscorein='cut -d\" -f9 | cut -d\ -f12 | tr "-" "0"'
alias alscoreout='cut -d\" -f9 | cut -d\ -f13 | tr "-" "0"'
alias alscores='cut -d\" -f9 | cut -d\ -f12,13 | tr " " ";" | tr "-" "0"'
alias col1='awk "{ print \$1 }"'
alias col2='awk "{ print \$2 }"'
alias col3='awk "{ print \$3 }"'
alias col4='awk "{ print \$4 }"'
alias col5='awk "{ print \$5 }"'
alias col6='awk "{ print \$6 }"'
alias col7='awk "{ print \$7 }"'
alias col8='awk "{ print \$8 }"'
alias col9='awk "{ print \$9 }"'
alias col10='awk "{ print \$10 }"'
alias col11='awk "{ print \$11 }"'
alias col12='awk "{ print \$12 }"'
alias col13='awk "{ print \$13 }"'
alias col14='awk "{ print \$14 }"'
alias col15='awk "{ print \$15 }"'
alias col16='awk "{ print \$16 }"'
alias col17='awk "{ print \$17 }"'
alias col18='awk "{ print \$18 }"'
alias col19='awk "{ print \$19 }"'
alias col20='awk "{ print \$20 }"'
alias col1-1='awk "{ print \$1, \$1 }"'
alias col1-2='awk "{ print \$1, \$2 }"'
alias col1-3='awk "{ print \$1, \$3 }"'
alias col1-4='awk "{ print \$1, \$4 }"'
alias col1-5='awk "{ print \$1, \$5 }"'
alias col1-6='awk "{ print \$1, \$6 }"'
alias col1-7='awk "{ print \$1, \$7 }"'
alias col1-8='awk "{ print \$1, \$8 }"'
alias col1-9='awk "{ print \$1, \$9 }"'
alias col1-10='awk "{ print \$1, \$10 }"'
alias meldata='grep -o "\[data [^]]*" | cut -d\" -f2'
alias melfile='grep -o "\[file [^]]*" | cut -d\" -f2'
alias melhostname='grep -o "\[hostname [^]]*" | cut -d\" -f2'
alias melid='grep -o "\[id [^]]*" | cut -d\" -f2'
alias melidmsg='sed -e "s/.*\[id \"//" -e "s/\(......\).*\[msg \"/\1 /" -e "s/\"\].*//" -e "s/(Total .*/(Total ...) .../" -e "s/Incoming and Outgoing Score: [0-9]* [0-9]*/Incoming and Outgoing Score: .../"'
alias melip='grep -o "\[client [^]]*" | cut -b9-'
alias melline='grep -o "\[line [^]]*" | cut -d\" -f2'
alias melmatch='grep -o " at [^\ ]*\. \[file" | sed -e "s/\. \[file//" | cut -b5-'
alias melmsg='grep -o "\[msg [^]]*" | cut -d\" -f2 | sed -e "s/(Total .*/(Total ...) .../"'
alias melsummary='grep -o -E " (at|against) .*\[file.*\[id \"[0-9]+.*\[msg \"[^\"]+" | tr -d \" | sed -e "s/ at the end of input at/ at/" -e "s/ required. /. /" -e "s/\[rev .*\[msg/[msg/" -e "s/\. / /" -e "s/(Total .*/(Total ...) .../" | tr -d \] | cut -d\ -f3,9,11- | sed -e "s/^\([^ ]*\) \([^ ]*\)/\2 \1/" | awk "{ printf \"%+6s %-35s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s\n\", \$1, \$2, \$3, \$4, \$5, \$6, \$7, \$8, \$9, \$10, \$11, \$12, \$13, \$14, \$15, \$16, \$17, \$18, \$19, \$20 }" | sed -e "s/\ *$//"'
# melsummary description: We grep for the various ModSec alert messages and take the content from the at/against via the parameter name, the id up and including the message. tr and sed and again tr are then used to strip this down. Now cut is used to extract (1) the parameter, (2) the id and (3) the message. Then we use sed to swap the position of the parameter and the id. Then we used awk to print the three fields in a clean table. This demands the used of a lot of %s fields, which results in a lot of empty spaces at the end of the line, which are finally removed.
alias meltags='tr "]" "\n" | tr "[" "\n" | grep "tag \"" | cut -b6- | tr -d "\""'
alias meltimestamp='cut -b2-25'
alias melunique_id='grep -o "\[unique_id [^]]*" | cut -d\" -f2'
alias meluri='grep -o "\[uri [^]]*" | cut -d\" -f2'
alias pathsegs1='cut -d\/ -f1-2'
alias pathsegs2='cut -d\/ -f1-3'
alias pathsegs3='cut -d\/ -f1-4'
alias pathsegs4='cut -d\/ -f1-5'
alias pathsegs5='cut -d\/ -f1-6'
alias sucs='sort | uniq -c | sort -n'
alias sucspercent='sort | uniq -c | sort -n | $HOME/bin/percent.awk'
alias swapcolumns='awk "{ t = \$1; \$1 = \$2; \$2 = t; print; }"'
alias greppl1='egrep "\"(901001|901450|905100|905110|910000|910100|910150|910160|910170|910180|911100|912120|912170|913100|913110|913120|920100|920120|920130|920140|920160|920170|920180|920190|920210|920220|920240|920250|920260|920270|920280|920290|920310|920311|920330|920340|920350|920360|920370|920380|920390|920400|920410|920420|920430|920440|920450|921100|921110|921120|921130|921140|921150|921160|930100|930110|930120|930130|931100|931110|931120|932100|932105|932110|932115|932120|932130|932140|932150|932160|932170|932171|933100|933110|933120|933130|933140|933150|933160|933170|933180|941100|941110|941120|941130|941140|941150|941160|941170|941180|941190|941200|941210|941220|941230|941240|941250|941260|941270|941280|941290|941300|941310|941350|942100|942140|942160|942170|942190|942220|942230|942240|942250|942270|942280|942290|942320|942350|942360|943100|943110|943120|949100|949110|950130|951110|951120|951130|951140|951150|951160|951170|951180|951190|951200|951210|951220|951230|951240|951250|951260|952100|952110|953100|953110|953120|954100|954110|954120|954130|959100|980100|980110|980120|980130|980140|9001000|9001110|9001112|9001114|9001116|9001120|9001122|9001124|9001126|9001128|9001140|9001150|9001170|9001180|9001182|9001184|9001200|9001202|9001204|9001206|9001208|9001210|9001212|9001214|9001216|9002000|9002001|9002100|9002120|9002130|9002150|9002160|9002200|9002400|9002401|9002410|9002420|9002520|9002530|9002540|9002700|9002710|9002720|9002730|9002740|9002750|9002800|9002810|9002820|9002900)\""'
alias greppl2='egrep "\"(912171|913101|913102|920200|920201|920230|920271|920300|920320|921151|931130|933151|941320|941330|941340|942110|942120|942130|942150|942180|942200|942210|942260|942300|942310|942330|942340|942370|942380|942390|942400|942410|942430|942440|942450|950100)\""'
alias greppl3='egrep "\"(920272|921180|933111|933131|933161|942251|942420|942431|942460)\""'
alias greppl4='egrep "\"(920202|920273|920274|920460|942421|942432)\""'
alias greppl1-2='egrep "\"(901001|901450|905100|905110|910000|910100|910150|910160|910170|910180|911100|912120|912170|913100|913110|913120|920100|920120|920130|920140|920160|920170|920180|920190|920210|920220|920240|920250|920260|920270|920280|920290|920310|920311|920330|920340|920350|920360|920370|920380|920390|920400|920410|920420|920430|920440|920450|921100|921110|921120|921130|921140|921150|921160|930100|930110|930120|930130|931100|931110|931120|932100|932105|932110|932115|932120|932130|932140|932150|932160|932170|932171|933100|933110|933120|933130|933140|933150|933160|933170|933180|941100|941110|941120|941130|941140|941150|941160|941170|941180|941190|941200|941210|941220|941230|941240|941250|941260|941270|941280|941290|941300|941310|941350|942100|942140|942160|942170|942190|942220|942230|942240|942250|942270|942280|942290|942320|942350|942360|943100|943110|943120|949100|949110|950130|951110|951120|951130|951140|951150|951160|951170|951180|951190|951200|951210|951220|951230|951240|951250|951260|952100|952110|953100|953110|953120|954100|954110|954120|954130|959100|980100|980110|980120|980130|980140|9001000|9001110|9001112|9001114|9001116|9001120|9001122|9001124|9001126|9001128|9001140|9001150|9001170|9001180|9001182|9001184|9001200|9001202|9001204|9001206|9001208|9001210|9001212|9001214|9001216|9002000|9002001|9002100|9002120|9002130|9002150|9002160|9002200|9002400|9002401|9002410|9002420|9002520|9002530|9002540|9002700|9002710|9002720|9002730|9002740|9002750|9002800|9002810|9002820|9002900|912171|913101|913102|920200|920201|920230|920271|920300|920320|921151|931130|933151|941320|941330|941340|942110|942120|942130|942150|942180|942200|942210|942260|942300|942310|942330|942340|942370|942380|942390|942400|942410|942430|942440|942450|950100)\""'
alias greppl1-3='egrep "\"(901001|901450|905100|905110|910000|910100|910150|910160|910170|910180|911100|912120|912170|913100|913110|913120|920100|920120|920130|920140|920160|920170|920180|920190|920210|920220|920240|920250|920260|920270|920280|920290|920310|920311|920330|920340|920350|920360|920370|920380|920390|920400|920410|920420|920430|920440|920450|921100|921110|921120|921130|921140|921150|921160|930100|930110|930120|930130|931100|931110|931120|932100|932105|932110|932115|932120|932130|932140|932150|932160|932170|932171|933100|933110|933120|933130|933140|933150|933160|933170|933180|941100|941110|941120|941130|941140|941150|941160|941170|941180|941190|941200|941210|941220|941230|941240|941250|941260|941270|941280|941290|941300|941310|941350|942100|942140|942160|942170|942190|942220|942230|942240|942250|942270|942280|942290|942320|942350|942360|943100|943110|943120|949100|949110|950130|951110|951120|951130|951140|951150|951160|951170|951180|951190|951200|951210|951220|951230|951240|951250|951260|952100|952110|953100|953110|953120|954100|954110|954120|954130|959100|980100|980110|980120|980130|980140|9001000|9001110|9001112|9001114|9001116|9001120|9001122|9001124|9001126|9001128|9001140|9001150|9001170|9001180|9001182|9001184|9001200|9001202|9001204|9001206|9001208|9001210|9001212|9001214|9001216|9002000|9002001|9002100|9002120|9002130|9002150|9002160|9002200|9002400|9002401|9002410|9002420|9002520|9002530|9002540|9002700|9002710|9002720|9002730|9002740|9002750|9002800|9002810|9002820|9002900|912171|913101|913102|920200|920201|920230|920271|920300|920320|921151|931130|933151|941320|941330|941340|942110|942120|942130|942150|942180|942200|942210|942260|942300|942310|942330|942340|942370|942380|942390|942400|942410|942430|942440|942450|950100|920272|921180|933111|933131|933161|942251|942420|942431|942460)\""'
alias greppl1-4='egrep "\"(901001|901450|905100|905110|910000|910100|910150|910160|910170|910180|911100|912120|912170|913100|913110|913120|920100|920120|920130|920140|920160|920170|920180|920190|920210|920220|920240|920250|920260|920270|920280|920290|920310|920311|920330|920340|920350|920360|920370|920380|920390|920400|920410|920420|920430|920440|920450|921100|921110|921120|921130|921140|921150|921160|930100|930110|930120|930130|931100|931110|931120|932100|932105|932110|932115|932120|932130|932140|932150|932160|932170|932171|933100|933110|933120|933130|933140|933150|933160|933170|933180|941100|941110|941120|941130|941140|941150|941160|941170|941180|941190|941200|941210|941220|941230|941240|941250|941260|941270|941280|941290|941300|941310|941350|942100|942140|942160|942170|942190|942220|942230|942240|942250|942270|942280|942290|942320|942350|942360|943100|943110|943120|949100|949110|950130|951110|951120|951130|951140|951150|951160|951170|951180|951190|951200|951210|951220|951230|951240|951250|951260|952100|952110|953100|953110|953120|954100|954110|954120|954130|959100|980100|980110|980120|980130|980140|9001000|9001110|9001112|9001114|9001116|9001120|9001122|9001124|9001126|9001128|9001140|9001150|9001170|9001180|9001182|9001184|9001200|9001202|9001204|9001206|9001208|9001210|9001212|9001214|9001216|9002000|9002001|9002100|9002120|9002130|9002150|9002160|9002200|9002400|9002401|9002410|9002420|9002520|9002530|9002540|9002700|9002710|9002720|9002730|9002740|9002750|9002800|9002810|9002820|9002900|912171|913101|913102|920200|920201|920230|920271|920300|920320|921151|931130|933151|941320|941330|941340|942110|942120|942130|942150|942180|942200|942210|942260|942300|942310|942330|942340|942370|942380|942390|942400|942410|942430|942440|942450|950100|920272|921180|933111|933131|933161|942251|942420|942431|942460|920202|920273|920274|920460|942421|942432)\""'
alias mappl='sed -e "s/901001\|901450\|905100\|905110\|910000\|910100\|910150\|910160\|910170\|910180\|911100\|912120\|912170\|913100\|913110\|913120\|920100\|920120\|920130\|920140\|920160\|920170\|920180\|920190\|920210\|920220\|920240\|920250\|920260\|920270\|920280\|920290\|920310\|920311\|920330\|920340\|920350\|920360\|920370\|920380\|920390\|920400\|920410\|920420\|920430\|920440\|920450\|921100\|921110\|921120\|921130\|921140\|921150\|921160\|930100\|930110\|930120\|930130\|931100\|931110\|931120\|932100\|932105\|932110\|932115\|932120\|932130\|932140\|932150\|932160\|932170\|932171\|933100\|933110\|933120\|933130\|933140\|933150\|933160\|933170\|933180\|941100\|941110\|941120\|941130\|941140\|941150\|941160\|941170\|941180\|941190\|941200\|941210\|941220\|941230\|941240\|941250\|941260\|941270\|941280\|941290\|941300\|941310\|941350\|942100\|942140\|942160\|942170\|942190\|942220\|942230\|942240\|942250\|942270\|942280\|942290\|942320\|942350\|942360\|943100\|943110\|943120\|949100\|949110\|950130\|951110\|951120\|951130\|951140\|951150\|951160\|951170\|951180\|951190\|951200\|951210\|951220\|951230\|951240\|951250\|951260\|952100\|952110\|953100\|953110\|953120\|954100\|954110\|954120\|954130\|959100\|980100\|980110\|980120\|980130\|980140\|9001000\|9001110\|9001112\|9001114\|9001116\|9001120\|9001122\|9001124\|9001126\|9001128\|9001140\|9001150\|9001170\|9001180\|9001182\|9001184\|9001200\|9001202\|9001204\|9001206\|9001208\|9001210\|9001212\|9001214\|9001216\|9002000\|9002001\|9002100\|9002120\|9002130\|9002150\|9002160\|9002200\|9002400\|9002401\|9002410\|9002420\|9002520\|9002530\|9002540\|9002700\|9002710\|9002720\|9002730\|9002740\|9002750\|9002800\|9002810\|9002820\|9002900/& PL1/" -e "s/912171\|913101\|913102\|920200\|920201\|920230\|920271\|920300\|920320\|921151\|931130\|933151\|941320\|941330\|941340\|942110\|942120\|942130\|942150\|942180\|942200\|942210\|942260\|942300\|942310\|942330\|942340\|942370\|942380\|942390\|942400\|942410\|942430\|942440\|942450\|950100/& PL2/" -e "s/920272\|921180\|933111\|933131\|933161\|942251\|942420\|942431\|942460/& PL3/" -e "s/920202\|920273\|920274\|920460\|942421\|942432/& PL4/"'