Skip to content
SEO Dashboard using R and ScreamingFrog for Reporting and Monitoring
R Batchfile
Branch: master
Clone or download
Latest commit c3809f7 Jun 27, 2016
Permalink
Type Name Latest commit message Commit time
Failed to load latest commit information.
certificat add certificate Jun 20, 2016
tools add new kibana.yml for kibana 4.5.1 Jun 24, 2016
websites/dataseo changes for kibana 4.5.1 in order to use ES 2.2.3 Jun 24, 2016
.gitignore detect null for GA Sessions Jun 22, 2016
LICENSE first commit Jun 5, 2016
README.md minor changes Jun 25, 2016
SEO-Dashboard.Rproj
TODO.md
build.bat add new directories Jun 27, 2016
run-filebeat.bat refacto Jun 21, 2016
run-kibana.bat
run-r.bat autoinstall packages Jun 22, 2016
step1_analyseUrl.R detect null for GA Sessions Jun 22, 2016
step2_createDashboard.R autoinstall packages Jun 22, 2016
step3_computeInternalPageRank.R autoinstall packages Jun 22, 2016

README.md

SEO-Dashboard

SEO Dashboard using R and ScreamingFrog for Reporting and Monitoring

Read more :


Paaslogs Configuration

Input section

input { beats { port => 5044 ssl => true ssl_certificate => "/etc/ssl/private/server.crt" ssl_key => "/etc/ssl/private/server.key" } }

Filter section

filter { mutate { rename => { "source" => "filename" } }

if [type] == "apache" {  
  mutate {
      add_field => { 
             "section" => "nohtml"
             "active" => "FALSE"
       }
    }
   grok {
       match => { "message" => "%{OVHCOMMONAPACHELOG}" }
       patterns_dir => "/opt/logstash/patterns"
   }
   if ("_grokparsefailure" in [tags]) {
       mutate {
          remove_tag => [ "_grokparsefailure" ]
        }
       grok {
          match => [ "message", "%{OVHCOMBINEDAPACHELOG}" ]
          patterns_dir => "/opt/logstash/patterns"
         }
    }
    elasticsearch { 
      hosts => "laas.runabove.com" 
      index => "XXyourindexXX" 
      user => "ra-logs-XXX" 
      password => "2OkHXXXXXXX"        
      ssl => true 
      query => 'type:csv AND request:"%{[request]}"'
      fields => [["section","section"],["active","active"],["speed","speed"],["compliant","compliant"],["depth","depth"],["inlinks","inlinks"],["outlinks","outlinks"],["status_title","status_title"],["status_description","status_description"],["status_h1","status_h1"],["group_inlinks","group_inlinks"],["group_wordcount","group_wordcount"]]
    }
   dns {
      action => "replace"
      reverse => [ "clientip" ]
   }
   if [clientip] =~ /googlebot.com/ { 
      mutate {
         add_field => { "bot" => "google" }
     }
   }
  if [clientip] =~ /search.msn.com/ { 
     mutate {
        add_field => { "bot" => "bing" }
     }
  }
  
  mutate {
     convert => { 
         "depth" => "integer"
         "inlinks" => "integer"
         "outlinks" => "integer"
     }
  }	  
  
}
if [type] == "csv" {
    csv {
        columns => ["request", "section","active", "speed", "compliant","depth","inlinks","outlinks","status_title","status_description","status_h1","group_inlinks","group_wordcount"]
        separator => ";"
    } 
}

}

Custom grok patterns

  • OVHCOMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} [%{HTTPDATE:timestamp}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion_num:float})?|%{DATA:rawrequest})" %{NUMBER:response_int:int} (?:%{NUMBER:bytes_int:int}|-)
  • OVHCOMBINEDAPACHELOG %{OVHCOMMONAPACHELOG} "%{NOTSPACE:referrer}" %{QS:agent}
  • OVHCOMMONIISLOG %{TIMESTAMP_ISO8601:log_timestamp} %{WORD:iisSite} %{IPORHOST:site} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:port} %{NOTSPACE:username} %{IPORHOST:clienthost} %{NOTSPACE:useragent} %{NOTSPACE:referer} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:scstatus} %{NUMBER:bytes:int} %{NUMBER:timetaken:int}

You can’t perform that action at this time.