Skip to content
Permalink
Browse files

tests-scheduler first draft

  • Loading branch information...
ilario committed Aug 17, 2019
1 parent a2e0534 commit 5c046474a32d44b59d725f0e671cf48eae86c834
@@ -0,0 +1,47 @@
#
# Copyright (C) 2019 Ilario Gelmetti
#
# This is free software, licensed under the GNU General Public License v3.
#

include $(TOPDIR)/rules.mk

GIT_COMMIT_DATE:=$(shell git log -n 1 --pretty=%ad --date=short . )
GIT_COMMIT_TSTAMP:=$(shell git log -n 1 --pretty=%at . )

PKG_NAME:=tests-scheduler
PKG_VERSION=$(GIT_COMMIT_DATE)-$(GIT_COMMIT_TSTAMP)

include $(INCLUDE_DIR)/package.mk

define Package/$(PKG_NAME)
CATEGORY:=LiMe
TITLE:=Tests scheduler at peak and night time
MAINTAINER:=Ilario Gelmetti <iochesonome@gmail.com>
URL:=https://libremesh.org
DEPENDS:=+at
PKGARCH:=all
endef

define Package/$(PKG_NAME)/config
endef

define Package/$(PKG_NAME)/description
Monitors the number of clients of the network for each hour,
uses this information for scheduling tests at the peak time and/or at the night time.
endef

define Build/Compile
endef

define Package/$(PKG_NAME)/install
$(INSTALL_DIR) $(1)/bin/
$(INSTALL_DIR) $(1)/etc/uci-defaults/
$(INSTALL_DIR) $(1)/etc/config/
$(INSTALL_BIN) ./files/bin/test-scheduler-at.lua $(1)/bin/tests-scheduler-at
$(INSTALL_BIN) ./files/bin/test-scheduler-probe.lua $(1)/bin/tests-scheduler-probe
$(INSTALL_BIN) ./files/etc/uci-defaults/* $(1)/etc/uci-defaults
$(INSTALL_CONF) ./files/etc/config/tests-scheduler $(1)/etc/config/
endef

$(eval $(call BuildPackage,$(PKG_NAME)))
@@ -0,0 +1,156 @@
#!/usr/bin/lua

local JSON = require("luci.jsonc")

local libuci_loaded, libuci = pcall(require, "uci")

local function config_uci_get(option)
local result
if libuci_loaded then
result = libuci:cursor():get("tests-scheduler","at",option)
else
result = nil
end
return result
end

local peakTestsList = config_uci_get(peak_test) or {arg[1]}

local nightTestsList = config_uci_get(night_test) or {arg[2]}

local dataFile = config_uci_get(data_file) or "/tmp/tests-scheduler-probe-data"

-- see if the file exists
function file_exists(file)
local f = io.open(file, "rb")
if f then f:close() end
return f ~= nil
end

-- get all lines from a file, returns an empty
-- list/table if the file does not exist
function lines_from(file)
if not file_exists(file) then return {} end
local lines = {}
for line in io.lines(file) do
lines[#lines + 1] = line
end
return lines
end


local function do_split(str,pat)
local tbl = {}
str:gsub(pat, function(x) tbl[#tbl+1]=x end)
return tbl
end

local function max(t)
if #t == 0 then return nil, nil end
local key, value = 1, tonumber(t[1])
for i = 2, #t do
temp = tonumber(t[i])
if value < temp then
key, value = i, temp
end
end
return key, value
end

local function min(t)
if #t == 0 then return nil, nil end
local key, value = 1, tonumber(t[1])
for i = 2, #t do
temp = tonumber(t[i])
if value > temp then
key, value = i, temp
end
end
return key, value
end

local data = lines_from(dataFile)
local complete = true
for _,val in pairs(data) do
if not tonumber(val) then
complete = false
break
end
end

if complete then
io.stderr:write("Found enough data in "..dataFile..", continuing.\n")
local peakHour1,_ = max(data)
local peakHour = peakHour1 - 1
for _,peakCommand in pairs(peakTestsList) do
local peakAt = "echo '"..peakCommand.."' | at -Mv "..peakHour..":30 2>&1"
local handlePeakAt = io.popen(peakAt, 'r')
local peakAtRaw = handlePeakAt:read("*a")
handlePeakAt:close()
local peakAtTime = do_split(peakAtRaw,"%C+")[1]
print(peakAtTime.."\t"..peakCommand)
end
if nightTestsList then
local datatemp = data
local nightHours = {}
for i = 1,6 do
nightHour1,_ = min(datatemp)
nightHours[i] = nightHour1 - 1
datatemp[nightHour1] = math.huge
end

local getCommand = "shared-state get tests-scheduler-night"
local handleAllTimesJson = io.popen(getCommand, "r")
local allTimesJson = handleAllTimesJson:read("*a")
handleAllTimesJson:close()
local allTimes = {}
for _,value in pairs(JSON.parse(allTimeJson)) do
allTimes[#allTimes + 1] = tonumber(value.data)
end

local hitsHours = {0,0,0,0,0,0}
for _,val in pairs(allTimes) do
for i = 1,6 do
local valHour = int(val/60)
if valHour == nightHours[i] then
hitsHours[i] = hitsHours[i] + 1
break
end
end
end

local minHourIndex,_ = min(hitsHours)
local nightHour = nightHours[minHourIndex]

local hits5minute = {0,0,0,0,0,0,0,0,0,0,0,0}
for _,val in pairs(allTimes) do
local valHour = int(val/60)
if valHour == nightHour then
local val5minute = 1 + int((val%60)/5)
hits5minute[val5minute] = hits5minute[val5minute] + 1
end
end

local min5minute1,_ = min(hits5minute)
local min5minute = min5minute1 - 1
local nightMinute = min5minute * 5 + math.random(0,4)
local myTime = nightHour * 60 + nightMinute
local myTimeTable = {}
local hostname = io.input("/proc/sys/kernel/hostname"):read("*line")
myTimeTable[hostname] = myTime
io.popen("shared-state insert tests-scheduler-night", "w"):write(JSON.stringify(myTimeTable))
for _,nightCommand in pairs(nightTestsList) do
local nightAt = "echo '"..nightCommand.."' | at -Mv "..nightHour..":"..nightMinute.." 2>&1"
local handleNightAt = io.popen(nightAt, 'r')
local nightAtRaw = handleNightAt:read("*a")
handleNightAt:close()
local nightAtTime = do_split(nightAtRaw,"%C+")[1]
print(nightAtTime.."\t"..nightCommand)
end
end
else
io.stderr:write("Data from at least one whole day is needed in "..dataFile..", stopping.\n")
os.exit(1)
end


@@ -0,0 +1,111 @@
#!/usr/bin/lua

local libuci_loaded, libuci = pcall(require, "uci")

local function config_uci_get(option)
local result
if libuci_loaded then
result = libuci:cursor():get("tests-scheduler","probe",option)
else
result = nil
end
return result
end

local defaultTestsList = {
"ip neigh show nud reachable",
"ping6 -n -c 2 ff02::1%br-lan",
"batctl tl"
}

local temp = config_uci_get(test)
local testsList = type(temp) == "table" and temp or defaultTestsList

-- stability indicates how much of the previous measurement data
-- (which could be from the previous day) is going to be considered
-- 0 means that just the very last is used, avoid using 1
-- consider that more than one measurement is performed each hour
local stability = tonumber(config_uci_get(stability)) or 0.9

local dataFile = config_uci_get(data_file) or "/tmp/tests-scheduler-probe-data"

local function do_test(test)
io.stderr:write("Command: "..test.."\n")
local handle = io.popen(test, 'r')
local output = handle:read("*a")
local rc = {handle:close()}
local _,count = output:gsub('\n', '\n')
local exitCode = tonumber(rc[3])
return count, exitCode
end

local function do_sum(arr, length)
return length == 1 and arr[1] or arr[length] + do_sum(arr, length -1)
end

local function do_tests_serie()
local results = {}
local i = 1
while testsList[i] do
local test = tostring(testsList[i])
local testResult,exitCode = do_test(test)
if exitCode == 0 then
io.stderr:write("Result: "..tostring(testResult).."\n")
results[#results + 1] = testResult
else
io.stderr:write("Failed with exit code "..exitCode.."\n")
end
i = i + 1
end
local sum = do_sum(results, #results)
return sum
end

-- see if the file exists
function file_exists(file)
local f = io.open(file, "rb")
if f then f:close() end
return f ~= nil
end

-- get all lines from a file, returns an empty
-- list/table if the file does not exist
function lines_from(file)
if not file_exists(file) then return {} end
local lines = {}
for line in io.lines(file) do
lines[#lines + 1] = line
end
return lines
end

local function calculate_all_data()
data = lines_from(dataFile)
local hour = os.date("*t")["hour"]
local hour1 = hour + 1
newResult = do_tests_serie()
for i = 1,24 do
if not data[i] then
data[i] = ""
end
end
data[hour1] = tonumber(data[hour1]) and
data[hour1] * stability + newResult * (1 - stability) or
newResult
return data,hour,newResult,data[hour1]
end

local function save_data(data, file)
-- empty the file
io.open(file,"w"):close()
local handle = io.open(file,"a")
io.output(handle)
for _,val in pairs(data) do
io.write(val.."\n")
end
io.close(handle)
end

local data,hour,result,cumulativeResult = calculate_all_data()
save_data(data, dataFile)
io.stdout:write(hour.."\t"..result.."\t"..cumulativeResult.."\n")
@@ -0,0 +1,11 @@
config probe 'probe'
list test 'ip neigh show nud reachable'
list test 'ping6 -n -c 2 ff02::1%br-lan'
list test 'batctl tl'
option stability 0.9
option data_file '/tmp/tests-scheduler-probe-data'

config at 'at'
option data_file '/tmp/tests-scheduler-probe-data'
list peak_test 'lime-report | gzip -c >> /tmp/tests-peak.gz'
list night_test '(date && bandwidth-test) >> /tmp/tests-night'
@@ -0,0 +1,12 @@
#!/bin/sh

unique_append()
{
grep -qF "$1" "$2" || echo "$1" >> "$2"
}

unique_append \
'*/15 * * * * ((/bin/tests-scheduler-probe)&)'\
/etc/crontabs/root

exit 0
@@ -0,0 +1,15 @@
#!/bin/sh

hour=$(awk 'BEGIN{srand();print int(rand()*24)}')
minute=$(awk 'BEGIN{srand();print int(rand()*60)}')

unique_append()
{
grep -qF "$1" "$2" || echo "$minute $hour $1" >> "$2"
}

unique_append \
'* * * ((/bin/tests-scheduler-at; shared-state sync tests-scheduler-night)&)'\
/etc/crontabs/root

exit 0

0 comments on commit 5c04647

Please sign in to comment.
You can’t perform that action at this time.