diff --git a/samples/cmb-getdates.sh b/samples/cmb-getdates.sh new file mode 100644 index 0000000..2ec7012 --- /dev/null +++ b/samples/cmb-getdates.sh @@ -0,0 +1,61 @@ +source ~/.bashrc +arg1="$1" +simfile="" +if [ "$arg1" = "sim" ] +then +simfile="sim" +elif [[ "$arg1" = *"safety"* ]] +then +simfile="$arg1" +fi +if [ "$2" != "" ] +then +arg2=$2 +fi +if [[ "$arg1" = "" || "$arg1" = "s3" || "$arg1" = "sim" || "$arg1" = "1" || "$arg1" = "2" || "$arg1" = "3" || "$arg1" = "4" || "$arg1" = *"safety"* ]] +then + cmbdates="" + IFS=$'\n' + if [ "$arg1" = "1" -o "$arg1" = "2" -o "$arg1" = "3" -o "$arg1" = "4" ] + then + echo $simfile + end=`expr $arg2 + 1` + start=`expr $arg2 + 1 - $arg1` + grabbingdates="tail -n+2 ${simfile}getdates.csv | sed -n \"$start,$end" + grabbingdates=$grabbingdates"p\" ${simfile}getdates.csv | sed 's/,/ /g'" + echo $grabbingdates + else + grabbingdates="tail -n+2 ${simfile}getdates.csv | sed 's/,/ /g'" + fi + for row in $(eval $grabbingdates) + do + IFS=$' ' + set -- $row + if [ "$cmbdates" = "" ] + then + if [ "$arg1" = "sim" ] + then + cmbdates="(t >= ($2 _ 1000) && t <= ($3 _ 1000))" + else + cmbdates="(t >= $2 && t <= $3)" + fi + firststart=$2 + else + if [ "$arg1" = "sim" ] + then + #cmbdates="$cmbdates || (t >= ($2 "'* 1000) && t <= ('"$3 "'* 1000))' + cmbdates="$cmbdates || (t >= ($2 _ 1000) && t <= ($3 _ 1000))" + else + cmbdates="$cmbdates || (t >= $2 && t <= $3)" + fi + fi + lastend=$3 + done + if [ "$arg1" = "sim" ] + then + cmbdates="'"$cmbdates"'" + else + cmbdates=$cmbdates"" + fi + echo $cmbdates +fi diff --git a/samples/runweekly.R b/samples/runweekly.R new file mode 100644 index 0000000..687b715 --- /dev/null +++ b/samples/runweekly.R @@ -0,0 +1,206 @@ +runweekly <- function(wstart,wend,verbose=F,plot=F,tplot=F,model="",addtotitle="",levered=100000,unlevered=100000,override="",balance=20000,marginutilization=20,leverage=50,compounding=T,commissions=0.000036,d=NULL,weektoday=F,systemcommand="",metrics=T,legend=F,infnull=F,meanrate=1,udbthread=1,normalized=T,pippercent=100) { + if (commissions!=0.000036) { + print(paste0("USING COMMISSIONS OF ",commissions)) + } + # print(paste0(wstart, ' to ', wend)) + if (compounding==T) { + metrictitle <- paste0(leverage,"x Leveraged Returns/",balance," Balance/",marginutilization,"% Margin Utilization/Compounding") + } else { + metrictitle <- paste0(leverage,"x Leveraged Returns/",balance," Balance/",marginutilization,"% Margin Utilization/Not Compounding") + } + if (systemcommand == "") { + print(metrictitle) + } + origdigits <- options()$digits + options(digits=15) + options(scipen = 999) + if (systemcommand == "") { + print(paste("Converting",model,"to Weekly Performance")) + } + if (addtotitle == "") { + addtotitle <- model + } else { + addtotitle <- paste(model,addtotitle,':') + } + if (is.null(d)) { + dates <- read.csv("safetygetdates.csv") + d <- dates[,2:3] + } + if (weektoday) { + wstart <- (wstart - 1)*6 + 1 + wend <- (wend - 1)*6 + 6 + } + if (systemcommand == "") { + plotattempt <- try(filteredtrades <- read.csv('modeltrades.csv',header=F)) + if (any(grepl("error",class(plotattempt)))) { + filteredtrades <- NULL + } + } else { + filteredtrades <- read.csv(pipe(systemcommand,open='r'),header=F) + } + if (length(filteredtrades)) { + xtstrades <- filteredtrades + xtsprofits <- xtstrades #[seq(2,nrow(xtstrades),2),] + + t1 <- Sys.time() + weekperf1 <- matrix(nrow=(wend - wstart + 1),ncol=1) + lastprofit <- 0 + weeklytrades <- NULL + if (wstart != 1) { + # print(wstart) + weeklysum <- 0 + ntrades <- 0 + weeklytrades <- as.numeric(xtsprofits[which(floor(as.numeric(xtsprofits[,4])/1000) >= d[(wstart - 1),1] & floor(as.numeric(xtsprofits[,4])/1000) <= d[(wstart - 1),2]),9]) + allweeklytrades <- xtsprofits[which(floor(as.numeric(xtsprofits[,4])/1000) >= d[(wstart - 1),1] & floor(as.numeric(xtsprofits[,4])/1000) <= d[(wstart - 1),2]),] + ntrades <- length(weeklytrades) + if (normalized) { + lastprofit <- ifelse(length(weeklytrades[ntrades]),tail(cumsum((allweeklytrades[which(allweeklytrades[,2]=="SELL"),5] - allweeklytrades[which(allweeklytrades[,2]=="BUY"),5])/allweeklytrades[which(allweeklytrades[,2]=="BUY"),5] - commissions),1),0) + } else { + lastprofit <- ifelse(length(weeklytrades[ntrades]),tail(cumsum(allweeklytrades[which(allweeklytrades[,2]=="SELL"),5] - allweeklytrades[which(allweeklytrades[,2]=="BUY"),5] - commissions),1),0) + } + if (verbose) { + print(paste((wstart - 1),ntrades,weeklysum,ifelse(length(weeklyprofit),weeklyprofit,0))) + print(lastprofit) + } + } + totaltrades <- 0 + for (week in wstart:wend) { + weeklysum <- 0 + ntrades <- 0 + weeklytrades <- as.numeric(xtsprofits[which(floor(as.numeric(xtsprofits[,4])/1000) >= d[week,1] & floor(as.numeric(xtsprofits[,4])/1000) <= d[week,2]),9]) +allweeklytrades <- xtsprofits[which(floor(as.numeric(xtsprofits[,4])/1000) >= d[week,1] & floor(as.numeric(xtsprofits[,4])/1000) <= d[week,2]),] +# print(allweeklytrades) + ntrades <- length(weeklytrades) + # print(paste0("Number of Trades: ",ntrades)) + totaltrades <- totaltrades + ntrades + # print(ntrades) + weeklysum <- weeklytrades[ntrades] - lastprofit + # print(weeklysum) + weeklyprofit <- weeklysum - (ntrades * commissions) #0.00008) + # print(weeklyprofit) + if (normalized) { + weekperf1[(week-wstart+1),1] <- ifelse(length(weeklyprofit),tail(cumsum((allweeklytrades[which(allweeklytrades[,2]=="SELL"),5] - allweeklytrades[which(allweeklytrades[,2]=="BUY"),5])/allweeklytrades[which(allweeklytrades[,2]=="BUY"),5] - commissions),1),0) + } else { + weekperf1[(week-wstart+1),1] <- ifelse(length(weeklyprofit),tail(cumsum(allweeklytrades[which(allweeklytrades[,2]=="SELL"),5] - allweeklytrades[which(allweeklytrades[,2]=="BUY"),5] - commissions),1),0) + } + # print(weekperf1[(week-wstart+1),1]) + if (length(weeklyprofit)) { + lastprofit <- weeklytrades[ntrades] + } + if (verbose) { + print(paste(week,ntrades,weeklysum,ifelse(length(weeklyprofit),weeklyprofit,0))) + } + } + # print(paste0('Total Number of Trades: ',totaltrades)) +# if (meanrate != 1) { +# print(paste('Converting pip profits to percent profit using meanrate of',meanrate)) +# weekperf1 <- weekperf1 / meanrate +# } + if (normalized) { + weekperf1 <- weekperf1*pippercent + } + cumweekperf <- cumsum(weekperf1) + if (plot) { + plot(wstart:wend,cumweekperf,type='l',col=3,main=paste(addtotitle)) #,'Week',wstart,'to',wend,'Performance')) + if (legend==T) {legend("topleft",legend = c(paste('Weekly Perf',model,collape=' ')), lty = c(1), col=c(3))} + abline(0,0) + } + if (tplot) { + # library(timeSeries) + # library(quantmod) + if (! "xts" %in% (.packages())) { + require(xts) + } + if (! "PerformanceAnalytics" %in% (.packages())) { + require(PerformanceAnalytics) + } + origperf <- xts(levered*(weekperf1/pippercent)/unlevered*100, order.by=seq(as.POSIXct(d[wstart,1],origin="1970-01-01"), + as.POSIXct(d[wend,1],origin="1970-01-01"),length.out=length(wstart:wend))) + t1 <- xts(levered*(cumweekperf/pippercent)/unlevered*100, order.by=seq(as.POSIXct(d[wstart,1],origin="1970-01-01"), + as.POSIXct(d[wend,1],origin="1970-01-01"),length.out=length(wstart:wend))) + if (metrics==T) { + # require(PerformanceAnalytics) + print(Return.annualized(origperf/100, scale = 52, geometric = FALSE)) + print(Return.annualized(origperf/100, scale = 52, geometric = TRUE)) + } + unleveredamount <- "Cumulative Net Profit [%]" + if (levered != 100000 || unlevered != 100000) { + unleveredamount <- paste0("Cumulative Profit [%]: Utilizing ",unlevered," Units Per Trade") + } + if (override!="") { + unleveredamount <- override + } + PerformanceAnalytics::chart.TimeSeries(t1,ylab=paste0(unleveredamount),main=paste(addtotitle,system(paste0('date -d @',d[wstart,1],' +%F'),intern=T),'to',system(paste0('date -d @',d[wend,2],' +%F'),intern=T),'Performance'),xlab = "Date (GMT)",cex.lab=.8) + if (legend==T) {legend("topleft",legend = c(paste('Weekly Perf',model,collape=' ')), lty = c(1), col=c(3))} + abline(a=0,b=0) + if (leverage!=1) { + sim_vec.xts <- xts(weekperf1/pippercent, order.by=seq(as.POSIXct(d[wstart,1],origin="1970-01-01"), + as.POSIXct(d[wend,1],origin="1970-01-01"),length.out=length(wstart:wend))) + temp <- sim_vec.xts + perf_adj <- rep(0,nrow(temp)) + bal <- balance #20000 + mar <- bal * (marginutilization / 100) #2000 + lot <- mar * leverage #100000 + for(i in 1:nrow(temp)) { + pl <- as.numeric((coredata(temp[i,1])) * lot) + perf_adj[i] <- as.numeric(pl / bal) + bal <- bal + pl + mar <- bal * (marginutilization / 100) # / 5 + if (compounding==T) { + lot <- mar * leverage #50 + } + # print(lot) + } + + temp[,1] <- perf_adj + weekperf1 <- temp*pippercent + + if (metrics==T) { + metrics <- as.table(rbind(Return.annualized(temp, scale = 52, geometric = T), + AverageDrawdown(temp), + maxDrawdown(temp), + AverageRecovery(temp), + SharpeRatio.annualized(temp, Rf = 0.004/52, scale = 52, geometric = TRUE), + UpsidePotentialRatio(temp, MAR = 0.004/52, method = "full"), + KellyRatio(temp, Rf = 0.004/52, method = "half"))) + print(metrics) + } + cumweekperf <- cumsum(weekperf1) + chart.TimeSeries(cumsum(temp)*100,main=metrictitle,ylab = "Cumulative Net Profit [%]",cex.lab=0.8,cex.main=0.7,xlab="Time") + } + } + } else { + print("FAILURE: zero lines in trade file: modeltrades.csv") + if (infnull) { + weekperf1 <- NULL + cumweekperf <- NULL + } else { + weekperf1 <- rep(-Inf,wend - wstart + 1) + cumweekperf <- rep(-Inf,wend - wstart + 1) + } + } + if (systemcommand=="") { + return(list('weekperf'=weekperf1,'cumweekperf'=cumweekperf)) + } else { + if (!is.null(length(weekperf1))) { + if (length(weekperf1) != 0) { + if (tail(cumweekperf,1)>0) { + print(paste('Result:',-tail(cumweekperf,1)*1/(max(rollmax(cumweekperf,2)-cumweekperf[-1])*sum(weekperf1<=0)/length(weekperf1))**4,-tail(cumweekperf,1),sum(weekperf1<=0),length(weekperf1),max(rollmax(cumweekperf,2)-cumweekperf[-1]))) + return(-tail(cumweekperf,1)*1/(max(rollmax(cumweekperf,2)-cumweekperf[-1])*sum(weekperf1<=0)/length(weekperf1))**4) + } else { + print(paste('Result:',-tail(cumweekperf,1)*(max(rollmax(cumweekperf,2)-cumweekperf[-1])*sum(weekperf1<=0)/length(weekperf1))**4,-tail(cumweekperf,1),sum(weekperf1<=0),length(weekperf1),max(rollmax(cumweekperf,2)-cumweekperf[-1]))) + return(-tail(cumweekperf,1)*(max(rollmax(cumweekperf,2)-cumweekperf[-1])*sum(weekperf1<=0)/length(weekperf1))**4) + } + } else { + print("Result: Inf") + return(Inf) + } + } else { + print("Result: Inf") + return(Inf) + } + } + options(scipen = 0) + options(digits=origdigits) +} + diff --git a/samples/safetygetdates.csv b/samples/safetygetdates.csv new file mode 100644 index 0000000..0f39b7d --- /dev/null +++ b/samples/safetygetdates.csv @@ -0,0 +1,732 @@ +"row","sunday","friday" +1,1199657760,1200065399 +2,1200262560,1200670199 +3,1200867360,1201274999 +4,1201472160,1201879799 +5,1202076960,1202484599 +6,1202681760,1203089399 +7,1203286560,1203694199 +8,1203891360,1204298999 +9,1204496160,1204903799 +10,1205097360,1205504999 +11,1205702160,1206109799 +12,1206306960,1206714599 +13,1206911760,1207319399 +14,1207516560,1207924199 +15,1208121360,1208528999 +16,1208726160,1209133799 +17,1209330960,1209738599 +18,1209935760,1210343399 +19,1210540560,1210948199 +20,1211145360,1211552999 +21,1211750160,1212157799 +22,1212354960,1212762599 +23,1212959760,1213367399 +24,1213564560,1213972199 +25,1214169360,1214576999 +26,1214774160,1215181799 +27,1215378960,1215786599 +28,1215983760,1216391399 +29,1216588560,1216996199 +30,1217193360,1217600999 +31,1217798160,1218205799 +32,1218402960,1218810599 +33,1219007760,1219415399 +34,1219612560,1220020199 +35,1220217360,1220624999 +36,1220822160,1221229799 +37,1221426960,1221834599 +38,1222031760,1222439399 +39,1222636560,1223044199 +40,1223241360,1223648999 +41,1223846160,1224253799 +42,1224450960,1224858599 +43,1225055760,1225463399 +44,1225664160,1226071799 +45,1226268960,1226676599 +46,1226873760,1227281399 +47,1227478560,1227886199 +48,1228083360,1228490999 +49,1228688160,1229095799 +50,1229292960,1229700599 +51,1229897760,1230305399 +52,1230502560,1230910199 +1,1231107360,1231514999 +2,1231712160,1232119799 +3,1232316960,1232724599 +4,1232921760,1233329399 +5,1233526560,1233934199 +6,1234131360,1234538999 +7,1234736160,1235143799 +8,1235340960,1235748599 +9,1235945760,1236353399 +10,1236546960,1236954599 +11,1237151760,1237559399 +12,1237756560,1238164199 +13,1238361360,1238768999 +14,1238966160,1239373799 +15,1239570960,1239978599 +16,1240175760,1240583399 +17,1240780560,1241188199 +18,1241385360,1241792999 +19,1241990160,1242397799 +20,1242594960,1243002599 +21,1243199760,1243607399 +22,1243804560,1244212199 +23,1244409360,1244816999 +24,1245014160,1245421799 +25,1245618960,1246026599 +26,1246223760,1246631399 +27,1246828560,1247236199 +28,1247433360,1247840999 +29,1248038160,1248445799 +30,1248642960,1249050599 +31,1249247760,1249655399 +32,1249852560,1250260199 +33,1250457360,1250864999 +34,1251062160,1251469799 +35,1251666960,1252074599 +36,1252271760,1252679399 +37,1252876560,1253284199 +38,1253481360,1253906999 +39,1254086160,1254511799 +40,1254690960,1255116599 +41,1255295760,1255721399 +42,1255900560,1256326199 +43,1256505360,1256930999 +44,1257113760,1257539399 +45,1257718560,1258144199 +46,1258323360,1258748999 +47,1258928160,1259353799 +48,1259532960,1259958599 +49,1260137760,1260563399 +50,1260742560,1261168199 +51,1261347360,1261772999 +52,1261952160,1262377799 +1,1262556960,1262982599 +2,1263161760,1263587399 +3,1263766560,1264192199 +4,1264371360,1264796999 +5,1264976160,1265401799 +6,1265580960,1266006599 +7,1266185760,1266611399 +8,1266790560,1267216199 +9,1267395360,1267820999 +10,1268000160,1268425799 +11,1268601360,1269026999 +12,1269206160,1269631799 +13,1269810960,1270236599 +14,1270415760,1270841399 +15,1271020560,1271446199 +16,1271625360,1272050999 +17,1272230160,1272655799 +18,1272834960,1273260599 +19,1273439760,1273865399 +20,1274044560,1274470199 +21,1274649360,1275074999 +22,1275254160,1275679799 +23,1275858960,1276284599 +24,1276463760,1276889399 +25,1277068560,1277494199 +26,1277673360,1278098999 +27,1278278160,1278703799 +28,1278882960,1279308599 +29,1279487760,1279913399 +30,1280092560,1280518199 +31,1280697360,1281122999 +32,1281302160,1281727799 +33,1281906960,1282332599 +34,1282511760,1282937399 +35,1283116560,1283542199 +36,1283721360,1284146999 +37,1284326160,1284751799 +38,1284930960,1285356599 +39,1285535760,1285961399 +40,1286140560,1286566199 +41,1286745360,1287170999 +42,1287350160,1287775799 +43,1287954960,1288380599 +44,1288559760,1288985399 +45,1289168160,1289593799 +46,1289772960,1290198599 +47,1290377760,1290803399 +48,1290982560,1291408199 +49,1291587360,1292012999 +50,1292192160,1292617799 +51,1292796960,1293222599 +52,1293401760,1293827399 +1,1294006560,1294432199 +2,1294611360,1295036999 +3,1295216160,1295641799 +4,1295820960,1296246599 +5,1296425760,1296851399 +6,1297030560,1297456199 +7,1297635360,1298060999 +8,1298240160,1298665799 +9,1298844960,1299270599 +10,1299449760,1299875399 +11,1300050960,1300476599 +12,1300655760,1301081399 +13,1301260560,1301686199 +14,1301865360,1302290999 +15,1302470160,1302895799 +16,1303074960,1303500599 +17,1303679760,1304105399 +18,1304284560,1304710199 +19,1304889360,1305314999 +20,1305494160,1305919799 +21,1306098960,1306524599 +22,1306703760,1307129399 +23,1307308560,1307734199 +24,1307913360,1308338999 +25,1308518160,1308943799 +26,1309122960,1309548599 +27,1309727760,1310153399 +28,1310332560,1310758199 +29,1310937360,1311362999 +30,1311542160,1311967799 +31,1312146960,1312572599 +32,1312751760,1313177399 +33,1313356560,1313782199 +34,1313961360,1314386999 +35,1314566160,1314991799 +36,1315170960,1315596599 +37,1315775760,1316201399 +38,1316380560,1316806199 +39,1316985360,1317410999 +40,1317590160,1318015799 +41,1318194960,1318620599 +42,1318799760,1319225399 +43,1319404560,1319830199 +44,1320009360,1320434999 +45,1320617760,1321043399 +46,1321222560,1321648199 +47,1321827360,1322252999 +48,1322432160,1322857799 +49,1323036960,1323462599 +50,1323641760,1324070999 +51,1324246560,1324675799 +52,1324851360,1325280599 +1,1325456160,1325885399 +2,1326060960,1326490199 +3,1326665760,1327094999 +4,1327270560,1327699799 +5,1327875360,1328304599 +6,1328480160,1328909399 +7,1329084960,1329514199 +8,1329689760,1330118999 +9,1330294560,1330723799 +10,1330899360,1331328599 +11,1331500560,1331929799 +12,1332105360,1332534599 +13,1332710160,1333139399 +14,1333314960,1333744199 +15,1333919760,1334348999 +16,1334524560,1334953799 +17,1335129360,1335558599 +18,1335734160,1336163399 +19,1336338960,1336768199 +20,1336943760,1337372999 +21,1337548560,1337977799 +22,1338153360,1338582599 +23,1338758160,1339187399 +24,1339362960,1339792199 +25,1339967760,1340396999 +26,1340572560,1341001799 +27,1341177360,1341606599 +28,1341782160,1342211399 +29,1342386960,1342816199 +30,1342991760,1343420999 +31,1343596560,1344025799 +32,1344201360,1344630599 +33,1344806160,1345235399 +34,1345410960,1345840199 +35,1346015760,1346444999 +36,1346620560,1347049799 +37,1347225360,1347654599 +38,1347830160,1348259399 +39,1348434960,1348864199 +40,1349039760,1349468999 +41,1349644560,1350073799 +42,1350249360,1350678599 +43,1350854160,1351283399 +44,1351458960,1351888199 +45,1352067360,1352496599 +46,1352672160,1353101399 +47,1353276960,1353706199 +48,1353881760,1354310999 +49,1354486560,1354915799 +50,1355091360,1355520599 +51,1355696160,1356125399 +52,1356300960,1356730199 +53,1356905760,1357334999 +1,1357510560,1357939799 +2,1358115360,1358544599 +3,1358720160,1359149399 +4,1359324960,1359754199 +5,1359929760,1360358999 +6,1360534560,1360963799 +7,1361139360,1361568599 +8,1361744160,1362173399 +9,1362348960,1362778199 +10,1362950160,1363379399 +11,1363554960,1363984199 +12,1364159760,1364588999 +13,1364764560,1365193799 +14,1365369360,1365798599 +15,1365974160,1366403399 +16,1366578960,1367008199 +17,1367183760,1367612999 +18,1367788560,1368217799 +19,1368393360,1368822599 +20,1368998160,1369427399 +21,1369602960,1370032199 +22,1370207760,1370636999 +23,1370812560,1371241799 +24,1371417360,1371846599 +25,1372022160,1372451399 +26,1372626960,1373056199 +27,1373231760,1373660999 +28,1373836560,1374265799 +29,1374441360,1374870599 +30,1375046160,1375475399 +31,1375650960,1376080199 +32,1376255760,1376684999 +33,1376860560,1377289799 +34,1377465360,1377894599 +35,1378070160,1378499399 +36,1378674960,1379104199 +37,1379279760,1379708999 +38,1379884560,1380313799 +39,1380489360,1380918599 +40,1381094160,1381523399 +41,1381698960,1382128199 +42,1382303760,1382732999 +43,1382908560,1383337799 +44,1383516960,1383946199 +45,1384121760,1384550999 +46,1384726560,1385155799 +47,1385331360,1385760599 +48,1385936160,1386365399 +49,1386540960,1386970199 +50,1387145760,1387574999 +51,1387750560,1388179799 +52,1388355360,1388784599 +1,1388960160,1389389399 +2,1389564960,1389994199 +3,1390169760,1390598999 +4,1390774560,1391203799 +5,1391379360,1391808599 +6,1391984160,1392413399 +7,1392588960,1393018199 +8,1393193760,1393622999 +9,1393798560,1394227799 +10,1394399760,1394828999 +11,1395004560,1395433799 +12,1395609360,1396038599 +13,1396214160,1396643399 +14,1396818960,1397248199 +15,1397423760,1397852999 +16,1398028560,1398457799 +17,1398633360,1399062599 +18,1399238160,1399667399 +19,1399842960,1400272199 +20,1400447760,1400876999 +21,1401052560,1401481799 +22,1401657360,1402086599 +23,1402262160,1402691399 +24,1402866960,1403296199 +25,1403471760,1403900999 +26,1404076560,1404505799 +27,1404681360,1405110599 +28,1405286160,1405715399 +29,1405890960,1406320199 +30,1406495760,1406924999 +31,1407100560,1407529799 +32,1407705360,1408134599 +33,1408310160,1408739399 +34,1408914960,1409344199 +35,1409519760,1409948999 +36,1410124560,1410553799 +37,1410729360,1411158599 +38,1411334160,1411763399 +39,1411938960,1412368199 +40,1412543760,1412972999 +41,1413148560,1413577799 +42,1413753360,1414182599 +43,1414358160,1414787399 +44,1414966560,1415395799 +45,1415571360,1416000599 +46,1416176160,1416605399 +47,1416780960,1417210199 +48,1417385760,1417814999 +49,1417990560,1418419799 +50,1418595360,1419024599 +51,1419200160,1419629399 +52,1419804960,1420234199 +1,1420409760,1420838999 +2,1421014560,1421443799 +3,1421619360,1422048599 +4,1422224160,1422653399 +5,1422828960,1423258199 +6,1423433760,1423862999 +7,1424038560,1424467799 +8,1424643360,1425072599 +9,1425248160,1425677399 +10,1425849360,1426278599 +11,1426454160,1426883399 +12,1427058960,1427488199 +13,1427663760,1428092999 +14,1428268560,1428697799 +15,1428873360,1429302599 +16,1429478160,1429907399 +17,1430082960,1430512199 +18,1430687760,1431116999 +19,1431292560,1431721799 +20,1431897360,1432326599 +21,1432502160,1432931399 +22,1433106960,1433536199 +23,1433711760,1434140999 +24,1434316560,1434745799 +25,1434921360,1435350599 +26,1435526160,1435955399 +27,1436130960,1436560199 +28,1436735760,1437164999 +29,1437340560,1437769799 +30,1437945360,1438374599 +31,1438550160,1438979399 +32,1439154960,1439584199 +33,1439759760,1440188999 +34,1440364560,1440793799 +35,1440969360,1441398599 +36,1441574160,1442003399 +37,1442178960,1442608199 +38,1442783760,1443212999 +39,1443388560,1443817799 +40,1443993360,1444422599 +41,1444598160,1445027399 +42,1445202960,1445632199 +43,1445807760,1446236999 +44,1446416160,1446845399 +45,1447020960,1447450199 +46,1447625760,1448054999 +47,1448230560,1448659799 +48,1448835360,1449264599 +49,1449440160,1449869399 +50,1450044960,1450474199 +51,1450649760,1451078999 +52,1451254560,1451683799 +1,1451859360,1452288599 +2,1452464160,1452893399 +3,1453068960,1453498199 +4,1453673760,1454102999 +5,1454278560,1454707799 +6,1454883360,1455312599 +7,1455488160,1455917399 +8,1456092960,1456522199 +9,1456697760,1457126999 +10,1457302560,1457731799 +11,1457903760,1458332999 +12,1458508560,1458937799 +13,1459113360,1459542599 +14,1459718160,1460147399 +15,1460322960,1460752199 +16,1460927760,1461356999 +17,1461532560,1461961799 +18,1462137360,1462566599 +19,1462742160,1463171399 +20,1463346960,1463776199 +21,1463951760,1464380999 +22,1464556560,1464985799 +23,1465161360,1465590599 +24,1465766160,1466195399 +25,1466370960,1466800199 +26,1466975760,1467404999 +27,1467580560,1468009799 +28,1468185360,1468614599 +29,1468790160,1469219399 +30,1469394960,1469824199 +31,1469999760,1470428999 +32,1470604560,1471033799 +33,1471209360,1471638599 +34,1471814160,1472243399 +35,1472418960,1472848199 +36,1473023760,1473452999 +37,1473628560,1474057799 +38,1474233360,1474662599 +39,1474838160,1475267399 +40,1475442960,1475872199 +41,1476047760,1476476999 +42,1476652560,1477081799 +43,1477257360,1477686599 +44,1477862160,1478291399 +45,1478470560,1478899799 +46,1479075360,1479504599 +47,1479680160,1480109399 +48,1480284960,1480714199 +49,1480889760,1481318999 +50,1481494560,1481923799 +51,1482099360,1482528599 +52,1482704160,1483133399 +1,1483308960,1483738199 +2,1483913760,1484342999 +3,1484518560,1484947799 +4,1485123360,1485552599 +5,1485728160,1486157399 +6,1486332960,1486762199 +7,1486937760,1487366999 +8,1487542560,1487971799 +9,1488147360,1488576599 +10,1488752160,1489181399 +11,1489353360,1489782599 +12,1489958160,1490387399 +13,1490562960,1490992199 +14,1491167760,1491596999 +15,1491772560,1492201799 +16,1492377360,1492806599 +17,1492982160,1493411399 +18,1493586960,1494016199 +19,1494191760,1494620999 +20,1494796560,1495225799 +21,1495401360,1495830599 +22,1496006160,1496435399 +23,1496610960,1497040199 +24,1497215760,1497644999 +25,1497820560,1498249799 +26,1498425360,1498854599 +27,1499030160,1499459399 +28,1499634960,1500064199 +29,1500239760,1500668999 +30,1500844560,1501273799 +31,1501449360,1501878599 +32,1502054160,1502483399 +33,1502658960,1503088199 +34,1503263760,1503692999 +35,1503868560,1504297799 +36,1504473360,1504902599 +37,1505078160,1505507399 +38,1505682960,1506112199 +39,1506287760,1506716999 +40,1506892560,1507321799 +41,1507497360,1507926599 +42,1508102160,1508531399 +43,1508706960,1509136199 +44,1509311760,1509740999 +45,1509920160,1510349399 +46,1510524960,1510954199 +47,1511129760,1511558999 +48,1511734560,1512163799 +49,1512339360,1512768599 +50,1512944160,1513373399 +51,1513548960,1513978199 +52,1514153760,1514582999 +53,1514758560,1515187799 +1,1515363360,1515792599 +2,1515968160,1516397399 +3,1516572960,1517002199 +4,1517177760,1517606999 +5,1517782560,1518211799 +6,1518387360,1518816599 +7,1518992160,1519421399 +8,1519596960,1520026199 +9,1520201760,1520630999 +10,1520802960,1521232199 +11,1521407760,1521836999 +12,1522012560,1522441799 +13,1522617360,1523046599 +14,1523222160,1523651399 +15,1523826960,1524256199 +16,1524431760,1524860999 +17,1525036560,1525465799 +18,1525641360,1526070599 +19,1526246160,1526675399 +20,1526850960,1527280199 +21,1527455760,1527884999 +22,1528060560,1528489799 +23,1528665360,1529094599 +24,1529270160,1529699399 +25,1529874960,1530304199 +26,1530479760,1530908999 +27,1531084560,1531513799 +28,1531689360,1532118599 +29,1532294160,1532723399 +30,1532898960,1533328199 +31,1533503760,1533932999 +32,1534108560,1534537799 +33,1534713360,1535142599 +34,1535318160,1535747399 +35,1535922960,1536352199 +36,1536527760,1536956999 +37,1537132560,1537561799 +38,1537737360,1538166599 +39,1538342160,1538771399 +40,1538946960,1539376199 +41,1539551760,1539980999 +42,1540156560,1540585799 +43,1540761360,1541190599 +44,1541369760,1541798999 +45,1541974560,1542403799 +46,1542579360,1543008599 +47,1543184160,1543613399 +48,1543788960,1544218199 +49,1544393760,1544822999 +50,1544998560,1545427799 +51,1545603360,1546032599 +52,1546208160,1546637399 +1,1546812960,1547242199 +2,1547417760,1547846999 +3,1548022560,1548451799 +4,1548627360,1549056599 +5,1549232160,1549661399 +6,1549836960,1550266199 +7,1550441760,1550870999 +8,1551046560,1551475799 +9,1551651360,1552080599 +10,1552252560,1552681799 +11,1552857360,1553286599 +12,1553462160,1553891399 +13,1554066960,1554496199 +14,1554671760,1555100999 +15,1555276560,1555705799 +16,1555881360,1556310599 +17,1556486160,1556915399 +18,1557090960,1557520199 +19,1557695760,1558124999 +20,1558300560,1558729799 +21,1558905360,1559334599 +22,1559510160,1559939399 +23,1560114960,1560544199 +24,1560719760,1561148999 +25,1561324560,1561753799 +26,1561929360,1562358599 +27,1562534160,1562963399 +28,1563138960,1563568199 +29,1563743760,1564172999 +30,1564348560,1564777799 +31,1564953360,1565382599 +32,1565558160,1565987399 +33,1566162960,1566592199 +34,1566767760,1567196999 +35,1567372560,1567801799 +36,1567977360,1568406599 +37,1568582160,1569011399 +38,1569186960,1569616199 +39,1569791760,1570220999 +40,1570396560,1570825799 +41,1571001360,1571430599 +42,1571606160,1572035399 +43,1572210960,1572640199 +44,1572819360,1573248599 +45,1573424160,1573853399 +46,1574028960,1574458199 +47,1574633760,1575062999 +48,1575238560,1575667799 +49,1575843360,1576272599 +50,1576448160,1576877399 +51,1577052960,1577482199 +52,1577657760,1578086999 +1,1578262560,1578691799 +2,1578867360,1579296599 +3,1579472160,1579901399 +4,1580076960,1580506199 +5,1580681760,1581110999 +6,1581286560,1581715799 +7,1581891360,1582320599 +8,1582496160,1582925399 +9,1583100960,1583530199 +10,1583702160,1584131399 +11,1584306960,1584736199 +12,1584911760,1585340999 +13,1585516560,1585945799 +14,1586121360,1586550599 +15,1586726160,1587155399 +16,1587330960,1587760199 +17,1587935760,1588364999 +18,1588540560,1588969799 +19,1589145360,1589574599 +20,1589750160,1590179399 +21,1590354960,1590784199 +22,1590959760,1591388999 +23,1591564560,1591993799 +24,1592169360,1592598599 +25,1592774160,1593203399 +26,1593378960,1593808199 +27,1593983760,1594412999 +28,1594588560,1595017799 +29,1595193360,1595622599 +30,1595798160,1596227399 +31,1596402960,1596832199 +32,1597007760,1597436999 +33,1597612560,1598041799 +34,1598217360,1598646599 +35,1598822160,1599251399 +36,1599426960,1599856199 +37,1600031760,1600460999 +38,1600636560,1601065799 +39,1601241360,1601670599 +40,1601846160,1602275399 +41,1602450960,1602880199 +42,1603055760,1603484999 +43,1603660560,1604089799 +44,1604268960,1604698199 +45,1604873760,1605302999 +46,1605478560,1605907799 +47,1606083360,1606512599 +48,1606688160,1607117399 +49,1607292960,1607722199 +50,1607897760,1608326999 +51,1608502560,1608931799 +52,1609107360,1609536599 +1,1609712160,1610141399 +2,1610316960,1610746199 +3,1610921760,1611350999 +4,1611526560,1611955799 +5,1612131360,1612560599 +6,1612736160,1613165399 +7,1613340960,1613770199 +8,1613945760,1614374999 +9,1614550560,1614979799 +10,1615155360,1615584599 +11,1615756560,1616185799 +12,1616361360,1616790599 +13,1616966160,1617395399 +14,1617570960,1618000199 +15,1618175760,1618604999 +16,1618780560,1619209799 +17,1619385360,1619814599 +18,1619990160,1620419399 +19,1620594960,1621024199 +20,1621199760,1621628999 +21,1621804560,1622233799 +22,1622409360,1622838599 +23,1623014160,1623443399 +24,1623618960,1624048199 +25,1624223760,1624652999 +26,1624828560,1625257799 +27,1625433360,1625862599 +28,1626038160,1626467399 +29,1626642960,1627072199 +30,1627247760,1627676999 +31,1627852560,1628281799 +32,1628457360,1628886599 +33,1629062160,1629491399 +34,1629666960,1630096199 +35,1630271760,1630700999 +36,1630876560,1631305799 +37,1631481360,1631910599 +38,1632086160,1632515399 +39,1632690960,1633120199 +40,1633295760,1633724999 +41,1633900560,1634329799 +42,1634505360,1634934599 +43,1635110160,1635539399 +44,1635714960,1636144199 +45,1636323360,1636752599 +46,1636928160,1637357399 +47,1637532960,1637962199 +48,1638137760,1638566999 +49,1638742560,1639171799 +50,1639347360,1639776599 +51,1639952160,1640381399 +52,1640556960,1640986199 + diff --git a/samples/sample_backtest.R b/samples/sample_backtest.R new file mode 100644 index 0000000..7281d1d --- /dev/null +++ b/samples/sample_backtest.R @@ -0,0 +1,223 @@ +pb_docker_name <- 'pb' +ISDOCKER <- F + +### Uncomment one of the following two settings options that matches your setup: + +# ### Option 1: Running R outside of pb_docker_name Docker container: +# currency <- "EURUSD" +# ISNEWTDB <- T +# ISDOCKER <- T +# pbDB <- "ohlc" +# pbTABLE <- "bymin" + +# ### Option 2: Running R inside of pb_docker_name Docker container: +# currency <- "EURUSD" +# ISNEWTDB <- T +# pbDB <- "ohlc" +# pbTABLE <- "bymin" +# startingwd <- getwd() +# neededwd <- "/home/ess" + +lotsize <- 1 # How many units to trade +offset <- 2 # Real-time Trading Execution Latency +com <- 0.000036 # Broker charge for executing each position +exp <- 30000000 # Expire and close out positions after exp minutes +stopscalar <- "1"; +sellbuycondition <- "4"; sellstop <- "0.1050"; selltarget <- "0.1050"; +buysellcondition <- "3"; buystop <- "0.1050"; buytarget <- "0.1050"; + +backtest <- function(normalized=T,runweekly=F,levered=F,balance=20000,marginutilization=10,leverage=50,compounding=F,metrics=F) { + t1 <- Sys.time() + if (exists('neededwd')) { + setwd(neededwd) + } + pippercent <- 100 + labelpippercent <- "%" + if (!normalized) { + pippercent <- 1 + labelpippercent <- "Pips" + } + if (ISDOCKER) { + dockerprefix1 <- paste0("docker exec ",pb_docker_name,' bash -c "source ~/.bashrc; cd /home/ess;') + dockerprefix2 <- '"' + } else { + dockerprefix1 <- "" + dockerprefix2 <- "" + } + if (sum(grepl('data.table',row.names(installed.packages()),fixed=T)) >= 1) { + if (! "data.table" %in% (.packages())) { + library('data.table') + } + signals <- fread(paste0(dockerprefix1," aq_udb -exp ",pbDB,':',pbTABLE," -filt 'entrysignal!=0'",dockerprefix2),stringsAsFactors = F,data.table = F) + } else { + signalscon <- pipe(paste0(dockerprefix1," aq_udb -exp ",pbDB,':',pbTABLE," -filt 'entrysignal!=0'",dockerprefix2),open='r') + signals <- read.csv(signalscon,stringsAsFactors = F) + close(signalscon) + } + start <- signals$t[1] + end <- signals$t[length(signals$t)] + tdbcon <- pipe(paste0(dockerprefix1," tdb -inf -db ",currency,dockerprefix2),open='r') + tdbinfo <- read.csv(tdbcon) + close(tdbcon) + tdbstart <- tdbinfo[1,1] + tdbend <- tdbinfo[1,2] + tdbcount <- tdbinfo[1,3] + simstart <- max(start,as.integer(tdbstart/1000)) + simend <- min(end,as.integer(tdbend/1000)) + print(paste0("Simulating from ",system(paste0('date -d @',simstart," +'%F'"),intern=T)," (",simstart,') to ',system(paste0('date -d @',simend," +'%F'"),intern=T)," (",simend,")")) + + buy.entrytimes <- signals[which(signals$entrysignal == 21),"t"] #Exit Short Positions at Buy Signals + sell.entrytimes <- signals[which(signals$entrysignal == 12),"t"] #Exit Long Positions at Sell Signals + buy.expiretimes <- buy.entrytimes*1000+exp*60*1000 + buy.expiretimes[which(buy.expiretimes>tdbend)] <- tdbend + sell.expiretimes <- sell.entrytimes*1000+exp*60*1000 + sell.expiretimes[which(sell.expiretimes>tdbend)] <- tdbend + if (ISNEWTDB) { + buyexpire <- cbind(entryexpire=buy.expiretimes,exitexpire=buy.expiretimes) + sellexpire <- cbind(entryexpire=sell.expiretimes,exitexpire=sell.expiretimes) + } else { + buyexpire <- cbind(entryexpire=buy.expiretimes) + sellexpire <- cbind(entryexpire=sell.expiretimes) + } + buy.entry <- cbind(type=paste0("\"BUY_SELL\""),t=(buy.entrytimes*1000 + offset*1000),buyexpire,price=10000,lo=0,hi=10000,conditioncolumn=paste0("\"trade(",buysellcondition,",",stopscalar,",",buystop,",",buytarget,",0,-1,-1)\""),amount=lotsize) + sell.entry <- cbind(type=paste0("\"SELL_BUY\""),t=(sell.entrytimes*1000 + offset*1000),sellexpire,price=0,lo=0,hi=10000,conditioncolumn=paste0("\"trade(",sellbuycondition,",",stopscalar,",",sellstop,",",selltarget,",0,-1,-1)\""),amount=lotsize) + entries <- rbind(buy.entry,sell.entry)[order(c(buy.entrytimes,sell.entrytimes)),] + write.table(entries,paste0(getwd(),"/financialentrysignals.csv"),row.names=F,quote = F,col.names=F,sep=',') + if (ISDOCKER) { + system(paste0('docker cp ',paste0(getwd(),"/financialentrysignals.csv"),' ',pb_docker_name,':/home/ess/financialentrysignals.csv')) + } + + buy.exittimes <- signals[which(signals$entrysignal == 21),"t"] #Exit Short Positions at Buy Signals + sell.exittimes <- signals[which(signals$entrysignal == 12),"t"] #Exit Long Positions at Sell Signals + buy.exit <- paste0(as.character(buy.exittimes * 1000 + offset * 1000),",4") + sell.exit <- paste0(as.character(sell.exittimes * 1000 + offset * 1000),",3") + buy.exit <- names(sort(sapply(buy.exit,function(x) as.numeric(strsplit(x,",")[[1]][1])))) + sell.exit <- names(sort(sapply(sell.exit,function(x) as.numeric(strsplit(x,",")[[1]][1])))) + exits <- sort(c(buy.exit,sell.exit)) + write.table(exits,paste0(getwd(),"/financialexitsignals.csv"),row.names=F,quote = F,col.names=F) + dir_for_trade_mod <- getwd() + if (ISDOCKER) { + system(paste0('docker cp ',paste0(getwd(),"/financialexitsignals.csv"),' ',pb_docker_name,':/home/ess/financialexitsignals.csv')) + dir_for_trade_mod <- "/home/ess" + } + + sim <- paste0(dockerprefix1,"cat ",dir_for_trade_mod, "/financialentrysignals.csv | tdb -trd,eok,qui - -db ",currency, + " -seq -mod 'trade(", + dir_for_trade_mod, "/financialexitsignals.csv",")' ",dockerprefix2) + + trades_list <- strsplit(system(sim,intern=TRUE)[-1],",") + + if(length(trades_list)>0) { + trades <- data.frame(matrix(unlist(trades_list),nrow=length(trades_list),byrow=TRUE),stringsAsFactors=FALSE) + if(sum(grepl("\"SKIPPED WITHOUT ACTION\"",trades[,3]))>0) { + trades <- trades[-which(trades[,3]=="\"SKIPPED WITHOUT ACTION\""),] + } + if(sum(grepl("\"NONE\"",trades[,2]))>0) { + trades <- trades[-which(trades[,2]=="\"NONE\""),] + } + if (ncol(trades)>10) { + trades[which(trades[,3]=="\"NONE\""),3] <- "\"ORDER SUCCESS\"" + names(trades) <- NULL + newtrades <- as.data.frame(matrix(0,nrow=nrow(trades)*2,ncol=10)) + newtrades[seq(1,nrow(trades)),] <- trades[,c(1:6,12:15)] + newtrades[seq((nrow(trades)+1),nrow(newtrades)),] <- trades[,c(1,7:11,12:15)] + trades <- newtrades[order(newtrades[,4]),] + } + weekperfs <- NULL + leveredweekperfs <- NULL + if (runweekly && (!ISDOCKER)) { + if ((sum(grepl('xts',row.names(installed.packages()),fixed=T)) && sum(grepl('PerformanceAnalytics',row.names(installed.packages()),fixed=T))) == T) { + meanrate <- 1 + if (normalized) { + meanrate <- mean(as.numeric(trades[,5])) + } + print("Running Weekly Performance, will take a few more seconds") + write.table(trades,'modeltrades.csv',row.names=F,col.names=F,sep=',',quote = F) + if (!exists('d')) { + dates <- read.csv(paste0(getwd(),"/funcs.custom/additionalfiles/safetygetdates.csv")) + d <- dates[,2:3] + } + source(paste0(getwd(),'/funcs.custom/additionalfiles/runweekly.R')) + weekperfs <- runweekly((which(d[,2]>simstart)[1] - 1),(which(d[,1]>simend)[1] - 1),tplot = T,compounding = F,leverage = 1,marginutilization = 100,balance = 100000,metrics = metrics,commissions = com,d = d,meanrate=meanrate,normalized=normalized,pippercent=pippercent) + if (levered) { + leveredweekperfs <- runweekly((which(d[,2]>simstart)[1] - 1),(which(d[,1]>simend)[1] - 1),tplot = T,compounding = compounding,leverage = leverage,marginutilization = marginutilization,balance = balance,metrics = metrics,commissions = com,d = d,meanrate=meanrate,normalized=normalized,pippercent=pippercent) + } + } else { + print("runweekly flag requires xts and PerformanceAnalytics Packages, please install them") + } + } + } + + if (normalized) { + profits <- (as.numeric(trades[which(trades[,2] =="\"SELL\""),5]) - as.numeric(trades[which(trades[,2] =="\"BUY\""),5]))/as.numeric(trades[which(trades[,2] =="\"BUY\""),5]) - com + } else { + profits <- as.numeric(trades[which(trades[,2] =="\"SELL\""),5]) - as.numeric(trades[which(trades[,2] =="\"BUY\""),5]) - com + } + profits <- profits * pippercent + + b_ind <- which(trades[,2]=="\"BUY\"" & trades[,3]=="\"ORDER SUCCESS\"") + s_ind <- which(trades[,2]=="\"SELL\"" & trades[,3]=="\"ORDER SUCCESS\"") + if (normalized) { + buyprofits <- (as.numeric(trades[b_ind+1,5])-as.numeric(trades[b_ind,5]))/as.numeric(trades[b_ind,5])-com + sellprofits <- (as.numeric(trades[s_ind,5])-as.numeric(trades[s_ind+1,5]))/as.numeric(trades[s_ind+1,5])-com + } else { + buyprofits <- as.numeric(trades[b_ind+1,5])-as.numeric(trades[b_ind,5])-com + sellprofits <- as.numeric(trades[s_ind,5])-as.numeric(trades[s_ind+1,5])-com + } + buyprofits <- buyprofits * pippercent + sellprofits <- sellprofits * pippercent + buyentrytime <- (as.numeric(trades[which(trades[,2]=="\"BUY\"" & trades[,3]=="\"ORDER SUCCESS\""),4])-offset*1000)/1000 + sellentrytime <- (as.numeric(trades[which(trades[,2]=="\"SELL\"" & trades[,3]=="\"ORDER SUCCESS\""),4])-offset*1000)/1000 + buyexittime <- (as.numeric(trades[which(trades[,2]=="\"SELL\"" & trades[,3]!="\"ORDER SUCCESS\""),4]))/1000 + sellexittime <- (as.numeric(trades[which(trades[,2]=="\"BUY\"" & trades[,3]!="\"ORDER SUCCESS\""),4]))/1000 + bt <- (buyexittime-buyentrytime)/60 + st <- (sellexittime-sellentrytime)/60 + par(mfrow=c(1,2)) + plot(buyprofits,bt,type="p",main="Long Position Net Profits",xlab=ifelse(normalized,"Net Profit [%]","Net Profit [Pips]"),ylab="Minutes") + abline(v=0,col=2) + plot(sellprofits,st,type="p",main="Short Position Net Profits",xlab=ifelse(normalized,"Net Profit [%]","Net Profit [Pips]"),ylab="Minutes") + abline(v=0,col=2) + + cumprofits <- cumsum(profits) + profit <- tail(cumprofits,1) + par(mfrow=c(1,1)) + plot(cumprofits,type="l",main="Cumulative Net Profit",xlab=paste0(system(paste0('date -d @',simstart," +'%F'"),intern=T),' to ',system(paste0('date -d @',simend," +'%F'"),intern=T)),ylab=ifelse(normalized,"Net Profit [%]","Net Profit [Pips]")) + + t2 <- Sys.time() + print(paste0("Simulated ",tdbcount," Data Points in ",t2 - t1," Seconds")) + print(paste0("Total Net Profit: ",profit,' ',labelpippercent)) + + leveredprofits <- NULL + if (levered) { + leveredprofits <- rep(0,length(profits)) + bal <- balance + mar <- bal / (100 / marginutilization) + lot <- mar * leverage + for(i in 1:length(profits)) { + pl <- (profits[i]/pippercent) * lot + leveredprofits[i] <- pl / bal + bal <- bal + pl + mar <- bal * (marginutilization / 100) + if (compounding==T) { + lot <- mar * leverage + } + # print(lot) + } + leveredprofits <- leveredprofits*pippercent + + print(paste0("Total Leveraged Net Profit: ",tail(cumsum(leveredprofits),1),' ',labelpippercent)) + plot(cumsum(leveredprofits),main=paste0(leverage,"x Leveraged Returns/",marginutilization,"% Margin Utilization/Not Compounding"),ylab = ifelse(normalized,"Cumulative Net Profit [%]","Cumulative Net Profit [Pips]"),xlab=paste0('Trades from ',system(paste0('date -d @',simstart," +'%F'"),intern=T),' to ',system(paste0('date -d @',simend," +'%F'"),intern=T)),type='l',cex.lab=0.8,cex.main=0.8) + + } + + if (exists('startingwd')) { + setwd(startingwd) + } + + return(list("totalprofit"=profit,"normalized"=normalized,"profits"=profits,"cumprofits"=cumprofits,"buyprofits"=buyprofits,"sellprofits"=sellprofits,"leveredprofits"=leveredprofits,"weekperfs"=weekperfs,"leveredweekperfs"=leveredweekperfs,"trades"=trades)) +} + +modelresults <- backtest() + +str(modelresults) + diff --git a/samples/sample_backtest.sh b/samples/sample_backtest.sh new file mode 100644 index 0000000..95b29b4 --- /dev/null +++ b/samples/sample_backtest.sh @@ -0,0 +1,112 @@ +######################################################################## +## sample_tdb.sh: Empowering Quick, Massive Simulation of Financial Data +######################################################################## + +setup_ta_lib() { + bash ~/mod_talib/installtalib.sh &>~/mod_talib/installtalib.txt + if [[ `grep "All tests succeeded" ~/mod_talib/installtalib.txt | wc -l` == 0 ]] + then + cat ~/mod_talib/installtalib.txt &>>task.log + exit + fi +} + +category() { + ess select local + ess category add EURUSD "/home/sampledata/Financial/EURUSD_Ticks_*.zip" --overwrite +} + +createdb () { + # Define Minute and Hour Aggregations and Features + ess server reset ### Reset old config ### + ess create database ohlc --port 0 + # Define a Minute-Aggregated Table + ess create table bymin s,pkey:curr s,+key:minute f,+first:open f,+max:high f,+min:low f:close f:nextclose f,+add:vol l:t f:max_$1_close f:min_$1_close f:linreg_angle_$2 i:entrysignal + # Define a Hour-Aggregated Table + ess create table byhour s,pkey:curr s,+key:hour f,+first:open f,+max:high f,+min:low f:close f:nextclose f,+add:vol l:t f:max_$1_close f:min_$1_close f:linreg_angle_$2 i:entrysignal + ess create variable i:fg_t i:vI1 i:vI2 s:vS1 s:vS2 f:vF1 f,+add:vF2 l:vL1 l,+add:vL2 f,+max:lmx f,+min:lmn + # Save these definitions and send them to any worker nodes if you have them + ess server commit +} + +read_from_files () { + # Stop any existing Time Series Databases and Start a new Time Series Database using the US/Eastern Timezone + tdb_ctl stop + TZ=US/Eastern tdb_ctl start + + # Filter out the data near market close to prevent low liquidity trades + filtertimes=`bash funcs.custom/additionalfiles/cmb-getdates.sh funcs.custom/additionalfiles/safety` + if [ -z "$filtertimes" ] + then + echo "Failed to filter out low liquidity trades, allowing all data" &>>/home/ess/task.log + filtertimes="t>0" + fi + + # Load the Time Series Database (tdb) from the Raw Data File, Enhance the Data, and Aggregate the Data by Minute and by Hour + ess stream EURUSD "*" "*" \ + "TZ='GMT' aq_pp -f+1 - -d s:utc f:ask f:bid f:askvol f:bidvol -eval s:curr '\"EURUSD\"' \ + -eval s:minute 'SubStr(utc,0,16)' -eval s:hour 'SubStr(utc,0,13)' -eval f:price '(ask + bid)/2' \ + -eval f:open price -eval f:high price -eval f:low price -eval f:close price -eval f:vol askvol \ + -eval l:t 'DateToTime(minute,\"%Y.%m.%d.%H.%M\")+60' \ + -if -filt '$filtertimes' \ + -imp,ddef ohlc:bymin -imp,ddef ohlc:byhour -endif \ + -eval s:DateToS 'ClipStr(utc,\"3->.\")' -eval s:milliseconds 'ClipStr(utc,\"1-<.\")' \ + -eval t 'DateToTime(DateToS,\"%Y.%m.%d.%H.%M.%S\")*1000+ToI(milliseconds)' \ + -o,notitle - -c t bid ask | tdb -imp - -db EURUSD" + + # Check the total number of rows imported into each database + # Tick Time-Series Data + tdb -inf -db EURUSD + # Minute-Aggregated Data + ess exec "aq_udb -cnt ohlc:bymin" + # Hour-Aggregated Data + ess exec "aq_udb -cnt ohlc:byhour" +} + +ta_lib () { + # Calculating TALIB Features: Maximum, Minimum, and Linear Regression Angle by Minute and by Hour + eval "aq_udb -scn ohlc:bymin -mod 'talib-umod(1,MAXIMUM,bymin.close,bymin.max_$1_close,$1)'" + eval "aq_udb -scn ohlc:bymin -mod 'talib-umod(1,MINIMUM,bymin.close,bymin.min_$1_close,$1)'" + eval "aq_udb -scn ohlc:bymin -mod 'talib-umod(1,LINEARREG_ANGLE,bymin.close,bymin.linreg_angle_$2,$2)'" + eval "aq_udb -scn ohlc:byhour -mod 'talib-umod(1,MAXIMUM,byhour.close,byhour.max_$1_close,$1)'" + eval "aq_udb -scn ohlc:byhour -mod 'talib-umod(1,MINIMUM,byhour.close,byhour.min_$1_close,$1)'" + eval "aq_udb -scn ohlc:byhour -mod 'talib-umod(1,LINEARREG_ANGLE,byhour.close,byhour.linreg_angle_$2,$2)'" +} + +getnextclose() { + # Calculate and Store the next period (future) Minute and Hour Prices + + # Reverse Data by Time (minute) Column + ess exec "aq_udb -ord,dec ohlc:bymin minute" + # Scan through database, calculating and storing the next period (future) Minute Price + tmpval=`ess exec "aq_udb -exp ohlc:bymin -lim_rec 1 -o,notitle - -c nextclose"` + ess exec "aq_udb -scn ohlc:bymin -pp bymin -bvar vF1 $tmpval -eval nextclose vF1 -eval vF1 close -endpp" + # Reorder Data by Time (minute) Column + ess exec "aq_udb -ord ohlc:bymin minute" + + # Reverse Data by Time (hour) Column + ess exec "aq_udb -ord,dec ohlc:byhour hour" + # Scan through database, calculating and storing the next period (future) Hour Price + tmpval=`ess exec "aq_udb -exp ohlc:byhour -lim_rec 1 -o,notitle - -c nextclose"` + ess exec "aq_udb -scn ohlc:byhour -pp byhour -bvar vF1 $tmpval -eval nextclose vF1 -eval vF1 close -endpp" + # Reorder Data by Time (hour) Column + ess exec "aq_udb -ord ohlc:byhour hour" +} + +gensignals() { + # Generate Entry Signals for Your Model + ess exec "aq_udb -scn ohlc:bymin -if -filt 'vol!=0 && (close == min_$1_close && linreg_angle_$2 <= 0.0) && max_$1_close - min_$1_close > (high - low)*5 && max_$1_close - min_$1_close < (high - low)*10' -eval entrysignal '21' -elif -filt 'vol!=0 && (close == max_$1_close && linreg_angle_$2 >= 0.006) && max_$1_close - min_$1_close > (high - low)*5 && max_$1_close - min_$1_close < (high - low)*10' -eval entrysignal '12' -endif" +} + +import(){ + if [ ! -e ~/mod_talib/installtalib.txt ] + then + setup_ta_lib + fi + category + createdb 80 10 + read_from_files + ta_lib 80 10 + getnextclose + gensignals 80 10 +} diff --git a/samples/sample_fx.sh b/samples/sample_fx.sh new file mode 100644 index 0000000..8f73d8f --- /dev/null +++ b/samples/sample_fx.sh @@ -0,0 +1,31 @@ +######################################################## +## sample_fx.sh: a financial data analysis +##################################################################### + +import(){ + category + createdb + read_from_files +} + +category() { + ess select local + ess category add EURUSD "/home/sampledata/Financial/EURUSD_Ticks_*.zip" --overwrite +} + +createdb () { + ess server reset ### Reset old config ### + ess create database demo --port 0 + ess create table tick s,pkey:utc f:ask f:bid f:askvol f:bidvol + ess create database ohlc --port 0 + ess create vector bymin s,pkey:minute f,+first:open f,+max:high f,+min:low f:close f,+add:vol + ess server commit +} + +read_from_files () { + ess stream EURUSD "*" "*" \ + "aq_pp -f+1 - -d s:utc f:ask f:bid f:askvol f:bidvol -imp demo:tick \ + -eval s:minute 'SubStr(utc,0,16)' -eval f:price ask*100 \ + -eval f:open price -eval f:high price -eval f:low price -eval f:close price -eval f:vol askvol -imp,ddef ohlc:bymin" + # +} diff --git a/samples/sample_talib.sh b/samples/sample_talib.sh new file mode 100644 index 0000000..5fa1ac6 --- /dev/null +++ b/samples/sample_talib.sh @@ -0,0 +1,119 @@ +##################################################################### +## sample_talib.sh: Enhancing Financial Data with TALIB features +##################################################################### + +setup_ta_lib() { + # For the first run only, install TA-Lib and make sure it is installed correctly + bash ~/mod_talib/installtalib.sh &>~/mod_talib/installtalib.txt + if [[ `grep "All tests succeeded" ~/mod_talib/installtalib.txt | wc -l` == 0 ]] + then + cat ~/mod_talib/installtalib.txt &>>task.log + exit + fi +} + +category() { + # Select the current machine's filesystem as your datastore / data location + ess select local + # Categorize the raw, compressed Financial Tick data for the EURUSD currency + ess category add EURUSD "/home/sampledata/Financial/EURUSD_Ticks_*.zip" --overwrite +} + +createdb () { + # Reset old config + ess server reset + # Create a database "demo", a table "tick", and some variables to store the enhanced tick data in + ess create database demo --port 0 + ess create table tick s,pkey:curr s,+key:utc f:ask f:bid f:avg f:askvol f:bidvol f:nextask f:nextbid f:nextavg \ + f:delta_NaskAsk f:delta_NbidBid f:delta_NavgAvg f:max_$1_avg f:min_$1_avg f:sma_$1_avg + ess create variable i:fg_t i:vI1 i:vI2 s:vS1 s:vS2 f:vF1 f,+add:vF2 l:vL1 l,+add:vL2 f,+max:lmx f,+min:lmn + # Save these definitions and send them to worker nodes if there are any + ess server commit +} + +read_from_files () { + # Load the Database from the Raw Data File and Enhance the Data + ess stream EURUSD "*" "*" \ + "aq_pp -f+1 - -d s:utc f:ask f:bid f:askvol f:bidvol \ + -eval s:curr '\"EURUSD\"' -eval f:avg '(ask + bid)/2' \ + -imp,ddef demo:tick" +} + +ta_lib () { + # Calculating TALIB Features: Maximum, Minimum, Simple Moving Average + eval "aq_udb -scn demo:tick -mod 'talib-umod(1,MAXIMUM,tick.avg,tick.max_$1_avg,$1)'" + eval "aq_udb -scn demo:tick -mod 'talib-umod(1,MINIMUM,tick.avg,tick.min_$1_avg,$1)'" + eval "aq_udb -scn demo:tick -mod 'talib-umod(1,SMA,tick.avg,tick.sma_$1_avg,$1)'" +} + +getnexttickprice() { + # Calculate and Store the next period (future) Tick Price + # Reverse Data by Time (utc) Column + ess exec "aq_udb -ord,dec demo:tick utc" + + # Scan through database, calculating and storing the next period (future) Tick Price + tmpval=`ess exec "aq_udb -exp demo:tick -lim_rec 1 -o,notitle - -c ask"` + ess exec "aq_udb -scn demo:tick -pp tick -bvar vF1 $tmpval -eval nextask vF1 -eval vF1 ask -endpp" + tmpval=`ess exec "aq_udb -exp demo:tick -lim_rec 1 -o,notitle - -c bid"` + ess exec "aq_udb -scn demo:tick -pp tick -bvar vF1 $tmpval -eval nextbid vF1 -eval vF1 bid -endpp" + tmpval=`ess exec "aq_udb -exp demo:tick -lim_rec 1 -o,notitle - -c avg"` + ess exec "aq_udb -scn demo:tick -pp tick -bvar vF1 $tmpval -eval nextavg vF1 -eval vF1 '(ask+bid)/2' -endpp" + + # Reorder Data by Time (utc) Column + ess exec "aq_udb -ord demo:tick utc" + + # Calculate and Store Difference in each tick price (ask/bid/average) to the corresponding next period tick price. + ess exec "aq_udb -scn demo:tick -eval delta_NaskAsk '(nextask - ask)' -eval delta_NbidBid '(nextbid - bid)' -eval delta_NavgAvg '(nextavg - avg)'" +} + +import(){ + # Call each of the previous functions in the proper order and ensure that you only install TA-Lib once + if [ ! -e ~/mod_talib/installtalib.txt ] + then + setup_ta_lib + fi + category + createdb 300 + read_from_files + ta_lib 300 + getnexttickprice +} + +### NEXT STEPS: + +# # To analyze this demo tick data, you can create the following additional features in the Pivot Billions GUI: + +# # Enter in the STANDARD tab of F(x): +# +# Label: delta_askbid +# Format: f +# Syntax: ask - bid + +# Label: delta_maxmin_300 +# Format: f +# Syntax: max_300_avg - min_300_avg + +# # Enter in the ADVANCED tab of F(x): +# +# Label: cat_delta_askbid +# Format: i +# Column Name: cat_delta_askbid +# Syntax: -if -filt 'var_delta_askbid < (2*(ask - avg))' -eval i:cat_delta_askbid '0' -else -eval cat_delta_askbid '1' -endif + +# Label: cat_delta_maxmin_300 +# Format: i +# Column Name: cat_delta_maxmin_300 +# Syntax: -if -filt 'var_delta_maxmin_300 <= 0.0005' -eval i:cat_delta_maxmin_300 '0' -else -eval cat_delta_maxmin_300 '1' -endif + +# Once you've created these columns, select the Pivot Icon to the left of f(x). +# Then under Dimensions click + then the box and select "cat_delta_askbid", +# then click + then the box and select "cat_delta_maxmin_300". +# Now under Values click + then the box and select "delta_NavgAvg". +# Click View. + +# You can now see the summary statistics and averages of the various categories we've created. +# It is clear that the thesholds we have set for both categories (>= 2*(ask - avg) and > 0.0005) greatly +# improve the average increase in the tick avg price one tick into the future and that they have +# predictive power and warrant further exploration. + +# Please feel free to explore the data further using PivotBillions. diff --git a/samples/sample_tlc.sh b/samples/sample_tlc.sh new file mode 100644 index 0000000..17384ba --- /dev/null +++ b/samples/sample_tlc.sh @@ -0,0 +1,32 @@ +################################################################################ +# tlc.sh: import TLC logs +################################################################################ +green_schema_2017_h1="s:vendor_id s:pickup_datetime s:dropoff_datetime s:store_and_fwd_flag s:rate_code_id s:pickup_location_id s:dropoff_location_id i:passenger_count f:trip_distance f:fare_amount s:extra s:mta_tax f:tip_amount f:tolls_amount s:ehail_fee s:improvement_surcharge f:total_amount s:payment_type s:trip_type" + +udbopt=",ddef,seg=1/20" +udbopt=",ddef" + +create_category () { + ess select local + ess category add green "/home/sampledata/TLC/green_*.csv.gz" --dateregex "_[:%Y:]-[:%m:]" --overwrite +} + +createdb () { + ess server reset ### Reset old config ### + ess create database demo --port 0 + ess create table green s:vendor_id s,pkey:pickup_datetime s:dropoff_datetime s:store_and_fwd_flag \ + s:rate_code_id s:pickup_location_id s:dropoff_location_id i:passenger_count f:trip_distance \ + f:fare_amount s:extra s:mta_tax f:tip_amount f:tolls_amount s:ehail_fee s:improvement_surcharge \ + f:total_amount s:payment_type s:trip_type + ess server commit +} + +import_green () { + ess stream green 2017-01-01 2017-01-31 "aq_pp -f+1,eok,qui - -d $green_schema_2017_h1 -imp$udbopt demo:green" +} + +import () { + create_category + createdb + import_green +} diff --git a/samples/sampledata/Financial/EURUSD_Ticks_2018.03.01_2018.03.31.csv.zip b/samples/sampledata/Financial/EURUSD_Ticks_2018.03.01_2018.03.31.csv.zip new file mode 100644 index 0000000..c9bdb70 Binary files /dev/null and b/samples/sampledata/Financial/EURUSD_Ticks_2018.03.01_2018.03.31.csv.zip differ diff --git a/samples/sampledata/TLC/green_tripdata_2017-01.csv.gz b/samples/sampledata/TLC/green_tripdata_2017-01.csv.gz new file mode 100644 index 0000000..c570a96 Binary files /dev/null and b/samples/sampledata/TLC/green_tripdata_2017-01.csv.gz differ diff --git a/samples/sampledata/TLC/green_tripdata_2017-02.csv.gz b/samples/sampledata/TLC/green_tripdata_2017-02.csv.gz new file mode 100644 index 0000000..4adc4f8 Binary files /dev/null and b/samples/sampledata/TLC/green_tripdata_2017-02.csv.gz differ diff --git a/samples/sampledata/TLC/green_tripdata_2017-03.csv.gz b/samples/sampledata/TLC/green_tripdata_2017-03.csv.gz new file mode 100644 index 0000000..49162ec Binary files /dev/null and b/samples/sampledata/TLC/green_tripdata_2017-03.csv.gz differ diff --git a/samples/sampledata/TLC/green_tripdata_2017-04.csv.gz b/samples/sampledata/TLC/green_tripdata_2017-04.csv.gz new file mode 100644 index 0000000..68a95f4 Binary files /dev/null and b/samples/sampledata/TLC/green_tripdata_2017-04.csv.gz differ diff --git a/samples/sampledata/TLC/green_tripdata_2017-05.csv.gz b/samples/sampledata/TLC/green_tripdata_2017-05.csv.gz new file mode 100644 index 0000000..814e946 Binary files /dev/null and b/samples/sampledata/TLC/green_tripdata_2017-05.csv.gz differ diff --git a/samples/sampledata/TLC/green_tripdata_2017-06.csv.gz b/samples/sampledata/TLC/green_tripdata_2017-06.csv.gz new file mode 100644 index 0000000..fab72b0 Binary files /dev/null and b/samples/sampledata/TLC/green_tripdata_2017-06.csv.gz differ diff --git a/samples/sampledata/TLC/green_tripdata_2017-07.csv.gz b/samples/sampledata/TLC/green_tripdata_2017-07.csv.gz new file mode 100644 index 0000000..df1a0b5 Binary files /dev/null and b/samples/sampledata/TLC/green_tripdata_2017-07.csv.gz differ diff --git a/samples/sampledata/TLC/green_tripdata_2017-08.csv.gz b/samples/sampledata/TLC/green_tripdata_2017-08.csv.gz new file mode 100644 index 0000000..ed41530 Binary files /dev/null and b/samples/sampledata/TLC/green_tripdata_2017-08.csv.gz differ diff --git a/samples/sampledata/TLC/green_tripdata_2017-09.csv.gz b/samples/sampledata/TLC/green_tripdata_2017-09.csv.gz new file mode 100644 index 0000000..6e78cc1 Binary files /dev/null and b/samples/sampledata/TLC/green_tripdata_2017-09.csv.gz differ diff --git a/samples/sampledata/TLC/green_tripdata_2017-10.csv.gz b/samples/sampledata/TLC/green_tripdata_2017-10.csv.gz new file mode 100644 index 0000000..83e7440 Binary files /dev/null and b/samples/sampledata/TLC/green_tripdata_2017-10.csv.gz differ diff --git a/samples/sampledata/TLC/green_tripdata_2017-11.csv.gz b/samples/sampledata/TLC/green_tripdata_2017-11.csv.gz new file mode 100644 index 0000000..046675f Binary files /dev/null and b/samples/sampledata/TLC/green_tripdata_2017-11.csv.gz differ diff --git a/samples/sampledata/TLC/green_tripdata_2017-12.csv.gz b/samples/sampledata/TLC/green_tripdata_2017-12.csv.gz new file mode 100644 index 0000000..5eb16a1 Binary files /dev/null and b/samples/sampledata/TLC/green_tripdata_2017-12.csv.gz differ