Weighted median

predict.R

@@ -4,16 +4,16 @@
 train <- read.csv('data/TrainingData.csv')
 submit <- read.csv('data/SubmissionZerosExceptNAs.csv')
 
-library(plyr)
+library(plyr);
+library(laeken);
 
 rmedian <- function (x) median(x, na.rm=T);
 
 medians <- colwise(rmedian, is.numeric)(train);
 
-mae <- function (x, y) {mean(abs(x - y))};
+mae <- function (x, y) {mean(abs(x - y), na.rm=T)};
 
 varsets <- list(
-    NULL,
     'hour',
     'chunkID',
     'month_most_common',
@@ -24,7 +24,7 @@ preds <- lapply(varsets, function (var) {
   medians.by.var <- ddply(train, var, colwise(rmedian, is.numeric));
   list(
     submit=join(submit[,1:5], medians.by.var, by=var),
-    train=join(train[,1:6], medians.by.var, by=var),
+    train=join(train[,1:6], medians.by.var, by=var)
     );
 });
 
@@ -33,17 +33,17 @@ for (target in colnames(submit)[6:44]) {
   idxs <- which(submit[,target] > -999999);
   cat(target, '\n');
   maes <- sapply(preds, function (pred) {
-    m <- mae(pred$train[idxs, target], train[idxs, target]);
+    m <- mae(pred$train[,target], train[,target]);
     cat(' ', m, '\n');
     m;
     });
-
-  submit[idxs, target] <- apply(sapply(preds, function (pred) pred$submit[idxs,target]), 1, rmedian);
+    wm <- function(x) weightedMedian(x, 1/maes, na.rm=T);
+    submit[idxs, target] <- apply(sapply(preds, function (pred) pred$submit[idxs,target]), 1, wm);
 }
 
 argv <- commandArgs(T);
 filename <- argv[1];
 
+write.csv(submit, filename);
 if (sum(is.na(submit)) > 0) stop('NAs detected');
 
-write.csv(submit, filename)