Estimate remaining time (estimate WAY under)

Author: JohnMount

Count/src/com/mzlabs/count/ctab/CTab.java

@@ -15,6 +15,7 @@
 import com.mzlabs.count.op.impl.ThreadedSum;
 import com.mzlabs.count.op.iter.OrderStepperTot;
 import com.mzlabs.count.util.IntVec;
+import com.mzlabs.count.util.LinearFitter;
 import com.mzlabs.count.zeroone.ZeroOneCounter;
 
 
@@ -219,17 +220,35 @@ public static BigInteger debugConfirmSqTables(final int rowsCols, final int tota
 	}
 
 	public static void main(final String[] args) {
-		System.out.println("n" + "\t" + "total" + "\t" + "count" + "\t" + "date" + "\t" + "cacheSizes");
-		for(int n=1;n<=10;++n) {
+		System.out.println("n" + "\t" + "total" + "\t" + "target" + "\t" + "count" + "\t" + "date" + "\t" + "cacheSizes" + "\t" + "tableFinishTimeEst");
+		for(int n=8;n<=10;++n) {
 			final CTab ctab = new CTab(n,true);
-			for(int total=0;total<=(n*n-3*n+2)/2;++total) {
+			final LinearFitter lf = new LinearFitter(1);
+			final int tLast = (n*n-3*n+2)/2;
+			for(int total=0;total<=tLast;++total) {
+				final Date startTime = new Date();
 				final BigInteger count = ctab.countSqTables(n,total);
 				final String cacheSizes = ctab.cacheSizesString();
-				ctab.clearCaches();
-				System.out.println("" + n + "\t" + total + "\t" + count + "\t" + new Date() + "\t" + cacheSizes);
+				final Date curTime = new Date();
+				long remainingTimeEstMS = 10000;
+				if(total>0) { 
+					// simplistic model: log(time) ~ a + b*size
+					final double[] x = { total };
+					final double y = 10000.0+curTime.getTime() - startTime.getTime();
+					lf.addObservation(x, Math.log(y), y);
+					final double[] beta = lf.solve();
+					double timeEstMS = 0.0;
+					for(int j=total+1;j<=tLast;++j) {
+						final double predict = LinearFitter.predict(beta,new double[] {j});
+						timeEstMS += Math.exp(predict);
+					}
+					remainingTimeEstMS = (long)Math.ceil(timeEstMS);
+				}
+				final Date finishTimeEst = new Date(curTime.getTime()+remainingTimeEstMS);
+				System.out.println("" + n + "\t" + total + "\t" + tLast + "\t" + count + "\t" + curTime + "\t" + cacheSizes + "\t" + finishTimeEst);
 			}
+			ctab.clearCaches();
 		}
-		//System.out.println("total evals: " + SolnCache.totalEvals);
 	}
 
 }

Count/src/com/mzlabs/count/util/LinearFitter.java

@@ -0,0 +1,68 @@
+package com.mzlabs.count.util;
+
+import com.winvector.linalg.LinalgFactory;
+import com.winvector.linalg.colt.ColtMatrix;
+
+
+/**
+ * Fit y ~ a + b.x (least squares)
+ * @author johnmount
+ *
+ */
+public final class LinearFitter {
+	private final ColtMatrix xTx;
+	private final double[] xTy;
+	
+	/**
+	 * 
+	 * @param n dimension of x-vectors
+	 */
+	public LinearFitter(final int n) {
+		final LinalgFactory<ColtMatrix> factory = ColtMatrix.factory;
+		xTx = factory.newMatrix(n+1,n+1,false);
+		xTy = new double[n+1];
+	}
+
+	/**
+	 * add a y ~ f(x) observation
+	 * @param x
+	 * @param y
+	 * @param wt weight of observation (set to 1.0 in many cases)
+	 */
+	public void addObservation(final double[] x, final double y, final double wt) {
+		final int n = xTx.rows()-1;
+		for(int i=0;i<=n;++i) {
+			final double xi = i<n?x[i]:1.0;
+			xTy[i] += wt*xi*y;
+			for(int j=0;j<=n;++j) {
+				final double xj = j<n?x[j]:1.0;
+				xTx.set(i,j,xTx.get(i, j)+wt*xi*xj);
+			}
+		}
+	}
+	
+	public double[] solve() {
+		final int n = xTx.rows()-1;
+		final double epsilon = 1.0e-5;
+		final double[] xTxii = new double[n+1];
+		for(int i=0;i<=n;++i) {
+			xTxii[i] = xTx.get(i,i);
+			xTx.set(i,i,xTxii[i]+epsilon);  // Ridge term
+		}
+		final double[] soln = xTx.solve(xTy);
+		for(int i=0;i<=n;++i) {
+			xTx.set(i,i,xTxii[i]);
+		}
+		return soln;
+	}
+	
+	public static double predict(final double[] soln, final double[] x) {
+		final int n = soln.length-1;
+		double sum = 0.0;
+		for(int i=0;i<=n;++i) {
+			final double xi = i<n?x[i]:1.0;
+			sum += xi*soln[i];
+		}
+		return sum;
+	}
+}