Opening per game speed analysis

bjedrzejewski · Aug 27, 2017 · a84e6b9 · a84e6b9
1 parent 884e566
commit a84e6b9
Show file tree

Hide file tree

Showing 2 changed files with 294 additions and 2 deletions.
diff --git a/openingPerSpeedSmallSet b/openingPerSpeedSmallSet
@@ -0,0 +1,261 @@
+ultraBullet performance
+bullet performance
+Sicilian Defense: French Variation : 0.367 from 105 games
+Sicilian Defense: Bowdler Attack : 0.398 from 157 games
+Hungarian Opening: Catalan Formation : 0.402 from 189 games
+Sicilian Defense: Marshall Counterattack : 0.418 from 104 games
+English Opening : 0.420 from 181 games
+Pirc Defense #4 : 0.442 from 502 games
+Sicilian Defense : 0.443 from 418 games
+Pirc Defense #5 : 0.445 from 110 games
+Hungarian Opening: Sicilian Invitation : 0.464 from 124 games
+French Defense: King's Indian Attack : 0.471 from 172 games
+French Defense #2 : 0.472 from 231 games
+Zukertort Opening: Sicilian Invitation : 0.474 from 137 games
+French Defense: La Bourdonnais Variation : 0.477 from 216 games
+Sicilian Defense: Old Sicilian : 0.477 from 220 games
+Sicilian Defense: McDonnell Attack : 0.478 from 359 games
+Scandinavian Defense: Mieses-Kotroc Variation : 0.479 from 607 games
+Queen's Gambit Declined : 0.481 from 103 games
+Bird Opening : 0.482 from 386 games
+Van't Kruijs Opening : 0.486 from 1642 games
+English Opening: Agincourt Defense : 0.486 from 143 games
+Scandinavian Defense : 0.489 from 482 games
+French Defense: Knight Variation : 0.494 from 595 games
+Queen's Pawn Game #2 : 0.495 from 211 games
+Caro-Kann Defense : 0.499 from 639 games
+Zukertort Opening: Tennison Gambit : 0.504 from 116 games
+Old Benoni Defense : 0.505 from 319 games
+Queen's Pawn Game : 0.507 from 143 games
+King's Pawn Game: Leonardis Variation : 0.510 from 249 games
+Gedult's Opening : 0.514 from 109 games
+Hungarian Opening: Slav Formation : 0.516 from 220 games
+Robatsch (Modern) Defense : 0.517 from 145 games
+Owen Defense : 0.522 from 1774 games
+King's Gambit Declined, Queen's Knight Defense : 0.523 from 128 games
+French Defense: Exchange Variation : 0.526 from 211 games
+King's Pawn Game : 0.530 from 118 games
+King's Pawn Game: Wayward Queen Attack : 0.531 from 112 games
+Bird Opening: Dutch Variation : 0.536 from 138 games
+Mieses Opening : 0.547 from 374 games
+Rat Defense: Small Center Defense : 0.548 from 145 games
+French Defense: Normal Variation : 0.549 from 195 games
+Englund Gambit : 0.551 from 136 games
+Hungarian Opening : 0.552 from 1182 games
+Queen's Pawn Game: Chigorin Variation : 0.552 from 201 games
+Zukertort Opening: Queenside Fianchetto Variation : 0.554 from 111 games
+English Defense #2 : 0.556 from 719 games
+King's Gambit Accepted, Stamma Gambit : 0.557 from 122 games
+Nimzo-Larsen Attack : 0.559 from 264 games
+French Defense: Queen's Knight : 0.559 from 101 games
+English Opening: King's English Variation, Reversed Sicilian : 0.561 from 106 games
+Queen's Pawn Game: Zukertort Variation : 0.566 from 190 games
+Modern Defense : 0.569 from 929 games
+Queen's Pawn : 0.584 from 347 games
+Queen's Gambit Declined: Queen's Knight Variation : 0.586 from 116 games
+Saragossa Opening : 0.597 from 458 games
+Nimzo-Larsen Attack: Symmetrical Variation : 0.610 from 123 games
+Philidor Defense #3 : 0.613 from 243 games
+Bishop's Opening : 0.616 from 151 games
+Philidor Defense #2 : 0.624 from 117 games
+Italian Game : 0.627 from 122 games
+Horwitz Defense : 0.651 from 545 games
+Zukertort Opening: Queen's Gambit Invitation : 0.656 from 131 games
+Old Indian Defense : 0.688 from 109 games
+Hungarian Opening: Symmetrical Variation : 0.720 from 186 games
+Queen's Gambit Accepted : 0.748 from 111 games
+Queen's Pawn Game: Levitsky Attack : 0.830 from 106 games
+blitz performance
+Mieses Opening : 0.306 from 111 games
+Center Game: Normal Variation : 0.387 from 137 games
+Englund Gambit Declined, Reversed French : 0.387 from 102 games
+Sicilian Defense: Smith-Morra Gambit #2 : 0.390 from 205 games
+Sicilian Defense: Bowdler Attack : 0.409 from 735 games
+Van't Kruijs Opening : 0.419 from 1475 games
+Nimzo-Larsen Attack: Classical Variation : 0.423 from 104 games
+French Defense #2 : 0.444 from 188 games
+King's Pawn Game: Leonardis Variation : 0.452 from 345 games
+Giuoco Piano : 0.453 from 139 games
+King's Pawn Game : 0.453 from 107 games
+Nimzowitsch Defense: Kennedy Variation, Linksspringer Variation : 0.455 from 155 games
+Italian Game: Two Knights Defense, Modern Bishop's Opening : 0.455 from 123 games
+Caro-Kann Defense: Advance Variation : 0.462 from 119 games
+King's Knight Opening: Normal Variation : 0.466 from 160 games
+French Defense: Steinitz Attack : 0.466 from 102 games
+Indian Game : 0.468 from 365 games
+Caro-Kann Defense: Exchange Variation : 0.472 from 107 games
+St. George Defense : 0.472 from 143 games
+Sicilian Defense : 0.476 from 533 games
+Scotch Game: Classical Variation : 0.476 from 104 games
+French Defense: La Bourdonnais Variation : 0.477 from 220 games
+French Defense: Exchange Variation : 0.479 from 243 games
+Nimzowitsch Defense : 0.482 from 112 games
+Three Knights Opening : 0.483 from 207 games
+French Defense: Knight Variation : 0.487 from 758 games
+Hungarian Opening : 0.487 from 317 games
+King's Pawn Game: Wayward Queen Attack : 0.493 from 419 games
+Sicilian Defense: French Variation : 0.496 from 131 games
+Four Knights Game: Italian Variation : 0.502 from 262 games
+Bird Opening: Dutch Variation : 0.504 from 131 games
+Owen Defense : 0.504 from 852 games
+Scandinavian Defense : 0.504 from 451 games
+Caro-Kann Defense : 0.505 from 543 games
+Polish Opening : 0.507 from 146 games
+Scandinavian Defense: Modern Variation #2 : 0.507 from 142 games
+Sicilian Defense: Old Sicilian : 0.511 from 453 games
+Queen's Gambit Declined : 0.511 from 355 games
+English Opening: Agincourt Defense : 0.517 from 119 games
+Italian Game : 0.519 from 241 games
+Sicilian Defense: McDonnell Attack : 0.519 from 309 games
+Dutch Defense : 0.522 from 112 games
+Queen's Pawn Game : 0.523 from 585 games
+Slav Defense : 0.525 from 118 games
+Sicilian Defense: Wing Gambit : 0.526 from 117 games
+Ruy Lopez: Cozio Defense : 0.526 from 170 games
+English Defense #2 : 0.527 from 312 games
+Philidor Defense #4 : 0.529 from 103 games
+Scotch Game : 0.530 from 428 games
+Queen's Gambit Declined: Three Knights Variation : 0.531 from 114 games
+Queen's Pawn Game #3 : 0.532 from 157 games
+Old Benoni Defense : 0.536 from 153 games
+Italian Game: Schilling-Kostic Gambit : 0.537 from 109 games
+Queen's Pawn Game: Mason Attack : 0.540 from 303 games
+Scandinavian Defense: Main Line : 0.545 from 178 games
+Queen's Pawn Game: Zukertort Variation : 0.546 from 445 games
+Sicilian Defense: Loewenthal Variation : 0.547 from 116 games
+Pirc Defense #4 : 0.548 from 218 games
+Bird Opening : 0.550 from 131 games
+Ruy Lopez : 0.550 from 109 games
+Queen's Pawn Game: Colle System : 0.554 from 121 games
+King's Gambit Declined, Queen's Knight Defense : 0.554 from 138 games
+Bishop's Opening: Berlin Defense : 0.556 from 284 games
+Philidor Defense: Exchange Variation : 0.559 from 102 games
+Ruy Lopez: Steinitz Defense : 0.562 from 402 games
+Ruy Lopez: Berlin Defense #2 : 0.568 from 103 games
+Bishop's Opening : 0.572 from 515 games
+King's Pawn Game: McConnell Defense : 0.572 from 167 games
+Old Indian Defense : 0.572 from 111 games
+Modern Defense : 0.573 from 626 games
+English Opening: King's English Variation : 0.575 from 193 games
+Italian Game: Anti-Fried Liver Defense : 0.575 from 279 games
+French Defense: Normal Variation : 0.576 from 341 games
+Vienna Game : 0.584 from 101 games
+Pirc Defense #5 : 0.586 from 105 games
+Philidor Defense #2 : 0.586 from 412 games
+Ruy Lopez: Classical Variation : 0.588 from 108 games
+Sicilian Defense: Open #3 : 0.597 from 113 games
+King's Gambit Accepted, Fischer Defense : 0.600 from 120 games
+Rat Defense: Small Center Defense : 0.602 from 197 games
+Philidor Defense #3 : 0.602 from 723 games
+Bishop's Opening: Boi Variation : 0.609 from 101 games
+King's Gambit : 0.609 from 197 games
+Queen's Pawn : 0.612 from 307 games
+Queen's Pawn Game #2 : 0.616 from 667 games
+Philidor Defense : 0.628 from 188 games
+Philidor Defense: Exchange Variation #2 : 0.628 from 129 games
+Horwitz Defense : 0.630 from 799 games
+Robatsch (Modern) Defense : 0.633 from 173 games
+King's Gambit Accepted, MacLeod Defense : 0.635 from 182 games
+Englund Gambit : 0.639 from 119 games
+French Defense: Advance Variation #3 : 0.648 from 105 games
+Queen's Gambit Refused: Marshall Defense : 0.653 from 226 games
+Queen's Gambit Accepted: Old Variation : 0.653 from 163 games
+Scandinavian Defense: Mieses-Kotroc Variation : 0.664 from 754 games
+King's Knight Opening : 0.687 from 134 games
+King's Gambit Accepted, King's Knight Gambit : 0.690 from 113 games
+Queen's Pawn Game: Chigorin Variation : 0.690 from 987 games
+Center Game #2 : 0.691 from 118 games
+Queen's Gambit Accepted : 0.693 from 168 games
+Danish Gambit : 0.723 from 146 games
+classical performance
+Center Game: Normal Variation : 0.306 from 191 games
+King's Pawn Game : 0.346 from 156 games
+Indian Game : 0.355 from 249 games
+Ruy Lopez: Exchange Variation : 0.369 from 126 games
+Sicilian Defense: Alapin Variation : 0.382 from 165 games
+Queen's Pawn Game : 0.394 from 533 games
+King's Pawn Game: Leonardis Variation : 0.399 from 412 games
+Van't Kruijs Opening : 0.400 from 859 games
+Sicilian Defense : 0.408 from 536 games
+Sicilian Defense: Smith-Morra Gambit #2 : 0.421 from 267 games
+Ruy Lopez: Classical Variation : 0.427 from 109 games
+King's Knight Opening: Normal Variation : 0.429 from 233 games
+King's Pawn Game: Macleod Attack : 0.431 from 102 games
+Slav Defense : 0.445 from 145 games
+Caro-Kann Defense: Hillbilly Attack : 0.452 from 115 games
+Sicilian Defense: Bowdler Attack : 0.456 from 814 games
+Hungarian Opening : 0.461 from 284 games
+King's Pawn Game: Wayward Queen Attack : 0.462 from 807 games
+Petrov's Defense : 0.464 from 111 games
+English Opening: King's English Variation : 0.466 from 116 games
+King's Pawn Game: Wayward Queen Attack, Kiddie Countergambit : 0.467 from 136 games
+Sicilian Defense: Staunton-Cochrane Variation : 0.467 from 228 games
+Sicilian Defense: Old Sicilian : 0.468 from 325 games
+French Defense #2 : 0.468 from 233 games
+Ruy Lopez : 0.468 from 125 games
+Queen's Pawn Game #3 : 0.475 from 202 games
+Sicilian Defense: French Variation : 0.477 from 132 games
+English Opening: The Whale : 0.478 from 289 games
+Ruy Lopez: Berlin Defense #2 : 0.480 from 147 games
+French Defense: Steinitz Attack : 0.481 from 156 games
+Queen's Pawn Game #2 : 0.483 from 643 games
+King's Pawn Game: Napoleon Attack : 0.483 from 238 games
+Three Knights Opening : 0.485 from 238 games
+Ruy Lopez: Steinitz Defense : 0.496 from 393 games
+Philidor Defense: Exchange Variation #2 : 0.496 from 134 games
+Queen's Pawn Game: Chigorin Variation : 0.496 from 558 games
+Polish Opening : 0.497 from 144 games
+Scandinavian Defense : 0.497 from 682 games
+Nimzowitsch Defense : 0.506 from 176 games
+Vienna Game : 0.507 from 142 games
+Italian Game: Schilling-Kostic Gambit : 0.509 from 113 games
+Sicilian Defense: McDonnell Attack : 0.514 from 140 games
+Scotch Game : 0.518 from 631 games
+Philidor Defense #2 : 0.520 from 691 games
+Owen Defense : 0.520 from 562 games
+French Defense: Knight Variation : 0.523 from 481 games
+English Defense #2 : 0.523 from 174 games
+Queen's Pawn Game: Mason Attack : 0.523 from 235 games
+Horwitz Defense : 0.526 from 602 games
+Bishop's Opening: Boi Variation : 0.526 from 174 games
+Alekhine Defense : 0.528 from 109 games
+Russian Game: Three Knights Game : 0.528 from 213 games
+Pirc Defense #4 : 0.531 from 229 games
+Caro-Kann Defense : 0.532 from 404 games
+Queen's Gambit Declined : 0.532 from 295 games
+Robatsch (Modern) Defense : 0.536 from 124 games
+Queen's Gambit Accepted : 0.537 from 219 games
+Scandinavian Defense: Main Line : 0.539 from 191 games
+Giuoco Piano : 0.543 from 138 games
+Nimzowitsch Defense: Scandinavian Variation, Advance Variation : 0.545 from 156 games
+Modern Defense : 0.548 from 425 games
+Bishop's Opening: Berlin Defense : 0.550 from 518 games
+Bishop's Opening : 0.557 from 706 games
+Nimzowitsch Defense: Kennedy Variation, Linksspringer Variation : 0.558 from 104 games
+Rat Defense: Small Center Defense : 0.564 from 164 games
+Ponziani Opening : 0.567 from 112 games
+Queen's Pawn Game: Zukertort Variation : 0.575 from 369 games
+King's Pawn Game: McConnell Defense : 0.575 from 187 games
+Four Knights Game: Italian Variation : 0.585 from 259 games
+Italian Game: Anti-Fried Liver Defense : 0.590 from 451 games
+Italian Game : 0.598 from 326 games
+Bird Opening: Dutch Variation : 0.603 from 102 games
+Center Game #2 : 0.604 from 178 games
+Queen's Gambit Refused: Marshall Defense : 0.605 from 256 games
+Queen's Gambit Accepted: Old Variation : 0.609 from 197 games
+Vienna Game: Stanley Variation : 0.618 from 106 games
+French Defense: Normal Variation : 0.618 from 326 games
+Philidor Defense : 0.625 from 277 games
+King's Gambit : 0.625 from 104 games
+Philidor Defense #3 : 0.631 from 742 games
+Elephant Gambit : 0.634 from 149 games
+Queen's Pawn : 0.634 from 242 games
+King's Pawn Game: Busch-Gass Gambit : 0.635 from 200 games
+King's Gambit Accepted, King's Knight Gambit : 0.636 from 107 games
+French Defense: Advance Variation #3 : 0.650 from 160 games
+Italian Game: Two Knights Defense, Modern Bishop's Opening : 0.658 from 111 games
+Russian Game: Urusov Gambit : 0.666 from 178 games
+Mikenas Defense : 0.688 from 109 games
+Scandinavian Defense: Mieses-Kotroc Variation : 0.701 from 936 games
+King's Pawn Game: Damiano Defense : 0.732 from 220 games
+King's Knight Opening : 0.781 from 169 games
diff --git a/src/main/java/ChessSparkMiner.java b/src/main/java/ChessSparkMiner.java
@@ -9,18 +9,35 @@
 
 public class ChessSparkMiner {
     public static void main(String[] args) {
-        String pgnFile = "/Users/bartoszjedrzejewski/github/chesssparkminer/lichess_db_standard_rated_2013-01.pgn.bz2"; // Should be some file on your system
+        String pgnFile = "/Users/bartoszjedrzejewski/github/chesssparkminer/lichess_db_standard_rated_2013-01.pgn"; // Should be some file on your system
         SparkConf conf = new SparkConf()
                 .setAppName("Chess Spark Miner")
                 .setMaster("local[2]");
 
         JavaSparkContext sc = new JavaSparkContext(conf);
+        sc.setLogLevel("ERROR");
         sc.hadoopConfiguration().set("textinputformat.record.delimiter", "[Event");
         JavaRDD<String> pgnData = sc.textFile(pgnFile);
 
         pgnData = pgnData.filter(line -> line.length() > 1);
 
-        printOpeningsPerformance(pgnData);
+
+        JavaRDD<String> ultraBullet = pgnData.filter(a -> getSpeedMode(a).equals("UltraBullet"));
+        JavaRDD<String> bullet = pgnData.filter(a -> getSpeedMode(a).equals("Bullet"));
+        JavaRDD<String> blitz = pgnData.filter(a -> getSpeedMode(a).equals("Blitz"));
+        JavaRDD<String> classical = pgnData.filter(a -> getSpeedMode(a).equals("Classical"));
+
+        System.out.println("ultraBullet performance");
+        printOpeningsPerformance(ultraBullet);
+
+        System.out.println("bullet performance");
+        printOpeningsPerformance(bullet);
+
+        System.out.println("blitz performance");
+        printOpeningsPerformance(blitz);
+
+        System.out.println("classical performance");
+        printOpeningsPerformance(classical);
 
         sc.stop();
     }
@@ -52,6 +69,20 @@ private static void printOpeningsPerformance(JavaRDD<String> pgnData) {
         }
     }
 
+    public static String getSpeedMode(String pgn){
+        pgn = pgn.substring(0, pgn.indexOf("]")); //to avoid words appearing in player names
+        if(pgn.contains("UltraBullet"))
+            return "UltraBullet";
+        else if(pgn.contains("Bullet"))
+            return "Bullet";
+        else if(pgn.contains("Blitz"))
+            return "Blitz";
+        else if(pgn.contains("Classical"))
+            return "Classical";
+        else
+            return "?";
+    }
+
 
     public static String getOpening(String pgn){
         int i = pgn.indexOf("[Opening \"");